From 2aa4a82499d4becd2284cdb482213d541b8804dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 28 Apr 2024 16:29:10 +0200 Subject: Adding upstream version 86.0.1. Signed-off-by: Daniel Baumann --- third_party/libwebrtc/webrtc/modules/BUILD.gn | 285 + third_party/libwebrtc/webrtc/modules/OWNERS | 4 + .../libwebrtc/webrtc/modules/audio_coding/BUILD.gn | 2139 ++ .../libwebrtc/webrtc/modules/audio_coding/DEPS | 7 + .../libwebrtc/webrtc/modules/audio_coding/OWNERS | 11 + .../audio_coding/acm2/acm_codec_database.cc | 343 + .../modules/audio_coding/acm2/acm_codec_database.h | 82 + .../modules/audio_coding/acm2/acm_receive_test.cc | 242 + .../modules/audio_coding/acm2/acm_receive_test.h | 96 + .../modules/audio_coding/acm2/acm_receiver.cc | 418 + .../modules/audio_coding/acm2/acm_receiver.h | 304 + .../audio_coding/acm2/acm_receiver_unittest.cc | 507 + .../modules/audio_coding/acm2/acm_resampler.cc | 65 + .../modules/audio_coding/acm2/acm_resampler.h | 39 + .../modules/audio_coding/acm2/acm_send_test.cc | 160 + .../modules/audio_coding/acm2/acm_send_test.h | 88 + .../audio_coding/acm2/audio_coding_module.cc | 1348 + .../acm2/audio_coding_module_unittest.cc | 1964 ++ .../modules/audio_coding/acm2/call_statistics.cc | 59 + .../modules/audio_coding/acm2/call_statistics.h | 64 + .../audio_coding/acm2/call_statistics_unittest.cc | 57 + .../modules/audio_coding/acm2/codec_manager.cc | 247 + .../modules/audio_coding/acm2/codec_manager.h | 75 + .../audio_coding/acm2/codec_manager_unittest.cc | 71 + .../modules/audio_coding/acm2/rent_a_codec.cc | 310 + .../modules/audio_coding/acm2/rent_a_codec.h | 201 + .../audio_coding/acm2/rent_a_codec_unittest.cc | 232 + .../webrtc/modules/audio_coding/audio_coding.gni | 33 + .../modules/audio_coding/audio_coding_gn/moz.build | 403 + .../audio_coding_module_typedefs_gn/moz.build | 187 + .../audio_format_conversion_gn/moz.build | 217 + .../audio_network_adaptor_config.cc | 34 + .../audio_network_adaptor_impl.cc | 205 + .../audio_network_adaptor_impl.h | 93 + .../audio_network_adaptor_impl_unittest.cc | 326 + .../audio_network_adaptor/bitrate_controller.cc | 76 + .../audio_network_adaptor/bitrate_controller.h | 54 + .../bitrate_controller_unittest.cc | 249 + .../audio_network_adaptor/channel_controller.cc | 61 + .../audio_network_adaptor/channel_controller.h | 53 + .../channel_controller_unittest.cc | 100 + .../audio_network_adaptor/config.proto | 164 + .../audio_network_adaptor/controller.cc | 19 + .../audio_network_adaptor/controller.h | 43 + .../audio_network_adaptor/controller_manager.cc | 441 + .../audio_network_adaptor/controller_manager.h | 123 + .../controller_manager_unittest.cc | 472 + .../audio_network_adaptor/debug_dump.proto | 42 + .../audio_network_adaptor/debug_dump_writer.cc | 165 + .../audio_network_adaptor/debug_dump_writer.h | 55 + .../audio_network_adaptor/dtx_controller.cc | 50 + .../audio_network_adaptor/dtx_controller.h | 49 + .../dtx_controller_unittest.cc | 81 + .../audio_network_adaptor/event_log_writer.cc | 72 + .../audio_network_adaptor/event_log_writer.h | 42 + .../event_log_writer_unittest.cc | 238 + .../fec_controller_plr_based.cc | 113 + .../fec_controller_plr_based.h | 72 + .../fec_controller_plr_based_unittest.cc | 488 + .../fec_controller_rplr_based.cc | 78 + .../fec_controller_rplr_based.h | 66 + .../fec_controller_rplr_based_unittest.cc | 520 + .../frame_length_controller.cc | 183 + .../frame_length_controller.h | 91 + .../frame_length_controller_unittest.cc | 381 + .../include/audio_network_adaptor.h | 52 + .../include/audio_network_adaptor_config.h | 49 + .../mock/mock_audio_network_adaptor.h | 49 + .../audio_network_adaptor/mock/mock_controller.h | 30 + .../mock/mock_controller_manager.h | 33 + .../mock/mock_debug_dump_writer.h | 40 + .../audio_network_adaptor/parse_ana_dump.py | 147 + .../audio_network_adaptor/util/threshold_curve.h | 118 + .../util/threshold_curve_unittest.cc | 631 + .../audio_network_adaptor_config_gn/moz.build | 217 + .../audio_network_adaptor_gn/moz.build | 238 + .../webrtc/modules/audio_coding/cng_gn/moz.build | 229 + .../webrtc/modules/audio_coding/codecs/OWNERS | 0 .../modules/audio_coding/codecs/audio_decoder.h | 20 + .../modules/audio_coding/codecs/audio_encoder.h | 20 + .../audio_coding/codecs/audio_format_conversion.cc | 88 + .../audio_coding/codecs/audio_format_conversion.h | 24 + .../codecs/builtin_audio_decoder_factory.h | 20 + .../builtin_audio_decoder_factory_unittest.cc | 136 + .../codecs/builtin_audio_encoder_factory.h | 20 + .../builtin_audio_encoder_factory_unittest.cc | 144 + .../audio_coding/codecs/cng/audio_encoder_cng.cc | 267 + .../audio_coding/codecs/cng/audio_encoder_cng.h | 96 + .../codecs/cng/audio_encoder_cng_unittest.cc | 508 + .../audio_coding/codecs/cng/cng_unittest.cc | 241 + .../modules/audio_coding/codecs/cng/webrtc_cng.cc | 444 + .../modules/audio_coding/codecs/cng/webrtc_cng.h | 99 + .../webrtc/modules/audio_coding/codecs/g711/OWNERS | 0 .../audio_coding/codecs/g711/audio_decoder_pcm.cc | 88 + .../audio_coding/codecs/g711/audio_decoder_pcm.h | 70 + .../audio_coding/codecs/g711/audio_encoder_pcm.cc | 142 + .../audio_coding/codecs/g711/audio_encoder_pcm.h | 124 + .../webrtc/modules/audio_coding/codecs/g711/g711.c | 73 + .../webrtc/modules/audio_coding/codecs/g711/g711.h | 344 + .../audio_coding/codecs/g711/g711_interface.c | 58 + .../audio_coding/codecs/g711/g711_interface.h | 135 + .../audio_coding/codecs/g711/test/testG711.cc | 170 + .../audio_coding/codecs/g722/audio_decoder_g722.cc | 162 + .../audio_coding/codecs/g722/audio_decoder_g722.h | 79 + .../audio_coding/codecs/g722/audio_encoder_g722.cc | 162 + .../audio_coding/codecs/g722/audio_encoder_g722.h | 68 + .../modules/audio_coding/codecs/g722/g722_decode.c | 400 + .../audio_coding/codecs/g722/g722_enc_dec.h | 160 + .../modules/audio_coding/codecs/g722/g722_encode.c | 430 + .../audio_coding/codecs/g722/g722_interface.c | 107 + .../audio_coding/codecs/g722/g722_interface.h | 182 + .../audio_coding/codecs/g722/test/testG722.cc | 158 + .../modules/audio_coding/codecs/ilbc/abs_quant.c | 80 + .../modules/audio_coding/codecs/ilbc/abs_quant.h | 39 + .../audio_coding/codecs/ilbc/abs_quant_loop.c | 87 + .../audio_coding/codecs/ilbc/abs_quant_loop.h | 33 + .../audio_coding/codecs/ilbc/audio_decoder_ilbc.cc | 110 + .../audio_coding/codecs/ilbc/audio_decoder_ilbc.h | 46 + .../audio_coding/codecs/ilbc/audio_encoder_ilbc.cc | 155 + .../audio_coding/codecs/ilbc/audio_encoder_ilbc.h | 54 + .../audio_coding/codecs/ilbc/augmented_cb_corr.c | 63 + .../audio_coding/codecs/ilbc/augmented_cb_corr.h | 41 + .../modules/audio_coding/codecs/ilbc/bw_expand.c | 42 + .../modules/audio_coding/codecs/ilbc/bw_expand.h | 36 + .../audio_coding/codecs/ilbc/cb_construct.c | 71 + .../audio_coding/codecs/ilbc/cb_construct.h | 39 + .../audio_coding/codecs/ilbc/cb_mem_energy.c | 79 + .../audio_coding/codecs/ilbc/cb_mem_energy.h | 34 + .../codecs/ilbc/cb_mem_energy_augmentation.c | 67 + .../codecs/ilbc/cb_mem_energy_augmentation.h | 31 + .../audio_coding/codecs/ilbc/cb_mem_energy_calc.c | 65 + .../audio_coding/codecs/ilbc/cb_mem_energy_calc.h | 33 + .../modules/audio_coding/codecs/ilbc/cb_search.c | 403 + .../modules/audio_coding/codecs/ilbc/cb_search.h | 35 + .../audio_coding/codecs/ilbc/cb_search_core.c | 113 + .../audio_coding/codecs/ilbc/cb_search_core.h | 40 + .../codecs/ilbc/cb_update_best_index.c | 88 + .../codecs/ilbc/cb_update_best_index.h | 38 + .../modules/audio_coding/codecs/ilbc/chebyshev.c | 74 + .../modules/audio_coding/codecs/ilbc/chebyshev.h | 37 + .../modules/audio_coding/codecs/ilbc/comp_corr.c | 49 + .../modules/audio_coding/codecs/ilbc/comp_corr.h | 39 + .../audio_coding/codecs/ilbc/complexityMeasures.m | 59 + .../modules/audio_coding/codecs/ilbc/constants.c | 666 + .../modules/audio_coding/codecs/ilbc/constants.h | 92 + .../codecs/ilbc/create_augmented_vec.c | 81 + .../codecs/ilbc/create_augmented_vec.h | 37 + .../modules/audio_coding/codecs/ilbc/decode.c | 257 + .../modules/audio_coding/codecs/ilbc/decode.h | 38 + .../audio_coding/codecs/ilbc/decode_residual.c | 185 + .../audio_coding/codecs/ilbc/decode_residual.h | 40 + .../codecs/ilbc/decoder_interpolate_lsf.c | 83 + .../codecs/ilbc/decoder_interpolate_lsf.h | 38 + .../modules/audio_coding/codecs/ilbc/defines.h | 220 + .../modules/audio_coding/codecs/ilbc/do_plc.c | 307 + .../modules/audio_coding/codecs/ilbc/do_plc.h | 41 + .../modules/audio_coding/codecs/ilbc/encode.c | 513 + .../modules/audio_coding/codecs/ilbc/encode.h | 35 + .../audio_coding/codecs/ilbc/energy_inverse.c | 46 + .../audio_coding/codecs/ilbc/energy_inverse.h | 32 + .../audio_coding/codecs/ilbc/enh_upsample.c | 110 + .../audio_coding/codecs/ilbc/enh_upsample.h | 33 + .../modules/audio_coding/codecs/ilbc/enhancer.c | 51 + .../modules/audio_coding/codecs/ilbc/enhancer.h | 39 + .../audio_coding/codecs/ilbc/enhancer_interface.c | 377 + .../audio_coding/codecs/ilbc/enhancer_interface.h | 34 + .../audio_coding/codecs/ilbc/filtered_cb_vecs.c | 48 + .../audio_coding/codecs/ilbc/filtered_cb_vecs.h | 38 + .../audio_coding/codecs/ilbc/frame_classify.c | 88 + .../audio_coding/codecs/ilbc/frame_classify.h | 29 + .../audio_coding/codecs/ilbc/gain_dequant.c | 45 + .../audio_coding/codecs/ilbc/gain_dequant.h | 36 + .../modules/audio_coding/codecs/ilbc/gain_quant.c | 103 + .../modules/audio_coding/codecs/ilbc/gain_quant.h | 35 + .../modules/audio_coding/codecs/ilbc/get_cd_vec.c | 126 + .../modules/audio_coding/codecs/ilbc/get_cd_vec.h | 36 + .../audio_coding/codecs/ilbc/get_lsp_poly.c | 82 + .../audio_coding/codecs/ilbc/get_lsp_poly.h | 47 + .../audio_coding/codecs/ilbc/get_sync_seq.c | 109 + .../audio_coding/codecs/ilbc/get_sync_seq.h | 40 + .../modules/audio_coding/codecs/ilbc/hp_input.c | 88 + .../modules/audio_coding/codecs/ilbc/hp_input.h | 34 + .../modules/audio_coding/codecs/ilbc/hp_output.c | 89 + .../modules/audio_coding/codecs/ilbc/hp_output.h | 34 + .../webrtc/modules/audio_coding/codecs/ilbc/ilbc.c | 287 + .../webrtc/modules/audio_coding/codecs/ilbc/ilbc.h | 258 + .../audio_coding/codecs/ilbc/ilbc_unittest.cc | 139 + .../audio_coding/codecs/ilbc/index_conv_dec.c | 38 + .../audio_coding/codecs/ilbc/index_conv_dec.h | 28 + .../audio_coding/codecs/ilbc/index_conv_enc.c | 42 + .../audio_coding/codecs/ilbc/index_conv_enc.h | 32 + .../modules/audio_coding/codecs/ilbc/init_decode.c | 96 + .../modules/audio_coding/codecs/ilbc/init_decode.h | 35 + .../modules/audio_coding/codecs/ilbc/init_encode.c | 71 + .../modules/audio_coding/codecs/ilbc/init_encode.h | 33 + .../modules/audio_coding/codecs/ilbc/interpolate.c | 46 + .../modules/audio_coding/codecs/ilbc/interpolate.h | 35 + .../audio_coding/codecs/ilbc/interpolate_samples.c | 51 + .../audio_coding/codecs/ilbc/interpolate_samples.h | 34 + .../modules/audio_coding/codecs/ilbc/lpc_encode.c | 60 + .../modules/audio_coding/codecs/ilbc/lpc_encode.h | 39 + .../modules/audio_coding/codecs/ilbc/lsf_check.c | 71 + .../modules/audio_coding/codecs/ilbc/lsf_check.h | 33 + .../codecs/ilbc/lsf_interpolate_to_poly_dec.c | 42 + .../codecs/ilbc/lsf_interpolate_to_poly_dec.h | 37 + .../codecs/ilbc/lsf_interpolate_to_poly_enc.c | 46 + .../codecs/ilbc/lsf_interpolate_to_poly_enc.h | 38 + .../modules/audio_coding/codecs/ilbc/lsf_to_lsp.c | 61 + .../modules/audio_coding/codecs/ilbc/lsf_to_lsp.h | 34 + .../modules/audio_coding/codecs/ilbc/lsf_to_poly.c | 86 + .../modules/audio_coding/codecs/ilbc/lsf_to_poly.h | 33 + .../modules/audio_coding/codecs/ilbc/lsp_to_lsf.c | 84 + .../modules/audio_coding/codecs/ilbc/lsp_to_lsf.h | 35 + .../modules/audio_coding/codecs/ilbc/my_corr.c | 54 + .../modules/audio_coding/codecs/ilbc/my_corr.h | 36 + .../audio_coding/codecs/ilbc/nearest_neighbor.c | 35 + .../audio_coding/codecs/ilbc/nearest_neighbor.h | 36 + .../modules/audio_coding/codecs/ilbc/pack_bits.c | 251 + .../modules/audio_coding/codecs/ilbc/pack_bits.h | 34 + .../modules/audio_coding/codecs/ilbc/poly_to_lsf.c | 31 + .../modules/audio_coding/codecs/ilbc/poly_to_lsf.h | 33 + .../modules/audio_coding/codecs/ilbc/poly_to_lsp.c | 158 + .../modules/audio_coding/codecs/ilbc/poly_to_lsp.h | 36 + .../modules/audio_coding/codecs/ilbc/refiner.c | 140 + .../modules/audio_coding/codecs/ilbc/refiner.h | 43 + .../codecs/ilbc/simple_interpolate_lsf.c | 131 + .../codecs/ilbc/simple_interpolate_lsf.h | 46 + .../audio_coding/codecs/ilbc/simple_lpc_analysis.c | 94 + .../audio_coding/codecs/ilbc/simple_lpc_analysis.h | 35 + .../audio_coding/codecs/ilbc/simple_lsf_dequant.c | 60 + .../audio_coding/codecs/ilbc/simple_lsf_dequant.h | 34 + .../audio_coding/codecs/ilbc/simple_lsf_quant.c | 47 + .../audio_coding/codecs/ilbc/simple_lsf_quant.h | 37 + .../modules/audio_coding/codecs/ilbc/smooth.c | 210 + .../modules/audio_coding/codecs/ilbc/smooth.h | 36 + .../audio_coding/codecs/ilbc/smooth_out_data.c | 44 + .../audio_coding/codecs/ilbc/smooth_out_data.h | 35 + .../modules/audio_coding/codecs/ilbc/sort_sq.c | 51 + .../modules/audio_coding/codecs/ilbc/sort_sq.h | 36 + .../modules/audio_coding/codecs/ilbc/split_vq.c | 61 + .../modules/audio_coding/codecs/ilbc/split_vq.h | 38 + .../audio_coding/codecs/ilbc/state_construct.c | 114 + .../audio_coding/codecs/ilbc/state_construct.h | 35 + .../audio_coding/codecs/ilbc/state_search.c | 119 + .../audio_coding/codecs/ilbc/state_search.h | 38 + .../modules/audio_coding/codecs/ilbc/swap_bytes.c | 35 + .../modules/audio_coding/codecs/ilbc/swap_bytes.h | 34 + .../modules/audio_coding/codecs/ilbc/test/empty.cc | 0 .../audio_coding/codecs/ilbc/test/iLBC_test.c | 240 + .../audio_coding/codecs/ilbc/test/iLBC_testLib.c | 215 + .../codecs/ilbc/test/iLBC_testprogram.c | 343 + .../codecs/ilbc/test/iLBCtestscript.txt | 73 + .../modules/audio_coding/codecs/ilbc/unpack_bits.c | 239 + .../modules/audio_coding/codecs/ilbc/unpack_bits.h | 34 + .../webrtc/modules/audio_coding/codecs/ilbc/vq3.c | 63 + .../webrtc/modules/audio_coding/codecs/ilbc/vq3.h | 36 + .../webrtc/modules/audio_coding/codecs/ilbc/vq4.c | 62 + .../webrtc/modules/audio_coding/codecs/ilbc/vq4.h | 36 + .../audio_coding/codecs/ilbc/window32_w32.c | 62 + .../audio_coding/codecs/ilbc/window32_w32.h | 35 + .../modules/audio_coding/codecs/ilbc/xcorr_coef.c | 140 + .../modules/audio_coding/codecs/ilbc/xcorr_coef.h | 38 + .../codecs/isac/audio_decoder_isac_t.h | 59 + .../codecs/isac/audio_decoder_isac_t_impl.h | 109 + .../codecs/isac/audio_encoder_isac_t.h | 98 + .../codecs/isac/audio_encoder_isac_t_impl.h | 189 + .../audio_coding/codecs/isac/bandwidth_info.h | 24 + .../modules/audio_coding/codecs/isac/empty.cc | 0 .../isac/fix/include/audio_decoder_isacfix.h | 22 + .../isac/fix/include/audio_encoder_isacfix.h | 22 + .../audio_coding/codecs/isac/fix/include/isacfix.h | 637 + .../codecs/isac/fix/source/arith_routines.c | 122 + .../codecs/isac/fix/source/arith_routines_hist.c | 401 + .../codecs/isac/fix/source/arith_routines_logist.c | 413 + .../codecs/isac/fix/source/arith_routins.h | 160 + .../isac/fix/source/audio_decoder_isacfix.cc | 20 + .../isac/fix/source/audio_encoder_isacfix.cc | 20 + .../codecs/isac/fix/source/bandwidth_estimator.c | 1037 + .../codecs/isac/fix/source/bandwidth_estimator.h | 135 + .../audio_coding/codecs/isac/fix/source/codec.h | 228 + .../audio_coding/codecs/isac/fix/source/decode.c | 221 + .../codecs/isac/fix/source/decode_bwe.c | 69 + .../codecs/isac/fix/source/decode_plc.c | 805 + .../audio_coding/codecs/isac/fix/source/encode.c | 635 + .../codecs/isac/fix/source/entropy_coding.c | 2056 ++ .../codecs/isac/fix/source/entropy_coding.h | 189 + .../codecs/isac/fix/source/entropy_coding_mips.c | 249 + .../codecs/isac/fix/source/entropy_coding_neon.c | 218 + .../audio_coding/codecs/isac/fix/source/fft.c | 415 + .../audio_coding/codecs/isac/fix/source/fft.h | 41 + .../codecs/isac/fix/source/filterbank_internal.h | 90 + .../codecs/isac/fix/source/filterbank_tables.c | 62 + .../codecs/isac/fix/source/filterbank_tables.h | 52 + .../codecs/isac/fix/source/filterbanks.c | 421 + .../codecs/isac/fix/source/filterbanks_mips.c | 242 + .../codecs/isac/fix/source/filterbanks_neon.c | 277 + .../codecs/isac/fix/source/filterbanks_unittest.cc | 106 + .../audio_coding/codecs/isac/fix/source/filters.c | 112 + .../codecs/isac/fix/source/filters_mips.c | 365 + .../codecs/isac/fix/source/filters_neon.c | 114 + .../codecs/isac/fix/source/filters_unittest.cc | 66 + .../codecs/isac/fix/source/initialize.c | 173 + .../codecs/isac/fix/source/isac_fix_type.h | 123 + .../audio_coding/codecs/isac/fix/source/isacfix.c | 1551 ++ .../audio_coding/codecs/isac/fix/source/lattice.c | 322 + .../codecs/isac/fix/source/lattice_armv7.S | 77 + .../codecs/isac/fix/source/lattice_c.c | 49 + .../codecs/isac/fix/source/lattice_mips.c | 329 + .../codecs/isac/fix/source/lattice_neon.c | 195 + .../codecs/isac/fix/source/lpc_masking_model.c | 948 + .../codecs/isac/fix/source/lpc_masking_model.h | 69 + .../isac/fix/source/lpc_masking_model_mips.c | 237 + .../isac/fix/source/lpc_masking_model_unittest.cc | 61 + .../codecs/isac/fix/source/lpc_tables.c | 1280 + .../codecs/isac/fix/source/lpc_tables.h | 97 + .../codecs/isac/fix/source/pitch_estimator.c | 435 + .../codecs/isac/fix/source/pitch_estimator.h | 65 + .../codecs/isac/fix/source/pitch_estimator_c.c | 112 + .../codecs/isac/fix/source/pitch_estimator_mips.c | 193 + .../codecs/isac/fix/source/pitch_filter.c | 248 + .../codecs/isac/fix/source/pitch_filter_armv6.S | 143 + .../codecs/isac/fix/source/pitch_filter_c.c | 73 + .../codecs/isac/fix/source/pitch_filter_mips.c | 133 + .../codecs/isac/fix/source/pitch_gain_tables.c | 149 + .../codecs/isac/fix/source/pitch_gain_tables.h | 44 + .../codecs/isac/fix/source/pitch_lag_tables.c | 306 + .../codecs/isac/fix/source/pitch_lag_tables.h | 101 + .../audio_coding/codecs/isac/fix/source/settings.h | 215 + .../isac/fix/source/spectrum_ar_model_tables.c | 193 + .../isac/fix/source/spectrum_ar_model_tables.h | 95 + .../audio_coding/codecs/isac/fix/source/structs.h | 382 + .../codecs/isac/fix/source/transform.c | 213 + .../codecs/isac/fix/source/transform_mips.c | 1294 + .../codecs/isac/fix/source/transform_neon.c | 479 + .../codecs/isac/fix/source/transform_tables.c | 109 + .../codecs/isac/fix/source/transform_unittest.cc | 193 + .../codecs/isac/fix/test/QA/ChannelFiles.txt | 3 + .../codecs/isac/fix/test/QA/InputFiles.txt | 31 + .../codecs/isac/fix/test/QA/InputFilesFew.txt | 6 + .../codecs/isac/fix/test/QA/ListOfTestCases.xls | Bin 0 -> 152064 bytes .../codecs/isac/fix/test/QA/diffiSAC.txt | 481 + .../codecs/isac/fix/test/QA/diffiSACPLC.txt | 20 + .../codecs/isac/fix/test/QA/runiSACLongtest.txt | 61 + .../codecs/isac/fix/test/QA/runiSACNB.txt | 45 + .../codecs/isac/fix/test/QA/runiSACPLC.txt | 23 + .../codecs/isac/fix/test/QA/runiSACRate.txt | 23 + .../codecs/isac/fix/test/QA/runiSACfault.txt | 40 + .../codecs/isac/fix/test/QA/runiSACfixfloat.txt | 47 + .../codecs/isac/fix/test/isac_speed_test.cc | 114 + .../audio_coding/codecs/isac/fix/test/kenny.cc | 845 + .../codecs/isac/locked_bandwidth_info.cc | 21 + .../codecs/isac/locked_bandwidth_info.h | 56 + .../codecs/isac/main/include/audio_decoder_isac.h | 22 + .../codecs/isac/main/include/audio_encoder_isac.h | 22 + .../audio_coding/codecs/isac/main/include/isac.h | 724 + .../codecs/isac/main/source/arith_routines.c | 60 + .../codecs/isac/main/source/arith_routines.h | 63 + .../codecs/isac/main/source/arith_routines_hist.c | 291 + .../isac/main/source/arith_routines_logist.c | 303 + .../codecs/isac/main/source/audio_decoder_isac.cc | 20 + .../codecs/isac/main/source/audio_encoder_isac.cc | 20 + .../main/source/audio_encoder_isac_unittest.cc | 56 + .../codecs/isac/main/source/bandwidth_estimator.c | 1031 + .../codecs/isac/main/source/bandwidth_estimator.h | 184 + .../audio_coding/codecs/isac/main/source/codec.h | 233 + .../audio_coding/codecs/isac/main/source/crc.c | 110 + .../audio_coding/codecs/isac/main/source/crc.h | 46 + .../audio_coding/codecs/isac/main/source/decode.c | 304 + .../codecs/isac/main/source/decode_bwe.c | 89 + .../audio_coding/codecs/isac/main/source/encode.c | 1258 + .../codecs/isac/main/source/encode_lpc_swb.c | 708 + .../codecs/isac/main/source/encode_lpc_swb.h | 282 + .../codecs/isac/main/source/entropy_coding.c | 2066 ++ .../codecs/isac/main/source/entropy_coding.h | 343 + .../audio_coding/codecs/isac/main/source/fft.c | 943 + .../audio_coding/codecs/isac/main/source/fft.h | 45 + .../codecs/isac/main/source/filter_functions.c | 263 + .../codecs/isac/main/source/filterbank_tables.c | 37 + .../codecs/isac/main/source/filterbank_tables.h | 46 + .../codecs/isac/main/source/filterbanks.c | 346 + .../codecs/isac/main/source/intialize.c | 171 + .../audio_coding/codecs/isac/main/source/isac.c | 2374 ++ .../codecs/isac/main/source/isac_float_type.h | 117 + .../codecs/isac/main/source/isac_unittest.cc | 111 + .../audio_coding/codecs/isac/main/source/lattice.c | 218 + .../codecs/isac/main/source/lpc_analysis.c | 535 + .../codecs/isac/main/source/lpc_analysis.h | 50 + .../codecs/isac/main/source/lpc_gain_swb_tables.c | 137 + .../codecs/isac/main/source/lpc_gain_swb_tables.h | 49 + .../isac/main/source/lpc_shape_swb12_tables.c | 159 + .../isac/main/source/lpc_shape_swb12_tables.h | 65 + .../isac/main/source/lpc_shape_swb16_tables.c | 248 + .../isac/main/source/lpc_shape_swb16_tables.h | 78 + .../codecs/isac/main/source/lpc_tables.c | 601 + .../codecs/isac/main/source/lpc_tables.h | 100 + .../codecs/isac/main/source/os_specific_inline.h | 41 + .../codecs/isac/main/source/pitch_estimator.c | 623 + .../codecs/isac/main/source/pitch_estimator.h | 75 + .../codecs/isac/main/source/pitch_filter.c | 390 + .../codecs/isac/main/source/pitch_gain_tables.c | 105 + .../codecs/isac/main/source/pitch_gain_tables.h | 45 + .../codecs/isac/main/source/pitch_lag_tables.c | 277 + .../codecs/isac/main/source/pitch_lag_tables.h | 114 + .../codecs/isac/main/source/settings.h | 205 + .../isac/main/source/spectrum_ar_model_tables.c | 139 + .../isac/main/source/spectrum_ar_model_tables.h | 78 + .../audio_coding/codecs/isac/main/source/structs.h | 495 + .../codecs/isac/main/source/transform.c | 125 + .../codecs/isac/main/test/QA/runiSACLongtest.txt | 433 + .../codecs/isac/main/test/QA/runiSACfault.txt | 80 + .../codecs/isac/main/test/QA/runiSACfixfloat.txt | 47 + .../main/test/ReleaseTest-API/ReleaseTest-API.cc | 949 + .../test/SwitchingSampRate/SwitchingSampRate.cc | 458 + .../codecs/isac/main/test/simpleKenny.c | 467 + .../audio_coding/codecs/isac/main/util/utility.c | 178 + .../audio_coding/codecs/isac/main/util/utility.h | 144 + .../modules/audio_coding/codecs/isac/unittest.cc | 257 + .../codecs/legacy_encoded_audio_frame.cc | 87 + .../codecs/legacy_encoded_audio_frame.h | 48 + .../codecs/legacy_encoded_audio_frame_unittest.cc | 170 + .../audio_coding/codecs/opus/audio_decoder_opus.cc | 168 + .../audio_coding/codecs/opus/audio_decoder_opus.h | 55 + .../audio_coding/codecs/opus/audio_encoder_opus.cc | 819 + .../audio_coding/codecs/opus/audio_encoder_opus.h | 181 + .../codecs/opus/audio_encoder_opus_unittest.cc | 877 + .../codecs/opus/opus_bandwidth_unittest.cc | 151 + .../codecs/opus/opus_complexity_unittest.cc | 98 + .../audio_coding/codecs/opus/opus_fec_test.cc | 248 + .../modules/audio_coding/codecs/opus/opus_inst.h | 36 + .../audio_coding/codecs/opus/opus_interface.c | 523 + .../audio_coding/codecs/opus/opus_interface.h | 432 + .../audio_coding/codecs/opus/opus_speed_test.cc | 142 + .../audio_coding/codecs/opus/opus_unittest.cc | 816 + .../codecs/pcm16b/audio_decoder_pcm16b.cc | 63 + .../codecs/pcm16b/audio_decoder_pcm16b.h | 44 + .../codecs/pcm16b/audio_encoder_pcm16b.cc | 60 + .../codecs/pcm16b/audio_encoder_pcm16b.h | 50 + .../modules/audio_coding/codecs/pcm16b/pcm16b.c | 34 + .../modules/audio_coding/codecs/pcm16b/pcm16b.h | 68 + .../audio_coding/codecs/pcm16b/pcm16b_common.cc | 25 + .../audio_coding/codecs/pcm16b/pcm16b_common.h | 22 + .../codecs/red/audio_encoder_copy_red.cc | 142 + .../codecs/red/audio_encoder_copy_red.h | 77 + .../codecs/red/audio_encoder_copy_red_unittest.cc | 308 + .../codecs/tools/audio_codec_speed_test.cc | 127 + .../codecs/tools/audio_codec_speed_test.h | 91 + .../modules/audio_coding/g711_c_gn/moz.build | 210 + .../webrtc/modules/audio_coding/g711_gn/moz.build | 218 + .../modules/audio_coding/g722_c_gn/moz.build | 214 + .../webrtc/modules/audio_coding/g722_gn/moz.build | 218 + .../modules/audio_coding/ilbc_c_gn/moz.build | 296 + .../webrtc/modules/audio_coding/ilbc_gn/moz.build | 229 + .../audio_coding/include/audio_coding_module.h | 821 + .../include/audio_coding_module_typedefs.h | 51 + .../modules/audio_coding/isac_c_gn/moz.build | 257 + .../modules/audio_coding/isac_common_gn/moz.build | 217 + .../audio_coding/isac_fix_c_arm_asm_gn/moz.build | 100 + .../modules/audio_coding/isac_fix_c_gn/moz.build | 438 + .../audio_coding/isac_fix_common_gn/moz.build | 210 + .../modules/audio_coding/isac_fix_gn/moz.build | 116 + .../webrtc/modules/audio_coding/isac_gn/moz.build | 228 + .../modules/audio_coding/isac_neon_gn/moz.build | 199 + .../legacy_encoded_audio_frame_gn/moz.build | 217 + .../modules/audio_coding/neteq/accelerate.cc | 101 + .../webrtc/modules/audio_coding/neteq/accelerate.h | 81 + .../audio_coding/neteq/audio_decoder_unittest.cc | 630 + .../audio_coding/neteq/audio_multi_vector.cc | 222 + .../audio_coding/neteq/audio_multi_vector.h | 139 + .../neteq/audio_multi_vector_unittest.cc | 324 + .../modules/audio_coding/neteq/audio_vector.cc | 386 + .../modules/audio_coding/neteq/audio_vector.h | 168 + .../audio_coding/neteq/audio_vector_unittest.cc | 388 + .../modules/audio_coding/neteq/background_noise.cc | 255 + .../modules/audio_coding/neteq/background_noise.h | 137 + .../neteq/background_noise_unittest.cc | 26 + .../audio_coding/neteq/buffer_level_filter.cc | 65 + .../audio_coding/neteq/buffer_level_filter.h | 49 + .../neteq/buffer_level_filter_unittest.cc | 162 + .../modules/audio_coding/neteq/comfort_noise.cc | 124 + .../modules/audio_coding/neteq/comfort_noise.h | 71 + .../audio_coding/neteq/comfort_noise_unittest.cc | 31 + .../audio_coding/neteq/cross_correlation.cc | 62 + .../modules/audio_coding/neteq/cross_correlation.h | 50 + .../modules/audio_coding/neteq/decision_logic.cc | 170 + .../modules/audio_coding/neteq/decision_logic.h | 168 + .../audio_coding/neteq/decision_logic_fax.cc | 103 + .../audio_coding/neteq/decision_logic_fax.h | 57 + .../audio_coding/neteq/decision_logic_normal.cc | 241 + .../audio_coding/neteq/decision_logic_normal.h | 107 + .../audio_coding/neteq/decision_logic_unittest.cc | 55 + .../modules/audio_coding/neteq/decoder_database.cc | 366 + .../modules/audio_coding/neteq/decoder_database.h | 250 + .../neteq/decoder_database_unittest.cc | 295 + .../webrtc/modules/audio_coding/neteq/defines.h | 52 + .../modules/audio_coding/neteq/delay_manager.cc | 469 + .../modules/audio_coding/neteq/delay_manager.h | 181 + .../audio_coding/neteq/delay_manager_unittest.cc | 414 + .../audio_coding/neteq/delay_peak_detector.cc | 128 + .../audio_coding/neteq/delay_peak_detector.h | 75 + .../neteq/delay_peak_detector_unittest.cc | 143 + .../modules/audio_coding/neteq/dsp_helper.cc | 368 + .../webrtc/modules/audio_coding/neteq/dsp_helper.h | 144 + .../audio_coding/neteq/dsp_helper_unittest.cc | 89 + .../modules/audio_coding/neteq/dtmf_buffer.cc | 249 + .../modules/audio_coding/neteq/dtmf_buffer.h | 114 + .../audio_coding/neteq/dtmf_buffer_unittest.cc | 301 + .../audio_coding/neteq/dtmf_tone_generator.cc | 218 + .../audio_coding/neteq/dtmf_tone_generator.h | 54 + .../neteq/dtmf_tone_generator_unittest.cc | 179 + .../webrtc/modules/audio_coding/neteq/expand.cc | 978 + .../webrtc/modules/audio_coding/neteq/expand.h | 161 + .../modules/audio_coding/neteq/expand_unittest.cc | 206 + .../modules/audio_coding/neteq/include/neteq.h | 314 + .../webrtc/modules/audio_coding/neteq/merge.cc | 387 + .../webrtc/modules/audio_coding/neteq/merge.h | 102 + .../modules/audio_coding/neteq/merge_unittest.cc | 39 + .../neteq/mock/mock_buffer_level_filter.h | 37 + .../neteq/mock/mock_decoder_database.h | 61 + .../audio_coding/neteq/mock/mock_delay_manager.h | 62 + .../neteq/mock/mock_delay_peak_detector.h | 35 + .../audio_coding/neteq/mock/mock_dtmf_buffer.h | 38 + .../neteq/mock/mock_dtmf_tone_generator.h | 35 + .../modules/audio_coding/neteq/mock/mock_expand.h | 64 + .../neteq/mock/mock_external_decoder_pcm16b.h | 98 + .../audio_coding/neteq/mock/mock_packet_buffer.h | 68 + .../neteq/mock/mock_red_payload_splitter.h | 29 + .../neteq/mock/mock_statistics_calculator.h | 27 + .../modules/audio_coding/neteq/nack_tracker.cc | 232 + .../modules/audio_coding/neteq/nack_tracker.h | 208 + .../audio_coding/neteq/nack_tracker_unittest.cc | 483 + .../webrtc/modules/audio_coding/neteq/neteq.cc | 43 + .../audio_coding/neteq/neteq_decoder_enum.cc | 85 + .../audio_coding/neteq/neteq_decoder_enum.h | 56 + .../neteq/neteq_external_decoder_unittest.cc | 457 + .../modules/audio_coding/neteq/neteq_impl.cc | 2139 ++ .../webrtc/modules/audio_coding/neteq/neteq_impl.h | 450 + .../audio_coding/neteq/neteq_impl_unittest.cc | 1515 ++ .../neteq/neteq_network_stats_unittest.cc | 336 + .../audio_coding/neteq/neteq_stereo_unittest.cc | 442 + .../modules/audio_coding/neteq/neteq_unittest.cc | 1804 ++ .../audio_coding/neteq/neteq_unittest.proto | 32 + .../webrtc/modules/audio_coding/neteq/normal.cc | 216 + .../webrtc/modules/audio_coding/neteq/normal.h | 75 + .../modules/audio_coding/neteq/normal_unittest.cc | 176 + .../webrtc/modules/audio_coding/neteq/packet.cc | 35 + .../webrtc/modules/audio_coding/neteq/packet.h | 124 + .../modules/audio_coding/neteq/packet_buffer.cc | 293 + .../modules/audio_coding/neteq/packet_buffer.h | 148 + .../audio_coding/neteq/packet_buffer_unittest.cc | 733 + .../modules/audio_coding/neteq/post_decode_vad.cc | 89 + .../modules/audio_coding/neteq/post_decode_vad.h | 72 + .../audio_coding/neteq/post_decode_vad_unittest.cc | 25 + .../audio_coding/neteq/preemptive_expand.cc | 114 + .../modules/audio_coding/neteq/preemptive_expand.h | 88 + .../modules/audio_coding/neteq/random_vector.cc | 57 + .../modules/audio_coding/neteq/random_vector.h | 50 + .../audio_coding/neteq/random_vector_unittest.cc | 25 + .../audio_coding/neteq/red_payload_splitter.cc | 162 + .../audio_coding/neteq/red_payload_splitter.h | 51 + .../neteq/red_payload_splitter_unittest.cc | 346 + .../webrtc/modules/audio_coding/neteq/rtcp.cc | 96 + .../webrtc/modules/audio_coding/neteq/rtcp.h | 58 + .../audio_coding/neteq/statistics_calculator.cc | 357 + .../audio_coding/neteq/statistics_calculator.h | 201 + .../neteq/statistics_calculator_unittest.cc | 107 + .../modules/audio_coding/neteq/sync_buffer.cc | 108 + .../modules/audio_coding/neteq/sync_buffer.h | 101 + .../audio_coding/neteq/sync_buffer_unittest.cc | 174 + .../neteq/test/delay_tool/parse_delay_file.m | 201 + .../neteq/test/delay_tool/plot_neteq_delay.m | 197 + .../neteq/test/neteq_ilbc_quality_test.cc | 79 + .../neteq/test/neteq_isac_quality_test.cc | 98 + .../neteq/test/neteq_opus_quality_test.cc | 174 + .../neteq/test/neteq_pcmu_quality_test.cc | 79 + .../neteq/test/neteq_performance_unittest.cc | 50 + .../audio_coding/neteq/test/neteq_speed_test.cc | 61 + .../modules/audio_coding/neteq/tick_timer.cc | 25 + .../webrtc/modules/audio_coding/neteq/tick_timer.h | 110 + .../audio_coding/neteq/tick_timer_unittest.cc | 135 + .../modules/audio_coding/neteq/time_stretch.cc | 214 + .../modules/audio_coding/neteq/time_stretch.h | 116 + .../audio_coding/neteq/time_stretch_unittest.cc | 127 + .../modules/audio_coding/neteq/timestamp_scaler.cc | 87 + .../modules/audio_coding/neteq/timestamp_scaler.h | 68 + .../neteq/timestamp_scaler_unittest.cc | 315 + .../webrtc/modules/audio_coding/neteq/tools/DEPS | 3 + .../audio_coding/neteq/tools/audio_checksum.h | 59 + .../modules/audio_coding/neteq/tools/audio_loop.cc | 59 + .../modules/audio_coding/neteq/tools/audio_loop.h | 59 + .../modules/audio_coding/neteq/tools/audio_sink.cc | 26 + .../modules/audio_coding/neteq/tools/audio_sink.h | 71 + .../neteq/tools/constant_pcm_packet_source.cc | 70 + .../neteq/tools/constant_pcm_packet_source.h | 55 + .../audio_coding/neteq/tools/encode_neteq_input.cc | 88 + .../audio_coding/neteq/tools/encode_neteq_input.h | 71 + .../neteq/tools/fake_decode_from_file.cc | 102 + .../neteq/tools/fake_decode_from_file.h | 73 + .../audio_coding/neteq/tools/input_audio_file.cc | 85 + .../audio_coding/neteq/tools/input_audio_file.h | 59 + .../neteq/tools/input_audio_file_unittest.cc | 59 + .../neteq/tools/neteq_delay_analyzer.cc | 256 + .../neteq/tools/neteq_delay_analyzer.h | 71 + .../neteq/tools/neteq_external_decoder_test.cc | 59 + .../neteq/tools/neteq_external_decoder_test.h | 65 + .../audio_coding/neteq/tools/neteq_input.cc | 32 + .../modules/audio_coding/neteq/tools/neteq_input.h | 83 + .../neteq/tools/neteq_packet_source_input.cc | 115 + .../neteq/tools/neteq_packet_source_input.h | 84 + .../neteq/tools/neteq_performance_test.cc | 133 + .../neteq/tools/neteq_performance_test.h | 32 + .../audio_coding/neteq/tools/neteq_quality_test.cc | 377 + .../audio_coding/neteq/tools/neteq_quality_test.h | 140 + .../neteq/tools/neteq_replacement_input.cc | 109 + .../neteq/tools/neteq_replacement_input.h | 51 + .../audio_coding/neteq/tools/neteq_rtpplay.cc | 670 + .../modules/audio_coding/neteq/tools/neteq_test.cc | 136 + .../modules/audio_coding/neteq/tools/neteq_test.h | 106 + .../audio_coding/neteq/tools/output_audio_file.h | 50 + .../audio_coding/neteq/tools/output_wav_file.h | 43 + .../modules/audio_coding/neteq/tools/packet.cc | 170 + .../modules/audio_coding/neteq/tools/packet.h | 117 + .../audio_coding/neteq/tools/packet_source.cc | 30 + .../audio_coding/neteq/tools/packet_source.h | 50 + .../audio_coding/neteq/tools/packet_unittest.cc | 202 + .../neteq/tools/resample_input_audio_file.cc | 48 + .../neteq/tools/resample_input_audio_file.h | 51 + .../neteq/tools/rtc_event_log_source.cc | 109 + .../neteq/tools/rtc_event_log_source.h | 65 + .../audio_coding/neteq/tools/rtp_analyze.cc | 174 + .../modules/audio_coding/neteq/tools/rtp_encode.cc | 357 + .../audio_coding/neteq/tools/rtp_file_source.cc | 103 + .../audio_coding/neteq/tools/rtp_file_source.h | 66 + .../audio_coding/neteq/tools/rtp_generator.cc | 61 + .../audio_coding/neteq/tools/rtp_generator.h | 83 + .../modules/audio_coding/neteq/tools/rtp_jitter.cc | 152 + .../modules/audio_coding/neteq/tools/rtpcat.cc | 45 + .../audio_coding/neteq_decoder_enum_gn/moz.build | 217 + .../webrtc/modules/audio_coding/neteq_gn/moz.build | 264 + .../modules/audio_coding/pcm16b_c_gn/moz.build | 209 + .../modules/audio_coding/pcm16b_gn/moz.build | 219 + .../modules/audio_coding/rent_a_codec_gn/moz.build | 390 + .../webrtc/modules/audio_coding/test/ACMTest.h | 21 + .../webrtc/modules/audio_coding/test/APITest.cc | 1106 + .../webrtc/modules/audio_coding/test/APITest.h | 162 + .../webrtc/modules/audio_coding/test/Channel.cc | 421 + .../webrtc/modules/audio_coding/test/Channel.h | 129 + .../modules/audio_coding/test/EncodeDecodeTest.cc | 344 + .../modules/audio_coding/test/EncodeDecodeTest.h | 123 + .../webrtc/modules/audio_coding/test/PCMFile.cc | 231 + .../webrtc/modules/audio_coding/test/PCMFile.h | 76 + .../modules/audio_coding/test/PacketLossTest.cc | 168 + .../modules/audio_coding/test/PacketLossTest.h | 67 + .../webrtc/modules/audio_coding/test/RTPFile.cc | 227 + .../webrtc/modules/audio_coding/test/RTPFile.h | 126 + .../modules/audio_coding/test/TestAllCodecs.cc | 485 + .../modules/audio_coding/test/TestAllCodecs.h | 83 + .../webrtc/modules/audio_coding/test/TestRedFec.cc | 471 + .../webrtc/modules/audio_coding/test/TestRedFec.h | 50 + .../webrtc/modules/audio_coding/test/TestStereo.cc | 847 + .../webrtc/modules/audio_coding/test/TestStereo.h | 116 + .../webrtc/modules/audio_coding/test/TestVADDTX.cc | 282 + .../webrtc/modules/audio_coding/test/TestVADDTX.h | 103 + .../webrtc/modules/audio_coding/test/Tester.cc | 139 + .../audio_coding/test/TwoWayCommunication.cc | 307 + .../audio_coding/test/TwoWayCommunication.h | 61 + .../webrtc/modules/audio_coding/test/delay_test.cc | 277 + .../webrtc/modules/audio_coding/test/iSACTest.cc | 348 + .../webrtc/modules/audio_coding/test/iSACTest.h | 80 + .../audio_coding/test/insert_packet_with_timing.cc | 322 + .../webrtc/modules/audio_coding/test/opus_test.cc | 390 + .../webrtc/modules/audio_coding/test/opus_test.h | 61 + .../audio_coding/test/target_delay_unittest.cc | 255 + .../webrtc/modules/audio_coding/test/utility.cc | 305 + .../webrtc/modules/audio_coding/test/utility.h | 138 + .../audio_coding/webrtc_opus_c_gn/moz.build | 280 + .../modules/audio_coding/webrtc_opus_gn/moz.build | 292 + .../libwebrtc/webrtc/modules/audio_device/BUILD.gn | 378 + .../libwebrtc/webrtc/modules/audio_device/DEPS | 31 + .../libwebrtc/webrtc/modules/audio_device/OWNERS | 9 + .../modules/audio_device/android/audio_common.h | 28 + .../audio_device/android/audio_device_template.h | 466 + .../audio_device/android/audio_device_unittest.cc | 1028 + .../modules/audio_device/android/audio_manager.cc | 311 + .../modules/audio_device/android/audio_manager.h | 216 + .../audio_device/android/audio_manager_unittest.cc | 241 + .../audio_device/android/audio_record_jni.cc | 259 + .../audio_device/android/audio_record_jni.h | 163 + .../audio_device/android/audio_track_jni.cc | 267 + .../modules/audio_device/android/audio_track_jni.h | 155 + .../modules/audio_device/android/build_info.cc | 59 + .../modules/audio_device/android/build_info.h | 85 + .../audio_device/android/ensure_initialized.cc | 47 + .../audio_device/android/ensure_initialized.h | 17 + .../java/src/org/webrtc/voiceengine/BuildInfo.java | 51 + .../org/webrtc/voiceengine/WebRtcAudioEffects.java | 321 + .../org/webrtc/voiceengine/WebRtcAudioManager.java | 370 + .../org/webrtc/voiceengine/WebRtcAudioRecord.java | 342 + .../org/webrtc/voiceengine/WebRtcAudioTrack.java | 522 + .../org/webrtc/voiceengine/WebRtcAudioUtils.java | 210 + .../audio_device/android/opensles_common.cc | 103 + .../modules/audio_device/android/opensles_common.h | 62 + .../audio_device/android/opensles_player.cc | 430 + .../modules/audio_device/android/opensles_player.h | 196 + .../audio_device/android/opensles_recorder.cc | 433 + .../audio_device/android/opensles_recorder.h | 193 + .../modules/audio_device/audio_device_buffer.cc | 500 + .../modules/audio_device/audio_device_buffer.h | 258 + .../modules/audio_device/audio_device_config.h | 31 + .../audio_device/audio_device_data_observer.cc | 305 + .../modules/audio_device/audio_device_generic.cc | 60 + .../modules/audio_device/audio_device_generic.h | 143 + .../audio_device/audio_device_generic_gn/moz.build | 253 + .../modules/audio_device/audio_device_gn/moz.build | 190 + .../modules/audio_device/audio_device_impl.cc | 903 + .../modules/audio_device/audio_device_impl.h | 174 + .../modules/audio_device/audio_device_unittest.cc | 750 + .../audio_device/dummy/audio_device_dummy.cc | 160 + .../audio_device/dummy/audio_device_dummy.h | 118 + .../audio_device/dummy/file_audio_device.cc | 511 + .../modules/audio_device/dummy/file_audio_device.h | 171 + .../dummy/file_audio_device_factory.cc | 59 + .../audio_device/dummy/file_audio_device_factory.h | 42 + .../modules/audio_device/fine_audio_buffer.cc | 92 + .../modules/audio_device/fine_audio_buffer.h | 91 + .../audio_device/fine_audio_buffer_unittest.cc | 145 + .../modules/audio_device/include/audio_device.h | 192 + .../include/audio_device_data_observer.h | 52 + .../audio_device/include/audio_device_defines.h | 152 + .../audio_device/include/fake_audio_device.h | 125 + .../audio_device/include/mock_audio_device.h | 101 + .../audio_device/include/mock_audio_transport.h | 68 + .../modules/audio_device/ios/audio_device_ios.h | 295 + .../modules/audio_device/ios/audio_device_ios.mm | 906 + .../ios/audio_device_not_implemented_ios.mm | 216 + .../audio_device/ios/audio_device_unittest_ios.mm | 880 + .../audio_device/ios/audio_session_observer.h | 42 + .../audio_device/ios/objc/RTCAudioSession.h | 11 + .../ios/objc/RTCAudioSessionConfiguration.h | 11 + .../ios/objc/RTCAudioSessionDelegateAdapter.h | 30 + .../ios/objc/RTCAudioSessionDelegateAdapter.mm | 89 + .../audio_device/ios/voice_processing_audio_unit.h | 137 + .../ios/voice_processing_audio_unit.mm | 467 + .../audio_device/linux/alsasymboltable_linux.cc | 41 + .../audio_device/linux/alsasymboltable_linux.h | 149 + .../audio_device/linux/audio_device_alsa_linux.cc | 1685 ++ .../audio_device/linux/audio_device_alsa_linux.h | 207 + .../audio_device/linux/audio_device_pulse_linux.cc | 2342 ++ .../audio_device/linux/audio_device_pulse_linux.h | 342 + .../linux/audio_mixer_manager_alsa_linux.cc | 972 + .../linux/audio_mixer_manager_alsa_linux.h | 72 + .../linux/audio_mixer_manager_pulse_linux.cc | 841 + .../linux/audio_mixer_manager_pulse_linux.h | 113 + .../linux/latebindingsymboltable_linux.cc | 105 + .../linux/latebindingsymboltable_linux.h | 178 + .../linux/pulseaudiosymboltable_linux.cc | 41 + .../linux/pulseaudiosymboltable_linux.h | 106 + .../modules/audio_device/mac/audio_device_mac.cc | 2574 ++ .../modules/audio_device/mac/audio_device_mac.h | 344 + .../audio_device/mac/audio_mixer_manager_mac.cc | 916 + .../audio_device/mac/audio_mixer_manager_mac.h | 72 + .../audio_device/mac/portaudio/pa_memorybarrier.h | 127 + .../audio_device/mac/portaudio/pa_ringbuffer.c | 237 + .../audio_device/mac/portaudio/pa_ringbuffer.h | 233 + .../audio_device/mock_audio_device_buffer.h | 34 + .../modules/audio_device/opensl/single_rw_fifo.cc | 123 + .../modules/audio_device/opensl/single_rw_fifo.h | 48 + .../audio_device/win/audio_device_core_win.cc | 4360 ++++ .../audio_device/win/audio_device_core_win.h | 334 + .../libwebrtc/webrtc/modules/audio_mixer/BUILD.gn | 95 + .../libwebrtc/webrtc/modules/audio_mixer/DEPS | 14 + .../libwebrtc/webrtc/modules/audio_mixer/OWNERS | 7 + .../modules/audio_mixer/audio_frame_manipulator.cc | 65 + .../modules/audio_mixer/audio_frame_manipulator.h | 30 + .../audio_frame_manipulator_gn/moz.build | 225 + .../audio_frame_manipulator_unittest.cc | 66 + .../webrtc/modules/audio_mixer/audio_mixer_impl.cc | 249 + .../webrtc/modules/audio_mixer/audio_mixer_impl.h | 114 + .../audio_mixer/audio_mixer_impl_gn/moz.build | 228 + .../audio_mixer/audio_mixer_impl_unittest.cc | 543 + .../audio_mixer/default_output_rate_calculator.cc | 39 + .../audio_mixer/default_output_rate_calculator.h | 35 + .../webrtc/modules/audio_mixer/frame_combiner.cc | 218 + .../webrtc/modules/audio_mixer/frame_combiner.h | 45 + .../modules/audio_mixer/frame_combiner_unittest.cc | 205 + .../modules/audio_mixer/gain_change_calculator.cc | 55 + .../modules/audio_mixer/gain_change_calculator.h | 38 + .../modules/audio_mixer/output_rate_calculator.h | 29 + .../modules/audio_mixer/sine_wave_generator.cc | 34 + .../modules/audio_mixer/sine_wave_generator.h | 38 + .../webrtc/modules/audio_processing/BUILD.gn | 905 + .../libwebrtc/webrtc/modules/audio_processing/DEPS | 14 + .../webrtc/modules/audio_processing/OWNERS | 9 + .../modules/audio_processing/aec/aec_common.h | 39 + .../modules/audio_processing/aec/aec_core.cc | 2057 ++ .../webrtc/modules/audio_processing/aec/aec_core.h | 335 + .../modules/audio_processing/aec/aec_core_mips.cc | 490 + .../modules/audio_processing/aec/aec_core_neon.cc | 737 + .../aec/aec_core_optimized_methods.h | 80 + .../modules/audio_processing/aec/aec_core_sse2.cc | 751 + .../modules/audio_processing/aec/aec_resampler.cc | 207 + .../modules/audio_processing/aec/aec_resampler.h | 39 + .../audio_processing/aec/echo_cancellation.cc | 868 + .../audio_processing/aec/echo_cancellation.h | 300 + .../aec/echo_cancellation_unittest.cc | 45 + .../audio_processing/aec/system_delay_unittest.cc | 601 + .../audio_processing/aec3/adaptive_fir_filter.cc | 525 + .../audio_processing/aec3/adaptive_fir_filter.h | 153 + .../aec3/adaptive_fir_filter_unittest.cc | 379 + .../modules/audio_processing/aec3/aec3_common.cc | 32 + .../modules/audio_processing/aec3/aec3_common.h | 120 + .../modules/audio_processing/aec3/aec3_fft.cc | 42 + .../modules/audio_processing/aec3/aec3_fft.h | 60 + .../audio_processing/aec3/aec3_fft_unittest.cc | 211 + .../modules/audio_processing/aec3/aec_state.cc | 321 + .../modules/audio_processing/aec3/aec_state.h | 183 + .../audio_processing/aec3/aec_state_unittest.cc | 231 + .../modules/audio_processing/aec3/block_framer.cc | 59 + .../modules/audio_processing/aec3/block_framer.h | 47 + .../audio_processing/aec3/block_framer_unittest.cc | 261 + .../audio_processing/aec3/block_processor.cc | 231 + .../audio_processing/aec3/block_processor.h | 62 + .../aec3/block_processor_metrics.cc | 103 + .../aec3/block_processor_metrics.h | 47 + .../aec3/block_processor_metrics_unittest.cc | 34 + .../aec3/block_processor_unittest.cc | 260 + .../aec3/cascaded_biquad_filter.cc | 58 + .../audio_processing/aec3/cascaded_biquad_filter.h | 58 + .../aec3/cascaded_biquad_filter_unittest.cc | 98 + .../aec3/comfort_noise_generator.cc | 219 + .../aec3/comfort_noise_generator.h | 68 + .../aec3/comfort_noise_generator_unittest.cc | 121 + .../modules/audio_processing/aec3/decimator.cc | 70 + .../modules/audio_processing/aec3/decimator.h | 39 + .../audio_processing/aec3/decimator_unittest.cc | 148 + .../aec3/downsampled_render_buffer.cc | 20 + .../aec3/downsampled_render_buffer.h | 30 + .../audio_processing/aec3/echo_canceller3.cc | 374 + .../audio_processing/aec3/echo_canceller3.h | 137 + .../aec3/echo_canceller3_unittest.cc | 748 + .../aec3/echo_path_delay_estimator.cc | 87 + .../aec3/echo_path_delay_estimator.h | 61 + .../aec3/echo_path_delay_estimator_unittest.cc | 232 + .../audio_processing/aec3/echo_path_variability.cc | 18 + .../audio_processing/aec3/echo_path_variability.h | 26 + .../aec3/echo_path_variability_unittest.cc | 39 + .../modules/audio_processing/aec3/echo_remover.cc | 256 + .../modules/audio_processing/aec3/echo_remover.h | 50 + .../audio_processing/aec3/echo_remover_metrics.cc | 282 + .../audio_processing/aec3/echo_remover_metrics.h | 77 + .../aec3/echo_remover_metrics_unittest.cc | 144 + .../audio_processing/aec3/echo_remover_unittest.cc | 216 + .../modules/audio_processing/aec3/erl_estimator.cc | 84 + .../modules/audio_processing/aec3/erl_estimator.h | 46 + .../aec3/erl_estimator_unittest.cc | 72 + .../audio_processing/aec3/erle_estimator.cc | 90 + .../modules/audio_processing/aec3/erle_estimator.h | 50 + .../aec3/erle_estimator_unittest.cc | 75 + .../modules/audio_processing/aec3/fft_data.h | 98 + .../audio_processing/aec3/fft_data_unittest.cc | 163 + .../modules/audio_processing/aec3/frame_blocker.cc | 71 + .../modules/audio_processing/aec3/frame_blocker.h | 48 + .../aec3/frame_blocker_unittest.cc | 341 + .../aec3/main_filter_update_gain.cc | 118 + .../aec3/main_filter_update_gain.h | 56 + .../aec3/main_filter_update_gain_unittest.cc | 291 + .../audio_processing/aec3/matched_filter.cc | 443 + .../modules/audio_processing/aec3/matched_filter.h | 129 + .../aec3/matched_filter_lag_aggregator.cc | 76 + .../aec3/matched_filter_lag_aggregator.h | 48 + .../aec3/matched_filter_lag_aggregator_unittest.cc | 126 + .../aec3/matched_filter_unittest.cc | 377 + .../aec3/mock/mock_block_processor.h | 39 + .../audio_processing/aec3/mock/mock_echo_remover.h | 43 + .../aec3/mock/mock_render_delay_buffer.h | 65 + .../aec3/mock/mock_render_delay_controller.h | 38 + .../audio_processing/aec3/output_selector.cc | 59 + .../audio_processing/aec3/output_selector.h | 41 + .../aec3/output_selector_unittest.cc | 69 + .../modules/audio_processing/aec3/render_buffer.cc | 95 + .../modules/audio_processing/aec3/render_buffer.h | 81 + .../aec3/render_buffer_unittest.cc | 46 + .../audio_processing/aec3/render_delay_buffer.cc | 253 + .../audio_processing/aec3/render_delay_buffer.h | 62 + .../aec3/render_delay_buffer_unittest.cc | 125 + .../aec3/render_delay_controller.cc | 179 + .../aec3/render_delay_controller.h | 45 + .../aec3/render_delay_controller_metrics.cc | 117 + .../aec3/render_delay_controller_metrics.h | 47 + .../render_delay_controller_metrics_unittest.cc | 33 + .../aec3/render_delay_controller_unittest.cc | 334 + .../aec3/render_signal_analyzer.cc | 126 + .../audio_processing/aec3/render_signal_analyzer.h | 58 + .../aec3/render_signal_analyzer_unittest.cc | 122 + .../aec3/residual_echo_estimator.cc | 292 + .../aec3/residual_echo_estimator.h | 83 + .../aec3/residual_echo_estimator_unittest.cc | 95 + .../aec3/shadow_filter_update_gain.cc | 69 + .../aec3/shadow_filter_update_gain.h | 44 + .../aec3/shadow_filter_update_gain_unittest.cc | 186 + .../modules/audio_processing/aec3/subtractor.cc | 132 + .../modules/audio_processing/aec3/subtractor.h | 76 + .../audio_processing/aec3/subtractor_output.h | 43 + .../audio_processing/aec3/subtractor_unittest.cc | 172 + .../audio_processing/aec3/suppression_filter.cc | 172 + .../audio_processing/aec3/suppression_filter.h | 44 + .../aec3/suppression_filter_unittest.cc | 181 + .../audio_processing/aec3/suppression_gain.cc | 426 + .../audio_processing/aec3/suppression_gain.h | 71 + .../aec3/suppression_gain_unittest.cc | 135 + .../modules/audio_processing/aec3/vector_math.h | 212 + .../audio_processing/aec3/vector_math_unittest.cc | 146 + .../modules/audio_processing/aec_dump/BUILD.gn | 114 + .../audio_processing/aec_dump/aec_dump_factory.h | 47 + .../aec_dump/aec_dump_gn/moz.build | 179 + .../audio_processing/aec_dump/aec_dump_impl.cc | 207 + .../audio_processing/aec_dump/aec_dump_impl.h | 80 + .../aec_dump/aec_dump_integration_test.cc | 91 + .../audio_processing/aec_dump/aec_dump_unittest.cc | 71 + .../aec_dump/capture_stream_info.cc | 69 + .../aec_dump/capture_stream_info.h | 66 + .../audio_processing/aec_dump/mock_aec_dump.cc | 19 + .../audio_processing/aec_dump/mock_aec_dump.h | 50 + .../aec_dump/null_aec_dump_factory.cc | 33 + .../aec_dump/null_aec_dump_factory_gn/moz.build | 217 + .../aec_dump/write_to_file_task.cc | 68 + .../audio_processing/aec_dump/write_to_file_task.h | 58 + .../aec_dump_interface_gn/moz.build | 217 + .../modules/audio_processing/aecm/aecm_core.cc | 1232 + .../modules/audio_processing/aecm/aecm_core.h | 436 + .../modules/audio_processing/aecm/aecm_core_c.cc | 773 + .../audio_processing/aecm/aecm_core_mips.cc | 1566 ++ .../audio_processing/aecm/aecm_core_neon.cc | 199 + .../modules/audio_processing/aecm/aecm_defines.h | 87 + .../audio_processing/aecm/echo_control_mobile.cc | 648 + .../audio_processing/aecm/echo_control_mobile.h | 209 + .../webrtc/modules/audio_processing/agc/agc.cc | 108 + .../webrtc/modules/audio_processing/agc/agc.h | 56 + .../audio_processing/agc/agc_manager_direct.cc | 461 + .../audio_processing/agc/agc_manager_direct.h | 113 + .../agc/agc_manager_direct_unittest.cc | 682 + .../audio_processing/agc/gain_map_internal.h | 275 + .../audio_processing/agc/legacy/analog_agc.c | 1390 ++ .../audio_processing/agc/legacy/analog_agc.h | 132 + .../audio_processing/agc/legacy/digital_agc.c | 703 + .../audio_processing/agc/legacy/digital_agc.h | 79 + .../audio_processing/agc/legacy/gain_control.h | 247 + .../audio_processing/agc/loudness_histogram.cc | 229 + .../audio_processing/agc/loudness_histogram.h | 92 + .../agc/loudness_histogram_unittest.cc | 106 + .../webrtc/modules/audio_processing/agc/mock_agc.h | 36 + .../webrtc/modules/audio_processing/agc/utility.cc | 35 + .../webrtc/modules/audio_processing/agc/utility.h | 23 + .../audio_processing/agc2/gain_controller2.cc | 75 + .../audio_processing/agc2/gain_controller2.h | 53 + .../agc2/gain_controller2_unittest.cc | 97 + .../modules/audio_processing/audio_buffer.cc | 475 + .../webrtc/modules/audio_processing/audio_buffer.h | 166 + .../audio_processing/audio_buffer_unittest.cc | 48 + .../audio_processing_c_gn/moz.build | 317 + .../audio_processing/audio_processing_gn/moz.build | 370 + .../audio_processing/audio_processing_impl.cc | 2011 ++ .../audio_processing/audio_processing_impl.h | 426 + .../audio_processing_impl_locking_unittest.cc | 1135 + .../audio_processing_impl_unittest.cc | 78 + .../audio_processing_neon_c_gn/moz.build | 183 + .../audio_processing_neon_gn/moz.build | 200 + .../audio_processing_performance_unittest.cc | 713 + .../audio_processing_sse2_gn/moz.build | 218 + .../audio_processing_statistics_gn/moz.build | 217 + .../audio_processing/audio_processing_unittest.cc | 3117 +++ .../audio_processing/beamformer/array_util.cc | 119 + .../audio_processing/beamformer/array_util.h | 117 + .../beamformer/array_util_unittest.cc | 185 + .../audio_processing/beamformer/complex_matrix.h | 96 + .../beamformer/complex_matrix_unittest.cc | 102 + .../beamformer/covariance_matrix_generator.cc | 114 + .../beamformer/covariance_matrix_generator.h | 54 + .../covariance_matrix_generator_unittest.cc | 230 + .../modules/audio_processing/beamformer/matrix.h | 369 + .../beamformer/matrix_test_helpers.h | 102 + .../audio_processing/beamformer/matrix_unittest.cc | 326 + .../beamformer/mock_nonlinear_beamformer.h | 39 + .../beamformer/nonlinear_beamformer.cc | 599 + .../beamformer/nonlinear_beamformer.h | 228 + .../beamformer/nonlinear_beamformer_test.cc | 96 + .../beamformer/nonlinear_beamformer_unittest.cc | 375 + .../webrtc/modules/audio_processing/common.h | 34 + .../modules/audio_processing/config_unittest.cc | 83 + .../webrtc/modules/audio_processing/debug.proto | 94 + .../echo_cancellation_bit_exact_unittest.cc | 357 + .../audio_processing/echo_cancellation_impl.cc | 516 + .../audio_processing/echo_cancellation_impl.h | 119 + .../echo_cancellation_impl_unittest.cc | 79 + .../audio_processing/echo_control_mobile_impl.cc | 391 + .../audio_processing/echo_control_mobile_impl.h | 84 + .../echo_control_mobile_unittest.cc | 224 + .../echo_detector/circular_buffer.cc | 49 + .../echo_detector/circular_buffer.h | 42 + .../echo_detector/circular_buffer_unittest.cc | 52 + .../echo_detector/mean_variance_estimator.cc | 47 + .../echo_detector/mean_variance_estimator.h | 33 + .../mean_variance_estimator_unittest.cc | 64 + .../audio_processing/echo_detector/moving_max.cc | 52 + .../audio_processing/echo_detector/moving_max.h | 36 + .../echo_detector/moving_max_unittest.cc | 67 + .../normalized_covariance_estimator.cc | 43 + .../normalized_covariance_estimator.h | 43 + .../normalized_covariance_estimator_unittest.cc | 40 + .../gain_control_for_experimental_agc.cc | 120 + .../gain_control_for_experimental_agc.h | 77 + .../modules/audio_processing/gain_control_impl.cc | 439 + .../modules/audio_processing/gain_control_impl.h | 97 + .../audio_processing/gain_control_unittest.cc | 441 + .../modules/audio_processing/include/aec_dump.cc | 40 + .../modules/audio_processing/include/aec_dump.h | 141 + .../audio_processing/include/audio_processing.cc | 35 + .../audio_processing/include/audio_processing.h | 1228 + .../include/audio_processing_statistics.cc | 22 + .../include/audio_processing_statistics.h | 56 + .../modules/audio_processing/include/config.cc | 23 + .../modules/audio_processing/include/config.h | 134 + .../include/mock_audio_processing.h | 241 + .../intelligibility/intelligibility_enhancer.cc | 392 + .../intelligibility/intelligibility_enhancer.h | 137 + .../intelligibility_enhancer_unittest.cc | 536 + .../intelligibility/intelligibility_utils.cc | 94 + .../intelligibility/intelligibility_utils.h | 86 + .../intelligibility_utils_unittest.cc | 79 + .../intelligibility/test/intelligibility_proc.cc | 96 + .../level_controller/biquad_filter.cc | 35 + .../level_controller/biquad_filter.h | 58 + .../level_controller/down_sampler.cc | 100 + .../level_controller/down_sampler.h | 40 + .../level_controller/gain_applier.cc | 160 + .../level_controller/gain_applier.h | 42 + .../level_controller/gain_selector.cc | 87 + .../level_controller/gain_selector.h | 40 + .../level_controller/level_controller.cc | 293 + .../level_controller/level_controller.h | 95 + .../level_controller_complexity_unittest.cc | 240 + .../level_controller/level_controller_constants.h | 23 + .../level_controller/level_controller_unittest.cc | 156 + .../level_controller/noise_level_estimator.cc | 72 + .../level_controller/noise_level_estimator.h | 37 + .../level_controller/noise_spectrum_estimator.cc | 68 + .../level_controller/noise_spectrum_estimator.h | 40 + .../level_controller/peak_level_estimator.cc | 74 + .../level_controller/peak_level_estimator.h | 37 + .../level_controller/saturating_gain_estimator.cc | 48 + .../level_controller/saturating_gain_estimator.h | 37 + .../level_controller/signal_classifier.cc | 171 + .../level_controller/signal_classifier.h | 67 + .../audio_processing/level_estimator_impl.cc | 66 + .../audio_processing/level_estimator_impl.h | 47 + .../audio_processing/level_estimator_unittest.cc | 93 + .../audio_processing/logging/apm_data_dumper.cc | 88 + .../audio_processing/logging/apm_data_dumper.h | 277 + .../modules/audio_processing/low_cut_filter.cc | 101 + .../modules/audio_processing/low_cut_filter.h | 36 + .../audio_processing/low_cut_filter_unittest.cc | 682 + .../audio_processing/noise_suppression_impl.cc | 213 + .../audio_processing/noise_suppression_impl.h | 56 + .../audio_processing/noise_suppression_unittest.cc | 284 + .../webrtc/modules/audio_processing/ns/defines.h | 49 + .../audio_processing/ns/noise_suppression.c | 71 + .../audio_processing/ns/noise_suppression.h | 135 + .../audio_processing/ns/noise_suppression_x.c | 61 + .../audio_processing/ns/noise_suppression_x.h | 113 + .../webrtc/modules/audio_processing/ns/ns_core.c | 1418 ++ .../webrtc/modules/audio_processing/ns/ns_core.h | 190 + .../webrtc/modules/audio_processing/ns/nsx_core.c | 2107 ++ .../webrtc/modules/audio_processing/ns/nsx_core.h | 263 + .../modules/audio_processing/ns/nsx_core_c.c | 260 + .../modules/audio_processing/ns/nsx_core_mips.c | 1002 + .../modules/audio_processing/ns/nsx_core_neon.c | 606 + .../modules/audio_processing/ns/nsx_defines.h | 64 + .../modules/audio_processing/ns/windows_private.h | 574 + .../audio_processing/render_queue_item_verifier.h | 36 + .../audio_processing/residual_echo_detector.cc | 204 + .../audio_processing/residual_echo_detector.h | 94 + .../residual_echo_detector_unittest.cc | 128 + .../webrtc/modules/audio_processing/rms_level.cc | 107 + .../webrtc/modules/audio_processing/rms_level.h | 75 + .../modules/audio_processing/rms_level_unittest.cc | 150 + .../modules/audio_processing/splitting_filter.cc | 108 + .../modules/audio_processing/splitting_filter.h | 69 + .../audio_processing/splitting_filter_unittest.cc | 106 + .../test/aec_dump_based_simulator.cc | 557 + .../test/aec_dump_based_simulator.h | 67 + .../test/android/apmtest/AndroidManifest.xml | 30 + .../test/android/apmtest/default.properties | 11 + .../test/android/apmtest/jni/main.c | 307 + .../test/android/apmtest/res/values/strings.xml | 4 + .../webrtc/modules/audio_processing/test/apmtest.m | 365 + .../audio_processing/test/audio_buffer_tools.cc | 55 + .../audio_processing/test/audio_buffer_tools.h | 35 + .../test/audio_processing_simulator.cc | 451 + .../test/audio_processing_simulator.h | 197 + .../audio_processing/test/audioproc_float.cc | 517 + .../audio_processing/test/bitexactness_tools.cc | 146 + .../audio_processing/test/bitexactness_tools.h | 56 + .../test/conversational_speech/BUILD.gn | 75 + .../test/conversational_speech/OWNERS | 6 + .../test/conversational_speech/README.md | 72 + .../test/conversational_speech/config.cc | 31 + .../test/conversational_speech/config.h | 41 + .../test/conversational_speech/generator.cc | 90 + .../conversational_speech/generator_unittest.cc | 688 + .../test/conversational_speech/mock_wavreader.cc | 32 + .../test/conversational_speech/mock_wavreader.h | 49 + .../mock_wavreader_factory.cc | 69 + .../conversational_speech/mock_wavreader_factory.h | 56 + .../test/conversational_speech/multiend_call.cc | 190 + .../test/conversational_speech/multiend_call.h | 93 + .../test/conversational_speech/simulator.cc | 221 + .../test/conversational_speech/simulator.h | 44 + .../test/conversational_speech/timing.cc | 66 + .../test/conversational_speech/timing.h | 46 + .../wavreader_abstract_factory.h | 34 + .../conversational_speech/wavreader_factory.cc | 71 + .../test/conversational_speech/wavreader_factory.h | 36 + .../conversational_speech/wavreader_interface.h | 41 + .../audio_processing/test/debug_dump_replayer.cc | 272 + .../audio_processing/test/debug_dump_replayer.h | 77 + .../audio_processing/test/debug_dump_test.cc | 620 + .../test/echo_canceller_test_tools.cc | 40 + .../test/echo_canceller_test_tools.h | 44 + .../test/echo_canceller_test_tools_unittest.cc | 71 + .../audio_processing/test/fake_recording_device.cc | 152 + .../audio_processing/test/fake_recording_device.h | 75 + .../test/fake_recording_device_unittest.cc | 231 + .../audio_processing/test/performance_timer.cc | 75 + .../audio_processing/test/performance_timer.h | 47 + .../audio_processing/test/protobuf_utils.cc | 43 + .../modules/audio_processing/test/protobuf_utils.h | 34 + .../test/py_quality_assessment/BUILD.gn | 174 + .../test/py_quality_assessment/OWNERS | 8 + .../test/py_quality_assessment/README.md | 125 + .../py_quality_assessment/apm_configs/default.json | 1 + .../apm_quality_assessment.py | 178 + .../apm_quality_assessment.sh | 91 + .../apm_quality_assessment_boxplot.py | 146 + .../apm_quality_assessment_export.py | 61 + .../apm_quality_assessment_gencfgs.py | 100 + .../apm_quality_assessment_optimize.py | 179 + .../apm_quality_assessment_unittest.py | 33 + .../test/py_quality_assessment/output/README.md | 1 + .../quality_assessment/__init__.py | 7 + .../quality_assessment/annotations.py | 293 + .../quality_assessment/annotations_unittest.py | 157 + .../quality_assessment/apm_configs/default.json | 1 + .../quality_assessment/apm_vad.cc | 94 + .../quality_assessment/audioproc_wrapper.py | 96 + .../quality_assessment/collect_data.py | 242 + .../quality_assessment/data_access.py | 154 + .../quality_assessment/echo_path_simulation.py | 136 + .../echo_path_simulation_factory.py | 48 + .../echo_path_simulation_unittest.py | 81 + .../quality_assessment/eval_scores.py | 341 + .../quality_assessment/eval_scores_factory.py | 51 + .../quality_assessment/eval_scores_unittest.py | 131 + .../quality_assessment/evaluation.py | 53 + .../quality_assessment/exceptions.py | 46 + .../quality_assessment/export.py | 402 + .../quality_assessment/export_unittest.py | 84 + .../quality_assessment/external_vad.py | 77 + .../quality_assessment/fake_external_vad.py | 24 + .../quality_assessment/fake_polqa.cc | 54 + .../quality_assessment/input_mixer.py | 95 + .../quality_assessment/input_mixer_unittest.py | 149 + .../quality_assessment/input_signal_creator.py | 67 + .../quality_assessment/results.css | 32 + .../quality_assessment/results.js | 376 + .../quality_assessment/signal_processing.py | 356 + .../signal_processing_unittest.py | 186 + .../quality_assessment/simulation.py | 422 + .../quality_assessment/simulation_unittest.py | 203 + .../quality_assessment/test_data_generation.py | 511 + .../test_data_generation_factory.py | 71 + .../test_data_generation_unittest.py | 206 + .../quality_assessment/vad.cc | 101 + .../audio_processing/test/simulator_buffers.cc | 85 + .../audio_processing/test/simulator_buffers.h | 66 + .../modules/audio_processing/test/test_utils.cc | 162 + .../modules/audio_processing/test/test_utils.h | 158 + .../modules/audio_processing/test/unittest.proto | 58 + .../webrtc/modules/audio_processing/test/unpack.cc | 346 + .../audio_processing/test/wav_based_simulator.cc | 185 + .../audio_processing/test/wav_based_simulator.h | 55 + .../audio_processing/three_band_filter_bank.cc | 216 + .../audio_processing/three_band_filter_bank.h | 69 + .../audio_processing/transient/click_annotate.cc | 113 + .../modules/audio_processing/transient/common.h | 27 + .../transient/daubechies_8_wavelet_coeffs.h | 63 + .../audio_processing/transient/dyadic_decimator.h | 70 + .../transient/dyadic_decimator_unittest.cc | 126 + .../audio_processing/transient/file_utils.cc | 258 + .../audio_processing/transient/file_utils.h | 118 + .../transient/file_utils_unittest.cc | 539 + .../audio_processing/transient/moving_moments.cc | 52 + .../audio_processing/transient/moving_moments.h | 52 + .../transient/moving_moments_unittest.cc | 207 + .../transient/test/plotDetection.m | 22 + .../transient/test/readDetection.m | 26 + .../audio_processing/transient/test/readPCM.m | 26 + .../transient/transient_detector.cc | 176 + .../transient/transient_detector.h | 87 + .../transient/transient_detector_unittest.cc | 104 + .../transient/transient_suppression_test.cc | 252 + .../transient/transient_suppressor.cc | 424 + .../transient/transient_suppressor.h | 120 + .../transient/transient_suppressor_unittest.cc | 85 + .../modules/audio_processing/transient/wpd_node.cc | 73 + .../modules/audio_processing/transient/wpd_node.h | 47 + .../transient/wpd_node_unittest.cc | 65 + .../modules/audio_processing/transient/wpd_tree.cc | 118 + .../modules/audio_processing/transient/wpd_tree.h | 92 + .../transient/wpd_tree_unittest.cc | 193 + .../modules/audio_processing/typing_detection.cc | 90 + .../modules/audio_processing/typing_detection.h | 93 + .../utility/block_mean_calculator.cc | 53 + .../utility/block_mean_calculator.h | 52 + .../utility/block_mean_calculator_unittest.cc | 58 + .../audio_processing/utility/delay_estimator.cc | 703 + .../audio_processing/utility/delay_estimator.h | 251 + .../utility/delay_estimator_internal.h | 48 + .../utility/delay_estimator_unittest.cc | 618 + .../utility/delay_estimator_wrapper.cc | 486 + .../utility/delay_estimator_wrapper.h | 244 + .../modules/audio_processing/utility/ooura_fft.cc | 543 + .../modules/audio_processing/utility/ooura_fft.h | 60 + .../audio_processing/utility/ooura_fft_mips.cc | 1185 + .../audio_processing/utility/ooura_fft_neon.cc | 352 + .../audio_processing/utility/ooura_fft_sse2.cc | 438 + .../utility/ooura_fft_tables_common.h | 54 + .../utility/ooura_fft_tables_neon_sse2.h | 94 + .../webrtc/modules/audio_processing/vad/common.h | 29 + .../webrtc/modules/audio_processing/vad/gmm.cc | 64 + .../webrtc/modules/audio_processing/vad/gmm.h | 45 + .../modules/audio_processing/vad/gmm_unittest.cc | 65 + .../audio_processing/vad/noise_gmm_tables.h | 85 + .../audio_processing/vad/pitch_based_vad.cc | 123 + .../modules/audio_processing/vad/pitch_based_vad.h | 58 + .../vad/pitch_based_vad_unittest.cc | 75 + .../modules/audio_processing/vad/pitch_internal.cc | 51 + .../modules/audio_processing/vad/pitch_internal.h | 26 + .../vad/pitch_internal_unittest.cc | 50 + .../audio_processing/vad/pole_zero_filter.cc | 106 + .../audio_processing/vad/pole_zero_filter.h | 52 + .../vad/pole_zero_filter_unittest.cc | 103 + .../modules/audio_processing/vad/standalone_vad.cc | 92 + .../modules/audio_processing/vad/standalone_vad.h | 69 + .../vad/standalone_vad_unittest.cc | 108 + .../modules/audio_processing/vad/vad_audio_proc.cc | 276 + .../modules/audio_processing/vad/vad_audio_proc.h | 93 + .../audio_processing/vad/vad_audio_proc_internal.h | 94 + .../vad/vad_audio_proc_unittest.cc | 63 + .../audio_processing/vad/vad_circular_buffer.cc | 137 + .../audio_processing/vad/vad_circular_buffer.h | 69 + .../vad/vad_circular_buffer_unittest.cc | 134 + .../vad/voice_activity_detector.cc | 85 + .../audio_processing/vad/voice_activity_detector.h | 69 + .../vad/voice_activity_detector_unittest.cc | 168 + .../audio_processing/vad/voice_gmm_tables.h | 85 + .../audio_processing/voice_detection_impl.cc | 155 + .../audio_processing/voice_detection_impl.h | 58 + .../audio_processing/voice_detection_unittest.cc | 123 + .../webrtc/modules/bitrate_controller/BUILD.gn | 68 + .../webrtc/modules/bitrate_controller/DEPS | 5 + .../webrtc/modules/bitrate_controller/OWNERS | 8 + .../bitrate_controller_gn/moz.build | 227 + .../bitrate_controller/bitrate_controller_impl.cc | 292 + .../bitrate_controller/bitrate_controller_impl.h | 108 + .../bitrate_controller_unittest.cc | 507 + .../include/bitrate_controller.h | 102 + .../include/mock/mock_bitrate_controller.h | 54 + .../send_side_bandwidth_estimation.cc | 427 + .../send_side_bandwidth_estimation.h | 107 + .../send_side_bandwidth_estimation_unittest.cc | 168 + .../webrtc/modules/congestion_controller/BUILD.gn | 122 + .../webrtc/modules/congestion_controller/DEPS | 5 + .../webrtc/modules/congestion_controller/OWNERS | 9 + .../acknowledged_bitrate_estimator.cc | 63 + .../acknowledged_bitrate_estimator.h | 44 + .../acknowledged_bitrate_estimator_unittest.cc | 134 + .../congestion_controller/bitrate_estimator.cc | 107 + .../congestion_controller/bitrate_estimator.h | 46 + .../congestion_controller_gn/moz.build | 238 + .../congestion_controller_unittests_helper.cc | 42 + .../congestion_controller_unittests_helper.h | 23 + .../congestion_controller/delay_based_bwe.cc | 337 + .../congestion_controller/delay_based_bwe.h | 97 + .../delay_based_bwe_unittest.cc | 234 + .../delay_based_bwe_unittest_helper.cc | 513 + .../delay_based_bwe_unittest_helper.h | 178 + .../include/mock/mock_congestion_observer.h | 31 + .../mock/mock_send_side_congestion_controller.h | 39 + .../include/receive_side_congestion_controller.h | 104 + .../include/send_side_congestion_controller.h | 166 + .../median_slope_estimator.cc | 93 + .../congestion_controller/median_slope_estimator.h | 72 + .../median_slope_estimator_unittest.cc | 72 + .../probe_bitrate_estimator.cc | 187 + .../probe_bitrate_estimator.h | 55 + .../probe_bitrate_estimator_unittest.cc | 221 + .../congestion_controller/probe_controller.cc | 301 + .../congestion_controller/probe_controller.h | 94 + .../probe_controller_unittest.cc | 270 + .../receive_side_congestion_controller.cc | 175 + .../receive_side_congestion_controller_unittest.cc | 75 + .../send_side_congestion_controller.cc | 458 + .../send_side_congestion_controller_unittest.cc | 494 + .../transport_feedback_adapter.cc | 218 + .../transport_feedback_adapter.h | 82 + .../transport_feedback_adapter_unittest.cc | 391 + .../congestion_controller/trendline_estimator.cc | 97 + .../congestion_controller/trendline_estimator.h | 71 + .../trendline_estimator_unittest.cc | 73 + .../webrtc/modules/desktop_capture/BUILD.gn | 428 + .../libwebrtc/webrtc/modules/desktop_capture/DEPS | 4 + .../webrtc/modules/desktop_capture/OWNERS | 10 + .../blank_detector_desktop_capturer_wrapper.cc | 118 + .../blank_detector_desktop_capturer_wrapper.h | 74 + ...k_detector_desktop_capturer_wrapper_unittest.cc | 163 + .../capture_result_desktop_capturer_wrapper.cc | 49 + .../capture_result_desktop_capturer_wrapper.h | 64 + .../desktop_capture/cropped_desktop_frame.cc | 62 + .../desktop_capture/cropped_desktop_frame.h | 30 + .../cropped_desktop_frame_unittest.cc | 66 + .../desktop_capture/cropping_window_capturer.cc | 124 + .../desktop_capture/cropping_window_capturer.h | 79 + .../cropping_window_capturer_win.cc | 267 + .../desktop_capture/desktop_and_cursor_composer.cc | 231 + .../desktop_capture/desktop_and_cursor_composer.h | 106 + .../desktop_and_cursor_composer_unittest.cc | 368 + .../desktop_capture_differ_sse2_gn/moz.build | 184 + .../desktop_capture_generic_gn/moz.build | 444 + .../desktop_capture/desktop_capture_gn/moz.build | 231 + .../desktop_capture_objc_gn/moz.build | 70 + .../desktop_capture/desktop_capture_options.cc | 41 + .../desktop_capture/desktop_capture_options.h | 150 + .../desktop_capture/desktop_capture_types.h | 69 + .../modules/desktop_capture/desktop_capturer.cc | 92 + .../modules/desktop_capture/desktop_capturer.h | 177 + .../desktop_capturer_differ_wrapper.cc | 225 + .../desktop_capturer_differ_wrapper.h | 62 + .../desktop_capturer_differ_wrapper_unittest.cc | 291 + .../desktop_capture/desktop_capturer_wrapper.cc | 60 + .../desktop_capture/desktop_capturer_wrapper.h | 45 + .../modules/desktop_capture/desktop_device_info.cc | 313 + .../modules/desktop_capture/desktop_device_info.h | 121 + .../desktop_capture/desktop_device_info_null.cc | 43 + .../modules/desktop_capture/desktop_frame.cc | 142 + .../webrtc/modules/desktop_capture/desktop_frame.h | 188 + .../desktop_capture/desktop_frame_generator.cc | 183 + .../desktop_capture/desktop_frame_generator.h | 121 + .../desktop_capture/desktop_frame_rotation.cc | 119 + .../desktop_capture/desktop_frame_rotation.h | 52 + .../desktop_frame_rotation_unittest.cc | 447 + .../modules/desktop_capture/desktop_frame_win.cc | 69 + .../modules/desktop_capture/desktop_frame_win.h | 51 + .../modules/desktop_capture/desktop_geometry.cc | 79 + .../modules/desktop_capture/desktop_geometry.h | 166 + .../desktop_capture/desktop_geometry_unittest.cc | 106 + .../modules/desktop_capture/desktop_region.cc | 572 + .../modules/desktop_capture/desktop_region.h | 169 + .../desktop_capture/desktop_region_unittest.cc | 710 + .../webrtc/modules/desktop_capture/differ_block.cc | 71 + .../webrtc/modules/desktop_capture/differ_block.h | 44 + .../desktop_capture/differ_block_unittest.cc | 86 + .../modules/desktop_capture/differ_vector_sse2.cc | 102 + .../modules/desktop_capture/differ_vector_sse2.h | 31 + .../desktop_capture/fake_desktop_capturer.cc | 81 + .../desktop_capture/fake_desktop_capturer.h | 76 + .../fallback_desktop_capturer_wrapper.cc | 180 + .../fallback_desktop_capturer_wrapper.h | 63 + .../fallback_desktop_capturer_wrapper_unittest.cc | 205 + .../linux/base_capturer_pipewire.cc | 940 + .../desktop_capture/linux/base_capturer_pipewire.h | 172 + .../linux/desktop_device_info_x11.cc | 49 + .../linux/desktop_device_info_x11.h | 31 + .../linux/mouse_cursor_monitor_x11.cc | 255 + .../linux/mouse_cursor_monitor_x11.h | 64 + .../linux/screen_capturer_pipewire.cc | 28 + .../linux/screen_capturer_pipewire.h | 33 + .../desktop_capture/linux/screen_capturer_x11.cc | 331 + .../desktop_capture/linux/screen_capturer_x11.h | 123 + .../desktop_capture/linux/shared_x_display.cc | 90 + .../desktop_capture/linux/shared_x_display.h | 81 + .../modules/desktop_capture/linux/shared_x_util.cc | 299 + .../modules/desktop_capture/linux/shared_x_util.h | 123 + .../linux/window_capturer_pipewire.cc | 28 + .../linux/window_capturer_pipewire.h | 33 + .../desktop_capture/linux/window_capturer_x11.cc | 254 + .../desktop_capture/linux/window_capturer_x11.h | 71 + .../desktop_capture/linux/window_finder_x11.cc | 53 + .../desktop_capture/linux/window_finder_x11.h | 35 + .../desktop_capture/linux/window_list_utils.cc | 261 + .../desktop_capture/linux/window_list_utils.h | 56 + .../modules/desktop_capture/linux/x_atom_cache.cc | 47 + .../modules/desktop_capture/linux/x_atom_cache.h | 43 + .../modules/desktop_capture/linux/x_error_trap.cc | 71 + .../modules/desktop_capture/linux/x_error_trap.h | 52 + .../desktop_capture/linux/x_server_pixel_buffer.cc | 356 + .../desktop_capture/linux/x_server_pixel_buffer.h | 84 + .../desktop_capture/mac/desktop_configuration.h | 90 + .../desktop_capture/mac/desktop_configuration.mm | 169 + .../mac/desktop_configuration_monitor.cc | 90 + .../mac/desktop_configuration_monitor.h | 61 + .../desktop_capture/mac/desktop_device_info_mac.h | 28 + .../desktop_capture/mac/desktop_device_info_mac.mm | 78 + .../mac/full_screen_chrome_window_detector.cc | 189 + .../mac/full_screen_chrome_window_detector.h | 62 + .../mac/scoped_pixel_buffer_object.cc | 55 + .../mac/scoped_pixel_buffer_object.h | 41 + .../desktop_capture/mac/window_list_utils.cc | 335 + .../desktop_capture/mac/window_list_utils.h | 93 + .../mock_desktop_capturer_callback.cc | 23 + .../mock_desktop_capturer_callback.h | 38 + .../webrtc/modules/desktop_capture/mouse_cursor.cc | 38 + .../webrtc/modules/desktop_capture/mouse_cursor.h | 49 + .../modules/desktop_capture/mouse_cursor_monitor.h | 110 + .../desktop_capture/mouse_cursor_monitor_linux.cc | 51 + .../desktop_capture/mouse_cursor_monitor_mac.mm | 329 + .../desktop_capture/mouse_cursor_monitor_null.cc | 39 + .../mouse_cursor_monitor_unittest.cc | 128 + .../desktop_capture/mouse_cursor_monitor_win.cc | 226 + .../desktop_capture/primitives_gn/moz.build | 188 + .../modules/desktop_capture/resolution_tracker.cc | 34 + .../modules/desktop_capture/resolution_tracker.h | 34 + .../webrtc/modules/desktop_capture/rgba_color.cc | 61 + .../webrtc/modules/desktop_capture/rgba_color.h | 59 + .../modules/desktop_capture/rgba_color_unittest.cc | 44 + .../desktop_capture/screen_capture_frame_queue.h | 85 + .../desktop_capture/screen_capturer_helper.cc | 102 + .../desktop_capture/screen_capturer_helper.h | 90 + .../screen_capturer_helper_unittest.cc | 187 + .../screen_capturer_integration_test.cc | 383 + .../desktop_capture/screen_capturer_linux.cc | 40 + .../modules/desktop_capture/screen_capturer_mac.mm | 766 + .../screen_capturer_mac_unittest.cc | 101 + .../desktop_capture/screen_capturer_null.cc | 21 + .../desktop_capture/screen_capturer_unittest.cc | 220 + .../modules/desktop_capture/screen_capturer_win.cc | 63 + .../modules/desktop_capture/screen_drawer.cc | 30 + .../webrtc/modules/desktop_capture/screen_drawer.h | 83 + .../modules/desktop_capture/screen_drawer_linux.cc | 181 + .../desktop_capture/screen_drawer_lock_posix.cc | 58 + .../desktop_capture/screen_drawer_lock_posix.h | 38 + .../modules/desktop_capture/screen_drawer_mac.cc | 29 + .../desktop_capture/screen_drawer_unittest.cc | 161 + .../modules/desktop_capture/screen_drawer_win.cc | 209 + .../desktop_capture/shared_desktop_frame.cc | 61 + .../modules/desktop_capture/shared_desktop_frame.h | 63 + .../modules/desktop_capture/shared_memory.cc | 28 + .../webrtc/modules/desktop_capture/shared_memory.h | 83 + .../webrtc/modules/desktop_capture/test_utils.cc | 49 + .../webrtc/modules/desktop_capture/test_utils.h | 27 + .../modules/desktop_capture/test_utils_unittest.cc | 107 + .../webrtc/modules/desktop_capture/win/cursor.cc | 248 + .../webrtc/modules/desktop_capture/win/cursor.h | 25 + .../win/cursor_test_data/1_24bpp.cur | Bin 0 -> 3262 bytes .../win/cursor_test_data/1_32bpp.cur | Bin 0 -> 4286 bytes .../win/cursor_test_data/1_8bpp.cur | Bin 0 -> 2238 bytes .../win/cursor_test_data/2_1bpp.cur | Bin 0 -> 326 bytes .../win/cursor_test_data/2_32bpp.cur | Bin 0 -> 4286 bytes .../win/cursor_test_data/3_32bpp.cur | Bin 0 -> 4286 bytes .../win/cursor_test_data/3_4bpp.cur | Bin 0 -> 766 bytes .../modules/desktop_capture/win/cursor_unittest.cc | 90 + .../win/cursor_unittest_resources.h | 24 + .../win/cursor_unittest_resources.rc | 28 + .../modules/desktop_capture/win/d3d_device.cc | 100 + .../modules/desktop_capture/win/d3d_device.h | 59 + .../webrtc/modules/desktop_capture/win/desktop.cc | 111 + .../webrtc/modules/desktop_capture/win/desktop.h | 63 + .../desktop_capture/win/desktop_device_info_win.cc | 81 + .../desktop_capture/win/desktop_device_info_win.h | 28 + .../win/display_configuration_monitor.cc | 37 + .../win/display_configuration_monitor.h | 38 + .../desktop_capture/win/dxgi_adapter_duplicator.cc | 187 + .../desktop_capture/win/dxgi_adapter_duplicator.h | 92 + .../modules/desktop_capture/win/dxgi_context.cc | 32 + .../modules/desktop_capture/win/dxgi_context.h | 61 + .../win/dxgi_duplicator_controller.cc | 484 + .../win/dxgi_duplicator_controller.h | 238 + .../modules/desktop_capture/win/dxgi_frame.cc | 76 + .../modules/desktop_capture/win/dxgi_frame.h | 63 + .../desktop_capture/win/dxgi_output_duplicator.cc | 395 + .../desktop_capture/win/dxgi_output_duplicator.h | 145 + .../modules/desktop_capture/win/dxgi_texture.cc | 81 + .../modules/desktop_capture/win/dxgi_texture.h | 73 + .../desktop_capture/win/dxgi_texture_mapping.cc | 58 + .../desktop_capture/win/dxgi_texture_mapping.h | 47 + .../desktop_capture/win/dxgi_texture_staging.cc | 133 + .../desktop_capture/win/dxgi_texture_staging.h | 68 + .../desktop_capture/win/scoped_gdi_object.h | 95 + .../desktop_capture/win/scoped_thread_desktop.cc | 55 + .../desktop_capture/win/scoped_thread_desktop.h | 54 + .../desktop_capture/win/screen_capture_utils.cc | 109 + .../desktop_capture/win/screen_capture_utils.h | 47 + .../win/screen_capture_utils_unittest.cc | 32 + .../win/screen_capturer_win_directx.cc | 211 + .../win/screen_capturer_win_directx.h | 95 + .../win/screen_capturer_win_directx_unittest.cc | 41 + .../desktop_capture/win/screen_capturer_win_gdi.cc | 228 + .../desktop_capture/win/screen_capturer_win_gdi.h | 85 + .../win/screen_capturer_win_magnifier.cc | 379 + .../win/screen_capturer_win_magnifier.h | 139 + .../modules/desktop_capture/win/win_shared.cc | 28 + .../modules/desktop_capture/win/win_shared.h | 12 + .../desktop_capture/win/window_capture_utils.cc | 156 + .../desktop_capture/win/window_capture_utils.h | 80 + .../desktop_capture/window_capturer_linux.cc | 40 + .../modules/desktop_capture/window_capturer_mac.mm | 236 + .../desktop_capture/window_capturer_null.cc | 71 + .../desktop_capture/window_capturer_unittest.cc | 95 + .../modules/desktop_capture/window_capturer_win.cc | 366 + .../modules/desktop_capture/window_finder.cc | 20 + .../webrtc/modules/desktop_capture/window_finder.h | 65 + .../modules/desktop_capture/window_finder_mac.h | 37 + .../modules/desktop_capture/window_finder_mac.mm | 62 + .../desktop_capture/window_finder_unittest.cc | 169 + .../modules/desktop_capture/window_finder_win.cc | 46 + .../modules/desktop_capture/window_finder_win.h | 30 + third_party/libwebrtc/webrtc/modules/include/DEPS | 3 + .../libwebrtc/webrtc/modules/include/module.h | 63 + .../webrtc/modules/include/module_common_types.h | 590 + .../modules/include/module_common_types_public.h | 110 + .../libwebrtc/webrtc/modules/media_file/BUILD.gn | 67 + .../libwebrtc/webrtc/modules/media_file/DEPS | 4 + .../libwebrtc/webrtc/modules/media_file/OWNERS | 8 + .../webrtc/modules/media_file/media_file.h | 180 + .../webrtc/modules/media_file/media_file_defines.h | 50 + .../modules/media_file/media_file_gn/moz.build | 229 + .../webrtc/modules/media_file/media_file_impl.cc | 893 + .../webrtc/modules/media_file/media_file_impl.h | 148 + .../modules/media_file/media_file_unittest.cc | 106 + .../modules/media_file/media_file_utility.cc | 1238 + .../webrtc/modules/media_file/media_file_utility.h | 281 + .../webrtc/modules/module_api_gn/moz.build | 187 + .../webrtc/modules/module_api_public_gn/moz.build | 179 + .../webrtc/modules/module_common_types_unittest.cc | 322 + .../libwebrtc/webrtc/modules/pacing/BUILD.gn | 90 + third_party/libwebrtc/webrtc/modules/pacing/DEPS | 4 + third_party/libwebrtc/webrtc/modules/pacing/OWNERS | 9 + .../webrtc/modules/pacing/alr_detector.cc | 130 + .../libwebrtc/webrtc/modules/pacing/alr_detector.h | 83 + .../webrtc/modules/pacing/alr_detector_unittest.cc | 170 + .../webrtc/modules/pacing/bitrate_prober.cc | 205 + .../webrtc/modules/pacing/bitrate_prober.h | 110 + .../modules/pacing/bitrate_prober_unittest.cc | 183 + .../webrtc/modules/pacing/interval_budget.cc | 66 + .../webrtc/modules/pacing/interval_budget.h | 43 + .../modules/pacing/interval_budget_unittest.cc | 119 + .../webrtc/modules/pacing/mock/mock_paced_sender.h | 44 + .../webrtc/modules/pacing/paced_sender.cc | 390 + .../libwebrtc/webrtc/modules/pacing/paced_sender.h | 205 + .../webrtc/modules/pacing/paced_sender_unittest.cc | 1147 + .../libwebrtc/webrtc/modules/pacing/pacer.h | 37 + .../webrtc/modules/pacing/pacing_gn/moz.build | 231 + .../webrtc/modules/pacing/packet_queue.cc | 151 + .../libwebrtc/webrtc/modules/pacing/packet_queue.h | 117 + .../webrtc/modules/pacing/packet_queue2.cc | 209 + .../webrtc/modules/pacing/packet_queue2.h | 111 + .../webrtc/modules/pacing/packet_router.cc | 294 + .../webrtc/modules/pacing/packet_router.h | 132 + .../modules/pacing/packet_router_unittest.cc | 829 + .../modules/remote_bitrate_estimator/BUILD.gn | 256 + .../webrtc/modules/remote_bitrate_estimator/DEPS | 10 + .../webrtc/modules/remote_bitrate_estimator/OWNERS | 10 + .../remote_bitrate_estimator/aimd_rate_control.cc | 360 + .../remote_bitrate_estimator/aimd_rate_control.h | 93 + .../aimd_rate_control_unittest.cc | 250 + .../remote_bitrate_estimator/bwe_defines.cc | 40 + .../remote_bitrate_estimator/bwe_simulations.cc | 554 + .../remote_bitrate_estimator/include/bwe_defines.h | 61 + .../include/remote_bitrate_estimator.h | 92 + .../include/send_time_history.h | 55 + .../remote_bitrate_estimator/inter_arrival.cc | 159 + .../remote_bitrate_estimator/inter_arrival.h | 95 + .../inter_arrival_unittest.cc | 536 + .../remote_bitrate_estimator/overuse_detector.cc | 161 + .../remote_bitrate_estimator/overuse_detector.h | 62 + .../overuse_detector_unittest.cc | 779 + .../remote_bitrate_estimator/overuse_estimator.cc | 167 + .../remote_bitrate_estimator/overuse_estimator.h | 73 + .../remote_bitrate_estimator_abs_send_time.cc | 416 + .../remote_bitrate_estimator_abs_send_time.h | 142 + ...ote_bitrate_estimator_abs_send_time_unittest.cc | 289 + .../remote_bitrate_estimator_gn/moz.build | 237 + .../remote_bitrate_estimator_single_stream.cc | 256 + .../remote_bitrate_estimator_single_stream.h | 75 + ...ote_bitrate_estimator_single_stream_unittest.cc | 72 + .../remote_bitrate_estimator_unittest_helper.cc | 616 + .../remote_bitrate_estimator_unittest_helper.h | 221 + .../remote_bitrate_estimators_test.cc | 246 + .../remote_estimator_proxy.cc | 201 + .../remote_estimator_proxy.h | 78 + .../remote_estimator_proxy_unittest.cc | 338 + .../remote_bitrate_estimator/send_time_history.cc | 89 + .../send_time_history_unittest.cc | 235 + .../test/bbr_paced_sender.cc | 140 + .../test/bbr_paced_sender.h | 93 + .../modules/remote_bitrate_estimator/test/bwe.cc | 285 + .../modules/remote_bitrate_estimator/test/bwe.h | 199 + .../remote_bitrate_estimator/test/bwe_test.cc | 996 + .../remote_bitrate_estimator/test/bwe_test.h | 196 + .../test/bwe_test_baselinefile.cc | 168 + .../test/bwe_test_baselinefile.h | 45 + .../test/bwe_test_fileutils.cc | 96 + .../test/bwe_test_fileutils.h | 59 + .../test/bwe_test_framework.cc | 829 + .../test/bwe_test_framework.h | 473 + .../test/bwe_test_framework_unittest.cc | 975 + .../test/bwe_test_logging.cc | 259 + .../test/bwe_test_logging.h | 351 + .../remote_bitrate_estimator/test/bwe_unittest.cc | 394 + .../test/estimators/bbr.cc | 553 + .../remote_bitrate_estimator/test/estimators/bbr.h | 242 + .../test/estimators/congestion_window.cc | 67 + .../test/estimators/congestion_window.h | 49 + .../test/estimators/congestion_window_unittest.cc | 63 + .../test/estimators/max_bandwidth_filter.cc | 86 + .../test/estimators/max_bandwidth_filter.h | 55 + .../estimators/max_bandwidth_filter_unittest.cc | 84 + .../test/estimators/min_rtt_filter.h | 73 + .../test/estimators/min_rtt_filter_unittest.cc | 42 + .../test/estimators/nada.cc | 293 + .../test/estimators/nada.h | 111 + .../test/estimators/nada_unittest.cc | 497 + .../test/estimators/remb.cc | 155 + .../test/estimators/remb.h | 87 + .../test/estimators/send_side.cc | 178 + .../test/estimators/send_side.h | 76 + .../test/estimators/tcp.cc | 53 + .../remote_bitrate_estimator/test/estimators/tcp.h | 38 + .../test/metric_recorder.cc | 445 + .../test/metric_recorder.h | 188 + .../test/metric_recorder_unittest.cc | 108 + .../modules/remote_bitrate_estimator/test/packet.h | 220 + .../test/packet_receiver.cc | 146 + .../test/packet_receiver.h | 71 + .../remote_bitrate_estimator/test/packet_sender.cc | 508 + .../remote_bitrate_estimator/test/packet_sender.h | 207 + .../remote_bitrate_estimator/test/plot_bars.sh | 286 + .../remote_bitrate_estimator/test/plot_dynamics.py | 182 + .../remote_bitrate_estimator/tools/bwe_rtp.cc | 143 + .../remote_bitrate_estimator/tools/bwe_rtp.h | 36 + .../remote_bitrate_estimator/tools/bwe_rtp_play.cc | 114 + .../remote_bitrate_estimator/tools/rtp_to_text.cc | 70 + .../libwebrtc/webrtc/modules/rtp_rtcp/BUILD.gn | 422 + third_party/libwebrtc/webrtc/modules/rtp_rtcp/DEPS | 6 + .../libwebrtc/webrtc/modules/rtp_rtcp/OWNERS | 10 + .../modules/rtp_rtcp/include/flexfec_receiver.h | 69 + .../modules/rtp_rtcp/include/flexfec_sender.h | 93 + .../modules/rtp_rtcp/include/receive_statistics.h | 89 + .../rtp_rtcp/include/remote_ntp_time_estimator.h | 53 + .../webrtc/modules/rtp_rtcp/include/rtp_cvo.h | 56 + .../rtp_rtcp/include/rtp_header_extension_map.h | 80 + .../modules/rtp_rtcp/include/rtp_header_parser.h | 45 + .../modules/rtp_rtcp/include/rtp_packet_observer.h | 19 + .../rtp_rtcp/include/rtp_payload_registry.h | 130 + .../webrtc/modules/rtp_rtcp/include/rtp_receiver.h | 115 + .../webrtc/modules/rtp_rtcp/include/rtp_rtcp.h | 466 + .../modules/rtp_rtcp/include/rtp_rtcp_defines.cc | 26 + .../modules/rtp_rtcp/include/rtp_rtcp_defines.h | 508 + .../modules/rtp_rtcp/include/ulpfec_receiver.h | 57 + .../mocks/mock_recovered_packet_receiver.h | 27 + .../modules/rtp_rtcp/mocks/mock_rtcp_rtt_stats.h | 25 + .../webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h | 216 + .../modules/rtp_rtcp/rtp_rtcp_format_gn/moz.build | 260 + .../webrtc/modules/rtp_rtcp/rtp_rtcp_gn/moz.build | 265 + .../webrtc/modules/rtp_rtcp/source/byte_io.h | 408 + .../modules/rtp_rtcp/source/byte_io_unittest.cc | 272 + .../webrtc/modules/rtp_rtcp/source/dtmf_queue.cc | 47 + .../webrtc/modules/rtp_rtcp/source/dtmf_queue.h | 41 + .../rtp_rtcp/source/fec_private_tables_bursty.h | 762 + .../rtp_rtcp/source/fec_private_tables_random.h | 24524 +++++++++++++++++++ .../modules/rtp_rtcp/source/fec_test_helper.cc | 217 + .../modules/rtp_rtcp/source/fec_test_helper.h | 134 + .../source/flexfec_header_reader_writer.cc | 315 + .../rtp_rtcp/source/flexfec_header_reader_writer.h | 86 + .../flexfec_header_reader_writer_unittest.cc | 559 + .../modules/rtp_rtcp/source/flexfec_receiver.cc | 156 + .../rtp_rtcp/source/flexfec_receiver_unittest.cc | 583 + .../modules/rtp_rtcp/source/flexfec_sender.cc | 170 + .../rtp_rtcp/source/flexfec_sender_unittest.cc | 323 + .../rtp_rtcp/source/forward_error_correction.cc | 779 + .../rtp_rtcp/source/forward_error_correction.h | 419 + .../source/forward_error_correction_internal.cc | 433 + .../source/forward_error_correction_internal.h | 108 + .../modules/rtp_rtcp/source/nack_rtx_unittest.cc | 290 + .../modules/rtp_rtcp/source/packet_loss_stats.cc | 137 + .../modules/rtp_rtcp/source/packet_loss_stats.h | 57 + .../rtp_rtcp/source/packet_loss_stats_unittest.cc | 197 + .../rtp_rtcp/source/playout_delay_oracle.cc | 66 + .../modules/rtp_rtcp/source/playout_delay_oracle.h | 81 + .../source/playout_delay_oracle_unittest.cc | 69 + .../rtp_rtcp/source/receive_statistics_impl.cc | 496 + .../rtp_rtcp/source/receive_statistics_impl.h | 134 + .../rtp_rtcp/source/receive_statistics_unittest.cc | 370 + .../rtp_rtcp/source/remote_ntp_time_estimator.cc | 83 + .../source/remote_ntp_time_estimator_unittest.cc | 144 + .../modules/rtp_rtcp/source/rtcp_nack_stats.cc | 31 + .../modules/rtp_rtcp/source/rtcp_nack_stats.h | 40 + .../rtp_rtcp/source/rtcp_nack_stats_unittest.cc | 64 + .../webrtc/modules/rtp_rtcp/source/rtcp_packet.cc | 86 + .../webrtc/modules/rtp_rtcp/source/rtcp_packet.h | 103 + .../modules/rtp_rtcp/source/rtcp_packet/app.cc | 97 + .../modules/rtp_rtcp/source/rtcp_packet/app.h | 60 + .../rtp_rtcp/source/rtcp_packet/app_unittest.cc | 110 + .../modules/rtp_rtcp/source/rtcp_packet/bye.cc | 137 + .../modules/rtp_rtcp/source/rtcp_packet/bye.h | 59 + .../rtp_rtcp/source/rtcp_packet/bye_unittest.cc | 147 + .../rtp_rtcp/source/rtcp_packet/common_header.cc | 89 + .../rtp_rtcp/source/rtcp_packet/common_header.h | 51 + .../source/rtcp_packet/common_header_unittest.cc | 105 + .../rtp_rtcp/source/rtcp_packet/compound_packet.cc | 47 + .../rtp_rtcp/source/rtcp_packet/compound_packet.h | 48 + .../source/rtcp_packet/compound_packet_unittest.cc | 161 + .../modules/rtp_rtcp/source/rtcp_packet/dlrr.cc | 94 + .../modules/rtp_rtcp/source/rtcp_packet/dlrr.h | 69 + .../rtp_rtcp/source/rtcp_packet/dlrr_unittest.cc | 92 + .../source/rtcp_packet/extended_jitter_report.cc | 101 + .../source/rtcp_packet/extended_jitter_report.h | 54 + .../rtcp_packet/extended_jitter_report_unittest.cc | 76 + .../source/rtcp_packet/extended_reports.cc | 216 + .../rtp_rtcp/source/rtcp_packet/extended_reports.h | 85 + .../rtcp_packet/extended_reports_unittest.cc | 254 + .../modules/rtp_rtcp/source/rtcp_packet/fir.cc | 111 + .../modules/rtp_rtcp/source/rtcp_packet/fir.h | 62 + .../rtp_rtcp/source/rtcp_packet/fir_unittest.cc | 99 + .../modules/rtp_rtcp/source/rtcp_packet/nack.cc | 174 + .../modules/rtp_rtcp/source/rtcp_packet/nack.h | 59 + .../rtp_rtcp/source/rtcp_packet/nack_unittest.cc | 178 + .../modules/rtp_rtcp/source/rtcp_packet/pli.cc | 73 + .../modules/rtp_rtcp/source/rtcp_packet/pli.h | 39 + .../rtp_rtcp/source/rtcp_packet/pli_unittest.cc | 59 + .../modules/rtp_rtcp/source/rtcp_packet/psfb.cc | 46 + .../modules/rtp_rtcp/source/rtcp_packet/psfb.h | 48 + .../source/rtcp_packet/rapid_resync_request.cc | 69 + .../source/rtcp_packet/rapid_resync_request.h | 41 + .../rtcp_packet/rapid_resync_request_unittest.cc | 66 + .../rtp_rtcp/source/rtcp_packet/receiver_report.cc | 110 + .../rtp_rtcp/source/rtcp_packet/receiver_report.h | 60 + .../source/rtcp_packet/receiver_report_unittest.cc | 161 + .../modules/rtp_rtcp/source/rtcp_packet/remb.cc | 141 + .../modules/rtp_rtcp/source/rtcp_packet/remb.h | 60 + .../rtp_rtcp/source/rtcp_packet/remb_unittest.cc | 128 + .../rtp_rtcp/source/rtcp_packet/report_block.cc | 90 + .../rtp_rtcp/source/rtcp_packet/report_block.h | 67 + .../source/rtcp_packet/report_block_unittest.cc | 86 + .../modules/rtp_rtcp/source/rtcp_packet/rrtr.cc | 49 + .../modules/rtp_rtcp/source/rtcp_packet/rrtr.h | 49 + .../rtp_rtcp/source/rtcp_packet/rrtr_unittest.cc | 51 + .../modules/rtp_rtcp/source/rtcp_packet/rtpfb.cc | 45 + .../modules/rtp_rtcp/source/rtcp_packet/rtpfb.h | 48 + .../modules/rtp_rtcp/source/rtcp_packet/sdes.cc | 196 + .../modules/rtp_rtcp/source/rtcp_packet/sdes.h | 56 + .../rtp_rtcp/source/rtcp_packet/sdes_unittest.cc | 249 + .../rtp_rtcp/source/rtcp_packet/sender_report.cc | 139 + .../rtp_rtcp/source/rtcp_packet/sender_report.h | 80 + .../source/rtcp_packet/sender_report_unittest.cc | 145 + .../rtp_rtcp/source/rtcp_packet/target_bitrate.cc | 127 + .../rtp_rtcp/source/rtcp_packet/target_bitrate.h | 62 + .../source/rtcp_packet/target_bitrate_unittest.cc | 96 + .../rtp_rtcp/source/rtcp_packet/tmmb_item.cc | 71 + .../rtp_rtcp/source/rtcp_packet/tmmb_item.h | 51 + .../modules/rtp_rtcp/source/rtcp_packet/tmmbn.cc | 110 + .../modules/rtp_rtcp/source/rtcp_packet/tmmbn.h | 56 + .../rtp_rtcp/source/rtcp_packet/tmmbn_unittest.cc | 108 + .../modules/rtp_rtcp/source/rtcp_packet/tmmbr.cc | 112 + .../modules/rtp_rtcp/source/rtcp_packet/tmmbr.h | 55 + .../rtp_rtcp/source/rtcp_packet/tmmbr_unittest.cc | 96 + .../source/rtcp_packet/transport_feedback.cc | 645 + .../source/rtcp_packet/transport_feedback.h | 106 + .../rtcp_packet/transport_feedback_unittest.cc | 470 + .../rtp_rtcp/source/rtcp_packet/voip_metric.cc | 107 + .../rtp_rtcp/source/rtcp_packet/voip_metric.h | 53 + .../source/rtcp_packet/voip_metric_unittest.cc | 93 + .../rtp_rtcp/source/rtcp_packet_unittest.cc | 42 + .../modules/rtp_rtcp/source/rtcp_receiver.cc | 1079 + .../webrtc/modules/rtp_rtcp/source/rtcp_receiver.h | 280 + .../rtp_rtcp/source/rtcp_receiver_unittest.cc | 1280 + .../webrtc/modules/rtp_rtcp/source/rtcp_sender.cc | 1027 + .../webrtc/modules/rtp_rtcp/source/rtcp_sender.h | 305 + .../rtp_rtcp/source/rtcp_sender_unittest.cc | 826 + .../modules/rtp_rtcp/source/rtcp_transceiver.cc | 137 + .../modules/rtp_rtcp/source/rtcp_transceiver.h | 71 + .../rtp_rtcp/source/rtcp_transceiver_config.cc | 79 + .../rtp_rtcp/source/rtcp_transceiver_config.h | 77 + .../rtp_rtcp/source/rtcp_transceiver_impl.cc | 292 + .../rtp_rtcp/source/rtcp_transceiver_impl.h | 82 + .../source/rtcp_transceiver_impl_unittest.cc | 585 + .../rtp_rtcp/source/rtcp_transceiver_unittest.cc | 135 + .../modules/rtp_rtcp/source/rtp_fec_unittest.cc | 1116 + .../webrtc/modules/rtp_rtcp/source/rtp_format.cc | 63 + .../webrtc/modules/rtp_rtcp/source/rtp_format.h | 70 + .../modules/rtp_rtcp/source/rtp_format_h264.cc | 669 + .../modules/rtp_rtcp/source/rtp_format_h264.h | 120 + .../rtp_rtcp/source/rtp_format_h264_unittest.cc | 946 + .../rtp_rtcp/source/rtp_format_video_generic.cc | 141 + .../rtp_rtcp/source/rtp_format_video_generic.h | 74 + .../source/rtp_format_video_generic_unittest.cc | 192 + .../modules/rtp_rtcp/source/rtp_format_vp8.cc | 548 + .../modules/rtp_rtcp/source/rtp_format_vp8.h | 169 + .../rtp_rtcp/source/rtp_format_vp8_test_helper.cc | 243 + .../rtp_rtcp/source/rtp_format_vp8_test_helper.h | 77 + .../rtp_rtcp/source/rtp_format_vp8_unittest.cc | 485 + .../modules/rtp_rtcp/source/rtp_format_vp9.cc | 765 + .../modules/rtp_rtcp/source/rtp_format_vp9.h | 101 + .../rtp_rtcp/source/rtp_format_vp9_unittest.cc | 810 + .../rtp_rtcp/source/rtp_header_extension_map.cc | 151 + .../source/rtp_header_extension_map_unittest.cc | 114 + .../rtp_rtcp/source/rtp_header_extensions.cc | 424 + .../rtp_rtcp/source/rtp_header_extensions.h | 198 + .../modules/rtp_rtcp/source/rtp_header_parser.cc | 79 + .../webrtc/modules/rtp_rtcp/source/rtp_packet.cc | 553 + .../webrtc/modules/rtp_rtcp/source/rtp_packet.h | 199 + .../modules/rtp_rtcp/source/rtp_packet_history.cc | 223 + .../modules/rtp_rtcp/source/rtp_packet_history.h | 85 + .../rtp_rtcp/source/rtp_packet_history_unittest.cc | 223 + .../modules/rtp_rtcp/source/rtp_packet_received.cc | 63 + .../modules/rtp_rtcp/source/rtp_packet_received.h | 68 + .../modules/rtp_rtcp/source/rtp_packet_to_send.h | 62 + .../modules/rtp_rtcp/source/rtp_packet_unittest.cc | 596 + .../rtp_rtcp/source/rtp_payload_registry.cc | 346 + .../source/rtp_payload_registry_unittest.cc | 226 + .../modules/rtp_rtcp/source/rtp_receiver_audio.cc | 308 + .../modules/rtp_rtcp/source/rtp_receiver_audio.h | 98 + .../modules/rtp_rtcp/source/rtp_receiver_impl.cc | 573 + .../modules/rtp_rtcp/source/rtp_receiver_impl.h | 117 + .../rtp_rtcp/source/rtp_receiver_strategy.cc | 45 + .../rtp_rtcp/source/rtp_receiver_strategy.h | 99 + .../rtp_rtcp/source/rtp_receiver_unittest.cc | 472 + .../modules/rtp_rtcp/source/rtp_receiver_video.cc | 137 + .../modules/rtp_rtcp/source/rtp_receiver_video.h | 57 + .../modules/rtp_rtcp/source/rtp_rtcp_config.h | 46 + .../modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 963 + .../webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h | 369 + .../rtp_rtcp/source/rtp_rtcp_impl_unittest.cc | 646 + .../webrtc/modules/rtp_rtcp/source/rtp_sender.cc | 1412 ++ .../webrtc/modules/rtp_rtcp/source/rtp_sender.h | 344 + .../modules/rtp_rtcp/source/rtp_sender_audio.cc | 339 + .../modules/rtp_rtcp/source/rtp_sender_audio.h | 98 + .../modules/rtp_rtcp/source/rtp_sender_unittest.cc | 2005 ++ .../modules/rtp_rtcp/source/rtp_sender_video.cc | 554 + .../modules/rtp_rtcp/source/rtp_sender_video.h | 169 + .../webrtc/modules/rtp_rtcp/source/rtp_utility.cc | 525 + .../webrtc/modules/rtp_rtcp/source/rtp_utility.h | 69 + .../rtp_rtcp/source/rtp_utility_unittest.cc | 279 + .../webrtc/modules/rtp_rtcp/source/time_util.cc | 90 + .../webrtc/modules/rtp_rtcp/source/time_util.h | 55 + .../modules/rtp_rtcp/source/time_util_unittest.cc | 132 + .../webrtc/modules/rtp_rtcp/source/tmmbr_help.cc | 182 + .../webrtc/modules/rtp_rtcp/source/tmmbr_help.h | 33 + .../modules/rtp_rtcp/source/ulpfec_generator.cc | 244 + .../modules/rtp_rtcp/source/ulpfec_generator.h | 110 + .../rtp_rtcp/source/ulpfec_generator_unittest.cc | 170 + .../rtp_rtcp/source/ulpfec_header_reader_writer.cc | 131 + .../rtp_rtcp/source/ulpfec_header_reader_writer.h | 67 + .../source/ulpfec_header_reader_writer_unittest.cc | 244 + .../rtp_rtcp/source/ulpfec_receiver_impl.cc | 260 + .../modules/rtp_rtcp/source/ulpfec_receiver_impl.h | 57 + .../rtp_rtcp/source/ulpfec_receiver_unittest.cc | 485 + .../rtp_rtcp/source/video_codec_information.h | 27 + .../modules/rtp_rtcp/test/testAPI/test_api.cc | 163 + .../modules/rtp_rtcp/test/testAPI/test_api.h | 72 + .../rtp_rtcp/test/testAPI/test_api_audio.cc | 299 + .../modules/rtp_rtcp/test/testAPI/test_api_rtcp.cc | 252 + .../rtp_rtcp/test/testAPI/test_api_video.cc | 183 + .../test/testFec/average_residual_loss_xor_codes.h | 191 + .../modules/rtp_rtcp/test/testFec/test_fec.cc | 478 + .../test/testFec/test_packet_masks_metrics.cc | 1081 + .../libwebrtc/webrtc/modules/utility/BUILD.gn | 72 + third_party/libwebrtc/webrtc/modules/utility/DEPS | 8 + .../libwebrtc/webrtc/modules/utility/OWNERS | 6 + .../utility/include/audio_frame_operations.h | 20 + .../modules/utility/include/helpers_android.h | 87 + .../webrtc/modules/utility/include/jvm_android.h | 188 + .../utility/include/mock/mock_process_thread.h | 44 + .../modules/utility/include/process_thread.h | 77 + .../modules/utility/source/helpers_android.cc | 123 + .../webrtc/modules/utility/source/jvm_android.cc | 289 + .../modules/utility/source/process_thread_impl.cc | 224 + .../modules/utility/source/process_thread_impl.h | 88 + .../utility/source/process_thread_impl_unittest.cc | 321 + .../webrtc/modules/utility/utility_gn/moz.build | 233 + .../webrtc/modules/video_capture/BUILD.gn | 182 + .../libwebrtc/webrtc/modules/video_capture/DEPS | 5 + .../libwebrtc/webrtc/modules/video_capture/OWNERS | 10 + .../modules/video_capture/device_info_impl.cc | 260 + .../modules/video_capture/device_info_impl.h | 61 + .../video_capture/external/device_info_external.cc | 50 + .../external/video_capture_external.cc | 26 + .../video_capture/linux/device_info_linux.cc | 460 + .../video_capture/linux/device_info_linux.h | 76 + .../video_capture/linux/video_capture_linux.cc | 435 + .../video_capture/linux/video_capture_linux.h | 67 + .../video_capture/test/video_capture_main_mac.mm | 17 + .../video_capture/test/video_capture_unittest.cc | 421 + .../webrtc/modules/video_capture/video_capture.h | 200 + .../modules/video_capture/video_capture_config.h | 33 + .../modules/video_capture/video_capture_defines.h | 99 + .../modules/video_capture/video_capture_factory.cc | 31 + .../modules/video_capture/video_capture_factory.h | 44 + .../modules/video_capture/video_capture_impl.cc | 285 + .../modules/video_capture/video_capture_impl.h | 122 + .../video_capture_internal_impl_gn/moz.build | 232 + .../video_capture_internal_impl_objc_gn/moz.build | 64 + .../video_capture_module_gn/moz.build | 229 + .../video_capture/windows/device_info_ds.cc | 702 + .../modules/video_capture/windows/device_info_ds.h | 103 + .../video_capture/windows/help_functions_ds.cc | 159 + .../video_capture/windows/help_functions_ds.h | 118 + .../video_capture/windows/sink_filter_ds.cc | 939 + .../modules/video_capture/windows/sink_filter_ds.h | 162 + .../video_capture/windows/video_capture_ds.cc | 323 + .../video_capture/windows/video_capture_ds.h | 74 + .../windows/video_capture_factory_windows.cc | 40 + .../libwebrtc/webrtc/modules/video_coding/BUILD.gn | 621 + .../libwebrtc/webrtc/modules/video_coding/DEPS | 19 + .../libwebrtc/webrtc/modules/video_coding/OWNERS | 10 + .../webrtc/modules/video_coding/codec_database.cc | 462 + .../webrtc/modules/video_coding/codec_database.h | 153 + .../codec_globals_headers_gn/moz.build | 171 + .../webrtc/modules/video_coding/codec_timer.cc | 56 + .../webrtc/modules/video_coding/codec_timer.h | 51 + .../webrtc/modules/video_coding/codecs/h264/DEPS | 5 + .../webrtc/modules/video_coding/codecs/h264/OWNERS | 2 + .../modules/video_coding/codecs/h264/h264.cc | 107 + .../video_coding/codecs/h264/h264_decoder_impl.cc | 420 + .../video_coding/codecs/h264/h264_decoder_impl.h | 87 + .../video_coding/codecs/h264/h264_encoder_impl.cc | 509 + .../video_coding/codecs/h264/h264_encoder_impl.h | 104 + .../codecs/h264/h264_encoder_impl_unittest.cc | 83 + .../video_coding/codecs/h264/include/h264.h | 54 + .../codecs/h264/include/h264_globals.h | 82 + .../codecs/h264/test/h264_impl_unittest.cc | 80 + .../modules/video_coding/codecs/i420/i420.cc | 249 + .../video_coding/codecs/i420/include/i420.h | 142 + .../codecs/interface/common_constants.h | 26 + .../interface/mock/mock_video_codec_interface.h | 81 + .../codecs/interface/video_codec_interface.h | 19 + .../codecs/interface/video_error_codes.h | 35 + .../modules/video_coding/codecs/stereo/OWNERS | 1 + .../codecs/stereo/include/stereo_decoder_adapter.h | 72 + .../codecs/stereo/include/stereo_encoder_adapter.h | 73 + .../codecs/stereo/stereo_decoder_adapter.cc | 200 + .../codecs/stereo/stereo_encoder_adapter.cc | 172 + .../codecs/stereo/test/stereo_adapter_unittest.cc | 129 + .../codecs/test/android_test_initializer.cc | 55 + .../codecs/test/android_test_initializer.h | 20 + .../codecs/test/mock/mock_packet_manipulator.h | 32 + .../codecs/test/objc_codec_h264_test.h | 26 + .../codecs/test/objc_codec_h264_test.mm | 29 + .../video_coding/codecs/test/packet_manipulator.cc | 95 + .../video_coding/codecs/test/packet_manipulator.h | 112 + .../codecs/test/packet_manipulator_unittest.cc | 96 + .../codecs/test/plot_webrtc_test_logs.py | 454 + .../modules/video_coding/codecs/test/stats.cc | 222 + .../modules/video_coding/codecs/test/stats.h | 83 + .../video_coding/codecs/test/stats_unittest.cc | 43 + .../video_coding/codecs/test/test_config.cc | 203 + .../modules/video_coding/codecs/test/test_config.h | 126 + .../codecs/test/test_config_unittest.cc | 162 + .../video_coding/codecs/test/video_codec_test.cc | 126 + .../video_coding/codecs/test/video_codec_test.h | 111 + .../video_coding/codecs/test/videoprocessor.cc | 376 + .../video_coding/codecs/test/videoprocessor.h | 250 + .../codecs/test/videoprocessor_integrationtest.cc | 715 + .../codecs/test/videoprocessor_integrationtest.h | 229 + .../test/videoprocessor_integrationtest_libvpx.cc | 385 + .../videoprocessor_integrationtest_mediacodec.cc | 110 + .../videoprocessor_integrationtest_openh264.cc | 100 + ...videoprocessor_integrationtest_parameterized.cc | 113 + .../videoprocessor_integrationtest_videotoolbox.cc | 85 + .../codecs/test/videoprocessor_unittest.cc | 176 + .../codecs/vp8/default_temporal_layers.cc | 603 + .../codecs/vp8/default_temporal_layers.h | 90 + .../codecs/vp8/default_temporal_layers_unittest.cc | 409 + .../modules/video_coding/codecs/vp8/include/vp8.h | 37 + .../codecs/vp8/include/vp8_common_types.h | 29 + .../video_coding/codecs/vp8/include/vp8_globals.h | 49 + .../video_coding/codecs/vp8/screenshare_layers.cc | 478 + .../video_coding/codecs/vp8/screenshare_layers.h | 127 + .../codecs/vp8/screenshare_layers_unittest.cc | 607 + .../codecs/vp8/simulcast_rate_allocator.cc | 163 + .../codecs/vp8/simulcast_rate_allocator.h | 51 + .../codecs/vp8/simulcast_test_utility.h | 755 + .../video_coding/codecs/vp8/simulcast_unittest.cc | 78 + .../video_coding/codecs/vp8/temporal_layers.cc | 135 + .../video_coding/codecs/vp8/temporal_layers.h | 205 + .../codecs/vp8/test/vp8_impl_unittest.cc | 516 + .../modules/video_coding/codecs/vp8/vp8_impl.cc | 1274 + .../modules/video_coding/codecs/vp8/vp8_impl.h | 167 + .../modules/video_coding/codecs/vp9/include/vp9.h | 38 + .../video_coding/codecs/vp9/include/vp9_globals.h | 210 + .../video_coding/codecs/vp9/screenshare_layers.cc | 93 + .../video_coding/codecs/vp9/screenshare_layers.h | 66 + .../codecs/vp9/test/vp9_impl_unittest.cc | 205 + .../codecs/vp9/vp9_frame_buffer_pool.cc | 140 + .../codecs/vp9/vp9_frame_buffer_pool.h | 124 + .../modules/video_coding/codecs/vp9/vp9_impl.cc | 961 + .../modules/video_coding/codecs/vp9/vp9_impl.h | 165 + .../modules/video_coding/codecs/vp9/vp9_noop.cc | 39 + .../codecs/vp9/vp9_screenshare_layers_unittest.cc | 324 + .../webrtc/modules/video_coding/decoding_state.cc | 366 + .../webrtc/modules/video_coding/decoding_state.h | 90 + .../video_coding/decoding_state_unittest.cc | 699 + .../webrtc/modules/video_coding/encoded_frame.cc | 218 + .../webrtc/modules/video_coding/encoded_frame.h | 141 + .../webrtc/modules/video_coding/fec_rate_table.h | 461 + .../webrtc/modules/video_coding/frame_buffer.cc | 306 + .../webrtc/modules/video_coding/frame_buffer.h | 96 + .../webrtc/modules/video_coding/frame_buffer2.cc | 586 + .../webrtc/modules/video_coding/frame_buffer2.h | 192 + .../modules/video_coding/frame_buffer2_unittest.cc | 563 + .../webrtc/modules/video_coding/frame_object.cc | 175 + .../webrtc/modules/video_coding/frame_object.h | 104 + .../webrtc/modules/video_coding/generic_decoder.cc | 261 + .../webrtc/modules/video_coding/generic_decoder.h | 118 + .../webrtc/modules/video_coding/generic_encoder.cc | 425 + .../webrtc/modules/video_coding/generic_encoder.h | 156 + .../video_coding/generic_encoder_unittest.cc | 269 + .../video_coding/h264_sprop_parameter_sets.cc | 51 + .../video_coding/h264_sprop_parameter_sets.h | 36 + .../h264_sprop_parameter_sets_unittest.cc | 47 + .../modules/video_coding/h264_sps_pps_tracker.cc | 262 + .../modules/video_coding/h264_sps_pps_tracker.h | 57 + .../video_coding/h264_sps_pps_tracker_unittest.cc | 379 + .../webrtc/modules/video_coding/histogram.cc | 61 + .../webrtc/modules/video_coding/histogram.h | 46 + .../modules/video_coding/histogram_unittest.cc | 76 + .../video_coding/include/mock/mock_vcm_callbacks.h | 45 + .../include/mock/mock_video_codec_interface.h | 87 + .../video_coding/include/video_codec_initializer.h | 61 + .../video_coding/include/video_codec_interface.h | 95 + .../modules/video_coding/include/video_coding.h | 421 + .../video_coding/include/video_coding_defines.h | 168 + .../video_coding/include/video_error_codes.h | 33 + .../modules/video_coding/inter_frame_delay.cc | 107 + .../modules/video_coding/inter_frame_delay.h | 67 + .../webrtc/modules/video_coding/internal_defines.h | 41 + .../webrtc/modules/video_coding/jitter_buffer.cc | 1308 + .../webrtc/modules/video_coding/jitter_buffer.h | 382 + .../modules/video_coding/jitter_buffer_common.h | 73 + .../modules/video_coding/jitter_buffer_unittest.cc | 2710 ++ .../modules/video_coding/jitter_estimator.cc | 443 + .../webrtc/modules/video_coding/jitter_estimator.h | 170 + .../modules/video_coding/jitter_estimator_tests.cc | 161 + .../webrtc/modules/video_coding/media_opt_util.cc | 660 + .../webrtc/modules/video_coding/media_opt_util.h | 360 + .../modules/video_coding/media_optimization.cc | 153 + .../modules/video_coding/media_optimization.h | 81 + .../webrtc/modules/video_coding/nack_fec_tables.h | 31 + .../webrtc/modules/video_coding/nack_module.cc | 265 + .../webrtc/modules/video_coding/nack_module.h | 102 + .../modules/video_coding/nack_module_unittest.cc | 317 + .../webrtc/modules/video_coding/packet.cc | 145 + .../libwebrtc/webrtc/modules/video_coding/packet.h | 57 + .../webrtc/modules/video_coding/packet_buffer.cc | 492 + .../webrtc/modules/video_coding/packet_buffer.h | 173 + .../video_coding/protection_bitrate_calculator.cc | 191 + .../video_coding/protection_bitrate_calculator.h | 79 + .../protection_bitrate_calculator_unittest.cc | 100 + .../webrtc/modules/video_coding/qp_parser.cc | 32 + .../webrtc/modules/video_coding/qp_parser.h | 30 + .../webrtc/modules/video_coding/receiver.cc | 290 + .../webrtc/modules/video_coding/receiver.h | 104 + .../modules/video_coding/receiver_unittest.cc | 507 + .../video_coding/rtp_frame_reference_finder.cc | 613 + .../video_coding/rtp_frame_reference_finder.h | 230 + .../rtp_frame_reference_finder_unittest.cc | 1324 + .../webrtc/modules/video_coding/rtt_filter.cc | 165 + .../webrtc/modules/video_coding/rtt_filter.h | 66 + .../webrtc/modules/video_coding/session_info.cc | 553 + .../webrtc/modules/video_coding/session_info.h | 166 + .../modules/video_coding/session_info_unittest.cc | 560 + .../modules/video_coding/test/stream_generator.cc | 129 + .../modules/video_coding/test/stream_generator.h | 72 + .../webrtc/modules/video_coding/test/test_util.h | 33 + .../webrtc/modules/video_coding/timestamp_map.cc | 63 + .../webrtc/modules/video_coding/timestamp_map.h | 48 + .../webrtc/modules/video_coding/timing.cc | 318 + .../libwebrtc/webrtc/modules/video_coding/timing.h | 147 + .../webrtc/modules/video_coding/timing_unittest.cc | 146 + .../utility/default_video_bitrate_allocator.cc | 46 + .../utility/default_video_bitrate_allocator.h | 33 + .../default_video_bitrate_allocator_unittest.cc | 80 + .../modules/video_coding/utility/frame_dropper.cc | 266 + .../modules/video_coding/utility/frame_dropper.h | 94 + .../video_coding/utility/frame_dropper_unittest.cc | 161 + .../video_coding/utility/ivf_file_writer.cc | 201 + .../modules/video_coding/utility/ivf_file_writer.h | 60 + .../utility/ivf_file_writer_unittest.cc | 183 + .../video_coding/utility/mock/mock_frame_dropper.h | 34 + .../modules/video_coding/utility/moving_average.cc | 46 + .../modules/video_coding/utility/moving_average.h | 35 + .../utility/moving_average_unittest.cc | 62 + .../modules/video_coding/utility/quality_scaler.cc | 205 + .../modules/video_coding/utility/quality_scaler.h | 87 + .../utility/quality_scaler_unittest.cc | 202 + .../utility/simulcast_rate_allocator_unittest.cc | 376 + .../video_coding/utility/vp8_header_parser.cc | 205 + .../video_coding/utility/vp8_header_parser.h | 68 + .../utility/vp9_uncompressed_header_parser.cc | 272 + .../utility/vp9_uncompressed_header_parser.h | 29 + .../video_coding/video_codec_initializer.cc | 250 + .../video_codec_initializer_unittest.cc | 218 + .../modules/video_coding/video_coding_gn/moz.build | 258 + .../modules/video_coding/video_coding_impl.cc | 280 + .../modules/video_coding/video_coding_impl.h | 229 + .../video_coding/video_coding_utility_gn/moz.build | 233 + .../video_coding/video_packet_buffer_unittest.cc | 837 + .../webrtc/modules/video_coding/video_receiver.cc | 440 + .../video_coding/video_receiver_unittest.cc | 223 + .../webrtc/modules/video_coding/video_sender.cc | 397 + .../modules/video_coding/video_sender_unittest.cc | 493 + .../modules/video_coding/webrtc_h264_gn/moz.build | 225 + .../modules/video_coding/webrtc_i420_gn/moz.build | 227 + .../video_coding/webrtc_stereo_gn/moz.build | 228 + .../modules/video_coding/webrtc_vp8_gn/moz.build | 231 + .../modules/video_coding/webrtc_vp9_gn/moz.build | 229 + .../webrtc/modules/video_processing/BUILD.gn | 122 + .../libwebrtc/webrtc/modules/video_processing/DEPS | 5 + .../webrtc/modules/video_processing/OWNERS | 7 + .../video_processing/denoiser_filter_gn/moz.build | 187 + .../modules/video_processing/test/denoiser_test.cc | 161 + .../video_processing/util/denoiser_filter.cc | 59 + .../video_processing/util/denoiser_filter.h | 56 + .../video_processing/util/denoiser_filter_c.cc | 136 + .../video_processing/util/denoiser_filter_c.h | 42 + .../video_processing/util/denoiser_filter_neon.cc | 195 + .../video_processing/util/denoiser_filter_neon.h | 42 + .../video_processing/util/denoiser_filter_sse2.cc | 209 + .../video_processing/util/denoiser_filter_sse2.h | 42 + .../video_processing/util/noise_estimation.cc | 113 + .../video_processing/util/noise_estimation.h | 63 + .../video_processing/util/skin_detection.cc | 97 + .../modules/video_processing/util/skin_detection.h | 30 + .../modules/video_processing/video_denoiser.cc | 345 + .../modules/video_processing/video_denoiser.h | 84 + .../video_processing/video_processing_gn/moz.build | 235 + .../video_processing_neon_gn/moz.build | 191 + .../video_processing_sse2_gn/moz.build | 232 + 2146 files changed, 449614 insertions(+) create mode 100644 third_party/libwebrtc/webrtc/modules/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding.gni create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_format_conversion_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.proto create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h create mode 100755 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/cng_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_decoder.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_encoder.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/test/testG711.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_decode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_enc_dec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_encode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/test/testG722.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/energy_inverse.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enh_upsample.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enh_upsample.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enhancer.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enhancer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/frame_classify.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/frame_classify.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/my_corr.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/my_corr.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/pack_bits.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/pack_bits.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_construct.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_construct.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_search.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_search.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/empty.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBCtestscript.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/empty.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_decoder_isacfix.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/isacfix.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_hist.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_logist.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routins.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_decoder_isacfix.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_encoder_isacfix.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/codec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_bwe.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_plc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/encode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_neon.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_internal.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks_neon.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_neon.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/initialize.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isac_fix_type.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_armv7.S create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_neon.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_c.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/settings.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_neon.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ChannelFiles.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFiles.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFilesFew.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ListOfTestCases.xls create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSAC.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSACPLC.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACLongtest.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACNB.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACPLC.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACRate.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfault.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfixfloat.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/isac_speed_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/kenny.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/isac.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_hist.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_logist.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_decoder_isac.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/codec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode_bwe.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filterbank_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filterbank_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filterbanks.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/intialize.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_float_type.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lattice.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_analysis.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_analysis.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/settings.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/transform.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACLongtest.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfault.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfixfloat.txt create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/SwitchingSampRate/SwitchingSampRate.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/simpleKenny.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_inst.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/g711_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/g711_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/g722_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/g722_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module_typedefs.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_common_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_arm_asm_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_common_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/isac_neon_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/include/neteq.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_expand.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.proto create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_isac_quality_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_checksum.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_audio_file.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_wav_file.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_encode.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtpcat.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq_decoder_enum_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/neteq_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/rent_a_codec_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/ACMTest.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/Tester.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/delay_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/insert_packet_with_timing.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/target_delay_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_template.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/BuildInfo.java create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioEffects.java create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioUtils.java create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_config.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_data_observer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/audio_device_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_data_observer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/include/fake_audio_device.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_device.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_transport.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.mm create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_not_implemented_ios.mm create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_unittest_ios.mm create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_session_observer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSession.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.mm create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_memorybarrier.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/mock_audio_device_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/output_rate_calculator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_mips.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_neon.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_optimized_methods.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_sse2.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec/system_delay_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_output.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_integration_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump_interface_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_c.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_neon.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/gain_map_internal.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/gain_control.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/mock_agc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_locking_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_c_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_performance_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_sse2_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_statistics_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_test_helpers.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/mock_nonlinear_beamformer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/config_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/debug.proto create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/config.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/config.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/include/mock_audio_processing.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/test/intelligibility_proc.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_complexity_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_constants.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_c.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_mips.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_neon.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/ns/windows_private.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/render_queue_item_verifier.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/rms_level_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/AndroidManifest.xml create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/default.properties create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/jni/main.c create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/res/values/strings.xml create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/apmtest.m create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/audioproc_float.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/README.md create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_abstract_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/README.md create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_configs/default.json create mode 100755 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py create mode 100755 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_boxplot.py create mode 100755 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py create mode 100755 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_optimize.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/output/README.md create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/__init__.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_configs/default.json create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_vad.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_factory.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_factory.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/exceptions.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/external_vad.py create mode 100755 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_external_vad.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_polqa.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_signal_creator.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_factory.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_unittest.py create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/vad.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/unittest.proto create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/unpack.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/click_annotate.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/daubechies_8_wavelet_coeffs.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/plotDetection.m create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readDetection.m create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readPCM.m create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppression_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_internal.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_mips.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_neon.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_sse2.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_neon_sse2.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/common.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/noise_gmm_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_gmm_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/include/bitrate_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/include/mock/mock_bitrate_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.cc create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.h create mode 100644 third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_congestion_observer.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_send_side_congestion_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/include/receive_side_congestion_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/include/send_side_congestion_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_differ_sse2_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_generic_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_objc_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_types.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_wrapper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_wrapper.h create mode 100755 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_device_info.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_device_info.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_device_info_null.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_rotation.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_rotation.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_rotation_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_win.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_geometry.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_geometry.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_geometry_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_region.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_region.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_region_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/differ_block.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/differ_block.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/differ_block_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/differ_vector_sse2.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/differ_vector_sse2.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/fake_desktop_capturer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/fake_desktop_capturer.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/base_capturer_pipewire.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/base_capturer_pipewire.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/desktop_device_info_x11.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/desktop_device_info_x11.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/mouse_cursor_monitor_x11.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/mouse_cursor_monitor_x11.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_pipewire.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_pipewire.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_x11.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_x11.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_display.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_display.h create mode 100755 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_util.cc create mode 100755 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_util.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_pipewire.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_pipewire.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_x11.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_x11.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_finder_x11.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_finder_x11.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_list_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_list_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_atom_cache.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_atom_cache.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_error_trap.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_error_trap.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_server_pixel_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_server_pixel_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_configuration.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_configuration.mm create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_configuration_monitor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_configuration_monitor.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_device_info_mac.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_device_info_mac.mm create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/full_screen_chrome_window_detector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/full_screen_chrome_window_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/scoped_pixel_buffer_object.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/scoped_pixel_buffer_object.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/window_list_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mac/window_list_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mock_desktop_capturer_callback.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mock_desktop_capturer_callback.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_mac.mm create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_null.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/primitives_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/resolution_tracker.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/resolution_tracker.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/rgba_color.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/rgba_color.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/rgba_color_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capture_frame_queue.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_helper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_helper.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_helper_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_integration_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_mac.mm create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_mac_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_null.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer_lock_posix.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer_lock_posix.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer_mac.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/screen_drawer_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/shared_desktop_frame.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/shared_desktop_frame.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/shared_memory.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/shared_memory.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/test_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/test_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/test_utils_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_test_data/1_24bpp.cur create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_test_data/1_32bpp.cur create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_test_data/1_8bpp.cur create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_test_data/2_1bpp.cur create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_test_data/2_32bpp.cur create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_test_data/3_32bpp.cur create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_test_data/3_4bpp.cur create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_unittest_resources.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor_unittest_resources.rc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/d3d_device.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/d3d_device.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/desktop.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/desktop.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/desktop_device_info_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/desktop_device_info_win.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/display_configuration_monitor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/display_configuration_monitor.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_adapter_duplicator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_adapter_duplicator.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_context.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_context.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_duplicator_controller.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_duplicator_controller.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_frame.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_frame.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_output_duplicator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_output_duplicator.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture_mapping.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture_mapping.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture_staging.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture_staging.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/scoped_gdi_object.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/scoped_thread_desktop.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/scoped_thread_desktop.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capture_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capture_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capture_utils_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_directx.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_directx.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_directx_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_gdi.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_gdi.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_magnifier.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_magnifier.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/win_shared.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/win_shared.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/window_capture_utils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/win/window_capture_utils.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_mac.mm create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_null.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder_mac.h create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder_mac.mm create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder_win.cc create mode 100644 third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder_win.h create mode 100644 third_party/libwebrtc/webrtc/modules/include/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/include/module.h create mode 100644 third_party/libwebrtc/webrtc/modules/include/module_common_types.h create mode 100644 third_party/libwebrtc/webrtc/modules/include/module_common_types_public.h create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file.h create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file_utility.cc create mode 100644 third_party/libwebrtc/webrtc/modules/media_file/media_file_utility.h create mode 100644 third_party/libwebrtc/webrtc/modules/module_api_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/module_api_public_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/module_common_types_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/alr_detector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/alr_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/alr_detector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/bitrate_prober.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/bitrate_prober.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/bitrate_prober_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/interval_budget.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/interval_budget.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/interval_budget_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/mock/mock_paced_sender.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/paced_sender.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/paced_sender.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/paced_sender_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/pacer.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/pacing_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/packet_queue.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/packet_queue.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/packet_queue2.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/packet_queue2.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/packet_router.cc create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/packet_router.h create mode 100644 third_party/libwebrtc/webrtc/modules/pacing/packet_router_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/aimd_rate_control.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/aimd_rate_control.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/aimd_rate_control_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/bwe_defines.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/include/send_time_history.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/inter_arrival.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/inter_arrival.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/inter_arrival_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/overuse_detector.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/overuse_detector.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/overuse_detector_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/overuse_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/overuse_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_abs_send_time.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_abs_send_time.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_abs_send_time_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_estimator_proxy.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_estimator_proxy.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/remote_estimator_proxy_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/send_time_history.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/send_time_history_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bbr_paced_sender.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bbr_paced_sender.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/bwe_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/bbr.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/bbr.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/congestion_window.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/congestion_window.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/congestion_window_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/max_bandwidth_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/max_bandwidth_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/max_bandwidth_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/min_rtt_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/min_rtt_filter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/nada_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/remb.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/remb.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/send_side.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/send_side.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/tcp.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/estimators/tcp.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/metric_recorder_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/packet.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h create mode 100755 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/plot_bars.sh create mode 100755 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/tools/bwe_rtp.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/tools/bwe_rtp.h create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/tools/bwe_rtp_play.cc create mode 100644 third_party/libwebrtc/webrtc/modules/remote_bitrate_estimator/tools/rtp_to_text.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/flexfec_receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/flexfec_sender.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/receive_statistics.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/remote_ntp_time_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_cvo.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_header_extension_map.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_header_parser.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_packet_observer.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_rtcp.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/include/ulpfec_receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/mocks/mock_recovered_packet_receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/mocks/mock_rtcp_rtt_stats.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/rtp_rtcp_format_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/rtp_rtcp_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/byte_io.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/byte_io_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/dtmf_queue.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/dtmf_queue.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/fec_private_tables_bursty.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/fec_private_tables_random.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/fec_test_helper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/fec_test_helper.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/flexfec_receiver.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/flexfec_receiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/flexfec_sender.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/flexfec_sender_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/forward_error_correction.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/forward_error_correction.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/forward_error_correction_internal.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/forward_error_correction_internal.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/nack_rtx_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/packet_loss_stats.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/packet_loss_stats.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/packet_loss_stats_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/playout_delay_oracle.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/playout_delay_oracle.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/playout_delay_oracle_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/receive_statistics_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/receive_statistics_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/receive_statistics_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/remote_ntp_time_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/remote_ntp_time_estimator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_nack_stats.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_nack_stats.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_nack_stats_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/app.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/app.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/app_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/bye.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/bye.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/bye_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/common_header.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/common_header.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/common_header_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/compound_packet.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/compound_packet.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/compound_packet_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/dlrr.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/dlrr.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/dlrr_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/extended_jitter_report.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/extended_jitter_report.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/extended_jitter_report_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/extended_reports.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/extended_reports.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/extended_reports_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/fir.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/fir.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/fir_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/nack.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/nack.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/nack_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/pli.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/pli.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/pli_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/psfb.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/psfb.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rapid_resync_request.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rapid_resync_request.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rapid_resync_request_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/receiver_report.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/receiver_report.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/receiver_report_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/remb.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/remb.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/remb_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/report_block.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/report_block.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/report_block_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rrtr.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rrtr.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rrtr_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rtpfb.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/rtpfb.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/sdes.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/sdes.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/sdes_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/sender_report.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/sender_report.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/sender_report_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/target_bitrate.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/target_bitrate.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/target_bitrate_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmb_item.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmb_item.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbn.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbn.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbn_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbr.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbr.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/tmmbr_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/voip_metric.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/voip_metric.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet/voip_metric_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_packet_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_receiver.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_sender.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_sender.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_sender_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_config.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_config.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_fec_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp8_test_helper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp8_test_helper.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp8_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_header_parser.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet_history.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet_history.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet_history_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet_received.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet_received.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_packet_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_payload_registry.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_payload_registry_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_audio.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_audio.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_strategy.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_strategy.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_video.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_receiver_video.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_rtcp_config.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_sender.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_sender.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_sender_audio.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_sender_audio.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_utility.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_utility.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/rtp_utility_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/time_util.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/time_util.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/time_util_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/tmmbr_help.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/tmmbr_help.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_header_reader_writer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_header_reader_writer.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_header_reader_writer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_receiver_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_receiver_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/ulpfec_receiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/source/video_codec_information.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testAPI/test_api.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testAPI/test_api.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testAPI/test_api_audio.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testAPI/test_api_rtcp.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testAPI/test_api_video.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testFec/average_residual_loss_xor_codes.h create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testFec/test_fec.cc create mode 100644 third_party/libwebrtc/webrtc/modules/rtp_rtcp/test/testFec/test_packet_masks_metrics.cc create mode 100644 third_party/libwebrtc/webrtc/modules/utility/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/utility/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/utility/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/utility/include/audio_frame_operations.h create mode 100644 third_party/libwebrtc/webrtc/modules/utility/include/helpers_android.h create mode 100644 third_party/libwebrtc/webrtc/modules/utility/include/jvm_android.h create mode 100644 third_party/libwebrtc/webrtc/modules/utility/include/mock/mock_process_thread.h create mode 100644 third_party/libwebrtc/webrtc/modules/utility/include/process_thread.h create mode 100644 third_party/libwebrtc/webrtc/modules/utility/source/helpers_android.cc create mode 100644 third_party/libwebrtc/webrtc/modules/utility/source/jvm_android.cc create mode 100644 third_party/libwebrtc/webrtc/modules/utility/source/process_thread_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/utility/source/process_thread_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/utility/source/process_thread_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/utility/utility_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/device_info_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/device_info_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/external/device_info_external.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/external/video_capture_external.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/linux/device_info_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/linux/device_info_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/linux/video_capture_linux.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/linux/video_capture_linux.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/test/video_capture_main_mac.mm create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/test/video_capture_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_config.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_factory.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_factory.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_internal_impl_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_internal_impl_objc_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/video_capture_module_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/device_info_ds.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/device_info_ds.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/help_functions_ds.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/help_functions_ds.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/sink_filter_ds.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/sink_filter_ds.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/video_capture_ds.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/video_capture_ds.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_capture/windows/video_capture_factory_windows.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codec_database.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codec_database.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codec_globals_headers_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codec_timer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codec_timer.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/h264.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/include/h264.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/include/h264_globals.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/i420/i420.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/i420/include/i420.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/interface/common_constants.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/interface/video_error_codes.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/stereo/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/android_test_initializer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/android_test_initializer.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/mock/mock_packet_manipulator.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.mm create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/packet_manipulator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/packet_manipulator.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/packet_manipulator_unittest.cc create mode 100755 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/stats.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/stats.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/stats_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/test_config.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/test_config.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/test_config_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/video_codec_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/video_codec_test.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/test/videoprocessor_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/include/vp8.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/simulcast_test_utility.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/temporal_layers.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/include/vp9.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/include/vp9_globals.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/vp9_noop.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/codecs/vp9/vp9_screenshare_layers_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/decoding_state.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/decoding_state.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/decoding_state_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/encoded_frame.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/encoded_frame.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/fec_rate_table.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/frame_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/frame_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/frame_buffer2.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/frame_buffer2.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/frame_buffer2_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/frame_object.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/frame_object.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/generic_decoder.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/generic_decoder.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/generic_encoder.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/generic_encoder.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/generic_encoder_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/h264_sprop_parameter_sets.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/h264_sprop_parameter_sets.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/h264_sprop_parameter_sets_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/h264_sps_pps_tracker.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/h264_sps_pps_tracker.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/histogram.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/histogram.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/histogram_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/include/mock/mock_vcm_callbacks.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/include/video_codec_initializer.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/include/video_codec_interface.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/include/video_coding.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/include/video_coding_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/include/video_error_codes.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/inter_frame_delay.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/inter_frame_delay.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/internal_defines.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/jitter_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/jitter_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/jitter_buffer_common.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/jitter_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/jitter_estimator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/jitter_estimator.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/jitter_estimator_tests.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/media_opt_util.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/media_opt_util.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/media_optimization.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/media_optimization.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/nack_fec_tables.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/nack_module.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/nack_module.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/nack_module_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/packet.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/packet.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/packet_buffer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/packet_buffer.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/protection_bitrate_calculator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/protection_bitrate_calculator.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/protection_bitrate_calculator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/qp_parser.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/qp_parser.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/receiver.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/receiver.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/receiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/rtp_frame_reference_finder.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/rtp_frame_reference_finder.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/rtt_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/rtt_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/session_info.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/session_info.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/session_info_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/test/stream_generator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/test/stream_generator.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/test/test_util.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/timestamp_map.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/timestamp_map.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/timing.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/timing.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/timing_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/default_video_bitrate_allocator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/frame_dropper.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/frame_dropper.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/frame_dropper_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/ivf_file_writer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/ivf_file_writer.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/ivf_file_writer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/mock/mock_frame_dropper.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/moving_average.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/moving_average.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/moving_average_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/quality_scaler.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/quality_scaler.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/vp8_header_parser.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/vp8_header_parser.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_codec_initializer.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_codec_initializer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_coding_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_coding_impl.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_coding_impl.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_coding_utility_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_packet_buffer_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_receiver.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_receiver_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_sender.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/video_sender_unittest.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/webrtc_h264_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/webrtc_i420_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/webrtc_stereo_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/webrtc_vp8_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_coding/webrtc_vp9_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/BUILD.gn create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/DEPS create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/OWNERS create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/denoiser_filter_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/test/denoiser_test.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter_c.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter_c.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter_neon.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter_neon.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/denoiser_filter_sse2.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/noise_estimation.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/noise_estimation.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/util/skin_detection.cc create mode 100755 third_party/libwebrtc/webrtc/modules/video_processing/util/skin_detection.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/video_denoiser.cc create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/video_denoiser.h create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/video_processing_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/video_processing_neon_gn/moz.build create mode 100644 third_party/libwebrtc/webrtc/modules/video_processing/video_processing_sse2_gn/moz.build (limited to 'third_party/libwebrtc/webrtc/modules') diff --git a/third_party/libwebrtc/webrtc/modules/BUILD.gn b/third_party/libwebrtc/webrtc/modules/BUILD.gn new file mode 100644 index 0000000000..619d8437cf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/BUILD.gn @@ -0,0 +1,285 @@ +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../webrtc.gni") +import("audio_coding/audio_coding.gni") + +group("modules") { + public_deps = [ + "audio_coding", + "audio_device", + "audio_mixer", + "audio_processing", + "bitrate_controller", + "congestion_controller", + "media_file", + "pacing", + "remote_bitrate_estimator", + "rtp_rtcp", + "utility", + "video_coding", + "video_processing", + ] + + if (is_win || is_mac || is_linux || is_bsd) { + public_deps += [ "desktop_capture" ] + } +} + +rtc_source_set("module_api_public") { + sources = [ + "include/module_common_types_public.h", + ] + deps = [ + "..:webrtc_common", + "../api:optional", + ] +} + +rtc_source_set("module_api") { + sources = [ + "include/module.h", + "include/module_common_types.h", + ] + deps = [ + ":module_api_public", + "..:webrtc_common", + "../api:optional", + "../api:video_frame_api", + "../api:video_frame_api_i420", + "../rtc_base:rtc_base_approved", + "video_coding:codec_globals_headers", + ] +} + +if (rtc_include_tests) { + modules_tests_resources = [ + "../resources/audio_coding/testfile32kHz.pcm", + "../resources/audio_coding/teststereo32kHz.pcm", + "../resources/foreman_cif.yuv", + "../resources/paris_qcif.yuv", + ] + + if (is_ios) { + bundle_data("modules_tests_bundle_data") { + testonly = true + sources = modules_tests_resources + outputs = [ + "{{bundle_resources_dir}}/{{source_file_part}}", + ] + } + } + + rtc_test("modules_tests") { + testonly = true + + deps = [ + "../test:test_main", + "../test:video_test_common", + "audio_coding:audio_coding_modules_tests", + "desktop_capture:desktop_capture_modules_tests", + "rtp_rtcp:rtp_rtcp_modules_tests", + "video_coding:video_coding_modules_tests", + "//testing/gmock", + "//testing/gtest", + ] + + data = modules_tests_resources + + if (is_android) { + deps += [ + # NOTE(brandtr): Including Java classes seems only to be possible from + # rtc_test targets. Therefore we include this target here, instead of + # in video_coding_modules_tests, where it is actually used. + "../sdk/android:libjingle_peerconnection_java", + "//testing/android/native_test:native_test_native_code", + ] + shard_timeout = 900 + } + + if (is_ios) { + deps += [ ":modules_tests_bundle_data" ] + } + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + if (is_win) { + cflags = [ + # TODO(phoglund): get rid of 4373 supression when + # http://code.google.com/p/webrtc/issues/detail?id=261 is solved. + # legacy warning for ignoring const / volatile in signatures. + "/wd4373", + ] + } + } + + modules_unittests_resources = [ + "../resources/att-downlink.rx", + "../resources/att-uplink.rx", + "../resources/audio_coding/neteq_opus.rtp", + "../resources/audio_coding/neteq_opus_dtx.rtp", + "../resources/audio_coding/neteq_universal_new.rtp", + "../resources/audio_coding/speech_mono_16kHz.pcm", + "../resources/audio_coding/speech_mono_32_48kHz.pcm", + "../resources/audio_coding/testfile32kHz.pcm", + "../resources/audio_coding/teststereo32kHz.pcm", + "../resources/audio_device/audio_short16.pcm", + "../resources/audio_device/audio_short44.pcm", + "../resources/audio_device/audio_short48.pcm", + "../resources/audio_processing/agc/agc_audio.pcm", + "../resources/audio_processing/agc/agc_no_circular_buffer.dat", + "../resources/audio_processing/agc/agc_pitch_gain.dat", + "../resources/audio_processing/agc/agc_pitch_lag.dat", + "../resources/audio_processing/agc/agc_spectral_peak.dat", + "../resources/audio_processing/agc/agc_vad.dat", + "../resources/audio_processing/agc/agc_voicing_prob.dat", + "../resources/audio_processing/agc/agc_with_circular_buffer.dat", + "../resources/audio_processing/output_data_fixed.pb", + "../resources/audio_processing/output_data_float.pb", + "../resources/audio_processing/output_data_mac.pb", + "../resources/audio_processing/transient/ajm-macbook-1-spke16m.pcm", + "../resources/audio_processing/transient/audio16kHz.pcm", + "../resources/audio_processing/transient/audio32kHz.pcm", + "../resources/audio_processing/transient/audio48kHz.pcm", + "../resources/audio_processing/transient/audio8kHz.pcm", + "../resources/audio_processing/transient/detect16kHz.dat", + "../resources/audio_processing/transient/detect32kHz.dat", + "../resources/audio_processing/transient/detect48kHz.dat", + "../resources/audio_processing/transient/detect8kHz.dat", + "../resources/audio_processing/transient/double-utils.dat", + "../resources/audio_processing/transient/float-utils.dat", + "../resources/audio_processing/transient/suppressed16kHz.pcm", + "../resources/audio_processing/transient/suppressed32kHz.pcm", + "../resources/audio_processing/transient/suppressed8kHz.pcm", + "../resources/audio_processing/transient/wpd0.dat", + "../resources/audio_processing/transient/wpd1.dat", + "../resources/audio_processing/transient/wpd2.dat", + "../resources/audio_processing/transient/wpd3.dat", + "../resources/audio_processing/transient/wpd4.dat", + "../resources/audio_processing/transient/wpd5.dat", + "../resources/audio_processing/transient/wpd6.dat", + "../resources/audio_processing/transient/wpd7.dat", + "../resources/deflicker_before_cif_short.yuv", + "../resources/far16_stereo.pcm", + "../resources/far32_stereo.pcm", + "../resources/far44_stereo.pcm", + "../resources/far48_stereo.pcm", + "../resources/far8_stereo.pcm", + "../resources/foremanColorEnhanced_cif_short.yuv", + "../resources/foreman_cif.yuv", + "../resources/foreman_cif_short.yuv", + "../resources/near16_stereo.pcm", + "../resources/near32_stereo.pcm", + "../resources/near44_stereo.pcm", + "../resources/near48_stereo.pcm", + "../resources/near8_stereo.pcm", + "../resources/ref03.aecdump", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin", + "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin", + "../resources/short_mixed_mono_48.dat", + "../resources/short_mixed_mono_48_arm.dat", + "../resources/short_mixed_mono_48.pcm", + "../resources/short_mixed_stereo_48.dat", + "../resources/short_mixed_stereo_48.pcm", + "../resources/sprint-downlink.rx", + "../resources/sprint-uplink.rx", + "../resources/synthetic-trace.rx", + "../resources/tmobile-downlink.rx", + "../resources/tmobile-uplink.rx", + "../resources/verizon3g-downlink.rx", + "../resources/verizon3g-uplink.rx", + "../resources/verizon4g-downlink.rx", + "../resources/verizon4g-uplink.rx", + "../resources/voice_engine/audio_tiny48.wav", + ] + if (is_ios) { + bundle_data("modules_unittests_bundle_data") { + testonly = true + sources = modules_unittests_resources + outputs = [ + "{{bundle_resources_dir}}/{{source_file_part}}", + ] + } + } + + rtc_test("modules_unittests") { + testonly = true + defines = [] + sources = [ + "module_common_types_unittest.cc", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + ":module_api", + "../test:test_main", + "audio_coding:audio_coding_unittests", + "audio_device:audio_device_unittests", + "audio_mixer:audio_mixer_unittests", + "audio_processing:audio_processing_unittests", + "bitrate_controller:bitrate_controller_unittests", + "congestion_controller:congestion_controller_unittests", + "desktop_capture:desktop_capture_unittests", + "media_file:media_file_unittests", + "pacing:pacing_unittests", + "remote_bitrate_estimator:remote_bitrate_estimator_unittests", + "rtp_rtcp:rtp_rtcp_unittests", + "utility:utility_unittests", + "video_coding:video_coding_unittests", + "video_processing:video_processing_unittests", + ] + + if (!build_with_mozilla) { + deps += [ "video_capture" ] + } + + data = modules_unittests_resources + + if (is_android) { + deps += [ + "../sdk/android:libjingle_peerconnection_java", + "//testing/android/native_test:native_test_support", + ] + shard_timeout = 900 + } + if (is_ios) { + info_plist = "../test/ios/Info.plist" + deps += [ ":modules_unittests_bundle_data" ] + configs += [ "..:common_objc" ] + ldflags = [ "-ObjC" ] + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/OWNERS b/third_party/libwebrtc/webrtc/modules/OWNERS new file mode 100644 index 0000000000..353039ca75 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/OWNERS @@ -0,0 +1,4 @@ +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/BUILD.gn b/third_party/libwebrtc/webrtc/modules/audio_coding/BUILD.gn new file mode 100644 index 0000000000..da86fee790 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/BUILD.gn @@ -0,0 +1,2139 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") +import("audio_coding.gni") +import("//build/config/arm.gni") +if (!build_with_mozilla) { + import("//third_party/protobuf/proto_library.gni") +} + +audio_codec_deps = [ + ":cng", + ":g711", + ":pcm16b", +] +if (rtc_include_ilbc) { + audio_codec_deps += [ ":ilbc" ] +} +if (rtc_include_opus) { + audio_codec_deps += [ ":webrtc_opus" ] +} +if (current_cpu == "arm") { + audio_codec_deps += [ ":isac_fix" ] +} else { + audio_codec_deps += [ ":isac" ] +} +audio_codec_deps += [ ":g722" ] +if (!build_with_mozilla && !build_with_chromium) { + audio_codec_deps += [ ":red" ] +} +audio_coding_deps = audio_codec_deps + [ + "../..:webrtc_common", + "../../common_audio", + "../../system_wrappers", + ] + +rtc_static_library("audio_format_conversion") { + sources = [ + "codecs/audio_format_conversion.cc", + "codecs/audio_format_conversion.h", + ] + deps = [ + "../..:webrtc_common", + "../../api:array_view", + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + ] +} + +rtc_static_library("rent_a_codec") { + sources = [ + "acm2/acm_codec_database.cc", + "acm2/acm_codec_database.h", + "acm2/rent_a_codec.cc", + "acm2/rent_a_codec.h", + ] + deps = [ + "../../api:array_view", + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../..:webrtc_common", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ":audio_coding_module_typedefs", + ":isac_common", + ":isac_fix_c", + ":neteq_decoder_enum", + ] + audio_codec_deps + + defines = audio_codec_defines + + if (build_with_mozilla) { + include_dirs = [ "/media/libopus/include" ] + } +} + +config("audio_coding_config") { + include_dirs = [ + "include", + "../include", + ] +} + +rtc_source_set("audio_coding_module_typedefs") { + sources = [ + "include/audio_coding_module_typedefs.h", + ] + deps = [ + "..:module_api", + "../..:webrtc_common", + ] +} + +rtc_static_library("audio_coding") { + sources = [ + "acm2/acm_receiver.cc", + "acm2/acm_receiver.h", + "acm2/acm_resampler.cc", + "acm2/acm_resampler.h", + "acm2/audio_coding_module.cc", + "acm2/call_statistics.cc", + "acm2/call_statistics.h", + "acm2/codec_manager.cc", + "acm2/codec_manager.h", + "include/audio_coding_module.h", + ] + + defines = [] + + public_configs = [ ":audio_coding_config" ] + + if (rtc_include_opus) { + public_deps = [ + ":webrtc_opus", + ] + } + + if (is_win) { + cflags = [ + # TODO(kjellander): Bug 261: fix this warning. + "/wd4373", # virtual function override. + ] + } + + deps = audio_coding_deps + [ + "..:module_api", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + ":audio_coding_module_typedefs", + ":neteq", + ":rent_a_codec", + "../../rtc_base:rtc_base_approved", + "../../api:optional", + "../../logging:rtc_event_log_api", + ] + defines = audio_coding_defines +} + +rtc_static_library("legacy_encoded_audio_frame") { + sources = [ + "codecs/legacy_encoded_audio_frame.cc", + "codecs/legacy_encoded_audio_frame.h", + ] + deps = [ + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + ] +} + +config("cng_config") { + include_dirs = [ "codecs/cng/include" ] +} + +rtc_static_library("cng") { + sources = [ + "codecs/cng/audio_encoder_cng.cc", + "codecs/cng/audio_encoder_cng.h", + "codecs/cng/webrtc_cng.cc", + "codecs/cng/webrtc_cng.h", + ] + + public_configs = [ ":cng_config" ] + + deps = [ + "../..:webrtc_common", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + ] +} + +config("red_config") { + include_dirs = [ "codecs/red" ] +} + +rtc_static_library("red") { + sources = [ + "codecs/red/audio_encoder_copy_red.cc", + "codecs/red/audio_encoder_copy_red.h", + ] + + public_configs = [ ":red_config" ] + + deps = [ + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + ] +} + +config("g711_config") { + include_dirs = [ "codecs/g711/include" ] +} + +rtc_static_library("g711") { + sources = [ + "codecs/g711/audio_decoder_pcm.cc", + "codecs/g711/audio_decoder_pcm.h", + "codecs/g711/audio_encoder_pcm.cc", + "codecs/g711/audio_encoder_pcm.h", + ] + + public_configs = [ ":g711_config" ] + + deps = [ + ":legacy_encoded_audio_frame", + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + ] + public_deps = [ + ":g711_c", + ] +} + +rtc_source_set("g711_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/g711/g711.c", + "codecs/g711/g711.h", + "codecs/g711/g711_interface.c", + "codecs/g711/g711_interface.h", + ] + deps = [ + "../..:webrtc_common", + ] +} + +config("g722_config") { + include_dirs = [ "codecs/g722/include" ] +} + +rtc_static_library("g722") { + sources = [ + "codecs/g722/audio_decoder_g722.cc", + "codecs/g722/audio_decoder_g722.h", + "codecs/g722/audio_encoder_g722.cc", + "codecs/g722/audio_encoder_g722.h", + ] + + public_configs = [ ":g722_config" ] + + deps = [ + ":legacy_encoded_audio_frame", + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/g722:audio_encoder_g722_config", + "../../rtc_base:rtc_base_approved", + ] + public_deps = [ + ":g722_c", + ] +} + +rtc_source_set("g722_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/g722/g722_decode.c", + "codecs/g722/g722_enc_dec.h", + "codecs/g722/g722_encode.c", + "codecs/g722/g722_interface.c", + "codecs/g722/g722_interface.h", + ] + deps = [ + "../..:webrtc_common", + ] +} + +config("ilbc_config") { + include_dirs = [ "codecs/ilbc/include" ] +} + +rtc_static_library("ilbc") { + sources = [ + "codecs/ilbc/audio_decoder_ilbc.cc", + "codecs/ilbc/audio_decoder_ilbc.h", + "codecs/ilbc/audio_encoder_ilbc.cc", + "codecs/ilbc/audio_encoder_ilbc.h", + ] + + public_configs = [ ":ilbc_config" ] + + deps = [ + ":legacy_encoded_audio_frame", + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/ilbc:audio_encoder_ilbc_config", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + ] + public_deps = [ + ":ilbc_c", + ] +} + +rtc_source_set("ilbc_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/ilbc/abs_quant.c", + "codecs/ilbc/abs_quant.h", + "codecs/ilbc/abs_quant_loop.c", + "codecs/ilbc/abs_quant_loop.h", + "codecs/ilbc/augmented_cb_corr.c", + "codecs/ilbc/augmented_cb_corr.h", + "codecs/ilbc/bw_expand.c", + "codecs/ilbc/bw_expand.h", + "codecs/ilbc/cb_construct.c", + "codecs/ilbc/cb_construct.h", + "codecs/ilbc/cb_mem_energy.c", + "codecs/ilbc/cb_mem_energy.h", + "codecs/ilbc/cb_mem_energy_augmentation.c", + "codecs/ilbc/cb_mem_energy_augmentation.h", + "codecs/ilbc/cb_mem_energy_calc.c", + "codecs/ilbc/cb_mem_energy_calc.h", + "codecs/ilbc/cb_search.c", + "codecs/ilbc/cb_search.h", + "codecs/ilbc/cb_search_core.c", + "codecs/ilbc/cb_search_core.h", + "codecs/ilbc/cb_update_best_index.c", + "codecs/ilbc/cb_update_best_index.h", + "codecs/ilbc/chebyshev.c", + "codecs/ilbc/chebyshev.h", + "codecs/ilbc/comp_corr.c", + "codecs/ilbc/comp_corr.h", + "codecs/ilbc/constants.c", + "codecs/ilbc/constants.h", + "codecs/ilbc/create_augmented_vec.c", + "codecs/ilbc/create_augmented_vec.h", + "codecs/ilbc/decode.c", + "codecs/ilbc/decode.h", + "codecs/ilbc/decode_residual.c", + "codecs/ilbc/decode_residual.h", + "codecs/ilbc/decoder_interpolate_lsf.c", + "codecs/ilbc/decoder_interpolate_lsf.h", + "codecs/ilbc/defines.h", + "codecs/ilbc/do_plc.c", + "codecs/ilbc/do_plc.h", + "codecs/ilbc/encode.c", + "codecs/ilbc/encode.h", + "codecs/ilbc/energy_inverse.c", + "codecs/ilbc/energy_inverse.h", + "codecs/ilbc/enh_upsample.c", + "codecs/ilbc/enh_upsample.h", + "codecs/ilbc/enhancer.c", + "codecs/ilbc/enhancer.h", + "codecs/ilbc/enhancer_interface.c", + "codecs/ilbc/enhancer_interface.h", + "codecs/ilbc/filtered_cb_vecs.c", + "codecs/ilbc/filtered_cb_vecs.h", + "codecs/ilbc/frame_classify.c", + "codecs/ilbc/frame_classify.h", + "codecs/ilbc/gain_dequant.c", + "codecs/ilbc/gain_dequant.h", + "codecs/ilbc/gain_quant.c", + "codecs/ilbc/gain_quant.h", + "codecs/ilbc/get_cd_vec.c", + "codecs/ilbc/get_cd_vec.h", + "codecs/ilbc/get_lsp_poly.c", + "codecs/ilbc/get_lsp_poly.h", + "codecs/ilbc/get_sync_seq.c", + "codecs/ilbc/get_sync_seq.h", + "codecs/ilbc/hp_input.c", + "codecs/ilbc/hp_input.h", + "codecs/ilbc/hp_output.c", + "codecs/ilbc/hp_output.h", + "codecs/ilbc/ilbc.c", + "codecs/ilbc/ilbc.h", + "codecs/ilbc/index_conv_dec.c", + "codecs/ilbc/index_conv_dec.h", + "codecs/ilbc/index_conv_enc.c", + "codecs/ilbc/index_conv_enc.h", + "codecs/ilbc/init_decode.c", + "codecs/ilbc/init_decode.h", + "codecs/ilbc/init_encode.c", + "codecs/ilbc/init_encode.h", + "codecs/ilbc/interpolate.c", + "codecs/ilbc/interpolate.h", + "codecs/ilbc/interpolate_samples.c", + "codecs/ilbc/interpolate_samples.h", + "codecs/ilbc/lpc_encode.c", + "codecs/ilbc/lpc_encode.h", + "codecs/ilbc/lsf_check.c", + "codecs/ilbc/lsf_check.h", + "codecs/ilbc/lsf_interpolate_to_poly_dec.c", + "codecs/ilbc/lsf_interpolate_to_poly_dec.h", + "codecs/ilbc/lsf_interpolate_to_poly_enc.c", + "codecs/ilbc/lsf_interpolate_to_poly_enc.h", + "codecs/ilbc/lsf_to_lsp.c", + "codecs/ilbc/lsf_to_lsp.h", + "codecs/ilbc/lsf_to_poly.c", + "codecs/ilbc/lsf_to_poly.h", + "codecs/ilbc/lsp_to_lsf.c", + "codecs/ilbc/lsp_to_lsf.h", + "codecs/ilbc/my_corr.c", + "codecs/ilbc/my_corr.h", + "codecs/ilbc/nearest_neighbor.c", + "codecs/ilbc/nearest_neighbor.h", + "codecs/ilbc/pack_bits.c", + "codecs/ilbc/pack_bits.h", + "codecs/ilbc/poly_to_lsf.c", + "codecs/ilbc/poly_to_lsf.h", + "codecs/ilbc/poly_to_lsp.c", + "codecs/ilbc/poly_to_lsp.h", + "codecs/ilbc/refiner.c", + "codecs/ilbc/refiner.h", + "codecs/ilbc/simple_interpolate_lsf.c", + "codecs/ilbc/simple_interpolate_lsf.h", + "codecs/ilbc/simple_lpc_analysis.c", + "codecs/ilbc/simple_lpc_analysis.h", + "codecs/ilbc/simple_lsf_dequant.c", + "codecs/ilbc/simple_lsf_dequant.h", + "codecs/ilbc/simple_lsf_quant.c", + "codecs/ilbc/simple_lsf_quant.h", + "codecs/ilbc/smooth.c", + "codecs/ilbc/smooth.h", + "codecs/ilbc/smooth_out_data.c", + "codecs/ilbc/smooth_out_data.h", + "codecs/ilbc/sort_sq.c", + "codecs/ilbc/sort_sq.h", + "codecs/ilbc/split_vq.c", + "codecs/ilbc/split_vq.h", + "codecs/ilbc/state_construct.c", + "codecs/ilbc/state_construct.h", + "codecs/ilbc/state_search.c", + "codecs/ilbc/state_search.h", + "codecs/ilbc/swap_bytes.c", + "codecs/ilbc/swap_bytes.h", + "codecs/ilbc/unpack_bits.c", + "codecs/ilbc/unpack_bits.h", + "codecs/ilbc/vq3.c", + "codecs/ilbc/vq3.h", + "codecs/ilbc/vq4.c", + "codecs/ilbc/vq4.h", + "codecs/ilbc/window32_w32.c", + "codecs/ilbc/window32_w32.h", + "codecs/ilbc/xcorr_coef.c", + "codecs/ilbc/xcorr_coef.h", + ] + + public_configs = [ ":ilbc_config" ] + + deps = [ + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + ] +} + +rtc_static_library("isac_common") { + sources = [ + "codecs/isac/audio_encoder_isac_t.h", + "codecs/isac/audio_encoder_isac_t_impl.h", + "codecs/isac/locked_bandwidth_info.cc", + "codecs/isac/locked_bandwidth_info.h", + ] + deps = [ + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + ] +} + +config("isac_config") { + include_dirs = [ "codecs/isac/main/include" ] +} + +rtc_static_library("isac") { + sources = [ + "codecs/isac/main/include/audio_decoder_isac.h", + "codecs/isac/main/include/audio_encoder_isac.h", + "codecs/isac/main/source/audio_decoder_isac.cc", + "codecs/isac/main/source/audio_encoder_isac.cc", + ] + + deps = [ + ":isac_common", + "../../api/audio_codecs:audio_codecs_api", + ] + public_deps = [ + ":isac_c", + ] +} + +rtc_static_library("isac_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/isac/main/include/isac.h", + "codecs/isac/main/source/arith_routines.c", + "codecs/isac/main/source/arith_routines.h", + "codecs/isac/main/source/arith_routines_hist.c", + "codecs/isac/main/source/arith_routines_logist.c", + "codecs/isac/main/source/bandwidth_estimator.c", + "codecs/isac/main/source/bandwidth_estimator.h", + "codecs/isac/main/source/codec.h", + "codecs/isac/main/source/crc.c", + "codecs/isac/main/source/crc.h", + "codecs/isac/main/source/decode.c", + "codecs/isac/main/source/decode_bwe.c", + "codecs/isac/main/source/encode.c", + "codecs/isac/main/source/encode_lpc_swb.c", + "codecs/isac/main/source/encode_lpc_swb.h", + "codecs/isac/main/source/entropy_coding.c", + "codecs/isac/main/source/entropy_coding.h", + "codecs/isac/main/source/fft.c", + "codecs/isac/main/source/fft.h", + "codecs/isac/main/source/filter_functions.c", + "codecs/isac/main/source/filterbank_tables.c", + "codecs/isac/main/source/filterbank_tables.h", + "codecs/isac/main/source/filterbanks.c", + "codecs/isac/main/source/intialize.c", + "codecs/isac/main/source/isac.c", + "codecs/isac/main/source/isac_float_type.h", + "codecs/isac/main/source/lattice.c", + "codecs/isac/main/source/lpc_analysis.c", + "codecs/isac/main/source/lpc_analysis.h", + "codecs/isac/main/source/lpc_gain_swb_tables.c", + "codecs/isac/main/source/lpc_gain_swb_tables.h", + "codecs/isac/main/source/lpc_shape_swb12_tables.c", + "codecs/isac/main/source/lpc_shape_swb12_tables.h", + "codecs/isac/main/source/lpc_shape_swb16_tables.c", + "codecs/isac/main/source/lpc_shape_swb16_tables.h", + "codecs/isac/main/source/lpc_tables.c", + "codecs/isac/main/source/lpc_tables.h", + "codecs/isac/main/source/os_specific_inline.h", + "codecs/isac/main/source/pitch_estimator.c", + "codecs/isac/main/source/pitch_estimator.h", + "codecs/isac/main/source/pitch_filter.c", + "codecs/isac/main/source/pitch_gain_tables.c", + "codecs/isac/main/source/pitch_gain_tables.h", + "codecs/isac/main/source/pitch_lag_tables.c", + "codecs/isac/main/source/pitch_lag_tables.h", + "codecs/isac/main/source/settings.h", + "codecs/isac/main/source/spectrum_ar_model_tables.c", + "codecs/isac/main/source/spectrum_ar_model_tables.h", + "codecs/isac/main/source/structs.h", + "codecs/isac/main/source/transform.c", + ] + + if (is_linux) { + libs = [ "m" ] + } + + public_configs = [ ":isac_config" ] + + deps = [ + ":isac_common", + "../..:webrtc_common", + "../../common_audio", + "../../rtc_base:compile_assert_c", + "../../rtc_base:rtc_base_approved", + ] +} + +config("isac_fix_config") { + include_dirs = [ "codecs/isac/fix/include" ] +} + +rtc_static_library("isac_fix") { + sources = [ + "codecs/isac/fix/source/audio_decoder_isacfix.cc", + "codecs/isac/fix/source/audio_encoder_isacfix.cc", + ] + + public_configs = [ ":isac_fix_config" ] + + deps = [ + ":isac_common", + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../system_wrappers", + ] + public_deps = [ + ":isac_fix_c", + ] + + if (rtc_build_with_neon) { + deps += [ ":isac_neon" ] + } +} + +rtc_source_set("isac_fix_common") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/isac/fix/source/codec.h", + "codecs/isac/fix/source/fft.c", + "codecs/isac/fix/source/fft.h", + "codecs/isac/fix/source/settings.h", + ] + public_configs = [ ":isac_fix_config" ] +} + +rtc_source_set("isac_fix_c_arm_asm") { + sources = [] + if (current_cpu == "arm" && arm_version >= 7) { + sources += [ + "codecs/isac/fix/source/lattice_armv7.S", + "codecs/isac/fix/source/pitch_filter_armv6.S", + ] + } +} + +rtc_source_set("isac_fix_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/isac/fix/include/audio_decoder_isacfix.h", + "codecs/isac/fix/include/audio_encoder_isacfix.h", + "codecs/isac/fix/include/isacfix.h", + "codecs/isac/fix/source/arith_routines.c", + "codecs/isac/fix/source/arith_routines_hist.c", + "codecs/isac/fix/source/arith_routines_logist.c", + "codecs/isac/fix/source/arith_routins.h", + "codecs/isac/fix/source/bandwidth_estimator.c", + "codecs/isac/fix/source/bandwidth_estimator.h", + "codecs/isac/fix/source/decode.c", + "codecs/isac/fix/source/decode_bwe.c", + "codecs/isac/fix/source/decode_plc.c", + "codecs/isac/fix/source/encode.c", + "codecs/isac/fix/source/entropy_coding.c", + "codecs/isac/fix/source/entropy_coding.h", + "codecs/isac/fix/source/filterbank_tables.c", + "codecs/isac/fix/source/filterbank_tables.h", + "codecs/isac/fix/source/filterbanks.c", + "codecs/isac/fix/source/filters.c", + "codecs/isac/fix/source/initialize.c", + "codecs/isac/fix/source/isac_fix_type.h", + "codecs/isac/fix/source/isacfix.c", + "codecs/isac/fix/source/lattice.c", + "codecs/isac/fix/source/lattice_c.c", + "codecs/isac/fix/source/lpc_masking_model.c", + "codecs/isac/fix/source/lpc_masking_model.h", + "codecs/isac/fix/source/lpc_tables.c", + "codecs/isac/fix/source/lpc_tables.h", + "codecs/isac/fix/source/pitch_estimator.c", + "codecs/isac/fix/source/pitch_estimator.h", + "codecs/isac/fix/source/pitch_estimator_c.c", + "codecs/isac/fix/source/pitch_filter.c", + "codecs/isac/fix/source/pitch_filter_c.c", + "codecs/isac/fix/source/pitch_gain_tables.c", + "codecs/isac/fix/source/pitch_gain_tables.h", + "codecs/isac/fix/source/pitch_lag_tables.c", + "codecs/isac/fix/source/pitch_lag_tables.h", + "codecs/isac/fix/source/spectrum_ar_model_tables.c", + "codecs/isac/fix/source/spectrum_ar_model_tables.h", + "codecs/isac/fix/source/structs.h", + "codecs/isac/fix/source/transform.c", + "codecs/isac/fix/source/transform_tables.c", + ] + + public_configs = [ ":isac_fix_config" ] + + deps = [ + ":isac_common", + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../rtc_base:compile_assert_c", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] + + public_deps = [ + ":isac_fix_common", + ] + + if (rtc_build_with_neon) { + deps += [ ":isac_neon" ] + } + + if (current_cpu == "arm" && arm_version >= 7) { + sources -= [ + "codecs/isac/fix/source/lattice_c.c", + "codecs/isac/fix/source/pitch_filter_c.c", + ] + deps += [ ":isac_fix_c_arm_asm" ] + } + + if (current_cpu == "mipsel") { + sources += [ + "codecs/isac/fix/source/entropy_coding_mips.c", + "codecs/isac/fix/source/filters_mips.c", + "codecs/isac/fix/source/lattice_mips.c", + "codecs/isac/fix/source/pitch_estimator_mips.c", + "codecs/isac/fix/source/transform_mips.c", + ] + sources -= [ + "codecs/isac/fix/source/lattice_c.c", + "codecs/isac/fix/source/pitch_estimator_c.c", + ] + if (mips_dsp_rev > 0) { + sources += [ "codecs/isac/fix/source/filterbanks_mips.c" ] + } + if (mips_dsp_rev > 1) { + sources += [ + "codecs/isac/fix/source/lpc_masking_model_mips.c", + "codecs/isac/fix/source/pitch_filter_mips.c", + ] + sources -= [ "codecs/isac/fix/source/pitch_filter_c.c" ] + } + } +} + +if (rtc_build_with_neon) { + rtc_static_library("isac_neon") { + sources = [ + "codecs/isac/fix/source/entropy_coding_neon.c", + "codecs/isac/fix/source/filterbanks_neon.c", + "codecs/isac/fix/source/filters_neon.c", + "codecs/isac/fix/source/lattice_neon.c", + "codecs/isac/fix/source/transform_neon.c", + ] + + if (current_cpu != "arm64") { + # Enable compilation for the NEON instruction set. This is needed + # since //build/config/arm.gni only enables NEON for iOS, not Android. + # This provides the same functionality as webrtc/build/arm_neon.gypi. + suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] + cflags = [ "-mfpu=neon" ] + } + + # Disable LTO on NEON targets due to compiler bug. + # TODO(fdegans): Enable this. See crbug.com/408997. + if (rtc_use_lto) { + cflags -= [ + "-flto", + "-ffat-lto-objects", + ] + } + + deps = [ + ":isac_fix_common", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + ] + } +} + +config("pcm16b_config") { + include_dirs = [ "codecs/pcm16b/include" ] +} + +rtc_static_library("pcm16b") { + sources = [ + "codecs/pcm16b/audio_decoder_pcm16b.cc", + "codecs/pcm16b/audio_decoder_pcm16b.h", + "codecs/pcm16b/audio_encoder_pcm16b.cc", + "codecs/pcm16b/audio_encoder_pcm16b.h", + "codecs/pcm16b/pcm16b_common.cc", + "codecs/pcm16b/pcm16b_common.h", + ] + + deps = [ + ":g711", + ":legacy_encoded_audio_frame", + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + ] + public_deps = [ + ":pcm16b_c", + ] + public_configs = [ ":pcm16b_config" ] +} + +rtc_source_set("pcm16b_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/pcm16b/pcm16b.c", + "codecs/pcm16b/pcm16b.h", + ] + + public_configs = [ ":pcm16b_config" ] + deps = [ + "../..:webrtc_common", + ] +} + +rtc_static_library("webrtc_opus") { + sources = [ + "codecs/opus/audio_decoder_opus.cc", + "codecs/opus/audio_decoder_opus.h", + "codecs/opus/audio_encoder_opus.cc", + "codecs/opus/audio_encoder_opus.h", + ] + + deps = [ + ":audio_network_adaptor", + "../..:webrtc_common", + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/opus:audio_encoder_opus_config", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_numerics", + "../../system_wrappers", + ] + public_deps = [ + ":webrtc_opus_c", + "../../rtc_base:protobuf_utils", + ] + + defines = audio_codec_defines + + if (rtc_build_opus) { + public_deps += [ rtc_opus_dir ] + } else if (build_with_mozilla) { + include_dirs = [ "/media/libopus/include" ] + } +} + +rtc_source_set("webrtc_opus_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "codecs/opus/opus_inst.h", + "codecs/opus/opus_interface.c", + "codecs/opus/opus_interface.h", + ] + + defines = audio_coding_defines + + if (rtc_build_opus) { + public_deps = [ + rtc_opus_dir, + ] + } else if (build_with_mozilla) { + include_dirs = [ "/media/libopus/include" ] + } + + deps = [ + "../..:webrtc_common", + "../../rtc_base:rtc_base_approved", + ] +} + +if (rtc_enable_protobuf) { + proto_library("ana_debug_dump_proto") { + sources = [ + "audio_network_adaptor/debug_dump.proto", + ] + deps = [ + ":ana_config_proto", + ] + proto_out_dir = "modules/audio_coding/audio_network_adaptor" + } + proto_library("ana_config_proto") { + sources = [ + "audio_network_adaptor/config.proto", + ] + proto_out_dir = "modules/audio_coding/audio_network_adaptor" + } +} + +rtc_static_library("audio_network_adaptor_config") { + sources = [ + "audio_network_adaptor/audio_network_adaptor_config.cc", + "audio_network_adaptor/include/audio_network_adaptor_config.h", + ] + deps = [ + "../../api:optional", + ] +} + +rtc_static_library("audio_network_adaptor") { + sources = [ + "audio_network_adaptor/audio_network_adaptor_impl.cc", + "audio_network_adaptor/audio_network_adaptor_impl.h", + "audio_network_adaptor/bitrate_controller.cc", + "audio_network_adaptor/bitrate_controller.h", + "audio_network_adaptor/channel_controller.cc", + "audio_network_adaptor/channel_controller.h", + "audio_network_adaptor/controller.cc", + "audio_network_adaptor/controller.h", + "audio_network_adaptor/controller_manager.cc", + "audio_network_adaptor/controller_manager.h", + "audio_network_adaptor/debug_dump_writer.cc", + "audio_network_adaptor/debug_dump_writer.h", + "audio_network_adaptor/dtx_controller.cc", + "audio_network_adaptor/dtx_controller.h", + "audio_network_adaptor/event_log_writer.cc", + "audio_network_adaptor/event_log_writer.h", + "audio_network_adaptor/fec_controller_plr_based.cc", + "audio_network_adaptor/fec_controller_plr_based.h", + "audio_network_adaptor/fec_controller_rplr_based.cc", + "audio_network_adaptor/fec_controller_rplr_based.h", + "audio_network_adaptor/frame_length_controller.cc", + "audio_network_adaptor/frame_length_controller.h", + "audio_network_adaptor/include/audio_network_adaptor.h", + "audio_network_adaptor/util/threshold_curve.h", + ] + + public_deps = [ + ":audio_network_adaptor_config", + ] + + deps = [ + "../..:webrtc_common", + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../logging:rtc_event_log_api", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] + + if (rtc_enable_protobuf) { + deps += [ + ":ana_config_proto", + ":ana_debug_dump_proto", + ] + } + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } +} + +rtc_source_set("neteq_decoder_enum") { + sources = [ + "neteq/neteq_decoder_enum.cc", + "neteq/neteq_decoder_enum.h", + ] + deps = [ + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + ] +} + +rtc_static_library("neteq") { + sources = [ + "neteq/accelerate.cc", + "neteq/accelerate.h", + "neteq/audio_multi_vector.cc", + "neteq/audio_multi_vector.h", + "neteq/audio_vector.cc", + "neteq/audio_vector.h", + "neteq/background_noise.cc", + "neteq/background_noise.h", + "neteq/buffer_level_filter.cc", + "neteq/buffer_level_filter.h", + "neteq/comfort_noise.cc", + "neteq/comfort_noise.h", + "neteq/cross_correlation.cc", + "neteq/cross_correlation.h", + "neteq/decision_logic.cc", + "neteq/decision_logic.h", + "neteq/decision_logic_fax.cc", + "neteq/decision_logic_fax.h", + "neteq/decision_logic_normal.cc", + "neteq/decision_logic_normal.h", + "neteq/decoder_database.cc", + "neteq/decoder_database.h", + "neteq/defines.h", + "neteq/delay_manager.cc", + "neteq/delay_manager.h", + "neteq/delay_peak_detector.cc", + "neteq/delay_peak_detector.h", + "neteq/dsp_helper.cc", + "neteq/dsp_helper.h", + "neteq/dtmf_buffer.cc", + "neteq/dtmf_buffer.h", + "neteq/dtmf_tone_generator.cc", + "neteq/dtmf_tone_generator.h", + "neteq/expand.cc", + "neteq/expand.h", + "neteq/include/neteq.h", + "neteq/merge.cc", + "neteq/merge.h", + "neteq/nack_tracker.cc", + "neteq/nack_tracker.h", + "neteq/neteq.cc", + "neteq/neteq_impl.cc", + "neteq/neteq_impl.h", + "neteq/normal.cc", + "neteq/normal.h", + "neteq/packet.cc", + "neteq/packet.h", + "neteq/packet_buffer.cc", + "neteq/packet_buffer.h", + "neteq/post_decode_vad.cc", + "neteq/post_decode_vad.h", + "neteq/preemptive_expand.cc", + "neteq/preemptive_expand.h", + "neteq/random_vector.cc", + "neteq/random_vector.h", + "neteq/red_payload_splitter.cc", + "neteq/red_payload_splitter.h", + "neteq/rtcp.cc", + "neteq/rtcp.h", + "neteq/statistics_calculator.cc", + "neteq/statistics_calculator.h", + "neteq/sync_buffer.cc", + "neteq/sync_buffer.h", + "neteq/tick_timer.cc", + "neteq/tick_timer.h", + "neteq/time_stretch.cc", + "neteq/time_stretch.h", + "neteq/timestamp_scaler.cc", + "neteq/timestamp_scaler.h", + ] + + deps = [ + ":audio_coding_module_typedefs", + ":cng", + ":neteq_decoder_enum", + "..:module_api", + "../..:webrtc_common", + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../rtc_base:gtest_prod", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] +} + +# Although providing only test support, this target must be outside of the +# rtc_include_tests conditional. The reason is that it supports fuzzer tests +# that ultimately are built and run as a part of the Chromium ecosystem, which +# does not set the rtc_include_tests flag. +rtc_source_set("neteq_tools_minimal") { + sources = [ + "neteq/tools/audio_sink.cc", + "neteq/tools/audio_sink.h", + "neteq/tools/encode_neteq_input.cc", + "neteq/tools/encode_neteq_input.h", + "neteq/tools/neteq_input.cc", + "neteq/tools/neteq_input.h", + "neteq/tools/neteq_test.cc", + "neteq/tools/neteq_test.h", + "neteq/tools/packet.cc", + "neteq/tools/packet.h", + "neteq/tools/packet_source.cc", + "neteq/tools/packet_source.h", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + ":neteq", + "..:module_api", + "../..:webrtc_common", + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../rtc_base:rtc_base_approved", + "../rtp_rtcp", + ] +} + +rtc_source_set("neteq_test_tools") { + testonly = true + sources = [ + "neteq/tools/audio_checksum.h", + "neteq/tools/audio_loop.cc", + "neteq/tools/audio_loop.h", + "neteq/tools/constant_pcm_packet_source.cc", + "neteq/tools/constant_pcm_packet_source.h", + "neteq/tools/output_audio_file.h", + "neteq/tools/output_wav_file.h", + "neteq/tools/rtp_file_source.cc", + "neteq/tools/rtp_file_source.h", + "neteq/tools/rtp_generator.cc", + "neteq/tools/rtp_generator.h", + ] + + public_configs = [ ":neteq_tools_config" ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + ":pcm16b", + "..:module_api", + "../..:webrtc_common", + "../../api:array_view", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_base_tests_utils", + "../../test:rtp_test_utils", + "../rtp_rtcp", + ] + + public_deps = [ + ":neteq_tools", + ":neteq_tools_minimal", + ] + + if (rtc_enable_protobuf) { + sources += [ + "neteq/tools/neteq_packet_source_input.cc", + "neteq/tools/neteq_packet_source_input.h", + ] + deps += [ ":rtc_event_log_source" ] + } +} + +config("neteq_tools_config") { + include_dirs = [ "tools" ] +} + +rtc_source_set("neteq_tools") { + sources = [ + "neteq/tools/fake_decode_from_file.cc", + "neteq/tools/fake_decode_from_file.h", + "neteq/tools/neteq_delay_analyzer.cc", + "neteq/tools/neteq_delay_analyzer.h", + "neteq/tools/neteq_replacement_input.cc", + "neteq/tools/neteq_replacement_input.h", + ] + + public_configs = [ ":neteq_tools_config" ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + "../..:webrtc_common", + "../../api:array_view", + "../../api:optional", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + "../rtp_rtcp", + ] + + public_deps = [ + ":neteq_input_audio_tools", + ":neteq_tools_minimal", + ] +} + +rtc_source_set("neteq_input_audio_tools") { + sources = [ + "neteq/tools/input_audio_file.cc", + "neteq/tools/input_audio_file.h", + "neteq/tools/resample_input_audio_file.cc", + "neteq/tools/resample_input_audio_file.h", + ] + + deps = [ + "../..:webrtc_common", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + ] +} + +if (rtc_enable_protobuf) { + rtc_static_library("rtc_event_log_source") { + testonly = true + + # TODO(kjellander): Remove (bugs.webrtc.org/6828) + # Needs call.h to be moved to webrtc/api first. + check_includes = false + + sources = [ + "neteq/tools/rtc_event_log_source.cc", + "neteq/tools/rtc_event_log_source.h", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + "../../logging:rtc_event_log_parser", + "../../rtc_base:rtc_base_approved", + ] + public_deps = [ + "../../logging:rtc_event_log_proto", + ] + } +} + +if (rtc_include_tests) { + group("audio_coding_tests") { + testonly = true + public_deps = [ + ":acm_receive_test", + ":acm_send_test", + ":audio_codec_speed_tests", + ":audio_decoder_unittests", + ":audio_decoder_unittests", + ":delay_test", + ":g711_test", + ":g722_test", + ":ilbc_test", + ":insert_packet_with_timing", + ":isac_api_test", + ":isac_fix_test", + ":isac_switch_samprate_test", + ":isac_test", + ":neteq_ilbc_quality_test", + ":neteq_isac_quality_test", + ":neteq_opus_quality_test", + ":neteq_pcmu_quality_test", + ":neteq_speed_test", + ":rtp_analyze", + ":rtp_encode", + ":rtp_jitter", + ":rtpcat", + ":webrtc_opus_fec_test", + ] + if (rtc_enable_protobuf) { + public_deps += [ ":neteq_rtpplay" ] + } + } + + rtc_source_set("audio_coding_modules_tests") { + testonly = true + + sources = [ + "test/ACMTest.h", + "test/APITest.cc", + "test/APITest.h", + "test/Channel.cc", + "test/Channel.h", + "test/EncodeDecodeTest.cc", + "test/EncodeDecodeTest.h", + "test/PCMFile.cc", + "test/PCMFile.h", + "test/PacketLossTest.cc", + "test/PacketLossTest.h", + "test/RTPFile.cc", + "test/RTPFile.h", + "test/TestAllCodecs.cc", + "test/TestAllCodecs.h", + "test/TestRedFec.cc", + "test/TestRedFec.h", + "test/TestStereo.cc", + "test/TestStereo.h", + "test/TestVADDTX.cc", + "test/TestVADDTX.h", + "test/Tester.cc", + "test/TwoWayCommunication.cc", + "test/TwoWayCommunication.h", + "test/iSACTest.cc", + "test/iSACTest.h", + "test/opus_test.cc", + "test/opus_test.h", + "test/target_delay_unittest.cc", + "test/utility.cc", + "test/utility.h", + ] + deps = [ + ":audio_coding", + ":audio_coding_module_typedefs", + ":audio_format_conversion", + ":pcm16b_c", + "..:module_api", + "../..:webrtc_common", + "../../api:optional", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + "../../test:test_support", + ] + defines = audio_coding_defines + if (is_win) { + cflags = [ + # TODO(kjellander): bugs.webrtc.org/261: Fix this warning. + "/wd4373", # virtual function override. + ] + } + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + + rtc_source_set("audio_coding_perf_tests") { + testonly = true + + sources = [ + "codecs/opus/opus_complexity_unittest.cc", + "neteq/test/neteq_performance_unittest.cc", + ] + deps = [ + ":neteq_test_support", + ":neteq_test_tools", + "../..:webrtc_common", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + "../../test:test_support", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + + rtc_source_set("acm_receive_test") { + testonly = true + sources = [ + "acm2/acm_receive_test.cc", + "acm2/acm_receive_test.h", + ] + + defines = audio_coding_defines + + deps = audio_coding_deps + [ + ":audio_coding", + ":audio_format_conversion", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + ":neteq_tools", + "../../rtc_base:rtc_base_approved", + "../../test:test_support", + "//testing/gtest", + ] + } + + rtc_source_set("acm_send_test") { + testonly = true + sources = [ + "acm2/acm_send_test.cc", + "acm2/acm_send_test.h", + ] + + defines = audio_coding_defines + + deps = audio_coding_deps + [ + ":audio_coding", + ":neteq_tools", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:rtc_base_approved", + "../../test:test_support", + "//testing/gtest", + ] + } + + rtc_executable("delay_test") { + testonly = true + sources = [ + "test/Channel.cc", + "test/Channel.h", + "test/PCMFile.cc", + "test/PCMFile.h", + "test/delay_test.cc", + "test/utility.cc", + "test/utility.h", + ] + + deps = [ + ":audio_coding", + ":audio_coding_module_typedefs", + ":audio_format_conversion", + "..:module_api", + "../../:webrtc_common", + "../../api:optional", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../../system_wrappers:system_wrappers_default", + "../../test:test_support", + "../rtp_rtcp", + "//testing/gtest", + ] + } # delay_test + + rtc_executable("insert_packet_with_timing") { + testonly = true + sources = [ + "test/Channel.cc", + "test/Channel.h", + "test/PCMFile.cc", + "test/PCMFile.h", + "test/insert_packet_with_timing.cc", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + ":audio_coding", + ":audio_format_conversion", + "..:module_api", + "../../:webrtc_common", + "../../api:optional", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../../system_wrappers:system_wrappers_default", + "../../test:test_support", + "../rtp_rtcp", + "//testing/gtest", + ] + } # insert_packet_with_timing + + audio_decoder_unittests_resources = + [ "../../resources/audio_coding/testfile32kHz.pcm" ] + + if (is_ios) { + bundle_data("audio_decoder_unittests_bundle_data") { + testonly = true + sources = audio_decoder_unittests_resources + outputs = [ + "{{bundle_resources_dir}}/{{source_file_part}}", + ] + } + } + + rtc_test("audio_decoder_unittests") { + testonly = true + sources = [ + "neteq/audio_decoder_unittest.cc", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [] + + defines = neteq_defines + + deps += audio_coding_deps + deps += [ + ":ilbc", + ":isac", + ":isac_fix", + ":neteq", + ":neteq_tools", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../common_audio", + "../../rtc_base:protobuf_utils", + "../../test:test_main", + "//testing/gtest", + ] + + data = audio_decoder_unittests_resources + + if (is_android) { + deps += [ "//testing/android/native_test:native_test_native_code" ] + shard_timeout = 900 + } + if (is_ios) { + deps += [ ":audio_decoder_unittests_bundle_data" ] + } + } # audio_decoder_unittests + + if (rtc_enable_protobuf) { + proto_library("neteq_unittest_proto") { + sources = [ + "neteq/neteq_unittest.proto", + ] + proto_out_dir = "modules/audio_coding/neteq" + } + + rtc_test("neteq_rtpplay") { + testonly = true + defines = [] + deps = [ + "..:module_api", + ] + sources = [ + "neteq/tools/neteq_rtpplay.cc", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + if (is_win) { + cflags = [ + # TODO(kjellander): bugs.webrtc.org/261: Fix this warning. + "/wd4373", # virtual function override. + ] + } + + deps += [ + ":neteq", + ":neteq_test_tools", + "../..:webrtc_common", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + "../../test:test_support", + ] + } + } + + audio_codec_speed_tests_resources = [ + "//resources/audio_coding/music_stereo_48kHz.pcm", + "//resources/audio_coding/speech_mono_16kHz.pcm", + "//resources/audio_coding/speech_mono_32_48kHz.pcm", + ] + + if (is_ios) { + bundle_data("audio_codec_speed_tests_data") { + testonly = true + sources = audio_codec_speed_tests_resources + outputs = [ + "{{bundle_resources_dir}}/{{source_file_part}}", + ] + } + } + + rtc_test("audio_codec_speed_tests") { + testonly = true + defines = [] + deps = [] + sources = [ + "codecs/isac/fix/test/isac_speed_test.cc", + "codecs/opus/opus_speed_test.cc", + "codecs/tools/audio_codec_speed_test.cc", + "codecs/tools/audio_codec_speed_test.h", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + data = audio_codec_speed_tests_resources + + if (is_android) { + deps += [ "//testing/android/native_test:native_test_native_code" ] + shard_timeout = 900 + } + + if (is_ios) { + deps += [ ":audio_codec_speed_tests_data" ] + } + + deps += [ + ":isac_fix", + ":webrtc_opus", + "../..:webrtc_common", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:metrics_default", + "../../system_wrappers:system_wrappers_default", + "../../test:field_trial", + "../../test:test_main", + "../audio_processing", + "//testing/gtest", + ] + + if (!build_with_mozilla) { + deps += [ "../../api:libjingle_peerconnection_api" ] + } + } + + rtc_source_set("neteq_test_support") { + testonly = true + sources = [ + "neteq/tools/neteq_external_decoder_test.cc", + "neteq/tools/neteq_external_decoder_test.h", + "neteq/tools/neteq_performance_test.cc", + "neteq/tools/neteq_performance_test.h", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + ":neteq", + ":neteq_test_tools", + ":pcm16b", + "..:module_api", + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../../test:test_support", + "//testing/gtest", + ] + } + + rtc_source_set("neteq_quality_test_support") { + testonly = true + sources = [ + "neteq/tools/neteq_quality_test.cc", + "neteq/tools/neteq_quality_test.h", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + ":neteq", + ":neteq_test_tools", + "..:module_api", + "../..:webrtc_common", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../rtc_base:rtc_base_approved", + "../../test:test_support", + "//testing/gtest", + ] + } + + rtc_executable("rtp_encode") { + testonly = true + + deps = audio_coding_deps + [ + ":audio_coding", + ":neteq_input_audio_tools", + "../../api/audio_codecs/g711:audio_encoder_g711", + "../../api/audio_codecs/L16:audio_encoder_L16", + "../../api/audio_codecs/g722:audio_encoder_g722", + "../../api/audio_codecs/ilbc:audio_encoder_ilbc", + "../../api/audio_codecs/isac:audio_encoder_isac", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + ] + + sources = [ + "neteq/tools/rtp_encode.cc", + ] + + defines = audio_coding_defines + } + + rtc_executable("rtp_jitter") { + testonly = true + + deps = audio_coding_deps + [ + "../rtp_rtcp:rtp_rtcp_format", + "../../api:array_view", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + ] + + sources = [ + "neteq/tools/rtp_jitter.cc", + ] + + defines = audio_coding_defines + } + + rtc_executable("rtpcat") { + testonly = true + + sources = [ + "neteq/tools/rtpcat.cc", + ] + + deps = [ + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + "../../test:rtp_test_utils", + "//testing/gtest", + ] + } + + rtc_executable("rtp_analyze") { + testonly = true + + sources = [ + "neteq/tools/rtp_analyze.cc", + ] + + deps = [ + ":neteq", + ":neteq_test_tools", + ":pcm16b", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + "//testing/gtest", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + + rtc_executable("neteq_opus_quality_test") { + testonly = true + + sources = [ + "neteq/test/neteq_opus_quality_test.cc", + ] + + deps = [ + ":neteq", + ":neteq_quality_test_support", + ":neteq_tools", + ":webrtc_opus", + "../../rtc_base:rtc_base_approved", + "../../test:test_main", + "//testing/gtest", + ] + } + + rtc_executable("neteq_speed_test") { + testonly = true + + sources = [ + "neteq/test/neteq_speed_test.cc", + ] + + deps = [ + ":neteq", + ":neteq_test_support", + "../..:webrtc_common", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + "../../test:test_support", + ] + } + + rtc_executable("neteq_ilbc_quality_test") { + testonly = true + + sources = [ + "neteq/test/neteq_ilbc_quality_test.cc", + ] + + deps = [ + ":ilbc", + ":neteq", + ":neteq_quality_test_support", + ":neteq_tools", + "../..:webrtc_common", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + "../../test:test_main", + "//testing/gtest", + ] + } + + rtc_executable("neteq_isac_quality_test") { + testonly = true + + sources = [ + "neteq/test/neteq_isac_quality_test.cc", + ] + + deps = [ + ":isac_fix", + ":neteq", + ":neteq_quality_test_support", + "../../rtc_base:rtc_base_approved", + "../../test:test_main", + "//testing/gtest", + ] + } + + rtc_executable("neteq_pcmu_quality_test") { + testonly = true + + sources = [ + "neteq/test/neteq_pcmu_quality_test.cc", + ] + + deps = [ + ":g711", + ":neteq", + ":neteq_quality_test_support", + "../../rtc_base:rtc_base_approved", + "../../test:test_main", + "//testing/gtest", + ] + } + + rtc_executable("isac_fix_test") { + testonly = true + + sources = [ + "codecs/isac/fix/test/kenny.cc", + ] + + deps = [ + ":isac_fix", + "../../test:test_support", + ] + + data = [ + "../../resources/speech_and_misc_wb.pcm", + ] + + if (is_win) { + cflags = [ + # Disable warnings to enable Win64 build, issue 1323. + "/wd4267", # size_t to int truncation + ] + } + } + + config("isac_test_warnings_config") { + if (is_win && is_clang) { + cflags = [ + # Disable warnings failing when compiling with Clang on Windows. + # https://bugs.chromium.org/p/webrtc/issues/detail?id=5366 + "-Wno-format", + ] + } + } + + rtc_source_set("isac_test_util") { + testonly = true + sources = [ + "codecs/isac/main/util/utility.c", + ] + } + + rtc_executable("isac_test") { + testonly = true + + sources = [ + "codecs/isac/main/test/simpleKenny.c", + ] + + include_dirs = [ + "codecs/isac/main/include", + "codecs/isac/main/test", + "codecs/isac/main/util", + ] + + deps = [ + ":isac", + ":isac_test_util", + "../../rtc_base:rtc_base_approved", + ] + + configs += [ ":isac_test_warnings_config" ] + } + + rtc_executable("g711_test") { + testonly = true + + sources = [ + "codecs/g711/test/testG711.cc", + ] + + deps = [ + ":g711", + ] + } + + rtc_executable("g722_test") { + testonly = true + + sources = [ + "codecs/g722/test/testG722.cc", + ] + + deps = [ + ":g722", + "../..:webrtc_common", + ] + } + + rtc_executable("isac_api_test") { + testonly = true + + sources = [ + "codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc", + ] + + deps = [ + ":isac", + ":isac_test_util", + "../../rtc_base:rtc_base_approved", + ] + + include_dirs = [ + "codecs/isac/main/include", + "codecs/isac/main/test", + "codecs/isac/main/util", + ] + } + + rtc_executable("isac_switch_samprate_test") { + testonly = true + + sources = [ + "codecs/isac/main/test/SwitchingSampRate/SwitchingSampRate.cc", + ] + + deps = [ + ":isac", + ":isac_test_util", + ] + + include_dirs = [ + "codecs/isac/main/include", + "codecs/isac/main/test", + "codecs/isac/main/util", + "../../common_audio/signal_processing/include", + ] + } + + rtc_executable("ilbc_test") { + testonly = true + + sources = [ + "codecs/ilbc/test/iLBC_test.c", + ] + + deps = [ + ":ilbc", + ] + } + + rtc_executable("webrtc_opus_fec_test") { + testonly = true + + sources = [ + "codecs/opus/opus_fec_test.cc", + ] + + deps = [ + ":webrtc_opus", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + "../../test:test_main", + "//testing/gtest", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + + rtc_source_set("audio_coding_unittests") { + testonly = true + + sources = [ + "acm2/acm_receiver_unittest.cc", + "acm2/audio_coding_module_unittest.cc", + "acm2/call_statistics_unittest.cc", + "acm2/codec_manager_unittest.cc", + "acm2/rent_a_codec_unittest.cc", + "audio_network_adaptor/audio_network_adaptor_impl_unittest.cc", + "audio_network_adaptor/bitrate_controller_unittest.cc", + "audio_network_adaptor/channel_controller_unittest.cc", + "audio_network_adaptor/controller_manager_unittest.cc", + "audio_network_adaptor/dtx_controller_unittest.cc", + "audio_network_adaptor/event_log_writer_unittest.cc", + "audio_network_adaptor/fec_controller_plr_based_unittest.cc", + "audio_network_adaptor/fec_controller_rplr_based_unittest.cc", + "audio_network_adaptor/frame_length_controller_unittest.cc", + "audio_network_adaptor/mock/mock_controller.h", + "audio_network_adaptor/mock/mock_controller_manager.h", + "audio_network_adaptor/util/threshold_curve_unittest.cc", + "codecs/builtin_audio_decoder_factory_unittest.cc", + "codecs/builtin_audio_encoder_factory_unittest.cc", + "codecs/cng/audio_encoder_cng_unittest.cc", + "codecs/cng/cng_unittest.cc", + "codecs/ilbc/ilbc_unittest.cc", + "codecs/isac/fix/source/filterbanks_unittest.cc", + "codecs/isac/fix/source/filters_unittest.cc", + "codecs/isac/fix/source/lpc_masking_model_unittest.cc", + "codecs/isac/fix/source/transform_unittest.cc", + "codecs/isac/main/source/audio_encoder_isac_unittest.cc", + "codecs/isac/main/source/isac_unittest.cc", + "codecs/isac/unittest.cc", + "codecs/legacy_encoded_audio_frame_unittest.cc", + "codecs/opus/audio_encoder_opus_unittest.cc", + "codecs/opus/opus_bandwidth_unittest.cc", + "codecs/opus/opus_unittest.cc", + "codecs/red/audio_encoder_copy_red_unittest.cc", + "neteq/audio_multi_vector_unittest.cc", + "neteq/audio_vector_unittest.cc", + "neteq/background_noise_unittest.cc", + "neteq/buffer_level_filter_unittest.cc", + "neteq/comfort_noise_unittest.cc", + "neteq/decision_logic_unittest.cc", + "neteq/decoder_database_unittest.cc", + "neteq/delay_manager_unittest.cc", + "neteq/delay_peak_detector_unittest.cc", + "neteq/dsp_helper_unittest.cc", + "neteq/dtmf_buffer_unittest.cc", + "neteq/dtmf_tone_generator_unittest.cc", + "neteq/expand_unittest.cc", + "neteq/merge_unittest.cc", + "neteq/mock/mock_buffer_level_filter.h", + "neteq/mock/mock_decoder_database.h", + "neteq/mock/mock_delay_manager.h", + "neteq/mock/mock_delay_peak_detector.h", + "neteq/mock/mock_dtmf_buffer.h", + "neteq/mock/mock_dtmf_tone_generator.h", + "neteq/mock/mock_expand.h", + "neteq/mock/mock_external_decoder_pcm16b.h", + "neteq/mock/mock_packet_buffer.h", + "neteq/mock/mock_red_payload_splitter.h", + "neteq/mock/mock_statistics_calculator.h", + "neteq/nack_tracker_unittest.cc", + "neteq/neteq_external_decoder_unittest.cc", + "neteq/neteq_impl_unittest.cc", + "neteq/neteq_network_stats_unittest.cc", + "neteq/neteq_stereo_unittest.cc", + "neteq/neteq_unittest.cc", + "neteq/normal_unittest.cc", + "neteq/packet_buffer_unittest.cc", + "neteq/post_decode_vad_unittest.cc", + "neteq/random_vector_unittest.cc", + "neteq/red_payload_splitter_unittest.cc", + "neteq/statistics_calculator_unittest.cc", + "neteq/sync_buffer_unittest.cc", + "neteq/tick_timer_unittest.cc", + "neteq/time_stretch_unittest.cc", + "neteq/timestamp_scaler_unittest.cc", + "neteq/tools/input_audio_file_unittest.cc", + "neteq/tools/packet_unittest.cc", + ] + + deps = [ + ":acm_receive_test", + ":acm_send_test", + ":audio_coding", + ":audio_coding_module_typedefs", + ":audio_format_conversion", + ":audio_network_adaptor", + ":cng", + ":g711", + ":ilbc", + ":isac", + ":isac_c", + ":isac_fix", + ":legacy_encoded_audio_frame", + ":neteq", + ":neteq_test_support", + ":neteq_test_tools", + ":pcm16b", + ":red", + ":rent_a_codec", + ":webrtc_opus", + "..:module_api", + "../..:webrtc_common", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../api/audio_codecs/opus:audio_decoder_opus", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../common_audio", + "../../common_audio:mock_common_audio", + "../../logging:rtc_event_log_api", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_base_tests_utils", + "../../system_wrappers:system_wrappers", + "../../test:audio_codec_mocks", + "../../test:field_trial", + "../../test:rtp_test_utils", + "../../test:test_common", + "../../test:test_support", + "//testing/gmock", + "//testing/gtest", + ] + + defines = audio_coding_defines + + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_NETEQ_UNITTEST_BITEXACT" ] + deps += [ + ":ana_config_proto", + ":neteq_unittest_proto", + ] + } + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } +} + +# For backwards compatibility only! Use +# webrtc/api/audio_codecs:audio_codecs_api instead. +# TODO(kwiberg): Remove this. +rtc_source_set("audio_decoder_interface") { + sources = [ + "codecs/audio_decoder.h", + ] + deps = [ + "../../api/audio_codecs:audio_codecs_api", + ] +} + +# For backwards compatibility only! Use +# webrtc/api/audio_codecs:audio_codecs_api instead. +# TODO(ossu): Remove this. +rtc_source_set("audio_encoder_interface") { + sources = [ + "codecs/audio_encoder.h", + ] + deps = [ + "../../api/audio_codecs:audio_codecs_api", + ] +} + +# For backwards compatibility only! Use +# webrtc/api/audio_codecs:builtin_audio_decoder_factory instead. +# TODO(kwiberg): Remove this. +rtc_source_set("builtin_audio_decoder_factory") { + sources = [ + "codecs/builtin_audio_decoder_factory.h", + ] + deps = [ + "../../api/audio_codecs:builtin_audio_decoder_factory", + ] +} + +# For backwards compatibility only! Use +# webrtc/api/audio_codecs:builtin_audio_decoder_factory instead. +# TODO(ossu): Remove this. +rtc_source_set("builtin_audio_encoder_factory") { + sources = [ + "codecs/builtin_audio_encoder_factory.h", + ] + deps = [ + "../../api/audio_codecs:builtin_audio_encoder_factory", + ] +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/DEPS b/third_party/libwebrtc/webrtc/modules/audio_coding/DEPS new file mode 100644 index 0000000000..3dc9624a4b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/DEPS @@ -0,0 +1,7 @@ +include_rules = [ + "+call", + "+common_audio", + "+logging/rtc_event_log", + "+audio_coding/neteq/neteq_unittest.pb.h", # Different path. + "+system_wrappers", +] diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_coding/OWNERS new file mode 100644 index 0000000000..c1adc561b8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/OWNERS @@ -0,0 +1,11 @@ +turaj@webrtc.org +henrik.lundin@webrtc.org +kwiberg@webrtc.org +minyue@webrtc.org +jan.skoglund@webrtc.org +ossu@webrtc.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.cc new file mode 100644 index 0000000000..4553b52c62 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.cc @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This file generates databases with information about all supported audio + * codecs. + */ + +// TODO(tlegrand): Change constant input pointers in all functions to constant +// references, where appropriate. +#include "modules/audio_coding/acm2/acm_codec_database.h" + +#include + +#include "rtc_base/checks.h" + +#if ((defined WEBRTC_CODEC_ISAC) && (defined WEBRTC_CODEC_ISACFX)) +#error iSAC and iSACFX codecs cannot be enabled at the same time +#endif + +namespace webrtc { + +namespace acm2 { + +namespace { + +// Checks if the bitrate is valid for iSAC. +bool IsISACRateValid(int rate) { + return (rate == -1) || ((rate <= 56000) && (rate >= 10000)); +} + +// Checks if the bitrate is valid for iLBC. +bool IsILBCRateValid(int rate, int frame_size_samples) { + if (((frame_size_samples == 240) || (frame_size_samples == 480)) && + (rate == 13300)) { + return true; + } else if (((frame_size_samples == 160) || (frame_size_samples == 320)) && + (rate == 15200)) { + return true; + } else { + return false; + } +} + +// Checks if the bitrate is valid for Opus. +bool IsOpusRateValid(int rate) { + return (rate >= 6000) && (rate <= 510000); +} + +} // namespace + +// Not yet used payload-types. +// 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, +// 67, 66, 65 + +const CodecInst ACMCodecDB::database_[] = { +#if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) + {103, "ISAC", 16000, 480, 1, 32000}, +# if (defined(WEBRTC_CODEC_ISAC)) + {104, "ISAC", 32000, 960, 1, 56000}, +# endif +#endif + // Mono + {107, "L16", 8000, 80, 1, 128000}, + {108, "L16", 16000, 160, 1, 256000}, + {109, "L16", 32000, 320, 1, 512000}, + // Stereo + {111, "L16", 8000, 80, 2, 128000}, + {112, "L16", 16000, 160, 2, 256000}, + {113, "L16", 32000, 320, 2, 512000}, + // G.711, PCM mu-law and A-law. + // Mono + {0, "PCMU", 8000, 160, 1, 64000}, + {8, "PCMA", 8000, 160, 1, 64000}, + // Stereo + {110, "PCMU", 8000, 160, 2, 64000}, + {118, "PCMA", 8000, 160, 2, 64000}, +#ifdef WEBRTC_CODEC_ILBC + {102, "ILBC", 8000, 240, 1, 13300}, +#endif + // Mono + {9, "G722", 16000, 320, 1, 64000}, + // Stereo + {119, "G722", 16000, 320, 2, 64000}, +#ifdef WEBRTC_CODEC_OPUS + // Opus internally supports 48, 24, 16, 12, 8 kHz. + // Mono and stereo. + {120, "opus", 48000, 960, 2, 64000}, +#endif + // Comfort noise for four different sampling frequencies. + {13, "CN", 8000, 240, 1, 0}, + {98, "CN", 16000, 480, 1, 0}, + {99, "CN", 32000, 960, 1, 0}, +#ifdef ENABLE_48000_HZ + {100, "CN", 48000, 1440, 1, 0}, +#endif + {106, "telephone-event", 8000, 240, 1, 0}, + {114, "telephone-event", 16000, 240, 1, 0}, + {115, "telephone-event", 32000, 240, 1, 0}, + {116, "telephone-event", 48000, 240, 1, 0}, +#ifdef WEBRTC_CODEC_RED + {127, "red", 8000, 0, 1, 0}, +#endif + // To prevent compile errors due to trailing commas. + {-1, "Null", -1, -1, 0, -1} +}; + +// Create database with all codec settings at compile time. +// Each entry needs the following parameters in the given order: +// Number of allowed packet sizes, a vector with the allowed packet sizes, +// Basic block samples, max number of channels that are supported. +const ACMCodecDB::CodecSettings ACMCodecDB::codec_settings_[] = { +#if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) + {2, {480, 960}, 0, 1}, +# if (defined(WEBRTC_CODEC_ISAC)) + {1, {960}, 0, 1}, +# endif +#endif + // Mono + {4, {80, 160, 240, 320}, 0, 2}, + {4, {160, 320, 480, 640}, 0, 2}, + {2, {320, 640}, 0, 2}, + // Stereo + {4, {80, 160, 240, 320}, 0, 2}, + {4, {160, 320, 480, 640}, 0, 2}, + {2, {320, 640}, 0, 2}, + // G.711, PCM mu-law and A-law. + // Mono + {6, {80, 160, 240, 320, 400, 480}, 0, 2}, + {6, {80, 160, 240, 320, 400, 480}, 0, 2}, + // Stereo + {6, {80, 160, 240, 320, 400, 480}, 0, 2}, + {6, {80, 160, 240, 320, 400, 480}, 0, 2}, +#ifdef WEBRTC_CODEC_ILBC + {4, {160, 240, 320, 480}, 0, 1}, +#endif + // Mono + {6, {160, 320, 480, 640, 800, 960}, 0, 2}, + // Stereo + {6, {160, 320, 480, 640, 800, 960}, 0, 2}, +#ifdef WEBRTC_CODEC_OPUS + // Opus supports frames shorter than 10ms, + // but it doesn't help us to use them. + // Mono and stereo. +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME + {5, {480, 960, 1920, 2880, 5760}, 0, 2}, +#else + {4, {480, 960, 1920, 2880}, 0, 2}, +#endif +#endif + // Comfort noise for three different sampling frequencies. + {1, {240}, 240, 1}, + {1, {480}, 480, 1}, + {1, {960}, 960, 1}, +// TODO(solenberg): What is this flag? It is never set in the build files. +#ifdef ENABLE_48000_HZ + {1, {1440}, 1440, 1}, +#endif + {1, {240}, 240, 1}, + {1, {240}, 240, 1}, + {1, {240}, 240, 1}, + {1, {240}, 240, 1}, +#ifdef WEBRTC_CODEC_RED + {1, {0}, 0, 1}, +#endif + // To prevent compile errors due to trailing commas. + {-1, {-1}, -1, 0} +}; + +// Create a database of all NetEQ decoders at compile time. +const NetEqDecoder ACMCodecDB::neteq_decoders_[] = { +#if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) + NetEqDecoder::kDecoderISAC, +# if (defined(WEBRTC_CODEC_ISAC)) + NetEqDecoder::kDecoderISACswb, +# endif +#endif + // Mono + NetEqDecoder::kDecoderPCM16B, NetEqDecoder::kDecoderPCM16Bwb, + NetEqDecoder::kDecoderPCM16Bswb32kHz, + // Stereo + NetEqDecoder::kDecoderPCM16B_2ch, NetEqDecoder::kDecoderPCM16Bwb_2ch, + NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch, + // G.711, PCM mu-las and A-law. + // Mono + NetEqDecoder::kDecoderPCMu, NetEqDecoder::kDecoderPCMa, + // Stereo + NetEqDecoder::kDecoderPCMu_2ch, NetEqDecoder::kDecoderPCMa_2ch, +#ifdef WEBRTC_CODEC_ILBC + NetEqDecoder::kDecoderILBC, +#endif + // Mono + NetEqDecoder::kDecoderG722, + // Stereo + NetEqDecoder::kDecoderG722_2ch, +#ifdef WEBRTC_CODEC_OPUS + // Mono and stereo. + NetEqDecoder::kDecoderOpus, +#endif + // Comfort noise for three different sampling frequencies. + NetEqDecoder::kDecoderCNGnb, NetEqDecoder::kDecoderCNGwb, + NetEqDecoder::kDecoderCNGswb32kHz, +#ifdef ENABLE_48000_HZ + NetEqDecoder::kDecoderCNGswb48kHz, +#endif + NetEqDecoder::kDecoderAVT, + NetEqDecoder::kDecoderAVT16kHz, + NetEqDecoder::kDecoderAVT32kHz, + NetEqDecoder::kDecoderAVT48kHz, +#ifdef WEBRTC_CODEC_RED + NetEqDecoder::kDecoderRED, +#endif +}; + +// Enumerator for error codes when asking for codec database id. +enum { + kInvalidCodec = -10, + kInvalidPayloadtype = -30, + kInvalidPacketSize = -40, + kInvalidRate = -50 +}; + +// Gets the codec id number from the database. If there is some mismatch in +// the codec settings, the function will return an error code. +// NOTE! The first mismatch found will generate the return value. +int ACMCodecDB::CodecNumber(const CodecInst& codec_inst) { + // Look for a matching codec in the database. + int codec_id = CodecId(codec_inst); + + // Checks if we found a matching codec. + if (codec_id == -1) { + return kInvalidCodec; + } + + // Checks the validity of payload type + if (!RentACodec::IsPayloadTypeValid(codec_inst.pltype)) { + return kInvalidPayloadtype; + } + + // Comfort Noise is special case, packet-size & rate is not checked. + if (STR_CASE_CMP(database_[codec_id].plname, "CN") == 0) { + return codec_id; + } + + // RED is special case, packet-size & rate is not checked. + if (STR_CASE_CMP(database_[codec_id].plname, "red") == 0) { + return codec_id; + } + + // Checks the validity of packet size. + if (codec_settings_[codec_id].num_packet_sizes > 0) { + bool packet_size_ok = false; + int i; + int packet_size_samples; + for (i = 0; i < codec_settings_[codec_id].num_packet_sizes; i++) { + packet_size_samples = + codec_settings_[codec_id].packet_sizes_samples[i]; + if (codec_inst.pacsize == packet_size_samples) { + packet_size_ok = true; + break; + } + } + + if (!packet_size_ok) { + return kInvalidPacketSize; + } + } + + if (codec_inst.pacsize < 1) { + return kInvalidPacketSize; + } + + // Check the validity of rate. Codecs with multiple rates have their own + // function for this. + if (STR_CASE_CMP("isac", codec_inst.plname) == 0) { + return IsISACRateValid(codec_inst.rate) ? codec_id : kInvalidRate; + } else if (STR_CASE_CMP("ilbc", codec_inst.plname) == 0) { + return IsILBCRateValid(codec_inst.rate, codec_inst.pacsize) + ? codec_id : kInvalidRate; + } else if (STR_CASE_CMP("opus", codec_inst.plname) == 0) { + return IsOpusRateValid(codec_inst.rate) + ? codec_id : kInvalidRate; + } + + return database_[codec_id].rate == codec_inst.rate ? codec_id : kInvalidRate; +} + +// Looks for a matching payload name, frequency, and channels in the +// codec list. Need to check all three since some codecs have several codec +// entries with different frequencies and/or channels. +// Does not check other codec settings, such as payload type and packet size. +// Returns the id of the codec, or -1 if no match is found. +int ACMCodecDB::CodecId(const CodecInst& codec_inst) { + return (CodecId(codec_inst.plname, codec_inst.plfreq, + codec_inst.channels)); +} + +int ACMCodecDB::CodecId(const char* payload_name, + int frequency, + size_t channels) { + for (const CodecInst& ci : RentACodec::Database()) { + bool name_match = false; + bool frequency_match = false; + bool channels_match = false; + + // Payload name, sampling frequency and number of channels need to match. + // NOTE! If |frequency| is -1, the frequency is not applicable, and is + // always treated as true, like for RED. + name_match = (STR_CASE_CMP(ci.plname, payload_name) == 0); + frequency_match = (frequency == ci.plfreq) || (frequency == -1); + // The number of channels must match for all codecs but Opus. + if (STR_CASE_CMP(payload_name, "opus") != 0) { + channels_match = (channels == ci.channels); + } else { + // For opus we just check that number of channels is valid. + channels_match = (channels == 1 || channels == 2); + } + + if (name_match && frequency_match && channels_match) { + // We have found a matching codec in the list. + return &ci - RentACodec::Database().data(); + } + } + + // We didn't find a matching codec. + return -1; +} +// Gets codec id number from database for the receiver. +int ACMCodecDB::ReceiverCodecNumber(const CodecInst& codec_inst) { + // Look for a matching codec in the database. + return CodecId(codec_inst); +} + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.h new file mode 100644 index 0000000000..81cd4be05a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This file generates databases with information about all supported audio + * codecs. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_CODEC_DATABASE_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_CODEC_DATABASE_H_ + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +namespace acm2 { + +// TODO(tlegrand): replace class ACMCodecDB with a namespace. +class ACMCodecDB { + public: + // kMaxNumCodecs - Maximum number of codecs that can be activated in one + // build. + // kMaxNumPacketSize - Maximum number of allowed packet sizes for one codec. + // These might need to be increased if adding a new codec to the database + static const int kMaxNumCodecs = 50; + static const int kMaxNumPacketSize = 6; + + // Codec specific settings + // + // num_packet_sizes - number of allowed packet sizes. + // packet_sizes_samples - list of the allowed packet sizes. + // basic_block_samples - assigned a value different from 0 if the codec + // requires to be fed with a specific number of samples + // that can be different from packet size. + // channel_support - number of channels supported to encode; + // 1 = mono, 2 = stereo, etc. + struct CodecSettings { + int num_packet_sizes; + int packet_sizes_samples[kMaxNumPacketSize]; + int basic_block_samples; + size_t channel_support; + }; + + // Returns codec id from database, given the information received in the input + // [codec_inst]. + // Input: + // [codec_inst] - Information about the codec for which we require the + // database id. + // Return: + // codec id if successful, otherwise < 0. + static int CodecNumber(const CodecInst& codec_inst); + static int CodecId(const CodecInst& codec_inst); + static int CodecId(const char* payload_name, int frequency, size_t channels); + static int ReceiverCodecNumber(const CodecInst& codec_inst); + + // Databases with information about the supported codecs + // database_ - stored information about all codecs: payload type, name, + // sampling frequency, packet size in samples, default channel + // support, and default rate. + // codec_settings_ - stored codec settings: number of allowed packet sizes, + // a vector with the allowed packet sizes, basic block + // samples, and max number of channels that are supported. + // neteq_decoders_ - list of supported decoders in NetEQ. + static const CodecInst database_[kMaxNumCodecs]; + static const CodecSettings codec_settings_[kMaxNumCodecs]; + static const NetEqDecoder neteq_decoders_[kMaxNumCodecs]; +}; + +} // namespace acm2 + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_ACM_CODEC_DATABASE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.cc new file mode 100644 index 0000000000..082506ab56 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.cc @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_receive_test.h" + +#include +#include + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +// Returns true if the codec should be registered, otherwise false. Changes +// the number of channels for the Opus codec to always be 1. +bool ModifyAndUseThisCodec(CodecInst* codec_param) { + if (STR_CASE_CMP(codec_param->plname, "CN") == 0 && + codec_param->plfreq == 48000) + return false; // Skip 48 kHz comfort noise. + + if (STR_CASE_CMP(codec_param->plname, "telephone-event") == 0) + return false; // Skip DTFM. + + return true; +} + +// Remaps payload types from ACM's default to those used in the resource file +// neteq_universal_new.rtp. Returns true if the codec should be registered, +// otherwise false. The payload types are set as follows (all are mono codecs): +// PCMu = 0; +// PCMa = 8; +// Comfort noise 8 kHz = 13 +// Comfort noise 16 kHz = 98 +// Comfort noise 32 kHz = 99 +// iLBC = 102 +// iSAC wideband = 103 +// iSAC super-wideband = 104 +// AVT/DTMF = 106 +// RED = 117 +// PCM16b 8 kHz = 93 +// PCM16b 16 kHz = 94 +// PCM16b 32 kHz = 95 +// G.722 = 94 +bool RemapPltypeAndUseThisCodec(const char* plname, + int plfreq, + size_t channels, + int* pltype) { + if (channels != 1) + return false; // Don't use non-mono codecs. + + // Re-map pltypes to those used in the NetEq test files. + if (STR_CASE_CMP(plname, "PCMU") == 0 && plfreq == 8000) { + *pltype = 0; + } else if (STR_CASE_CMP(plname, "PCMA") == 0 && plfreq == 8000) { + *pltype = 8; + } else if (STR_CASE_CMP(plname, "CN") == 0 && plfreq == 8000) { + *pltype = 13; + } else if (STR_CASE_CMP(plname, "CN") == 0 && plfreq == 16000) { + *pltype = 98; + } else if (STR_CASE_CMP(plname, "CN") == 0 && plfreq == 32000) { + *pltype = 99; + } else if (STR_CASE_CMP(plname, "ILBC") == 0) { + *pltype = 102; + } else if (STR_CASE_CMP(plname, "ISAC") == 0 && plfreq == 16000) { + *pltype = 103; + } else if (STR_CASE_CMP(plname, "ISAC") == 0 && plfreq == 32000) { + *pltype = 104; + } else if (STR_CASE_CMP(plname, "telephone-event") == 0 && plfreq == 8000) { + *pltype = 106; + } else if (STR_CASE_CMP(plname, "telephone-event") == 0 && plfreq == 16000) { + *pltype = 114; + } else if (STR_CASE_CMP(plname, "telephone-event") == 0 && plfreq == 32000) { + *pltype = 115; + } else if (STR_CASE_CMP(plname, "telephone-event") == 0 && plfreq == 48000) { + *pltype = 116; + } else if (STR_CASE_CMP(plname, "red") == 0) { + *pltype = 117; + } else if (STR_CASE_CMP(plname, "L16") == 0 && plfreq == 8000) { + *pltype = 93; + } else if (STR_CASE_CMP(plname, "L16") == 0 && plfreq == 16000) { + *pltype = 94; + } else if (STR_CASE_CMP(plname, "L16") == 0 && plfreq == 32000) { + *pltype = 95; + } else if (STR_CASE_CMP(plname, "G722") == 0) { + *pltype = 9; + } else { + // Don't use any other codecs. + return false; + } + return true; +} + +AudioCodingModule::Config MakeAcmConfig( + Clock* clock, + rtc::scoped_refptr decoder_factory) { + AudioCodingModule::Config config; + config.clock = clock; + config.decoder_factory = std::move(decoder_factory); + return config; +} + +} // namespace + +AcmReceiveTestOldApi::AcmReceiveTestOldApi( + PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz, + NumOutputChannels exptected_output_channels, + rtc::scoped_refptr decoder_factory) + : clock_(0), + acm_(webrtc::AudioCodingModule::Create( + MakeAcmConfig(&clock_, std::move(decoder_factory)))), + packet_source_(packet_source), + audio_sink_(audio_sink), + output_freq_hz_(output_freq_hz), + exptected_output_channels_(exptected_output_channels) {} + +AcmReceiveTestOldApi::~AcmReceiveTestOldApi() = default; + +void AcmReceiveTestOldApi::RegisterDefaultCodecs() { + CodecInst my_codec_param; + for (int n = 0; n < acm_->NumberOfCodecs(); n++) { + ASSERT_EQ(0, acm_->Codec(n, &my_codec_param)) << "Failed to get codec."; + if (ModifyAndUseThisCodec(&my_codec_param)) { + ASSERT_EQ(true, + acm_->RegisterReceiveCodec(my_codec_param.pltype, + CodecInstToSdp(my_codec_param))) + << "Couldn't register receive codec.\n"; + } + } +} + +void AcmReceiveTestOldApi::RegisterNetEqTestCodecs() { + CodecInst my_codec_param; + for (int n = 0; n < acm_->NumberOfCodecs(); n++) { + ASSERT_EQ(0, acm_->Codec(n, &my_codec_param)) << "Failed to get codec."; + if (!ModifyAndUseThisCodec(&my_codec_param)) { + // Skip this codec. + continue; + } + + if (RemapPltypeAndUseThisCodec(my_codec_param.plname, + my_codec_param.plfreq, + my_codec_param.channels, + &my_codec_param.pltype)) { + ASSERT_EQ(true, + acm_->RegisterReceiveCodec(my_codec_param.pltype, + CodecInstToSdp(my_codec_param))) + << "Couldn't register receive codec.\n"; + } + } +} + +void AcmReceiveTestOldApi::Run() { + for (std::unique_ptr packet(packet_source_->NextPacket()); packet; + packet = packet_source_->NextPacket()) { + // Pull audio until time to insert packet. + while (clock_.TimeInMilliseconds() < packet->time_ms()) { + AudioFrame output_frame; + bool muted; + EXPECT_EQ(0, + acm_->PlayoutData10Ms(output_freq_hz_, &output_frame, &muted)); + ASSERT_EQ(output_freq_hz_, output_frame.sample_rate_hz_); + ASSERT_FALSE(muted); + const size_t samples_per_block = + static_cast(output_freq_hz_ * 10 / 1000); + EXPECT_EQ(samples_per_block, output_frame.samples_per_channel_); + if (exptected_output_channels_ != kArbitraryChannels) { + if (output_frame.speech_type_ == webrtc::AudioFrame::kPLC) { + // Don't check number of channels for PLC output, since each test run + // usually starts with a short period of mono PLC before decoding the + // first packet. + } else { + EXPECT_EQ(exptected_output_channels_, output_frame.num_channels_); + } + } + ASSERT_TRUE(audio_sink_->WriteAudioFrame(output_frame)); + clock_.AdvanceTimeMilliseconds(10); + AfterGetAudio(); + } + + // Insert packet after converting from RTPHeader to WebRtcRTPHeader. + WebRtcRTPHeader header; + header.header = packet->header(); + header.frameType = kAudioFrameSpeech; + memset(&header.type.Audio, 0, sizeof(RTPAudioHeader)); + EXPECT_EQ(0, + acm_->IncomingPacket( + packet->payload(), + static_cast(packet->payload_length_bytes()), + header)) + << "Failure when inserting packet:" << std::endl + << " PT = " << static_cast(header.header.payloadType) << std::endl + << " TS = " << header.header.timestamp << std::endl + << " SN = " << header.header.sequenceNumber; + } +} + +AcmReceiveTestToggleOutputFreqOldApi::AcmReceiveTestToggleOutputFreqOldApi( + PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz_1, + int output_freq_hz_2, + int toggle_period_ms, + NumOutputChannels exptected_output_channels) + : AcmReceiveTestOldApi(packet_source, + audio_sink, + output_freq_hz_1, + exptected_output_channels, + CreateBuiltinAudioDecoderFactory()), + output_freq_hz_1_(output_freq_hz_1), + output_freq_hz_2_(output_freq_hz_2), + toggle_period_ms_(toggle_period_ms), + last_toggle_time_ms_(clock_.TimeInMilliseconds()) {} + +void AcmReceiveTestToggleOutputFreqOldApi::AfterGetAudio() { + if (clock_.TimeInMilliseconds() >= last_toggle_time_ms_ + toggle_period_ms_) { + output_freq_hz_ = (output_freq_hz_ == output_freq_hz_1_) + ? output_freq_hz_2_ + : output_freq_hz_1_; + last_toggle_time_ms_ = clock_.TimeInMilliseconds(); + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.h new file mode 100644 index 0000000000..657a12467a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receive_test.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_RECEIVE_TEST_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_RECEIVE_TEST_H_ + +#include +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +class AudioCodingModule; +class AudioDecoder; +struct CodecInst; + +namespace test { +class AudioSink; +class PacketSource; + +class AcmReceiveTestOldApi { + public: + enum NumOutputChannels { + kArbitraryChannels = 0, + kMonoOutput = 1, + kStereoOutput = 2 + }; + + AcmReceiveTestOldApi(PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz, + NumOutputChannels exptected_output_channels, + rtc::scoped_refptr decoder_factory); + virtual ~AcmReceiveTestOldApi(); + + // Registers the codecs with default parameters from ACM. + void RegisterDefaultCodecs(); + + // Registers codecs with payload types matching the pre-encoded NetEq test + // files. + void RegisterNetEqTestCodecs(); + + // Runs the test and returns true if successful. + void Run(); + + AudioCodingModule* get_acm() { return acm_.get(); } + + protected: + // Method is called after each block of output audio is received from ACM. + virtual void AfterGetAudio() {} + + SimulatedClock clock_; + std::unique_ptr acm_; + PacketSource* packet_source_; + AudioSink* audio_sink_; + int output_freq_hz_; + NumOutputChannels exptected_output_channels_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AcmReceiveTestOldApi); +}; + +// This test toggles the output frequency every |toggle_period_ms|. The test +// starts with |output_freq_hz_1|. Except for the toggling, it does the same +// thing as AcmReceiveTestOldApi. +class AcmReceiveTestToggleOutputFreqOldApi : public AcmReceiveTestOldApi { + public: + AcmReceiveTestToggleOutputFreqOldApi( + PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz_1, + int output_freq_hz_2, + int toggle_period_ms, + NumOutputChannels exptected_output_channels); + + protected: + void AfterGetAudio() override; + + const int output_freq_hz_1_; + const int output_freq_hz_2_; + const int toggle_period_ms_; + int64_t last_toggle_time_ms_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_ACM2_ACM_RECEIVE_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.cc new file mode 100644 index 0000000000..8b943ec3a7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.cc @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_receiver.h" + +#include // malloc + +#include // sort +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/audio_coding/acm2/call_statistics.h" +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +namespace acm2 { + +AcmReceiver::AcmReceiver(const AudioCodingModule::Config& config) + : last_audio_buffer_(new int16_t[AudioFrame::kMaxDataSizeSamples]), + neteq_(NetEq::Create(config.neteq_config, config.decoder_factory)), + clock_(config.clock), + resampled_last_output_frame_(true) { + RTC_DCHECK(clock_); + memset(last_audio_buffer_.get(), 0, AudioFrame::kMaxDataSizeSamples); +} + +AcmReceiver::~AcmReceiver() = default; + +int AcmReceiver::SetMinimumDelay(int delay_ms) { + if (neteq_->SetMinimumDelay(delay_ms)) + return 0; + RTC_LOG(LERROR) << "AcmReceiver::SetExtraDelay " << delay_ms; + return -1; +} + +int AcmReceiver::SetMaximumDelay(int delay_ms) { + if (neteq_->SetMaximumDelay(delay_ms)) + return 0; + RTC_LOG(LERROR) << "AcmReceiver::SetExtraDelay " << delay_ms; + return -1; +} + +int AcmReceiver::LeastRequiredDelayMs() const { + return neteq_->LeastRequiredDelayMs(); +} + +rtc::Optional AcmReceiver::last_packet_sample_rate_hz() const { + rtc::CritScope lock(&crit_sect_); + return last_packet_sample_rate_hz_; +} + +int AcmReceiver::last_output_sample_rate_hz() const { + return neteq_->last_output_sample_rate_hz(); +} + +int AcmReceiver::InsertPacket(const WebRtcRTPHeader& rtp_header, + rtc::ArrayView incoming_payload) { + uint32_t receive_timestamp = 0; + const RTPHeader* header = &rtp_header.header; // Just a shorthand. + + if (incoming_payload.empty()) { + neteq_->InsertEmptyPacket(rtp_header.header); + return 0; + } + + { + rtc::CritScope lock(&crit_sect_); + + const rtc::Optional ci = + RtpHeaderToDecoder(*header, incoming_payload[0]); + if (!ci) { + RTC_LOG_F(LS_ERROR) << "Payload-type " + << static_cast(header->payloadType) + << " is not registered."; + return -1; + } + receive_timestamp = NowInTimestamp(ci->plfreq); + + if (STR_CASE_CMP(ci->plname, "cn") == 0) { + if (last_audio_decoder_ && last_audio_decoder_->channels > 1) { + // This is a CNG and the audio codec is not mono, so skip pushing in + // packets into NetEq. + return 0; + } + } else { + last_audio_decoder_ = ci; + last_audio_format_ = neteq_->GetDecoderFormat(ci->pltype); + last_audio_format_clockrate_hz_ = last_audio_format_->clockrate_hz; + RTC_DCHECK(last_audio_format_); + last_packet_sample_rate_hz_ = ci->plfreq; + } + } // |crit_sect_| is released. + + if (neteq_->InsertPacket(rtp_header.header, incoming_payload, + receive_timestamp) < 0) { + RTC_LOG(LERROR) << "AcmReceiver::InsertPacket " + << static_cast(header->payloadType) + << " Failed to insert packet"; + return -1; + } + return 0; +} + +int AcmReceiver::GetAudio(int desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) { + RTC_DCHECK(muted); + + if (neteq_->GetAudio(audio_frame, muted) != NetEq::kOK) { + RTC_LOG(LERROR) << "AcmReceiver::GetAudio - NetEq Failed."; + return -1; + } + + const int current_sample_rate_hz = neteq_->last_output_sample_rate_hz(); + + // Update if resampling is required. + const bool need_resampling = + (desired_freq_hz != -1) && (current_sample_rate_hz != desired_freq_hz); + + if (need_resampling && !resampled_last_output_frame_) { + // Prime the resampler with the last frame. + int16_t temp_output[AudioFrame::kMaxDataSizeSamples]; + int samples_per_channel_int = resampler_.Resample10Msec( + last_audio_buffer_.get(), current_sample_rate_hz, desired_freq_hz, + audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples, + temp_output); + if (samples_per_channel_int < 0) { + RTC_LOG(LERROR) << "AcmReceiver::GetAudio - " + "Resampling last_audio_buffer_ failed."; + return -1; + } + } + + // TODO(henrik.lundin) Glitches in the output may appear if the output rate + // from NetEq changes. See WebRTC issue 3923. + if (need_resampling) { + // TODO(yujo): handle this more efficiently for muted frames. + int samples_per_channel_int = resampler_.Resample10Msec( + audio_frame->data(), current_sample_rate_hz, desired_freq_hz, + audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples, + audio_frame->mutable_data()); + if (samples_per_channel_int < 0) { + RTC_LOG(LERROR) + << "AcmReceiver::GetAudio - Resampling audio_buffer_ failed."; + return -1; + } + audio_frame->samples_per_channel_ = + static_cast(samples_per_channel_int); + audio_frame->sample_rate_hz_ = desired_freq_hz; + RTC_DCHECK_EQ( + audio_frame->sample_rate_hz_, + rtc::dchecked_cast(audio_frame->samples_per_channel_ * 100)); + resampled_last_output_frame_ = true; + } else { + resampled_last_output_frame_ = false; + // We might end up here ONLY if codec is changed. + } + + // Store current audio in |last_audio_buffer_| for next time. + memcpy(last_audio_buffer_.get(), audio_frame->data(), + sizeof(int16_t) * audio_frame->samples_per_channel_ * + audio_frame->num_channels_); + + call_stats_.DecodedByNetEq(audio_frame->speech_type_, *muted); + return 0; +} + +void AcmReceiver::SetCodecs(const std::map& codecs) { + neteq_->SetCodecs(codecs); +} + +int32_t AcmReceiver::AddCodec(int acm_codec_id, + uint8_t payload_type, + size_t channels, + int /*sample_rate_hz*/, + AudioDecoder* audio_decoder, + const std::string& name) { + // TODO(kwiberg): This function has been ignoring the |sample_rate_hz| + // argument for a long time. Arguably, it should simply be removed. + + const auto neteq_decoder = [acm_codec_id, channels]() -> NetEqDecoder { + if (acm_codec_id == -1) + return NetEqDecoder::kDecoderArbitrary; // External decoder. + const rtc::Optional cid = + RentACodec::CodecIdFromIndex(acm_codec_id); + RTC_DCHECK(cid) << "Invalid codec index: " << acm_codec_id; + const rtc::Optional ned = + RentACodec::NetEqDecoderFromCodecId(*cid, channels); + RTC_DCHECK(ned) << "Invalid codec ID: " << static_cast(*cid); + return *ned; + }(); + const rtc::Optional new_format = + NetEqDecoderToSdpAudioFormat(neteq_decoder); + + rtc::CritScope lock(&crit_sect_); + + const auto old_format = neteq_->GetDecoderFormat(payload_type); + if (old_format && new_format && *old_format == *new_format) { + // Re-registering the same codec. Do nothing and return. + return 0; + } + + if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK) { + RTC_LOG(LERROR) << "Cannot remove payload " + << static_cast(payload_type); + return -1; + } + + int ret_val; + if (!audio_decoder) { + ret_val = neteq_->RegisterPayloadType(neteq_decoder, name, payload_type); + } else { + ret_val = neteq_->RegisterExternalDecoder( + audio_decoder, neteq_decoder, name, payload_type); + } + if (ret_val != NetEq::kOK) { + RTC_LOG(LERROR) << "AcmReceiver::AddCodec " << acm_codec_id + << static_cast(payload_type) + << " channels: " << channels; + return -1; + } + return 0; +} + +bool AcmReceiver::AddCodec(int rtp_payload_type, + const SdpAudioFormat& audio_format) { + const auto old_format = neteq_->GetDecoderFormat(rtp_payload_type); + if (old_format && *old_format == audio_format) { + // Re-registering the same codec. Do nothing and return. + return true; + } + + if (neteq_->RemovePayloadType(rtp_payload_type) != NetEq::kOK) { + RTC_LOG(LERROR) + << "AcmReceiver::AddCodec: Could not remove existing decoder" + " for payload type " + << rtp_payload_type; + return false; + } + + const bool success = + neteq_->RegisterPayloadType(rtp_payload_type, audio_format); + if (!success) { + RTC_LOG(LERROR) << "AcmReceiver::AddCodec failed for payload type " + << rtp_payload_type << ", decoder format " << audio_format; + } + return success; +} + +void AcmReceiver::FlushBuffers() { + neteq_->FlushBuffers(); +} + +void AcmReceiver::RemoveAllCodecs() { + rtc::CritScope lock(&crit_sect_); + neteq_->RemoveAllPayloadTypes(); + last_audio_decoder_ = rtc::nullopt; + last_audio_format_ = rtc::nullopt; + last_packet_sample_rate_hz_ = rtc::nullopt; +} + +int AcmReceiver::RemoveCodec(uint8_t payload_type) { + rtc::CritScope lock(&crit_sect_); + if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK) { + RTC_LOG(LERROR) << "AcmReceiver::RemoveCodec " + << static_cast(payload_type); + return -1; + } + if (last_audio_decoder_ && payload_type == last_audio_decoder_->pltype) { + last_audio_decoder_ = rtc::nullopt; + last_audio_format_ = rtc::nullopt; + last_packet_sample_rate_hz_ = rtc::nullopt; + } + return 0; +} + +rtc::Optional AcmReceiver::GetPlayoutTimestamp() { + return neteq_->GetPlayoutTimestamp(); +} + +int AcmReceiver::FilteredCurrentDelayMs() const { + return neteq_->FilteredCurrentDelayMs(); +} + +int AcmReceiver::TargetDelayMs() const { + return neteq_->TargetDelayMs(); +} + +int AcmReceiver::LastAudioCodec(CodecInst* codec) const { + rtc::CritScope lock(&crit_sect_); + if (!last_audio_decoder_) { + return -1; + } + *codec = *last_audio_decoder_; + return 0; +} + +rtc::Optional AcmReceiver::LastAudioFormat() const { + rtc::CritScope lock(&crit_sect_); + return last_audio_format_; +} + +int AcmReceiver::LastAudioSampleRate() const { + return last_audio_format_clockrate_hz_; +} + +void AcmReceiver::GetNetworkStatistics(NetworkStatistics* acm_stat) { + NetEqNetworkStatistics neteq_stat; + // NetEq function always returns zero, so we don't check the return value. + neteq_->NetworkStatistics(&neteq_stat); + + acm_stat->currentBufferSize = neteq_stat.current_buffer_size_ms; + acm_stat->preferredBufferSize = neteq_stat.preferred_buffer_size_ms; + acm_stat->jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false; + acm_stat->currentPacketLossRate = neteq_stat.packet_loss_rate; + acm_stat->currentExpandRate = neteq_stat.expand_rate; + acm_stat->currentSpeechExpandRate = neteq_stat.speech_expand_rate; + acm_stat->currentPreemptiveRate = neteq_stat.preemptive_rate; + acm_stat->currentAccelerateRate = neteq_stat.accelerate_rate; + acm_stat->currentSecondaryDecodedRate = neteq_stat.secondary_decoded_rate; + acm_stat->currentSecondaryDiscardedRate = neteq_stat.secondary_discarded_rate; + acm_stat->clockDriftPPM = neteq_stat.clockdrift_ppm; + acm_stat->addedSamples = neteq_stat.added_zero_samples; + acm_stat->meanWaitingTimeMs = neteq_stat.mean_waiting_time_ms; + acm_stat->medianWaitingTimeMs = neteq_stat.median_waiting_time_ms; + acm_stat->minWaitingTimeMs = neteq_stat.min_waiting_time_ms; + acm_stat->maxWaitingTimeMs = neteq_stat.max_waiting_time_ms; + + NetEqLifetimeStatistics neteq_lifetime_stat = neteq_->GetLifetimeStatistics(); + acm_stat->totalSamplesReceived = neteq_lifetime_stat.total_samples_received; + acm_stat->concealedSamples = neteq_lifetime_stat.concealed_samples; + acm_stat->concealmentEvents = neteq_lifetime_stat.concealment_events; + acm_stat->jitterBufferDelayMs = neteq_lifetime_stat.jitter_buffer_delay_ms; +} + +int AcmReceiver::DecoderByPayloadType(uint8_t payload_type, + CodecInst* codec) const { + rtc::CritScope lock(&crit_sect_); + const rtc::Optional ci = neteq_->GetDecoder(payload_type); + if (ci) { + *codec = *ci; + return 0; + } else { + RTC_LOG(LERROR) << "AcmReceiver::DecoderByPayloadType " + << static_cast(payload_type); + return -1; + } +} + +int AcmReceiver::EnableNack(size_t max_nack_list_size) { + neteq_->EnableNack(max_nack_list_size); + return 0; +} + +void AcmReceiver::DisableNack() { + neteq_->DisableNack(); +} + +std::vector AcmReceiver::GetNackList( + int64_t round_trip_time_ms) const { + return neteq_->GetNackList(round_trip_time_ms); +} + +void AcmReceiver::ResetInitialDelay() { + neteq_->SetMinimumDelay(0); + // TODO(turajs): Should NetEq Buffer be flushed? +} + +const rtc::Optional AcmReceiver::RtpHeaderToDecoder( + const RTPHeader& rtp_header, + uint8_t first_payload_byte) const { + const rtc::Optional ci = + neteq_->GetDecoder(rtp_header.payloadType); + if (ci && STR_CASE_CMP(ci->plname, "red") == 0) { + // This is a RED packet. Get the payload of the audio codec. + return neteq_->GetDecoder(first_payload_byte & 0x7f); + } else { + return ci; + } +} + +uint32_t AcmReceiver::NowInTimestamp(int decoder_sampling_rate) const { + // Down-cast the time to (32-6)-bit since we only care about + // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. + // We masked 6 most significant bits of 32-bit so there is no overflow in + // the conversion from milliseconds to timestamp. + const uint32_t now_in_ms = static_cast( + clock_->TimeInMilliseconds() & 0x03ffffff); + return static_cast( + (decoder_sampling_rate / 1000) * now_in_ms); +} + +void AcmReceiver::GetDecodingCallStatistics( + AudioDecodingCallStats* stats) const { + rtc::CritScope lock(&crit_sect_); + *stats = call_stats_.GetDecodingStatistics(); +} + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.h new file mode 100644 index 0000000000..514111585b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.h @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_RECEIVER_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_RECEIVER_H_ + +#include +#include +#include +#include +#include + +#include "api/array_view.h" +#include "api/optional.h" +#include "common_audio/vad/include/webrtc_vad.h" +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/audio_coding/acm2/call_statistics.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/thread_annotations.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +struct CodecInst; +class NetEq; + +namespace acm2 { + +class AcmReceiver { + public: + // Constructor of the class + explicit AcmReceiver(const AudioCodingModule::Config& config); + + // Destructor of the class. + ~AcmReceiver(); + + // + // Inserts a payload with its associated RTP-header into NetEq. + // + // Input: + // - rtp_header : RTP header for the incoming payload containing + // information about payload type, sequence number, + // timestamp, SSRC and marker bit. + // - incoming_payload : Incoming audio payload. + // - length_payload : Length of incoming audio payload in bytes. + // + // Return value : 0 if OK. + // <0 if NetEq returned an error. + // + int InsertPacket(const WebRtcRTPHeader& rtp_header, + rtc::ArrayView incoming_payload); + + // + // Asks NetEq for 10 milliseconds of decoded audio. + // + // Input: + // -desired_freq_hz : specifies the sampling rate [Hz] of the output + // audio. If set -1 indicates to resampling is + // is required and the audio returned at the + // sampling rate of the decoder. + // + // Output: + // -audio_frame : an audio frame were output data and + // associated parameters are written to. + // -muted : if true, the sample data in audio_frame is not + // populated, and must be interpreted as all zero. + // + // Return value : 0 if OK. + // -1 if NetEq returned an error. + // + int GetAudio(int desired_freq_hz, AudioFrame* audio_frame, bool* muted); + + // Replace the current set of decoders with the specified set. + void SetCodecs(const std::map& codecs); + + // + // Adds a new codec to the NetEq codec database. + // + // Input: + // - acm_codec_id : ACM codec ID; -1 means external decoder. + // - payload_type : payload type. + // - sample_rate_hz : sample rate. + // - audio_decoder : pointer to a decoder object. If it's null, then + // NetEq will internally create a decoder object + // based on the value of |acm_codec_id| (which + // mustn't be -1). Otherwise, NetEq will use the + // given decoder for the given payload type. NetEq + // won't take ownership of the decoder; it's up to + // the caller to delete it when it's no longer + // needed. + // + // Providing an existing decoder object here is + // necessary for external decoders, but may also be + // used for built-in decoders if NetEq doesn't have + // all the info it needs to construct them properly + // (e.g. iSAC, where the decoder needs to be paired + // with an encoder). + // + // Return value : 0 if OK. + // <0 if NetEq returned an error. + // + int AddCodec(int acm_codec_id, + uint8_t payload_type, + size_t channels, + int sample_rate_hz, + AudioDecoder* audio_decoder, + const std::string& name); + + // Adds a new decoder to the NetEq codec database. Returns true iff + // successful. + bool AddCodec(int rtp_payload_type, const SdpAudioFormat& audio_format); + + // + // Sets a minimum delay for packet buffer. The given delay is maintained, + // unless channel condition dictates a higher delay. + // + // Input: + // - delay_ms : minimum delay in milliseconds. + // + // Return value : 0 if OK. + // <0 if NetEq returned an error. + // + int SetMinimumDelay(int delay_ms); + + // + // Sets a maximum delay [ms] for the packet buffer. The target delay does not + // exceed the given value, even if channel condition requires so. + // + // Input: + // - delay_ms : maximum delay in milliseconds. + // + // Return value : 0 if OK. + // <0 if NetEq returned an error. + // + int SetMaximumDelay(int delay_ms); + + // + // Get least required delay computed based on channel conditions. Note that + // this is before applying any user-defined limits (specified by calling + // (SetMinimumDelay() and/or SetMaximumDelay()). + // + int LeastRequiredDelayMs() const; + + // + // Resets the initial delay to zero. + // + void ResetInitialDelay(); + + // Returns the sample rate of the decoder associated with the last incoming + // packet. If no packet of a registered non-CNG codec has been received, the + // return value is empty. Also, if the decoder was unregistered since the last + // packet was inserted, the return value is empty. + rtc::Optional last_packet_sample_rate_hz() const; + + // Returns last_output_sample_rate_hz from the NetEq instance. + int last_output_sample_rate_hz() const; + + // + // Get the current network statistics from NetEq. + // + // Output: + // - statistics : The current network statistics. + // + void GetNetworkStatistics(NetworkStatistics* statistics); + + // + // Flushes the NetEq packet and speech buffers. + // + void FlushBuffers(); + + // + // Removes a payload-type from the NetEq codec database. + // + // Input: + // - payload_type : the payload-type to be removed. + // + // Return value : 0 if OK. + // -1 if an error occurred. + // + int RemoveCodec(uint8_t payload_type); + + // + // Remove all registered codecs. + // + void RemoveAllCodecs(); + + // Returns the RTP timestamp for the last sample delivered by GetAudio(). + // The return value will be empty if no valid timestamp is available. + rtc::Optional GetPlayoutTimestamp(); + + // Returns the current total delay from NetEq (packet buffer and sync buffer) + // in ms, with smoothing applied to even out short-time fluctuations due to + // jitter. The packet buffer part of the delay is not updated during DTX/CNG + // periods. + // + int FilteredCurrentDelayMs() const; + + // Returns the current target delay for NetEq in ms. + // + int TargetDelayMs() const; + + // + // Get the audio codec associated with the last non-CNG/non-DTMF received + // payload. If no non-CNG/non-DTMF packet is received -1 is returned, + // otherwise return 0. + // + int LastAudioCodec(CodecInst* codec) const; + + rtc::Optional LastAudioFormat() const; + int LastAudioSampleRate() const; + + // + // Get a decoder given its registered payload-type. + // + // Input: + // -payload_type : the payload-type of the codec to be retrieved. + // + // Output: + // -codec : codec associated with the given payload-type. + // + // Return value : 0 if succeeded. + // -1 if failed, e.g. given payload-type is not + // registered. + // + int DecoderByPayloadType(uint8_t payload_type, + CodecInst* codec) const; + + // + // Enable NACK and set the maximum size of the NACK list. If NACK is already + // enabled then the maximum NACK list size is modified accordingly. + // + // Input: + // -max_nack_list_size : maximum NACK list size + // should be positive (none zero) and less than or + // equal to |Nack::kNackListSizeLimit| + // Return value + // : 0 if succeeded. + // -1 if failed + // + int EnableNack(size_t max_nack_list_size); + + // Disable NACK. + void DisableNack(); + + // + // Get a list of packets to be retransmitted. + // + // Input: + // -round_trip_time_ms : estimate of the round-trip-time (in milliseconds). + // Return value : list of packets to be retransmitted. + // + std::vector GetNackList(int64_t round_trip_time_ms) const; + + // + // Get statistics of calls to GetAudio(). + void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const; + + private: + struct Decoder { + int acm_codec_id; + uint8_t payload_type; + // This field is meaningful for codecs where both mono and + // stereo versions are registered under the same ID. + size_t channels; + int sample_rate_hz; + }; + + const rtc::Optional RtpHeaderToDecoder(const RTPHeader& rtp_header, + uint8_t first_payload_byte) + const RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + uint32_t NowInTimestamp(int decoder_sampling_rate) const; + + rtc::CriticalSection crit_sect_; + rtc::Optional last_audio_decoder_ RTC_GUARDED_BY(crit_sect_); + rtc::Optional last_audio_format_ RTC_GUARDED_BY(crit_sect_); + ACMResampler resampler_ RTC_GUARDED_BY(crit_sect_); + + // After construction, this is only ever touched on the thread that calls + // AcmReceiver::GetAudio, and only modified in this method. + std::unique_ptr last_audio_buffer_; + CallStatistics call_stats_ RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr neteq_; // NetEq is thread-safe; no lock needed. + const Clock* const clock_; + std::atomic resampled_last_output_frame_; + rtc::Optional last_packet_sample_rate_hz_ RTC_GUARDED_BY(crit_sect_); + std::atomic last_audio_format_clockrate_hz_; +}; + +} // namespace acm2 + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_ACM_RECEIVER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc new file mode 100644 index 0000000000..8d0b2f100e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_receiver.h" + +#include // std::min +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +namespace acm2 { +namespace { + +bool CodecsEqual(const CodecInst& codec_a, const CodecInst& codec_b) { + if (strcmp(codec_a.plname, codec_b.plname) != 0 || + codec_a.plfreq != codec_b.plfreq || + codec_a.pltype != codec_b.pltype || + codec_b.channels != codec_a.channels) + return false; + return true; +} + +struct CodecIdInst { + explicit CodecIdInst(RentACodec::CodecId codec_id) { + const auto codec_ix = RentACodec::CodecIndexFromId(codec_id); + EXPECT_TRUE(codec_ix); + id = *codec_ix; + const auto codec_inst = RentACodec::CodecInstById(codec_id); + EXPECT_TRUE(codec_inst); + inst = *codec_inst; + } + int id; + CodecInst inst; +}; + +} // namespace + +class AcmReceiverTestOldApi : public AudioPacketizationCallback, + public ::testing::Test { + protected: + AcmReceiverTestOldApi() + : timestamp_(0), + packet_sent_(false), + last_packet_send_timestamp_(timestamp_), + last_frame_type_(kEmptyFrame) { + config_.decoder_factory = CreateBuiltinAudioDecoderFactory(); + } + + ~AcmReceiverTestOldApi() {} + + void SetUp() override { + acm_.reset(AudioCodingModule::Create(config_)); + receiver_.reset(new AcmReceiver(config_)); + ASSERT_TRUE(receiver_.get() != NULL); + ASSERT_TRUE(acm_.get() != NULL); + codecs_ = RentACodec::Database(); + + acm_->InitializeReceiver(); + acm_->RegisterTransportCallback(this); + + rtp_header_.header.sequenceNumber = 0; + rtp_header_.header.timestamp = 0; + rtp_header_.header.markerBit = false; + rtp_header_.header.ssrc = 0x12345678; // Arbitrary. + rtp_header_.header.numCSRCs = 0; + rtp_header_.header.payloadType = 0; + rtp_header_.frameType = kAudioFrameSpeech; + rtp_header_.type.Audio.isCNG = false; + } + + void TearDown() override {} + + void InsertOnePacketOfSilence(int codec_id) { + CodecInst codec = + *RentACodec::CodecInstById(*RentACodec::CodecIdFromIndex(codec_id)); + if (timestamp_ == 0) { // This is the first time inserting audio. + ASSERT_EQ(0, acm_->RegisterSendCodec(codec)); + } else { + auto current_codec = acm_->SendCodec(); + ASSERT_TRUE(current_codec); + if (!CodecsEqual(codec, *current_codec)) + ASSERT_EQ(0, acm_->RegisterSendCodec(codec)); + } + AudioFrame frame; + // Frame setup according to the codec. + frame.sample_rate_hz_ = codec.plfreq; + frame.samples_per_channel_ = codec.plfreq / 100; // 10 ms. + frame.num_channels_ = codec.channels; + frame.Mute(); + packet_sent_ = false; + last_packet_send_timestamp_ = timestamp_; + while (!packet_sent_) { + frame.timestamp_ = timestamp_; + timestamp_ += rtc::checked_cast(frame.samples_per_channel_); + ASSERT_GE(acm_->Add10MsData(frame), 0); + } + } + + template + void AddSetOfCodecs(const RentACodec::CodecId(&ids)[N]) { + for (auto id : ids) { + const auto i = RentACodec::CodecIndexFromId(id); + ASSERT_TRUE(i); + ASSERT_EQ(0, receiver_->AddCodec(*i, codecs_[*i].pltype, + codecs_[*i].channels, codecs_[*i].plfreq, + nullptr, codecs_[*i].plname)); + } + } + + int SendData(FrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + const RTPFragmentationHeader* fragmentation) override { + if (frame_type == kEmptyFrame) + return 0; + + rtp_header_.header.payloadType = payload_type; + rtp_header_.frameType = frame_type; + if (frame_type == kAudioFrameSpeech) + rtp_header_.type.Audio.isCNG = false; + else + rtp_header_.type.Audio.isCNG = true; + rtp_header_.header.timestamp = timestamp; + + int ret_val = receiver_->InsertPacket( + rtp_header_, + rtc::ArrayView(payload_data, payload_len_bytes)); + if (ret_val < 0) { + assert(false); + return -1; + } + rtp_header_.header.sequenceNumber++; + packet_sent_ = true; + last_frame_type_ = frame_type; + return 0; + } + + AudioCodingModule::Config config_; + std::unique_ptr receiver_; + rtc::ArrayView codecs_; + std::unique_ptr acm_; + WebRtcRTPHeader rtp_header_; + uint32_t timestamp_; + bool packet_sent_; // Set when SendData is called reset when inserting audio. + uint32_t last_packet_send_timestamp_; + FrameType last_frame_type_; +}; + +#if defined(WEBRTC_ANDROID) +#define MAYBE_AddCodecGetCodec DISABLED_AddCodecGetCodec +#else +#define MAYBE_AddCodecGetCodec AddCodecGetCodec +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_AddCodecGetCodec) { + // Add codec. + for (size_t n = 0; n < codecs_.size(); ++n) { + if (n & 0x1) { // Just add codecs with odd index. + EXPECT_EQ( + 0, receiver_->AddCodec(rtc::checked_cast(n), codecs_[n].pltype, + codecs_[n].channels, codecs_[n].plfreq, NULL, + codecs_[n].plname)); + } + } + // Get codec and compare. + for (size_t n = 0; n < codecs_.size(); ++n) { + CodecInst my_codec; + if (n & 0x1) { + // Codecs with odd index should match the reference. + EXPECT_EQ(0, receiver_->DecoderByPayloadType(codecs_[n].pltype, + &my_codec)); + EXPECT_TRUE(CodecsEqual(codecs_[n], my_codec)); + } else { + // Codecs with even index are not registered. + EXPECT_EQ(-1, receiver_->DecoderByPayloadType(codecs_[n].pltype, + &my_codec)); + } + } +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_AddCodecChangePayloadType DISABLED_AddCodecChangePayloadType +#else +#define MAYBE_AddCodecChangePayloadType AddCodecChangePayloadType +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_AddCodecChangePayloadType) { + const CodecIdInst codec1(RentACodec::CodecId::kPCMA); + CodecInst codec2 = codec1.inst; + ++codec2.pltype; + CodecInst test_codec; + + // Register the same codec with different payloads. + EXPECT_EQ(0, receiver_->AddCodec(codec1.id, codec1.inst.pltype, + codec1.inst.channels, codec1.inst.plfreq, + nullptr, codec1.inst.plname)); + EXPECT_EQ(0, receiver_->AddCodec(codec1.id, codec2.pltype, codec2.channels, + codec2.plfreq, NULL, codec2.plname)); + + // Both payload types should exist. + EXPECT_EQ(0, + receiver_->DecoderByPayloadType(codec1.inst.pltype, &test_codec)); + EXPECT_EQ(true, CodecsEqual(codec1.inst, test_codec)); + EXPECT_EQ(0, receiver_->DecoderByPayloadType(codec2.pltype, &test_codec)); + EXPECT_EQ(true, CodecsEqual(codec2, test_codec)); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_AddCodecChangeCodecId DISABLED_AddCodecChangeCodecId +#else +#define MAYBE_AddCodecChangeCodecId AddCodecChangeCodecId +#endif +TEST_F(AcmReceiverTestOldApi, AddCodecChangeCodecId) { + const CodecIdInst codec1(RentACodec::CodecId::kPCMU); + CodecIdInst codec2(RentACodec::CodecId::kPCMA); + codec2.inst.pltype = codec1.inst.pltype; + CodecInst test_codec; + + // Register the same payload type with different codec ID. + EXPECT_EQ(0, receiver_->AddCodec(codec1.id, codec1.inst.pltype, + codec1.inst.channels, codec1.inst.plfreq, + nullptr, codec1.inst.plname)); + EXPECT_EQ(0, receiver_->AddCodec(codec2.id, codec2.inst.pltype, + codec2.inst.channels, codec2.inst.plfreq, + nullptr, codec2.inst.plname)); + + // Make sure that the last codec is used. + EXPECT_EQ(0, + receiver_->DecoderByPayloadType(codec2.inst.pltype, &test_codec)); + EXPECT_EQ(true, CodecsEqual(codec2.inst, test_codec)); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_AddCodecRemoveCodec DISABLED_AddCodecRemoveCodec +#else +#define MAYBE_AddCodecRemoveCodec AddCodecRemoveCodec +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_AddCodecRemoveCodec) { + const CodecIdInst codec(RentACodec::CodecId::kPCMA); + const int payload_type = codec.inst.pltype; + EXPECT_EQ( + 0, receiver_->AddCodec(codec.id, codec.inst.pltype, codec.inst.channels, + codec.inst.plfreq, nullptr, codec.inst.plname)); + + // Remove non-existing codec should not fail. ACM1 legacy. + EXPECT_EQ(0, receiver_->RemoveCodec(payload_type + 1)); + + // Remove an existing codec. + EXPECT_EQ(0, receiver_->RemoveCodec(payload_type)); + + // Ask for the removed codec, must fail. + CodecInst ci; + EXPECT_EQ(-1, receiver_->DecoderByPayloadType(payload_type, &ci)); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_SampleRate DISABLED_SampleRate +#else +#define MAYBE_SampleRate SampleRate +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_SampleRate) { + const RentACodec::CodecId kCodecId[] = {RentACodec::CodecId::kISAC, + RentACodec::CodecId::kISACSWB}; + AddSetOfCodecs(kCodecId); + + AudioFrame frame; + const int kOutSampleRateHz = 8000; // Different than codec sample rate. + for (const auto codec_id : kCodecId) { + const CodecIdInst codec(codec_id); + const int num_10ms_frames = codec.inst.pacsize / (codec.inst.plfreq / 100); + InsertOnePacketOfSilence(codec.id); + for (int k = 0; k < num_10ms_frames; ++k) { + bool muted; + EXPECT_EQ(0, receiver_->GetAudio(kOutSampleRateHz, &frame, &muted)); + } + EXPECT_EQ(codec.inst.plfreq, receiver_->last_output_sample_rate_hz()); + } +} + +class AcmReceiverTestFaxModeOldApi : public AcmReceiverTestOldApi { + protected: + AcmReceiverTestFaxModeOldApi() { + config_.neteq_config.playout_mode = kPlayoutFax; + } + + void RunVerifyAudioFrame(RentACodec::CodecId codec_id) { + // Make sure "fax mode" is enabled. This will avoid delay changes unless the + // packet-loss concealment is made. We do this in order to make the + // timestamp increments predictable; in normal mode, NetEq may decide to do + // accelerate or pre-emptive expand operations after some time, offsetting + // the timestamp. + EXPECT_EQ(kPlayoutFax, config_.neteq_config.playout_mode); + + const RentACodec::CodecId kCodecId[] = {codec_id}; + AddSetOfCodecs(kCodecId); + + const CodecIdInst codec(codec_id); + const int output_sample_rate_hz = codec.inst.plfreq; + const size_t output_channels = codec.inst.channels; + const size_t samples_per_ms = rtc::checked_cast( + rtc::CheckedDivExact(output_sample_rate_hz, 1000)); + const int num_10ms_frames = rtc::CheckedDivExact( + codec.inst.pacsize, rtc::checked_cast(10 * samples_per_ms)); + const AudioFrame::VADActivity expected_vad_activity = + output_sample_rate_hz > 16000 ? AudioFrame::kVadActive + : AudioFrame::kVadPassive; + + // Expect the first output timestamp to be 5*fs/8000 samples before the + // first inserted timestamp (because of NetEq's look-ahead). (This value is + // defined in Expand::overlap_length_.) + uint32_t expected_output_ts = last_packet_send_timestamp_ - + rtc::CheckedDivExact(5 * output_sample_rate_hz, 8000); + + AudioFrame frame; + bool muted; + EXPECT_EQ(0, receiver_->GetAudio(output_sample_rate_hz, &frame, &muted)); + // Expect timestamp = 0 before first packet is inserted. + EXPECT_EQ(0u, frame.timestamp_); + for (int i = 0; i < 5; ++i) { + InsertOnePacketOfSilence(codec.id); + for (int k = 0; k < num_10ms_frames; ++k) { + EXPECT_EQ(0, + receiver_->GetAudio(output_sample_rate_hz, &frame, &muted)); + EXPECT_EQ(expected_output_ts, frame.timestamp_); + expected_output_ts += rtc::checked_cast(10 * samples_per_ms); + EXPECT_EQ(10 * samples_per_ms, frame.samples_per_channel_); + EXPECT_EQ(output_sample_rate_hz, frame.sample_rate_hz_); + EXPECT_EQ(output_channels, frame.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, frame.speech_type_); + EXPECT_EQ(expected_vad_activity, frame.vad_activity_); + EXPECT_FALSE(muted); + } + } + } +}; + +#if defined(WEBRTC_ANDROID) +#define MAYBE_VerifyAudioFramePCMU DISABLED_VerifyAudioFramePCMU +#else +#define MAYBE_VerifyAudioFramePCMU VerifyAudioFramePCMU +#endif +TEST_F(AcmReceiverTestFaxModeOldApi, MAYBE_VerifyAudioFramePCMU) { + RunVerifyAudioFrame(RentACodec::CodecId::kPCMU); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_VerifyAudioFrameISAC DISABLED_VerifyAudioFrameISAC +#else +#define MAYBE_VerifyAudioFrameISAC VerifyAudioFrameISAC +#endif +TEST_F(AcmReceiverTestFaxModeOldApi, MAYBE_VerifyAudioFrameISAC) { + RunVerifyAudioFrame(RentACodec::CodecId::kISAC); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_VerifyAudioFrameOpus DISABLED_VerifyAudioFrameOpus +#else +#define MAYBE_VerifyAudioFrameOpus VerifyAudioFrameOpus +#endif +TEST_F(AcmReceiverTestFaxModeOldApi, MAYBE_VerifyAudioFrameOpus) { + RunVerifyAudioFrame(RentACodec::CodecId::kOpus); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_PostdecodingVad DISABLED_PostdecodingVad +#else +#define MAYBE_PostdecodingVad PostdecodingVad +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_PostdecodingVad) { + EXPECT_TRUE(config_.neteq_config.enable_post_decode_vad); + const CodecIdInst codec(RentACodec::CodecId::kPCM16Bwb); + ASSERT_EQ( + 0, receiver_->AddCodec(codec.id, codec.inst.pltype, codec.inst.channels, + codec.inst.plfreq, nullptr, "")); + const int kNumPackets = 5; + const int num_10ms_frames = codec.inst.pacsize / (codec.inst.plfreq / 100); + AudioFrame frame; + for (int n = 0; n < kNumPackets; ++n) { + InsertOnePacketOfSilence(codec.id); + for (int k = 0; k < num_10ms_frames; ++k) { + bool muted; + ASSERT_EQ(0, receiver_->GetAudio(codec.inst.plfreq, &frame, &muted)); + } + } + EXPECT_EQ(AudioFrame::kVadPassive, frame.vad_activity_); +} + +class AcmReceiverTestPostDecodeVadPassiveOldApi : public AcmReceiverTestOldApi { + protected: + AcmReceiverTestPostDecodeVadPassiveOldApi() { + config_.neteq_config.enable_post_decode_vad = false; + } +}; + +#if defined(WEBRTC_ANDROID) +#define MAYBE_PostdecodingVad DISABLED_PostdecodingVad +#else +#define MAYBE_PostdecodingVad PostdecodingVad +#endif +TEST_F(AcmReceiverTestPostDecodeVadPassiveOldApi, MAYBE_PostdecodingVad) { + EXPECT_FALSE(config_.neteq_config.enable_post_decode_vad); + const CodecIdInst codec(RentACodec::CodecId::kPCM16Bwb); + ASSERT_EQ( + 0, receiver_->AddCodec(codec.id, codec.inst.pltype, codec.inst.channels, + codec.inst.plfreq, nullptr, "")); + const int kNumPackets = 5; + const int num_10ms_frames = codec.inst.pacsize / (codec.inst.plfreq / 100); + AudioFrame frame; + for (int n = 0; n < kNumPackets; ++n) { + InsertOnePacketOfSilence(codec.id); + for (int k = 0; k < num_10ms_frames; ++k) { + bool muted; + ASSERT_EQ(0, receiver_->GetAudio(codec.inst.plfreq, &frame, &muted)); + } + } + EXPECT_EQ(AudioFrame::kVadUnknown, frame.vad_activity_); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_LastAudioCodec DISABLED_LastAudioCodec +#else +#define MAYBE_LastAudioCodec LastAudioCodec +#endif +#if defined(WEBRTC_CODEC_ISAC) +TEST_F(AcmReceiverTestOldApi, MAYBE_LastAudioCodec) { + const RentACodec::CodecId kCodecId[] = { + RentACodec::CodecId::kISAC, RentACodec::CodecId::kPCMA, + RentACodec::CodecId::kISACSWB, RentACodec::CodecId::kPCM16Bswb32kHz}; + AddSetOfCodecs(kCodecId); + + const RentACodec::CodecId kCngId[] = { + // Not including full-band. + RentACodec::CodecId::kCNNB, RentACodec::CodecId::kCNWB, + RentACodec::CodecId::kCNSWB}; + AddSetOfCodecs(kCngId); + + // Register CNG at sender side. + for (auto id : kCngId) + ASSERT_EQ(0, acm_->RegisterSendCodec(CodecIdInst(id).inst)); + + CodecInst codec; + // No audio payload is received. + EXPECT_EQ(-1, receiver_->LastAudioCodec(&codec)); + + // Start with sending DTX. + ASSERT_EQ(0, acm_->SetVAD(true, true, VADVeryAggr)); + packet_sent_ = false; + InsertOnePacketOfSilence(CodecIdInst(kCodecId[0]).id); // Enough to test + // with one codec. + ASSERT_TRUE(packet_sent_); + EXPECT_EQ(kAudioFrameCN, last_frame_type_); + + // Has received, only, DTX. Last Audio codec is undefined. + EXPECT_EQ(-1, receiver_->LastAudioCodec(&codec)); + EXPECT_FALSE(receiver_->last_packet_sample_rate_hz()); + + for (auto id : kCodecId) { + const CodecIdInst c(id); + + // Set DTX off to send audio payload. + acm_->SetVAD(false, false, VADAggr); + packet_sent_ = false; + InsertOnePacketOfSilence(c.id); + + // Sanity check if Actually an audio payload received, and it should be + // of type "speech." + ASSERT_TRUE(packet_sent_); + ASSERT_EQ(kAudioFrameSpeech, last_frame_type_); + EXPECT_EQ(c.inst.plfreq, receiver_->last_packet_sample_rate_hz()); + + // Set VAD on to send DTX. Then check if the "Last Audio codec" returns + // the expected codec. + acm_->SetVAD(true, true, VADAggr); + + // Do as many encoding until a DTX is sent. + while (last_frame_type_ != kAudioFrameCN) { + packet_sent_ = false; + InsertOnePacketOfSilence(c.id); + ASSERT_TRUE(packet_sent_); + } + EXPECT_EQ(c.inst.plfreq, receiver_->last_packet_sample_rate_hz()); + EXPECT_EQ(0, receiver_->LastAudioCodec(&codec)); + EXPECT_TRUE(CodecsEqual(c.inst, codec)); + } +} +#endif + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.cc new file mode 100644 index 0000000000..b97ced23de --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_resampler.h" + +#include +#include + +#include "common_audio/resampler/include/resampler.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace acm2 { + +ACMResampler::ACMResampler() { +} + +ACMResampler::~ACMResampler() { +} + +int ACMResampler::Resample10Msec(const int16_t* in_audio, + int in_freq_hz, + int out_freq_hz, + size_t num_audio_channels, + size_t out_capacity_samples, + int16_t* out_audio) { + size_t in_length = in_freq_hz * num_audio_channels / 100; + if (in_freq_hz == out_freq_hz) { + if (out_capacity_samples < in_length) { + assert(false); + return -1; + } + memcpy(out_audio, in_audio, in_length * sizeof(int16_t)); + return static_cast(in_length / num_audio_channels); + } + + if (resampler_.InitializeIfNeeded(in_freq_hz, out_freq_hz, + num_audio_channels) != 0) { + RTC_LOG(LS_ERROR) << "InitializeIfNeeded(" << in_freq_hz << ", " + << out_freq_hz << ", " << num_audio_channels + << ") failed."; + return -1; + } + + int out_length = + resampler_.Resample(in_audio, in_length, out_audio, out_capacity_samples); + if (out_length == -1) { + RTC_LOG(LS_ERROR) << "Resample(" << in_audio << ", " << in_length << ", " + << out_audio << ", " << out_capacity_samples + << ") failed."; + return -1; + } + + return static_cast(out_length / num_audio_channels); +} + +} // namespace acm2 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.h new file mode 100644 index 0000000000..d7d7bcf165 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_RESAMPLER_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_RESAMPLER_H_ + +#include "common_audio/resampler/include/push_resampler.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace acm2 { + +class ACMResampler { + public: + ACMResampler(); + ~ACMResampler(); + + int Resample10Msec(const int16_t* in_audio, + int in_freq_hz, + int out_freq_hz, + size_t num_audio_channels, + size_t out_capacity_samples, + int16_t* out_audio); + + private: + PushResampler resampler_; +}; + +} // namespace acm2 +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_ACM_RESAMPLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.cc new file mode 100644 index 0000000000..307c9066b1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_send_test.h" + +#include +#include +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +AcmSendTestOldApi::AcmSendTestOldApi(InputAudioFile* audio_source, + int source_rate_hz, + int test_duration_ms) + : clock_(0), + acm_(webrtc::AudioCodingModule::Create(&clock_)), + audio_source_(audio_source), + source_rate_hz_(source_rate_hz), + input_block_size_samples_( + static_cast(source_rate_hz_ * kBlockSizeMs / 1000)), + codec_registered_(false), + test_duration_ms_(test_duration_ms), + frame_type_(kAudioFrameSpeech), + payload_type_(0), + timestamp_(0), + sequence_number_(0) { + input_frame_.sample_rate_hz_ = source_rate_hz_; + input_frame_.num_channels_ = 1; + input_frame_.samples_per_channel_ = input_block_size_samples_; + assert(input_block_size_samples_ * input_frame_.num_channels_ <= + AudioFrame::kMaxDataSizeSamples); + acm_->RegisterTransportCallback(this); +} + +AcmSendTestOldApi::~AcmSendTestOldApi() = default; + +bool AcmSendTestOldApi::RegisterCodec(const char* payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples) { + CodecInst codec; + RTC_CHECK_EQ(0, AudioCodingModule::Codec(payload_name, &codec, + sampling_freq_hz, channels)); + codec.pltype = payload_type; + codec.pacsize = frame_size_samples; + codec_registered_ = (acm_->RegisterSendCodec(codec) == 0); + input_frame_.num_channels_ = channels; + assert(input_block_size_samples_ * input_frame_.num_channels_ <= + AudioFrame::kMaxDataSizeSamples); + return codec_registered_; +} + +bool AcmSendTestOldApi::RegisterExternalCodec( + AudioEncoder* external_speech_encoder) { + acm_->RegisterExternalSendCodec(external_speech_encoder); + input_frame_.num_channels_ = external_speech_encoder->NumChannels(); + assert(input_block_size_samples_ * input_frame_.num_channels_ <= + AudioFrame::kMaxDataSizeSamples); + return codec_registered_ = true; +} + +std::unique_ptr AcmSendTestOldApi::NextPacket() { + assert(codec_registered_); + if (filter_.test(static_cast(payload_type_))) { + // This payload type should be filtered out. Since the payload type is the + // same throughout the whole test run, no packet at all will be delivered. + // We can just as well signal that the test is over by returning NULL. + return nullptr; + } + // Insert audio and process until one packet is produced. + while (clock_.TimeInMilliseconds() < test_duration_ms_) { + clock_.AdvanceTimeMilliseconds(kBlockSizeMs); + RTC_CHECK(audio_source_->Read(input_block_size_samples_, + input_frame_.mutable_data())); + if (input_frame_.num_channels_ > 1) { + InputAudioFile::DuplicateInterleaved(input_frame_.data(), + input_block_size_samples_, + input_frame_.num_channels_, + input_frame_.mutable_data()); + } + data_to_send_ = false; + RTC_CHECK_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += static_cast(input_block_size_samples_); + if (data_to_send_) { + // Encoded packet received. + return CreatePacket(); + } + } + // Test ended. + return nullptr; +} + +// This method receives the callback from ACM when a new packet is produced. +int32_t AcmSendTestOldApi::SendData( + FrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + const RTPFragmentationHeader* fragmentation) { + // Store the packet locally. + frame_type_ = frame_type; + payload_type_ = payload_type; + timestamp_ = timestamp; + last_payload_vec_.assign(payload_data, payload_data + payload_len_bytes); + assert(last_payload_vec_.size() == payload_len_bytes); + data_to_send_ = true; + return 0; +} + +std::unique_ptr AcmSendTestOldApi::CreatePacket() { + const size_t kRtpHeaderSize = 12; + size_t allocated_bytes = last_payload_vec_.size() + kRtpHeaderSize; + uint8_t* packet_memory = new uint8_t[allocated_bytes]; + // Populate the header bytes. + packet_memory[0] = 0x80; + packet_memory[1] = static_cast(payload_type_); + packet_memory[2] = (sequence_number_ >> 8) & 0xFF; + packet_memory[3] = (sequence_number_) & 0xFF; + packet_memory[4] = (timestamp_ >> 24) & 0xFF; + packet_memory[5] = (timestamp_ >> 16) & 0xFF; + packet_memory[6] = (timestamp_ >> 8) & 0xFF; + packet_memory[7] = timestamp_ & 0xFF; + // Set SSRC to 0x12345678. + packet_memory[8] = 0x12; + packet_memory[9] = 0x34; + packet_memory[10] = 0x56; + packet_memory[11] = 0x78; + + ++sequence_number_; + + // Copy the payload data. + memcpy(packet_memory + kRtpHeaderSize, + &last_payload_vec_[0], + last_payload_vec_.size()); + std::unique_ptr packet( + new Packet(packet_memory, allocated_bytes, clock_.TimeInMilliseconds())); + RTC_DCHECK(packet); + RTC_DCHECK(packet->valid_header()); + return packet; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.h new file mode 100644 index 0000000000..6aea0f1b52 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_send_test.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_SEND_TEST_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_SEND_TEST_H_ + +#include +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "rtc_base/constructormagic.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +class AudioEncoder; + +namespace test { +class InputAudioFile; +class Packet; + +class AcmSendTestOldApi : public AudioPacketizationCallback, + public PacketSource { + public: + AcmSendTestOldApi(InputAudioFile* audio_source, + int source_rate_hz, + int test_duration_ms); + ~AcmSendTestOldApi() override; + + // Registers the send codec. Returns true on success, false otherwise. + bool RegisterCodec(const char* payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples); + + // Registers an external send codec. Returns true on success, false otherwise. + bool RegisterExternalCodec(AudioEncoder* external_speech_encoder); + + // Inherited from PacketSource. + std::unique_ptr NextPacket() override; + + // Inherited from AudioPacketizationCallback. + int32_t SendData(FrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + const RTPFragmentationHeader* fragmentation) override; + + AudioCodingModule* acm() { return acm_.get(); } + + private: + static const int kBlockSizeMs = 10; + + // Creates a Packet object from the last packet produced by ACM (and received + // through the SendData method as a callback). + std::unique_ptr CreatePacket(); + + SimulatedClock clock_; + std::unique_ptr acm_; + InputAudioFile* audio_source_; + int source_rate_hz_; + const size_t input_block_size_samples_; + AudioFrame input_frame_; + bool codec_registered_; + int test_duration_ms_; + // The following member variables are set whenever SendData() is called. + FrameType frame_type_; + int payload_type_; + uint32_t timestamp_; + uint16_t sequence_number_; + std::vector last_payload_vec_; + bool data_to_send_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AcmSendTestOldApi); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_ACM2_ACM_SEND_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module.cc new file mode 100644 index 0000000000..931a443b77 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module.cc @@ -0,0 +1,1348 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/include/audio_coding_module.h" + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/acm2/acm_receiver.h" +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/audio_coding/acm2/codec_manager.h" +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +struct EncoderFactory { + AudioEncoder* external_speech_encoder = nullptr; + acm2::CodecManager codec_manager; + acm2::RentACodec rent_a_codec; +}; + +class AudioCodingModuleImpl final : public AudioCodingModule { + public: + explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config); + ~AudioCodingModuleImpl() override; + + ///////////////////////////////////////// + // Sender + // + + // Can be called multiple times for Codec, CNG, RED. + int RegisterSendCodec(const CodecInst& send_codec) override; + + void RegisterExternalSendCodec( + AudioEncoder* external_speech_encoder) override; + + void ModifyEncoder(rtc::FunctionView*)> + modifier) override; + + void QueryEncoder( + rtc::FunctionView query) override; + + // Get current send codec. + rtc::Optional SendCodec() const override; + + // Get current send frequency. + int SendFrequency() const override; + + // Sets the bitrate to the specified value in bits/sec. In case the codec does + // not support the requested value it will choose an appropriate value + // instead. + void SetBitRate(int bitrate_bps) override; + + // Register a transport callback which will be + // called to deliver the encoded buffers. + int RegisterTransportCallback(AudioPacketizationCallback* transport) override; + + // Add 10 ms of raw (PCM) audio data to the encoder. + int Add10MsData(const AudioFrame& audio_frame) override; + + ///////////////////////////////////////// + // (RED) Redundant Coding + // + + // Configure RED status i.e. on/off. + int SetREDStatus(bool enable_red) override; + + // Get RED status. + bool REDStatus() const override; + + ///////////////////////////////////////// + // (FEC) Forward Error Correction (codec internal) + // + + // Configure FEC status i.e. on/off. + int SetCodecFEC(bool enabled_codec_fec) override; + + // Get FEC status. + bool CodecFEC() const override; + + // Set target packet loss rate + int SetPacketLossRate(int loss_rate) override; + + ///////////////////////////////////////// + // (VAD) Voice Activity Detection + // and + // (CNG) Comfort Noise Generation + // + + int SetVAD(bool enable_dtx = true, + bool enable_vad = false, + ACMVADMode mode = VADNormal) override; + + int VAD(bool* dtx_enabled, + bool* vad_enabled, + ACMVADMode* mode) const override; + + int RegisterVADCallback(ACMVADCallback* vad_callback) override; + + ///////////////////////////////////////// + // Receiver + // + + // Initialize receiver, resets codec database etc. + int InitializeReceiver() override; + + // Get current receive frequency. + int ReceiveFrequency() const override; + + // Get current playout frequency. + int PlayoutFrequency() const override; + + void SetReceiveCodecs(const std::map& codecs) override; + + bool RegisterReceiveCodec(int rtp_payload_type, + const SdpAudioFormat& audio_format) override; + + int RegisterReceiveCodec(const CodecInst& receive_codec) override; + int RegisterReceiveCodec( + const CodecInst& receive_codec, + rtc::FunctionView()> isac_factory) override; + + int RegisterExternalReceiveCodec(int rtp_payload_type, + AudioDecoder* external_decoder, + int sample_rate_hz, + int num_channels, + const std::string& name) override; + + // Get current received codec. + int ReceiveCodec(CodecInst* current_codec) const override; + + rtc::Optional ReceiveFormat() const override; + int ReceiveSampleRate() const override; + + // Incoming packet from network parsed and ready for decode. + int IncomingPacket(const uint8_t* incoming_payload, + const size_t payload_length, + const WebRtcRTPHeader& rtp_info) override; + + // Minimum playout delay. + int SetMinimumPlayoutDelay(int time_ms) override; + + // Maximum playout delay. + int SetMaximumPlayoutDelay(int time_ms) override; + + // Smallest latency NetEq will maintain. + int LeastRequiredDelayMs() const override; + + RTC_DEPRECATED int32_t PlayoutTimestamp(uint32_t* timestamp) override; + + rtc::Optional PlayoutTimestamp() override; + + int FilteredCurrentDelayMs() const override; + + int TargetDelayMs() const override; + + // Get 10 milliseconds of raw audio data to play out, and + // automatic resample to the requested frequency if > 0. + int PlayoutData10Ms(int desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) override; + int PlayoutData10Ms(int desired_freq_hz, AudioFrame* audio_frame) override; + + ///////////////////////////////////////// + // Statistics + // + + int GetNetworkStatistics(NetworkStatistics* statistics) override; + + int SetOpusApplication(OpusApplicationMode application) override; + + // If current send codec is Opus, informs it about the maximum playback rate + // the receiver will render. + int SetOpusMaxPlaybackRate(int frequency_hz) override; + + int EnableOpusDtx() override; + + int DisableOpusDtx() override; + + int UnregisterReceiveCodec(uint8_t payload_type) override; + + int EnableNack(size_t max_nack_list_size) override; + + void DisableNack() override; + + std::vector GetNackList(int64_t round_trip_time_ms) const override; + + void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override; + + ANAStats GetANAStats() const override; + + private: + struct InputData { + uint32_t input_timestamp; + const int16_t* audio; + size_t length_per_channel; + size_t audio_channel; + // If a re-mix is required (up or down), this buffer will store a re-mixed + // version of the input. + int16_t buffer[WEBRTC_10MS_PCM_AUDIO]; + }; + + // This member class writes values to the named UMA histogram, but only if + // the value has changed since the last time (and always for the first call). + class ChangeLogger { + public: + explicit ChangeLogger(const std::string& histogram_name) + : histogram_name_(histogram_name) {} + // Logs the new value if it is different from the last logged value, or if + // this is the first call. + void MaybeLog(int value); + + private: + int last_value_ = 0; + int first_time_ = true; + const std::string histogram_name_; + }; + + int RegisterReceiveCodecUnlocked( + const CodecInst& codec, + rtc::FunctionView()> isac_factory) + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + + int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data) + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + int Encode(const InputData& input_data) + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + + int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + + bool HaveValidEncoder(const char* caller_name) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + + // Preprocessing of input audio, including resampling and down-mixing if + // required, before pushing audio into encoder's buffer. + // + // in_frame: input audio-frame + // ptr_out: pointer to output audio_frame. If no preprocessing is required + // |ptr_out| will be pointing to |in_frame|, otherwise pointing to + // |preprocess_frame_|. + // + // Return value: + // -1: if encountering an error. + // 0: otherwise. + int PreprocessToAddData(const AudioFrame& in_frame, + const AudioFrame** ptr_out) + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + + // Change required states after starting to receive the codec corresponding + // to |index|. + int UpdateUponReceivingCodec(int index); + + rtc::CriticalSection acm_crit_sect_; + rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_); + uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_); + uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_); + acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_); + acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock. + ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_); + + std::unique_ptr encoder_factory_ + RTC_GUARDED_BY(acm_crit_sect_); + + // Current encoder stack, either obtained from + // encoder_factory_->rent_a_codec.RentEncoderStack or provided by a call to + // RegisterEncoder. + std::unique_ptr encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_); + + std::unique_ptr isac_decoder_16k_ + RTC_GUARDED_BY(acm_crit_sect_); + std::unique_ptr isac_decoder_32k_ + RTC_GUARDED_BY(acm_crit_sect_); + + // This is to keep track of CN instances where we can send DTMFs. + uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_); + + bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_); + + AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_); + bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_); + + bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_); + uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_); + uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_); + + rtc::CriticalSection callback_crit_sect_; + AudioPacketizationCallback* packetization_callback_ + RTC_GUARDED_BY(callback_crit_sect_); + ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_); + + int codec_histogram_bins_log_[static_cast( + AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)]; + int number_of_consecutive_empty_packets_; +}; + +// Adds a codec usage sample to the histogram. +void UpdateCodecTypeHistogram(size_t codec_type) { + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.Encoder.CodecType", static_cast(codec_type), + static_cast( + webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)); +} + +// Stereo-to-mono can be used as in-place. +int DownMix(const AudioFrame& frame, + size_t length_out_buff, + int16_t* out_buff) { + RTC_DCHECK_EQ(frame.num_channels_, 2); + RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_); + + if (!frame.muted()) { + const int16_t* frame_data = frame.data(); + for (size_t n = 0; n < frame.samples_per_channel_; ++n) { + out_buff[n] = static_cast( + (static_cast(frame_data[2 * n]) + + static_cast(frame_data[2 * n + 1])) >> 1); + } + } else { + std::fill(out_buff, out_buff + frame.samples_per_channel_, 0); + } + return 0; +} + +// Mono-to-stereo can be used as in-place. +int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) { + RTC_DCHECK_EQ(frame.num_channels_, 1); + RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_); + + if (!frame.muted()) { + const int16_t* frame_data = frame.data(); + for (size_t n = frame.samples_per_channel_; n != 0; --n) { + size_t i = n - 1; + int16_t sample = frame_data[i]; + out_buff[2 * i + 1] = sample; + out_buff[2 * i] = sample; + } + } else { + std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0); + } + return 0; +} + +void ConvertEncodedInfoToFragmentationHeader( + const AudioEncoder::EncodedInfo& info, + RTPFragmentationHeader* frag) { + if (info.redundant.empty()) { + frag->fragmentationVectorSize = 0; + return; + } + + frag->VerifyAndAllocateFragmentationHeader( + static_cast(info.redundant.size())); + frag->fragmentationVectorSize = static_cast(info.redundant.size()); + size_t offset = 0; + for (size_t i = 0; i < info.redundant.size(); ++i) { + frag->fragmentationOffset[i] = offset; + offset += info.redundant[i].encoded_bytes; + frag->fragmentationLength[i] = info.redundant[i].encoded_bytes; + frag->fragmentationTimeDiff[i] = rtc::dchecked_cast( + info.encoded_timestamp - info.redundant[i].encoded_timestamp); + frag->fragmentationPlType[i] = info.redundant[i].payload_type; + } +} + +// Wraps a raw AudioEncoder pointer. The idea is that you can put one of these +// in a unique_ptr, to protect the contained raw pointer from being deleted +// when the unique_ptr expires. (This is of course a bad idea in general, but +// backwards compatibility.) +class RawAudioEncoderWrapper final : public AudioEncoder { + public: + RawAudioEncoderWrapper(AudioEncoder* enc) : enc_(enc) {} + int SampleRateHz() const override { return enc_->SampleRateHz(); } + size_t NumChannels() const override { return enc_->NumChannels(); } + int RtpTimestampRateHz() const override { return enc_->RtpTimestampRateHz(); } + size_t Num10MsFramesInNextPacket() const override { + return enc_->Num10MsFramesInNextPacket(); + } + size_t Max10MsFramesInAPacket() const override { + return enc_->Max10MsFramesInAPacket(); + } + int GetTargetBitrate() const override { return enc_->GetTargetBitrate(); } + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override { + return enc_->Encode(rtp_timestamp, audio, encoded); + } + void Reset() override { return enc_->Reset(); } + bool SetFec(bool enable) override { return enc_->SetFec(enable); } + bool SetDtx(bool enable) override { return enc_->SetDtx(enable); } + bool SetApplication(Application application) override { + return enc_->SetApplication(application); + } + void SetMaxPlaybackRate(int frequency_hz) override { + return enc_->SetMaxPlaybackRate(frequency_hz); + } + + private: + AudioEncoder* enc_; +}; + +// Return false on error. +bool CreateSpeechEncoderIfNecessary(EncoderFactory* ef) { + auto* sp = ef->codec_manager.GetStackParams(); + if (sp->speech_encoder) { + // Do nothing; we already have a speech encoder. + } else if (ef->codec_manager.GetCodecInst()) { + RTC_DCHECK(!ef->external_speech_encoder); + // We have no speech encoder, but we have a specification for making one. + std::unique_ptr enc = + ef->rent_a_codec.RentEncoder(*ef->codec_manager.GetCodecInst()); + if (!enc) + return false; // Encoder spec was bad. + sp->speech_encoder = std::move(enc); + } else if (ef->external_speech_encoder) { + RTC_DCHECK(!ef->codec_manager.GetCodecInst()); + // We have an external speech encoder. + sp->speech_encoder = std::unique_ptr( + new RawAudioEncoderWrapper(ef->external_speech_encoder)); + } + return true; +} + +void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) { + if (value != last_value_ || first_time_) { + first_time_ = false; + last_value_ = value; + RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value); + } +} + +AudioCodingModuleImpl::AudioCodingModuleImpl( + const AudioCodingModule::Config& config) + : expected_codec_ts_(0xD87F3F9F), + expected_in_ts_(0xD87F3F9F), + receiver_(config), + bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"), + encoder_factory_(new EncoderFactory), + encoder_stack_(nullptr), + previous_pltype_(255), + receiver_initialized_(false), + first_10ms_data_(false), + first_frame_(true), + packetization_callback_(NULL), + vad_callback_(NULL), + codec_histogram_bins_log_(), + number_of_consecutive_empty_packets_(0) { + if (InitializeReceiverSafe() < 0) { + RTC_LOG(LS_ERROR) << "Cannot initialize receiver"; + } + RTC_LOG(LS_INFO) << "Created"; +} + +AudioCodingModuleImpl::~AudioCodingModuleImpl() = default; + +int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) { + AudioEncoder::EncodedInfo encoded_info; + uint8_t previous_pltype; + + // Check if there is an encoder before. + if (!HaveValidEncoder("Process")) + return -1; + + if(!first_frame_) { + RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_)) + << "Time should not move backwards"; + } + + // Scale the timestamp to the codec's RTP timestamp rate. + uint32_t rtp_timestamp = + first_frame_ ? input_data.input_timestamp + : last_rtp_timestamp_ + + rtc::CheckedDivExact( + input_data.input_timestamp - last_timestamp_, + static_cast(rtc::CheckedDivExact( + encoder_stack_->SampleRateHz(), + encoder_stack_->RtpTimestampRateHz()))); + last_timestamp_ = input_data.input_timestamp; + last_rtp_timestamp_ = rtp_timestamp; + first_frame_ = false; + + // Clear the buffer before reuse - encoded data will get appended. + encode_buffer_.Clear(); + encoded_info = encoder_stack_->Encode( + rtp_timestamp, rtc::ArrayView( + input_data.audio, input_data.audio_channel * + input_data.length_per_channel), + &encode_buffer_); + + bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000); + if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) { + // Not enough data. + return 0; + } + previous_pltype = previous_pltype_; // Read it while we have the critsect. + + // Log codec type to histogram once every 500 packets. + if (encoded_info.encoded_bytes == 0) { + ++number_of_consecutive_empty_packets_; + } else { + size_t codec_type = static_cast(encoded_info.encoder_type); + codec_histogram_bins_log_[codec_type] += + number_of_consecutive_empty_packets_ + 1; + number_of_consecutive_empty_packets_ = 0; + if (codec_histogram_bins_log_[codec_type] >= 500) { + codec_histogram_bins_log_[codec_type] -= 500; + UpdateCodecTypeHistogram(codec_type); + } + } + + RTPFragmentationHeader my_fragmentation; + ConvertEncodedInfoToFragmentationHeader(encoded_info, &my_fragmentation); + FrameType frame_type; + if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) { + frame_type = kEmptyFrame; + encoded_info.payload_type = previous_pltype; + } else { + RTC_DCHECK_GT(encode_buffer_.size(), 0); + frame_type = encoded_info.speech ? kAudioFrameSpeech : kAudioFrameCN; + } + + { + rtc::CritScope lock(&callback_crit_sect_); + if (packetization_callback_) { + packetization_callback_->SendData( + frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp, + encode_buffer_.data(), encode_buffer_.size(), + my_fragmentation.fragmentationVectorSize > 0 ? &my_fragmentation + : nullptr); + } + + if (vad_callback_) { + // Callback with VAD decision. + vad_callback_->InFrameType(frame_type); + } + } + previous_pltype_ = encoded_info.payload_type; + return static_cast(encode_buffer_.size()); +} + +///////////////////////////////////////// +// Sender +// + +// Can be called multiple times for Codec, CNG, RED. +int AudioCodingModuleImpl::RegisterSendCodec(const CodecInst& send_codec) { + rtc::CritScope lock(&acm_crit_sect_); + if (!encoder_factory_->codec_manager.RegisterEncoder(send_codec)) { + return -1; + } + if (encoder_factory_->codec_manager.GetCodecInst()) { + encoder_factory_->external_speech_encoder = nullptr; + } + if (!CreateSpeechEncoderIfNecessary(encoder_factory_.get())) { + return -1; + } + auto* sp = encoder_factory_->codec_manager.GetStackParams(); + if (sp->speech_encoder) + encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp); + return 0; +} + +void AudioCodingModuleImpl::RegisterExternalSendCodec( + AudioEncoder* external_speech_encoder) { + rtc::CritScope lock(&acm_crit_sect_); + encoder_factory_->codec_manager.UnsetCodecInst(); + encoder_factory_->external_speech_encoder = external_speech_encoder; + RTC_CHECK(CreateSpeechEncoderIfNecessary(encoder_factory_.get())); + auto* sp = encoder_factory_->codec_manager.GetStackParams(); + RTC_CHECK(sp->speech_encoder); + encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp); +} + +void AudioCodingModuleImpl::ModifyEncoder( + rtc::FunctionView*)> modifier) { + rtc::CritScope lock(&acm_crit_sect_); + + // Wipe the encoder factory, so that everything that relies on it will fail. + // We don't want the complexity of supporting swapping back and forth. + if (encoder_factory_) { + encoder_factory_.reset(); + RTC_CHECK(!encoder_stack_); // Ensure we hadn't started using the factory. + } + + modifier(&encoder_stack_); +} + +void AudioCodingModuleImpl::QueryEncoder( + rtc::FunctionView query) { + rtc::CritScope lock(&acm_crit_sect_); + query(encoder_stack_.get()); +} + +// Get current send codec. +rtc::Optional AudioCodingModuleImpl::SendCodec() const { + rtc::CritScope lock(&acm_crit_sect_); + if (encoder_factory_) { + auto* ci = encoder_factory_->codec_manager.GetCodecInst(); + if (ci) { + return *ci; + } + CreateSpeechEncoderIfNecessary(encoder_factory_.get()); + const std::unique_ptr& enc = + encoder_factory_->codec_manager.GetStackParams()->speech_encoder; + if (enc) { + return acm2::CodecManager::ForgeCodecInst(enc.get()); + } + return rtc::nullopt; + } else { + return encoder_stack_ + ? rtc::Optional( + acm2::CodecManager::ForgeCodecInst(encoder_stack_.get())) + : rtc::nullopt; + } +} + +// Get current send frequency. +int AudioCodingModuleImpl::SendFrequency() const { + rtc::CritScope lock(&acm_crit_sect_); + + if (!encoder_stack_) { + RTC_LOG(LS_ERROR) << "SendFrequency Failed, no codec is registered"; + return -1; + } + + return encoder_stack_->SampleRateHz(); +} + +void AudioCodingModuleImpl::SetBitRate(int bitrate_bps) { + rtc::CritScope lock(&acm_crit_sect_); + if (encoder_stack_) { + encoder_stack_->OnReceivedUplinkBandwidth(bitrate_bps, rtc::nullopt); + } +} + +// Register a transport callback which will be called to deliver +// the encoded buffers. +int AudioCodingModuleImpl::RegisterTransportCallback( + AudioPacketizationCallback* transport) { + rtc::CritScope lock(&callback_crit_sect_); + packetization_callback_ = transport; + return 0; +} + +// Add 10MS of raw (PCM) audio data to the encoder. +int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) { + InputData input_data; + rtc::CritScope lock(&acm_crit_sect_); + int r = Add10MsDataInternal(audio_frame, &input_data); + return r < 0 ? r : Encode(input_data); +} + +int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame, + InputData* input_data) { + if (audio_frame.samples_per_channel_ == 0) { + assert(false); + RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero"; + return -1; + } + + if (audio_frame.sample_rate_hz_ > 48000) { + assert(false); + RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid"; + return -1; + } + + // If the length and frequency matches. We currently just support raw PCM. + if (static_cast(audio_frame.sample_rate_hz_ / 100) != + audio_frame.samples_per_channel_) { + RTC_LOG(LS_ERROR) + << "Cannot Add 10 ms audio, input frequency and length doesn't match"; + return -1; + } + + if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2) { + RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels."; + return -1; + } + + // Do we have a codec registered? + if (!HaveValidEncoder("Add10MsData")) { + return -1; + } + + const AudioFrame* ptr_frame; + // Perform a resampling, also down-mix if it is required and can be + // performed before resampling (a down mix prior to resampling will take + // place if both primary and secondary encoders are mono and input is in + // stereo). + if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) { + return -1; + } + + // Check whether we need an up-mix or down-mix? + const size_t current_num_channels = encoder_stack_->NumChannels(); + const bool same_num_channels = + ptr_frame->num_channels_ == current_num_channels; + + if (!same_num_channels) { + if (ptr_frame->num_channels_ == 1) { + if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0) + return -1; + } else { + if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0) + return -1; + } + } + + // When adding data to encoders this pointer is pointing to an audio buffer + // with correct number of channels. + const int16_t* ptr_audio = ptr_frame->data(); + + // For pushing data to primary, point the |ptr_audio| to correct buffer. + if (!same_num_channels) + ptr_audio = input_data->buffer; + + // TODO(yujo): Skip encode of muted frames. + input_data->input_timestamp = ptr_frame->timestamp_; + input_data->audio = ptr_audio; + input_data->length_per_channel = ptr_frame->samples_per_channel_; + input_data->audio_channel = current_num_channels; + + return 0; +} + +// Perform a resampling and down-mix if required. We down-mix only if +// encoder is mono and input is stereo. In case of dual-streaming, both +// encoders has to be mono for down-mix to take place. +// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing +// is required, |*ptr_out| points to |in_frame|. +// TODO(yujo): Make this more efficient for muted frames. +int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame, + const AudioFrame** ptr_out) { + const bool resample = + in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz(); + + // This variable is true if primary codec and secondary codec (if exists) + // are both mono and input is stereo. + // TODO(henrik.lundin): This condition should probably be + // in_frame.num_channels_ > encoder_stack_->NumChannels() + const bool down_mix = + in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1; + + if (!first_10ms_data_) { + expected_in_ts_ = in_frame.timestamp_; + expected_codec_ts_ = in_frame.timestamp_; + first_10ms_data_ = true; + } else if (in_frame.timestamp_ != expected_in_ts_) { + RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_ + << ", expected: " << expected_in_ts_; + expected_codec_ts_ += + (in_frame.timestamp_ - expected_in_ts_) * + static_cast( + static_cast(encoder_stack_->SampleRateHz()) / + static_cast(in_frame.sample_rate_hz_)); + expected_in_ts_ = in_frame.timestamp_; + } + + + if (!down_mix && !resample) { + // No pre-processing is required. + if (expected_in_ts_ == expected_codec_ts_) { + // If we've never resampled, we can use the input frame as-is + *ptr_out = &in_frame; + } else { + // Otherwise we'll need to alter the timestamp. Since in_frame is const, + // we'll have to make a copy of it. + preprocess_frame_.CopyFrom(in_frame); + preprocess_frame_.timestamp_ = expected_codec_ts_; + *ptr_out = &preprocess_frame_; + } + + expected_in_ts_ += static_cast(in_frame.samples_per_channel_); + expected_codec_ts_ += static_cast(in_frame.samples_per_channel_); + return 0; + } + + *ptr_out = &preprocess_frame_; + preprocess_frame_.num_channels_ = in_frame.num_channels_; + int16_t audio[WEBRTC_10MS_PCM_AUDIO]; + const int16_t* src_ptr_audio = in_frame.data(); + if (down_mix) { + // If a resampling is required the output of a down-mix is written into a + // local buffer, otherwise, it will be written to the output frame. + int16_t* dest_ptr_audio = resample ? + audio : preprocess_frame_.mutable_data(); + if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0) + return -1; + preprocess_frame_.num_channels_ = 1; + // Set the input of the resampler is the down-mixed signal. + src_ptr_audio = audio; + } + + preprocess_frame_.timestamp_ = expected_codec_ts_; + preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_; + preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_; + // If it is required, we have to do a resampling. + if (resample) { + // The result of the resampler is written to output frame. + int16_t* dest_ptr_audio = preprocess_frame_.mutable_data(); + + int samples_per_channel = resampler_.Resample10Msec( + src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(), + preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples, + dest_ptr_audio); + + if (samples_per_channel < 0) { + RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed"; + return -1; + } + preprocess_frame_.samples_per_channel_ = + static_cast(samples_per_channel); + preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz(); + } + + expected_codec_ts_ += + static_cast(preprocess_frame_.samples_per_channel_); + expected_in_ts_ += static_cast(in_frame.samples_per_channel_); + + return 0; +} + +///////////////////////////////////////// +// (RED) Redundant Coding +// + +bool AudioCodingModuleImpl::REDStatus() const { + rtc::CritScope lock(&acm_crit_sect_); + return encoder_factory_->codec_manager.GetStackParams()->use_red; +} + +// Configure RED status i.e on/off. +int AudioCodingModuleImpl::SetREDStatus(bool enable_red) { +#ifdef WEBRTC_CODEC_RED + rtc::CritScope lock(&acm_crit_sect_); + CreateSpeechEncoderIfNecessary(encoder_factory_.get()); + if (!encoder_factory_->codec_manager.SetCopyRed(enable_red)) { + return -1; + } + auto* sp = encoder_factory_->codec_manager.GetStackParams(); + if (sp->speech_encoder) + encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp); + return 0; +#else + RTC_LOG(LS_WARNING) << " WEBRTC_CODEC_RED is undefined"; + return -1; +#endif +} + +///////////////////////////////////////// +// (FEC) Forward Error Correction (codec internal) +// + +bool AudioCodingModuleImpl::CodecFEC() const { + rtc::CritScope lock(&acm_crit_sect_); + return encoder_factory_->codec_manager.GetStackParams()->use_codec_fec; +} + +int AudioCodingModuleImpl::SetCodecFEC(bool enable_codec_fec) { + rtc::CritScope lock(&acm_crit_sect_); + CreateSpeechEncoderIfNecessary(encoder_factory_.get()); + if (!encoder_factory_->codec_manager.SetCodecFEC(enable_codec_fec)) { + return -1; + } + auto* sp = encoder_factory_->codec_manager.GetStackParams(); + if (sp->speech_encoder) + encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp); + if (enable_codec_fec) { + return sp->use_codec_fec ? 0 : -1; + } else { + RTC_DCHECK(!sp->use_codec_fec); + return 0; + } +} + +int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) { + rtc::CritScope lock(&acm_crit_sect_); + if (HaveValidEncoder("SetPacketLossRate")) { + encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0); + } + return 0; +} + +///////////////////////////////////////// +// (VAD) Voice Activity Detection +// +int AudioCodingModuleImpl::SetVAD(bool enable_dtx, + bool enable_vad, + ACMVADMode mode) { + // Note: |enable_vad| is not used; VAD is enabled based on the DTX setting. + RTC_DCHECK_EQ(enable_dtx, enable_vad); + rtc::CritScope lock(&acm_crit_sect_); + CreateSpeechEncoderIfNecessary(encoder_factory_.get()); + if (!encoder_factory_->codec_manager.SetVAD(enable_dtx, mode)) { + return -1; + } + auto* sp = encoder_factory_->codec_manager.GetStackParams(); + if (sp->speech_encoder) + encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp); + return 0; +} + +// Get VAD/DTX settings. +int AudioCodingModuleImpl::VAD(bool* dtx_enabled, bool* vad_enabled, + ACMVADMode* mode) const { + rtc::CritScope lock(&acm_crit_sect_); + const auto* sp = encoder_factory_->codec_manager.GetStackParams(); + *dtx_enabled = *vad_enabled = sp->use_cng; + *mode = sp->vad_mode; + return 0; +} + +///////////////////////////////////////// +// Receiver +// + +int AudioCodingModuleImpl::InitializeReceiver() { + rtc::CritScope lock(&acm_crit_sect_); + return InitializeReceiverSafe(); +} + +// Initialize receiver, resets codec database etc. +int AudioCodingModuleImpl::InitializeReceiverSafe() { + // If the receiver is already initialized then we want to destroy any + // existing decoders. After a call to this function, we should have a clean + // start-up. + if (receiver_initialized_) + receiver_.RemoveAllCodecs(); + receiver_.ResetInitialDelay(); + receiver_.SetMinimumDelay(0); + receiver_.SetMaximumDelay(0); + receiver_.FlushBuffers(); + + receiver_initialized_ = true; + return 0; +} + +// Get current receive frequency. +int AudioCodingModuleImpl::ReceiveFrequency() const { + const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz(); + return last_packet_sample_rate ? *last_packet_sample_rate + : receiver_.last_output_sample_rate_hz(); +} + +// Get current playout frequency. +int AudioCodingModuleImpl::PlayoutFrequency() const { + return receiver_.last_output_sample_rate_hz(); +} + +void AudioCodingModuleImpl::SetReceiveCodecs( + const std::map& codecs) { + rtc::CritScope lock(&acm_crit_sect_); + receiver_.SetCodecs(codecs); +} + +bool AudioCodingModuleImpl::RegisterReceiveCodec( + int rtp_payload_type, + const SdpAudioFormat& audio_format) { + rtc::CritScope lock(&acm_crit_sect_); + RTC_DCHECK(receiver_initialized_); + + if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) { + RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type + << " for decoder."; + return false; + } + + return receiver_.AddCodec(rtp_payload_type, audio_format); +} + +int AudioCodingModuleImpl::RegisterReceiveCodec(const CodecInst& codec) { + rtc::CritScope lock(&acm_crit_sect_); + auto* ef = encoder_factory_.get(); + return RegisterReceiveCodecUnlocked( + codec, [&] { return ef->rent_a_codec.RentIsacDecoder(codec.plfreq); }); +} + +int AudioCodingModuleImpl::RegisterReceiveCodec( + const CodecInst& codec, + rtc::FunctionView()> isac_factory) { + rtc::CritScope lock(&acm_crit_sect_); + return RegisterReceiveCodecUnlocked(codec, isac_factory); +} + +int AudioCodingModuleImpl::RegisterReceiveCodecUnlocked( + const CodecInst& codec, + rtc::FunctionView()> isac_factory) { + RTC_DCHECK(receiver_initialized_); + if (codec.channels > 2) { + RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << codec.channels; + return -1; + } + + auto codec_id = acm2::RentACodec::CodecIdByParams(codec.plname, codec.plfreq, + codec.channels); + if (!codec_id) { + RTC_LOG_F(LS_ERROR) + << "Wrong codec params to be registered as receive codec"; + return -1; + } + auto codec_index = acm2::RentACodec::CodecIndexFromId(*codec_id); + RTC_CHECK(codec_index) << "Invalid codec ID: " << static_cast(*codec_id); + + // Check if the payload-type is valid. + if (!acm2::RentACodec::IsPayloadTypeValid(codec.pltype)) { + RTC_LOG_F(LS_ERROR) << "Invalid payload type " << codec.pltype << " for " + << codec.plname; + return -1; + } + + AudioDecoder* isac_decoder = nullptr; + if (STR_CASE_CMP(codec.plname, "isac") == 0) { + std::unique_ptr& saved_isac_decoder = + codec.plfreq == 16000 ? isac_decoder_16k_ : isac_decoder_32k_; + if (!saved_isac_decoder) { + saved_isac_decoder = isac_factory(); + } + isac_decoder = saved_isac_decoder.get(); + } + return receiver_.AddCodec(*codec_index, codec.pltype, codec.channels, + codec.plfreq, isac_decoder, codec.plname); +} + +int AudioCodingModuleImpl::RegisterExternalReceiveCodec( + int rtp_payload_type, + AudioDecoder* external_decoder, + int sample_rate_hz, + int num_channels, + const std::string& name) { + rtc::CritScope lock(&acm_crit_sect_); + RTC_DCHECK(receiver_initialized_); + if (num_channels > 2 || num_channels < 0) { + RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << num_channels; + return -1; + } + + // Check if the payload-type is valid. + if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) { + RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type + << " for external decoder."; + return -1; + } + + return receiver_.AddCodec(-1 /* external */, rtp_payload_type, num_channels, + sample_rate_hz, external_decoder, name); +} + +// Get current received codec. +int AudioCodingModuleImpl::ReceiveCodec(CodecInst* current_codec) const { + rtc::CritScope lock(&acm_crit_sect_); + return receiver_.LastAudioCodec(current_codec); +} + +rtc::Optional AudioCodingModuleImpl::ReceiveFormat() const { + rtc::CritScope lock(&acm_crit_sect_); + return receiver_.LastAudioFormat(); +} + +int AudioCodingModuleImpl::ReceiveSampleRate() const { + return receiver_.LastAudioSampleRate(); +} + +// Incoming packet from network parsed and ready for decode. +int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload, + const size_t payload_length, + const WebRtcRTPHeader& rtp_header) { + RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr); + return receiver_.InsertPacket( + rtp_header, + rtc::ArrayView(incoming_payload, payload_length)); +} + +// Minimum playout delay (Used for lip-sync). +int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) { + if ((time_ms < 0) || (time_ms > 10000)) { + RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds."; + return -1; + } + return receiver_.SetMinimumDelay(time_ms); +} + +int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) { + if ((time_ms < 0) || (time_ms > 10000)) { + RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds."; + return -1; + } + return receiver_.SetMaximumDelay(time_ms); +} + +// Get 10 milliseconds of raw audio data to play out. +// Automatic resample to the requested frequency. +int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) { + // GetAudio always returns 10 ms, at the requested sample rate. + if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) { + RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed"; + return -1; + } + return 0; +} + +int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz, + AudioFrame* audio_frame) { + bool muted; + int ret = PlayoutData10Ms(desired_freq_hz, audio_frame, &muted); + RTC_DCHECK(!muted); + return ret; +} + +///////////////////////////////////////// +// Statistics +// + +// TODO(turajs) change the return value to void. Also change the corresponding +// NetEq function. +int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) { + receiver_.GetNetworkStatistics(statistics); + return 0; +} + +int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) { + RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()"; + rtc::CritScope lock(&callback_crit_sect_); + vad_callback_ = vad_callback; + return 0; +} + +int AudioCodingModuleImpl::SetOpusApplication(OpusApplicationMode application) { + rtc::CritScope lock(&acm_crit_sect_); + if (!HaveValidEncoder("SetOpusApplication")) { + return -1; + } + AudioEncoder::Application app; + switch (application) { + case kVoip: + app = AudioEncoder::Application::kSpeech; + break; + case kAudio: + app = AudioEncoder::Application::kAudio; + break; + default: + FATAL(); + return 0; + } + return encoder_stack_->SetApplication(app) ? 0 : -1; +} + +// Informs Opus encoder of the maximum playback rate the receiver will render. +int AudioCodingModuleImpl::SetOpusMaxPlaybackRate(int frequency_hz) { + rtc::CritScope lock(&acm_crit_sect_); + if (!HaveValidEncoder("SetOpusMaxPlaybackRate")) { + return -1; + } + encoder_stack_->SetMaxPlaybackRate(frequency_hz); + return 0; +} + +int AudioCodingModuleImpl::EnableOpusDtx() { + rtc::CritScope lock(&acm_crit_sect_); + if (!HaveValidEncoder("EnableOpusDtx")) { + return -1; + } + return encoder_stack_->SetDtx(true) ? 0 : -1; +} + +int AudioCodingModuleImpl::DisableOpusDtx() { + rtc::CritScope lock(&acm_crit_sect_); + if (!HaveValidEncoder("DisableOpusDtx")) { + return -1; + } + return encoder_stack_->SetDtx(false) ? 0 : -1; +} + +int32_t AudioCodingModuleImpl::PlayoutTimestamp(uint32_t* timestamp) { + rtc::Optional ts = PlayoutTimestamp(); + if (!ts) + return -1; + *timestamp = *ts; + return 0; +} + +rtc::Optional AudioCodingModuleImpl::PlayoutTimestamp() { + return receiver_.GetPlayoutTimestamp(); +} + +int AudioCodingModuleImpl::FilteredCurrentDelayMs() const { + return receiver_.FilteredCurrentDelayMs(); +} + +int AudioCodingModuleImpl::TargetDelayMs() const { + return receiver_.TargetDelayMs(); +} + +bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const { + if (!encoder_stack_) { + RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered."; + return false; + } + return true; +} + +int AudioCodingModuleImpl::UnregisterReceiveCodec(uint8_t payload_type) { + return receiver_.RemoveCodec(payload_type); +} + +int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) { + return receiver_.EnableNack(max_nack_list_size); +} + +void AudioCodingModuleImpl::DisableNack() { + receiver_.DisableNack(); +} + +std::vector AudioCodingModuleImpl::GetNackList( + int64_t round_trip_time_ms) const { + return receiver_.GetNackList(round_trip_time_ms); +} + +int AudioCodingModuleImpl::LeastRequiredDelayMs() const { + return receiver_.LeastRequiredDelayMs(); +} + +void AudioCodingModuleImpl::GetDecodingCallStatistics( + AudioDecodingCallStats* call_stats) const { + receiver_.GetDecodingCallStatistics(call_stats); +} + +ANAStats AudioCodingModuleImpl::GetANAStats() const { + rtc::CritScope lock(&acm_crit_sect_); + if (encoder_stack_) + return encoder_stack_->GetANAStats(); + // If no encoder is set, return default stats. + return ANAStats(); +} + +} // namespace + +AudioCodingModule::Config::Config() + : neteq_config(), clock(Clock::GetRealTimeClock()) { + // Post-decode VAD is disabled by default in NetEq, however, Audio + // Conference Mixer relies on VAD decisions and fails without them. + neteq_config.enable_post_decode_vad = true; +} + +AudioCodingModule::Config::Config(const Config&) = default; +AudioCodingModule::Config::~Config() = default; + +AudioCodingModule* AudioCodingModule::Create(int id) { + RTC_UNUSED(id); + return Create(); +} + +// Create module +AudioCodingModule* AudioCodingModule::Create() { + Config config; + config.clock = Clock::GetRealTimeClock(); + config.decoder_factory = CreateBuiltinAudioDecoderFactory(); + return Create(config); +} + +AudioCodingModule* AudioCodingModule::Create(Clock* clock) { + Config config; + config.clock = clock; + config.decoder_factory = CreateBuiltinAudioDecoderFactory(); + return Create(config); +} + +AudioCodingModule* AudioCodingModule::Create(const Config& config) { + if (!config.decoder_factory) { + // TODO(ossu): Backwards compatibility. Will be removed after a deprecation + // cycle. + Config config_copy = config; + config_copy.decoder_factory = CreateBuiltinAudioDecoderFactory(); + return new AudioCodingModuleImpl(config_copy); + } + return new AudioCodingModuleImpl(config); +} + +int AudioCodingModule::NumberOfCodecs() { + return static_cast(acm2::RentACodec::NumberOfCodecs()); +} + +int AudioCodingModule::Codec(int list_id, CodecInst* codec) { + auto codec_id = acm2::RentACodec::CodecIdFromIndex(list_id); + if (!codec_id) + return -1; + auto ci = acm2::RentACodec::CodecInstById(*codec_id); + if (!ci) + return -1; + *codec = *ci; + return 0; +} + +int AudioCodingModule::Codec(const char* payload_name, + CodecInst* codec, + int sampling_freq_hz, + size_t channels) { + rtc::Optional ci = acm2::RentACodec::CodecInstByParams( + payload_name, sampling_freq_hz, channels); + if (ci) { + *codec = *ci; + return 0; + } else { + // We couldn't find a matching codec, so set the parameters to unacceptable + // values and return. + codec->plname[0] = '\0'; + codec->pltype = -1; + codec->pacsize = 0; + codec->rate = 0; + codec->plfreq = 0; + return -1; + } +} + +int AudioCodingModule::Codec(const char* payload_name, + int sampling_freq_hz, + size_t channels) { + rtc::Optional ci = + acm2::RentACodec::CodecIdByParams(payload_name, sampling_freq_hz, + channels); + if (!ci) + return -1; + rtc::Optional i = acm2::RentACodec::CodecIndexFromId(*ci); + return i ? *i : -1; +} + +// Checks the validity of the parameters of the given codec +bool AudioCodingModule::IsCodecValid(const CodecInst& codec) { + bool valid = acm2::RentACodec::IsCodecValid(codec); + if (!valid) + RTC_LOG(LS_ERROR) << "Invalid codec setting"; + return valid; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc new file mode 100644 index 0000000000..f0f4cdf6fe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc @@ -0,0 +1,1964 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/acm2/acm_receive_test.h" +#include "modules/audio_coding/acm2/acm_send_test.h" +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/neteq/tools/audio_checksum.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "modules/audio_coding/neteq/tools/constant_pcm_packet_source.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_wav_file.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/md5digest.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/refcountedobject.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/event_wrapper.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder.h" +#include "test/mock_audio_encoder.h" +#include "test/testsupport/fileutils.h" + +using ::testing::AtLeast; +using ::testing::Invoke; +using ::testing::_; + +namespace webrtc { + +namespace { +const int kSampleRateHz = 16000; +const int kNumSamples10ms = kSampleRateHz / 100; +const int kFrameSizeMs = 10; // Multiple of 10. +const int kFrameSizeSamples = kFrameSizeMs / 10 * kNumSamples10ms; +const int kPayloadSizeBytes = kFrameSizeSamples * sizeof(int16_t); +const uint8_t kPayloadType = 111; +} // namespace + +class RtpUtility { + public: + RtpUtility(int samples_per_packet, uint8_t payload_type) + : samples_per_packet_(samples_per_packet), payload_type_(payload_type) {} + + virtual ~RtpUtility() {} + + void Populate(WebRtcRTPHeader* rtp_header) { + rtp_header->header.sequenceNumber = 0xABCD; + rtp_header->header.timestamp = 0xABCDEF01; + rtp_header->header.payloadType = payload_type_; + rtp_header->header.markerBit = false; + rtp_header->header.ssrc = 0x1234; + rtp_header->header.numCSRCs = 0; + rtp_header->frameType = kAudioFrameSpeech; + + rtp_header->header.payload_type_frequency = kSampleRateHz; + rtp_header->type.Audio.channel = 1; + rtp_header->type.Audio.isCNG = false; + } + + void Forward(WebRtcRTPHeader* rtp_header) { + ++rtp_header->header.sequenceNumber; + rtp_header->header.timestamp += samples_per_packet_; + } + + private: + int samples_per_packet_; + uint8_t payload_type_; +}; + +class PacketizationCallbackStubOldApi : public AudioPacketizationCallback { + public: + PacketizationCallbackStubOldApi() + : num_calls_(0), + last_frame_type_(kEmptyFrame), + last_payload_type_(-1), + last_timestamp_(0) {} + + int32_t SendData(FrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + const RTPFragmentationHeader* fragmentation) override { + rtc::CritScope lock(&crit_sect_); + ++num_calls_; + last_frame_type_ = frame_type; + last_payload_type_ = payload_type; + last_timestamp_ = timestamp; + last_payload_vec_.assign(payload_data, payload_data + payload_len_bytes); + return 0; + } + + int num_calls() const { + rtc::CritScope lock(&crit_sect_); + return num_calls_; + } + + int last_payload_len_bytes() const { + rtc::CritScope lock(&crit_sect_); + return rtc::checked_cast(last_payload_vec_.size()); + } + + FrameType last_frame_type() const { + rtc::CritScope lock(&crit_sect_); + return last_frame_type_; + } + + int last_payload_type() const { + rtc::CritScope lock(&crit_sect_); + return last_payload_type_; + } + + uint32_t last_timestamp() const { + rtc::CritScope lock(&crit_sect_); + return last_timestamp_; + } + + void SwapBuffers(std::vector* payload) { + rtc::CritScope lock(&crit_sect_); + last_payload_vec_.swap(*payload); + } + + private: + int num_calls_ RTC_GUARDED_BY(crit_sect_); + FrameType last_frame_type_ RTC_GUARDED_BY(crit_sect_); + int last_payload_type_ RTC_GUARDED_BY(crit_sect_); + uint32_t last_timestamp_ RTC_GUARDED_BY(crit_sect_); + std::vector last_payload_vec_ RTC_GUARDED_BY(crit_sect_); + rtc::CriticalSection crit_sect_; +}; + +class AudioCodingModuleTestOldApi : public ::testing::Test { + protected: + AudioCodingModuleTestOldApi() + : rtp_utility_(new RtpUtility(kFrameSizeSamples, kPayloadType)), + clock_(Clock::GetRealTimeClock()) {} + + ~AudioCodingModuleTestOldApi() {} + + void TearDown() {} + + void SetUp() { + acm_.reset(AudioCodingModule::Create(clock_)); + + rtp_utility_->Populate(&rtp_header_); + + input_frame_.sample_rate_hz_ = kSampleRateHz; + input_frame_.num_channels_ = 1; + input_frame_.samples_per_channel_ = kSampleRateHz * 10 / 1000; // 10 ms. + static_assert(kSampleRateHz * 10 / 1000 <= AudioFrame::kMaxDataSizeSamples, + "audio frame too small"); + input_frame_.Mute(); + + ASSERT_EQ(0, acm_->RegisterTransportCallback(&packet_cb_)); + + SetUpL16Codec(); + } + + // Set up L16 codec. + virtual void SetUpL16Codec() { + audio_format_ = SdpAudioFormat("L16", kSampleRateHz, 1); + ASSERT_EQ(0, AudioCodingModule::Codec("L16", &codec_, kSampleRateHz, 1)); + codec_.pltype = kPayloadType; + } + + virtual void RegisterCodec() { + EXPECT_EQ(true, acm_->RegisterReceiveCodec(kPayloadType, *audio_format_)); + EXPECT_EQ(0, acm_->RegisterSendCodec(codec_)); + } + + virtual void InsertPacketAndPullAudio() { + InsertPacket(); + PullAudio(); + } + + virtual void InsertPacket() { + const uint8_t kPayload[kPayloadSizeBytes] = {0}; + ASSERT_EQ(0, + acm_->IncomingPacket(kPayload, kPayloadSizeBytes, rtp_header_)); + rtp_utility_->Forward(&rtp_header_); + } + + virtual void PullAudio() { + AudioFrame audio_frame; + bool muted; + ASSERT_EQ(0, acm_->PlayoutData10Ms(-1, &audio_frame, &muted)); + ASSERT_FALSE(muted); + } + + virtual void InsertAudio() { + ASSERT_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += kNumSamples10ms; + } + + virtual void VerifyEncoding() { + int last_length = packet_cb_.last_payload_len_bytes(); + EXPECT_TRUE(last_length == 2 * codec_.pacsize || last_length == 0) + << "Last encoded packet was " << last_length << " bytes."; + } + + virtual void InsertAudioAndVerifyEncoding() { + InsertAudio(); + VerifyEncoding(); + } + + std::unique_ptr rtp_utility_; + std::unique_ptr acm_; + PacketizationCallbackStubOldApi packet_cb_; + WebRtcRTPHeader rtp_header_; + AudioFrame input_frame_; + + // These two have to be kept in sync for now. In the future, we'll be able to + // eliminate the CodecInst and keep only the SdpAudioFormat. + rtc::Optional audio_format_; + CodecInst codec_; + + Clock* clock_; +}; + +// Check if the statistics are initialized correctly. Before any call to ACM +// all fields have to be zero. +#if defined(WEBRTC_ANDROID) +#define MAYBE_InitializedToZero DISABLED_InitializedToZero +#else +#define MAYBE_InitializedToZero InitializedToZero +#endif +TEST_F(AudioCodingModuleTestOldApi, MAYBE_InitializedToZero) { + RegisterCodec(); + AudioDecodingCallStats stats; + acm_->GetDecodingCallStatistics(&stats); + EXPECT_EQ(0, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(0, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(0, stats.decoded_plc); + EXPECT_EQ(0, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); +} + +// Insert some packets and pull audio. Check statistics are valid. Then, +// simulate packet loss and check if PLC and PLC-to-CNG statistics are +// correctly updated. +#if defined(WEBRTC_ANDROID) +#define MAYBE_NetEqCalls DISABLED_NetEqCalls +#else +#define MAYBE_NetEqCalls NetEqCalls +#endif +TEST_F(AudioCodingModuleTestOldApi, MAYBE_NetEqCalls) { + RegisterCodec(); + AudioDecodingCallStats stats; + const int kNumNormalCalls = 10; + + for (int num_calls = 0; num_calls < kNumNormalCalls; ++num_calls) { + InsertPacketAndPullAudio(); + } + acm_->GetDecodingCallStatistics(&stats); + EXPECT_EQ(kNumNormalCalls, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(kNumNormalCalls, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(0, stats.decoded_plc); + EXPECT_EQ(0, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); + + const int kNumPlc = 3; + const int kNumPlcCng = 5; + + // Simulate packet-loss. NetEq first performs PLC then PLC fades to CNG. + for (int n = 0; n < kNumPlc + kNumPlcCng; ++n) { + PullAudio(); + } + acm_->GetDecodingCallStatistics(&stats); + EXPECT_EQ(kNumNormalCalls + kNumPlc + kNumPlcCng, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(kNumNormalCalls, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(kNumPlc, stats.decoded_plc); + EXPECT_EQ(kNumPlcCng, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); + // TODO(henrik.lundin) Add a test with muted state enabled. +} + +TEST_F(AudioCodingModuleTestOldApi, VerifyOutputFrame) { + AudioFrame audio_frame; + const int kSampleRateHz = 32000; + bool muted; + EXPECT_EQ(0, acm_->PlayoutData10Ms(kSampleRateHz, &audio_frame, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(0u, audio_frame.timestamp_); + EXPECT_GT(audio_frame.num_channels_, 0u); + EXPECT_EQ(static_cast(kSampleRateHz / 100), + audio_frame.samples_per_channel_); + EXPECT_EQ(kSampleRateHz, audio_frame.sample_rate_hz_); +} + +// The below test is temporarily disabled on Windows due to problems +// with clang debug builds. +// TODO(tommi): Re-enable when we've figured out what the problem is. +// http://crbug.com/615050 +#if !defined(WEBRTC_WIN) && defined(__clang__) && RTC_DCHECK_IS_ON && \ + GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST_F(AudioCodingModuleTestOldApi, FailOnZeroDesiredFrequency) { + AudioFrame audio_frame; + bool muted; + EXPECT_DEATH(acm_->PlayoutData10Ms(0, &audio_frame, &muted), + "dst_sample_rate_hz"); +} +#endif + +// Checks that the transport callback is invoked once for each speech packet. +// Also checks that the frame type is kAudioFrameSpeech. +TEST_F(AudioCodingModuleTestOldApi, TransportCallbackIsInvokedForEachPacket) { + const int k10MsBlocksPerPacket = 3; + codec_.pacsize = k10MsBlocksPerPacket * kSampleRateHz / 100; + RegisterCodec(); + const int kLoops = 10; + for (int i = 0; i < kLoops; ++i) { + EXPECT_EQ(i / k10MsBlocksPerPacket, packet_cb_.num_calls()); + if (packet_cb_.num_calls() > 0) + EXPECT_EQ(kAudioFrameSpeech, packet_cb_.last_frame_type()); + InsertAudioAndVerifyEncoding(); + } + EXPECT_EQ(kLoops / k10MsBlocksPerPacket, packet_cb_.num_calls()); + EXPECT_EQ(kAudioFrameSpeech, packet_cb_.last_frame_type()); +} + +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +// Verifies that the RTP timestamp series is not reset when the codec is +// changed. +TEST_F(AudioCodingModuleTestOldApi, TimestampSeriesContinuesWhenCodecChanges) { + RegisterCodec(); // This registers the default codec. + uint32_t expected_ts = input_frame_.timestamp_; + int blocks_per_packet = codec_.pacsize / (kSampleRateHz / 100); + // Encode 5 packets of the first codec type. + const int kNumPackets1 = 5; + for (int j = 0; j < kNumPackets1; ++j) { + for (int i = 0; i < blocks_per_packet; ++i) { + EXPECT_EQ(j, packet_cb_.num_calls()); + InsertAudio(); + } + EXPECT_EQ(j + 1, packet_cb_.num_calls()); + EXPECT_EQ(expected_ts, packet_cb_.last_timestamp()); + expected_ts += codec_.pacsize; + } + + // Change codec. + ASSERT_EQ(0, AudioCodingModule::Codec("ISAC", &codec_, kSampleRateHz, 1)); + RegisterCodec(); + blocks_per_packet = codec_.pacsize / (kSampleRateHz / 100); + // Encode another 5 packets. + const int kNumPackets2 = 5; + for (int j = 0; j < kNumPackets2; ++j) { + for (int i = 0; i < blocks_per_packet; ++i) { + EXPECT_EQ(kNumPackets1 + j, packet_cb_.num_calls()); + InsertAudio(); + } + EXPECT_EQ(kNumPackets1 + j + 1, packet_cb_.num_calls()); + EXPECT_EQ(expected_ts, packet_cb_.last_timestamp()); + expected_ts += codec_.pacsize; + } +} +#endif + +// Introduce this class to set different expectations on the number of encoded +// bytes. This class expects all encoded packets to be 9 bytes (matching one +// CNG SID frame) or 0 bytes. This test depends on |input_frame_| containing +// (near-)zero values. It also introduces a way to register comfort noise with +// a custom payload type. +class AudioCodingModuleTestWithComfortNoiseOldApi + : public AudioCodingModuleTestOldApi { + protected: + void RegisterCngCodec(int rtp_payload_type) { + EXPECT_EQ(true, + acm_->RegisterReceiveCodec( + rtp_payload_type, SdpAudioFormat("cn", kSampleRateHz, 1))); + + CodecInst codec; + EXPECT_EQ(0, AudioCodingModule::Codec("CN", &codec, kSampleRateHz, 1)); + codec.pltype = rtp_payload_type; + EXPECT_EQ(0, acm_->RegisterSendCodec(codec)); + } + + void VerifyEncoding() override { + int last_length = packet_cb_.last_payload_len_bytes(); + EXPECT_TRUE(last_length == 9 || last_length == 0) + << "Last encoded packet was " << last_length << " bytes."; + } + + void DoTest(int blocks_per_packet, int cng_pt) { + const int kLoops = 40; + // This array defines the expected frame types, and when they should arrive. + // We expect a frame to arrive each time the speech encoder would have + // produced a packet, and once every 100 ms the frame should be non-empty, + // that is contain comfort noise. + const struct { + int ix; + FrameType type; + } expectation[] = {{2, kAudioFrameCN}, + {5, kEmptyFrame}, + {8, kEmptyFrame}, + {11, kAudioFrameCN}, + {14, kEmptyFrame}, + {17, kEmptyFrame}, + {20, kAudioFrameCN}, + {23, kEmptyFrame}, + {26, kEmptyFrame}, + {29, kEmptyFrame}, + {32, kAudioFrameCN}, + {35, kEmptyFrame}, + {38, kEmptyFrame}}; + for (int i = 0; i < kLoops; ++i) { + int num_calls_before = packet_cb_.num_calls(); + EXPECT_EQ(i / blocks_per_packet, num_calls_before); + InsertAudioAndVerifyEncoding(); + int num_calls = packet_cb_.num_calls(); + if (num_calls == num_calls_before + 1) { + EXPECT_EQ(expectation[num_calls - 1].ix, i); + EXPECT_EQ(expectation[num_calls - 1].type, packet_cb_.last_frame_type()) + << "Wrong frame type for lap " << i; + EXPECT_EQ(cng_pt, packet_cb_.last_payload_type()); + } else { + EXPECT_EQ(num_calls, num_calls_before); + } + } + } +}; + +// Checks that the transport callback is invoked once per frame period of the +// underlying speech encoder, even when comfort noise is produced. +// Also checks that the frame type is kAudioFrameCN or kEmptyFrame. +// This test and the next check the same thing, but differ in the order of +// speech codec and CNG registration. +TEST_F(AudioCodingModuleTestWithComfortNoiseOldApi, + TransportCallbackTestForComfortNoiseRegisterCngLast) { + const int k10MsBlocksPerPacket = 3; + codec_.pacsize = k10MsBlocksPerPacket * kSampleRateHz / 100; + RegisterCodec(); + const int kCngPayloadType = 105; + RegisterCngCodec(kCngPayloadType); + ASSERT_EQ(0, acm_->SetVAD(true, true)); + DoTest(k10MsBlocksPerPacket, kCngPayloadType); +} + +TEST_F(AudioCodingModuleTestWithComfortNoiseOldApi, + TransportCallbackTestForComfortNoiseRegisterCngFirst) { + const int k10MsBlocksPerPacket = 3; + codec_.pacsize = k10MsBlocksPerPacket * kSampleRateHz / 100; + const int kCngPayloadType = 105; + RegisterCngCodec(kCngPayloadType); + RegisterCodec(); + ASSERT_EQ(0, acm_->SetVAD(true, true)); + DoTest(k10MsBlocksPerPacket, kCngPayloadType); +} + +// A multi-threaded test for ACM. This base class is using the PCM16b 16 kHz +// codec, while the derive class AcmIsacMtTest is using iSAC. +class AudioCodingModuleMtTestOldApi : public AudioCodingModuleTestOldApi { + protected: + static const int kNumPackets = 500; + static const int kNumPullCalls = 500; + + AudioCodingModuleMtTestOldApi() + : AudioCodingModuleTestOldApi(), + send_thread_(CbSendThread, this, "send"), + insert_packet_thread_(CbInsertPacketThread, this, "insert_packet"), + pull_audio_thread_(CbPullAudioThread, this, "pull_audio"), + test_complete_(EventWrapper::Create()), + send_count_(0), + insert_packet_count_(0), + pull_audio_count_(0), + next_insert_packet_time_ms_(0), + fake_clock_(new SimulatedClock(0)) { + clock_ = fake_clock_.get(); + } + + void SetUp() { + AudioCodingModuleTestOldApi::SetUp(); + RegisterCodec(); // Must be called before the threads start below. + StartThreads(); + } + + void StartThreads() { + send_thread_.Start(); + send_thread_.SetPriority(rtc::kRealtimePriority); + insert_packet_thread_.Start(); + insert_packet_thread_.SetPriority(rtc::kRealtimePriority); + pull_audio_thread_.Start(); + pull_audio_thread_.SetPriority(rtc::kRealtimePriority); + } + + void TearDown() { + AudioCodingModuleTestOldApi::TearDown(); + pull_audio_thread_.Stop(); + send_thread_.Stop(); + insert_packet_thread_.Stop(); + } + + EventTypeWrapper RunTest() { + return test_complete_->Wait(10 * 60 * 1000); // 10 minutes' timeout. + } + + virtual bool TestDone() { + if (packet_cb_.num_calls() > kNumPackets) { + rtc::CritScope lock(&crit_sect_); + if (pull_audio_count_ > kNumPullCalls) { + // Both conditions for completion are met. End the test. + return true; + } + } + return false; + } + + static bool CbSendThread(void* context) { + return reinterpret_cast(context) + ->CbSendImpl(); + } + + // The send thread doesn't have to care about the current simulated time, + // since only the AcmReceiver is using the clock. + bool CbSendImpl() { + SleepMs(1); + if (HasFatalFailure()) { + // End the test early if a fatal failure (ASSERT_*) has occurred. + test_complete_->Set(); + } + ++send_count_; + InsertAudioAndVerifyEncoding(); + if (TestDone()) { + test_complete_->Set(); + } + return true; + } + + static bool CbInsertPacketThread(void* context) { + return reinterpret_cast(context) + ->CbInsertPacketImpl(); + } + + bool CbInsertPacketImpl() { + SleepMs(1); + { + rtc::CritScope lock(&crit_sect_); + if (clock_->TimeInMilliseconds() < next_insert_packet_time_ms_) { + return true; + } + next_insert_packet_time_ms_ += 10; + } + // Now we're not holding the crit sect when calling ACM. + ++insert_packet_count_; + InsertPacket(); + return true; + } + + static bool CbPullAudioThread(void* context) { + return reinterpret_cast(context) + ->CbPullAudioImpl(); + } + + bool CbPullAudioImpl() { + SleepMs(1); + { + rtc::CritScope lock(&crit_sect_); + // Don't let the insert thread fall behind. + if (next_insert_packet_time_ms_ < clock_->TimeInMilliseconds()) { + return true; + } + ++pull_audio_count_; + } + // Now we're not holding the crit sect when calling ACM. + PullAudio(); + fake_clock_->AdvanceTimeMilliseconds(10); + return true; + } + + rtc::PlatformThread send_thread_; + rtc::PlatformThread insert_packet_thread_; + rtc::PlatformThread pull_audio_thread_; + const std::unique_ptr test_complete_; + int send_count_; + int insert_packet_count_; + int pull_audio_count_ RTC_GUARDED_BY(crit_sect_); + rtc::CriticalSection crit_sect_; + int64_t next_insert_packet_time_ms_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr fake_clock_; +}; + +#if defined(WEBRTC_IOS) +#define MAYBE_DoTest DISABLED_DoTest +#else +#define MAYBE_DoTest DoTest +#endif +TEST_F(AudioCodingModuleMtTestOldApi, MAYBE_DoTest) { + EXPECT_EQ(kEventSignaled, RunTest()); +} + +// This is a multi-threaded ACM test using iSAC. The test encodes audio +// from a PCM file. The most recent encoded frame is used as input to the +// receiving part. Depending on timing, it may happen that the same RTP packet +// is inserted into the receiver multiple times, but this is a valid use-case, +// and simplifies the test code a lot. +class AcmIsacMtTestOldApi : public AudioCodingModuleMtTestOldApi { + protected: + static const int kNumPackets = 500; + static const int kNumPullCalls = 500; + + AcmIsacMtTestOldApi() + : AudioCodingModuleMtTestOldApi(), last_packet_number_(0) {} + + ~AcmIsacMtTestOldApi() {} + + void SetUp() override { + AudioCodingModuleTestOldApi::SetUp(); + RegisterCodec(); // Must be called before the threads start below. + + // Set up input audio source to read from specified file, loop after 5 + // seconds, and deliver blocks of 10 ms. + const std::string input_file_name = + webrtc::test::ResourcePath("audio_coding/speech_mono_16kHz", "pcm"); + audio_loop_.Init(input_file_name, 5 * kSampleRateHz, kNumSamples10ms); + + // Generate one packet to have something to insert. + int loop_counter = 0; + while (packet_cb_.last_payload_len_bytes() == 0) { + InsertAudio(); + ASSERT_LT(loop_counter++, 10); + } + // Set |last_packet_number_| to one less that |num_calls| so that the packet + // will be fetched in the next InsertPacket() call. + last_packet_number_ = packet_cb_.num_calls() - 1; + + StartThreads(); + } + + void RegisterCodec() override { + static_assert(kSampleRateHz == 16000, "test designed for iSAC 16 kHz"); + audio_format_ = SdpAudioFormat("isac", kSampleRateHz, 1); + AudioCodingModule::Codec("ISAC", &codec_, kSampleRateHz, 1); + codec_.pltype = kPayloadType; + + // Register iSAC codec in ACM, effectively unregistering the PCM16B codec + // registered in AudioCodingModuleTestOldApi::SetUp(); + EXPECT_EQ(true, acm_->RegisterReceiveCodec(kPayloadType, *audio_format_)); + EXPECT_EQ(0, acm_->RegisterSendCodec(codec_)); + } + + void InsertPacket() override { + int num_calls = packet_cb_.num_calls(); // Store locally for thread safety. + if (num_calls > last_packet_number_) { + // Get the new payload out from the callback handler. + // Note that since we swap buffers here instead of directly inserting + // a pointer to the data in |packet_cb_|, we avoid locking the callback + // for the duration of the IncomingPacket() call. + packet_cb_.SwapBuffers(&last_payload_vec_); + ASSERT_GT(last_payload_vec_.size(), 0u); + rtp_utility_->Forward(&rtp_header_); + last_packet_number_ = num_calls; + } + ASSERT_GT(last_payload_vec_.size(), 0u); + ASSERT_EQ( + 0, + acm_->IncomingPacket( + &last_payload_vec_[0], last_payload_vec_.size(), rtp_header_)); + } + + void InsertAudio() override { + // TODO(kwiberg): Use std::copy here. Might be complications because AFAICS + // this call confuses the number of samples with the number of bytes, and + // ends up copying only half of what it should. + memcpy(input_frame_.mutable_data(), audio_loop_.GetNextBlock().data(), + kNumSamples10ms); + AudioCodingModuleTestOldApi::InsertAudio(); + } + + // Override the verification function with no-op, since iSAC produces variable + // payload sizes. + void VerifyEncoding() override {} + + // This method is the same as AudioCodingModuleMtTestOldApi::TestDone(), but + // here it is using the constants defined in this class (i.e., shorter test + // run). + bool TestDone() override { + if (packet_cb_.num_calls() > kNumPackets) { + rtc::CritScope lock(&crit_sect_); + if (pull_audio_count_ > kNumPullCalls) { + // Both conditions for completion are met. End the test. + return true; + } + } + return false; + } + + int last_packet_number_; + std::vector last_payload_vec_; + test::AudioLoop audio_loop_; +}; + +#if defined(WEBRTC_IOS) +#define MAYBE_DoTest DISABLED_DoTest +#else +#define MAYBE_DoTest DoTest +#endif +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +TEST_F(AcmIsacMtTestOldApi, MAYBE_DoTest) { + EXPECT_EQ(kEventSignaled, RunTest()); +} +#endif + +class AcmReRegisterIsacMtTestOldApi : public AudioCodingModuleTestOldApi { + protected: + static const int kRegisterAfterNumPackets = 5; + static const int kNumPackets = 10; + static const int kPacketSizeMs = 30; + static const int kPacketSizeSamples = kPacketSizeMs * 16; + + AcmReRegisterIsacMtTestOldApi() + : AudioCodingModuleTestOldApi(), + receive_thread_(CbReceiveThread, this, "receive"), + codec_registration_thread_(CbCodecRegistrationThread, + this, + "codec_registration"), + test_complete_(EventWrapper::Create()), + codec_registered_(false), + receive_packet_count_(0), + next_insert_packet_time_ms_(0), + fake_clock_(new SimulatedClock(0)) { + AudioEncoderIsacFloatImpl::Config config; + config.payload_type = kPayloadType; + isac_encoder_.reset(new AudioEncoderIsacFloatImpl(config)); + clock_ = fake_clock_.get(); + } + + void SetUp() override { + AudioCodingModuleTestOldApi::SetUp(); + // Set up input audio source to read from specified file, loop after 5 + // seconds, and deliver blocks of 10 ms. + const std::string input_file_name = + webrtc::test::ResourcePath("audio_coding/speech_mono_16kHz", "pcm"); + audio_loop_.Init(input_file_name, 5 * kSampleRateHz, kNumSamples10ms); + RegisterCodec(); // Must be called before the threads start below. + StartThreads(); + } + + void RegisterCodec() override { + static_assert(kSampleRateHz == 16000, "test designed for iSAC 16 kHz"); + AudioCodingModule::Codec("ISAC", &codec_, kSampleRateHz, 1); + codec_.pltype = kPayloadType; + + // Register iSAC codec in ACM, effectively unregistering the PCM16B codec + // registered in AudioCodingModuleTestOldApi::SetUp(); + // Only register the decoder for now. The encoder is registered later. + ASSERT_EQ(true, acm_->RegisterReceiveCodec(codec_.pltype, + CodecInstToSdp(codec_))); + } + + void StartThreads() { + receive_thread_.Start(); + receive_thread_.SetPriority(rtc::kRealtimePriority); + codec_registration_thread_.Start(); + codec_registration_thread_.SetPriority(rtc::kRealtimePriority); + } + + void TearDown() override { + AudioCodingModuleTestOldApi::TearDown(); + receive_thread_.Stop(); + codec_registration_thread_.Stop(); + } + + EventTypeWrapper RunTest() { + return test_complete_->Wait(10 * 60 * 1000); // 10 minutes' timeout. + } + + static bool CbReceiveThread(void* context) { + return reinterpret_cast(context) + ->CbReceiveImpl(); + } + + bool CbReceiveImpl() { + SleepMs(1); + rtc::Buffer encoded; + AudioEncoder::EncodedInfo info; + { + rtc::CritScope lock(&crit_sect_); + if (clock_->TimeInMilliseconds() < next_insert_packet_time_ms_) { + return true; + } + next_insert_packet_time_ms_ += kPacketSizeMs; + ++receive_packet_count_; + + // Encode new frame. + uint32_t input_timestamp = rtp_header_.header.timestamp; + while (info.encoded_bytes == 0) { + info = + isac_encoder_->Encode(input_timestamp, audio_loop_.GetNextBlock(), + &encoded); + input_timestamp += 160; // 10 ms at 16 kHz. + } + EXPECT_EQ(rtp_header_.header.timestamp + kPacketSizeSamples, + input_timestamp); + EXPECT_EQ(rtp_header_.header.timestamp, info.encoded_timestamp); + EXPECT_EQ(rtp_header_.header.payloadType, info.payload_type); + } + // Now we're not holding the crit sect when calling ACM. + + // Insert into ACM. + EXPECT_EQ(0, acm_->IncomingPacket(encoded.data(), info.encoded_bytes, + rtp_header_)); + + // Pull audio. + for (int i = 0; i < rtc::CheckedDivExact(kPacketSizeMs, 10); ++i) { + AudioFrame audio_frame; + bool muted; + EXPECT_EQ(0, acm_->PlayoutData10Ms(-1 /* default output frequency */, + &audio_frame, &muted)); + if (muted) { + ADD_FAILURE(); + return false; + } + fake_clock_->AdvanceTimeMilliseconds(10); + } + rtp_utility_->Forward(&rtp_header_); + return true; + } + + static bool CbCodecRegistrationThread(void* context) { + return reinterpret_cast(context) + ->CbCodecRegistrationImpl(); + } + + bool CbCodecRegistrationImpl() { + SleepMs(1); + if (HasFatalFailure()) { + // End the test early if a fatal failure (ASSERT_*) has occurred. + test_complete_->Set(); + } + rtc::CritScope lock(&crit_sect_); + if (!codec_registered_ && + receive_packet_count_ > kRegisterAfterNumPackets) { + // Register the iSAC encoder. + EXPECT_EQ(0, acm_->RegisterSendCodec(codec_)); + codec_registered_ = true; + } + if (codec_registered_ && receive_packet_count_ > kNumPackets) { + test_complete_->Set(); + } + return true; + } + + rtc::PlatformThread receive_thread_; + rtc::PlatformThread codec_registration_thread_; + const std::unique_ptr test_complete_; + rtc::CriticalSection crit_sect_; + bool codec_registered_ RTC_GUARDED_BY(crit_sect_); + int receive_packet_count_ RTC_GUARDED_BY(crit_sect_); + int64_t next_insert_packet_time_ms_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr isac_encoder_; + std::unique_ptr fake_clock_; + test::AudioLoop audio_loop_; +}; + +#if defined(WEBRTC_IOS) +#define MAYBE_DoTest DISABLED_DoTest +#else +#define MAYBE_DoTest DoTest +#endif +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +TEST_F(AcmReRegisterIsacMtTestOldApi, MAYBE_DoTest) { + EXPECT_EQ(kEventSignaled, RunTest()); +} +#endif + +// Disabling all of these tests on iOS until file support has been added. +// See https://code.google.com/p/webrtc/issues/detail?id=4752 for details. +#if !defined(WEBRTC_IOS) + +class AcmReceiverBitExactnessOldApi : public ::testing::Test { + public: + static std::string PlatformChecksum(std::string others, + std::string win64, + std::string android_arm32, + std::string android_arm64, + std::string android_arm64_clang) { +#if defined(_WIN32) && defined(WEBRTC_ARCH_64_BITS) + return win64; +#elif defined(WEBRTC_ANDROID) && defined(WEBRTC_ARCH_ARM) + return android_arm32; +#elif defined(WEBRTC_ANDROID) && defined(WEBRTC_ARCH_ARM64) +#if defined(__clang__) + // Android ARM64 with Clang compiler + return android_arm64_clang; +#else + // Android ARM64 with non-Clang compiler + return android_arm64; +#endif // __clang__ +#else + return others; +#endif + } + + protected: + struct ExternalDecoder { + int rtp_payload_type; + AudioDecoder* external_decoder; + int sample_rate_hz; + int num_channels; + std::string name; + }; + + void Run(int output_freq_hz, const std::string& checksum_ref) { + Run(output_freq_hz, checksum_ref, CreateBuiltinAudioDecoderFactory(), + [](AudioCodingModule*) {}); + } + + void Run(int output_freq_hz, + const std::string& checksum_ref, + rtc::scoped_refptr decoder_factory, + rtc::FunctionView decoder_reg) { + const std::string input_file_name = + webrtc::test::ResourcePath("audio_coding/neteq_universal_new", "rtp"); + std::unique_ptr packet_source( + test::RtpFileSource::Create(input_file_name)); +#ifdef WEBRTC_ANDROID + // Filter out iLBC and iSAC-swb since they are not supported on Android. + packet_source->FilterOutPayloadType(102); // iLBC. + packet_source->FilterOutPayloadType(104); // iSAC-swb. +#endif + + test::AudioChecksum checksum; + const std::string output_file_name = + webrtc::test::OutputPath() + + ::testing::UnitTest::GetInstance() + ->current_test_info() + ->test_case_name() + + "_" + ::testing::UnitTest::GetInstance()->current_test_info()->name() + + "_output.wav"; + test::OutputWavFile output_file(output_file_name, output_freq_hz); + test::AudioSinkFork output(&checksum, &output_file); + + test::AcmReceiveTestOldApi test( + packet_source.get(), &output, output_freq_hz, + test::AcmReceiveTestOldApi::kArbitraryChannels, + std::move(decoder_factory)); + ASSERT_NO_FATAL_FAILURE(test.RegisterNetEqTestCodecs()); + decoder_reg(test.get_acm()); + test.Run(); + + std::string checksum_string = checksum.Finish(); + EXPECT_EQ(checksum_ref, checksum_string); + + // Delete the output file. + remove(output_file_name.c_str()); + } +}; + +#if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) && \ + defined(WEBRTC_CODEC_ILBC) +TEST_F(AcmReceiverBitExactnessOldApi, 8kHzOutput) { + Run(8000, PlatformChecksum("2adede965c6f87de7142c51552111d08", + "028c0fc414b1c9ab7e582dccdf381e98", + "36c95170c1393d4b765d1c17b61ef977", + "4598140b5e4f7ee66c5adad609e65a3e", + "bac5db6dff44323be401060f1279a532")); +} + +TEST_F(AcmReceiverBitExactnessOldApi, 16kHzOutput) { + Run(16000, PlatformChecksum("c2550a3db7632de409e8db0093df1c12", + "edd31f4b6665cd5b9041fb93f2316594", + "22128bca51650cb61c80bed63b595603", + "f2aad418af974a3b1694d5ae5cc2c3c7", + "61c3cb9386b9503feebcb829c9be54bd")); +} + +TEST_F(AcmReceiverBitExactnessOldApi, 32kHzOutput) { + Run(32000, PlatformChecksum("85e28d7950132d56f90b099c90f82153", + "7b903f5c89997f271b405e63c245ef45", + "8b8fc6c6fd1dcdcfb3dd90e1ce597f10", + "100869c8dcde51346c2073e52a272d98", + "fdec5301dc649a47d407382b587e14da")); +} + +TEST_F(AcmReceiverBitExactnessOldApi, 48kHzOutput) { + Run(48000, PlatformChecksum("ab611510e8fd6d5210a23cc04d3f0e8e", + "d8609bc9b495d81f29779344c68bcc47", + "ec5ebb90cda0ea5bb89e79d698af65de", + "bd44bf97e7899186532f91235cef444d", + "0baae2972cca142027d4af44f95f0bd5")); +} + +TEST_F(AcmReceiverBitExactnessOldApi, 48kHzOutputExternalDecoder) { + class ADFactory : public AudioDecoderFactory { + public: + ADFactory() + : mock_decoder_(new MockAudioDecoder()), + pcmu_decoder_(1), + decode_forwarder_(&pcmu_decoder_), + fact_(CreateBuiltinAudioDecoderFactory()) { + // Set expectations on the mock decoder and also delegate the calls to + // the real decoder. + EXPECT_CALL(*mock_decoder_, IncomingPacket(_, _, _, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly( + Invoke(&pcmu_decoder_, &AudioDecoderPcmU::IncomingPacket)); + EXPECT_CALL(*mock_decoder_, SampleRateHz()) + .Times(AtLeast(1)) + .WillRepeatedly( + Invoke(&pcmu_decoder_, &AudioDecoderPcmU::SampleRateHz)); + EXPECT_CALL(*mock_decoder_, Channels()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&pcmu_decoder_, &AudioDecoderPcmU::Channels)); + EXPECT_CALL(*mock_decoder_, DecodeInternal(_, _, _, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&decode_forwarder_, &DecodeForwarder::Decode)); + EXPECT_CALL(*mock_decoder_, HasDecodePlc()) + .Times(AtLeast(1)) + .WillRepeatedly( + Invoke(&pcmu_decoder_, &AudioDecoderPcmU::HasDecodePlc)); + EXPECT_CALL(*mock_decoder_, PacketDuration(_, _)) + .Times(AtLeast(1)) + .WillRepeatedly( + Invoke(&pcmu_decoder_, &AudioDecoderPcmU::PacketDuration)); + EXPECT_CALL(*mock_decoder_, Die()); + } + std::vector GetSupportedDecoders() override { + return fact_->GetSupportedDecoders(); + } + bool IsSupportedDecoder(const SdpAudioFormat& format) override { + return format.name == "MockPCMu" ? true + : fact_->IsSupportedDecoder(format); + } + std::unique_ptr MakeAudioDecoder( + const SdpAudioFormat& format) override { + return format.name == "MockPCMu" ? std::move(mock_decoder_) + : fact_->MakeAudioDecoder(format); + } + + private: + // Class intended to forward a call from a mock DecodeInternal to Decode on + // the real decoder's Decode. DecodeInternal for the real decoder isn't + // public. + class DecodeForwarder { + public: + explicit DecodeForwarder(AudioDecoder* decoder) : decoder_(decoder) {} + int Decode(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + AudioDecoder::SpeechType* speech_type) { + return decoder_->Decode(encoded, encoded_len, sample_rate_hz, + decoder_->PacketDuration(encoded, encoded_len) * + decoder_->Channels() * sizeof(int16_t), + decoded, speech_type); + } + + private: + AudioDecoder* const decoder_; + }; + + std::unique_ptr mock_decoder_; + AudioDecoderPcmU pcmu_decoder_; + DecodeForwarder decode_forwarder_; + rtc::scoped_refptr fact_; // Fallback factory. + }; + + rtc::scoped_refptr> factory( + new rtc::RefCountedObject); + Run(48000, PlatformChecksum("ab611510e8fd6d5210a23cc04d3f0e8e", + "d8609bc9b495d81f29779344c68bcc47", + "ec5ebb90cda0ea5bb89e79d698af65de", + "bd44bf97e7899186532f91235cef444d", + "0baae2972cca142027d4af44f95f0bd5"), + factory, [](AudioCodingModule* acm) { + acm->RegisterReceiveCodec(0, {"MockPCMu", 8000, 1}); + }); +} +#endif + +// This test verifies bit exactness for the send-side of ACM. The test setup is +// a chain of three different test classes: +// +// test::AcmSendTest -> AcmSenderBitExactness -> test::AcmReceiveTest +// +// The receiver side is driving the test by requesting new packets from +// AcmSenderBitExactness::NextPacket(). This method, in turn, asks for the +// packet from test::AcmSendTest::NextPacket, which inserts audio from the +// input file until one packet is produced. (The input file loops indefinitely.) +// Before passing the packet to the receiver, this test class verifies the +// packet header and updates a payload checksum with the new payload. The +// decoded output from the receiver is also verified with a (separate) checksum. +class AcmSenderBitExactnessOldApi : public ::testing::Test, + public test::PacketSource { + protected: + static const int kTestDurationMs = 1000; + + AcmSenderBitExactnessOldApi() + : frame_size_rtp_timestamps_(0), + packet_count_(0), + payload_type_(0), + last_sequence_number_(0), + last_timestamp_(0) {} + + // Sets up the test::AcmSendTest object. Returns true on success, otherwise + // false. + bool SetUpSender() { + const std::string input_file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + // Note that |audio_source_| will loop forever. The test duration is set + // explicitly by |kTestDurationMs|. + audio_source_.reset(new test::InputAudioFile(input_file_name)); + static const int kSourceRateHz = 32000; + send_test_.reset(new test::AcmSendTestOldApi( + audio_source_.get(), kSourceRateHz, kTestDurationMs)); + return send_test_.get() != NULL; + } + + // Registers a send codec in the test::AcmSendTest object. Returns true on + // success, false on failure. + bool RegisterSendCodec(const char* payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples, + int frame_size_rtp_timestamps) { + payload_type_ = payload_type; + frame_size_rtp_timestamps_ = frame_size_rtp_timestamps; + return send_test_->RegisterCodec(payload_name, + sampling_freq_hz, + channels, + payload_type, + frame_size_samples); + } + + bool RegisterExternalSendCodec(AudioEncoder* external_speech_encoder, + int payload_type) { + payload_type_ = payload_type; + frame_size_rtp_timestamps_ = rtc::checked_cast( + external_speech_encoder->Num10MsFramesInNextPacket() * + external_speech_encoder->RtpTimestampRateHz() / 100); + return send_test_->RegisterExternalCodec(external_speech_encoder); + } + + // Runs the test. SetUpSender() and RegisterSendCodec() must have been called + // before calling this method. + void Run(const std::string& audio_checksum_ref, + const std::string& payload_checksum_ref, + int expected_packets, + test::AcmReceiveTestOldApi::NumOutputChannels expected_channels) { + // Set up the receiver used to decode the packets and verify the decoded + // output. + test::AudioChecksum audio_checksum; + const std::string output_file_name = + webrtc::test::OutputPath() + + ::testing::UnitTest::GetInstance() + ->current_test_info() + ->test_case_name() + + "_" + ::testing::UnitTest::GetInstance()->current_test_info()->name() + + "_output.wav"; + const int kOutputFreqHz = 8000; + test::OutputWavFile output_file(output_file_name, kOutputFreqHz); + // Have the output audio sent both to file and to the checksum calculator. + test::AudioSinkFork output(&audio_checksum, &output_file); + test::AcmReceiveTestOldApi receive_test(this, &output, kOutputFreqHz, + expected_channels, + CreateBuiltinAudioDecoderFactory()); + ASSERT_NO_FATAL_FAILURE(receive_test.RegisterDefaultCodecs()); + + // This is where the actual test is executed. + receive_test.Run(); + + // Extract and verify the audio checksum. + std::string checksum_string = audio_checksum.Finish(); + EXPECT_EQ(audio_checksum_ref, checksum_string); + + // Extract and verify the payload checksum. + char checksum_result[rtc::Md5Digest::kSize]; + payload_checksum_.Finish(checksum_result, rtc::Md5Digest::kSize); + checksum_string = rtc::hex_encode(checksum_result, rtc::Md5Digest::kSize); + EXPECT_EQ(payload_checksum_ref, checksum_string); + + // Verify number of packets produced. + EXPECT_EQ(expected_packets, packet_count_); + + // Delete the output file. + remove(output_file_name.c_str()); + } + + // Inherited from test::PacketSource. + std::unique_ptr NextPacket() override { + auto packet = send_test_->NextPacket(); + if (!packet) + return NULL; + + VerifyPacket(packet.get()); + // TODO(henrik.lundin) Save the packet to file as well. + + // Pass it on to the caller. The caller becomes the owner of |packet|. + return packet; + } + + // Verifies the packet. + void VerifyPacket(const test::Packet* packet) { + EXPECT_TRUE(packet->valid_header()); + // (We can check the header fields even if valid_header() is false.) + EXPECT_EQ(payload_type_, packet->header().payloadType); + if (packet_count_ > 0) { + // This is not the first packet. + uint16_t sequence_number_diff = + packet->header().sequenceNumber - last_sequence_number_; + EXPECT_EQ(1, sequence_number_diff); + uint32_t timestamp_diff = packet->header().timestamp - last_timestamp_; + EXPECT_EQ(frame_size_rtp_timestamps_, timestamp_diff); + } + ++packet_count_; + last_sequence_number_ = packet->header().sequenceNumber; + last_timestamp_ = packet->header().timestamp; + // Update the checksum. + payload_checksum_.Update(packet->payload(), packet->payload_length_bytes()); + } + + void SetUpTest(const char* codec_name, + int codec_sample_rate_hz, + int channels, + int payload_type, + int codec_frame_size_samples, + int codec_frame_size_rtp_timestamps) { + ASSERT_TRUE(SetUpSender()); + ASSERT_TRUE(RegisterSendCodec(codec_name, + codec_sample_rate_hz, + channels, + payload_type, + codec_frame_size_samples, + codec_frame_size_rtp_timestamps)); + } + + void SetUpTestExternalEncoder(AudioEncoder* external_speech_encoder, + int payload_type) { + ASSERT_TRUE(SetUpSender()); + ASSERT_TRUE( + RegisterExternalSendCodec(external_speech_encoder, payload_type)); + } + + std::unique_ptr send_test_; + std::unique_ptr audio_source_; + uint32_t frame_size_rtp_timestamps_; + int packet_count_; + uint8_t payload_type_; + uint16_t last_sequence_number_; + uint32_t last_timestamp_; + rtc::Md5Digest payload_checksum_; +}; + +class AcmSenderBitExactnessNewApi : public AcmSenderBitExactnessOldApi {}; + +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +TEST_F(AcmSenderBitExactnessOldApi, IsacWb30ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("ISAC", 16000, 1, 103, 480, 480)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "2c9cb15d4ed55b5a0cadd04883bc73b0", + "9336a9b993cbd8a751f0e8958e66c89c", + "bd4682225f7c4ad5f2049f6769713ac2", + "343f1f42be0607c61e6516aece424609", + "2c9cb15d4ed55b5a0cadd04883bc73b0"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "3c79f16f34218271f3dca4e2b1dfe1bb", + "d42cb5195463da26c8129bbfe73a22e6", + "83de248aea9c3c2bd680b6952401b4ca", + "3c79f16f34218271f3dca4e2b1dfe1bb", + "3c79f16f34218271f3dca4e2b1dfe1bb"), + 33, test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, IsacWb60ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("ISAC", 16000, 1, 103, 960, 960)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "1ad29139a04782a33daad8c2b9b35875", + "14d63c5f08127d280e722e3191b73bdd", + "edcf26694c289e3d9691faf79b74f09f", + "ef75e900e6f375e3061163c53fd09a63", + "1ad29139a04782a33daad8c2b9b35875"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "9e0a0ab743ad987b55b8e14802769c56", + "ebe04a819d3a9d83a83a17f271e1139a", + "97aeef98553b5a4b5a68f8b716e8eaf0", + "9e0a0ab743ad987b55b8e14802769c56", + "9e0a0ab743ad987b55b8e14802769c56"), + 16, test::AcmReceiveTestOldApi::kMonoOutput); +} +#endif + +#if defined(WEBRTC_ANDROID) +#define MAYBE_IsacSwb30ms DISABLED_IsacSwb30ms +#else +#define MAYBE_IsacSwb30ms IsacSwb30ms +#endif +#if defined(WEBRTC_CODEC_ISAC) +TEST_F(AcmSenderBitExactnessOldApi, MAYBE_IsacSwb30ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("ISAC", 32000, 1, 104, 960, 960)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "5683b58da0fbf2063c7adc2e6bfb3fb8", + "2b3c387d06f00b7b7aad4c9be56fb83d", "android_arm32_audio", + "android_arm64_audio", "android_arm64_clang_audio"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "ce86106a93419aefb063097108ec94ab", + "bcc2041e7744c7ebd9f701866856849c", "android_arm32_payload", + "android_arm64_payload", "android_arm64_clang_payload"), + 33, test::AcmReceiveTestOldApi::kMonoOutput); +} +#endif + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_8000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 1, 107, 80, 80)); + Run("de4a98e1406f8b798d99cd0704e862e2", + "c1edd36339ce0326cc4550041ad719a0", + 100, + test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_16000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 16000, 1, 108, 160, 160)); + Run("ae646d7b68384a1269cc080dd4501916", + "ad786526383178b08d80d6eee06e9bad", + 100, + test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_32000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 32000, 1, 109, 320, 320)); + Run("7fe325e8fbaf755e3c5df0b11a4774fb", + "5ef82ea885e922263606c6fdbc49f651", + 100, + test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_stereo_8000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 2, 111, 80, 80)); + Run("fb263b74e7ac3de915474d77e4744ceb", + "62ce5adb0d4965d0a52ec98ae7f98974", + 100, + test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_stereo_16000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 16000, 2, 112, 160, 160)); + Run("d09e9239553649d7ac93e19d304281fd", + "41ca8edac4b8c71cd54fd9f25ec14870", + 100, + test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_stereo_32000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 32000, 2, 113, 320, 320)); + Run("5f025d4f390982cc26b3d92fe02e3044", + "50e58502fb04421bf5b857dda4c96879", + 100, + test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcmu_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMU", 8000, 1, 0, 160, 160)); + Run("81a9d4c0bb72e9becc43aef124c981e9", + "8f9b8750bd80fe26b6cbf6659b89f0f9", + 50, + test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcma_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMA", 8000, 1, 8, 160, 160)); + Run("39611f798969053925a49dc06d08de29", + "6ad745e55aa48981bfc790d0eeef2dd1", + 50, + test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcmu_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMU", 8000, 2, 110, 160, 160)); + Run("437bec032fdc5cbaa0d5175430af7b18", + "60b6f25e8d1e74cb679cfe756dd9bca5", + 50, + test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcma_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMA", 8000, 2, 118, 160, 160)); + Run("a5c6d83c5b7cedbeff734238220a4b0c", + "92b282c83efd20e7eeef52ba40842cf7", + 50, + test::AcmReceiveTestOldApi::kStereoOutput); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_Ilbc_30ms DISABLED_Ilbc_30ms +#else +#define MAYBE_Ilbc_30ms Ilbc_30ms +#endif +#if defined(WEBRTC_CODEC_ILBC) +TEST_F(AcmSenderBitExactnessOldApi, MAYBE_Ilbc_30ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("ILBC", 8000, 1, 102, 240, 240)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "7b6ec10910debd9af08011d3ed5249f7", + "7b6ec10910debd9af08011d3ed5249f7", "android_arm32_audio", + "android_arm64_audio", "android_arm64_clang_audio"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "cfae2e9f6aba96e145f2bcdd5050ce78", + "cfae2e9f6aba96e145f2bcdd5050ce78", "android_arm32_payload", + "android_arm64_payload", "android_arm64_clang_payload"), + 33, test::AcmReceiveTestOldApi::kMonoOutput); +} +#endif + +#if defined(WEBRTC_ANDROID) +#define MAYBE_G722_20ms DISABLED_G722_20ms +#else +#define MAYBE_G722_20ms G722_20ms +#endif +TEST_F(AcmSenderBitExactnessOldApi, MAYBE_G722_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("G722", 16000, 1, 9, 320, 160)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "e99c89be49a46325d03c0d990c292d68", + "e99c89be49a46325d03c0d990c292d68", "android_arm32_audio", + "android_arm64_audio", "android_arm64_clang_audio"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "fc68a87e1380614e658087cb35d5ca10", + "fc68a87e1380614e658087cb35d5ca10", "android_arm32_payload", + "android_arm64_payload", "android_arm64_clang_payload"), + 50, test::AcmReceiveTestOldApi::kMonoOutput); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_G722_stereo_20ms DISABLED_G722_stereo_20ms +#else +#define MAYBE_G722_stereo_20ms G722_stereo_20ms +#endif +TEST_F(AcmSenderBitExactnessOldApi, MAYBE_G722_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("G722", 16000, 2, 119, 320, 160)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "e280aed283e499d37091b481ca094807", + "e280aed283e499d37091b481ca094807", "android_arm32_audio", + "android_arm64_audio", "android_arm64_clang_audio"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "66516152eeaa1e650ad94ff85f668dac", + "66516152eeaa1e650ad94ff85f668dac", "android_arm32_payload", + "android_arm64_payload", "android_arm64_clang_payload"), + 50, test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Opus_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 2, 120, 960, 960)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "3e285b74510e62062fbd8142dacd16e9", + "3e285b74510e62062fbd8142dacd16e9", + "439e97ad1932c49923b5da029c17dd5e", + "038ec90f5f3fc2320f3090f8ecef6bb7", + "038ec90f5f3fc2320f3090f8ecef6bb7"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "78cf8f03157358acdc69f6835caa0d9b", + "78cf8f03157358acdc69f6835caa0d9b", + "ab88b1a049c36bdfeb7e8b057ef6982a", + "27fef7b799393347ec3b5694369a1c36", + "27fef7b799393347ec3b5694369a1c36"), + 50, test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessNewApi, MAYBE_OpusFromFormat_stereo_20ms) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}})); + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(*config, 120); + ASSERT_NO_FATAL_FAILURE(SetUpTestExternalEncoder(encoder.get(), 120)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "3e285b74510e62062fbd8142dacd16e9", + "3e285b74510e62062fbd8142dacd16e9", + "439e97ad1932c49923b5da029c17dd5e", + "038ec90f5f3fc2320f3090f8ecef6bb7", + "038ec90f5f3fc2320f3090f8ecef6bb7"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "78cf8f03157358acdc69f6835caa0d9b", + "78cf8f03157358acdc69f6835caa0d9b", + "ab88b1a049c36bdfeb7e8b057ef6982a", + "27fef7b799393347ec3b5694369a1c36", + "27fef7b799393347ec3b5694369a1c36"), + 50, test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Opus_stereo_20ms_voip) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 2, 120, 960, 960)); + // If not set, default will be kAudio in case of stereo. + EXPECT_EQ(0, send_test_->acm()->SetOpusApplication(kVoip)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "b0325df4e8104f04e03af23c0b75800e", + "b0325df4e8104f04e03af23c0b75800e", + "1c81121f5d9286a5a865d01dbab22ce8", + "11d547f89142e9ef03f37d7ca7f32379", + "11d547f89142e9ef03f37d7ca7f32379"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "4eab2259b6fe24c22dd242a113e0b3d9", + "4eab2259b6fe24c22dd242a113e0b3d9", + "839ea60399447268ee0f0262a50b75fd", + "1815fd5589cad0c6f6cf946c76b81aeb", + "1815fd5589cad0c6f6cf946c76b81aeb"), + 50, test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessNewApi, OpusFromFormat_stereo_20ms_voip) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}})); + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(*config, 120); + ASSERT_NO_FATAL_FAILURE(SetUpTestExternalEncoder(encoder.get(), 120)); + // If not set, default will be kAudio in case of stereo. + EXPECT_EQ(0, send_test_->acm()->SetOpusApplication(kVoip)); + Run(AcmReceiverBitExactnessOldApi::PlatformChecksum( + "b0325df4e8104f04e03af23c0b75800e", + "b0325df4e8104f04e03af23c0b75800e", + "1c81121f5d9286a5a865d01dbab22ce8", + "11d547f89142e9ef03f37d7ca7f32379", + "11d547f89142e9ef03f37d7ca7f32379"), + AcmReceiverBitExactnessOldApi::PlatformChecksum( + "4eab2259b6fe24c22dd242a113e0b3d9", + "4eab2259b6fe24c22dd242a113e0b3d9", + "839ea60399447268ee0f0262a50b75fd", + "1815fd5589cad0c6f6cf946c76b81aeb", + "1815fd5589cad0c6f6cf946c76b81aeb"), + 50, test::AcmReceiveTestOldApi::kStereoOutput); +} + +// This test is for verifying the SetBitRate function. The bitrate is changed at +// the beginning, and the number of generated bytes are checked. +class AcmSetBitRateTest : public ::testing::Test { + protected: + static const int kTestDurationMs = 1000; + + // Sets up the test::AcmSendTest object. Returns true on success, otherwise + // false. + bool SetUpSender() { + const std::string input_file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + // Note that |audio_source_| will loop forever. The test duration is set + // explicitly by |kTestDurationMs|. + audio_source_.reset(new test::InputAudioFile(input_file_name)); + static const int kSourceRateHz = 32000; + send_test_.reset(new test::AcmSendTestOldApi( + audio_source_.get(), kSourceRateHz, kTestDurationMs)); + return send_test_.get(); + } + + // Registers a send codec in the test::AcmSendTest object. Returns true on + // success, false on failure. + virtual bool RegisterSendCodec(const char* payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples, + int frame_size_rtp_timestamps) { + return send_test_->RegisterCodec(payload_name, sampling_freq_hz, channels, + payload_type, frame_size_samples); + } + + bool RegisterExternalSendCodec(AudioEncoder* external_speech_encoder, + int payload_type) { + return send_test_->RegisterExternalCodec(external_speech_encoder); + } + + void RunInner(int expected_total_bits) { + int nr_bytes = 0; + while (std::unique_ptr next_packet = + send_test_->NextPacket()) { + nr_bytes += rtc::checked_cast(next_packet->payload_length_bytes()); + } + EXPECT_EQ(expected_total_bits, nr_bytes * 8); + } + + void SetUpTest(const char* codec_name, + int codec_sample_rate_hz, + int channels, + int payload_type, + int codec_frame_size_samples, + int codec_frame_size_rtp_timestamps) { + ASSERT_TRUE(SetUpSender()); + ASSERT_TRUE(RegisterSendCodec(codec_name, codec_sample_rate_hz, channels, + payload_type, codec_frame_size_samples, + codec_frame_size_rtp_timestamps)); + } + + std::unique_ptr send_test_; + std::unique_ptr audio_source_; +}; + +class AcmSetBitRateOldApi : public AcmSetBitRateTest { + protected: + // Runs the test. SetUpSender() must have been called and a codec must be set + // up before calling this method. + void Run(int target_bitrate_bps, int expected_total_bits) { + ASSERT_TRUE(send_test_->acm()); + send_test_->acm()->SetBitRate(target_bitrate_bps); + RunInner(expected_total_bits); + } +}; + +class AcmSetBitRateNewApi : public AcmSetBitRateTest { + protected: + // Runs the test. SetUpSender() must have been called and a codec must be set + // up before calling this method. + void Run(int expected_total_bits) { RunInner(expected_total_bits); } +}; + +TEST_F(AcmSetBitRateOldApi, Opus_48khz_20ms_10kbps) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 1, 107, 960, 960)); +#if defined(WEBRTC_ANDROID) + Run(10000, 8640); +#else + Run(10000, 8680); +#endif // WEBRTC_ANDROID +} + +TEST_F(AcmSetBitRateNewApi, OpusFromFormat_48khz_20ms_10kbps) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"maxaveragebitrate", "10000"}})); + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(*config, 107); + ASSERT_TRUE(SetUpSender()); + ASSERT_TRUE(RegisterExternalSendCodec(encoder.get(), 107)); +#if defined(WEBRTC_ANDROID) + RunInner(8640); +#else + RunInner(8680); +#endif // WEBRTC_ANDROID +} + +TEST_F(AcmSetBitRateOldApi, Opus_48khz_20ms_50kbps) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 1, 107, 960, 960)); +#if defined(WEBRTC_ANDROID) + Run(50000, 45792); +#else + Run(50000, 45520); +#endif // WEBRTC_ANDROID +} + +TEST_F(AcmSetBitRateNewApi, OpusFromFormat_48khz_20ms_50kbps) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"maxaveragebitrate", "50000"}})); + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(*config, 107); + ASSERT_TRUE(SetUpSender()); + ASSERT_TRUE(RegisterExternalSendCodec(encoder.get(), 107)); +#if defined(WEBRTC_ANDROID) + RunInner(45792); +#else + RunInner(45520); +#endif // WEBRTC_ANDROID +} + +// The result on the Android platforms is inconsistent for this test case. +// On android_rel the result is different from android and android arm64 rel. +#if defined(WEBRTC_ANDROID) +#define MAYBE_Opus_48khz_20ms_100kbps DISABLED_Opus_48khz_20ms_100kbps +#define MAYBE_OpusFromFormat_48khz_20ms_100kbps \ + DISABLED_OpusFromFormat_48khz_20ms_100kbps +#else +#define MAYBE_Opus_48khz_20ms_100kbps Opus_48khz_20ms_100kbps +#define MAYBE_OpusFromFormat_48khz_20ms_100kbps \ + OpusFromFormat_48khz_20ms_100kbps +#endif +TEST_F(AcmSetBitRateOldApi, MAYBE_Opus_48khz_20ms_100kbps) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 1, 107, 960, 960)); + Run(100000, 100832); +} + +TEST_F(AcmSetBitRateNewApi, MAYBE_OpusFromFormat_48khz_20ms_100kbps) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"maxaveragebitrate", "100000"}})); + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(*config, 107); + ASSERT_TRUE(SetUpSender()); + ASSERT_TRUE(RegisterExternalSendCodec(encoder.get(), 107)); + RunInner(100832); +} + +// These next 2 tests ensure that the SetBitRate function has no effect on PCM +TEST_F(AcmSetBitRateOldApi, Pcm16_8khz_10ms_8kbps) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 1, 107, 80, 80)); + Run(8000, 128000); +} + +TEST_F(AcmSetBitRateOldApi, Pcm16_8khz_10ms_32kbps) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 1, 107, 80, 80)); + Run(32000, 128000); +} + +// This test is for verifying the SetBitRate function. The bitrate is changed +// in the middle, and the number of generated bytes are before and after the +// change are checked. +class AcmChangeBitRateOldApi : public AcmSetBitRateOldApi { + protected: + AcmChangeBitRateOldApi() : sampling_freq_hz_(0), frame_size_samples_(0) {} + + // Registers a send codec in the test::AcmSendTest object. Returns true on + // success, false on failure. + bool RegisterSendCodec(const char* payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples, + int frame_size_rtp_timestamps) override { + frame_size_samples_ = frame_size_samples; + sampling_freq_hz_ = sampling_freq_hz; + return AcmSetBitRateOldApi::RegisterSendCodec( + payload_name, sampling_freq_hz, channels, payload_type, + frame_size_samples, frame_size_rtp_timestamps); + } + + // Runs the test. SetUpSender() and RegisterSendCodec() must have been called + // before calling this method. + void Run(int target_bitrate_bps, + int expected_before_switch_bits, + int expected_after_switch_bits) { + ASSERT_TRUE(send_test_->acm()); + int nr_packets = + sampling_freq_hz_ * kTestDurationMs / (frame_size_samples_ * 1000); + int nr_bytes_before = 0, nr_bytes_after = 0; + int packet_counter = 0; + while (std::unique_ptr next_packet = + send_test_->NextPacket()) { + if (packet_counter == nr_packets / 2) + send_test_->acm()->SetBitRate(target_bitrate_bps); + if (packet_counter < nr_packets / 2) + nr_bytes_before += rtc::checked_cast( + next_packet->payload_length_bytes()); + else + nr_bytes_after += rtc::checked_cast( + next_packet->payload_length_bytes()); + packet_counter++; + } + EXPECT_EQ(expected_before_switch_bits, nr_bytes_before * 8); + EXPECT_EQ(expected_after_switch_bits, nr_bytes_after * 8); + } + + uint32_t sampling_freq_hz_; + uint32_t frame_size_samples_; +}; + +TEST_F(AcmChangeBitRateOldApi, Opus_48khz_20ms_10kbps_2) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 1, 107, 960, 960)); +#if defined(WEBRTC_ANDROID) + Run(10000, 29512, 4800); +#else + Run(10000, 32200, 5368); +#endif // WEBRTC_ANDROID +} + +TEST_F(AcmChangeBitRateOldApi, Opus_48khz_20ms_50kbps_2) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 1, 107, 960, 960)); +#if defined(WEBRTC_ANDROID) + Run(50000, 29512, 23304); +#else + Run(50000, 32200, 23920); +#endif // WEBRTC_ANDROID +} + +TEST_F(AcmChangeBitRateOldApi, Opus_48khz_20ms_100kbps_2) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 1, 107, 960, 960)); +#if defined(WEBRTC_ANDROID) + #if defined(WEBRTC_ARCH_ARM64) + Run(100000, 29512, 50440); + #else + Run(100000, 29512, 50496); + #endif // WEBRTC_ARCH_ARM64 +#else + Run(100000, 32200, 50448); +#endif // WEBRTC_ANDROID +} + +// These next 2 tests ensure that the SetBitRate function has no effect on PCM +TEST_F(AcmChangeBitRateOldApi, Pcm16_8khz_10ms_8kbps) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 1, 107, 80, 80)); + Run(8000, 64000, 64000); +} + +TEST_F(AcmChangeBitRateOldApi, Pcm16_8khz_10ms_32kbps) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 1, 107, 80, 80)); + Run(32000, 64000, 64000); +} + +TEST_F(AcmSenderBitExactnessOldApi, External_Pcmu_20ms) { + CodecInst codec_inst; + codec_inst.channels = 1; + codec_inst.pacsize = 160; + codec_inst.pltype = 0; + AudioEncoderPcmU encoder(codec_inst); + MockAudioEncoder mock_encoder; + // Set expectations on the mock encoder and also delegate the calls to the + // real encoder. + EXPECT_CALL(mock_encoder, Die()); + EXPECT_CALL(mock_encoder, SampleRateHz()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::SampleRateHz)); + EXPECT_CALL(mock_encoder, NumChannels()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::NumChannels)); + EXPECT_CALL(mock_encoder, RtpTimestampRateHz()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::RtpTimestampRateHz)); + EXPECT_CALL(mock_encoder, Num10MsFramesInNextPacket()) + .Times(AtLeast(1)) + .WillRepeatedly( + Invoke(&encoder, &AudioEncoderPcmU::Num10MsFramesInNextPacket)); + EXPECT_CALL(mock_encoder, GetTargetBitrate()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::GetTargetBitrate)); + EXPECT_CALL(mock_encoder, EncodeImpl(_, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, + static_cast< + AudioEncoder::EncodedInfo(AudioEncoder::*)( + uint32_t, + rtc::ArrayView, + rtc::Buffer*)>(&AudioEncoderPcmU::Encode))); + EXPECT_CALL(mock_encoder, SetFec(_)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::SetFec)); + ASSERT_NO_FATAL_FAILURE( + SetUpTestExternalEncoder(&mock_encoder, codec_inst.pltype)); + Run("81a9d4c0bb72e9becc43aef124c981e9", "8f9b8750bd80fe26b6cbf6659b89f0f9", + 50, test::AcmReceiveTestOldApi::kMonoOutput); +} + +// This test fixture is implemented to run ACM and change the desired output +// frequency during the call. The input packets are simply PCM16b-wb encoded +// payloads with a constant value of |kSampleValue|. The test fixture itself +// acts as PacketSource in between the receive test class and the constant- +// payload packet source class. The output is both written to file, and analyzed +// in this test fixture. +class AcmSwitchingOutputFrequencyOldApi : public ::testing::Test, + public test::PacketSource, + public test::AudioSink { + protected: + static const size_t kTestNumPackets = 50; + static const int kEncodedSampleRateHz = 16000; + static const size_t kPayloadLenSamples = 30 * kEncodedSampleRateHz / 1000; + static const int kPayloadType = 108; // Default payload type for PCM16b-wb. + + AcmSwitchingOutputFrequencyOldApi() + : first_output_(true), + num_packets_(0), + packet_source_(kPayloadLenSamples, + kSampleValue, + kEncodedSampleRateHz, + kPayloadType), + output_freq_2_(0), + has_toggled_(false) {} + + void Run(int output_freq_1, int output_freq_2, int toggle_period_ms) { + // Set up the receiver used to decode the packets and verify the decoded + // output. + const std::string output_file_name = + webrtc::test::OutputPath() + + ::testing::UnitTest::GetInstance() + ->current_test_info() + ->test_case_name() + + "_" + ::testing::UnitTest::GetInstance()->current_test_info()->name() + + "_output.pcm"; + test::OutputAudioFile output_file(output_file_name); + // Have the output audio sent both to file and to the WriteArray method in + // this class. + test::AudioSinkFork output(this, &output_file); + test::AcmReceiveTestToggleOutputFreqOldApi receive_test( + this, + &output, + output_freq_1, + output_freq_2, + toggle_period_ms, + test::AcmReceiveTestOldApi::kMonoOutput); + ASSERT_NO_FATAL_FAILURE(receive_test.RegisterDefaultCodecs()); + output_freq_2_ = output_freq_2; + + // This is where the actual test is executed. + receive_test.Run(); + + // Delete output file. + remove(output_file_name.c_str()); + } + + // Inherited from test::PacketSource. + std::unique_ptr NextPacket() override { + // Check if it is time to terminate the test. The packet source is of type + // ConstantPcmPacketSource, which is infinite, so we must end the test + // "manually". + if (num_packets_++ > kTestNumPackets) { + EXPECT_TRUE(has_toggled_); + return NULL; // Test ended. + } + + // Get the next packet from the source. + return packet_source_.NextPacket(); + } + + // Inherited from test::AudioSink. + bool WriteArray(const int16_t* audio, size_t num_samples) override { + // Skip checking the first output frame, since it has a number of zeros + // due to how NetEq is initialized. + if (first_output_) { + first_output_ = false; + return true; + } + for (size_t i = 0; i < num_samples; ++i) { + EXPECT_EQ(kSampleValue, audio[i]); + } + if (num_samples == + static_cast(output_freq_2_ / 100)) // Size of 10 ms frame. + has_toggled_ = true; + // The return value does not say if the values match the expectation, just + // that the method could process the samples. + return true; + } + + const int16_t kSampleValue = 1000; + bool first_output_; + size_t num_packets_; + test::ConstantPcmPacketSource packet_source_; + int output_freq_2_; + bool has_toggled_; +}; + +TEST_F(AcmSwitchingOutputFrequencyOldApi, TestWithoutToggling) { + Run(16000, 16000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle16KhzTo32Khz) { + Run(16000, 32000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle32KhzTo16Khz) { + Run(32000, 16000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle16KhzTo8Khz) { + Run(16000, 8000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle8KhzTo16Khz) { + Run(8000, 16000, 1000); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.cc new file mode 100644 index 0000000000..a506ead30e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/call_statistics.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace acm2 { + +void CallStatistics::DecodedByNetEq(AudioFrame::SpeechType speech_type, + bool muted) { + ++decoding_stat_.calls_to_neteq; + if (muted) { + ++decoding_stat_.decoded_muted_output; + } + switch (speech_type) { + case AudioFrame::kNormalSpeech: { + ++decoding_stat_.decoded_normal; + break; + } + case AudioFrame::kPLC: { + ++decoding_stat_.decoded_plc; + break; + } + case AudioFrame::kCNG: { + ++decoding_stat_.decoded_cng; + break; + } + case AudioFrame::kPLCCNG: { + ++decoding_stat_.decoded_plc_cng; + break; + } + case AudioFrame::kUndefined: { + // If the audio is decoded by NetEq, |kUndefined| is not an option. + RTC_NOTREACHED(); + } + } +} + +void CallStatistics::DecodedBySilenceGenerator() { + ++decoding_stat_.calls_to_silence_generator; +} + +const AudioDecodingCallStats& CallStatistics::GetDecodingStatistics() const { + return decoding_stat_; +} + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.h new file mode 100644 index 0000000000..9dd052f003 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_ +#define MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_ + +#include "common_types.h" // NOLINT(build/include) +#include "modules/include/module_common_types.h" + +// +// This class is for book keeping of calls to ACM. It is not useful to log API +// calls which are supposed to be called every 10ms, e.g. PlayoutData10Ms(), +// however, it is useful to know the number of such calls in a given time +// interval. The current implementation covers calls to PlayoutData10Ms() with +// detailed accounting of the decoded speech type. +// +// Thread Safety +// ============= +// Please note that this class in not thread safe. The class must be protected +// if different APIs are called from different threads. +// + +namespace webrtc { + +namespace acm2 { + +class CallStatistics { + public: + CallStatistics() {} + ~CallStatistics() {} + + // Call this method to indicate that NetEq engaged in decoding. |speech_type| + // is the audio-type according to NetEq, and |muted| indicates if the decoded + // frame was produced in muted state. + void DecodedByNetEq(AudioFrame::SpeechType speech_type, bool muted); + + // Call this method to indicate that a decoding call resulted in generating + // silence, i.e. call to NetEq is bypassed and the output audio is zero. + void DecodedBySilenceGenerator(); + + // Get statistics for decoding. The statistics include the number of calls to + // NetEq and silence generator, as well as the type of speech pulled of off + // NetEq, c.f. declaration of AudioDecodingCallStats for detailed description. + const AudioDecodingCallStats& GetDecodingStatistics() const; + + private: + // Reset the decoding statistics. + void ResetDecodingStatistics(); + + AudioDecodingCallStats decoding_stat_; +}; + +} // namespace acm2 + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics_unittest.cc new file mode 100644 index 0000000000..77c3863ea8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics_unittest.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/call_statistics.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace acm2 { + +TEST(CallStatisticsTest, InitializedZero) { + CallStatistics call_stats; + AudioDecodingCallStats stats; + + stats = call_stats.GetDecodingStatistics(); + EXPECT_EQ(0, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(0, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(0, stats.decoded_plc); + EXPECT_EQ(0, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); +} + +TEST(CallStatisticsTest, AllCalls) { + CallStatistics call_stats; + AudioDecodingCallStats stats; + + call_stats.DecodedBySilenceGenerator(); + call_stats.DecodedByNetEq(AudioFrame::kNormalSpeech, false); + call_stats.DecodedByNetEq(AudioFrame::kPLC, false); + call_stats.DecodedByNetEq(AudioFrame::kPLCCNG, true); // Let this be muted. + call_stats.DecodedByNetEq(AudioFrame::kCNG, false); + + stats = call_stats.GetDecodingStatistics(); + EXPECT_EQ(4, stats.calls_to_neteq); + EXPECT_EQ(1, stats.calls_to_silence_generator); + EXPECT_EQ(1, stats.decoded_normal); + EXPECT_EQ(1, stats.decoded_cng); + EXPECT_EQ(1, stats.decoded_plc); + EXPECT_EQ(1, stats.decoded_plc_cng); + EXPECT_EQ(1, stats.decoded_muted_output); +} + +} // namespace acm2 + +} // namespace webrtc + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.cc new file mode 100644 index 0000000000..a101d3debb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.cc @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/codec_manager.h" + +#include "rtc_base/checks.h" +//#include "rtc_base/format_macros.h" +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "rtc_base/logging.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace acm2 { + +namespace { + +// Check if the given codec is a valid to be registered as send codec. +int IsValidSendCodec(const CodecInst& send_codec) { + if ((send_codec.channels != 1) && (send_codec.channels != 2)) { + RTC_LOG(LS_ERROR) << "Wrong number of channels (" << send_codec.channels + << "), only mono and stereo are supported)"; + return -1; + } + + auto maybe_codec_id = RentACodec::CodecIdByInst(send_codec); + if (!maybe_codec_id) { + RTC_LOG(LS_ERROR) << "Invalid codec setting for the send codec."; + return -1; + } + + // Telephone-event cannot be a send codec. + if (!STR_CASE_CMP(send_codec.plname, "telephone-event")) { + RTC_LOG(LS_ERROR) << "telephone-event cannot be a send codec"; + return -1; + } + + if (!RentACodec::IsSupportedNumChannels(*maybe_codec_id, send_codec.channels) + .value_or(false)) { + RTC_LOG(LS_ERROR) << send_codec.channels + << " number of channels not supported for " + << send_codec.plname << "."; + return -1; + } + return RentACodec::CodecIndexFromId(*maybe_codec_id).value_or(-1); +} + +bool IsOpus(const CodecInst& codec) { + return +#ifdef WEBRTC_CODEC_OPUS + !STR_CASE_CMP(codec.plname, "opus") || +#endif + false; +} + +} // namespace + +CodecManager::CodecManager() { + thread_checker_.DetachFromThread(); +} + +CodecManager::~CodecManager() = default; + +bool CodecManager::RegisterEncoder(const CodecInst& send_codec) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + int codec_id = IsValidSendCodec(send_codec); + + // Check for reported errors from function IsValidSendCodec(). + if (codec_id < 0) { + return false; + } + + switch (RentACodec::RegisterRedPayloadType( + &codec_stack_params_.red_payload_types, send_codec)) { + case RentACodec::RegistrationResult::kOk: + return true; + case RentACodec::RegistrationResult::kBadFreq: + RTC_LOG(LS_ERROR) + << "RegisterSendCodec() failed, invalid frequency for RED" + " registration"; + return false; + case RentACodec::RegistrationResult::kSkip: + break; + } + switch (RentACodec::RegisterCngPayloadType( + &codec_stack_params_.cng_payload_types, send_codec)) { + case RentACodec::RegistrationResult::kOk: + return true; + case RentACodec::RegistrationResult::kBadFreq: + RTC_LOG(LS_ERROR) + << "RegisterSendCodec() failed, invalid frequency for CNG" + " registration"; + return false; + case RentACodec::RegistrationResult::kSkip: + break; + } + + if (IsOpus(send_codec)) { + // VAD/DTX not supported. + codec_stack_params_.use_cng = false; + } + + send_codec_inst_ = send_codec; + recreate_encoder_ = true; // Caller must recreate it. + return true; +} + +CodecInst CodecManager::ForgeCodecInst( + const AudioEncoder* external_speech_encoder) { + CodecInst ci; + ci.channels = external_speech_encoder->NumChannels(); + ci.plfreq = external_speech_encoder->SampleRateHz(); + ci.pacsize = rtc::CheckedDivExact( + static_cast(external_speech_encoder->Max10MsFramesInAPacket() * + ci.plfreq), + 100); + ci.pltype = -1; // Not valid. + ci.rate = -1; // Not valid. + static const char kName[] = "external"; + memcpy(ci.plname, kName, sizeof(kName)); + return ci; +} + +bool CodecManager::SetCopyRed(bool enable) { + if (enable && codec_stack_params_.use_codec_fec) { + RTC_LOG(LS_WARNING) << "Codec internal FEC and RED cannot be co-enabled."; + return false; + } + if (enable && send_codec_inst_ && + codec_stack_params_.red_payload_types.count(send_codec_inst_->plfreq) < + 1) { + RTC_LOG(LS_WARNING) << "Cannot enable RED at " << send_codec_inst_->plfreq + << " Hz."; + return false; + } + codec_stack_params_.use_red = enable; + return true; +} + +bool CodecManager::SetVAD(bool enable, ACMVADMode mode) { + // Sanity check of the mode. + RTC_DCHECK(mode == VADNormal || mode == VADLowBitrate || mode == VADAggr || + mode == VADVeryAggr); + + // Check that the send codec is mono. We don't support VAD/DTX for stereo + // sending. + const bool stereo_send = + codec_stack_params_.speech_encoder + ? (codec_stack_params_.speech_encoder->NumChannels() != 1) + : false; + if (enable && stereo_send) { + RTC_LOG(LS_ERROR) << "VAD/DTX not supported for stereo sending"; + return false; + } + + // TODO(kwiberg): This doesn't protect Opus when injected as an external + // encoder. + if (send_codec_inst_ && IsOpus(*send_codec_inst_)) { + // VAD/DTX not supported, but don't fail. + enable = false; + } + + codec_stack_params_.use_cng = enable; + codec_stack_params_.vad_mode = mode; + return true; +} + +bool CodecManager::SetCodecFEC(bool enable_codec_fec) { + if (enable_codec_fec && codec_stack_params_.use_red) { + RTC_LOG(LS_WARNING) << "Codec internal FEC and RED cannot be co-enabled."; + return false; + } + + codec_stack_params_.use_codec_fec = enable_codec_fec; + return true; +} + +bool CodecManager::MakeEncoder(RentACodec* rac, AudioCodingModule* acm) { + RTC_DCHECK(rac); + RTC_DCHECK(acm); + + if (!recreate_encoder_) { + bool error = false; + // Try to re-use the speech encoder we've given to the ACM. + acm->ModifyEncoder([&](std::unique_ptr* encoder) { + if (!*encoder) { + // There is no existing encoder. + recreate_encoder_ = true; + return; + } + + // Extract the speech encoder from the ACM. + std::unique_ptr enc = std::move(*encoder); + while (true) { + auto sub_enc = enc->ReclaimContainedEncoders(); + if (sub_enc.empty()) { + break; + } + RTC_CHECK_EQ(1, sub_enc.size()); + + // Replace enc with its sub encoder. We need to put the sub encoder in + // a temporary first, since otherwise the old value of enc would be + // destroyed before the new value got assigned, which would be bad + // since the new value is a part of the old value. + auto tmp_enc = std::move(sub_enc[0]); + enc = std::move(tmp_enc); + } + + // Wrap it in a new encoder stack and put it back. + codec_stack_params_.speech_encoder = std::move(enc); + *encoder = rac->RentEncoderStack(&codec_stack_params_); + if (!*encoder) { + error = true; + } + }); + if (error) { + return false; + } + if (!recreate_encoder_) { + return true; + } + } + + if (!send_codec_inst_) { + // We don't have the information we need to create a new speech encoder. + // (This is not an error.) + return true; + } + + codec_stack_params_.speech_encoder = rac->RentEncoder(*send_codec_inst_); + auto stack = rac->RentEncoderStack(&codec_stack_params_); + if (!stack) { + return false; + } + acm->SetEncoder(std::move(stack)); + recreate_encoder_ = false; + return true; +} + +} // namespace acm2 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.h new file mode 100644 index 0000000000..7485426cea --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_CODEC_MANAGER_H_ +#define MODULES_AUDIO_CODING_ACM2_CODEC_MANAGER_H_ + +#include + +#include "api/optional.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +class AudioDecoder; +class AudioEncoder; + +namespace acm2 { + +class CodecManager final { + public: + CodecManager(); + ~CodecManager(); + + // Parses the given specification. On success, returns true and updates the + // stored CodecInst and stack parameters; on error, returns false. + bool RegisterEncoder(const CodecInst& send_codec); + + static CodecInst ForgeCodecInst(const AudioEncoder* external_speech_encoder); + + const CodecInst* GetCodecInst() const { + return send_codec_inst_ ? &*send_codec_inst_ : nullptr; + } + + void UnsetCodecInst() { send_codec_inst_ = rtc::nullopt; } + + const RentACodec::StackParameters* GetStackParams() const { + return &codec_stack_params_; + } + RentACodec::StackParameters* GetStackParams() { return &codec_stack_params_; } + + bool SetCopyRed(bool enable); + + bool SetVAD(bool enable, ACMVADMode mode); + + bool SetCodecFEC(bool enable_codec_fec); + + // Uses the provided Rent-A-Codec to create a new encoder stack, if we have a + // complete specification; if so, it is then passed to set_encoder. On error, + // returns false. + bool MakeEncoder(RentACodec* rac, AudioCodingModule* acm); + + private: + rtc::ThreadChecker thread_checker_; + rtc::Optional send_codec_inst_; + RentACodec::StackParameters codec_stack_params_; + bool recreate_encoder_ = true; // Need to recreate encoder? + + RTC_DISALLOW_COPY_AND_ASSIGN(CodecManager); +}; + +} // namespace acm2 +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_ACM2_CODEC_MANAGER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager_unittest.cc new file mode 100644 index 0000000000..e041b5e97a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager_unittest.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/acm2/codec_manager.h" +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "test/gtest.h" +#include "test/mock_audio_encoder.h" + +namespace webrtc { +namespace acm2 { + +using ::testing::Return; + +namespace { + +// Create a MockAudioEncoder with some reasonable default behavior. +std::unique_ptr CreateMockEncoder() { + auto enc = std::unique_ptr(new MockAudioEncoder); + EXPECT_CALL(*enc, SampleRateHz()).WillRepeatedly(Return(8000)); + EXPECT_CALL(*enc, NumChannels()).WillRepeatedly(Return(1)); + EXPECT_CALL(*enc, Max10MsFramesInAPacket()).WillRepeatedly(Return(1)); + EXPECT_CALL(*enc, Die()); + return enc; +} + +} // namespace + +TEST(CodecManagerTest, ExternalEncoderFec) { + auto enc0 = CreateMockEncoder(); + auto enc1 = CreateMockEncoder(); + auto enc2 = CreateMockEncoder(); + { + ::testing::InSequence s; + EXPECT_CALL(*enc0, SetFec(false)).WillOnce(Return(true)); + EXPECT_CALL(*enc1, SetFec(true)).WillOnce(Return(true)); + EXPECT_CALL(*enc2, SetFec(true)).WillOnce(Return(false)); + } + + CodecManager cm; + RentACodec rac; + + // use_codec_fec starts out false. + EXPECT_FALSE(cm.GetStackParams()->use_codec_fec); + cm.GetStackParams()->speech_encoder = std::move(enc0); + EXPECT_TRUE(rac.RentEncoderStack(cm.GetStackParams())); + EXPECT_FALSE(cm.GetStackParams()->use_codec_fec); + + // Set it to true. + EXPECT_EQ(true, cm.SetCodecFEC(true)); + EXPECT_TRUE(cm.GetStackParams()->use_codec_fec); + cm.GetStackParams()->speech_encoder = std::move(enc1); + EXPECT_TRUE(rac.RentEncoderStack(cm.GetStackParams())); + EXPECT_TRUE(cm.GetStackParams()->use_codec_fec); + + // Switch to a codec that doesn't support it. + cm.GetStackParams()->speech_encoder = std::move(enc2); + EXPECT_TRUE(rac.RentEncoderStack(cm.GetStackParams())); + EXPECT_FALSE(cm.GetStackParams()->use_codec_fec); +} + +} // namespace acm2 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.cc new file mode 100644 index 0000000000..78db38dad6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.cc @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/rent_a_codec.h" + +#include +#include + +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "rtc_base/logging.h" +#include "modules/audio_coding/codecs/g722/audio_encoder_g722.h" +#ifdef WEBRTC_CODEC_ILBC +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#endif +#ifdef WEBRTC_CODEC_ISACFX +#include "modules/audio_coding/codecs/isac/fix/include/audio_decoder_isacfix.h" // nogncheck +#include "modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h" // nogncheck +#endif +#ifdef WEBRTC_CODEC_ISAC +#include "modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h" // nogncheck +#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h" // nogncheck +#endif +#ifdef WEBRTC_CODEC_OPUS +#include "modules/audio_coding/codecs/opus/audio_encoder_opus.h" +#endif +#include "modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h" +#ifdef WEBRTC_CODEC_RED +#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" +#endif +#include "modules/audio_coding/acm2/acm_codec_database.h" + +#if defined(WEBRTC_CODEC_ISACFX) || defined(WEBRTC_CODEC_ISAC) +#include "modules/audio_coding/codecs/isac/locked_bandwidth_info.h" +#endif + +namespace webrtc { +namespace acm2 { + +rtc::Optional RentACodec::CodecIdByParams( + const char* payload_name, + int sampling_freq_hz, + size_t channels) { + return CodecIdFromIndex( + ACMCodecDB::CodecId(payload_name, sampling_freq_hz, channels)); +} + +rtc::Optional RentACodec::CodecInstById(CodecId codec_id) { + rtc::Optional mi = CodecIndexFromId(codec_id); + return mi ? rtc::Optional(Database()[*mi]) + : rtc::nullopt; +} + +rtc::Optional RentACodec::CodecIdByInst( + const CodecInst& codec_inst) { + return CodecIdFromIndex(ACMCodecDB::CodecNumber(codec_inst)); +} + +rtc::Optional RentACodec::CodecInstByParams(const char* payload_name, + int sampling_freq_hz, + size_t channels) { + rtc::Optional codec_id = + CodecIdByParams(payload_name, sampling_freq_hz, channels); + if (!codec_id) + return rtc::nullopt; + rtc::Optional ci = CodecInstById(*codec_id); + RTC_DCHECK(ci); + + // Keep the number of channels from the function call. For most codecs it + // will be the same value as in default codec settings, but not for all. + ci->channels = channels; + + return ci; +} + +bool RentACodec::IsCodecValid(const CodecInst& codec_inst) { + return ACMCodecDB::CodecNumber(codec_inst) >= 0; +} + +rtc::Optional RentACodec::IsSupportedNumChannels(CodecId codec_id, + size_t num_channels) { + auto i = CodecIndexFromId(codec_id); + return i ? rtc::Optional( + ACMCodecDB::codec_settings_[*i].channel_support >= + num_channels) + : rtc::nullopt; +} + +rtc::ArrayView RentACodec::Database() { + return rtc::ArrayView(ACMCodecDB::database_, + NumberOfCodecs()); +} + +rtc::Optional RentACodec::NetEqDecoderFromCodecId( + CodecId codec_id, + size_t num_channels) { + rtc::Optional i = CodecIndexFromId(codec_id); + if (!i) + return rtc::nullopt; + const NetEqDecoder ned = ACMCodecDB::neteq_decoders_[*i]; + return (ned == NetEqDecoder::kDecoderOpus && num_channels == 2) + ? NetEqDecoder::kDecoderOpus_2ch + : ned; +} + +RentACodec::RegistrationResult RentACodec::RegisterCngPayloadType( + std::map* pt_map, + const CodecInst& codec_inst) { + if (STR_CASE_CMP(codec_inst.plname, "CN") != 0) + return RegistrationResult::kSkip; + switch (codec_inst.plfreq) { + case 8000: + case 16000: + case 32000: + case 48000: + (*pt_map)[codec_inst.plfreq] = codec_inst.pltype; + return RegistrationResult::kOk; + default: + return RegistrationResult::kBadFreq; + } +} + +RentACodec::RegistrationResult RentACodec::RegisterRedPayloadType( + std::map* pt_map, + const CodecInst& codec_inst) { + if (STR_CASE_CMP(codec_inst.plname, "RED") != 0) + return RegistrationResult::kSkip; + switch (codec_inst.plfreq) { + case 8000: + (*pt_map)[codec_inst.plfreq] = codec_inst.pltype; + return RegistrationResult::kOk; + default: + return RegistrationResult::kBadFreq; + } +} + +namespace { + +// Returns a new speech encoder, or null on error. +// TODO(kwiberg): Don't handle errors here (bug 5033) +std::unique_ptr CreateEncoder( + const CodecInst& speech_inst, + const rtc::scoped_refptr& bwinfo) { +#if defined(WEBRTC_CODEC_ISACFX) + if (STR_CASE_CMP(speech_inst.plname, "isac") == 0) + return std::unique_ptr( + new AudioEncoderIsacFixImpl(speech_inst, bwinfo)); +#endif +#if defined(WEBRTC_CODEC_ISAC) + if (STR_CASE_CMP(speech_inst.plname, "isac") == 0) + return std::unique_ptr( + new AudioEncoderIsacFloatImpl(speech_inst, bwinfo)); +#endif +#ifdef WEBRTC_CODEC_OPUS + if (STR_CASE_CMP(speech_inst.plname, "opus") == 0) + return std::unique_ptr(new AudioEncoderOpusImpl(speech_inst)); +#endif + if (STR_CASE_CMP(speech_inst.plname, "pcmu") == 0) + return std::unique_ptr(new AudioEncoderPcmU(speech_inst)); + if (STR_CASE_CMP(speech_inst.plname, "pcma") == 0) + return std::unique_ptr(new AudioEncoderPcmA(speech_inst)); + if (STR_CASE_CMP(speech_inst.plname, "l16") == 0) + return std::unique_ptr(new AudioEncoderPcm16B(speech_inst)); +#ifdef WEBRTC_CODEC_ILBC + if (STR_CASE_CMP(speech_inst.plname, "ilbc") == 0) + return std::unique_ptr(new AudioEncoderIlbcImpl(speech_inst)); +#endif + if (STR_CASE_CMP(speech_inst.plname, "g722") == 0) + return std::unique_ptr(new AudioEncoderG722Impl(speech_inst)); + RTC_LOG_F(LS_ERROR) << "Could not create encoder of type " + << speech_inst.plname; + return std::unique_ptr(); +} + +std::unique_ptr CreateRedEncoder( + std::unique_ptr encoder, + int red_payload_type) { +#ifdef WEBRTC_CODEC_RED + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type; + config.speech_encoder = std::move(encoder); + return std::unique_ptr( + new AudioEncoderCopyRed(std::move(config))); +#else + return std::unique_ptr(); +#endif +} + +std::unique_ptr CreateCngEncoder( + std::unique_ptr encoder, + int payload_type, + ACMVADMode vad_mode) { + AudioEncoderCng::Config config; + config.num_channels = encoder->NumChannels(); + config.payload_type = payload_type; + config.speech_encoder = std::move(encoder); + switch (vad_mode) { + case VADNormal: + config.vad_mode = Vad::kVadNormal; + break; + case VADLowBitrate: + config.vad_mode = Vad::kVadLowBitrate; + break; + case VADAggr: + config.vad_mode = Vad::kVadAggressive; + break; + case VADVeryAggr: + config.vad_mode = Vad::kVadVeryAggressive; + break; + default: + FATAL(); + } + return std::unique_ptr(new AudioEncoderCng(std::move(config))); +} + +std::unique_ptr CreateIsacDecoder( + int sample_rate_hz, + const rtc::scoped_refptr& bwinfo) { +#if defined(WEBRTC_CODEC_ISACFX) + return std::unique_ptr( + new AudioDecoderIsacFixImpl(sample_rate_hz, bwinfo)); +#elif defined(WEBRTC_CODEC_ISAC) + return std::unique_ptr( + new AudioDecoderIsacFloatImpl(sample_rate_hz, bwinfo)); +#else + FATAL() << "iSAC is not supported."; + return std::unique_ptr(); +#endif +} + +} // namespace + +RentACodec::RentACodec() { +#if defined(WEBRTC_CODEC_ISACFX) || defined(WEBRTC_CODEC_ISAC) + isac_bandwidth_info_ = new LockedIsacBandwidthInfo; +#endif +} +RentACodec::~RentACodec() = default; + +std::unique_ptr RentACodec::RentEncoder( + const CodecInst& codec_inst) { + return CreateEncoder(codec_inst, isac_bandwidth_info_); +} + +RentACodec::StackParameters::StackParameters() { + // Register the default payload types for RED and CNG. + for (const CodecInst& ci : RentACodec::Database()) { + RentACodec::RegisterCngPayloadType(&cng_payload_types, ci); + RentACodec::RegisterRedPayloadType(&red_payload_types, ci); + } +} + +RentACodec::StackParameters::~StackParameters() = default; + +std::unique_ptr RentACodec::RentEncoderStack( + StackParameters* param) { + if (!param->speech_encoder) + return nullptr; + + if (param->use_codec_fec) { + // Switch FEC on. On failure, remember that FEC is off. + if (!param->speech_encoder->SetFec(true)) + param->use_codec_fec = false; + } else { + // Switch FEC off. This shouldn't fail. + const bool success = param->speech_encoder->SetFec(false); + RTC_DCHECK(success); + } + + auto pt = [¶m](const std::map& m) { + auto it = m.find(param->speech_encoder->SampleRateHz()); + return it == m.end() ? rtc::nullopt + : rtc::Optional(it->second); + }; + auto cng_pt = pt(param->cng_payload_types); + param->use_cng = + param->use_cng && cng_pt && param->speech_encoder->NumChannels() == 1; + auto red_pt = pt(param->red_payload_types); + param->use_red = param->use_red && red_pt; + + if (param->use_cng || param->use_red) { + // The RED and CNG encoders need to be in sync with the speech encoder, so + // reset the latter to ensure its buffer is empty. + param->speech_encoder->Reset(); + } + std::unique_ptr encoder_stack = + std::move(param->speech_encoder); + if (param->use_red) { + encoder_stack = CreateRedEncoder(std::move(encoder_stack), *red_pt); + } + if (param->use_cng) { + encoder_stack = + CreateCngEncoder(std::move(encoder_stack), *cng_pt, param->vad_mode); + } + return encoder_stack; +} + +std::unique_ptr RentACodec::RentIsacDecoder(int sample_rate_hz) { + return CreateIsacDecoder(sample_rate_hz, isac_bandwidth_info_); +} + +} // namespace acm2 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.h new file mode 100644 index 0000000000..f8fac4c01b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_RENT_A_CODEC_H_ +#define MODULES_AUDIO_CODING_ACM2_RENT_A_CODEC_H_ + +#include +#include +#include + +#include "api/array_view.h" +#include "api/audio_codecs/audio_decoder.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/optional.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/neteq/neteq_decoder_enum.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +struct CodecInst; +class LockedIsacBandwidthInfo; + +namespace acm2 { + +class RentACodec { + public: + enum class CodecId { +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) + kISAC, +#endif +#ifdef WEBRTC_CODEC_ISAC + kISACSWB, +#endif + // Mono + kPCM16B, + kPCM16Bwb, + kPCM16Bswb32kHz, + // Stereo + kPCM16B_2ch, + kPCM16Bwb_2ch, + kPCM16Bswb32kHz_2ch, + // Mono + kPCMU, + kPCMA, + // Stereo + kPCMU_2ch, + kPCMA_2ch, +#ifdef WEBRTC_CODEC_ILBC + kILBC, +#endif + kG722, // Mono + kG722_2ch, // Stereo +#ifdef WEBRTC_CODEC_OPUS + kOpus, // Mono and stereo +#endif + kCNNB, + kCNWB, + kCNSWB, +#ifdef ENABLE_48000_HZ + kCNFB, +#endif + kAVT, + kAVT16kHz, + kAVT32kHz, + kAVT48kHz, +#ifdef WEBRTC_CODEC_RED + kRED, +#endif + kNumCodecs, // Implementation detail. Don't use. + +// Set unsupported codecs to -1. +#if !defined(WEBRTC_CODEC_ISAC) && !defined(WEBRTC_CODEC_ISACFX) + kISAC = -1, +#endif +#ifndef WEBRTC_CODEC_ISAC + kISACSWB = -1, +#endif + // 48 kHz not supported, always set to -1. + kPCM16Bswb48kHz = -1, +#ifndef WEBRTC_CODEC_ILBC + kILBC = -1, +#endif +#ifndef WEBRTC_CODEC_OPUS + kOpus = -1, // Mono and stereo +#endif +#ifndef WEBRTC_CODEC_RED + kRED = -1, +#endif +#ifndef ENABLE_48000_HZ + kCNFB = -1, +#endif + + kNone = -1 + }; + + static inline size_t NumberOfCodecs() { + return static_cast(CodecId::kNumCodecs); + } + + static inline rtc::Optional CodecIndexFromId(CodecId codec_id) { + const int i = static_cast(codec_id); + return i >= 0 && i < static_cast(NumberOfCodecs()) + ? rtc::Optional(i) + : rtc::nullopt; + } + + static inline rtc::Optional CodecIdFromIndex(int codec_index) { + return static_cast(codec_index) < NumberOfCodecs() + ? rtc::Optional( + static_cast(codec_index)) + : rtc::nullopt; + } + + static rtc::Optional CodecIdByParams(const char* payload_name, + int sampling_freq_hz, + size_t channels); + static rtc::Optional CodecInstById(CodecId codec_id); + static rtc::Optional CodecIdByInst(const CodecInst& codec_inst); + static rtc::Optional CodecInstByParams(const char* payload_name, + int sampling_freq_hz, + size_t channels); + static bool IsCodecValid(const CodecInst& codec_inst); + + static inline bool IsPayloadTypeValid(int payload_type) { + return payload_type >= 0 && payload_type <= 127; + } + + static rtc::ArrayView Database(); + + static rtc::Optional IsSupportedNumChannels(CodecId codec_id, + size_t num_channels); + + static rtc::Optional NetEqDecoderFromCodecId( + CodecId codec_id, + size_t num_channels); + + // Parse codec_inst and extract payload types. If the given CodecInst was for + // the wrong sort of codec, return kSkip; otherwise, if the rate was illegal, + // return kBadFreq; otherwise, update the given RTP timestamp rate (Hz) -> + // payload type map and return kOk. + enum class RegistrationResult { kOk, kSkip, kBadFreq }; + static RegistrationResult RegisterCngPayloadType(std::map* pt_map, + const CodecInst& codec_inst); + static RegistrationResult RegisterRedPayloadType(std::map* pt_map, + const CodecInst& codec_inst); + + RentACodec(); + ~RentACodec(); + + // Creates and returns an audio encoder built to the given specification. + // Returns null in case of error. + std::unique_ptr RentEncoder(const CodecInst& codec_inst); + + struct StackParameters { + StackParameters(); + ~StackParameters(); + + std::unique_ptr speech_encoder; + + bool use_codec_fec = false; + bool use_red = false; + bool use_cng = false; + ACMVADMode vad_mode = VADNormal; + + // Maps from RTP timestamp rate (in Hz) to payload type. + std::map cng_payload_types; + std::map red_payload_types; + }; + + // Creates and returns an audio encoder stack constructed to the given + // specification. If the specification isn't compatible with the encoder, it + // will be changed to match (things will be switched off). The speech encoder + // will be stolen. If the specification isn't complete, returns nullptr. + std::unique_ptr RentEncoderStack(StackParameters* param); + + // Creates and returns an iSAC decoder. + std::unique_ptr RentIsacDecoder(int sample_rate_hz); + + private: + std::unique_ptr speech_encoder_; + std::unique_ptr cng_encoder_; + std::unique_ptr red_encoder_; + rtc::scoped_refptr isac_bandwidth_info_; + + RTC_DISALLOW_COPY_AND_ASSIGN(RentACodec); +}; + +} // namespace acm2 +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_RENT_A_CODEC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec_unittest.cc new file mode 100644 index 0000000000..9eded20d23 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec_unittest.cc @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "rtc_base/arraysize.h" +#include "test/gtest.h" +#include "test/mock_audio_encoder.h" + +namespace webrtc { +namespace acm2 { + +using ::testing::Return; + +namespace { + +const int kDataLengthSamples = 80; +const int kPacketSizeSamples = 2 * kDataLengthSamples; +const int16_t kZeroData[kDataLengthSamples] = {0}; +const CodecInst kDefaultCodecInst = {0, "pcmu", 8000, kPacketSizeSamples, + 1, 64000}; +const int kCngPt = 13; + +class Marker final { + public: + MOCK_METHOD1(Mark, void(std::string desc)); +}; + +} // namespace + +class RentACodecTestF : public ::testing::Test { + protected: + void CreateCodec() { + auto speech_encoder = rent_a_codec_.RentEncoder(kDefaultCodecInst); + ASSERT_TRUE(speech_encoder); + RentACodec::StackParameters param; + param.use_cng = true; + param.speech_encoder = std::move(speech_encoder); + encoder_ = rent_a_codec_.RentEncoderStack(¶m); + } + + void EncodeAndVerify(size_t expected_out_length, + uint32_t expected_timestamp, + int expected_payload_type, + int expected_send_even_if_empty) { + rtc::Buffer out; + AudioEncoder::EncodedInfo encoded_info; + encoded_info = + encoder_->Encode(timestamp_, kZeroData, &out); + timestamp_ += kDataLengthSamples; + EXPECT_TRUE(encoded_info.redundant.empty()); + EXPECT_EQ(expected_out_length, encoded_info.encoded_bytes); + EXPECT_EQ(expected_timestamp, encoded_info.encoded_timestamp); + if (expected_payload_type >= 0) + EXPECT_EQ(expected_payload_type, encoded_info.payload_type); + if (expected_send_even_if_empty >= 0) + EXPECT_EQ(static_cast(expected_send_even_if_empty), + encoded_info.send_even_if_empty); + } + + RentACodec rent_a_codec_; + std::unique_ptr encoder_; + uint32_t timestamp_ = 0; +}; + +// This test verifies that CNG frames are delivered as expected. Since the frame +// size is set to 20 ms, we expect the first encode call to produce no output +// (which is signaled as 0 bytes output of type kNoEncoding). The next encode +// call should produce one SID frame of 9 bytes. The third call should not +// result in any output (just like the first one). The fourth and final encode +// call should produce an "empty frame", which is like no output, but with +// AudioEncoder::EncodedInfo::send_even_if_empty set to true. (The reason to +// produce an empty frame is to drive sending of DTMF packets in the RTP/RTCP +// module.) +TEST_F(RentACodecTestF, VerifyCngFrames) { + CreateCodec(); + uint32_t expected_timestamp = timestamp_; + // Verify no frame. + { + SCOPED_TRACE("First encoding"); + EncodeAndVerify(0, expected_timestamp, -1, -1); + } + + // Verify SID frame delivered. + { + SCOPED_TRACE("Second encoding"); + EncodeAndVerify(9, expected_timestamp, kCngPt, 1); + } + + // Verify no frame. + { + SCOPED_TRACE("Third encoding"); + EncodeAndVerify(0, expected_timestamp, -1, -1); + } + + // Verify NoEncoding. + expected_timestamp += 2 * kDataLengthSamples; + { + SCOPED_TRACE("Fourth encoding"); + EncodeAndVerify(0, expected_timestamp, kCngPt, 1); + } +} + +TEST(RentACodecTest, ExternalEncoder) { + const int kSampleRateHz = 8000; + auto* external_encoder = new MockAudioEncoder; + EXPECT_CALL(*external_encoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(*external_encoder, NumChannels()).WillRepeatedly(Return(1)); + EXPECT_CALL(*external_encoder, SetFec(false)).WillRepeatedly(Return(true)); + + RentACodec rac; + RentACodec::StackParameters param; + param.speech_encoder = std::unique_ptr(external_encoder); + std::unique_ptr encoder_stack = rac.RentEncoderStack(¶m); + EXPECT_EQ(external_encoder, encoder_stack.get()); + const int kPacketSizeSamples = kSampleRateHz / 100; + int16_t audio[kPacketSizeSamples] = {0}; + rtc::Buffer encoded; + AudioEncoder::EncodedInfo info; + + Marker marker; + { + ::testing::InSequence s; + info.encoded_timestamp = 0; + EXPECT_CALL( + *external_encoder, + EncodeImpl(0, rtc::ArrayView(audio), &encoded)) + .WillOnce(Return(info)); + EXPECT_CALL(marker, Mark("A")); + EXPECT_CALL(marker, Mark("B")); + EXPECT_CALL(*external_encoder, Die()); + EXPECT_CALL(marker, Mark("C")); + } + + info = encoder_stack->Encode(0, audio, &encoded); + EXPECT_EQ(0u, info.encoded_timestamp); + marker.Mark("A"); + + // Change to internal encoder. + CodecInst codec_inst = kDefaultCodecInst; + codec_inst.pacsize = kPacketSizeSamples; + param.speech_encoder = rac.RentEncoder(codec_inst); + ASSERT_TRUE(param.speech_encoder); + AudioEncoder* enc = param.speech_encoder.get(); + std::unique_ptr stack = rac.RentEncoderStack(¶m); + EXPECT_EQ(enc, stack.get()); + + // Don't expect any more calls to the external encoder. + info = stack->Encode(1, audio, &encoded); + marker.Mark("B"); + encoder_stack.reset(); + marker.Mark("C"); +} + +// Verify that the speech encoder's Reset method is called when CNG or RED +// (or both) are switched on, but not when they're switched off. +void TestCngAndRedResetSpeechEncoder(bool use_cng, bool use_red) { + auto make_enc = [] { + auto speech_encoder = + std::unique_ptr(new MockAudioEncoder); + EXPECT_CALL(*speech_encoder, NumChannels()).WillRepeatedly(Return(1)); + EXPECT_CALL(*speech_encoder, Max10MsFramesInAPacket()) + .WillRepeatedly(Return(2)); + EXPECT_CALL(*speech_encoder, SampleRateHz()).WillRepeatedly(Return(8000)); + EXPECT_CALL(*speech_encoder, SetFec(false)).WillRepeatedly(Return(true)); + return speech_encoder; + }; + auto speech_encoder1 = make_enc(); + auto speech_encoder2 = make_enc(); + Marker marker; + { + ::testing::InSequence s; + EXPECT_CALL(marker, Mark("disabled")); + EXPECT_CALL(*speech_encoder1, Die()); + EXPECT_CALL(marker, Mark("enabled")); + if (use_cng || use_red) + EXPECT_CALL(*speech_encoder2, Reset()); + EXPECT_CALL(*speech_encoder2, Die()); + } + + RentACodec::StackParameters param1, param2; + param1.speech_encoder = std::move(speech_encoder1); + param2.speech_encoder = std::move(speech_encoder2); + param2.use_cng = use_cng; + param2.use_red = use_red; + marker.Mark("disabled"); + RentACodec rac; + rac.RentEncoderStack(¶m1); + marker.Mark("enabled"); + rac.RentEncoderStack(¶m2); +} + +TEST(RentACodecTest, CngResetsSpeechEncoder) { + TestCngAndRedResetSpeechEncoder(true, false); +} + +TEST(RentACodecTest, RedResetsSpeechEncoder) { + TestCngAndRedResetSpeechEncoder(false, true); +} + +TEST(RentACodecTest, CngAndRedResetsSpeechEncoder) { + TestCngAndRedResetSpeechEncoder(true, true); +} + +TEST(RentACodecTest, NoCngAndRedNoSpeechEncoderReset) { + TestCngAndRedResetSpeechEncoder(false, false); +} + +TEST(RentACodecTest, RentEncoderError) { + const CodecInst codec_inst = { + 0, "Robert'); DROP TABLE Students;", 8000, 160, 1, 64000}; + RentACodec rent_a_codec; + EXPECT_FALSE(rent_a_codec.RentEncoder(codec_inst)); +} + +TEST(RentACodecTest, RentEncoderStackWithoutSpeechEncoder) { + RentACodec::StackParameters sp; + EXPECT_EQ(nullptr, sp.speech_encoder); + EXPECT_EQ(nullptr, RentACodec().RentEncoderStack(&sp)); +} + +} // namespace acm2 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding.gni b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding.gni new file mode 100644 index 0000000000..9b0aba856a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding.gni @@ -0,0 +1,33 @@ +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +audio_codec_defines = [] +if (rtc_include_ilbc) { + audio_codec_defines += [ "WEBRTC_CODEC_ILBC" ] +} +if (rtc_include_opus) { + audio_codec_defines += [ "WEBRTC_CODEC_OPUS" ] +} +if (rtc_opus_support_120ms_ptime) { + audio_codec_defines += [ "WEBRTC_OPUS_SUPPORT_120MS_PTIME=1" ] +} else { + audio_codec_defines += [ "WEBRTC_OPUS_SUPPORT_120MS_PTIME=0" ] +} +if (current_cpu == "arm") { + audio_codec_defines += [ "WEBRTC_CODEC_ISACFX" ] +} else { + audio_codec_defines += [ "WEBRTC_CODEC_ISAC" ] +} +if (!build_with_mozilla && !build_with_chromium) { + audio_codec_defines += [ "WEBRTC_CODEC_RED" ] +} + +audio_coding_defines = audio_codec_defines +neteq_defines = audio_codec_defines diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_gn/moz.build new file mode 100644 index 0000000000..e32712660f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_gn/moz.build @@ -0,0 +1,403 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_CODEC_ILBC"] = True +DEFINES["WEBRTC_CODEC_OPUS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_OPUS_SUPPORT_120MS_PTIME"] = "1" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/include/", + "/third_party/libwebrtc/webrtc/modules/include/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/codec_manager.cc" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_receiver.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_resampler.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/audio_coding_module.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/call_statistics.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_CODEC_ISACFX"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/" + ] + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "ppc64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +Library("audio_coding_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build new file mode 100644 index 0000000000..fc1cd25686 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build @@ -0,0 +1,187 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +Library("audio_coding_module_typedefs_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_format_conversion_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_format_conversion_gn/moz.build new file mode 100644 index 0000000000..852f5a8e07 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_format_conversion_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_format_conversion_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc new file mode 100644 index 0000000000..16fd2a1b9a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +AudioEncoderRuntimeConfig::AudioEncoderRuntimeConfig() = default; + +AudioEncoderRuntimeConfig::AudioEncoderRuntimeConfig( + const AudioEncoderRuntimeConfig& other) = default; + +AudioEncoderRuntimeConfig::~AudioEncoderRuntimeConfig() = default; + +AudioEncoderRuntimeConfig& AudioEncoderRuntimeConfig::operator=( + const AudioEncoderRuntimeConfig& other) = default; + +bool AudioEncoderRuntimeConfig::operator==( + const AudioEncoderRuntimeConfig& other) const { + return bitrate_bps == other.bitrate_bps && + frame_length_ms == other.frame_length_ms && + uplink_packet_loss_fraction == other.uplink_packet_loss_fraction && + enable_fec == other.enable_fec && enable_dtx == other.enable_dtx && + num_channels == other.num_channels; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc new file mode 100644 index 0000000000..55e53094c4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" + +#include + +#include "rtc_base/logging.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { +constexpr int kEventLogMinBitrateChangeBps = 5000; +constexpr float kEventLogMinBitrateChangeFraction = 0.25; +constexpr float kEventLogMinPacketLossChangeFraction = 0.5; +} // namespace + +AudioNetworkAdaptorImpl::Config::Config() : event_log(nullptr){}; + +AudioNetworkAdaptorImpl::Config::~Config() = default; + +AudioNetworkAdaptorImpl::AudioNetworkAdaptorImpl( + const Config& config, + std::unique_ptr controller_manager, + std::unique_ptr debug_dump_writer) + : config_(config), + controller_manager_(std::move(controller_manager)), + debug_dump_writer_(std::move(debug_dump_writer)), + event_log_writer_( + config.event_log + ? new EventLogWriter(config.event_log, + kEventLogMinBitrateChangeBps, + kEventLogMinBitrateChangeFraction, + kEventLogMinPacketLossChangeFraction) + : nullptr), + enable_bitrate_adaptation_( + webrtc::field_trial::IsEnabled("WebRTC-Audio-BitrateAdaptation")), + enable_dtx_adaptation_( + webrtc::field_trial::IsEnabled("WebRTC-Audio-DtxAdaptation")), + enable_fec_adaptation_( + webrtc::field_trial::IsEnabled("WebRTC-Audio-FecAdaptation")), + enable_channel_adaptation_( + webrtc::field_trial::IsEnabled("WebRTC-Audio-ChannelAdaptation")), + enable_frame_length_adaptation_(webrtc::field_trial::IsEnabled( + "WebRTC-Audio-FrameLengthAdaptation")) { + RTC_DCHECK(controller_manager_); +} + +AudioNetworkAdaptorImpl::~AudioNetworkAdaptorImpl() = default; + +void AudioNetworkAdaptorImpl::SetUplinkBandwidth(int uplink_bandwidth_bps) { + last_metrics_.uplink_bandwidth_bps = uplink_bandwidth_bps; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + last_metrics_.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + last_metrics_.uplink_recoverable_packet_loss_fraction = + uplink_recoverable_packet_loss_fraction; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_recoverable_packet_loss_fraction = + uplink_recoverable_packet_loss_fraction; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetRtt(int rtt_ms) { + last_metrics_.rtt_ms = rtt_ms; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.rtt_ms = rtt_ms; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetTargetAudioBitrate( + int target_audio_bitrate_bps) { + last_metrics_.target_audio_bitrate_bps = target_audio_bitrate_bps; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.target_audio_bitrate_bps = target_audio_bitrate_bps; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetOverhead(size_t overhead_bytes_per_packet) { + last_metrics_.overhead_bytes_per_packet = overhead_bytes_per_packet; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + UpdateNetworkMetrics(network_metrics); +} + +AudioEncoderRuntimeConfig AudioNetworkAdaptorImpl::GetEncoderRuntimeConfig() { + AudioEncoderRuntimeConfig config; + for (auto& controller : + controller_manager_->GetSortedControllers(last_metrics_)) + controller->MakeDecision(&config); + + // Update ANA stats. + auto increment_opt = [](rtc::Optional& a) { + a = a.value_or(0) + 1; + }; + if (prev_config_) { + if (config.bitrate_bps != prev_config_->bitrate_bps) { + increment_opt(stats_.bitrate_action_counter); + } + if (config.enable_dtx != prev_config_->enable_dtx) { + increment_opt(stats_.dtx_action_counter); + } + if (config.enable_fec != prev_config_->enable_fec) { + increment_opt(stats_.fec_action_counter); + } + if (config.frame_length_ms && prev_config_->frame_length_ms) { + if (*config.frame_length_ms > *prev_config_->frame_length_ms) { + increment_opt(stats_.frame_length_increase_counter); + } else if (*config.frame_length_ms < *prev_config_->frame_length_ms) { + increment_opt(stats_.frame_length_decrease_counter); + } + } + if (config.num_channels != prev_config_->num_channels) { + increment_opt(stats_.channel_action_counter); + } + if (config.uplink_packet_loss_fraction) { + stats_.uplink_packet_loss_fraction = *config.uplink_packet_loss_fraction; + } + } + prev_config_ = config; + + // Prevent certain controllers from taking action (determined by field trials) + if (!enable_bitrate_adaptation_ && config.bitrate_bps) { + config.bitrate_bps.reset(); + } + if (!enable_dtx_adaptation_ && config.enable_dtx) { + config.enable_dtx.reset(); + } + if (!enable_fec_adaptation_ && config.enable_fec) { + config.enable_fec.reset(); + config.uplink_packet_loss_fraction.reset(); + } + if (!enable_frame_length_adaptation_ && config.frame_length_ms) { + config.frame_length_ms.reset(); + } + if (!enable_channel_adaptation_ && config.num_channels) { + config.num_channels.reset(); + } + + if (debug_dump_writer_) + debug_dump_writer_->DumpEncoderRuntimeConfig(config, rtc::TimeMillis()); + + if (event_log_writer_) + event_log_writer_->MaybeLogEncoderConfig(config); + + return config; +} + +void AudioNetworkAdaptorImpl::StartDebugDump(FILE* file_handle) { + debug_dump_writer_ = DebugDumpWriter::Create(file_handle); +} + +void AudioNetworkAdaptorImpl::StopDebugDump() { + debug_dump_writer_.reset(nullptr); +} + +ANAStats AudioNetworkAdaptorImpl::GetStats() const { + return stats_; +} + +void AudioNetworkAdaptorImpl::DumpNetworkMetrics() { + if (debug_dump_writer_) + debug_dump_writer_->DumpNetworkMetrics(last_metrics_, rtc::TimeMillis()); +} + +void AudioNetworkAdaptorImpl::UpdateNetworkMetrics( + const Controller::NetworkMetrics& network_metrics) { + for (auto& controller : controller_manager_->GetControllers()) + controller->UpdateNetworkMetrics(network_metrics); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h new file mode 100644 index 0000000000..14000fed14 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_AUDIO_NETWORK_ADAPTOR_IMPL_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_AUDIO_NETWORK_ADAPTOR_IMPL_H_ + +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" +#include "modules/audio_coding/audio_network_adaptor/event_log_writer.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class RtcEventLog; + +class AudioNetworkAdaptorImpl final : public AudioNetworkAdaptor { + public: + struct Config { + Config(); + ~Config(); + RtcEventLog* event_log; + }; + + AudioNetworkAdaptorImpl( + const Config& config, + std::unique_ptr controller_manager, + std::unique_ptr debug_dump_writer = nullptr); + + ~AudioNetworkAdaptorImpl() override; + + void SetUplinkBandwidth(int uplink_bandwidth_bps) override; + + void SetUplinkPacketLossFraction(float uplink_packet_loss_fraction) override; + + void SetUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; + + void SetRtt(int rtt_ms) override; + + void SetTargetAudioBitrate(int target_audio_bitrate_bps) override; + + void SetOverhead(size_t overhead_bytes_per_packet) override; + + AudioEncoderRuntimeConfig GetEncoderRuntimeConfig() override; + + void StartDebugDump(FILE* file_handle) override; + + void StopDebugDump() override; + + ANAStats GetStats() const override; + + private: + void DumpNetworkMetrics(); + + void UpdateNetworkMetrics(const Controller::NetworkMetrics& network_metrics); + + const Config config_; + + std::unique_ptr controller_manager_; + + std::unique_ptr debug_dump_writer_; + + const std::unique_ptr event_log_writer_; + + Controller::NetworkMetrics last_metrics_; + + rtc::Optional prev_config_; + + ANAStats stats_; + + const bool enable_bitrate_adaptation_; + const bool enable_dtx_adaptation_; + const bool enable_fec_adaptation_; + const bool enable_channel_adaptation_; + const bool enable_frame_length_adaptation_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioNetworkAdaptorImpl); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_AUDIO_NETWORK_ADAPTOR_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc new file mode 100644 index 0000000000..c437918263 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "logging/rtc_event_log/events/rtc_event.h" +#include "logging/rtc_event_log/events/rtc_event_audio_network_adaptation.h" +#include "logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_controller.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h" +#include "rtc_base/fakeclock.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SetArgPointee; + +namespace { + +constexpr size_t kNumControllers = 2; + +constexpr int64_t kClockInitialTimeMs = 12345678; + +MATCHER_P(NetworkMetricsIs, metric, "") { + return arg.uplink_bandwidth_bps == metric.uplink_bandwidth_bps && + arg.target_audio_bitrate_bps == metric.target_audio_bitrate_bps && + arg.rtt_ms == metric.rtt_ms && + arg.overhead_bytes_per_packet == metric.overhead_bytes_per_packet && + arg.uplink_packet_loss_fraction == + metric.uplink_packet_loss_fraction && + arg.uplink_recoverable_packet_loss_fraction == + metric.uplink_recoverable_packet_loss_fraction; +} + +MATCHER_P(IsRtcEventAnaConfigEqualTo, config, "") { + if (arg->GetType() != RtcEvent::Type::AudioNetworkAdaptation) { + return false; + } + auto ana_event = static_cast(arg); + return *ana_event->config_ == config; +} + +MATCHER_P(EncoderRuntimeConfigIs, config, "") { + return arg.bitrate_bps == config.bitrate_bps && + arg.frame_length_ms == config.frame_length_ms && + arg.uplink_packet_loss_fraction == + config.uplink_packet_loss_fraction && + arg.enable_fec == config.enable_fec && + arg.enable_dtx == config.enable_dtx && + arg.num_channels == config.num_channels; +} + +struct AudioNetworkAdaptorStates { + std::unique_ptr audio_network_adaptor; + std::vector> mock_controllers; + std::unique_ptr event_log; + MockDebugDumpWriter* mock_debug_dump_writer; +}; + +AudioNetworkAdaptorStates CreateAudioNetworkAdaptor() { + AudioNetworkAdaptorStates states; + std::vector controllers; + for (size_t i = 0; i < kNumControllers; ++i) { + auto controller = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*controller, Die()); + controllers.push_back(controller.get()); + states.mock_controllers.push_back(std::move(controller)); + } + + auto controller_manager = std::unique_ptr( + new NiceMock()); + + EXPECT_CALL(*controller_manager, Die()); + EXPECT_CALL(*controller_manager, GetControllers()) + .WillRepeatedly(Return(controllers)); + EXPECT_CALL(*controller_manager, GetSortedControllers(_)) + .WillRepeatedly(Return(controllers)); + + states.event_log.reset(new NiceMock()); + + auto debug_dump_writer = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*debug_dump_writer, Die()); + states.mock_debug_dump_writer = debug_dump_writer.get(); + + AudioNetworkAdaptorImpl::Config config; + config.event_log = states.event_log.get(); + // AudioNetworkAdaptorImpl governs the lifetime of controller manager. + states.audio_network_adaptor.reset(new AudioNetworkAdaptorImpl( + config, + std::move(controller_manager), std::move(debug_dump_writer))); + + return states; +} + +void SetExpectCallToUpdateNetworkMetrics( + const std::vector>& controllers, + const Controller::NetworkMetrics& check) { + for (auto& mock_controller : controllers) { + EXPECT_CALL(*mock_controller, + UpdateNetworkMetrics(NetworkMetricsIs(check))); + } +} + +} // namespace + +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetUplinkBandwidth) { + auto states = CreateAudioNetworkAdaptor(); + constexpr int kBandwidth = 16000; + Controller::NetworkMetrics check; + check.uplink_bandwidth_bps = kBandwidth; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetUplinkBandwidth(kBandwidth); +} + +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetUplinkPacketLossFraction) { + auto states = CreateAudioNetworkAdaptor(); + constexpr float kPacketLoss = 0.7f; + Controller::NetworkMetrics check; + check.uplink_packet_loss_fraction = kPacketLoss; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetUplinkPacketLossFraction(kPacketLoss); +} + +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetUplinkRecoverablePacketLossFraction) { + auto states = CreateAudioNetworkAdaptor(); + constexpr float kRecoverablePacketLoss = 0.1f; + Controller::NetworkMetrics check; + check.uplink_recoverable_packet_loss_fraction = kRecoverablePacketLoss; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetUplinkRecoverablePacketLossFraction( + kRecoverablePacketLoss); +} + +TEST(AudioNetworkAdaptorImplTest, UpdateNetworkMetricsIsCalledOnSetRtt) { + auto states = CreateAudioNetworkAdaptor(); + constexpr int kRtt = 100; + Controller::NetworkMetrics check; + check.rtt_ms = kRtt; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetRtt(kRtt); +} + +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetTargetAudioBitrate) { + auto states = CreateAudioNetworkAdaptor(); + constexpr int kTargetAudioBitrate = 15000; + Controller::NetworkMetrics check; + check.target_audio_bitrate_bps = kTargetAudioBitrate; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetTargetAudioBitrate(kTargetAudioBitrate); +} + +TEST(AudioNetworkAdaptorImplTest, UpdateNetworkMetricsIsCalledOnSetOverhead) { + auto states = CreateAudioNetworkAdaptor(); + constexpr size_t kOverhead = 64; + Controller::NetworkMetrics check; + check.overhead_bytes_per_packet = kOverhead; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetOverhead(kOverhead); +} + +TEST(AudioNetworkAdaptorImplTest, + MakeDecisionIsCalledOnGetEncoderRuntimeConfig) { + auto states = CreateAudioNetworkAdaptor(); + for (auto& mock_controller : states.mock_controllers) + EXPECT_CALL(*mock_controller, MakeDecision(_)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); +} + +TEST(AudioNetworkAdaptorImplTest, + DumpEncoderRuntimeConfigIsCalledOnGetEncoderRuntimeConfig) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-BitrateAdaptation/Enabled/WebRTC-Audio-FecAdaptation/" + "Enabled/"); + rtc::ScopedFakeClock fake_clock; + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kClockInitialTimeMs)); + auto states = CreateAudioNetworkAdaptor(); + AudioEncoderRuntimeConfig config; + config.bitrate_bps = 32000; + config.enable_fec = true; + + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config)); + + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpEncoderRuntimeConfig(EncoderRuntimeConfigIs(config), + kClockInitialTimeMs)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); +} + +TEST(AudioNetworkAdaptorImplTest, + DumpNetworkMetricsIsCalledOnSetNetworkMetrics) { + rtc::ScopedFakeClock fake_clock; + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kClockInitialTimeMs)); + + auto states = CreateAudioNetworkAdaptor(); + + constexpr int kBandwidth = 16000; + constexpr float kPacketLoss = 0.7f; + const auto kRecoverablePacketLoss = 0.2f; + constexpr int kRtt = 100; + constexpr int kTargetAudioBitrate = 15000; + constexpr size_t kOverhead = 64; + + Controller::NetworkMetrics check; + check.uplink_bandwidth_bps = kBandwidth; + int64_t timestamp_check = kClockInitialTimeMs; + + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetUplinkBandwidth(kBandwidth); + + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(100)); + timestamp_check += 100; + check.uplink_packet_loss_fraction = kPacketLoss; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetUplinkPacketLossFraction(kPacketLoss); + + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(50)); + timestamp_check += 50; + check.uplink_recoverable_packet_loss_fraction = kRecoverablePacketLoss; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetUplinkRecoverablePacketLossFraction( + kRecoverablePacketLoss); + + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(200)); + timestamp_check += 200; + check.rtt_ms = kRtt; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetRtt(kRtt); + + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(150)); + timestamp_check += 150; + check.target_audio_bitrate_bps = kTargetAudioBitrate; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetTargetAudioBitrate(kTargetAudioBitrate); + + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(50)); + timestamp_check += 50; + check.overhead_bytes_per_packet = kOverhead; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetOverhead(kOverhead); +} + +TEST(AudioNetworkAdaptorImplTest, LogRuntimeConfigOnGetEncoderRuntimeConfig) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-BitrateAdaptation/Enabled/WebRTC-Audio-FecAdaptation/" + "Enabled/"); + auto states = CreateAudioNetworkAdaptor(); + + AudioEncoderRuntimeConfig config; + config.bitrate_bps = 32000; + config.enable_fec = true; + + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config)); + + EXPECT_CALL(*states.event_log, LogProxy(IsRtcEventAnaConfigEqualTo(config))) + .Times(1); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); +} + +TEST(AudioNetworkAdaptorImplTest, TestANAStats) { + auto states = CreateAudioNetworkAdaptor(); + + // Simulate some adaptation, otherwise the stats will not show anything. + AudioEncoderRuntimeConfig config1, config2; + config1.bitrate_bps = 32000; + config1.num_channels = 2; + config1.enable_fec = true; + config1.enable_dtx = true; + config1.frame_length_ms = 120; + config1.uplink_packet_loss_fraction = 0.1f; + config2.bitrate_bps = 16000; + config2.num_channels = 1; + config2.enable_fec = false; + config2.enable_dtx = false; + config2.frame_length_ms = 60; + config1.uplink_packet_loss_fraction = 0.1f; + + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config1)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config2)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config1)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); + + auto ana_stats = states.audio_network_adaptor->GetStats(); + + EXPECT_EQ(ana_stats.bitrate_action_counter, 2); + EXPECT_EQ(ana_stats.channel_action_counter, 2); + EXPECT_EQ(ana_stats.dtx_action_counter, 2); + EXPECT_EQ(ana_stats.fec_action_counter, 2); + EXPECT_EQ(ana_stats.frame_length_increase_counter, 1); + EXPECT_EQ(ana_stats.frame_length_decrease_counter, 1); + EXPECT_EQ(ana_stats.uplink_packet_loss_fraction, 0.1f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc new file mode 100644 index 0000000000..6850926c27 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h" + +#include + +#include "rtc_base/checks.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace audio_network_adaptor { + +BitrateController::Config::Config(int initial_bitrate_bps, + int initial_frame_length_ms, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset) + : initial_bitrate_bps(initial_bitrate_bps), + initial_frame_length_ms(initial_frame_length_ms), + fl_increase_overhead_offset(fl_increase_overhead_offset), + fl_decrease_overhead_offset(fl_decrease_overhead_offset) {} + +BitrateController::Config::~Config() = default; + +BitrateController::BitrateController(const Config& config) + : config_(config), + bitrate_bps_(config_.initial_bitrate_bps), + frame_length_ms_(config_.initial_frame_length_ms) { + RTC_DCHECK_GT(bitrate_bps_, 0); + RTC_DCHECK_GT(frame_length_ms_, 0); +} + +BitrateController::~BitrateController() = default; + +void BitrateController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.target_audio_bitrate_bps) + target_audio_bitrate_bps_ = network_metrics.target_audio_bitrate_bps; + if (network_metrics.overhead_bytes_per_packet) + overhead_bytes_per_packet_ = network_metrics.overhead_bytes_per_packet; +} + +void BitrateController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on |bitrate_bps| should not have been made. + RTC_DCHECK(!config->bitrate_bps); + if (target_audio_bitrate_bps_ && overhead_bytes_per_packet_) { + // Current implementation of BitrateController can only work when + // |metrics.target_audio_bitrate_bps| includes overhead is enabled. This is + // currently governed by the following field trial. + RTC_DCHECK( + webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")); + if (config->frame_length_ms) + frame_length_ms_ = *config->frame_length_ms; + int offset = config->last_fl_change_increase + ? config_.fl_increase_overhead_offset + : config_.fl_decrease_overhead_offset; + // Check that + // -(*overhead_bytes_per_packet_) <= offset <= (*overhead_bytes_per_packet_) + RTC_DCHECK_GE(*overhead_bytes_per_packet_, -offset); + RTC_DCHECK_LE(offset, *overhead_bytes_per_packet_); + int overhead_rate_bps = static_cast( + (*overhead_bytes_per_packet_ + offset) * 8 * 1000 / frame_length_ms_); + bitrate_bps_ = std::max(0, *target_audio_bitrate_bps_ - overhead_rate_bps); + } + config->bitrate_bps = bitrate_bps_; +} + +} // namespace audio_network_adaptor +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h new file mode 100644 index 0000000000..601f794b6a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_BITRATE_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_BITRATE_CONTROLLER_H_ + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace audio_network_adaptor { + +class BitrateController final : public Controller { + public: + struct Config { + Config(int initial_bitrate_bps, + int initial_frame_length_ms, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset); + ~Config(); + int initial_bitrate_bps; + int initial_frame_length_ms; + int fl_increase_overhead_offset; + int fl_decrease_overhead_offset; + }; + + explicit BitrateController(const Config& config); + + ~BitrateController() override; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + const Config config_; + int bitrate_bps_; + int frame_length_ms_; + rtc::Optional target_audio_bitrate_bps_; + rtc::Optional overhead_bytes_per_packet_; + RTC_DISALLOW_COPY_AND_ASSIGN(BitrateController); +}; + +} // namespace audio_network_adaptor +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_BITRATE_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc new file mode 100644 index 0000000000..9864511c48 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace audio_network_adaptor { + +namespace { + +void UpdateNetworkMetrics( + BitrateController* controller, + const rtc::Optional& target_audio_bitrate_bps, + const rtc::Optional& overhead_bytes_per_packet) { + // UpdateNetworkMetrics can accept multiple network metric updates at once. + // However, currently, the most used case is to update one metric at a time. + // To reflect this fact, we separate the calls. + if (target_audio_bitrate_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.target_audio_bitrate_bps = target_audio_bitrate_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + if (overhead_bytes_per_packet) { + Controller::NetworkMetrics network_metrics; + network_metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + controller->UpdateNetworkMetrics(network_metrics); + } +} + +void CheckDecision(BitrateController* controller, + const rtc::Optional& frame_length_ms, + int expected_bitrate_bps) { + AudioEncoderRuntimeConfig config; + config.frame_length_ms = frame_length_ms; + controller->MakeDecision(&config); + EXPECT_EQ(expected_bitrate_bps, config.bitrate_bps); +} + +} // namespace + +// These tests are named AnaBitrateControllerTest to distinguish from +// BitrateControllerTest in +// modules/bitrate_controller/bitrate_controller_unittest.cc. + +TEST(AnaBitrateControllerTest, OutputInitValueWhenTargetBitrateUnknown) { + constexpr int kInitialBitrateBps = 32000; + constexpr int kInitialFrameLengthMs = 20; + constexpr size_t kOverheadBytesPerPacket = 64; + BitrateController controller(BitrateController::Config( + kInitialBitrateBps, kInitialFrameLengthMs, 0, 0)); + UpdateNetworkMetrics(&controller, rtc::nullopt, kOverheadBytesPerPacket); + CheckDecision(&controller, kInitialFrameLengthMs * 2, kInitialBitrateBps); +} + +TEST(AnaBitrateControllerTest, OutputInitValueWhenOverheadUnknown) { + constexpr int kInitialBitrateBps = 32000; + constexpr int kInitialFrameLengthMs = 20; + constexpr int kTargetBitrateBps = 48000; + BitrateController controller(BitrateController::Config( + kInitialBitrateBps, kInitialFrameLengthMs, 0, 0)); + UpdateNetworkMetrics(&controller, kTargetBitrateBps, rtc::nullopt); + CheckDecision(&controller, kInitialFrameLengthMs * 2, kInitialBitrateBps); +} + +TEST(AnaBitrateControllerTest, ChangeBitrateOnTargetBitrateChanged) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = + kTargetBitrateBps - + kOverheadBytesPerPacket * 8 * 1000 / kInitialFrameLengthMs; + // Frame length unchanged, bitrate changes in accordance with + // |metrics.target_audio_bitrate_bps| and |metrics.overhead_bytes_per_packet|. + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kInitialFrameLengthMs, kBitrateBps); +} + +TEST(AnaBitrateControllerTest, UpdateMultipleNetworkMetricsAtOnce) { + // This test is similar to ChangeBitrateOnTargetBitrateChanged. But instead of + // using ::UpdateNetworkMetrics(...), which calls + // BitrateController::UpdateNetworkMetrics(...) multiple times, we + // we call it only once. This is to verify that + // BitrateController::UpdateNetworkMetrics(...) can handle multiple + // network updates at once. This is, however, not a common use case in current + // audio_network_adaptor_impl.cc. + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = + kTargetBitrateBps - + kOverheadBytesPerPacket * 8 * 1000 / kInitialFrameLengthMs; + Controller::NetworkMetrics network_metrics; + network_metrics.target_audio_bitrate_bps = kTargetBitrateBps; + network_metrics.overhead_bytes_per_packet = kOverheadBytesPerPacket; + controller.UpdateNetworkMetrics(network_metrics); + CheckDecision(&controller, kInitialFrameLengthMs, kBitrateBps); +} + +TEST(AnaBitrateControllerTest, TreatUnknownFrameLengthAsFrameLengthUnchanged) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = + kTargetBitrateBps - + kOverheadBytesPerPacket * 8 * 1000 / kInitialFrameLengthMs; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, rtc::nullopt, kBitrateBps); +} + +TEST(AnaBitrateControllerTest, IncreaseBitrateOnFrameLengthIncreased) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = + kTargetBitrateBps - + kOverheadBytesPerPacket * 8 * 1000 / kInitialFrameLengthMs; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, rtc::nullopt, kBitrateBps); + + constexpr int kFrameLengthMs = 60; + constexpr size_t kPacketOverheadRateDiff = + kOverheadBytesPerPacket * 8 * 1000 / 20 - + kOverheadBytesPerPacket * 8 * 1000 / 60; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kFrameLengthMs, + kBitrateBps + kPacketOverheadRateDiff); +} + +TEST(AnaBitrateControllerTest, DecreaseBitrateOnFrameLengthDecreased) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + constexpr int kInitialFrameLengthMs = 60; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = + kTargetBitrateBps - + kOverheadBytesPerPacket * 8 * 1000 / kInitialFrameLengthMs; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, rtc::nullopt, kBitrateBps); + + constexpr int kFrameLengthMs = 20; + constexpr size_t kPacketOverheadRateDiff = + kOverheadBytesPerPacket * 8 * 1000 / 20 - + kOverheadBytesPerPacket * 8 * 1000 / 60; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kFrameLengthMs, + kBitrateBps - kPacketOverheadRateDiff); +} + +TEST(AnaBitrateControllerTest, BitrateNeverBecomesNegative) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + BitrateController controller(BitrateController::Config(32000, 20, 0, 0)); + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kFrameLengthMs = 60; + // Set a target rate smaller than overhead rate, the bitrate is bounded by 0. + constexpr int kTargetBitrateBps = + kOverheadBytesPerPacket * 8 * 1000 / kFrameLengthMs - 1; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kFrameLengthMs, 0); +} + +TEST(AnaBitrateControllerTest, CheckBehaviorOnChangingCondition) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + BitrateController controller(BitrateController::Config(32000, 20, 0, 0)); + + // Start from an arbitrary overall bitrate. + int overall_bitrate = 34567; + size_t overhead_bytes_per_packet = 64; + int frame_length_ms = 20; + int current_bitrate = rtc::checked_cast( + overall_bitrate - overhead_bytes_per_packet * 8 * 1000 / frame_length_ms); + + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: increase overall bitrate. + overall_bitrate += 100; + current_bitrate += 100; + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: change frame length. + frame_length_ms = 60; + current_bitrate += rtc::checked_cast( + overhead_bytes_per_packet * 8 * 1000 / 20 - + overhead_bytes_per_packet * 8 * 1000 / 60); + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: change overhead. + overhead_bytes_per_packet -= 30; + current_bitrate += 30 * 8 * 1000 / frame_length_ms; + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: change frame length. + frame_length_ms = 20; + current_bitrate -= rtc::checked_cast( + overhead_bytes_per_packet * 8 * 1000 / 20 - + overhead_bytes_per_packet * 8 * 1000 / 60); + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: decrease overall bitrate and frame length. + overall_bitrate -= 100; + current_bitrate -= 100; + frame_length_ms = 60; + current_bitrate += rtc::checked_cast( + overhead_bytes_per_packet * 8 * 1000 / 20 - + overhead_bytes_per_packet * 8 * 1000 / 60); + + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); +} + +} // namespace audio_network_adaptor +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc new file mode 100644 index 0000000000..a1c30dbf3f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/audio_network_adaptor/channel_controller.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +ChannelController::Config::Config(size_t num_encoder_channels, + size_t intial_channels_to_encode, + int channel_1_to_2_bandwidth_bps, + int channel_2_to_1_bandwidth_bps) + : num_encoder_channels(num_encoder_channels), + intial_channels_to_encode(intial_channels_to_encode), + channel_1_to_2_bandwidth_bps(channel_1_to_2_bandwidth_bps), + channel_2_to_1_bandwidth_bps(channel_2_to_1_bandwidth_bps) {} + +ChannelController::ChannelController(const Config& config) + : config_(config), channels_to_encode_(config_.intial_channels_to_encode) { + RTC_DCHECK_GT(config_.intial_channels_to_encode, 0lu); + // Currently, we require |intial_channels_to_encode| to be <= 2. + RTC_DCHECK_LE(config_.intial_channels_to_encode, 2lu); + RTC_DCHECK_GE(config_.num_encoder_channels, + config_.intial_channels_to_encode); +} + +ChannelController::~ChannelController() = default; + +void ChannelController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; +} + +void ChannelController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on |num_channels| should not have been made. + RTC_DCHECK(!config->num_channels); + + if (uplink_bandwidth_bps_) { + if (channels_to_encode_ == 2 && + *uplink_bandwidth_bps_ <= config_.channel_2_to_1_bandwidth_bps) { + channels_to_encode_ = 1; + } else if (channels_to_encode_ == 1 && + *uplink_bandwidth_bps_ >= config_.channel_1_to_2_bandwidth_bps) { + channels_to_encode_ = + std::min(static_cast(2), config_.num_encoder_channels); + } + } + config->num_channels = channels_to_encode_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h new file mode 100644 index 0000000000..f53ddd6b19 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CHANNEL_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CHANNEL_CONTROLLER_H_ + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ChannelController final : public Controller { + public: + struct Config { + Config(size_t num_encoder_channels, + size_t intial_channels_to_encode, + int channel_1_to_2_bandwidth_bps, + int channel_2_to_1_bandwidth_bps); + size_t num_encoder_channels; + size_t intial_channels_to_encode; + // Uplink bandwidth above which the number of encoded channels should switch + // from 1 to 2. + int channel_1_to_2_bandwidth_bps; + // Uplink bandwidth below which the number of encoded channels should switch + // from 2 to 1. + int channel_2_to_1_bandwidth_bps; + }; + + explicit ChannelController(const Config& config); + + ~ChannelController() override; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + const Config config_; + size_t channels_to_encode_; + rtc::Optional uplink_bandwidth_bps_; + RTC_DISALLOW_COPY_AND_ASSIGN(ChannelController); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CHANNEL_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc new file mode 100644 index 0000000000..64e5daee2b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/audio_network_adaptor/channel_controller.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kNumChannels = 2; +constexpr int kChannel1To2BandwidthBps = 31000; +constexpr int kChannel2To1BandwidthBps = 29000; +constexpr int kMediumBandwidthBps = + (kChannel1To2BandwidthBps + kChannel2To1BandwidthBps) / 2; + +std::unique_ptr CreateChannelController(int init_channels) { + std::unique_ptr controller( + new ChannelController(ChannelController::Config( + kNumChannels, init_channels, kChannel1To2BandwidthBps, + kChannel2To1BandwidthBps))); + return controller; +} + +void CheckDecision(ChannelController* controller, + const rtc::Optional& uplink_bandwidth_bps, + size_t expected_num_channels) { + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + AudioEncoderRuntimeConfig config; + controller->MakeDecision(&config); + EXPECT_EQ(expected_num_channels, config.num_channels); +} + +} // namespace + +TEST(ChannelControllerTest, OutputInitValueWhenUplinkBandwidthUnknown) { + constexpr int kInitChannels = 2; + auto controller = CreateChannelController(kInitChannels); + CheckDecision(controller.get(), rtc::nullopt, kInitChannels); +} + +TEST(ChannelControllerTest, SwitchTo2ChannelsOnHighUplinkBandwidth) { + constexpr int kInitChannels = 1; + auto controller = CreateChannelController(kInitChannels); + // Use high bandwidth to check output switch to 2. + CheckDecision(controller.get(), kChannel1To2BandwidthBps, 2); +} + +TEST(ChannelControllerTest, SwitchTo1ChannelOnLowUplinkBandwidth) { + constexpr int kInitChannels = 2; + auto controller = CreateChannelController(kInitChannels); + // Use low bandwidth to check output switch to 1. + CheckDecision(controller.get(), kChannel2To1BandwidthBps, 1); +} + +TEST(ChannelControllerTest, Maintain1ChannelOnMediumUplinkBandwidth) { + constexpr int kInitChannels = 1; + auto controller = CreateChannelController(kInitChannels); + // Use between-thresholds bandwidth to check output remains at 1. + CheckDecision(controller.get(), kMediumBandwidthBps, 1); +} + +TEST(ChannelControllerTest, Maintain2ChannelsOnMediumUplinkBandwidth) { + constexpr int kInitChannels = 2; + auto controller = CreateChannelController(kInitChannels); + // Use between-thresholds bandwidth to check output remains at 2. + CheckDecision(controller.get(), kMediumBandwidthBps, 2); +} + +TEST(ChannelControllerTest, CheckBehaviorOnChangingUplinkBandwidth) { + constexpr int kInitChannels = 1; + auto controller = CreateChannelController(kInitChannels); + + // Use between-thresholds bandwidth to check output remains at 1. + CheckDecision(controller.get(), kMediumBandwidthBps, 1); + + // Use high bandwidth to check output switch to 2. + CheckDecision(controller.get(), kChannel1To2BandwidthBps, 2); + + // Use between-thresholds bandwidth to check output remains at 2. + CheckDecision(controller.get(), kMediumBandwidthBps, 2); + + // Use low bandwidth to check output switch to 1. + CheckDecision(controller.get(), kChannel2To1BandwidthBps, 1); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.proto b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.proto new file mode 100644 index 0000000000..6d1cd42d46 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.proto @@ -0,0 +1,164 @@ +syntax = "proto2"; +option optimize_for = LITE_RUNTIME; +option java_package = "org.webrtc.AudioNetworkAdaptor"; +option java_outer_classname = "Config"; +package webrtc.audio_network_adaptor.config; + +message FecController { + message Threshold { + // Threshold defines a curve in the bandwidth/packet-loss domain. The + // curve is characterized by the two conjunction points: A and B. + // + // packet ^ | + // loss | A| + // | \ A: (low_bandwidth_bps, low_bandwidth_packet_loss) + // | \ B: (high_bandwidth_bps, high_bandwidth_packet_loss) + // | B\________ + // |---------------> bandwidth + optional int32 low_bandwidth_bps = 1; + optional float low_bandwidth_packet_loss = 2; + optional int32 high_bandwidth_bps = 3; + optional float high_bandwidth_packet_loss = 4; + } + + // |fec_enabling_threshold| defines a curve, above which FEC should be + // enabled. |fec_disabling_threshold| defines a curve, under which FEC + // should be disabled. See below + // + // packet-loss ^ | | + // | | | FEC + // | \ \ ON + // | FEC \ \_______ fec_enabling_threshold + // | OFF \_________ fec_disabling_threshold + // |-----------------> bandwidth + optional Threshold fec_enabling_threshold = 1; + optional Threshold fec_disabling_threshold = 2; + + // |time_constant_ms| is the time constant for an exponential filter, which + // is used for smoothing the packet loss fraction. + optional int32 time_constant_ms = 3; +} + +message FecControllerRplrBased { + message Threshold { + // Threshold defines a curve in the bandwidth/recoverable-packet-loss + // domain. + // The curve is characterized by the two conjunction points: A and B. + // + // recoverable ^ + // packet | | + // loss | A| + // | \ A: (low_bandwidth_bps, + // | \ low_bandwidth_recoverable_packet_loss) + // | \ B: (high_bandwidth_bps, + // | \ high_bandwidth_recoverable_packet_loss) + // | B\________ + // |---------------> bandwidth + optional int32 low_bandwidth_bps = 1; + optional float low_bandwidth_recoverable_packet_loss = 2; + optional int32 high_bandwidth_bps = 3; + optional float high_bandwidth_recoverable_packet_loss = 4; + } + + // |fec_enabling_threshold| defines a curve, above which FEC should be + // enabled. |fec_disabling_threshold| defines a curve, under which FEC + // should be disabled. See below + // + // packet-loss ^ | | + // | | | FEC + // | \ \ ON + // | FEC \ \_______ fec_enabling_threshold + // | OFF \_________ fec_disabling_threshold + // |-----------------> bandwidth + optional Threshold fec_enabling_threshold = 1; + optional Threshold fec_disabling_threshold = 2; +} + +message FrameLengthController { + // Uplink packet loss fraction below which frame length can increase. + optional float fl_increasing_packet_loss_fraction = 1; + + // Uplink packet loss fraction above which frame length should decrease. + optional float fl_decreasing_packet_loss_fraction = 2; + + // Uplink bandwidth below which frame length can switch from 20ms to 60ms. + optional int32 fl_20ms_to_60ms_bandwidth_bps = 3; + + // Uplink bandwidth above which frame length should switch from 60ms to 20ms. + optional int32 fl_60ms_to_20ms_bandwidth_bps = 4; + + // Uplink bandwidth below which frame length can switch from 60ms to 120ms. + optional int32 fl_60ms_to_120ms_bandwidth_bps = 5; + + // Uplink bandwidth above which frame length should switch from 120ms to 60ms. + optional int32 fl_120ms_to_60ms_bandwidth_bps = 6; + + // Offset to apply to the per-packet overhead when increasing frame length. + optional int32 fl_increase_overhead_offset = 7; + + // Offset to apply to the per-packet overhead when decreasing frame length. + optional int32 fl_decrease_overhead_offset = 8; +} + +message ChannelController { + // Uplink bandwidth above which the number of encoded channels should switch + // from 1 to 2. + optional int32 channel_1_to_2_bandwidth_bps = 1; + + // Uplink bandwidth below which the number of encoded channels should switch + // from 2 to 1. + optional int32 channel_2_to_1_bandwidth_bps = 2; +} + +message DtxController { + // Uplink bandwidth below which DTX should be switched on. + optional int32 dtx_enabling_bandwidth_bps = 1; + + // Uplink bandwidth above which DTX should be switched off. + optional int32 dtx_disabling_bandwidth_bps = 2; +} + +message BitrateController { + // Offset to apply to per-packet overhead when the frame length is increased. + optional int32 fl_increase_overhead_offset = 1; + // Offset to apply to per-packet overhead when the frame length is decreased. + optional int32 fl_decrease_overhead_offset = 2; +} + +message Controller { + message ScoringPoint { + // |ScoringPoint| is a subspace of network condition. It is used for + // comparing the significance of controllers. + optional int32 uplink_bandwidth_bps = 1; + optional float uplink_packet_loss_fraction = 2; + } + + // The distance from |scoring_point| to a given network condition defines + // the significance of this controller with respect that network condition. + // Shorter distance means higher significance. The significances of + // controllers determine their order in the processing pipeline. Controllers + // without |scoring_point| follow their default order in + // |ControllerManager::controllers|. + optional ScoringPoint scoring_point = 1; + + oneof controller { + FecController fec_controller = 21; + FrameLengthController frame_length_controller = 22; + ChannelController channel_controller = 23; + DtxController dtx_controller = 24; + BitrateController bitrate_controller = 25; + FecControllerRplrBased fec_controller_rplr_based = 26; + } +} + +message ControllerManager { + repeated Controller controllers = 1; + + // Least time since last reordering for a new reordering to be made. + optional int32 min_reordering_time_ms = 2; + + // Least squared distance from last scoring point for a new reordering to be + // made. + optional float min_reordering_squared_distance = 3; +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.cc new file mode 100644 index 0000000000..5e2dc859bf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.cc @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/controller.h" + +namespace webrtc { + +Controller::NetworkMetrics::NetworkMetrics() = default; + +Controller::NetworkMetrics::~NetworkMetrics() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.h new file mode 100644 index 0000000000..af2f569f89 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_H_ + +#include "api/optional.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" + +namespace webrtc { + +class Controller { + public: + struct NetworkMetrics { + NetworkMetrics(); + ~NetworkMetrics(); + rtc::Optional uplink_bandwidth_bps; + rtc::Optional uplink_packet_loss_fraction; + rtc::Optional uplink_recoverable_packet_loss_fraction; + rtc::Optional target_audio_bitrate_bps; + rtc::Optional rtt_ms; + rtc::Optional overhead_bytes_per_packet; + }; + + virtual ~Controller() = default; + + // Informs network metrics update to this controller. Any non-empty field + // indicates an update on the corresponding network metric. + virtual void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) = 0; + + virtual void MakeDecision(AudioEncoderRuntimeConfig* config) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc new file mode 100644 index 0000000000..313aa628c7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc @@ -0,0 +1,441 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" + +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h" +#include "modules/audio_coding/audio_network_adaptor/channel_controller.h" +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" +#include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" +#include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" +#include "modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.h" +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" +#include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/timeutils.h" + +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/config.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +namespace { + +#if WEBRTC_ENABLE_PROTOBUF + +std::unique_ptr CreateFecControllerPlrBased( + const audio_network_adaptor::config::FecController& config, + bool initial_fec_enabled) { + RTC_CHECK(config.has_fec_enabling_threshold()); + RTC_CHECK(config.has_fec_disabling_threshold()); + RTC_CHECK(config.has_time_constant_ms()); + + auto& fec_enabling_threshold = config.fec_enabling_threshold(); + RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_bps()); + RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_packet_loss()); + RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_bps()); + RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_packet_loss()); + + auto& fec_disabling_threshold = config.fec_disabling_threshold(); + RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_bps()); + RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_packet_loss()); + RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_bps()); + RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_packet_loss()); + + return std::unique_ptr( + new FecControllerPlrBased(FecControllerPlrBased::Config( + initial_fec_enabled, + ThresholdCurve(fec_enabling_threshold.low_bandwidth_bps(), + fec_enabling_threshold.low_bandwidth_packet_loss(), + fec_enabling_threshold.high_bandwidth_bps(), + fec_enabling_threshold.high_bandwidth_packet_loss()), + ThresholdCurve(fec_disabling_threshold.low_bandwidth_bps(), + fec_disabling_threshold.low_bandwidth_packet_loss(), + fec_disabling_threshold.high_bandwidth_bps(), + fec_disabling_threshold.high_bandwidth_packet_loss()), + config.time_constant_ms()))); +} + +std::unique_ptr CreateFecControllerRplrBased( + const audio_network_adaptor::config::FecControllerRplrBased& config, + bool initial_fec_enabled) { + RTC_CHECK(config.has_fec_enabling_threshold()); + RTC_CHECK(config.has_fec_disabling_threshold()); + + auto& fec_enabling_threshold = config.fec_enabling_threshold(); + RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_bps()); + RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_recoverable_packet_loss()); + RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_bps()); + RTC_CHECK( + fec_enabling_threshold.has_high_bandwidth_recoverable_packet_loss()); + + auto& fec_disabling_threshold = config.fec_disabling_threshold(); + RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_bps()); + RTC_CHECK( + fec_disabling_threshold.has_low_bandwidth_recoverable_packet_loss()); + RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_bps()); + RTC_CHECK( + fec_disabling_threshold.has_high_bandwidth_recoverable_packet_loss()); + + return std::unique_ptr( + new FecControllerRplrBased(FecControllerRplrBased::Config( + initial_fec_enabled, + ThresholdCurve( + fec_enabling_threshold.low_bandwidth_bps(), + fec_enabling_threshold.low_bandwidth_recoverable_packet_loss(), + fec_enabling_threshold.high_bandwidth_bps(), + fec_enabling_threshold.high_bandwidth_recoverable_packet_loss()), + ThresholdCurve( + fec_disabling_threshold.low_bandwidth_bps(), + fec_disabling_threshold.low_bandwidth_recoverable_packet_loss(), + fec_disabling_threshold.high_bandwidth_bps(), + fec_disabling_threshold + .high_bandwidth_recoverable_packet_loss())))); +} + +std::unique_ptr CreateFrameLengthController( + const audio_network_adaptor::config::FrameLengthController& config, + rtc::ArrayView encoder_frame_lengths_ms, + int initial_frame_length_ms, + int min_encoder_bitrate_bps) { + RTC_CHECK(config.has_fl_increasing_packet_loss_fraction()); + RTC_CHECK(config.has_fl_decreasing_packet_loss_fraction()); + RTC_CHECK(config.has_fl_20ms_to_60ms_bandwidth_bps()); + RTC_CHECK(config.has_fl_60ms_to_20ms_bandwidth_bps()); + + std::map + fl_changing_bandwidths_bps = { + {FrameLengthController::Config::FrameLengthChange(20, 60), + config.fl_20ms_to_60ms_bandwidth_bps()}, + {FrameLengthController::Config::FrameLengthChange(60, 20), + config.fl_60ms_to_20ms_bandwidth_bps()}}; + + if (config.has_fl_60ms_to_120ms_bandwidth_bps() && + config.has_fl_120ms_to_60ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert(std::make_pair( + FrameLengthController::Config::FrameLengthChange(60, 120), + config.fl_60ms_to_120ms_bandwidth_bps())); + fl_changing_bandwidths_bps.insert(std::make_pair( + FrameLengthController::Config::FrameLengthChange(120, 60), + config.fl_120ms_to_60ms_bandwidth_bps())); + } + + int fl_increase_overhead_offset = 0; + if (config.has_fl_increase_overhead_offset()) { + fl_increase_overhead_offset = config.fl_increase_overhead_offset(); + } + int fl_decrease_overhead_offset = 0; + if (config.has_fl_decrease_overhead_offset()) { + fl_decrease_overhead_offset = config.fl_decrease_overhead_offset(); + } + + FrameLengthController::Config ctor_config( + std::vector(), initial_frame_length_ms, min_encoder_bitrate_bps, + config.fl_increasing_packet_loss_fraction(), + config.fl_decreasing_packet_loss_fraction(), fl_increase_overhead_offset, + fl_decrease_overhead_offset, std::move(fl_changing_bandwidths_bps)); + + for (auto frame_length : encoder_frame_lengths_ms) + ctor_config.encoder_frame_lengths_ms.push_back(frame_length); + + return std::unique_ptr( + new FrameLengthController(ctor_config)); +} + +std::unique_ptr CreateChannelController( + const audio_network_adaptor::config::ChannelController& config, + size_t num_encoder_channels, + size_t intial_channels_to_encode) { + RTC_CHECK(config.has_channel_1_to_2_bandwidth_bps()); + RTC_CHECK(config.has_channel_2_to_1_bandwidth_bps()); + + return std::unique_ptr(new ChannelController( + ChannelController::Config(num_encoder_channels, intial_channels_to_encode, + config.channel_1_to_2_bandwidth_bps(), + config.channel_2_to_1_bandwidth_bps()))); +} + +std::unique_ptr CreateDtxController( + const audio_network_adaptor::config::DtxController& dtx_config, + bool initial_dtx_enabled) { + RTC_CHECK(dtx_config.has_dtx_enabling_bandwidth_bps()); + RTC_CHECK(dtx_config.has_dtx_disabling_bandwidth_bps()); + + return std::unique_ptr(new DtxController(DtxController::Config( + initial_dtx_enabled, dtx_config.dtx_enabling_bandwidth_bps(), + dtx_config.dtx_disabling_bandwidth_bps()))); +} + +using audio_network_adaptor::BitrateController; +std::unique_ptr CreateBitrateController( + const audio_network_adaptor::config::BitrateController& bitrate_config, + int initial_bitrate_bps, + int initial_frame_length_ms) { + int fl_increase_overhead_offset = 0; + if (bitrate_config.has_fl_increase_overhead_offset()) { + fl_increase_overhead_offset = bitrate_config.fl_increase_overhead_offset(); + } + int fl_decrease_overhead_offset = 0; + if (bitrate_config.has_fl_decrease_overhead_offset()) { + fl_decrease_overhead_offset = bitrate_config.fl_decrease_overhead_offset(); + } + return std::unique_ptr( + new BitrateController(BitrateController::Config( + initial_bitrate_bps, initial_frame_length_ms, + fl_increase_overhead_offset, fl_decrease_overhead_offset))); +} +#endif // WEBRTC_ENABLE_PROTOBUF + +} // namespace + +ControllerManagerImpl::Config::Config(int min_reordering_time_ms, + float min_reordering_squared_distance) + : min_reordering_time_ms(min_reordering_time_ms), + min_reordering_squared_distance(min_reordering_squared_distance) {} + +ControllerManagerImpl::Config::~Config() = default; + +std::unique_ptr ControllerManagerImpl::Create( + const ProtoString& config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled) { + return Create(config_string, num_encoder_channels, encoder_frame_lengths_ms, + min_encoder_bitrate_bps, intial_channels_to_encode, + initial_frame_length_ms, initial_bitrate_bps, + initial_fec_enabled, initial_dtx_enabled, nullptr); +} + +std::unique_ptr ControllerManagerImpl::Create( + const ProtoString& config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled, + DebugDumpWriter* debug_dump_writer) { +#if WEBRTC_ENABLE_PROTOBUF + audio_network_adaptor::config::ControllerManager controller_manager_config; + RTC_CHECK(controller_manager_config.ParseFromString(config_string)); + if (debug_dump_writer) + debug_dump_writer->DumpControllerManagerConfig(controller_manager_config, + rtc::TimeMillis()); + + std::vector> controllers; + std::map> scoring_points; + + for (int i = 0; i < controller_manager_config.controllers_size(); ++i) { + auto& controller_config = controller_manager_config.controllers(i); + std::unique_ptr controller; + switch (controller_config.controller_case()) { + case audio_network_adaptor::config::Controller::kFecController: + controller = CreateFecControllerPlrBased( + controller_config.fec_controller(), initial_fec_enabled); + break; + case audio_network_adaptor::config::Controller::kFecControllerRplrBased: + controller = CreateFecControllerRplrBased( + controller_config.fec_controller_rplr_based(), initial_fec_enabled); + break; + case audio_network_adaptor::config::Controller::kFrameLengthController: + controller = CreateFrameLengthController( + controller_config.frame_length_controller(), + encoder_frame_lengths_ms, initial_frame_length_ms, + min_encoder_bitrate_bps); + break; + case audio_network_adaptor::config::Controller::kChannelController: + controller = CreateChannelController( + controller_config.channel_controller(), num_encoder_channels, + intial_channels_to_encode); + break; + case audio_network_adaptor::config::Controller::kDtxController: + controller = CreateDtxController(controller_config.dtx_controller(), + initial_dtx_enabled); + break; + case audio_network_adaptor::config::Controller::kBitrateController: + controller = CreateBitrateController( + controller_config.bitrate_controller(), initial_bitrate_bps, + initial_frame_length_ms); + break; + default: + RTC_NOTREACHED(); + } + if (controller_config.has_scoring_point()) { + auto& scoring_point = controller_config.scoring_point(); + RTC_CHECK(scoring_point.has_uplink_bandwidth_bps()); + RTC_CHECK(scoring_point.has_uplink_packet_loss_fraction()); + scoring_points[controller.get()] = std::make_pair( + scoring_point.uplink_bandwidth_bps(), + scoring_point.uplink_packet_loss_fraction()); + } + controllers.push_back(std::move(controller)); + } + + if (scoring_points.size() == 0) { + return std::unique_ptr(new ControllerManagerImpl( + ControllerManagerImpl::Config(0, 0), std::move(controllers), + scoring_points)); + } else { + RTC_CHECK(controller_manager_config.has_min_reordering_time_ms()); + RTC_CHECK(controller_manager_config.has_min_reordering_squared_distance()); + return std::unique_ptr(new ControllerManagerImpl( + ControllerManagerImpl::Config( + controller_manager_config.min_reordering_time_ms(), + controller_manager_config.min_reordering_squared_distance()), + std::move(controllers), scoring_points)); + } + +#else + RTC_NOTREACHED(); + return nullptr; +#endif // WEBRTC_ENABLE_PROTOBUF +} + +ControllerManagerImpl::ControllerManagerImpl(const Config& config) + : ControllerManagerImpl( + config, + std::vector>(), + std::map>()) {} + +ControllerManagerImpl::ControllerManagerImpl( + const Config& config, + std::vector> controllers, + const std::map>& scoring_points) + : config_(config), + controllers_(std::move(controllers)), + last_reordering_time_ms_(rtc::nullopt), + last_scoring_point_(0, 0.0) { + for (auto& controller : controllers_) + default_sorted_controllers_.push_back(controller.get()); + sorted_controllers_ = default_sorted_controllers_; + for (auto& controller_point : scoring_points) { + controller_scoring_points_.insert(std::make_pair( + controller_point.first, ScoringPoint(controller_point.second.first, + controller_point.second.second))); + } +} + +ControllerManagerImpl::~ControllerManagerImpl() = default; + +std::vector ControllerManagerImpl::GetSortedControllers( + const Controller::NetworkMetrics& metrics) { + if (controller_scoring_points_.size() == 0) + return default_sorted_controllers_; + + if (!metrics.uplink_bandwidth_bps || !metrics.uplink_packet_loss_fraction) + return sorted_controllers_; + + const int64_t now_ms = rtc::TimeMillis(); + if (last_reordering_time_ms_ && + now_ms - *last_reordering_time_ms_ < config_.min_reordering_time_ms) + return sorted_controllers_; + + ScoringPoint scoring_point(*metrics.uplink_bandwidth_bps, + *metrics.uplink_packet_loss_fraction); + + if (last_reordering_time_ms_ && + last_scoring_point_.SquaredDistanceTo(scoring_point) < + config_.min_reordering_squared_distance) + return sorted_controllers_; + + // Sort controllers according to the distances of |scoring_point| to the + // scoring points of controllers. + // + // A controller that does not associate with any scoring point + // are treated as if + // 1) they are less important than any controller that has a scoring point, + // 2) they are equally important to any controller that has no scoring point, + // and their relative order will follow |default_sorted_controllers_|. + std::vector sorted_controllers(default_sorted_controllers_); + std::stable_sort( + sorted_controllers.begin(), sorted_controllers.end(), + [this, &scoring_point](const Controller* lhs, const Controller* rhs) { + auto lhs_scoring_point = controller_scoring_points_.find(lhs); + auto rhs_scoring_point = controller_scoring_points_.find(rhs); + + if (lhs_scoring_point == controller_scoring_points_.end()) + return false; + + if (rhs_scoring_point == controller_scoring_points_.end()) + return true; + + return lhs_scoring_point->second.SquaredDistanceTo(scoring_point) < + rhs_scoring_point->second.SquaredDistanceTo(scoring_point); + }); + + if (sorted_controllers_ != sorted_controllers) { + sorted_controllers_ = sorted_controllers; + last_reordering_time_ms_ = now_ms; + last_scoring_point_ = scoring_point; + } + return sorted_controllers_; +} + +std::vector ControllerManagerImpl::GetControllers() const { + return default_sorted_controllers_; +} + +ControllerManagerImpl::ScoringPoint::ScoringPoint( + int uplink_bandwidth_bps, + float uplink_packet_loss_fraction) + : uplink_bandwidth_bps(uplink_bandwidth_bps), + uplink_packet_loss_fraction(uplink_packet_loss_fraction) {} + +namespace { + +constexpr int kMinUplinkBandwidthBps = 0; +constexpr int kMaxUplinkBandwidthBps = 120000; + +float NormalizeUplinkBandwidth(int uplink_bandwidth_bps) { + uplink_bandwidth_bps = + std::min(kMaxUplinkBandwidthBps, + std::max(kMinUplinkBandwidthBps, uplink_bandwidth_bps)); + return static_cast(uplink_bandwidth_bps - kMinUplinkBandwidthBps) / + (kMaxUplinkBandwidthBps - kMinUplinkBandwidthBps); +} + +float NormalizePacketLossFraction(float uplink_packet_loss_fraction) { + // |uplink_packet_loss_fraction| is seldom larger than 0.3, so we scale it up + // by 3.3333f. + return std::min(uplink_packet_loss_fraction * 3.3333f, 1.0f); +} + +} // namespace + +float ControllerManagerImpl::ScoringPoint::SquaredDistanceTo( + const ScoringPoint& scoring_point) const { + float diff_normalized_bitrate_bps = + NormalizeUplinkBandwidth(scoring_point.uplink_bandwidth_bps) - + NormalizeUplinkBandwidth(uplink_bandwidth_bps); + float diff_normalized_packet_loss = + NormalizePacketLossFraction(scoring_point.uplink_packet_loss_fraction) - + NormalizePacketLossFraction(uplink_packet_loss_fraction); + return std::pow(diff_normalized_bitrate_bps, 2) + + std::pow(diff_normalized_packet_loss, 2); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h new file mode 100644 index 0000000000..5c63f2f514 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_MANAGER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_MANAGER_H_ + +#include +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/protobuf_utils.h" + +namespace webrtc { + +class DebugDumpWriter; + +class ControllerManager { + public: + virtual ~ControllerManager() = default; + + // Sort controllers based on their significance. + virtual std::vector GetSortedControllers( + const Controller::NetworkMetrics& metrics) = 0; + + virtual std::vector GetControllers() const = 0; +}; + +class ControllerManagerImpl final : public ControllerManager { + public: + struct Config { + Config(int min_reordering_time_ms, float min_reordering_squared_distance); + ~Config(); + // Least time since last reordering for a new reordering to be made. + int min_reordering_time_ms; + // Least squared distance from last scoring point for a new reordering to be + // made. + float min_reordering_squared_distance; + }; + + static std::unique_ptr Create( + const ProtoString& config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled); + + static std::unique_ptr Create( + const ProtoString& config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled, + DebugDumpWriter* debug_dump_writer); + + explicit ControllerManagerImpl(const Config& config); + + // Dependency injection for testing. + ControllerManagerImpl( + const Config& config, + std::vector> controllers, + const std::map>& + chracteristic_points); + + ~ControllerManagerImpl() override; + + // Sort controllers based on their significance. + std::vector GetSortedControllers( + const Controller::NetworkMetrics& metrics) override; + + std::vector GetControllers() const override; + + private: + // Scoring point is a subset of NetworkMetrics that is used for comparing the + // significance of controllers. + struct ScoringPoint { + // TODO(eladalon): Do we want to experiment with RPLR-based scoring? + ScoringPoint(int uplink_bandwidth_bps, float uplink_packet_loss_fraction); + + // Calculate the normalized [0,1] distance between two scoring points. + float SquaredDistanceTo(const ScoringPoint& scoring_point) const; + + int uplink_bandwidth_bps; + float uplink_packet_loss_fraction; + }; + + const Config config_; + + std::vector> controllers_; + + rtc::Optional last_reordering_time_ms_; + ScoringPoint last_scoring_point_; + + std::vector default_sorted_controllers_; + + std::vector sorted_controllers_; + + // |scoring_points_| saves the scoring points of various + // controllers. + std::map controller_scoring_points_; + + RTC_DISALLOW_COPY_AND_ASSIGN(ControllerManagerImpl); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_MANAGER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc new file mode 100644 index 0000000000..576661cd35 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc @@ -0,0 +1,472 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_controller.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h" +#include "rtc_base/fakeclock.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/protobuf_utils.h" +#include "test/gtest.h" + +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/config.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +using ::testing::_; +using ::testing::NiceMock; + +namespace { + +constexpr size_t kNumControllers = 4; +constexpr int kChracteristicBandwithBps[2] = {15000, 0}; +constexpr float kChracteristicPacketLossFraction[2] = {0.2f, 0.0f}; +constexpr int kMinReorderingTimeMs = 200; +constexpr int kFactor = 100; +constexpr float kMinReorderingSquareDistance = 1.0f / kFactor / kFactor; + +// |kMinUplinkBandwidthBps| and |kMaxUplinkBandwidthBps| are copied from +// controller_manager.cc +constexpr int kMinUplinkBandwidthBps = 0; +constexpr int kMaxUplinkBandwidthBps = 120000; +constexpr int kMinBandwithChangeBps = + (kMaxUplinkBandwidthBps - kMinUplinkBandwidthBps) / kFactor; + +struct ControllerManagerStates { + std::unique_ptr controller_manager; + std::vector mock_controllers; +}; + +ControllerManagerStates CreateControllerManager() { + ControllerManagerStates states; + std::vector> controllers; + std::map> chracteristic_points; + for (size_t i = 0; i < kNumControllers; ++i) { + auto controller = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*controller, Die()); + states.mock_controllers.push_back(controller.get()); + controllers.push_back(std::move(controller)); + } + + // Assign characteristic points to the last two controllers. + chracteristic_points[states.mock_controllers[kNumControllers - 2]] = + std::make_pair(kChracteristicBandwithBps[0], + kChracteristicPacketLossFraction[0]); + chracteristic_points[states.mock_controllers[kNumControllers - 1]] = + std::make_pair(kChracteristicBandwithBps[1], + kChracteristicPacketLossFraction[1]); + + states.controller_manager.reset(new ControllerManagerImpl( + ControllerManagerImpl::Config(kMinReorderingTimeMs, + kMinReorderingSquareDistance), + std::move(controllers), chracteristic_points)); + return states; +} + +// |expected_order| contains the expected indices of all controllers in the +// vector of controllers returned by GetSortedControllers(). A negative index +// means that we do not care about its exact place, but we do check that it +// exists in the vector. +void CheckControllersOrder( + ControllerManagerStates* states, + const rtc::Optional& uplink_bandwidth_bps, + const rtc::Optional& uplink_packet_loss_fraction, + const std::vector& expected_order) { + RTC_DCHECK_EQ(kNumControllers, expected_order.size()); + Controller::NetworkMetrics metrics; + metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + metrics.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + auto check = states->controller_manager->GetSortedControllers(metrics); + EXPECT_EQ(states->mock_controllers.size(), check.size()); + for (size_t i = 0; i < states->mock_controllers.size(); ++i) { + if (expected_order[i] >= 0) { + EXPECT_EQ(states->mock_controllers[i], check[expected_order[i]]); + } else { + EXPECT_NE(check.end(), std::find(check.begin(), check.end(), + states->mock_controllers[i])); + } + } +} + +} // namespace + +TEST(ControllerManagerTest, GetControllersReturnAllControllers) { + auto states = CreateControllerManager(); + auto check = states.controller_manager->GetControllers(); + // Verify that controllers in |check| are one-to-one mapped to those in + // |mock_controllers_|. + EXPECT_EQ(states.mock_controllers.size(), check.size()); + for (auto& controller : check) + EXPECT_NE(states.mock_controllers.end(), + std::find(states.mock_controllers.begin(), + states.mock_controllers.end(), controller)); +} + +TEST(ControllerManagerTest, ControllersInDefaultOrderOnEmptyNetworkMetrics) { + auto states = CreateControllerManager(); + // |network_metrics| are empty, and the controllers are supposed to follow the + // default order. + CheckControllersOrder(&states, rtc::nullopt, rtc::nullopt, + {0, 1, 2, 3}); +} + +TEST(ControllerManagerTest, ControllersWithoutCharPointAtEndAndInDefaultOrder) { + auto states = CreateControllerManager(); + CheckControllersOrder(&states, 0, + 0.0, + {kNumControllers - 2, kNumControllers - 1, -1, -1}); +} + +TEST(ControllerManagerTest, ControllersWithCharPointDependOnNetworkMetrics) { + auto states = CreateControllerManager(); + CheckControllersOrder(&states, kChracteristicBandwithBps[1], + kChracteristicPacketLossFraction[1], + {kNumControllers - 2, kNumControllers - 1, 1, 0}); +} + +TEST(ControllerManagerTest, DoNotReorderBeforeMinReordingTime) { + rtc::ScopedFakeClock fake_clock; + auto states = CreateControllerManager(); + CheckControllersOrder(&states, kChracteristicBandwithBps[0], + kChracteristicPacketLossFraction[0], + {kNumControllers - 2, kNumControllers - 1, 0, 1}); + fake_clock.AdvanceTime( + rtc::TimeDelta::FromMilliseconds(kMinReorderingTimeMs - 1)); + // Move uplink bandwidth and packet loss fraction to the other controller's + // characteristic point, which would cause controller manager to reorder the + // controllers if time had reached min reordering time. + CheckControllersOrder(&states, kChracteristicBandwithBps[1], + kChracteristicPacketLossFraction[1], + {kNumControllers - 2, kNumControllers - 1, 0, 1}); +} + +TEST(ControllerManagerTest, ReorderBeyondMinReordingTimeAndMinDistance) { + rtc::ScopedFakeClock fake_clock; + auto states = CreateControllerManager(); + constexpr int kBandwidthBps = + (kChracteristicBandwithBps[0] + kChracteristicBandwithBps[1]) / 2; + constexpr float kPacketLossFraction = (kChracteristicPacketLossFraction[0] + + kChracteristicPacketLossFraction[1]) / + 2.0f; + // Set network metrics to be in the middle between the characteristic points + // of two controllers. + CheckControllersOrder(&states, kBandwidthBps, kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 0, 1}); + fake_clock.AdvanceTime( + rtc::TimeDelta::FromMilliseconds(kMinReorderingTimeMs)); + // Then let network metrics move a little towards the other controller. + CheckControllersOrder(&states, kBandwidthBps - kMinBandwithChangeBps - 1, + kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 1, 0}); +} + +TEST(ControllerManagerTest, DoNotReorderIfNetworkMetricsChangeTooSmall) { + rtc::ScopedFakeClock fake_clock; + auto states = CreateControllerManager(); + constexpr int kBandwidthBps = + (kChracteristicBandwithBps[0] + kChracteristicBandwithBps[1]) / 2; + constexpr float kPacketLossFraction = (kChracteristicPacketLossFraction[0] + + kChracteristicPacketLossFraction[1]) / + 2.0f; + // Set network metrics to be in the middle between the characteristic points + // of two controllers. + CheckControllersOrder(&states, kBandwidthBps, kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 0, 1}); + fake_clock.AdvanceTime( + rtc::TimeDelta::FromMilliseconds(kMinReorderingTimeMs)); + // Then let network metrics move a little towards the other controller. + CheckControllersOrder(&states, kBandwidthBps - kMinBandwithChangeBps + 1, + kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 0, 1}); +} + +#if WEBRTC_ENABLE_PROTOBUF + +namespace { + +void AddBitrateControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + config->add_controllers()->mutable_bitrate_controller(); +} + +void AddChannelControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config = + config->add_controllers()->mutable_channel_controller(); + controller_config->set_channel_1_to_2_bandwidth_bps(31000); + controller_config->set_channel_2_to_1_bandwidth_bps(29000); +} + +void AddDtxControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config = config->add_controllers()->mutable_dtx_controller(); + controller_config->set_dtx_enabling_bandwidth_bps(55000); + controller_config->set_dtx_disabling_bandwidth_bps(65000); +} + +void AddFecControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config_ext = config->add_controllers(); + auto controller_config = controller_config_ext->mutable_fec_controller(); + auto fec_enabling_threshold = + controller_config->mutable_fec_enabling_threshold(); + fec_enabling_threshold->set_low_bandwidth_bps(17000); + fec_enabling_threshold->set_low_bandwidth_packet_loss(0.1f); + fec_enabling_threshold->set_high_bandwidth_bps(64000); + fec_enabling_threshold->set_high_bandwidth_packet_loss(0.05f); + auto fec_disabling_threshold = + controller_config->mutable_fec_disabling_threshold(); + fec_disabling_threshold->set_low_bandwidth_bps(15000); + fec_disabling_threshold->set_low_bandwidth_packet_loss(0.08f); + fec_disabling_threshold->set_high_bandwidth_bps(64000); + fec_disabling_threshold->set_high_bandwidth_packet_loss(0.01f); + controller_config->set_time_constant_ms(500); + + auto scoring_point = controller_config_ext->mutable_scoring_point(); + scoring_point->set_uplink_bandwidth_bps(kChracteristicBandwithBps[0]); + scoring_point->set_uplink_packet_loss_fraction( + kChracteristicPacketLossFraction[0]); +} + +void AddFrameLengthControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config_ext = config->add_controllers(); + auto controller_config = + controller_config_ext->mutable_frame_length_controller(); + controller_config->set_fl_decreasing_packet_loss_fraction(0.05f); + controller_config->set_fl_increasing_packet_loss_fraction(0.04f); + controller_config->set_fl_20ms_to_60ms_bandwidth_bps(72000); + controller_config->set_fl_60ms_to_20ms_bandwidth_bps(88000); + + auto scoring_point = controller_config_ext->mutable_scoring_point(); + scoring_point->set_uplink_bandwidth_bps(kChracteristicBandwithBps[1]); + scoring_point->set_uplink_packet_loss_fraction( + kChracteristicPacketLossFraction[1]); +} + +constexpr int kInitialBitrateBps = 24000; +constexpr size_t kIntialChannelsToEncode = 1; +constexpr bool kInitialDtxEnabled = true; +constexpr bool kInitialFecEnabled = true; +constexpr int kInitialFrameLengthMs = 60; +constexpr int kMinBitrateBps = 6000; + +ControllerManagerStates CreateControllerManager( + const ProtoString& config_string) { + ControllerManagerStates states; + constexpr size_t kNumEncoderChannels = 2; + const std::vector encoder_frame_lengths_ms = {20, 60}; + states.controller_manager = ControllerManagerImpl::Create( + config_string, kNumEncoderChannels, encoder_frame_lengths_ms, + kMinBitrateBps, kIntialChannelsToEncode, kInitialFrameLengthMs, + kInitialBitrateBps, kInitialFecEnabled, kInitialDtxEnabled); + return states; +} + +enum class ControllerType : int8_t { + FEC, + CHANNEL, + DTX, + FRAME_LENGTH, + BIT_RATE +}; + +void CheckControllersOrder(const std::vector& controllers, + const std::vector& expected_types) { + ASSERT_EQ(expected_types.size(), controllers.size()); + + // We also check that the controllers follow the initial settings. + AudioEncoderRuntimeConfig encoder_config; + + for (size_t i = 0; i < controllers.size(); ++i) { + AudioEncoderRuntimeConfig encoder_config; + // We check the order of |controllers| by judging their decisions. + controllers[i]->MakeDecision(&encoder_config); + + // Since controllers are not provided with network metrics, they give the + // initial values. + switch (expected_types[i]) { + case ControllerType::FEC: + EXPECT_EQ(kInitialFecEnabled, encoder_config.enable_fec); + break; + case ControllerType::CHANNEL: + EXPECT_EQ(kIntialChannelsToEncode, encoder_config.num_channels); + break; + case ControllerType::DTX: + EXPECT_EQ(kInitialDtxEnabled, encoder_config.enable_dtx); + break; + case ControllerType::FRAME_LENGTH: + EXPECT_EQ(kInitialFrameLengthMs, encoder_config.frame_length_ms); + break; + case ControllerType::BIT_RATE: + EXPECT_EQ(kInitialBitrateBps, encoder_config.bitrate_bps); + } + } +} + +MATCHER_P(ControllerManagerEqual, value, "") { + ProtoString value_string; + ProtoString arg_string; + EXPECT_TRUE(arg.SerializeToString(&arg_string)); + EXPECT_TRUE(value.SerializeToString(&value_string)); + return arg_string == value_string; +} + +} // namespace + +TEST(ControllerManagerTest, DebugDumpLoggedWhenCreateFromConfigString) { + audio_network_adaptor::config::ControllerManager config; + config.set_min_reordering_time_ms(kMinReorderingTimeMs); + config.set_min_reordering_squared_distance(kMinReorderingSquareDistance); + + AddFecControllerConfig(&config); + AddChannelControllerConfig(&config); + AddDtxControllerConfig(&config); + AddFrameLengthControllerConfig(&config); + AddBitrateControllerConfig(&config); + + ProtoString config_string; + config.SerializeToString(&config_string); + + constexpr size_t kNumEncoderChannels = 2; + const std::vector encoder_frame_lengths_ms = {20, 60}; + + constexpr int64_t kClockInitialTimeMs = 12345678; + rtc::ScopedFakeClock fake_clock; + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kClockInitialTimeMs)); + auto debug_dump_writer = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*debug_dump_writer, Die()); + EXPECT_CALL(*debug_dump_writer, + DumpControllerManagerConfig(ControllerManagerEqual(config), + kClockInitialTimeMs)); + + ControllerManagerImpl::Create(config_string, kNumEncoderChannels, + encoder_frame_lengths_ms, kMinBitrateBps, + kIntialChannelsToEncode, kInitialFrameLengthMs, + kInitialBitrateBps, kInitialFecEnabled, + kInitialDtxEnabled, debug_dump_writer.get()); +} + +TEST(ControllerManagerTest, CreateFromConfigStringAndCheckDefaultOrder) { + audio_network_adaptor::config::ControllerManager config; + config.set_min_reordering_time_ms(kMinReorderingTimeMs); + config.set_min_reordering_squared_distance(kMinReorderingSquareDistance); + + AddFecControllerConfig(&config); + AddChannelControllerConfig(&config); + AddDtxControllerConfig(&config); + AddFrameLengthControllerConfig(&config); + AddBitrateControllerConfig(&config); + + ProtoString config_string; + config.SerializeToString(&config_string); + + auto states = CreateControllerManager(config_string); + Controller::NetworkMetrics metrics; + + auto controllers = states.controller_manager->GetSortedControllers(metrics); + CheckControllersOrder( + controllers, + std::vector{ + ControllerType::FEC, ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::FRAME_LENGTH, ControllerType::BIT_RATE}); +} + +TEST(ControllerManagerTest, CreateCharPointFreeConfigAndCheckDefaultOrder) { + audio_network_adaptor::config::ControllerManager config; + + // Following controllers have no characteristic points. + AddChannelControllerConfig(&config); + AddDtxControllerConfig(&config); + AddBitrateControllerConfig(&config); + + ProtoString config_string; + config.SerializeToString(&config_string); + + auto states = CreateControllerManager(config_string); + Controller::NetworkMetrics metrics; + + auto controllers = states.controller_manager->GetSortedControllers(metrics); + CheckControllersOrder( + controllers, + std::vector{ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); +} + +TEST(ControllerManagerTest, CreateFromConfigStringAndCheckReordering) { + rtc::ScopedFakeClock fake_clock; + audio_network_adaptor::config::ControllerManager config; + config.set_min_reordering_time_ms(kMinReorderingTimeMs); + config.set_min_reordering_squared_distance(kMinReorderingSquareDistance); + + AddChannelControllerConfig(&config); + + // Internally associated with characteristic point 0. + AddFecControllerConfig(&config); + + AddDtxControllerConfig(&config); + + // Internally associated with characteristic point 1. + AddFrameLengthControllerConfig(&config); + + AddBitrateControllerConfig(&config); + + ProtoString config_string; + config.SerializeToString(&config_string); + + auto states = CreateControllerManager(config_string); + + Controller::NetworkMetrics metrics; + metrics.uplink_bandwidth_bps = kChracteristicBandwithBps[0]; + metrics.uplink_packet_loss_fraction = kChracteristicPacketLossFraction[0]; + + auto controllers = states.controller_manager->GetSortedControllers(metrics); + CheckControllersOrder(controllers, + std::vector{ + ControllerType::FEC, ControllerType::FRAME_LENGTH, + ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); + + metrics.uplink_bandwidth_bps = kChracteristicBandwithBps[1]; + metrics.uplink_packet_loss_fraction = kChracteristicPacketLossFraction[1]; + fake_clock.AdvanceTime( + rtc::TimeDelta::FromMilliseconds(kMinReorderingTimeMs - 1)); + controllers = states.controller_manager->GetSortedControllers(metrics); + // Should not reorder since min reordering time is not met. + CheckControllersOrder(controllers, + std::vector{ + ControllerType::FEC, ControllerType::FRAME_LENGTH, + ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); + + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(1)); + controllers = states.controller_manager->GetSortedControllers(metrics); + // Reorder now. + CheckControllersOrder(controllers, + std::vector{ + ControllerType::FRAME_LENGTH, ControllerType::FEC, + ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); +} +#endif // WEBRTC_ENABLE_PROTOBUF + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto new file mode 100644 index 0000000000..93b31c3fe8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto @@ -0,0 +1,42 @@ +syntax = "proto2"; +option optimize_for = LITE_RUNTIME; +package webrtc.audio_network_adaptor.debug_dump; + +import "config.proto"; + +message NetworkMetrics { + optional int32 uplink_bandwidth_bps = 1; + optional float uplink_packet_loss_fraction = 2; + optional int32 target_audio_bitrate_bps = 3; + optional int32 rtt_ms = 4; + optional int32 uplink_recoverable_packet_loss_fraction = 5; +} + +message EncoderRuntimeConfig { + optional int32 bitrate_bps = 1; + optional int32 frame_length_ms = 2; + // Note: This is what we tell the encoder. It doesn't have to reflect + // the actual NetworkMetrics; it's subject to our decision. + optional float uplink_packet_loss_fraction = 3; + optional bool enable_fec = 4; + optional bool enable_dtx = 5; + // Some encoders can encode fewer channels than the actual input to make + // better use of the bandwidth. |num_channels| sets the number of channels + // to encode. + optional uint32 num_channels = 6; +} + +message Event { + enum Type { + NETWORK_METRICS = 0; + ENCODER_RUNTIME_CONFIG = 1; + CONTROLLER_MANAGER_CONFIG = 2; + } + required Type type = 1; + required uint32 timestamp = 2; + optional NetworkMetrics network_metrics = 3; + optional EncoderRuntimeConfig encoder_runtime_config = 4; + optional webrtc.audio_network_adaptor.config.ControllerManager + controller_manager_config = 5; +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc new file mode 100644 index 0000000000..97de177164 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" + +#include "rtc_base/checks.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/protobuf_utils.h" + +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/debug_dump.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +#if WEBRTC_ENABLE_PROTOBUF +namespace { + +using audio_network_adaptor::debug_dump::Event; +using audio_network_adaptor::debug_dump::NetworkMetrics; +using audio_network_adaptor::debug_dump::EncoderRuntimeConfig; + +void DumpEventToFile(const Event& event, FileWrapper* dump_file) { + RTC_CHECK(dump_file->is_open()); + ProtoString dump_data; + event.SerializeToString(&dump_data); + int32_t size = event.ByteSize(); + dump_file->Write(&size, sizeof(size)); + dump_file->Write(dump_data.data(), dump_data.length()); +} + +} // namespace +#endif // WEBRTC_ENABLE_PROTOBUF + +class DebugDumpWriterImpl final : public DebugDumpWriter { + public: + explicit DebugDumpWriterImpl(FILE* file_handle); + ~DebugDumpWriterImpl() override = default; + + void DumpEncoderRuntimeConfig(const AudioEncoderRuntimeConfig& config, + int64_t timestamp) override; + + void DumpNetworkMetrics(const Controller::NetworkMetrics& metrics, + int64_t timestamp) override; + +#if WEBRTC_ENABLE_PROTOBUF + void DumpControllerManagerConfig( + const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp) override; +#endif + + private: + std::unique_ptr dump_file_; +}; + +DebugDumpWriterImpl::DebugDumpWriterImpl(FILE* file_handle) + : dump_file_(FileWrapper::Create()) { +#if WEBRTC_ENABLE_PROTOBUF + dump_file_->OpenFromFileHandle(file_handle); + RTC_CHECK(dump_file_->is_open()); +#else + RTC_NOTREACHED(); +#endif +} + +void DebugDumpWriterImpl::DumpNetworkMetrics( + const Controller::NetworkMetrics& metrics, + int64_t timestamp) { +#if WEBRTC_ENABLE_PROTOBUF + Event event; + event.set_timestamp(timestamp); + event.set_type(Event::NETWORK_METRICS); + auto dump_metrics = event.mutable_network_metrics(); + + if (metrics.uplink_bandwidth_bps) + dump_metrics->set_uplink_bandwidth_bps(*metrics.uplink_bandwidth_bps); + + if (metrics.uplink_packet_loss_fraction) { + dump_metrics->set_uplink_packet_loss_fraction( + *metrics.uplink_packet_loss_fraction); + } + + if (metrics.target_audio_bitrate_bps) { + dump_metrics->set_target_audio_bitrate_bps( + *metrics.target_audio_bitrate_bps); + } + + if (metrics.rtt_ms) + dump_metrics->set_rtt_ms(*metrics.rtt_ms); + + if (metrics.uplink_recoverable_packet_loss_fraction) { + dump_metrics->set_uplink_recoverable_packet_loss_fraction( + *metrics.uplink_recoverable_packet_loss_fraction); + } + + DumpEventToFile(event, dump_file_.get()); +#endif // WEBRTC_ENABLE_PROTOBUF +} + +void DebugDumpWriterImpl::DumpEncoderRuntimeConfig( + const AudioEncoderRuntimeConfig& config, + int64_t timestamp) { +#if WEBRTC_ENABLE_PROTOBUF + Event event; + event.set_timestamp(timestamp); + event.set_type(Event::ENCODER_RUNTIME_CONFIG); + auto dump_config = event.mutable_encoder_runtime_config(); + + if (config.bitrate_bps) + dump_config->set_bitrate_bps(*config.bitrate_bps); + + if (config.frame_length_ms) + dump_config->set_frame_length_ms(*config.frame_length_ms); + + if (config.uplink_packet_loss_fraction) { + dump_config->set_uplink_packet_loss_fraction( + *config.uplink_packet_loss_fraction); + } + + if (config.enable_fec) + dump_config->set_enable_fec(*config.enable_fec); + + if (config.enable_dtx) + dump_config->set_enable_dtx(*config.enable_dtx); + + if (config.num_channels) + dump_config->set_num_channels(*config.num_channels); + + DumpEventToFile(event, dump_file_.get()); +#endif // WEBRTC_ENABLE_PROTOBUF +} + +#if WEBRTC_ENABLE_PROTOBUF +void DebugDumpWriterImpl::DumpControllerManagerConfig( + const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp) { + Event event; + event.set_timestamp(timestamp); + event.set_type(Event::CONTROLLER_MANAGER_CONFIG); + event.mutable_controller_manager_config()->CopyFrom( + controller_manager_config); + DumpEventToFile(event, dump_file_.get()); +} +#endif // WEBRTC_ENABLE_PROTOBUF + +std::unique_ptr DebugDumpWriter::Create(FILE* file_handle) { + return std::unique_ptr(new DebugDumpWriterImpl(file_handle)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h new file mode 100644 index 0000000000..e40c8328af --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DEBUG_DUMP_WRITER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DEBUG_DUMP_WRITER_H_ + +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/ignore_wundef.h" +#include "system_wrappers/include/file_wrapper.h" +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/config.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +class DebugDumpWriter { + public: + static std::unique_ptr Create(FILE* file_handle); + + virtual ~DebugDumpWriter() = default; + + virtual void DumpEncoderRuntimeConfig(const AudioEncoderRuntimeConfig& config, + int64_t timestamp) = 0; + + virtual void DumpNetworkMetrics(const Controller::NetworkMetrics& metrics, + int64_t timestamp) = 0; + +#if WEBRTC_ENABLE_PROTOBUF + virtual void DumpControllerManagerConfig( + const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp) = 0; +#endif +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DEBUG_DUMP_WRITER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc new file mode 100644 index 0000000000..cbfea95bd1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +DtxController::Config::Config(bool initial_dtx_enabled, + int dtx_enabling_bandwidth_bps, + int dtx_disabling_bandwidth_bps) + : initial_dtx_enabled(initial_dtx_enabled), + dtx_enabling_bandwidth_bps(dtx_enabling_bandwidth_bps), + dtx_disabling_bandwidth_bps(dtx_disabling_bandwidth_bps) {} + +DtxController::DtxController(const Config& config) + : config_(config), dtx_enabled_(config_.initial_dtx_enabled) {} + +DtxController::~DtxController() = default; + +void DtxController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; +} + +void DtxController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on |enable_dtx| should not have been made. + RTC_DCHECK(!config->enable_dtx); + + if (uplink_bandwidth_bps_) { + if (dtx_enabled_ && + *uplink_bandwidth_bps_ >= config_.dtx_disabling_bandwidth_bps) { + dtx_enabled_ = false; + } else if (!dtx_enabled_ && + *uplink_bandwidth_bps_ <= config_.dtx_enabling_bandwidth_bps) { + dtx_enabled_ = true; + } + } + config->enable_dtx = dtx_enabled_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h new file mode 100644 index 0000000000..8a2427ed7c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DTX_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DTX_CONTROLLER_H_ + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class DtxController final : public Controller { + public: + struct Config { + Config(bool initial_dtx_enabled, + int dtx_enabling_bandwidth_bps, + int dtx_disabling_bandwidth_bps); + bool initial_dtx_enabled; + // Uplink bandwidth below which DTX should be switched on. + int dtx_enabling_bandwidth_bps; + // Uplink bandwidth above which DTX should be switched off. + int dtx_disabling_bandwidth_bps; + }; + + explicit DtxController(const Config& config); + + ~DtxController() override; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + const Config config_; + bool dtx_enabled_; + rtc::Optional uplink_bandwidth_bps_; + RTC_DISALLOW_COPY_AND_ASSIGN(DtxController); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DTX_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc new file mode 100644 index 0000000000..e38e65d8f5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kDtxEnablingBandwidthBps = 55000; +constexpr int kDtxDisablingBandwidthBps = 65000; +constexpr int kMediumBandwidthBps = + (kDtxEnablingBandwidthBps + kDtxDisablingBandwidthBps) / 2; + +std::unique_ptr CreateController(int initial_dtx_enabled) { + std::unique_ptr controller(new DtxController( + DtxController::Config(initial_dtx_enabled, kDtxEnablingBandwidthBps, + kDtxDisablingBandwidthBps))); + return controller; +} + +void CheckDecision(DtxController* controller, + const rtc::Optional& uplink_bandwidth_bps, + bool expected_dtx_enabled) { + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + AudioEncoderRuntimeConfig config; + controller->MakeDecision(&config); + EXPECT_EQ(expected_dtx_enabled, config.enable_dtx); +} + +} // namespace + +TEST(DtxControllerTest, OutputInitValueWhenUplinkBandwidthUnknown) { + constexpr bool kInitialDtxEnabled = true; + auto controller = CreateController(kInitialDtxEnabled); + CheckDecision(controller.get(), rtc::nullopt, kInitialDtxEnabled); +} + +TEST(DtxControllerTest, TurnOnDtxForLowUplinkBandwidth) { + auto controller = CreateController(false); + CheckDecision(controller.get(), kDtxEnablingBandwidthBps, true); +} + +TEST(DtxControllerTest, TurnOffDtxForHighUplinkBandwidth) { + auto controller = CreateController(true); + CheckDecision(controller.get(), kDtxDisablingBandwidthBps, false); +} + +TEST(DtxControllerTest, MaintainDtxOffForMediumUplinkBandwidth) { + auto controller = CreateController(false); + CheckDecision(controller.get(), kMediumBandwidthBps, false); +} + +TEST(DtxControllerTest, MaintainDtxOnForMediumUplinkBandwidth) { + auto controller = CreateController(true); + CheckDecision(controller.get(), kMediumBandwidthBps, true); +} + +TEST(DtxControllerTest, CheckBehaviorOnChangingUplinkBandwidth) { + auto controller = CreateController(false); + CheckDecision(controller.get(), kMediumBandwidthBps, false); + CheckDecision(controller.get(), kDtxEnablingBandwidthBps, true); + CheckDecision(controller.get(), kMediumBandwidthBps, true); + CheckDecision(controller.get(), kDtxDisablingBandwidthBps, false); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc new file mode 100644 index 0000000000..9cdbc54c67 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include + +#include "logging/rtc_event_log/events/rtc_event_audio_network_adaptation.h" +#include "logging/rtc_event_log/rtc_event_log.h" +#include "modules/audio_coding/audio_network_adaptor/event_log_writer.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { + +EventLogWriter::EventLogWriter(RtcEventLog* event_log, + int min_bitrate_change_bps, + float min_bitrate_change_fraction, + float min_packet_loss_change_fraction) + : event_log_(event_log), + min_bitrate_change_bps_(min_bitrate_change_bps), + min_bitrate_change_fraction_(min_bitrate_change_fraction), + min_packet_loss_change_fraction_(min_packet_loss_change_fraction) { + RTC_DCHECK(event_log_); +} + +EventLogWriter::~EventLogWriter() = default; + +void EventLogWriter::MaybeLogEncoderConfig( + const AudioEncoderRuntimeConfig& config) { + if (last_logged_config_.num_channels != config.num_channels) + return LogEncoderConfig(config); + if (last_logged_config_.enable_dtx != config.enable_dtx) + return LogEncoderConfig(config); + if (last_logged_config_.enable_fec != config.enable_fec) + return LogEncoderConfig(config); + if (last_logged_config_.frame_length_ms != config.frame_length_ms) + return LogEncoderConfig(config); + if ((!last_logged_config_.bitrate_bps && config.bitrate_bps) || + (last_logged_config_.bitrate_bps && config.bitrate_bps && + std::abs(*last_logged_config_.bitrate_bps - *config.bitrate_bps) >= + std::min(static_cast(*last_logged_config_.bitrate_bps * + min_bitrate_change_fraction_), + min_bitrate_change_bps_))) { + return LogEncoderConfig(config); + } + if ((!last_logged_config_.uplink_packet_loss_fraction && + config.uplink_packet_loss_fraction) || + (last_logged_config_.uplink_packet_loss_fraction && + config.uplink_packet_loss_fraction && + fabs(*last_logged_config_.uplink_packet_loss_fraction - + *config.uplink_packet_loss_fraction) >= + min_packet_loss_change_fraction_ * + *last_logged_config_.uplink_packet_loss_fraction)) { + return LogEncoderConfig(config); + } +} + +void EventLogWriter::LogEncoderConfig(const AudioEncoderRuntimeConfig& config) { + auto config_copy = rtc::MakeUnique(config); + event_log_->Log( + rtc::MakeUnique(std::move(config_copy))); + last_logged_config_ = config; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h new file mode 100644 index 0000000000..fca8e53b66 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_EVENT_LOG_WRITER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_EVENT_LOG_WRITER_H_ + +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +class RtcEventLog; + +class EventLogWriter final { + public: + EventLogWriter(RtcEventLog* event_log, + int min_bitrate_change_bps, + float min_bitrate_change_fraction, + float min_packet_loss_change_fraction); + ~EventLogWriter(); + void MaybeLogEncoderConfig(const AudioEncoderRuntimeConfig& config); + + private: + void LogEncoderConfig(const AudioEncoderRuntimeConfig& config); + + RtcEventLog* const event_log_; + const int min_bitrate_change_bps_; + const float min_bitrate_change_fraction_; + const float min_packet_loss_change_fraction_; + AudioEncoderRuntimeConfig last_logged_config_; + RTC_DISALLOW_COPY_AND_ASSIGN(EventLogWriter); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_EVENT_LOG_WRITER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc new file mode 100644 index 0000000000..df975947ca --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "logging/rtc_event_log/events/rtc_event_audio_network_adaptation.h" +#include "logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "modules/audio_coding/audio_network_adaptor/event_log_writer.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kMinBitrateChangeBps = 5000; +constexpr float kMinPacketLossChangeFraction = 0.5; +constexpr float kMinBitrateChangeFraction = 0.25; + +constexpr int kHighBitrateBps = 70000; +constexpr int kLowBitrateBps = 10000; +constexpr int kFrameLengthMs = 60; +constexpr bool kEnableFec = true; +constexpr bool kEnableDtx = true; +constexpr float kPacketLossFraction = 0.05f; +constexpr size_t kNumChannels = 1; + +MATCHER_P(IsRtcEventAnaConfigEqualTo, config, "") { + if (arg->GetType() != RtcEvent::Type::AudioNetworkAdaptation) { + return false; + } + auto ana_event = static_cast(arg); + return *ana_event->config_ == config; +} + +struct EventLogWriterStates { + std::unique_ptr event_log_writer; + std::unique_ptr> event_log; + AudioEncoderRuntimeConfig runtime_config; +}; + +EventLogWriterStates CreateEventLogWriter() { + EventLogWriterStates state; + state.event_log.reset(new testing::StrictMock()); + state.event_log_writer.reset(new EventLogWriter( + state.event_log.get(), kMinBitrateChangeBps, kMinBitrateChangeFraction, + kMinPacketLossChangeFraction)); + state.runtime_config.bitrate_bps = kHighBitrateBps; + state.runtime_config.frame_length_ms = kFrameLengthMs; + state.runtime_config.uplink_packet_loss_fraction = kPacketLossFraction; + state.runtime_config.enable_fec = kEnableFec; + state.runtime_config.enable_dtx = kEnableDtx; + state.runtime_config.num_channels = kNumChannels; + return state; +} +} // namespace + +TEST(EventLogWriterTest, FirstConfigIsLogged) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, SameConfigIsNotLogged) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogFecStateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.enable_fec = !kEnableFec; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogDtxStateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.enable_dtx = !kEnableDtx; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogChannelChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.num_channels = kNumChannels + 1; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogFrameLengthChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.frame_length_ms = 20; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, DoNotLogSmallBitrateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.bitrate_bps = kHighBitrateBps + kMinBitrateChangeBps - 1; + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogLargeBitrateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + // At high bitrate, the min fraction rule requires a larger change than the + // min change rule. We make sure that the min change rule applies. + RTC_DCHECK_GT(kHighBitrateBps * kMinBitrateChangeFraction, + kMinBitrateChangeBps); + state.runtime_config.bitrate_bps = kHighBitrateBps + kMinBitrateChangeBps; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogMinBitrateChangeFractionOnLowBitrateChange) { + auto state = CreateEventLogWriter(); + state.runtime_config.bitrate_bps = kLowBitrateBps; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + // At high bitrate, the min change rule requires a larger change than the min + // fraction rule. We make sure that the min fraction rule applies. + state.runtime_config.bitrate_bps = + kLowBitrateBps + kLowBitrateBps * kMinBitrateChangeFraction; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, DoNotLogSmallPacketLossFractionChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.uplink_packet_loss_fraction = + kPacketLossFraction + kMinPacketLossChangeFraction * kPacketLossFraction - + 0.001f; + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogLargePacketLossFractionChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.uplink_packet_loss_fraction = + kPacketLossFraction + kMinPacketLossChangeFraction * kPacketLossFraction; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogJustOnceOnMultipleChanges) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.uplink_packet_loss_fraction = + kPacketLossFraction + kMinPacketLossChangeFraction * kPacketLossFraction; + state.runtime_config.frame_length_ms = 20; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogAfterGradualChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.bitrate_bps = kHighBitrateBps + kMinBitrateChangeBps; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + for (int bitrate_bps = kHighBitrateBps; + bitrate_bps <= kHighBitrateBps + kMinBitrateChangeBps; bitrate_bps++) { + state.runtime_config.bitrate_bps = bitrate_bps; + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc new file mode 100644 index 0000000000..62f356dd07 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { +class NullSmoothingFilter final : public SmoothingFilter { + public: + void AddSample(float sample) override { last_sample_ = sample; } + + rtc::Optional GetAverage() override { return last_sample_; } + + bool SetTimeConstantMs(int time_constant_ms) override { + RTC_NOTREACHED(); + return false; + } + + private: + rtc::Optional last_sample_; +}; +} + +FecControllerPlrBased::Config::Config( + bool initial_fec_enabled, + const ThresholdCurve& fec_enabling_threshold, + const ThresholdCurve& fec_disabling_threshold, + int time_constant_ms) + : initial_fec_enabled(initial_fec_enabled), + fec_enabling_threshold(fec_enabling_threshold), + fec_disabling_threshold(fec_disabling_threshold), + time_constant_ms(time_constant_ms) {} + +FecControllerPlrBased::FecControllerPlrBased( + const Config& config, + std::unique_ptr smoothing_filter) + : config_(config), + fec_enabled_(config.initial_fec_enabled), + packet_loss_smoother_(std::move(smoothing_filter)) { + RTC_DCHECK(config_.fec_disabling_threshold <= config_.fec_enabling_threshold); +} + +FecControllerPlrBased::FecControllerPlrBased(const Config& config) + : FecControllerPlrBased( + config, + webrtc::field_trial::FindFullName("UseTwccPlrForAna") == "Enabled" + ? std::unique_ptr(new NullSmoothingFilter()) + : std::unique_ptr( + new SmoothingFilterImpl(config.time_constant_ms))) {} + +FecControllerPlrBased::~FecControllerPlrBased() = default; + +void FecControllerPlrBased::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + if (network_metrics.uplink_packet_loss_fraction) { + packet_loss_smoother_->AddSample( + *network_metrics.uplink_packet_loss_fraction); + } +} + +void FecControllerPlrBased::MakeDecision(AudioEncoderRuntimeConfig* config) { + RTC_DCHECK(!config->enable_fec); + RTC_DCHECK(!config->uplink_packet_loss_fraction); + + const auto& packet_loss = packet_loss_smoother_->GetAverage(); + + fec_enabled_ = fec_enabled_ ? !FecDisablingDecision(packet_loss) + : FecEnablingDecision(packet_loss); + + config->enable_fec = fec_enabled_; + + config->uplink_packet_loss_fraction = packet_loss ? *packet_loss : 0.0; +} + +bool FecControllerPlrBased::FecEnablingDecision( + const rtc::Optional& packet_loss) const { + if (!uplink_bandwidth_bps_ || !packet_loss) { + return false; + } else { + // Enable when above the curve or exactly on it. + return !config_.fec_enabling_threshold.IsBelowCurve( + {static_cast(*uplink_bandwidth_bps_), *packet_loss}); + } +} + +bool FecControllerPlrBased::FecDisablingDecision( + const rtc::Optional& packet_loss) const { + if (!uplink_bandwidth_bps_ || !packet_loss) { + return false; + } else { + // Disable when below the curve. + return config_.fec_disabling_threshold.IsBelowCurve( + {static_cast(*uplink_bandwidth_bps_), *packet_loss}); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h new file mode 100644 index 0000000000..c2735370d0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_PLR_BASED_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_PLR_BASED_H_ + +#include + +#include "common_audio/smoothing_filter.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class FecControllerPlrBased final : public Controller { + public: + struct Config { + // |fec_enabling_threshold| defines a curve, above which FEC should be + // enabled. |fec_disabling_threshold| defines a curve, under which FEC + // should be disabled. See below + // + // packet-loss ^ | | + // | | | FEC + // | \ \ ON + // | FEC \ \_______ fec_enabling_threshold + // | OFF \_________ fec_disabling_threshold + // |-----------------> bandwidth + Config(bool initial_fec_enabled, + const ThresholdCurve& fec_enabling_threshold, + const ThresholdCurve& fec_disabling_threshold, + int time_constant_ms); + bool initial_fec_enabled; + ThresholdCurve fec_enabling_threshold; + ThresholdCurve fec_disabling_threshold; + int time_constant_ms; + }; + + // Dependency injection for testing. + FecControllerPlrBased(const Config& config, + std::unique_ptr smoothing_filter); + + explicit FecControllerPlrBased(const Config& config); + + ~FecControllerPlrBased() override; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + bool FecEnablingDecision(const rtc::Optional& packet_loss) const; + bool FecDisablingDecision(const rtc::Optional& packet_loss) const; + + const Config config_; + bool fec_enabled_; + rtc::Optional uplink_bandwidth_bps_; + const std::unique_ptr packet_loss_smoother_; + + RTC_DISALLOW_COPY_AND_ASSIGN(FecControllerPlrBased); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_PLR_BASED_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc new file mode 100644 index 0000000000..8636aa96b1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc @@ -0,0 +1,488 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "common_audio/mocks/mock_smoothing_filter.h" +#include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::_; + +namespace { + +// The test uses the following settings: +// +// packet-loss ^ | | +// | A| C| FEC +// | \ \ ON +// | FEC \ D\_______ +// | OFF B\_________ +// |-----------------> bandwidth +// +// A : (kDisablingBandwidthLow, kDisablingPacketLossAtLowBw) +// B : (kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw) +// C : (kEnablingBandwidthLow, kEnablingPacketLossAtLowBw) +// D : (kEnablingBandwidthHigh, kEnablingPacketLossAtHighBw) + +constexpr int kDisablingBandwidthLow = 15000; +constexpr float kDisablingPacketLossAtLowBw = 0.08f; +constexpr int kDisablingBandwidthHigh = 64000; +constexpr float kDisablingPacketLossAtHighBw = 0.01f; +constexpr int kEnablingBandwidthLow = 17000; +constexpr float kEnablingPacketLossAtLowBw = 0.1f; +constexpr int kEnablingBandwidthHigh = 64000; +constexpr float kEnablingPacketLossAtHighBw = 0.05f; + +constexpr float kEpsilon = 1e-5f; + +struct FecControllerPlrBasedTestStates { + std::unique_ptr controller; + MockSmoothingFilter* packet_loss_smoother; +}; + +FecControllerPlrBasedTestStates CreateFecControllerPlrBased( + bool initial_fec_enabled, + const ThresholdCurve& enabling_curve, + const ThresholdCurve& disabling_curve) { + FecControllerPlrBasedTestStates states; + std::unique_ptr mock_smoothing_filter( + new NiceMock()); + states.packet_loss_smoother = mock_smoothing_filter.get(); + states.controller.reset(new FecControllerPlrBased( + FecControllerPlrBased::Config(initial_fec_enabled, enabling_curve, + disabling_curve, 0), + std::move(mock_smoothing_filter))); + return states; +} + +FecControllerPlrBasedTestStates CreateFecControllerPlrBased( + bool initial_fec_enabled) { + return CreateFecControllerPlrBased( + initial_fec_enabled, + ThresholdCurve(kEnablingBandwidthLow, kEnablingPacketLossAtLowBw, + kEnablingBandwidthHigh, kEnablingPacketLossAtHighBw), + ThresholdCurve(kDisablingBandwidthLow, kDisablingPacketLossAtLowBw, + kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw)); +} + +void UpdateNetworkMetrics(FecControllerPlrBasedTestStates* states, + const rtc::Optional& uplink_bandwidth_bps, + const rtc::Optional& uplink_packet_loss) { + // UpdateNetworkMetrics can accept multiple network metric updates at once. + // However, currently, the most used case is to update one metric at a time. + // To reflect this fact, we separate the calls. + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + states->controller->UpdateNetworkMetrics(network_metrics); + } + if (uplink_packet_loss) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_packet_loss_fraction = uplink_packet_loss; + EXPECT_CALL(*states->packet_loss_smoother, AddSample(*uplink_packet_loss)); + states->controller->UpdateNetworkMetrics(network_metrics); + // This is called during CheckDecision(). + EXPECT_CALL(*states->packet_loss_smoother, GetAverage()) + .WillOnce(Return(*uplink_packet_loss)); + } +} + +// Checks that the FEC decision and |uplink_packet_loss_fraction| given by +// |states->controller->MakeDecision| matches |expected_enable_fec| and +// |expected_uplink_packet_loss_fraction|, respectively. +void CheckDecision(FecControllerPlrBasedTestStates* states, + bool expected_enable_fec, + float expected_uplink_packet_loss_fraction) { + AudioEncoderRuntimeConfig config; + states->controller->MakeDecision(&config); + EXPECT_EQ(expected_enable_fec, config.enable_fec); + EXPECT_EQ(expected_uplink_packet_loss_fraction, + config.uplink_packet_loss_fraction); +} + +} // namespace + +TEST(FecControllerPlrBasedTest, OutputInitValueBeforeAnyInputsAreReceived) { + for (bool initial_fec_enabled : {false, true}) { + auto states = CreateFecControllerPlrBased(initial_fec_enabled); + CheckDecision(&states, initial_fec_enabled, 0); + } +} + +TEST(FecControllerPlrBasedTest, OutputInitValueWhenUplinkBandwidthUnknown) { + // Regardless of the initial FEC state and the packet-loss rate, + // the initial FEC state is maintained as long as the BWE is unknown. + for (bool initial_fec_enabled : {false, true}) { + for (float packet_loss : + {kDisablingPacketLossAtLowBw - kEpsilon, kDisablingPacketLossAtLowBw, + kDisablingPacketLossAtLowBw + kEpsilon, + kEnablingPacketLossAtLowBw - kEpsilon, kEnablingPacketLossAtLowBw, + kEnablingPacketLossAtLowBw + kEpsilon}) { + auto states = CreateFecControllerPlrBased(initial_fec_enabled); + UpdateNetworkMetrics(&states, rtc::nullopt, packet_loss); + CheckDecision(&states, initial_fec_enabled, packet_loss); + } + } +} + +TEST(FecControllerPlrBasedTest, + OutputInitValueWhenUplinkPacketLossFractionUnknown) { + // Regardless of the initial FEC state and the BWE, the initial FEC state + // is maintained as long as the packet-loss rate is unknown. + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {kDisablingBandwidthLow - 1, kDisablingBandwidthLow, + kDisablingBandwidthLow + 1, kEnablingBandwidthLow - 1, + kEnablingBandwidthLow, kEnablingBandwidthLow + 1}) { + auto states = CreateFecControllerPlrBased(initial_fec_enabled); + UpdateNetworkMetrics(&states, bandwidth, rtc::nullopt); + CheckDecision(&states, initial_fec_enabled, 0.0); + } + } +} + +TEST(FecControllerPlrBasedTest, EnableFecForHighBandwidth) { + auto states = CreateFecControllerPlrBased(false); + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); +} + +TEST(FecControllerPlrBasedTest, UpdateMultipleNetworkMetricsAtOnce) { + // This test is similar to EnableFecForHighBandwidth. But instead of + // using ::UpdateNetworkMetrics(...), which calls + // FecControllerPlrBased::UpdateNetworkMetrics(...) multiple times, we + // we call it only once. This is to verify that + // FecControllerPlrBased::UpdateNetworkMetrics(...) can handle multiple + // network updates at once. This is, however, not a common use case in current + // audio_network_adaptor_impl.cc. + auto states = CreateFecControllerPlrBased(false); + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = kEnablingBandwidthHigh; + network_metrics.uplink_packet_loss_fraction = kEnablingPacketLossAtHighBw; + EXPECT_CALL(*states.packet_loss_smoother, GetAverage()) + .WillOnce(Return(kEnablingPacketLossAtHighBw)); + states.controller->UpdateNetworkMetrics(network_metrics); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForHighBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = kEnablingPacketLossAtHighBw * 0.99f; + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, EnableFecForMediumBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = + (kEnablingPacketLossAtLowBw + kEnablingPacketLossAtHighBw) / 2.0; + UpdateNetworkMetrics(&states, + (kEnablingBandwidthHigh + kEnablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, true, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForMediumBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = + kEnablingPacketLossAtLowBw * 0.49f + kEnablingPacketLossAtHighBw * 0.51f; + UpdateNetworkMetrics(&states, + (kEnablingBandwidthHigh + kEnablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, EnableFecForLowBandwidth) { + auto states = CreateFecControllerPlrBased(false); + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, + kEnablingPacketLossAtLowBw); + CheckDecision(&states, true, kEnablingPacketLossAtLowBw); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForLowBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = kEnablingPacketLossAtLowBw * 0.99f; + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForVeryLowBandwidth) { + auto states = CreateFecControllerPlrBased(false); + // Below |kEnablingBandwidthLow|, no packet loss fraction can cause FEC to + // turn on. + UpdateNetworkMetrics(&states, kEnablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); +} + +TEST(FecControllerPlrBasedTest, DisableFecForHighBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = kDisablingPacketLossAtHighBw - kEpsilon; + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOnForHighBandwidth) { + // Note: Disabling happens when the value is strictly below the threshold. + auto states = CreateFecControllerPlrBased(true); + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, + kDisablingPacketLossAtHighBw); + CheckDecision(&states, true, kDisablingPacketLossAtHighBw); +} + +TEST(FecControllerPlrBasedTest, DisableFecOnMediumBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = + (kDisablingPacketLossAtLowBw + kDisablingPacketLossAtHighBw) / 2.0f - + kEpsilon; + UpdateNetworkMetrics(&states, + (kDisablingBandwidthHigh + kDisablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOnForMediumBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = kDisablingPacketLossAtLowBw * 0.51f + + kDisablingPacketLossAtHighBw * 0.49f - kEpsilon; + UpdateNetworkMetrics(&states, + (kEnablingBandwidthHigh + kDisablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, true, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, DisableFecForLowBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = kDisablingPacketLossAtLowBw - kEpsilon; + UpdateNetworkMetrics(&states, kDisablingBandwidthLow, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, DisableFecForVeryLowBandwidth) { + auto states = CreateFecControllerPlrBased(true); + // Below |kEnablingBandwidthLow|, any packet loss fraction can cause FEC to + // turn off. + UpdateNetworkMetrics(&states, kDisablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); +} + +TEST(FecControllerPlrBasedTest, CheckBehaviorOnChangingNetworkMetrics) { + // In this test, we let the network metrics to traverse from 1 to 5. + // packet-loss ^ 1 | | + // | | 2| + // | \ \ 3 + // | \4 \_______ + // | \_________ + // |---------5-------> bandwidth + + auto states = CreateFecControllerPlrBased(true); + UpdateNetworkMetrics(&states, kDisablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); + + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, + kEnablingPacketLossAtLowBw * 0.99f); + CheckDecision(&states, false, kEnablingPacketLossAtLowBw * 0.99f); + + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, + kDisablingPacketLossAtHighBw); + CheckDecision(&states, true, kDisablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh + 1, 0.0); + CheckDecision(&states, false, 0.0); +} + +TEST(FecControllerPlrBasedTest, CheckBehaviorOnSpecialCurves) { + // We test a special configuration, where the points to define the FEC + // enabling/disabling curves are placed like the following, otherwise the test + // is the same as CheckBehaviorOnChangingNetworkMetrics. + // + // packet-loss ^ | | + // | | C| + // | | | + // | | D|_______ + // | A|___B______ + // |-----------------> bandwidth + + constexpr int kEnablingBandwidthHigh = kEnablingBandwidthLow; + constexpr float kDisablingPacketLossAtLowBw = kDisablingPacketLossAtHighBw; + FecControllerPlrBasedTestStates states; + std::unique_ptr mock_smoothing_filter( + new NiceMock()); + states.packet_loss_smoother = mock_smoothing_filter.get(); + states.controller.reset(new FecControllerPlrBased( + FecControllerPlrBased::Config( + true, + ThresholdCurve(kEnablingBandwidthLow, kEnablingPacketLossAtLowBw, + kEnablingBandwidthHigh, kEnablingPacketLossAtHighBw), + ThresholdCurve(kDisablingBandwidthLow, kDisablingPacketLossAtLowBw, + kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw), + 0), + std::move(mock_smoothing_filter))); + + UpdateNetworkMetrics(&states, kDisablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); + + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, + kEnablingPacketLossAtHighBw * 0.99f); + CheckDecision(&states, false, kEnablingPacketLossAtHighBw * 0.99f); + + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, + kDisablingPacketLossAtHighBw); + CheckDecision(&states, true, kDisablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh + 1, 0.0); + CheckDecision(&states, false, 0.0); +} + +TEST(FecControllerPlrBasedTest, SingleThresholdCurveForEnablingAndDisabling) { + // Note: To avoid numerical errors, keep kPacketLossAtLowBw and + // kPacketLossAthighBw as (negative) integer powers of 2. + // This is mostly relevant for the O3 case. + constexpr int kBandwidthLow = 10000; + constexpr float kPacketLossAtLowBw = 0.25f; + constexpr int kBandwidthHigh = 20000; + constexpr float kPacketLossAtHighBw = 0.125f; + auto curve = ThresholdCurve(kBandwidthLow, kPacketLossAtLowBw, kBandwidthHigh, + kPacketLossAtHighBw); + + // B* stands for "below-curve", O* for "on-curve", and A* for "above-curve". + // + // // + // packet-loss ^ // + // | | // + // | B1 O1 // + // | | // + // | O2 // + // | \ A1 // + // | \ // + // | O3 A2 // + // | B2 \ // + // | \ // + // | O4--O5---- // + // | // + // | B3 // + // |-----------------> bandwidth // + + struct NetworkState { + int bandwidth; + float packet_loss; + }; + + std::vector below{ + {kBandwidthLow - 1, kPacketLossAtLowBw + 0.1f}, // B1 + {(kBandwidthLow + kBandwidthHigh) / 2, + (kPacketLossAtLowBw + kPacketLossAtHighBw) / 2 - kEpsilon}, // B2 + {kBandwidthHigh + 1, kPacketLossAtHighBw - kEpsilon} // B3 + }; + + std::vector on{ + {kBandwidthLow, kPacketLossAtLowBw + 0.1f}, // O1 + {kBandwidthLow, kPacketLossAtLowBw}, // O2 + {(kBandwidthLow + kBandwidthHigh) / 2, + (kPacketLossAtLowBw + kPacketLossAtHighBw) / 2}, // O3 + {kBandwidthHigh, kPacketLossAtHighBw}, // O4 + {kBandwidthHigh + 1, kPacketLossAtHighBw}, // O5 + }; + + std::vector above{ + {(kBandwidthLow + kBandwidthHigh) / 2, + (kPacketLossAtLowBw + kPacketLossAtHighBw) / 2 + kEpsilon}, // A1 + {kBandwidthHigh + 1, kPacketLossAtHighBw + kEpsilon}, // A2 + }; + + // Test that FEC is turned off whenever we're below the curve, independent + // of the starting FEC state. + for (NetworkState net_state : below) { + for (bool initial_fec_enabled : {false, true}) { + auto states = + CreateFecControllerPlrBased(initial_fec_enabled, curve, curve); + UpdateNetworkMetrics(&states, net_state.bandwidth, net_state.packet_loss); + CheckDecision(&states, false, net_state.packet_loss); + } + } + + // Test that FEC is turned on whenever we're on the curve or above it, + // independent of the starting FEC state. + for (std::vector states_list : {on, above}) { + for (NetworkState net_state : states_list) { + for (bool initial_fec_enabled : {false, true}) { + auto states = + CreateFecControllerPlrBased(initial_fec_enabled, curve, curve); + UpdateNetworkMetrics(&states, net_state.bandwidth, + net_state.packet_loss); + CheckDecision(&states, true, net_state.packet_loss); + } + } + } +} + +TEST(FecControllerPlrBasedTest, FecAlwaysOff) { + ThresholdCurve always_off_curve(0, 1.0f + kEpsilon, 0, 1.0f + kEpsilon); + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {0, 10000}) { + for (float packet_loss : {0.0f, 0.5f, 1.0f}) { + auto states = CreateFecControllerPlrBased( + initial_fec_enabled, always_off_curve, always_off_curve); + UpdateNetworkMetrics(&states, bandwidth, packet_loss); + CheckDecision(&states, false, packet_loss); + } + } + } +} + +TEST(FecControllerPlrBasedTest, FecAlwaysOn) { + ThresholdCurve always_on_curve(0, 0.0f, 0, 0.0f); + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {0, 10000}) { + for (float packet_loss : {0.0f, 0.5f, 1.0f}) { + auto states = CreateFecControllerPlrBased( + initial_fec_enabled, always_on_curve, always_on_curve); + UpdateNetworkMetrics(&states, bandwidth, packet_loss); + CheckDecision(&states, true, packet_loss); + } + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(FecControllerPlrBasedDeathTest, InvalidConfig) { + FecControllerPlrBasedTestStates states; + std::unique_ptr mock_smoothing_filter( + new NiceMock()); + states.packet_loss_smoother = mock_smoothing_filter.get(); + EXPECT_DEATH( + states.controller.reset(new FecControllerPlrBased( + FecControllerPlrBased::Config( + true, + ThresholdCurve(kDisablingBandwidthLow - 1, + kEnablingPacketLossAtLowBw, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw), + ThresholdCurve( + kDisablingBandwidthLow, kDisablingPacketLossAtLowBw, + kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw), + 0), + std::move(mock_smoothing_filter))), + "Check failed"); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.cc new file mode 100644 index 0000000000..c8cfd31607 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +FecControllerRplrBased::Config::Config( + bool initial_fec_enabled, + const ThresholdCurve& fec_enabling_threshold, + const ThresholdCurve& fec_disabling_threshold) + : initial_fec_enabled(initial_fec_enabled), + fec_enabling_threshold(fec_enabling_threshold), + fec_disabling_threshold(fec_disabling_threshold) {} + +FecControllerRplrBased::FecControllerRplrBased(const Config& config) + : config_(config), fec_enabled_(config.initial_fec_enabled) { + RTC_DCHECK(config_.fec_disabling_threshold <= config_.fec_enabling_threshold); +} + +FecControllerRplrBased::~FecControllerRplrBased() = default; + +void FecControllerRplrBased::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + if (network_metrics.uplink_recoverable_packet_loss_fraction) { + uplink_recoverable_packet_loss_ = + network_metrics.uplink_recoverable_packet_loss_fraction; + } +} + +void FecControllerRplrBased::MakeDecision(AudioEncoderRuntimeConfig* config) { + RTC_DCHECK(!config->enable_fec); + RTC_DCHECK(!config->uplink_packet_loss_fraction); + + fec_enabled_ = fec_enabled_ ? !FecDisablingDecision() : FecEnablingDecision(); + + config->enable_fec = fec_enabled_; + config->uplink_packet_loss_fraction = + uplink_recoverable_packet_loss_ ? *uplink_recoverable_packet_loss_ : 0.0; +} + +bool FecControllerRplrBased::FecEnablingDecision() const { + if (!uplink_bandwidth_bps_ || !uplink_recoverable_packet_loss_) { + return false; + } else { + // Enable when above the curve or exactly on it. + return !config_.fec_enabling_threshold.IsBelowCurve( + {static_cast(*uplink_bandwidth_bps_), + *uplink_recoverable_packet_loss_}); + } +} + +bool FecControllerRplrBased::FecDisablingDecision() const { + if (!uplink_bandwidth_bps_ || !uplink_recoverable_packet_loss_) { + return false; + } else { + // Disable when below the curve. + return config_.fec_disabling_threshold.IsBelowCurve( + {static_cast(*uplink_bandwidth_bps_), + *uplink_recoverable_packet_loss_}); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.h new file mode 100644 index 0000000000..ade55aea5d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_RPLR_BASED_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_RPLR_BASED_H_ + +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class FecControllerRplrBased final : public Controller { + public: + struct Config { + // |fec_enabling_threshold| defines a curve, above which FEC should be + // enabled. |fec_disabling_threshold| defines a curve, under which FEC + // should be disabled. See below + // + // recoverable + // packet-loss ^ | | + // | | | FEC + // | \ \ ON + // | FEC \ \_______ fec_enabling_threshold + // | OFF \_________ fec_disabling_threshold + // |-----------------> bandwidth + Config(bool initial_fec_enabled, + const ThresholdCurve& fec_enabling_threshold, + const ThresholdCurve& fec_disabling_threshold); + bool initial_fec_enabled; + ThresholdCurve fec_enabling_threshold; + ThresholdCurve fec_disabling_threshold; + }; + + explicit FecControllerRplrBased(const Config& config); + + ~FecControllerRplrBased() override; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + bool FecEnablingDecision() const; + bool FecDisablingDecision() const; + + const Config config_; + bool fec_enabled_; + rtc::Optional uplink_bandwidth_bps_; + rtc::Optional uplink_recoverable_packet_loss_; + + RTC_DISALLOW_COPY_AND_ASSIGN(FecControllerRplrBased); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_RPLR_BASED_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based_unittest.cc new file mode 100644 index 0000000000..0fc003beab --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based_unittest.cc @@ -0,0 +1,520 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +// The test uses the following settings: +// +// recoverable ^ +// packet-loss | | | +// | A| C| FEC +// | \ \ ON +// | FEC \ D\_______ +// | OFF B\_________ +// |-----------------> bandwidth +// +// A : (kDisablingBandwidthLow, kDisablingRecoverablePacketLossAtLowBw) +// B : (kDisablingBandwidthHigh, kDisablingRecoverablePacketLossAtHighBw) +// C : (kEnablingBandwidthLow, kEnablingRecoverablePacketLossAtLowBw) +// D : (kEnablingBandwidthHigh, kEnablingRecoverablePacketLossAtHighBw) + +constexpr int kDisablingBandwidthLow = 15000; +constexpr float kDisablingRecoverablePacketLossAtLowBw = 0.08f; +constexpr int kDisablingBandwidthHigh = 64000; +constexpr float kDisablingRecoverablePacketLossAtHighBw = 0.01f; +constexpr int kEnablingBandwidthLow = 17000; +constexpr float kEnablingRecoverablePacketLossAtLowBw = 0.1f; +constexpr int kEnablingBandwidthHigh = 64000; +constexpr float kEnablingRecoverablePacketLossAtHighBw = 0.05f; + +constexpr float kEpsilon = 1e-5f; + +rtc::Optional GetRandomProbabilityOrUnknown() { + std::random_device rd; + std::mt19937 generator(rd()); + std::uniform_real_distribution<> distribution(0, 1); + + return (distribution(generator) < 0.2) + ? rtc::nullopt + : rtc::Optional(distribution(generator)); +} + +std::unique_ptr CreateFecControllerRplrBased( + bool initial_fec_enabled) { + return std::unique_ptr( + new FecControllerRplrBased(FecControllerRplrBased::Config( + initial_fec_enabled, + ThresholdCurve( + kEnablingBandwidthLow, kEnablingRecoverablePacketLossAtLowBw, + kEnablingBandwidthHigh, kEnablingRecoverablePacketLossAtHighBw), + ThresholdCurve(kDisablingBandwidthLow, + kDisablingRecoverablePacketLossAtLowBw, + kDisablingBandwidthHigh, + kDisablingRecoverablePacketLossAtHighBw)))); +} + +void UpdateNetworkMetrics( + FecControllerRplrBased* controller, + const rtc::Optional& uplink_bandwidth_bps, + const rtc::Optional& uplink_packet_loss, + const rtc::Optional& uplink_recoveralbe_packet_loss) { + // UpdateNetworkMetrics can accept multiple network metric updates at once. + // However, currently, the most used case is to update one metric at a time. + // To reflect this fact, we separate the calls. + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + if (uplink_packet_loss) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_packet_loss_fraction = uplink_packet_loss; + controller->UpdateNetworkMetrics(network_metrics); + } + if (uplink_recoveralbe_packet_loss) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_recoverable_packet_loss_fraction = + uplink_recoveralbe_packet_loss; + controller->UpdateNetworkMetrics(network_metrics); + } +} + +void UpdateNetworkMetrics( + FecControllerRplrBased* controller, + const rtc::Optional& uplink_bandwidth_bps, + const rtc::Optional& uplink_recoveralbe_packet_loss) { + // FecControllerRplrBased doesn't currently use the PLR (general packet-loss + // rate) at all. (This might be changed in the future.) The unit-tests will + // use a random value (including unknown), to show this does not interfere. + UpdateNetworkMetrics(controller, uplink_bandwidth_bps, + GetRandomProbabilityOrUnknown(), + uplink_recoveralbe_packet_loss); +} + +// Checks that the FEC decision and |uplink_packet_loss_fraction| given by +// |states->controller->MakeDecision| matches |expected_enable_fec| and +// |expected_uplink_packet_loss_fraction|, respectively. +void CheckDecision(FecControllerRplrBased* controller, + bool expected_enable_fec, + float expected_uplink_packet_loss_fraction) { + AudioEncoderRuntimeConfig config; + controller->MakeDecision(&config); + + // Less compact than comparing optionals, but yields more readable errors. + EXPECT_TRUE(config.enable_fec); + if (config.enable_fec) { + EXPECT_EQ(expected_enable_fec, *config.enable_fec); + } + EXPECT_TRUE(config.uplink_packet_loss_fraction); + if (config.uplink_packet_loss_fraction) { + EXPECT_EQ(expected_uplink_packet_loss_fraction, + *config.uplink_packet_loss_fraction); + } +} + +} // namespace + +TEST(FecControllerRplrBasedTest, OutputInitValueBeforeAnyInputsAreReceived) { + for (bool initial_fec_enabled : {false, true}) { + auto controller = CreateFecControllerRplrBased(initial_fec_enabled); + CheckDecision(controller.get(), initial_fec_enabled, 0); + } +} + +TEST(FecControllerRplrBasedTest, OutputInitValueWhenUplinkBandwidthUnknown) { + // Regardless of the initial FEC state and the recoverable-packet-loss + // rate, the initial FEC state is maintained as long as the BWE is unknown. + for (bool initial_fec_enabled : {false, true}) { + for (float recoverable_packet_loss : + {kDisablingRecoverablePacketLossAtHighBw - kEpsilon, + kDisablingRecoverablePacketLossAtHighBw, + kDisablingRecoverablePacketLossAtHighBw + kEpsilon, + kEnablingRecoverablePacketLossAtHighBw - kEpsilon, + kEnablingRecoverablePacketLossAtHighBw, + kEnablingRecoverablePacketLossAtHighBw + kEpsilon}) { + auto controller = CreateFecControllerRplrBased(initial_fec_enabled); + UpdateNetworkMetrics(controller.get(), rtc::nullopt, + recoverable_packet_loss); + CheckDecision(controller.get(), initial_fec_enabled, + recoverable_packet_loss); + } + } +} + +TEST(FecControllerRplrBasedTest, + OutputInitValueWhenUplinkRecoverablePacketLossFractionUnknown) { + // Regardless of the initial FEC state and the BWE, the initial FEC state + // is maintained as long as the recoverable-packet-loss rate is unknown. + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {kDisablingBandwidthLow - 1, kDisablingBandwidthLow, + kDisablingBandwidthLow + 1, kEnablingBandwidthLow - 1, + kEnablingBandwidthLow, kEnablingBandwidthLow + 1}) { + auto controller = CreateFecControllerRplrBased(initial_fec_enabled); + UpdateNetworkMetrics(controller.get(), bandwidth, rtc::nullopt); + CheckDecision(controller.get(), initial_fec_enabled, 0.0); + } + } +} + +TEST(FecControllerRplrBasedTest, EnableFecForHighBandwidth) { + auto controller = CreateFecControllerRplrBased(false); + UpdateNetworkMetrics(controller.get(), kEnablingBandwidthHigh, + kEnablingRecoverablePacketLossAtHighBw); + CheckDecision(controller.get(), true, kEnablingRecoverablePacketLossAtHighBw); +} + +TEST(FecControllerRplrBasedTest, UpdateMultipleNetworkMetricsAtOnce) { + // This test is similar to EnableFecForHighBandwidth. But instead of + // using ::UpdateNetworkMetrics(...), which calls + // FecControllerRplrBasedTest::UpdateNetworkMetrics(...) multiple times, we + // we call it only once. This is to verify that + // FecControllerRplrBasedTest::UpdateNetworkMetrics(...) can handle multiple + // network updates at once. This is, however, not a common use case in current + // audio_network_adaptor_impl.cc. + auto controller = CreateFecControllerRplrBased(false); + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = kEnablingBandwidthHigh; + network_metrics.uplink_packet_loss_fraction = GetRandomProbabilityOrUnknown(); + network_metrics.uplink_recoverable_packet_loss_fraction = + kEnablingRecoverablePacketLossAtHighBw; + controller->UpdateNetworkMetrics(network_metrics); + CheckDecision(controller.get(), true, kEnablingRecoverablePacketLossAtHighBw); +} + +TEST(FecControllerRplrBasedTest, MaintainFecOffForHighBandwidth) { + auto controller = CreateFecControllerRplrBased(false); + constexpr float kRecoverablePacketLoss = + kEnablingRecoverablePacketLossAtHighBw * 0.99f; + UpdateNetworkMetrics(controller.get(), kEnablingBandwidthHigh, + kRecoverablePacketLoss); + CheckDecision(controller.get(), false, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, EnableFecForMediumBandwidth) { + auto controller = CreateFecControllerRplrBased(false); + constexpr float kRecoverablePacketLoss = + (kEnablingRecoverablePacketLossAtLowBw + + kEnablingRecoverablePacketLossAtHighBw) / 2.0; + UpdateNetworkMetrics( + controller.get(), + (kEnablingBandwidthHigh + kEnablingBandwidthLow) / 2, + kRecoverablePacketLoss); + CheckDecision(controller.get(), true, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, MaintainFecOffForMediumBandwidth) { + auto controller = CreateFecControllerRplrBased(false); + constexpr float kRecoverablePacketLoss = + kEnablingRecoverablePacketLossAtLowBw * 0.49f + + kEnablingRecoverablePacketLossAtHighBw * 0.51f; + UpdateNetworkMetrics(controller.get(), + (kEnablingBandwidthHigh + kEnablingBandwidthLow) / 2, + kRecoverablePacketLoss); + CheckDecision(controller.get(), false, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, EnableFecForLowBandwidth) { + auto controller = CreateFecControllerRplrBased(false); + UpdateNetworkMetrics(controller.get(), kEnablingBandwidthLow, + kEnablingRecoverablePacketLossAtLowBw); + CheckDecision(controller.get(), true, kEnablingRecoverablePacketLossAtLowBw); +} + +TEST(FecControllerRplrBasedTest, MaintainFecOffForLowBandwidth) { + auto controller = CreateFecControllerRplrBased(false); + constexpr float kRecoverablePacketLoss = + kEnablingRecoverablePacketLossAtLowBw * 0.99f; + UpdateNetworkMetrics(controller.get(), kEnablingBandwidthLow, + kRecoverablePacketLoss); + CheckDecision(controller.get(), false, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, MaintainFecOffForVeryLowBandwidth) { + auto controller = CreateFecControllerRplrBased(false); + // Below |kEnablingBandwidthLow|, no recoverable packet loss fraction can + // cause FEC to turn on. + UpdateNetworkMetrics(controller.get(), kEnablingBandwidthLow - 1, 1.0); + CheckDecision(controller.get(), false, 1.0); +} + +TEST(FecControllerRplrBasedTest, DisableFecForHighBandwidth) { + auto controller = CreateFecControllerRplrBased(true); + constexpr float kRecoverablePacketLoss = + kDisablingRecoverablePacketLossAtHighBw - kEpsilon; + UpdateNetworkMetrics(controller.get(), kDisablingBandwidthHigh, + kRecoverablePacketLoss); + CheckDecision(controller.get(), false, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, MaintainFecOnForHighBandwidth) { + // Note: Disabling happens when the value is strictly below the threshold. + auto controller = CreateFecControllerRplrBased(true); + UpdateNetworkMetrics(controller.get(), kDisablingBandwidthHigh, + kDisablingRecoverablePacketLossAtHighBw); + CheckDecision(controller.get(), true, + kDisablingRecoverablePacketLossAtHighBw); +} + +TEST(FecControllerRplrBasedTest, DisableFecOnMediumBandwidth) { + auto controller = CreateFecControllerRplrBased(true); + constexpr float kRecoverablePacketLoss = + ((kDisablingRecoverablePacketLossAtLowBw + + kDisablingRecoverablePacketLossAtHighBw) / 2.0f) - kEpsilon; + UpdateNetworkMetrics( + controller.get(), + (kDisablingBandwidthHigh + kDisablingBandwidthLow) / 2, + kRecoverablePacketLoss); + CheckDecision(controller.get(), false, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, MaintainFecOnForMediumBandwidth) { + auto controller = CreateFecControllerRplrBased(true); + constexpr float kRecoverablePacketLoss = + kDisablingRecoverablePacketLossAtLowBw * 0.51f + + kDisablingRecoverablePacketLossAtHighBw * 0.49f - kEpsilon; + UpdateNetworkMetrics(controller.get(), + (kEnablingBandwidthHigh + kDisablingBandwidthLow) / 2, + kRecoverablePacketLoss); + CheckDecision(controller.get(), true, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, DisableFecForLowBandwidth) { + auto controller = CreateFecControllerRplrBased(true); + constexpr float kRecoverablePacketLoss = + kDisablingRecoverablePacketLossAtLowBw - kEpsilon; + UpdateNetworkMetrics(controller.get(), kDisablingBandwidthLow, + kRecoverablePacketLoss); + CheckDecision(controller.get(), false, kRecoverablePacketLoss); +} + +TEST(FecControllerRplrBasedTest, DisableFecForVeryLowBandwidth) { + auto controller = CreateFecControllerRplrBased(true); + // Below |kEnablingBandwidthLow|, any recoverable packet loss fraction can + // cause FEC to turn off. + UpdateNetworkMetrics(controller.get(), kDisablingBandwidthLow - 1, 1.0); + CheckDecision(controller.get(), false, 1.0); +} + +TEST(FecControllerRplrBasedTest, CheckBehaviorOnChangingNetworkMetrics) { + // In this test, we let the network metrics to traverse from 1 to 5. + // + // recoverable ^ + // packet-loss | 1 | | + // | | 2| + // | \ \ 3 + // | \4 \_______ + // | \_________ + // |---------5-------> bandwidth + + auto controller = CreateFecControllerRplrBased(true); + UpdateNetworkMetrics(controller.get(), kDisablingBandwidthLow - 1, 1.0); + CheckDecision(controller.get(), false, 1.0); + + UpdateNetworkMetrics(controller.get(), kEnablingBandwidthLow, + kEnablingRecoverablePacketLossAtLowBw * 0.99f); + CheckDecision(controller.get(), false, + kEnablingRecoverablePacketLossAtLowBw * 0.99f); + + UpdateNetworkMetrics(controller.get(), kEnablingBandwidthHigh, + kEnablingRecoverablePacketLossAtHighBw); + CheckDecision(controller.get(), true, kEnablingRecoverablePacketLossAtHighBw); + + UpdateNetworkMetrics(controller.get(), kDisablingBandwidthHigh, + kDisablingRecoverablePacketLossAtHighBw); + CheckDecision(controller.get(), true, + kDisablingRecoverablePacketLossAtHighBw); + + UpdateNetworkMetrics(controller.get(), kDisablingBandwidthHigh + 1, 0.0); + CheckDecision(controller.get(), false, 0.0); +} + +TEST(FecControllerRplrBasedTest, CheckBehaviorOnSpecialCurves) { + // We test a special configuration, where the points to define the FEC + // enabling/disabling curves are placed like the following, otherwise the test + // is the same as CheckBehaviorOnChangingNetworkMetrics. + // + // recoverable ^ + // packet-loss | | | + // | | C| + // | | | + // | | D|_______ + // | A|___B______ + // |-----------------> bandwidth + + constexpr int kEnablingBandwidthHigh = kEnablingBandwidthLow; + constexpr float kDisablingRecoverablePacketLossAtLowBw = + kDisablingRecoverablePacketLossAtHighBw; + FecControllerRplrBased controller(FecControllerRplrBased::Config( + true, + ThresholdCurve( + kEnablingBandwidthLow, kEnablingRecoverablePacketLossAtLowBw, + kEnablingBandwidthHigh, kEnablingRecoverablePacketLossAtHighBw), + ThresholdCurve( + kDisablingBandwidthLow, kDisablingRecoverablePacketLossAtLowBw, + kDisablingBandwidthHigh, kDisablingRecoverablePacketLossAtHighBw))); + + UpdateNetworkMetrics(&controller, kDisablingBandwidthLow - 1, 1.0); + CheckDecision(&controller, false, 1.0); + + UpdateNetworkMetrics(&controller, kEnablingBandwidthLow, + kEnablingRecoverablePacketLossAtHighBw * 0.99f); + CheckDecision(&controller, false, + kEnablingRecoverablePacketLossAtHighBw * 0.99f); + + UpdateNetworkMetrics(&controller, kEnablingBandwidthHigh, + kEnablingRecoverablePacketLossAtHighBw); + CheckDecision(&controller, true, kEnablingRecoverablePacketLossAtHighBw); + + UpdateNetworkMetrics(&controller, kDisablingBandwidthHigh, + kDisablingRecoverablePacketLossAtHighBw); + CheckDecision(&controller, true, kDisablingRecoverablePacketLossAtHighBw); + + UpdateNetworkMetrics(&controller, kDisablingBandwidthHigh + 1, 0.0); + CheckDecision(&controller, false, 0.0); +} + +TEST(FecControllerRplrBasedTest, SingleThresholdCurveForEnablingAndDisabling) { + // Note: To avoid numerical errors, keep kRecoverablePacketLossAtLowBw and + // kRecoverablePacketLossAthighBw as (negative) integer powers of 2. + // This is mostly relevant for the O3 case. + constexpr int kBandwidthLow = 10000; + constexpr float kRecoverablePacketLossAtLowBw = 0.25f; + constexpr int kBandwidthHigh = 20000; + constexpr float kRecoverablePacketLossAtHighBw = 0.125f; + auto curve = ThresholdCurve(kBandwidthLow, kRecoverablePacketLossAtLowBw, + kBandwidthHigh, kRecoverablePacketLossAtHighBw); + + // B* stands for "below-curve", O* for "on-curve", and A* for "above-curve". + // + // // + // recoverable ^ // + // packet-loss | | // + // | B1 O1 // + // | | // + // | O2 // + // | \ A1 // + // | \ // + // | O3 A2 // + // | B2 \ // + // | \ // + // | O4--O5---- // + // | // + // | B3 // + // |-----------------> bandwidth // + + struct NetworkState { + int bandwidth; + float recoverable_packet_loss; + }; + + std::vector below{ + {kBandwidthLow - 1, kRecoverablePacketLossAtLowBw + 0.1f}, // B1 + {(kBandwidthLow + kBandwidthHigh) / 2, + (kRecoverablePacketLossAtLowBw + kRecoverablePacketLossAtHighBw) / 2 - + kEpsilon}, // B2 + {kBandwidthHigh + 1, kRecoverablePacketLossAtHighBw - kEpsilon} // B3 + }; + + std::vector on{ + {kBandwidthLow, kRecoverablePacketLossAtLowBw + 0.1f}, // O1 + {kBandwidthLow, kRecoverablePacketLossAtLowBw}, // O2 + {(kBandwidthLow + kBandwidthHigh) / 2, + (kRecoverablePacketLossAtLowBw + kRecoverablePacketLossAtHighBw) / + 2}, // O3 + {kBandwidthHigh, kRecoverablePacketLossAtHighBw}, // O4 + {kBandwidthHigh + 1, kRecoverablePacketLossAtHighBw}, // O5 + }; + + std::vector above{ + {(kBandwidthLow + kBandwidthHigh) / 2, + (kRecoverablePacketLossAtLowBw + kRecoverablePacketLossAtHighBw) / 2 + + kEpsilon}, // A1 + {kBandwidthHigh + 1, kRecoverablePacketLossAtHighBw + kEpsilon}, // A2 + }; + + // Test that FEC is turned off whenever we're below the curve, independent + // of the starting FEC state. + for (NetworkState net_state : below) { + for (bool initial_fec_enabled : {false, true}) { + FecControllerRplrBased controller( + FecControllerRplrBased::Config(initial_fec_enabled, curve, curve)); + UpdateNetworkMetrics(&controller, net_state.bandwidth, + net_state.recoverable_packet_loss); + CheckDecision(&controller, false, net_state.recoverable_packet_loss); + } + } + + // Test that FEC is turned on whenever we're on the curve or above it, + // independent of the starting FEC state. + for (std::vector states_list : {on, above}) { + for (NetworkState net_state : states_list) { + for (bool initial_fec_enabled : {false, true}) { + FecControllerRplrBased controller( + FecControllerRplrBased::Config(initial_fec_enabled, curve, curve)); + UpdateNetworkMetrics(&controller, net_state.bandwidth, + net_state.recoverable_packet_loss); + CheckDecision(&controller, true, net_state.recoverable_packet_loss); + } + } + } +} + +TEST(FecControllerRplrBasedTest, FecAlwaysOff) { + ThresholdCurve always_off_curve(0, 1.0f + kEpsilon, 0, 1.0f + kEpsilon); + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {0, 10000}) { + for (float recoverable_packet_loss : {0.0f, 0.5f, 1.0f}) { + FecControllerRplrBased controller(FecControllerRplrBased::Config( + initial_fec_enabled, always_off_curve, always_off_curve)); + UpdateNetworkMetrics(&controller, bandwidth, recoverable_packet_loss); + CheckDecision(&controller, false, recoverable_packet_loss); + } + } + } +} + +TEST(FecControllerRplrBasedTest, FecAlwaysOn) { + ThresholdCurve always_on_curve(0, 0.0f, 0, 0.0f); + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {0, 10000}) { + for (float recoverable_packet_loss : {0.0f, 0.5f, 1.0f}) { + FecControllerRplrBased controller(FecControllerRplrBased::Config( + initial_fec_enabled, always_on_curve, always_on_curve)); + UpdateNetworkMetrics(&controller, bandwidth, recoverable_packet_loss); + CheckDecision(&controller, true, recoverable_packet_loss); + } + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(FecControllerRplrBasedDeathTest, InvalidConfig) { + EXPECT_DEATH( + FecControllerRplrBased controller(FecControllerRplrBased::Config( + true, + ThresholdCurve( + kDisablingBandwidthLow - 1, kEnablingRecoverablePacketLossAtLowBw, + kEnablingBandwidthHigh, kEnablingRecoverablePacketLossAtHighBw), + ThresholdCurve(kDisablingBandwidthLow, + kDisablingRecoverablePacketLossAtLowBw, + kDisablingBandwidthHigh, + kDisablingRecoverablePacketLossAtHighBw))), + "Check failed"); +} +#endif +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc new file mode 100644 index 0000000000..6c3cae07bc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +namespace { +constexpr int kPreventOveruseMarginBps = 5000; + +int OverheadRateBps(size_t overhead_bytes_per_packet, int frame_length_ms) { + return static_cast(overhead_bytes_per_packet * 8 * 1000 / + frame_length_ms); +} +} + +FrameLengthController::Config::Config( + const std::vector& encoder_frame_lengths_ms, + int initial_frame_length_ms, + int min_encoder_bitrate_bps, + float fl_increasing_packet_loss_fraction, + float fl_decreasing_packet_loss_fraction, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset, + std::map fl_changing_bandwidths_bps) + : encoder_frame_lengths_ms(encoder_frame_lengths_ms), + initial_frame_length_ms(initial_frame_length_ms), + min_encoder_bitrate_bps(min_encoder_bitrate_bps), + fl_increasing_packet_loss_fraction(fl_increasing_packet_loss_fraction), + fl_decreasing_packet_loss_fraction(fl_decreasing_packet_loss_fraction), + fl_increase_overhead_offset(fl_increase_overhead_offset), + fl_decrease_overhead_offset(fl_decrease_overhead_offset), + fl_changing_bandwidths_bps(std::move(fl_changing_bandwidths_bps)) {} + +FrameLengthController::Config::Config(const Config& other) = default; + +FrameLengthController::Config::~Config() = default; + +FrameLengthController::FrameLengthController(const Config& config) + : config_(config) { + frame_length_ms_ = std::find(config_.encoder_frame_lengths_ms.begin(), + config_.encoder_frame_lengths_ms.end(), + config_.initial_frame_length_ms); + // |encoder_frame_lengths_ms| must contain |initial_frame_length_ms|. + RTC_DCHECK(frame_length_ms_ != config_.encoder_frame_lengths_ms.end()); +} + +FrameLengthController::~FrameLengthController() = default; + +void FrameLengthController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + if (network_metrics.uplink_packet_loss_fraction) + uplink_packet_loss_fraction_ = network_metrics.uplink_packet_loss_fraction; + if (network_metrics.overhead_bytes_per_packet) + overhead_bytes_per_packet_ = network_metrics.overhead_bytes_per_packet; +} + +void FrameLengthController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on |frame_length_ms| should not have been made. + RTC_DCHECK(!config->frame_length_ms); + + if (FrameLengthIncreasingDecision(*config)) { + ++frame_length_ms_; + prev_decision_increase_ = true; + } else if (FrameLengthDecreasingDecision(*config)) { + --frame_length_ms_; + prev_decision_increase_ = false; + } + config->last_fl_change_increase = prev_decision_increase_; + config->frame_length_ms = *frame_length_ms_; +} + +FrameLengthController::Config::FrameLengthChange::FrameLengthChange( + int from_frame_length_ms, + int to_frame_length_ms) + : from_frame_length_ms(from_frame_length_ms), + to_frame_length_ms(to_frame_length_ms) {} + +bool FrameLengthController::Config::FrameLengthChange::operator<( + const FrameLengthChange& rhs) const { + return from_frame_length_ms < rhs.from_frame_length_ms || + (from_frame_length_ms == rhs.from_frame_length_ms && + to_frame_length_ms < rhs.to_frame_length_ms); +} + +bool FrameLengthController::FrameLengthIncreasingDecision( + const AudioEncoderRuntimeConfig& config) const { + // Increase frame length if + // 1. |uplink_bandwidth_bps| is known to be smaller or equal than + // |min_encoder_bitrate_bps| plus |prevent_overuse_margin_bps| plus the + // current overhead rate OR all the following: + // 2. longer frame length is available AND + // 3. |uplink_bandwidth_bps| is known to be smaller than a threshold AND + // 4. |uplink_packet_loss_fraction| is known to be smaller than a threshold. + + auto longer_frame_length_ms = std::next(frame_length_ms_); + if (longer_frame_length_ms == config_.encoder_frame_lengths_ms.end()) + return false; + + auto increase_threshold = config_.fl_changing_bandwidths_bps.find( + Config::FrameLengthChange(*frame_length_ms_, *longer_frame_length_ms)); + + if (increase_threshold == config_.fl_changing_bandwidths_bps.end()) + return false; + + // Check that + // -(*overhead_bytes_per_packet_) <= offset <= (*overhead_bytes_per_packet_) + RTC_DCHECK( + !overhead_bytes_per_packet_ || + (overhead_bytes_per_packet_ && + static_cast(std::max(0, -config_.fl_increase_overhead_offset)) <= + *overhead_bytes_per_packet_ && + static_cast(std::max(0, config_.fl_increase_overhead_offset)) <= + *overhead_bytes_per_packet_)); + + if (uplink_bandwidth_bps_ && overhead_bytes_per_packet_ && + *uplink_bandwidth_bps_ <= + config_.min_encoder_bitrate_bps + kPreventOveruseMarginBps + + OverheadRateBps(*overhead_bytes_per_packet_ + + config_.fl_increase_overhead_offset, + *frame_length_ms_)) { + return true; + } + + return (uplink_bandwidth_bps_ && + *uplink_bandwidth_bps_ <= increase_threshold->second) && + (uplink_packet_loss_fraction_ && + *uplink_packet_loss_fraction_ <= + config_.fl_increasing_packet_loss_fraction); +} + +bool FrameLengthController::FrameLengthDecreasingDecision( + const AudioEncoderRuntimeConfig& config) const { + // Decrease frame length if + // 1. shorter frame length is available AND + // 2. |uplink_bandwidth_bps| is known to be bigger than + // |min_encoder_bitrate_bps| plus |prevent_overuse_margin_bps| plus the + // overhead which would be produced with the shorter frame length AND + // one or more of the followings: + // 3. |uplink_bandwidth_bps| is known to be larger than a threshold, + // 4. |uplink_packet_loss_fraction| is known to be larger than a threshold, + if (frame_length_ms_ == config_.encoder_frame_lengths_ms.begin()) + return false; + + auto shorter_frame_length_ms = std::prev(frame_length_ms_); + auto decrease_threshold = config_.fl_changing_bandwidths_bps.find( + Config::FrameLengthChange(*frame_length_ms_, *shorter_frame_length_ms)); + + if (decrease_threshold == config_.fl_changing_bandwidths_bps.end()) + return false; + + if (uplink_bandwidth_bps_ && overhead_bytes_per_packet_ && + *uplink_bandwidth_bps_ <= + config_.min_encoder_bitrate_bps + kPreventOveruseMarginBps + + OverheadRateBps(*overhead_bytes_per_packet_ + + config_.fl_decrease_overhead_offset, + *shorter_frame_length_ms)) { + return false; + } + + return (uplink_bandwidth_bps_ && + *uplink_bandwidth_bps_ >= decrease_threshold->second) || + (uplink_packet_loss_fraction_ && + *uplink_packet_loss_fraction_ >= + config_.fl_decreasing_packet_loss_fraction); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h new file mode 100644 index 0000000000..c254b3dcf6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_H_ + +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Determines target frame length based on the network metrics and the decision +// of FEC controller. +class FrameLengthController final : public Controller { + public: + struct Config { + struct FrameLengthChange { + FrameLengthChange(int from_frame_length_ms, int to_frame_length_ms); + bool operator<(const FrameLengthChange& rhs) const; + int from_frame_length_ms; + int to_frame_length_ms; + }; + Config(const std::vector& encoder_frame_lengths_ms, + int initial_frame_length_ms, + int min_encoder_bitrate_bps, + float fl_increasing_packet_loss_fraction, + float fl_decreasing_packet_loss_fraction, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset, + std::map fl_changing_bandwidths_bps); + Config(const Config& other); + ~Config(); + std::vector encoder_frame_lengths_ms; + int initial_frame_length_ms; + int min_encoder_bitrate_bps; + // Uplink packet loss fraction below which frame length can increase. + float fl_increasing_packet_loss_fraction; + // Uplink packet loss fraction below which frame length should decrease. + float fl_decreasing_packet_loss_fraction; + // Offset to apply to overhead calculation when increasing frame length. + int fl_increase_overhead_offset; + // Offset to apply to overhead calculation when decreasing frame length. + int fl_decrease_overhead_offset; + std::map fl_changing_bandwidths_bps; + }; + + explicit FrameLengthController(const Config& config); + + ~FrameLengthController() override; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + bool FrameLengthIncreasingDecision( + const AudioEncoderRuntimeConfig& config) const; + + bool FrameLengthDecreasingDecision( + const AudioEncoderRuntimeConfig& config) const; + + const Config config_; + + std::vector::const_iterator frame_length_ms_; + + rtc::Optional uplink_bandwidth_bps_; + + rtc::Optional uplink_packet_loss_fraction_; + + rtc::Optional overhead_bytes_per_packet_; + + // True if the previous frame length decision was an increase, otherwise + // false. + bool prev_decision_increase_ = false; + + RTC_DISALLOW_COPY_AND_ASSIGN(FrameLengthController); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc new file mode 100644 index 0000000000..1f984470c0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr float kFlIncreasingPacketLossFraction = 0.04f; +constexpr float kFlDecreasingPacketLossFraction = 0.05f; +constexpr int kFlIncreaseOverheadOffset = 0; +constexpr int kFlDecreaseOverheadOffset = 0; +constexpr int kMinEncoderBitrateBps = 6000; +constexpr int kPreventOveruseMarginBps = 5000; +constexpr size_t kOverheadBytesPerPacket = 20; +constexpr int kFl20msTo60msBandwidthBps = 40000; +constexpr int kFl60msTo20msBandwidthBps = 50000; +constexpr int kFl60msTo120msBandwidthBps = 30000; +constexpr int kFl120msTo60msBandwidthBps = 40000; +constexpr int kMediumBandwidthBps = + (kFl60msTo20msBandwidthBps + kFl20msTo60msBandwidthBps) / 2; +constexpr float kMediumPacketLossFraction = + (kFlDecreasingPacketLossFraction + kFlIncreasingPacketLossFraction) / 2; + +int VeryLowBitrate(int frame_length_ms) { + return kMinEncoderBitrateBps + kPreventOveruseMarginBps + + (kOverheadBytesPerPacket * 8 * 1000 / frame_length_ms); +} + +std::unique_ptr CreateController( + const std::map& + frame_length_change_criteria, + const std::vector& encoder_frame_lengths_ms, + int initial_frame_length_ms) { + std::unique_ptr controller( + new FrameLengthController(FrameLengthController::Config( + encoder_frame_lengths_ms, initial_frame_length_ms, + kMinEncoderBitrateBps, kFlIncreasingPacketLossFraction, + kFlDecreasingPacketLossFraction, kFlIncreaseOverheadOffset, + kFlDecreaseOverheadOffset, frame_length_change_criteria))); + + return controller; +} + +std::map +CreateChangeCriteriaFor20msAnd60ms() { + return std::map{ + {FrameLengthController::Config::FrameLengthChange(20, 60), + kFl20msTo60msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 20), + kFl60msTo20msBandwidthBps}}; +} + +std::map +CreateChangeCriteriaFor20ms60msAnd120ms() { + return std::map{ + {FrameLengthController::Config::FrameLengthChange(20, 60), + kFl20msTo60msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 20), + kFl60msTo20msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 120), + kFl60msTo120msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(120, 60), + kFl120msTo60msBandwidthBps}}; +} + +void UpdateNetworkMetrics( + FrameLengthController* controller, + const rtc::Optional& uplink_bandwidth_bps, + const rtc::Optional& uplink_packet_loss_fraction, + const rtc::Optional& overhead_bytes_per_packet) { + // UpdateNetworkMetrics can accept multiple network metric updates at once. + // However, currently, the most used case is to update one metric at a time. + // To reflect this fact, we separate the calls. + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + if (uplink_packet_loss_fraction) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + controller->UpdateNetworkMetrics(network_metrics); + } + if (overhead_bytes_per_packet) { + Controller::NetworkMetrics network_metrics; + network_metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + controller->UpdateNetworkMetrics(network_metrics); + } +} + +void CheckDecision(FrameLengthController* controller, + int expected_frame_length_ms) { + AudioEncoderRuntimeConfig config; + controller->MakeDecision(&config); + EXPECT_EQ(expected_frame_length_ms, config.frame_length_ms); +} + +} // namespace + +TEST(FrameLengthControllerTest, DecreaseTo20MsOnHighUplinkBandwidth) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 60); + UpdateNetworkMetrics(controller.get(), kFl60msTo20msBandwidthBps, + rtc::nullopt, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, DecreaseTo20MsOnHighUplinkPacketLossFraction) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 60); + UpdateNetworkMetrics(controller.get(), rtc::nullopt, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, + Maintain60MsIf20MsNotInReceiverFrameLengthRange) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {60}, 60); + // Set FEC on that would cause frame length to decrease if receiver frame + // length range included 20ms. + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, Maintain60MsOnMultipleConditions) { + // Maintain 60ms frame length if + // 1. |uplink_bandwidth_bps| is at medium level, + // 2. |uplink_packet_loss_fraction| is at medium, + // 3. FEC is not decided ON. + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 60); + UpdateNetworkMetrics(controller.get(), kMediumBandwidthBps, + kMediumPacketLossFraction, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, IncreaseTo60MsOnMultipleConditions) { + // Increase to 60ms frame length if + // 1. |uplink_bandwidth_bps| is known to be smaller than a threshold AND + // 2. |uplink_packet_loss_fraction| is known to be smaller than a threshold + // AND + // 3. FEC is not decided or OFF. + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 20); + UpdateNetworkMetrics(controller.get(), kFl20msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, IncreaseTo60MsOnVeryLowUplinkBandwidth) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 20); + // We set packet loss fraction to kFlDecreasingPacketLossFraction, which + // should have prevented frame length to increase, if the uplink bandwidth + // was not this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(20), + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, Maintain60MsOnVeryLowUplinkBandwidth) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 60); + // We set packet loss fraction to FlDecreasingPacketLossFraction, which should + // have caused the frame length to decrease, if the uplink bandwidth was not + // this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(20), + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, UpdateMultipleNetworkMetricsAtOnce) { + // This test is similar to IncreaseTo60MsOnMultipleConditions. But instead of + // using ::UpdateNetworkMetrics(...), which calls + // FrameLengthController::UpdateNetworkMetrics(...) multiple times, we + // we call it only once. This is to verify that + // FrameLengthController::UpdateNetworkMetrics(...) can handle multiple + // network updates at once. This is, however, not a common use case in current + // audio_network_adaptor_impl.cc. + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 20); + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = kFl20msTo60msBandwidthBps; + network_metrics.uplink_packet_loss_fraction = kFlIncreasingPacketLossFraction; + controller->UpdateNetworkMetrics(network_metrics); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, + Maintain20MsIf60MsNotInReceiverFrameLengthRange) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20}, 20); + // Use a low uplink bandwidth and a low uplink packet loss fraction that would + // cause frame length to increase if receiver frame length included 60ms. + UpdateNetworkMetrics(controller.get(), + kFl20msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain20MsOnMediumUplinkBandwidth) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 20); + UpdateNetworkMetrics(controller.get(), + kMediumBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain20MsOnMediumUplinkPacketLossFraction) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60}, 20); + // Use a low uplink bandwidth that would cause frame length to increase if + // uplink packet loss fraction was low. + UpdateNetworkMetrics(controller.get(), + kFl20msTo60msBandwidthBps, + kMediumPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain60MsWhenNo120msCriteriaIsSet) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20, 60, 120}, 60); + UpdateNetworkMetrics(controller.get(), + kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, From120MsTo20MsOnHighUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + {20, 60, 120}, 120); + // It takes two steps for frame length to go from 120ms to 20ms. + UpdateNetworkMetrics(controller.get(), + kFl60msTo20msBandwidthBps, + rtc::nullopt, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), + kFl60msTo20msBandwidthBps, + rtc::nullopt, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, From120MsTo20MsOnHighUplinkPacketLossFraction) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + {20, 60, 120}, 120); + // It takes two steps for frame length to go from 120ms to 20ms. + UpdateNetworkMetrics(controller.get(), rtc::nullopt, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), rtc::nullopt, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain120MsOnVeryLowUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + {20, 60, 120}, 120); + // We set packet loss fraction to FlDecreasingPacketLossFraction, which should + // have caused the frame length to decrease, if the uplink bandwidth was not + // this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(60), + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); +} + +TEST(FrameLengthControllerTest, From60MsTo120MsOnVeryLowUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + {20, 60, 120}, 60); + // We set packet loss fraction to FlDecreasingPacketLossFraction, which should + // have prevented frame length to increase, if the uplink bandwidth was not + // this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(60), + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); +} + +TEST(FrameLengthControllerTest, From20MsTo120MsOnMultipleConditions) { + // Increase to 120ms frame length if + // 1. |uplink_bandwidth_bps| is known to be smaller than a threshold AND + // 2. |uplink_packet_loss_fraction| is known to be smaller than a threshold. + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + {20, 60, 120}, 20); + // It takes two steps for frame length to go from 20ms to 120ms. + UpdateNetworkMetrics(controller.get(), + kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + UpdateNetworkMetrics(controller.get(), + kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); +} + +TEST(FrameLengthControllerTest, Stall60MsIf120MsNotInReceiverFrameLengthRange) { + auto controller = + CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), {20, 60}, 20); + UpdateNetworkMetrics(controller.get(), + kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + UpdateNetworkMetrics(controller.get(), + kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, CheckBehaviorOnChangingNetworkMetrics) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + {20, 60, 120}, 20); + UpdateNetworkMetrics(controller.get(), + kMediumBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); + + UpdateNetworkMetrics(controller.get(), + kFl20msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), + kFl60msTo120msBandwidthBps, + kMediumPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), + kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); + + UpdateNetworkMetrics(controller.get(), + kFl120msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), + kMediumBandwidthBps, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h new file mode 100644 index 0000000000..7687446cac --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_H_ + +#include "api/audio_codecs/audio_encoder.h" +#include "api/optional.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +// An AudioNetworkAdaptor optimizes the audio experience by suggesting a +// suitable runtime configuration (bit rate, frame length, FEC, etc.) to the +// encoder based on network metrics. +class AudioNetworkAdaptor { + public: + virtual ~AudioNetworkAdaptor() = default; + + virtual void SetUplinkBandwidth(int uplink_bandwidth_bps) = 0; + + virtual void SetUplinkPacketLossFraction( + float uplink_packet_loss_fraction) = 0; + + virtual void SetUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) = 0; + + virtual void SetRtt(int rtt_ms) = 0; + + virtual void SetTargetAudioBitrate(int target_audio_bitrate_bps) = 0; + + virtual void SetOverhead(size_t overhead_bytes_per_packet) = 0; + + virtual AudioEncoderRuntimeConfig GetEncoderRuntimeConfig() = 0; + + virtual void StartDebugDump(FILE* file_handle) = 0; + + virtual void StopDebugDump() = 0; + + virtual ANAStats GetStats() const = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h new file mode 100644 index 0000000000..874fc97eb2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_CONFIG_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_CONFIG_H_ + +#include "api/optional.h" + +namespace webrtc { + +struct AudioEncoderRuntimeConfig { + AudioEncoderRuntimeConfig(); + AudioEncoderRuntimeConfig(const AudioEncoderRuntimeConfig& other); + ~AudioEncoderRuntimeConfig(); + AudioEncoderRuntimeConfig& operator=(const AudioEncoderRuntimeConfig& other); + bool operator==(const AudioEncoderRuntimeConfig& other) const; + rtc::Optional bitrate_bps; + rtc::Optional frame_length_ms; + // Note: This is what we tell the encoder. It doesn't have to reflect + // the actual NetworkMetrics; it's subject to our decision. + rtc::Optional uplink_packet_loss_fraction; + rtc::Optional enable_fec; + rtc::Optional enable_dtx; + + // Some encoders can encode fewer channels than the actual input to make + // better use of the bandwidth. |num_channels| sets the number of channels + // to encode. + rtc::Optional num_channels; + + // This is true if the last frame length change was an increase, and otherwise + // false. + // The value of this boolean is used to apply a different offset to the + // per-packet overhead that is reported by the BWE. The exact offset value + // is most important right after a frame length change, because the frame + // length change affects the overhead. In the steady state, the exact value is + // not important because the BWE will compensate. + bool last_fl_change_increase = false; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_CONFIG_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h new file mode 100644 index 0000000000..15dc741e29 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ + +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockAudioNetworkAdaptor : public AudioNetworkAdaptor { + public: + virtual ~MockAudioNetworkAdaptor() { Die(); } + MOCK_METHOD0(Die, void()); + + MOCK_METHOD1(SetUplinkBandwidth, void(int uplink_bandwidth_bps)); + + MOCK_METHOD1(SetUplinkPacketLossFraction, + void(float uplink_packet_loss_fraction)); + + MOCK_METHOD1(SetUplinkRecoverablePacketLossFraction, + void(float uplink_recoverable_packet_loss_fraction)); + + MOCK_METHOD1(SetRtt, void(int rtt_ms)); + + MOCK_METHOD1(SetTargetAudioBitrate, void(int target_audio_bitrate_bps)); + + MOCK_METHOD1(SetOverhead, void(size_t overhead_bytes_per_packet)); + + MOCK_METHOD0(GetEncoderRuntimeConfig, AudioEncoderRuntimeConfig()); + + MOCK_METHOD1(StartDebugDump, void(FILE* file_handle)); + + MOCK_METHOD0(StopDebugDump, void()); + + MOCK_CONST_METHOD0(GetStats, ANAStats()); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h new file mode 100644 index 0000000000..df28e9e26f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_H_ + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockController : public Controller { + public: + virtual ~MockController() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD1(UpdateNetworkMetrics, + void(const NetworkMetrics& network_metrics)); + MOCK_METHOD1(MakeDecision, void(AudioEncoderRuntimeConfig* config)); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h new file mode 100644 index 0000000000..8d410a742d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ + +#include + +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockControllerManager : public ControllerManager { + public: + virtual ~MockControllerManager() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD1( + GetSortedControllers, + std::vector(const Controller::NetworkMetrics& metrics)); + MOCK_CONST_METHOD0(GetControllers, std::vector()); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h new file mode 100644 index 0000000000..06650abbd6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_DEBUG_DUMP_WRITER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_DEBUG_DUMP_WRITER_H_ + +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDebugDumpWriter : public DebugDumpWriter { + public: + virtual ~MockDebugDumpWriter() { Die(); } + MOCK_METHOD0(Die, void()); + + MOCK_METHOD2(DumpEncoderRuntimeConfig, + void(const AudioEncoderRuntimeConfig& config, + int64_t timestamp)); + MOCK_METHOD2(DumpNetworkMetrics, + void(const Controller::NetworkMetrics& metrics, + int64_t timestamp)); +#if WEBRTC_ENABLE_PROTOBUF + MOCK_METHOD2(DumpControllerManagerConfig, + void(const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp)); +#endif +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_DEBUG_DUMP_WRITER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py new file mode 100755 index 0000000000..a52b065f12 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py @@ -0,0 +1,147 @@ +#!/usr/bin/python2 +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# To run this script please copy "out///pyproto/webrtc/modules/ +# audio_coding/audio_network_adaptor/debug_dump_pb2.py" to this folder. +# The you can run this script with: +# "python parse_ana_dump.py -m uplink_bandwidth_bps -f dump_file.dat" +# You can add as may metrics or decisions to the plot as you like. +# form more information call: +# "python parse_ana_dump.py --help" + +import struct +from optparse import OptionParser + +import matplotlib.pyplot as plt + +import debug_dump_pb2 + + +def GetNextMessageSize(file_to_parse): + data = file_to_parse.read(4) + if data == '': + return 0 + return struct.unpack(' bandwidth + // + // If either a.x == b.x or a.y == b.y, the curve can be defined + // by a single point. (We merge the two points into one - either the lower or + // the leftmost one - for easier treatment.) + // + // y-axis ^ | + // | | + // | | + // | | + // | P|__________ + // |---------------> bandwidth + ThresholdCurve(const Point& left, const Point& right) + : a(GetPoint(left, right, true)), + b(GetPoint(left, right, false)), + slope(b.x - a.x == 0.0f ? 0.0f : (b.y - a.y) / (b.x - a.x)), + offset(a.y - slope * a.x) { + // TODO(eladalon): We might want to introduce some numerical validations. + } + + ThresholdCurve(float a_x, float a_y, float b_x, float b_y) + : ThresholdCurve(Point{a_x, a_y}, Point{b_x, b_y}) {} + + // Checks if a point is strictly below the curve. + bool IsBelowCurve(const Point& p) const { + if (p.x < a.x) { + return true; + } else if (p.x == a.x) { + // In principle, we could merge this into the next else, but to avoid + // numerical errors, we treat it separately. + return p.y < a.y; + } else if (a.x < p.x && p.x < b.x) { + return p.y < offset + slope * p.x; + } else { // if (b.x <= p.x) + return p.y < b.y; + } + } + + // Checks if a point is strictly above the curve. + bool IsAboveCurve(const Point& p) const { + if (p.x <= a.x) { + return false; + } else if (a.x < p.x && p.x < b.x) { + return p.y > offset + slope * p.x; + } else { // if (b.x <= p.x) + return p.y > b.y; + } + } + + bool operator<=(const ThresholdCurve& rhs) const { + // This curve is <= the rhs curve if no point from this curve is + // above a corresponding point from the rhs curve. + return !IsBelowCurve(rhs.a) && !IsBelowCurve(rhs.b) && + !rhs.IsAboveCurve(a) && !rhs.IsAboveCurve(b); + } + + private: + static const Point& GetPoint(const Point& left, + const Point& right, + bool is_for_left) { + RTC_DCHECK_LE(left.x, right.x); + RTC_DCHECK_GE(left.y, right.y); + + // Same X-value or Y-value triggers merging both points to the + // lower and/or left of the two points, respectively. + if (left.x == right.x) { + return right; + } else if (left.y == right.y) { + return left; + } + + // If unmerged, boolean flag determines which of the points is desired. + return is_for_left ? left : right; + } + + const Point a; + const Point b; + const float slope; + const float offset; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_UTIL_THRESHOLD_CURVE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc new file mode 100644 index 0000000000..0375e76609 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc @@ -0,0 +1,631 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" +#include "test/gtest.h" + +// A threshold curve divides 2D space into three domains - below, on and above +// the threshold curve. +// The curve is defined by two points. Those points, P1 and P2, are ordered so +// that (P1.x <= P2.x && P1.y >= P2.y). +// The part of the curve which is between the two points is hereon referred +// to as the "segment". +// A "ray" extends from P1 directly upwards into infinity; that's the "vertical +// ray". Likewise, a "horizontal ray" extends from P2 directly rightwards. +// +// ^ | // +// | | vertical ray // +// | | // +// | | // +// | P1| // +// | \ // +// | \ segment // +// | \ // +// | \ horizontal ray // +// | P2 ------------------ // +// *---------------------------> // + +namespace webrtc { + +namespace { +enum RelativePosition { kBelow, kOn, kAbove }; + +void CheckRelativePosition(const ThresholdCurve& curve, + ThresholdCurve::Point point, + RelativePosition pos) { + RTC_CHECK(pos == kBelow || pos == kOn || pos == kAbove); + + EXPECT_EQ(pos == kBelow, curve.IsBelowCurve(point)); + EXPECT_EQ(pos == kAbove, curve.IsAboveCurve(point)); +} +} // namespace + +// Test that the curve correctly reports the below/above position of points, +// when the curve is a "normal" one - P1 and P2 are different in both their +// X and Y values. +TEST(ThresholdCurveTest, PointPositionToCommonCurve) { + // The points (P1-P2) define the curve. // + // All other points are above/below/on the curve. // + // // + // ^ // + // | | // + // | A F J R V // + // | | // + // | B P1 K S W // + // | \ // + // | \ // + // | \ L // + // | \ // + // | C G M T X // + // | \ // + // | N \ // + // | \ // + // | D H O P2--Y---------------- // + // | E I Q U Z // + // *----------------------------------> // + constexpr ThresholdCurve::Point p1{1000, 2000}; + constexpr ThresholdCurve::Point p2{2000, 1000}; + + RTC_CHECK_GT((p1.x + p2.x) / 2, p1.x); + RTC_CHECK_LT((p1.x + p2.x) / 2, p2.x); + RTC_CHECK_LT((p1.y + p2.y) / 2, p1.y); + RTC_CHECK_GT((p1.y + p2.y) / 2, p2.y); + + const ThresholdCurve curve(p1, p2); + + { + // All cases where the point lies to the left of P1. + constexpr float x = p1.x - 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kBelow); // A + CheckRelativePosition(curve, {x, p1.y + 0}, kBelow); // B + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kBelow); // C + CheckRelativePosition(curve, {x, p2.y + 0}, kBelow); // D + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // E + } + + { + // All cases where the point has the same x-value as P1. + constexpr float x = p1.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kOn); // F + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // P1 + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kBelow); // G + CheckRelativePosition(curve, {x, p2.y + 0}, kBelow); // H + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // I + } + + { + // To make sure we're really covering all of the cases, make sure that P1 + // and P2 were chosen so that L would really be below K, and O would really + // be below N. (This would not hold if the Y values are too close together.) + RTC_CHECK_LT(((p1.y + p2.y) / 2) + 1, p1.y); + RTC_CHECK_LT(p2.y, ((p1.y + p2.y) / 2) - 1); + + // All cases where the point's x-value is between P1 and P2. + constexpr float x = (p1.x + p2.x) / 2; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // J + CheckRelativePosition(curve, {x, p1.y + 0}, kAbove); // K + CheckRelativePosition(curve, {x, ((p1.y + p2.y) / 2) + 1}, kAbove); // L + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kOn); // M + CheckRelativePosition(curve, {x, ((p1.y + p2.y) / 2) - 1}, kBelow); // N + CheckRelativePosition(curve, {x, p2.y + 0}, kBelow); // O + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // Q + } + + { + // All cases where the point has the same x-value as P2. + constexpr float x = p2.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // R + CheckRelativePosition(curve, {x, p1.y + 0}, kAbove); // S + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kAbove); // T + CheckRelativePosition(curve, {x, p2.y + 0}, kOn); // P2 + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // U + } + + { + // All cases where the point lies to the right of P2. + constexpr float x = p2.x + 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // V + CheckRelativePosition(curve, {x, p1.y + 0}, kAbove); // W + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kAbove); // X + CheckRelativePosition(curve, {x, p2.y + 0}, kOn); // Y + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // Z + } +} + +// Test that the curve correctly reports the below/above position of points, +// when the curve is defined by two points with the same Y value. +TEST(ThresholdCurveTest, PointPositionToCurveWithHorizaontalSegment) { + // The points (P1-P2) define the curve. + // All other points are above/below/on the curve. + // + // ^ + // | | + // | | + // | A D F I K + // | | + // | | + // | B P1--G--P2-L-- + // | C E H J M + // *------------------> + + constexpr ThresholdCurve::Point p1{100, 200}; + constexpr ThresholdCurve::Point p2{p1.x + 1, p1.y}; + + RTC_CHECK_GT((p1.x + p2.x) / 2, p1.x); + RTC_CHECK_LT((p1.x + p2.x) / 2, p2.x); + + const ThresholdCurve curve(p1, p2); + + { + // All cases where the point lies to the left of P1. + constexpr float x = p1.x - 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kBelow); // A + CheckRelativePosition(curve, {x, p1.y + 0}, kBelow); // B + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // C + } + + { + // All cases where the point has the same x-value as P1. + constexpr float x = p1.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kOn); // D + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // P1 + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // E + } + + { + // All cases where the point's x-value is between P1 and P2. + constexpr float x = (p1.x + p2.x) / 2; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // F + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // G + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // H + } + + { + // All cases where the point has the same x-value as P2. + constexpr float x = p2.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // I + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // P2 + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // J + } + + { + // All cases where the point lies to the right of P2. + constexpr float x = p2.x + 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // K + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // L + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // M + } +} + +// Test that the curve correctly reports the below/above position of points, +// when the curve is defined by two points with the same X value. +TEST(ThresholdCurveTest, PointPositionToCurveWithVerticalSegment) { + // The points (P1-P2) define the curve. + // All other points are above/below/on the curve. + // + // ^ + // | | + // | A B C + // | | + // | D P1 E + // | | + // | F G H + // | | + // | I P2--J------ + // | K L M + // *------------------> + + constexpr ThresholdCurve::Point p1{100, 200}; + constexpr ThresholdCurve::Point p2{p1.x, p1.y - 1}; + + constexpr float left = p1.x - 1; + constexpr float on = p1.x; + constexpr float right = p1.x + 1; + + RTC_CHECK_LT((p1.y + p2.y) / 2, p1.y); + RTC_CHECK_GT((p1.y + p2.y) / 2, p2.y); + + const ThresholdCurve curve(p1, p2); + + { + // All cases where the point lies above P1. + constexpr float y = p1.y + 1; + CheckRelativePosition(curve, {left, y}, kBelow); // A + CheckRelativePosition(curve, {on, y}, kOn); // B + CheckRelativePosition(curve, {right, y}, kAbove); // C + } + + { + // All cases where the point has the same y-value as P1. + constexpr float y = p1.y; + CheckRelativePosition(curve, {left, y}, kBelow); // D + CheckRelativePosition(curve, {on, y}, kOn); // P1 + CheckRelativePosition(curve, {right, y}, kAbove); // E + } + + { + // All cases where the point's y-value is between P1 and P2. + constexpr float y = (p1.y + p2.y) / 2; + CheckRelativePosition(curve, {left, y}, kBelow); // F + CheckRelativePosition(curve, {on, y}, kOn); // G + CheckRelativePosition(curve, {right, y}, kAbove); // H + } + + { + // All cases where the point has the same y-value as P2. + constexpr float y = p2.y; + CheckRelativePosition(curve, {left, y}, kBelow); // I + CheckRelativePosition(curve, {on, y}, kOn); // P2 + CheckRelativePosition(curve, {right, y}, kOn); // J + } + + { + // All cases where the point lies below P2. + constexpr float y = p2.y - 1; + CheckRelativePosition(curve, {left, y}, kBelow); // K + CheckRelativePosition(curve, {on, y}, kBelow); // L + CheckRelativePosition(curve, {right, y}, kBelow); // M + } +} + +// Test that the curve correctly reports the below/above position of points, +// when the curve is defined by two points which are identical. +TEST(ThresholdCurveTest, PointPositionCurveWithNullSegment) { + // The points (P1-P2) define the curve. + // All other points are above/below/on the curve. + // + // ^ + // | | + // | A D F + // | | + // | B P---G------ + // | C E H + // *------------------> + + constexpr ThresholdCurve::Point p{100, 200}; + + const ThresholdCurve curve(p, p); + + { + // All cases where the point lies to the left of P. + constexpr float x = p.x - 1; + CheckRelativePosition(curve, {x, p.y + 1}, kBelow); // A + CheckRelativePosition(curve, {x, p.y + 0}, kBelow); // B + CheckRelativePosition(curve, {x, p.y - 1}, kBelow); // C + } + + { + // All cases where the point has the same x-value as P. + constexpr float x = p.x + 0; + CheckRelativePosition(curve, {x, p.y + 1}, kOn); // D + CheckRelativePosition(curve, {x, p.y + 0}, kOn); // P + CheckRelativePosition(curve, {x, p.y - 1}, kBelow); // E + } + + { + // All cases where the point lies to the right of P. + constexpr float x = p.x + 1; + CheckRelativePosition(curve, {x, p.y + 1}, kAbove); // F + CheckRelativePosition(curve, {x, p.y + 0}, kOn); // G + CheckRelativePosition(curve, {x, p.y - 1}, kBelow); // H + } +} + +// Test that the relative position of two curves is computed correctly when +// the two curves have the same projection on the X-axis. +TEST(ThresholdCurveTest, TwoCurvesSegmentHasSameProjectionAxisX) { + // ^ // + // | C1 + C2 // + // | | // + // | |\ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ -------- C2 // + // | --------- C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + // Same x-values, but higher on Y. (Can be parallel, but doesn't have to be.) + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 20}; + constexpr ThresholdCurve::Point c2_right{c1_right.x, c1_right.y + 10}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the higher curve's projection on the X-axis is a strict subset of the +// lower curve's projection on the X-axis (on both ends). +TEST(ThresholdCurveTest, TwoCurvesSegmentOfHigherSubsetProjectionAxisX) { + // ^ // + // | C1 C2 // + // | | | // + // | | | // + // | \ | // + // | \ | // + // | \ \ // + // | \ \ // + // | \ --------- C2 // + // | \ // + // | \ // + // | ---------C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{6, 11}; + constexpr ThresholdCurve::Point c2_right{9, 7}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the higher curve's right point is above lower curve's horizontal ray (meaning +// the higher curve's projection on the X-axis extends further right than +// the lower curve's). +TEST(ThresholdCurveTest, + TwoCurvesRightPointOfHigherCurveAboveHorizontalRayOfLower) { + // ^ // + // | C1 + C2 // + // | | // + // | |\ // + // | | \ // + // | | \ // + // | | \ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ ----- C2 // + // | --------- C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 1, c1_right.y + 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the higher curve's points are on the lower curve's rays (left point on the +// veritcal ray, right point on the horizontal ray). +TEST(ThresholdCurveTest, TwoCurvesPointsOfHigherOnRaysOfLower) { + // ^ + // | C1 + C2 // + // | | // + // | |\ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | ----- C1 + C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + // Same x-values, but one of the points is higher on Y (the other isn't). + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 2}; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 3, c1_right.y}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the second curve's segment intersects the first curve's vertical ray. +TEST(ThresholdCurveTest, SecondCurveCrossesVerticalRayOfFirstCurve) { + // ^ // + // | C2 C1 // + // | | | // + // | \| // + // | | // + // | |\ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ ------- C2 // + // | -------- C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{c1_left.x - 1, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right{c1_right.x, c1_right.y + 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the second curve's segment intersects the first curve's horizontal ray. +TEST(ThresholdCurveTest, SecondCurveCrossesHorizontalRayOfFirstCurve) { + // ^ // + // | C1 + C2 // + // | | // + // | |\ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | ----------- C1 // + // | \ // + // | ------- C2 // + // *--------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 2, c1_right.y - 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the second curve's segment intersects the first curve's segment. +TEST(ThresholdCurveTest, TwoCurvesWithCrossingSegments) { + // ^ // + // | C2 C1 // + // | | | // + // | | | // + // | | \ // + // | | \ // + // | -_ \ // + // | -_ \ // + // | -_\ // + // | -_ // + // | \-_ // + // | \ ---------- C2 // + // | ----------- C1 // + // | // + // | // + // *-------------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{4, 9}; + constexpr ThresholdCurve::Point c2_right{10, 6}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + // The test is structured so that the two curves intersect at (8, 7). + RTC_CHECK(!c1_curve.IsAboveCurve({8, 7})); + RTC_CHECK(!c1_curve.IsBelowCurve({8, 7})); + RTC_CHECK(!c2_curve.IsAboveCurve({8, 7})); + RTC_CHECK(!c2_curve.IsBelowCurve({8, 7})); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// both curves are identical. +TEST(ThresholdCurveTest, IdenticalCurves) { + // ^ // + // | C1 + C2 // + // | | // + // | | // + // | \ // + // | \ // + // | \ // + // | ------- C1 + C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point left{5, 10}; + constexpr ThresholdCurve::Point right{10, 5}; + + const ThresholdCurve c1_curve(left, right); + const ThresholdCurve c2_curve(left, right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_TRUE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// they are "nearly identical" - the first curve's segment is contained within +// the second curve's segment, but the second curve's segment extends further +// to the left (which also produces separate vertical rays for the curves). +TEST(ThresholdCurveTest, NearlyIdenticalCurvesSecondContinuesOnOtherLeftSide) { + // ^ // + // | C2 C1 // + // | | | // + // | | | // + // | \| // + // | | // + // | \ // + // | \ // + // | \ // + // | ----- C1 + C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_left); + + constexpr ThresholdCurve::Point c2_left{c1_left.x - 1, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right = c1_right; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_TRUE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// they are "nearly identical" - the first curve's segment is contained within +// the second curve's segment, but the second curve's segment extends further +// to the right (which also produces separate horizontal rays for the curves). +TEST(ThresholdCurveTest, NearlyIdenticalCurvesSecondContinuesOnOtherRightSide) { + // ^ // + // | C1 + C2 // + // | | // + // | | // + // | \ // + // | \ // + // | \ // + // | \----------- C1 // + // | \ // + // | ---------- C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_left); + + constexpr ThresholdCurve::Point c2_left = c1_left; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 1, c1_right.y - 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_TRUE(c2_curve <= c1_curve); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// The higher-left point must be given as the first point, and the lower-right +// point must be given as the second. +// This necessarily produces a non-positive slope. +TEST(ThresholdCurveTest, WrongOrderPoints) { + std::unique_ptr curve; + constexpr ThresholdCurve::Point left{5, 10}; + constexpr ThresholdCurve::Point right{10, 5}; + EXPECT_DEATH(curve.reset(new ThresholdCurve(right, left)), ""); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build new file mode 100644 index 0000000000..9928930df2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_network_adaptor_config_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build new file mode 100644 index 0000000000..ae65bba82c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build @@ -0,0 +1,238 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/fec_controller_rplr_based.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_network_adaptor_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/cng_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/cng_gn/moz.build new file mode 100644 index 0000000000..1f448d86d4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/cng_gn/moz.build @@ -0,0 +1,229 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("cng_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/OWNERS new file mode 100644 index 0000000000..e69de29bb2 diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_decoder.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_decoder.h new file mode 100644 index 0000000000..b7b15cdd6e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_decoder.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is for backwards compatibility only! Use +// webrtc/api/audio_codecs/audio_decoder.h instead! +// TODO(kwiberg): Remove it. + +#ifndef MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ +#define MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ + +#include "api/audio_codecs/audio_decoder.h" + +#endif // MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_encoder.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_encoder.h new file mode 100644 index 0000000000..010ae6705f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_encoder.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is for backwards compatibility only! Use +// webrtc/api/audio_codecs/audio_encoder.h instead! +// TODO(ossu): Remove it. + +#ifndef MODULES_AUDIO_CODING_CODECS_AUDIO_ENCODER_H_ +#define MODULES_AUDIO_CODING_CODECS_AUDIO_ENCODER_H_ + +#include "api/audio_codecs/audio_encoder.h" + +#endif // MODULES_AUDIO_CODING_CODECS_AUDIO_ENCODER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.cc new file mode 100644 index 0000000000..a99a28cf2b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/audio_format_conversion.h" + +#include + +#include "api/array_view.h" +#include "api/optional.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/sanitizer.h" + +namespace webrtc { + +namespace { + +CodecInst MakeCodecInst(int payload_type, + const char* name, + int sample_rate, + size_t num_channels) { + // Create a CodecInst with some fields set. The remaining fields are zeroed, + // but we tell MSan to consider them uninitialized. + CodecInst ci = {0}; + rtc::MsanMarkUninitialized(rtc::MakeArrayView(&ci, 1)); + ci.pltype = payload_type; + strncpy(ci.plname, name, sizeof(ci.plname)); + ci.plname[sizeof(ci.plname) - 1] = '\0'; + ci.plfreq = sample_rate; + ci.channels = num_channels; + return ci; +} + +} // namespace + +SdpAudioFormat CodecInstToSdp(const CodecInst& ci) { + if (STR_CASE_CMP(ci.plname, "g722") == 0) { + RTC_CHECK_EQ(16000, ci.plfreq); + RTC_CHECK(ci.channels == 1 || ci.channels == 2); + return {"g722", 8000, ci.channels}; + } else if (STR_CASE_CMP(ci.plname, "opus") == 0) { + RTC_CHECK_EQ(48000, ci.plfreq); + RTC_CHECK(ci.channels == 1 || ci.channels == 2); + return ci.channels == 1 + ? SdpAudioFormat("opus", 48000, 2) + : SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}}); + } else { + return {ci.plname, ci.plfreq, ci.channels}; + } +} + +CodecInst SdpToCodecInst(int payload_type, const SdpAudioFormat& audio_format) { + if (STR_CASE_CMP(audio_format.name.c_str(), "g722") == 0) { + RTC_CHECK_EQ(8000, audio_format.clockrate_hz); + RTC_CHECK(audio_format.num_channels == 1 || audio_format.num_channels == 2); + return MakeCodecInst(payload_type, "g722", 16000, + audio_format.num_channels); + } else if (STR_CASE_CMP(audio_format.name.c_str(), "opus") == 0) { + RTC_CHECK_EQ(48000, audio_format.clockrate_hz); + RTC_CHECK_EQ(2, audio_format.num_channels); + const int num_channels = [&] { + auto stereo = audio_format.parameters.find("stereo"); + if (stereo != audio_format.parameters.end()) { + if (stereo->second == "0") { + return 1; + } else if (stereo->second == "1") { + return 2; + } else { + RTC_CHECK(false); // Bad stereo parameter. + } + } + return 1; // Default to mono. + }(); + return MakeCodecInst(payload_type, "opus", 48000, num_channels); + } else { + return MakeCodecInst(payload_type, audio_format.name.c_str(), + audio_format.clockrate_hz, audio_format.num_channels); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.h new file mode 100644 index 0000000000..d981741c3c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/audio_format_conversion.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_AUDIO_FORMAT_CONVERSION_H_ +#define MODULES_AUDIO_CODING_CODECS_AUDIO_FORMAT_CONVERSION_H_ + +#include "api/audio_codecs/audio_format.h" +#include "common_types.h" // NOLINT(build/include) + +namespace webrtc { + +SdpAudioFormat CodecInstToSdp(const CodecInst& codec_inst); +CodecInst SdpToCodecInst(int payload_type, const SdpAudioFormat& audio_format); + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_AUDIO_FORMAT_CONVERSION_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory.h new file mode 100644 index 0000000000..7494ac0e6d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is for backwards compatibility only! Use +// webrtc/api/audio_codecs/builtin_audio_decoder_factory.h instead! +// TODO(kwiberg): Remove it. + +#ifndef MODULES_AUDIO_CODING_CODECS_BUILTIN_AUDIO_DECODER_FACTORY_H_ +#define MODULES_AUDIO_CODING_CODECS_BUILTIN_AUDIO_DECODER_FACTORY_H_ + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" + +#endif // MODULES_AUDIO_CODING_CODECS_BUILTIN_AUDIO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc new file mode 100644 index 0000000000..27b89b6b34 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(AudioDecoderFactoryTest, CreateUnknownDecoder) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("rey", 8000, 1))); +} + +TEST(AudioDecoderFactoryTest, CreatePcmu) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // PCMu supports 8 kHz, and any number of channels. + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 0))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 1))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 2))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 3))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 16000, 1))); +} + +TEST(AudioDecoderFactoryTest, CreatePcma) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // PCMa supports 8 kHz, and any number of channels. + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 0))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 1))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 2))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 3))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("pcma", 16000, 1))); +} + +TEST(AudioDecoderFactoryTest, CreateIlbc) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // iLBC supports 8 kHz, 1 channel. + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 8000, 0))); +#ifdef WEBRTC_CODEC_ILBC + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 8000, 1))); +#endif + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 8000, 2))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 16000, 1))); +} + +TEST(AudioDecoderFactoryTest, CreateIsac) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // iSAC supports 16 kHz, 1 channel. The float implementation additionally + // supports 32 kHz, 1 channel. + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("isac", 16000, 0))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("isac", 16000, 1))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("isac", 16000, 2))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("isac", 8000, 1))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("isac", 48000, 1))); +#ifdef WEBRTC_ARCH_ARM + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("isac", 32000, 1))); +#else + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("isac", 32000, 1))); +#endif +} + +TEST(AudioDecoderFactoryTest, CreateL16) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // L16 supports any clock rate, any number of channels. + const int clockrates[] = {8000, 16000, 32000, 48000}; + const int num_channels[] = {1, 2, 3, 4711}; + for (int clockrate : clockrates) { + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("l16", clockrate, 0))); + for (int channels : num_channels) { + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("l16", clockrate, channels))); + } + } +} + +TEST(AudioDecoderFactoryTest, CreateG722) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // g722 supports 8 kHz, 1-2 channels. + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 0))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 1))); + EXPECT_TRUE(adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 2))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 3))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("g722", 16000, 1))); + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("g722", 32000, 1))); + + // g722 actually uses a 16 kHz sample rate instead of the nominal 8 kHz. + std::unique_ptr dec = + adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 1)); + EXPECT_EQ(16000, dec->SampleRateHz()); +} + +TEST(AudioDecoderFactoryTest, CreateOpus) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // Opus supports 48 kHz, 2 channels, and wants a "stereo" parameter whose + // value is either "0" or "1". + for (int hz : {8000, 16000, 32000, 48000}) { + for (int channels : {0, 1, 2, 3}) { + for (std::string stereo : {"XX", "0", "1", "2"}) { + std::map params; + if (stereo != "XX") { + params["stereo"] = stereo; + } + const bool good = (hz == 48000 && channels == 2 && + (stereo == "XX" || stereo == "0" || stereo == "1")); + EXPECT_EQ(good, static_cast(adf->MakeAudioDecoder(SdpAudioFormat( + "opus", hz, channels, std::move(params))))); + } + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h new file mode 100644 index 0000000000..6ec765ab97 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is for backwards compatibility only! Use +// webrtc/api/audio_codecs/builtin_audio_decoder_factory.h instead! +// TODO(ossu): Remove it. + +#ifndef MODULES_AUDIO_CODING_CODECS_BUILTIN_AUDIO_ENCODER_FACTORY_H_ +#define MODULES_AUDIO_CODING_CODECS_BUILTIN_AUDIO_ENCODER_FACTORY_H_ + +#include "api/audio_codecs/builtin_audio_encoder_factory.h" + +#endif // MODULES_AUDIO_CODING_CODECS_BUILTIN_AUDIO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc new file mode 100644 index 0000000000..984bfd24f6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +class AudioEncoderFactoryTest + : public ::testing::TestWithParam> { +}; + +TEST_P(AudioEncoderFactoryTest, SupportsAtLeastOneFormat) { + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + EXPECT_FALSE(supported_encoders.empty()); +} + +TEST_P(AudioEncoderFactoryTest, CanQueryAllSupportedFormats) { + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + for (const auto& spec : supported_encoders) { + auto info = factory->QueryAudioEncoder(spec.format); + EXPECT_TRUE(info); + } +} + +TEST_P(AudioEncoderFactoryTest, CanConstructAllSupportedEncoders) { + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + for (const auto& spec : supported_encoders) { + auto info = factory->QueryAudioEncoder(spec.format); + auto encoder = factory->MakeAudioEncoder(127, spec.format); + EXPECT_TRUE(encoder); + EXPECT_EQ(encoder->SampleRateHz(), info->sample_rate_hz); + EXPECT_EQ(encoder->NumChannels(), info->num_channels); + EXPECT_EQ(encoder->RtpTimestampRateHz(), spec.format.clockrate_hz); + } +} + +TEST_P(AudioEncoderFactoryTest, CanRunAllSupportedEncoders) { + constexpr int kTestPayloadType = 127; + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + for (const auto& spec : supported_encoders) { + auto encoder = factory->MakeAudioEncoder(kTestPayloadType, spec.format); + EXPECT_TRUE(encoder); + encoder->Reset(); + const int num_samples = rtc::checked_cast( + encoder->SampleRateHz() * encoder->NumChannels() / 100); + rtc::Buffer out; + rtc::BufferT audio; + audio.SetData(num_samples, [](rtc::ArrayView audio) { + for (size_t i = 0; i != audio.size(); ++i) { + // Just put some numbers in there, ensure they're within range. + audio[i] = + static_cast(i & std::numeric_limits::max()); + } + return audio.size(); + }); + // This is here to stop the test going forever with a broken encoder. + constexpr int kMaxEncodeCalls = 100; + int blocks = 0; + for (; blocks < kMaxEncodeCalls; ++blocks) { + AudioEncoder::EncodedInfo info = encoder->Encode( + blocks * encoder->RtpTimestampRateHz() / 100, audio, &out); + EXPECT_EQ(info.encoded_bytes, out.size()); + if (info.encoded_bytes > 0) { + EXPECT_EQ(0u, info.encoded_timestamp); + EXPECT_EQ(kTestPayloadType, info.payload_type); + break; + } + } + ASSERT_LT(blocks, kMaxEncodeCalls); + const unsigned int next_timestamp = + blocks * encoder->RtpTimestampRateHz() / 100; + out.Clear(); + for (; blocks < kMaxEncodeCalls; ++blocks) { + AudioEncoder::EncodedInfo info = encoder->Encode( + blocks * encoder->RtpTimestampRateHz() / 100, audio, &out); + EXPECT_EQ(info.encoded_bytes, out.size()); + if (info.encoded_bytes > 0) { + EXPECT_EQ(next_timestamp, info.encoded_timestamp); + EXPECT_EQ(kTestPayloadType, info.payload_type); + break; + } + } + ASSERT_LT(blocks, kMaxEncodeCalls); + } +} + +INSTANTIATE_TEST_CASE_P(BuiltinAudioEncoderFactoryTest, + AudioEncoderFactoryTest, + ::testing::Values(CreateBuiltinAudioEncoderFactory())); + +TEST(BuiltinAudioEncoderFactoryTest, SupportsTheExpectedFormats) { + using ::testing::ElementsAreArray; + // Check that we claim to support the formats we expect from build flags, and + // we've ordered them correctly. + auto factory = CreateBuiltinAudioEncoderFactory(); + auto specs = factory->GetSupportedEncoders(); + + const std::vector supported_formats = [&specs] { + std::vector formats; + for (const auto& spec : specs) { + formats.push_back(spec.format); + } + return formats; + }(); + + const std::vector expected_formats = { +#ifdef WEBRTC_CODEC_OPUS + {"opus", 48000, 2, {{"minptime", "10"}, {"useinbandfec", "1"}}}, +#endif +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) + {"isac", 16000, 1}, +#endif +#ifdef WEBRTC_CODEC_ISAC + {"isac", 32000, 1}, +#endif + {"G722", 8000, 1}, +#ifdef WEBRTC_CODEC_ILBC + {"ilbc", 8000, 1}, +#endif + {"pcmu", 8000, 1}, + {"pcma", 8000, 1} + }; + + ASSERT_THAT(supported_formats, ElementsAreArray(expected_formats)); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc new file mode 100644 index 0000000000..78148ab350 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" + +#include +#include +#include +#include + +namespace webrtc { + +namespace { + +const int kMaxFrameSizeMs = 60; + +} // namespace + +AudioEncoderCng::Config::Config() = default; +AudioEncoderCng::Config::Config(Config&&) = default; +AudioEncoderCng::Config::~Config() = default; + +bool AudioEncoderCng::Config::IsOk() const { + if (num_channels != 1) + return false; + if (!speech_encoder) + return false; + if (num_channels != speech_encoder->NumChannels()) + return false; + if (sid_frame_interval_ms < + static_cast(speech_encoder->Max10MsFramesInAPacket() * 10)) + return false; + if (num_cng_coefficients > WEBRTC_CNG_MAX_LPC_ORDER || + num_cng_coefficients <= 0) + return false; + return true; +} + +AudioEncoderCng::AudioEncoderCng(Config&& config) + : speech_encoder_( + ([&] { RTC_CHECK(config.IsOk()) << "Invalid configuration."; }(), + std::move(config.speech_encoder))), + cng_payload_type_(config.payload_type), + num_cng_coefficients_(config.num_cng_coefficients), + sid_frame_interval_ms_(config.sid_frame_interval_ms), + last_frame_active_(true), + vad_(config.vad ? std::unique_ptr(config.vad) + : CreateVad(config.vad_mode)), + cng_encoder_(new ComfortNoiseEncoder(SampleRateHz(), + sid_frame_interval_ms_, + num_cng_coefficients_)) { +} + +AudioEncoderCng::~AudioEncoderCng() = default; + +int AudioEncoderCng::SampleRateHz() const { + return speech_encoder_->SampleRateHz(); +} + +size_t AudioEncoderCng::NumChannels() const { + return 1; +} + +int AudioEncoderCng::RtpTimestampRateHz() const { + return speech_encoder_->RtpTimestampRateHz(); +} + +size_t AudioEncoderCng::Num10MsFramesInNextPacket() const { + return speech_encoder_->Num10MsFramesInNextPacket(); +} + +size_t AudioEncoderCng::Max10MsFramesInAPacket() const { + return speech_encoder_->Max10MsFramesInAPacket(); +} + +int AudioEncoderCng::GetTargetBitrate() const { + return speech_encoder_->GetTargetBitrate(); +} + +AudioEncoder::EncodedInfo AudioEncoderCng::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + const size_t samples_per_10ms_frame = SamplesPer10msFrame(); + RTC_CHECK_EQ(speech_buffer_.size(), + rtp_timestamps_.size() * samples_per_10ms_frame); + rtp_timestamps_.push_back(rtp_timestamp); + RTC_DCHECK_EQ(samples_per_10ms_frame, audio.size()); + speech_buffer_.insert(speech_buffer_.end(), audio.cbegin(), audio.cend()); + const size_t frames_to_encode = speech_encoder_->Num10MsFramesInNextPacket(); + if (rtp_timestamps_.size() < frames_to_encode) { + return EncodedInfo(); + } + RTC_CHECK_LE(frames_to_encode * 10, kMaxFrameSizeMs) + << "Frame size cannot be larger than " << kMaxFrameSizeMs + << " ms when using VAD/CNG."; + + // Group several 10 ms blocks per VAD call. Call VAD once or twice using the + // following split sizes: + // 10 ms = 10 + 0 ms; 20 ms = 20 + 0 ms; 30 ms = 30 + 0 ms; + // 40 ms = 20 + 20 ms; 50 ms = 30 + 20 ms; 60 ms = 30 + 30 ms. + size_t blocks_in_first_vad_call = + (frames_to_encode > 3 ? 3 : frames_to_encode); + if (frames_to_encode == 4) + blocks_in_first_vad_call = 2; + RTC_CHECK_GE(frames_to_encode, blocks_in_first_vad_call); + const size_t blocks_in_second_vad_call = + frames_to_encode - blocks_in_first_vad_call; + + // Check if all of the buffer is passive speech. Start with checking the first + // block. + Vad::Activity activity = vad_->VoiceActivity( + &speech_buffer_[0], samples_per_10ms_frame * blocks_in_first_vad_call, + SampleRateHz()); + if (activity == Vad::kPassive && blocks_in_second_vad_call > 0) { + // Only check the second block if the first was passive. + activity = vad_->VoiceActivity( + &speech_buffer_[samples_per_10ms_frame * blocks_in_first_vad_call], + samples_per_10ms_frame * blocks_in_second_vad_call, SampleRateHz()); + } + + EncodedInfo info; + switch (activity) { + case Vad::kPassive: { + info = EncodePassive(frames_to_encode, encoded); + last_frame_active_ = false; + break; + } + case Vad::kActive: { + info = EncodeActive(frames_to_encode, encoded); + last_frame_active_ = true; + break; + } + case Vad::kError: { + FATAL(); // Fails only if fed invalid data. + break; + } + } + + speech_buffer_.erase( + speech_buffer_.begin(), + speech_buffer_.begin() + frames_to_encode * samples_per_10ms_frame); + rtp_timestamps_.erase(rtp_timestamps_.begin(), + rtp_timestamps_.begin() + frames_to_encode); + return info; +} + +void AudioEncoderCng::Reset() { + speech_encoder_->Reset(); + speech_buffer_.clear(); + rtp_timestamps_.clear(); + last_frame_active_ = true; + vad_->Reset(); + cng_encoder_.reset( + new ComfortNoiseEncoder(SampleRateHz(), sid_frame_interval_ms_, + num_cng_coefficients_)); +} + +bool AudioEncoderCng::SetFec(bool enable) { + return speech_encoder_->SetFec(enable); +} + +bool AudioEncoderCng::SetDtx(bool enable) { + return speech_encoder_->SetDtx(enable); +} + +bool AudioEncoderCng::SetApplication(Application application) { + return speech_encoder_->SetApplication(application); +} + +void AudioEncoderCng::SetMaxPlaybackRate(int frequency_hz) { + speech_encoder_->SetMaxPlaybackRate(frequency_hz); +} + +rtc::ArrayView> +AudioEncoderCng::ReclaimContainedEncoders() { + return rtc::ArrayView>(&speech_encoder_, 1); +} + +void AudioEncoderCng::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkPacketLossFraction( + uplink_packet_loss_fraction); +} + +void AudioEncoderCng::OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkRecoverablePacketLossFraction( + uplink_recoverable_packet_loss_fraction); +} + +void AudioEncoderCng::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional bwe_period_ms) { + speech_encoder_->OnReceivedUplinkBandwidth(target_audio_bitrate_bps, + bwe_period_ms); +} + +AudioEncoder::EncodedInfo AudioEncoderCng::EncodePassive( + size_t frames_to_encode, + rtc::Buffer* encoded) { + bool force_sid = last_frame_active_; + bool output_produced = false; + const size_t samples_per_10ms_frame = SamplesPer10msFrame(); + AudioEncoder::EncodedInfo info; + + for (size_t i = 0; i < frames_to_encode; ++i) { + // It's important not to pass &info.encoded_bytes directly to + // WebRtcCng_Encode(), since later loop iterations may return zero in + // that value, in which case we don't want to overwrite any value from + // an earlier iteration. + size_t encoded_bytes_tmp = + cng_encoder_->Encode( + rtc::ArrayView( + &speech_buffer_[i * samples_per_10ms_frame], + samples_per_10ms_frame), + force_sid, encoded); + + if (encoded_bytes_tmp > 0) { + RTC_CHECK(!output_produced); + info.encoded_bytes = encoded_bytes_tmp; + output_produced = true; + force_sid = false; + } + } + + info.encoded_timestamp = rtp_timestamps_.front(); + info.payload_type = cng_payload_type_; + info.send_even_if_empty = true; + info.speech = false; + return info; +} + +AudioEncoder::EncodedInfo AudioEncoderCng::EncodeActive( + size_t frames_to_encode, + rtc::Buffer* encoded) { + const size_t samples_per_10ms_frame = SamplesPer10msFrame(); + AudioEncoder::EncodedInfo info; + for (size_t i = 0; i < frames_to_encode; ++i) { + info = + speech_encoder_->Encode(rtp_timestamps_.front(), + rtc::ArrayView( + &speech_buffer_[i * samples_per_10ms_frame], + samples_per_10ms_frame), + encoded); + if (i + 1 == frames_to_encode) { + RTC_CHECK_GT(info.encoded_bytes, 0) << "Encoder didn't deliver data."; + } else { + RTC_CHECK_EQ(info.encoded_bytes, 0) + << "Encoder delivered data too early."; + } + } + return info; +} + +size_t AudioEncoderCng::SamplesPer10msFrame() const { + return rtc::CheckedDivExact(10 * SampleRateHz(), 1000); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h new file mode 100644 index 0000000000..4491289f28 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_CNG_AUDIO_ENCODER_CNG_H_ +#define MODULES_AUDIO_CODING_CODECS_CNG_AUDIO_ENCODER_CNG_H_ + +#include +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "common_audio/vad/include/vad.h" +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class Vad; + +class AudioEncoderCng final : public AudioEncoder { + public: + struct Config { + Config(); + Config(Config&&); + ~Config(); + bool IsOk() const; + + size_t num_channels = 1; + int payload_type = 13; + std::unique_ptr speech_encoder; + Vad::Aggressiveness vad_mode = Vad::kVadNormal; + int sid_frame_interval_ms = 100; + int num_cng_coefficients = 8; + // The Vad pointer is mainly for testing. If a NULL pointer is passed, the + // AudioEncoderCng creates (and destroys) a Vad object internally. If an + // object is passed, the AudioEncoderCng assumes ownership of the Vad + // object. + Vad* vad = nullptr; + }; + + explicit AudioEncoderCng(Config&& config); + ~AudioEncoderCng() override; + + int SampleRateHz() const override; + size_t NumChannels() const override; + int RtpTimestampRateHz() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + void Reset() override; + bool SetFec(bool enable) override; + bool SetDtx(bool enable) override; + bool SetApplication(Application application) override; + void SetMaxPlaybackRate(int frequency_hz) override; + rtc::ArrayView> ReclaimContainedEncoders() + override; + void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) override; + void OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional bwe_period_ms) override; + + private: + EncodedInfo EncodePassive(size_t frames_to_encode, + rtc::Buffer* encoded); + EncodedInfo EncodeActive(size_t frames_to_encode, + rtc::Buffer* encoded); + size_t SamplesPer10msFrame() const; + + std::unique_ptr speech_encoder_; + const int cng_payload_type_; + const int num_cng_coefficients_; + const int sid_frame_interval_ms_; + std::vector speech_buffer_; + std::vector rtp_timestamps_; + bool last_frame_active_; + std::unique_ptr vad_; + std::unique_ptr cng_encoder_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderCng); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_CNG_AUDIO_ENCODER_CNG_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc new file mode 100644 index 0000000000..6e246c9a55 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "common_audio/vad/mock/mock_vad.h" +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/mock_audio_encoder.h" + +using ::testing::Return; +using ::testing::_; +using ::testing::SetArgPointee; +using ::testing::InSequence; +using ::testing::Invoke; + +namespace webrtc { + +namespace { +static const size_t kMaxNumSamples = 48 * 10 * 2; // 10 ms @ 48 kHz stereo. +static const size_t kMockReturnEncodedBytes = 17; +static const int kCngPayloadType = 18; +} + +class AudioEncoderCngTest : public ::testing::Test { + protected: + AudioEncoderCngTest() + : mock_encoder_owner_(new MockAudioEncoder), + mock_encoder_(mock_encoder_owner_.get()), + mock_vad_(new MockVad), + timestamp_(4711), + num_audio_samples_10ms_(0), + sample_rate_hz_(8000) { + memset(audio_, 0, kMaxNumSamples * 2); + EXPECT_CALL(*mock_encoder_, NumChannels()).WillRepeatedly(Return(1)); + EXPECT_CALL(*mock_encoder_, Die()).Times(1); + } + + void TearDown() override { + EXPECT_CALL(*mock_vad_, Die()).Times(1); + cng_.reset(); + } + + AudioEncoderCng::Config MakeCngConfig() { + AudioEncoderCng::Config config; + config.speech_encoder = std::move(mock_encoder_owner_); + EXPECT_TRUE(config.speech_encoder); + + // Let the AudioEncoderCng object use a MockVad instead of its internally + // created Vad object. + config.vad = mock_vad_; + config.payload_type = kCngPayloadType; + + return config; + } + + void CreateCng(AudioEncoderCng::Config&& config) { + num_audio_samples_10ms_ = static_cast(10 * sample_rate_hz_ / 1000); + ASSERT_LE(num_audio_samples_10ms_, kMaxNumSamples); + if (config.speech_encoder) { + EXPECT_CALL(*mock_encoder_, SampleRateHz()) + .WillRepeatedly(Return(sample_rate_hz_)); + // Max10MsFramesInAPacket() is just used to verify that the SID frame + // period is not too small. The return value does not matter that much, + // as long as it is smaller than 10. + EXPECT_CALL(*mock_encoder_, Max10MsFramesInAPacket()) + .WillOnce(Return(1u)); + } + cng_.reset(new AudioEncoderCng(std::move(config))); + } + + void Encode() { + ASSERT_TRUE(cng_) << "Must call CreateCng() first."; + encoded_info_ = cng_->Encode( + timestamp_, + rtc::ArrayView(audio_, num_audio_samples_10ms_), + &encoded_); + timestamp_ += static_cast(num_audio_samples_10ms_); + } + + // Expect |num_calls| calls to the encoder, all successful. The last call + // claims to have encoded |kMockReturnEncodedBytes| bytes, and all the + // preceding ones 0 bytes. + void ExpectEncodeCalls(size_t num_calls) { + InSequence s; + AudioEncoder::EncodedInfo info; + for (size_t j = 0; j < num_calls - 1; ++j) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Return(info)); + } + info.encoded_bytes = kMockReturnEncodedBytes; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce( + Invoke(MockAudioEncoder::FakeEncoding(kMockReturnEncodedBytes))); + } + + // Verifies that the cng_ object waits until it has collected + // |blocks_per_frame| blocks of audio, and then dispatches all of them to + // the underlying codec (speech or cng). + void CheckBlockGrouping(size_t blocks_per_frame, bool active_speech) { + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(blocks_per_frame)); + auto config = MakeCngConfig(); + const int num_cng_coefficients = config.num_cng_coefficients; + CreateCng(std::move(config)); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillRepeatedly(Return(active_speech ? Vad::kActive : Vad::kPassive)); + + // Don't expect any calls to the encoder yet. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)).Times(0); + for (size_t i = 0; i < blocks_per_frame - 1; ++i) { + Encode(); + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + } + if (active_speech) + ExpectEncodeCalls(blocks_per_frame); + Encode(); + if (active_speech) { + EXPECT_EQ(kMockReturnEncodedBytes, encoded_info_.encoded_bytes); + } else { + EXPECT_EQ(static_cast(num_cng_coefficients + 1), + encoded_info_.encoded_bytes); + } + } + + // Verifies that the audio is partitioned into larger blocks before calling + // the VAD. + void CheckVadInputSize(int input_frame_size_ms, + int expected_first_block_size_ms, + int expected_second_block_size_ms) { + const size_t blocks_per_frame = + static_cast(input_frame_size_ms / 10); + + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(blocks_per_frame)); + + // Expect nothing to happen before the last block is sent to cng_. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)).Times(0); + for (size_t i = 0; i < blocks_per_frame - 1; ++i) { + Encode(); + } + + // Let the VAD decision be passive, since an active decision may lead to + // early termination of the decision loop. + InSequence s; + EXPECT_CALL( + *mock_vad_, + VoiceActivity(_, expected_first_block_size_ms * sample_rate_hz_ / 1000, + sample_rate_hz_)).WillOnce(Return(Vad::kPassive)); + if (expected_second_block_size_ms > 0) { + EXPECT_CALL(*mock_vad_, + VoiceActivity( + _, expected_second_block_size_ms * sample_rate_hz_ / 1000, + sample_rate_hz_)).WillOnce(Return(Vad::kPassive)); + } + + // With this call to Encode(), |mock_vad_| should be called according to the + // above expectations. + Encode(); + } + + // Tests a frame with both active and passive speech. Returns true if the + // decision was active speech, false if it was passive. + bool CheckMixedActivePassive(Vad::Activity first_type, + Vad::Activity second_type) { + // Set the speech encoder frame size to 60 ms, to ensure that the VAD will + // be called twice. + const size_t blocks_per_frame = 6; + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(blocks_per_frame)); + InSequence s; + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(first_type)); + if (first_type == Vad::kPassive) { + // Expect a second call to the VAD only if the first frame was passive. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(second_type)); + } + encoded_info_.payload_type = 0; + for (size_t i = 0; i < blocks_per_frame; ++i) { + Encode(); + } + return encoded_info_.payload_type != kCngPayloadType; + } + + std::unique_ptr cng_; + std::unique_ptr mock_encoder_owner_; + MockAudioEncoder* mock_encoder_; + MockVad* mock_vad_; // Ownership is transferred to |cng_|. + uint32_t timestamp_; + int16_t audio_[kMaxNumSamples]; + size_t num_audio_samples_10ms_; + rtc::Buffer encoded_; + AudioEncoder::EncodedInfo encoded_info_; + int sample_rate_hz_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderCngTest); +}; + +TEST_F(AudioEncoderCngTest, CreateAndDestroy) { + CreateCng(MakeCngConfig()); +} + +TEST_F(AudioEncoderCngTest, CheckFrameSizePropagation) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillOnce(Return(17U)); + EXPECT_EQ(17U, cng_->Num10MsFramesInNextPacket()); +} + +TEST_F(AudioEncoderCngTest, CheckTargetAudioBitratePropagation) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, + OnReceivedUplinkBandwidth(4711, rtc::Optional())); + cng_->OnReceivedUplinkBandwidth(4711, rtc::nullopt); +} + +TEST_F(AudioEncoderCngTest, CheckPacketLossFractionPropagation) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, OnReceivedUplinkPacketLossFraction(0.5)); + cng_->OnReceivedUplinkPacketLossFraction(0.5); +} + +TEST_F(AudioEncoderCngTest, EncodeCallsVad) { + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(1U)); + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kPassive)); + Encode(); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects1BlockPassiveSpeech) { + CheckBlockGrouping(1, false); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects2BlocksPassiveSpeech) { + CheckBlockGrouping(2, false); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects3BlocksPassiveSpeech) { + CheckBlockGrouping(3, false); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects1BlockActiveSpeech) { + CheckBlockGrouping(1, true); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects2BlocksActiveSpeech) { + CheckBlockGrouping(2, true); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects3BlocksActiveSpeech) { + CheckBlockGrouping(3, true); +} + +TEST_F(AudioEncoderCngTest, EncodePassive) { + const size_t kBlocksPerFrame = 3; + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(kBlocksPerFrame)); + auto config = MakeCngConfig(); + const auto sid_frame_interval_ms = config.sid_frame_interval_ms; + const auto num_cng_coefficients = config.num_cng_coefficients; + CreateCng(std::move(config)); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillRepeatedly(Return(Vad::kPassive)); + // Expect no calls at all to the speech encoder mock. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)).Times(0); + uint32_t expected_timestamp = timestamp_; + for (size_t i = 0; i < 100; ++i) { + Encode(); + // Check if it was time to call the cng encoder. This is done once every + // |kBlocksPerFrame| calls. + if ((i + 1) % kBlocksPerFrame == 0) { + // Now check if a SID interval has elapsed. + if ((i % (sid_frame_interval_ms / 10)) < kBlocksPerFrame) { + // If so, verify that we got a CNG encoding. + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); + EXPECT_FALSE(encoded_info_.speech); + EXPECT_EQ(static_cast(num_cng_coefficients) + 1, + encoded_info_.encoded_bytes); + EXPECT_EQ(expected_timestamp, encoded_info_.encoded_timestamp); + } + expected_timestamp += rtc::checked_cast( + kBlocksPerFrame * num_audio_samples_10ms_); + } else { + // Otherwise, expect no output. + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + } + } +} + +// Verifies that the correct action is taken for frames with both active and +// passive speech. +TEST_F(AudioEncoderCngTest, MixedActivePassive) { + CreateCng(MakeCngConfig()); + + // All of the frame is active speech. + ExpectEncodeCalls(6); + EXPECT_TRUE(CheckMixedActivePassive(Vad::kActive, Vad::kActive)); + EXPECT_TRUE(encoded_info_.speech); + + // First half of the frame is active speech. + ExpectEncodeCalls(6); + EXPECT_TRUE(CheckMixedActivePassive(Vad::kActive, Vad::kPassive)); + EXPECT_TRUE(encoded_info_.speech); + + // Second half of the frame is active speech. + ExpectEncodeCalls(6); + EXPECT_TRUE(CheckMixedActivePassive(Vad::kPassive, Vad::kActive)); + EXPECT_TRUE(encoded_info_.speech); + + // All of the frame is passive speech. Expect no calls to |mock_encoder_|. + EXPECT_FALSE(CheckMixedActivePassive(Vad::kPassive, Vad::kPassive)); + EXPECT_FALSE(encoded_info_.speech); +} + +// These tests verify that the audio is partitioned into larger blocks before +// calling the VAD. +// The parameters for CheckVadInputSize are: +// CheckVadInputSize(frame_size, expected_first_block_size, +// expected_second_block_size); +TEST_F(AudioEncoderCngTest, VadInputSize10Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(10, 10, 0); +} +TEST_F(AudioEncoderCngTest, VadInputSize20Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(20, 20, 0); +} +TEST_F(AudioEncoderCngTest, VadInputSize30Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(30, 30, 0); +} +TEST_F(AudioEncoderCngTest, VadInputSize40Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(40, 20, 20); +} +TEST_F(AudioEncoderCngTest, VadInputSize50Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(50, 30, 20); +} +TEST_F(AudioEncoderCngTest, VadInputSize60Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(60, 30, 30); +} + +// Verifies that the correct payload type is set when CNG is encoded. +TEST_F(AudioEncoderCngTest, VerifyCngPayloadType) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)).Times(0); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()).WillOnce(Return(1U)); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kPassive)); + encoded_info_.payload_type = 0; + Encode(); + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); +} + +// Verifies that a SID frame is encoded immediately as the signal changes from +// active speech to passive. +TEST_F(AudioEncoderCngTest, VerifySidFrameAfterSpeech) { + auto config = MakeCngConfig(); + const auto num_cng_coefficients = config.num_cng_coefficients; + CreateCng(std::move(config)); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(1U)); + // Start with encoding noise. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .Times(2) + .WillRepeatedly(Return(Vad::kPassive)); + Encode(); + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); + EXPECT_EQ(static_cast(num_cng_coefficients) + 1, + encoded_info_.encoded_bytes); + // Encode again, and make sure we got no frame at all (since the SID frame + // period is 100 ms by default). + Encode(); + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + + // Now encode active speech. + encoded_info_.payload_type = 0; + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kActive)); + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce( + Invoke(MockAudioEncoder::FakeEncoding(kMockReturnEncodedBytes))); + Encode(); + EXPECT_EQ(kMockReturnEncodedBytes, encoded_info_.encoded_bytes); + + // Go back to noise again, and verify that a SID frame is emitted. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kPassive)); + Encode(); + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); + EXPECT_EQ(static_cast(num_cng_coefficients) + 1, + encoded_info_.encoded_bytes); +} + +// Resetting the CNG should reset both the VAD and the encoder. +TEST_F(AudioEncoderCngTest, Reset) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, Reset()).Times(1); + EXPECT_CALL(*mock_vad_, Reset()).Times(1); + cng_->Reset(); +} + +#if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// This test fixture tests various error conditions that makes the +// AudioEncoderCng die via CHECKs. +class AudioEncoderCngDeathTest : public AudioEncoderCngTest { + protected: + AudioEncoderCngDeathTest() : AudioEncoderCngTest() { + EXPECT_CALL(*mock_vad_, Die()).Times(1); + delete mock_vad_; + mock_vad_ = nullptr; + } + + // Override AudioEncoderCngTest::TearDown, since that one expects a call to + // the destructor of |mock_vad_|. In this case, that object is already + // deleted. + void TearDown() override { + cng_.reset(); + } + + AudioEncoderCng::Config MakeCngConfig() { + // Don't provide a Vad mock object, since it would leak when the test dies. + auto config = AudioEncoderCngTest::MakeCngConfig(); + config.vad = nullptr; + return config; + } + + void TryWrongNumCoefficients(int num) { + EXPECT_DEATH( + [&] { + auto config = MakeCngConfig(); + config.num_cng_coefficients = num; + CreateCng(std::move(config)); + }(), + "Invalid configuration"); + } +}; + +TEST_F(AudioEncoderCngDeathTest, WrongFrameSize) { + CreateCng(MakeCngConfig()); + num_audio_samples_10ms_ *= 2; // 20 ms frame. + EXPECT_DEATH(Encode(), ""); + num_audio_samples_10ms_ = 0; // Zero samples. + EXPECT_DEATH(Encode(), ""); +} + +TEST_F(AudioEncoderCngDeathTest, WrongNumCoefficientsA) { + TryWrongNumCoefficients(-1); +} + +TEST_F(AudioEncoderCngDeathTest, WrongNumCoefficientsB) { + TryWrongNumCoefficients(0); +} + +TEST_F(AudioEncoderCngDeathTest, WrongNumCoefficientsC) { + TryWrongNumCoefficients(13); +} + +TEST_F(AudioEncoderCngDeathTest, NullSpeechEncoder) { + auto config = MakeCngConfig(); + config.speech_encoder = nullptr; + EXPECT_DEATH(CreateCng(std::move(config)), ""); +} + +TEST_F(AudioEncoderCngDeathTest, StereoEncoder) { + EXPECT_CALL(*mock_encoder_, NumChannels()).WillRepeatedly(Return(2)); + EXPECT_DEATH(CreateCng(MakeCngConfig()), "Invalid configuration"); +} + +TEST_F(AudioEncoderCngDeathTest, StereoConfig) { + EXPECT_DEATH( + [&] { + auto config = MakeCngConfig(); + config.num_channels = 2; + CreateCng(std::move(config)); + }(), + "Invalid configuration"); +} + +TEST_F(AudioEncoderCngDeathTest, EncoderFrameSizeTooLarge) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(7U)); + for (int i = 0; i < 6; ++i) + Encode(); + EXPECT_DEATH(Encode(), + "Frame size cannot be larger than 60 ms when using VAD/CNG."); +} + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc new file mode 100644 index 0000000000..54e5189d30 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include +#include + +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +enum { + kSidShortIntervalUpdate = 1, + kSidNormalIntervalUpdate = 100, + kSidLongIntervalUpdate = 10000 +}; + +enum : size_t { + kCNGNumParamsLow = 0, + kCNGNumParamsNormal = 8, + kCNGNumParamsHigh = WEBRTC_CNG_MAX_LPC_ORDER, + kCNGNumParamsTooHigh = WEBRTC_CNG_MAX_LPC_ORDER + 1 +}; + +enum { + kNoSid, + kForceSid +}; + +class CngTest : public ::testing::Test { + protected: + virtual void SetUp(); + + void TestCngEncode(int sample_rate_hz, int quality); + + int16_t speech_data_[640]; // Max size of CNG internal buffers. +}; + +void CngTest::SetUp() { + FILE* input_file; + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + input_file = fopen(file_name.c_str(), "rb"); + ASSERT_TRUE(input_file != NULL); + ASSERT_EQ(640, static_cast(fread(speech_data_, sizeof(int16_t), + 640, input_file))); + fclose(input_file); + input_file = NULL; +} + +void CngTest::TestCngEncode(int sample_rate_hz, int quality) { + const size_t num_samples_10ms = rtc::CheckedDivExact(sample_rate_hz, 100); + rtc::Buffer sid_data; + + ComfortNoiseEncoder cng_encoder(sample_rate_hz, kSidNormalIntervalUpdate, + quality); + EXPECT_EQ(0U, cng_encoder.Encode(rtc::ArrayView( + speech_data_, num_samples_10ms), + kNoSid, &sid_data)); + EXPECT_EQ(static_cast(quality + 1), + cng_encoder.Encode( + rtc::ArrayView(speech_data_, num_samples_10ms), + kForceSid, &sid_data)); +} + +#if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Create CNG encoder, init with faulty values, free CNG encoder. +TEST_F(CngTest, CngInitFail) { + // Call with too few parameters. + EXPECT_DEATH({ ComfortNoiseEncoder(8000, kSidNormalIntervalUpdate, + kCNGNumParamsLow); }, ""); + // Call with too many parameters. + EXPECT_DEATH({ ComfortNoiseEncoder(8000, kSidNormalIntervalUpdate, + kCNGNumParamsTooHigh); }, ""); +} + +// Encode Cng with too long input vector. +TEST_F(CngTest, CngEncodeTooLong) { + rtc::Buffer sid_data; + + // Create encoder. + ComfortNoiseEncoder cng_encoder(8000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + // Run encoder with too much data. + EXPECT_DEATH( + cng_encoder.Encode(rtc::ArrayView(speech_data_, 641), + kNoSid, &sid_data), + ""); +} +#endif // GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +TEST_F(CngTest, CngEncode8000) { + TestCngEncode(8000, kCNGNumParamsNormal); +} + +TEST_F(CngTest, CngEncode16000) { + TestCngEncode(16000, kCNGNumParamsNormal); +} + +TEST_F(CngTest, CngEncode32000) { + TestCngEncode(32000, kCNGNumParamsHigh); +} + +TEST_F(CngTest, CngEncode48000) { + TestCngEncode(48000, kCNGNumParamsNormal); +} + +TEST_F(CngTest, CngEncode64000) { + TestCngEncode(64000, kCNGNumParamsNormal); +} + +// Update SID parameters, for both 9 and 16 parameters. +TEST_F(CngTest, CngUpdateSid) { + rtc::Buffer sid_data; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // Run normal Encode and UpdateSid. + EXPECT_EQ(kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + cng_decoder.UpdateSid(sid_data); + + // Reinit with new length. + cng_encoder.Reset(16000, kSidNormalIntervalUpdate, kCNGNumParamsHigh); + cng_decoder.Reset(); + + // Expect 0 because of unstable parameters after switching length. + EXPECT_EQ(0U, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + EXPECT_EQ( + kCNGNumParamsHigh + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_ + 160, 160), + kForceSid, &sid_data)); + cng_decoder.UpdateSid( + rtc::ArrayView(sid_data.data(), kCNGNumParamsNormal + 1)); +} + +// Update SID parameters, with wrong parameters or without calling decode. +TEST_F(CngTest, CngUpdateSidErroneous) { + rtc::Buffer sid_data; + + // Encode. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + EXPECT_EQ(kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + + // First run with valid parameters, then with too many CNG parameters. + // The function will operate correctly by only reading the maximum number of + // parameters, skipping the extra. + EXPECT_EQ(kCNGNumParamsNormal + 1, sid_data.size()); + cng_decoder.UpdateSid(sid_data); + + // Make sure the input buffer is large enough. Since Encode() appends data, we + // need to set the size manually only afterwards, or the buffer will be bigger + // than anticipated. + sid_data.SetSize(kCNGNumParamsTooHigh + 1); + cng_decoder.UpdateSid(sid_data); +} + +// Test to generate cng data, by forcing SID. Both normal and faulty condition. +TEST_F(CngTest, CngGenerate) { + rtc::Buffer sid_data; + int16_t out_data[640]; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // Normal Encode. + EXPECT_EQ(kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + + // Normal UpdateSid. + cng_decoder.UpdateSid(sid_data); + + // Two normal Generate, one with new_period. + EXPECT_TRUE(cng_decoder.Generate(rtc::ArrayView(out_data, 640), 1)); + EXPECT_TRUE(cng_decoder.Generate(rtc::ArrayView(out_data, 640), 0)); + + // Call Genereate with too much data. + EXPECT_FALSE(cng_decoder.Generate(rtc::ArrayView(out_data, 641), 0)); +} + +// Test automatic SID. +TEST_F(CngTest, CngAutoSid) { + rtc::Buffer sid_data; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // Normal Encode, 100 msec, where no SID data should be generated. + for (int i = 0; i < 10; i++) { + EXPECT_EQ(0U, cng_encoder.Encode( + rtc::ArrayView(speech_data_, 160), kNoSid, &sid_data)); + } + + // We have reached 100 msec, and SID data should be generated. + EXPECT_EQ(kCNGNumParamsNormal + 1, cng_encoder.Encode( + rtc::ArrayView(speech_data_, 160), kNoSid, &sid_data)); +} + +// Test automatic SID, with very short interval. +TEST_F(CngTest, CngAutoSidShort) { + rtc::Buffer sid_data; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidShortIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // First call will never generate SID, unless forced to. + EXPECT_EQ(0U, cng_encoder.Encode( + rtc::ArrayView(speech_data_, 160), kNoSid, &sid_data)); + + // Normal Encode, 100 msec, SID data should be generated all the time. + for (int i = 0; i < 10; i++) { + EXPECT_EQ(kCNGNumParamsNormal + 1, cng_encoder.Encode( + rtc::ArrayView(speech_data_, 160), kNoSid, &sid_data)); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc new file mode 100644 index 0000000000..8b8e57eb44 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" + +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { + +const size_t kCngMaxOutsizeOrder = 640; + +// TODO(ossu): Rename the left-over WebRtcCng according to style guide. +void WebRtcCng_K2a16(int16_t* k, int useOrder, int16_t* a); + +const int32_t WebRtcCng_kDbov[94] = { + 1081109975, 858756178, 682134279, 541838517, 430397633, 341876992, + 271562548, 215709799, 171344384, 136103682, 108110997, 85875618, + 68213428, 54183852, 43039763, 34187699, 27156255, 21570980, + 17134438, 13610368, 10811100, 8587562, 6821343, 5418385, + 4303976, 3418770, 2715625, 2157098, 1713444, 1361037, + 1081110, 858756, 682134, 541839, 430398, 341877, + 271563, 215710, 171344, 136104, 108111, 85876, + 68213, 54184, 43040, 34188, 27156, 21571, + 17134, 13610, 10811, 8588, 6821, 5418, + 4304, 3419, 2716, 2157, 1713, 1361, + 1081, 859, 682, 542, 430, 342, + 272, 216, 171, 136, 108, 86, + 68, 54, 43, 34, 27, 22, + 17, 14, 11, 9, 7, 5, + 4, 3, 3, 2, 2, 1, + 1, 1, 1, 1 +}; + +const int16_t WebRtcCng_kCorrWindow[WEBRTC_CNG_MAX_LPC_ORDER] = { + 32702, 32636, 32570, 32505, 32439, 32374, + 32309, 32244, 32179, 32114, 32049, 31985 +}; + +} // namespace + +ComfortNoiseDecoder::ComfortNoiseDecoder() { + /* Needed to get the right function pointers in SPLIB. */ + WebRtcSpl_Init(); + Reset(); +} + +void ComfortNoiseDecoder::Reset() { + dec_seed_ = 7777; /* For debugging only. */ + dec_target_energy_ = 0; + dec_used_energy_ = 0; + for (auto& c : dec_target_reflCoefs_) + c = 0; + for (auto& c : dec_used_reflCoefs_) + c = 0; + for (auto& c : dec_filtstate_) + c = 0; + for (auto& c : dec_filtstateLow_) + c = 0; + dec_order_ = 5; + dec_target_scale_factor_ = 0; + dec_used_scale_factor_ = 0; +} + +void ComfortNoiseDecoder::UpdateSid(rtc::ArrayView sid) { + int16_t refCs[WEBRTC_CNG_MAX_LPC_ORDER]; + int32_t targetEnergy; + size_t length = sid.size(); + /* Throw away reflection coefficients of higher order than we can handle. */ + if (length > (WEBRTC_CNG_MAX_LPC_ORDER + 1)) + length = WEBRTC_CNG_MAX_LPC_ORDER + 1; + + dec_order_ = static_cast(length - 1); + + uint8_t sid0 = std::min(sid[0], 93); + targetEnergy = WebRtcCng_kDbov[sid0]; + /* Take down target energy to 75%. */ + targetEnergy = targetEnergy >> 1; + targetEnergy += targetEnergy >> 2; + + dec_target_energy_ = targetEnergy; + + /* Reconstruct coeffs with tweak for WebRtc implementation of RFC3389. */ + if (dec_order_ == WEBRTC_CNG_MAX_LPC_ORDER) { + for (size_t i = 0; i < (dec_order_); i++) { + refCs[i] = sid[i + 1] << 8; /* Q7 to Q15*/ + dec_target_reflCoefs_[i] = refCs[i]; + } + } else { + for (size_t i = 0; i < (dec_order_); i++) { + refCs[i] = (sid[i + 1] - 127) << 8; /* Q7 to Q15. */ + dec_target_reflCoefs_[i] = refCs[i]; + } + } + + for (size_t i = (dec_order_); i < WEBRTC_CNG_MAX_LPC_ORDER; i++) { + refCs[i] = 0; + dec_target_reflCoefs_[i] = refCs[i]; + } +} + +bool ComfortNoiseDecoder::Generate(rtc::ArrayView out_data, + bool new_period) { + int16_t excitation[kCngMaxOutsizeOrder]; + int16_t low[kCngMaxOutsizeOrder]; + int16_t lpPoly[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t ReflBetaStd = 26214; /* 0.8 in q15. */ + int16_t ReflBetaCompStd = 6553; /* 0.2 in q15. */ + int16_t ReflBetaNewP = 19661; /* 0.6 in q15. */ + int16_t ReflBetaCompNewP = 13107; /* 0.4 in q15. */ + int16_t Beta, BetaC; /* These are in Q15. */ + int32_t targetEnergy; + int16_t En; + int16_t temp16; + const size_t num_samples = out_data.size(); + + if (num_samples > kCngMaxOutsizeOrder) { + return false; + } + + if (new_period) { + dec_used_scale_factor_ = dec_target_scale_factor_; + Beta = ReflBetaNewP; + BetaC = ReflBetaCompNewP; + } else { + Beta = ReflBetaStd; + BetaC = ReflBetaCompStd; + } + + /* Calculate new scale factor in Q13 */ + dec_used_scale_factor_ = + rtc::checked_cast( + WEBRTC_SPL_MUL_16_16_RSFT(dec_used_scale_factor_, Beta >> 2, 13) + + WEBRTC_SPL_MUL_16_16_RSFT(dec_target_scale_factor_, BetaC >> 2, 13)); + + dec_used_energy_ = dec_used_energy_ >> 1; + dec_used_energy_ += dec_target_energy_ >> 1; + + /* Do the same for the reflection coeffs, albeit in Q15. */ + for (size_t i = 0; i < WEBRTC_CNG_MAX_LPC_ORDER; i++) { + dec_used_reflCoefs_[i] = (int16_t) WEBRTC_SPL_MUL_16_16_RSFT( + dec_used_reflCoefs_[i], Beta, 15); + dec_used_reflCoefs_[i] += (int16_t) WEBRTC_SPL_MUL_16_16_RSFT( + dec_target_reflCoefs_[i], BetaC, 15); + } + + /* Compute the polynomial coefficients. */ + WebRtcCng_K2a16(dec_used_reflCoefs_, WEBRTC_CNG_MAX_LPC_ORDER, lpPoly); + + + targetEnergy = dec_used_energy_; + + /* Calculate scaling factor based on filter energy. */ + En = 8192; /* 1.0 in Q13. */ + for (size_t i = 0; i < (WEBRTC_CNG_MAX_LPC_ORDER); i++) { + /* Floating point value for reference. + E *= 1.0 - (dec_used_reflCoefs_[i] / 32768.0) * + (dec_used_reflCoefs_[i] / 32768.0); + */ + + /* Same in fixed point. */ + /* K(i).^2 in Q15. */ + temp16 = (int16_t) WEBRTC_SPL_MUL_16_16_RSFT( + dec_used_reflCoefs_[i], dec_used_reflCoefs_[i], 15); + /* 1 - K(i).^2 in Q15. */ + temp16 = 0x7fff - temp16; + En = (int16_t) WEBRTC_SPL_MUL_16_16_RSFT(En, temp16, 15); + } + + /* float scaling= sqrt(E * dec_target_energy_ / (1 << 24)); */ + + /* Calculate sqrt(En * target_energy / excitation energy) */ + targetEnergy = WebRtcSpl_Sqrt(dec_used_energy_); + + En = (int16_t) WebRtcSpl_Sqrt(En) << 6; + En = (En * 3) >> 1; /* 1.5 estimates sqrt(2). */ + dec_used_scale_factor_ = (int16_t)((En * targetEnergy) >> 12); + + /* Generate excitation. */ + /* Excitation energy per sample is 2.^24 - Q13 N(0,1). */ + for (size_t i = 0; i < num_samples; i++) { + excitation[i] = WebRtcSpl_RandN(&dec_seed_) >> 1; + } + + /* Scale to correct energy. */ + WebRtcSpl_ScaleVector(excitation, excitation, dec_used_scale_factor_, + num_samples, 13); + + /* |lpPoly| - Coefficients in Q12. + * |excitation| - Speech samples. + * |nst->dec_filtstate| - State preservation. + * |out_data| - Filtered speech samples. */ + WebRtcSpl_FilterAR(lpPoly, WEBRTC_CNG_MAX_LPC_ORDER + 1, excitation, + num_samples, dec_filtstate_, WEBRTC_CNG_MAX_LPC_ORDER, + dec_filtstateLow_, WEBRTC_CNG_MAX_LPC_ORDER, + out_data.data(), low, num_samples); + + return true; +} + +ComfortNoiseEncoder::ComfortNoiseEncoder(int fs, int interval, int quality) + : enc_nrOfCoefs_(quality), + enc_sampfreq_(fs), + enc_interval_(interval), + enc_msSinceSid_(0), + enc_Energy_(0), + enc_reflCoefs_{0}, + enc_corrVector_{0}, + enc_seed_(7777) /* For debugging only. */ { + RTC_CHECK_GT(quality, 0); + RTC_CHECK_LE(quality, WEBRTC_CNG_MAX_LPC_ORDER); + /* Needed to get the right function pointers in SPLIB. */ + WebRtcSpl_Init(); +} + +void ComfortNoiseEncoder::Reset(int fs, int interval, int quality) { + RTC_CHECK_GT(quality, 0); + RTC_CHECK_LE(quality, WEBRTC_CNG_MAX_LPC_ORDER); + enc_nrOfCoefs_ = quality; + enc_sampfreq_ = fs; + enc_interval_ = interval; + enc_msSinceSid_ = 0; + enc_Energy_ = 0; + for (auto& c : enc_reflCoefs_) + c = 0; + for (auto& c : enc_corrVector_) + c = 0; + enc_seed_ = 7777; /* For debugging only. */ +} + +size_t ComfortNoiseEncoder::Encode(rtc::ArrayView speech, + bool force_sid, + rtc::Buffer* output) { + int16_t arCoefs[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int32_t corrVector[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t refCs[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t hanningW[kCngMaxOutsizeOrder]; + int16_t ReflBeta = 19661; /* 0.6 in q15. */ + int16_t ReflBetaComp = 13107; /* 0.4 in q15. */ + int32_t outEnergy; + int outShifts; + size_t i; + int stab; + int acorrScale; + size_t index; + size_t ind, factor; + int32_t* bptr; + int32_t blo, bhi; + int16_t negate; + const int16_t* aptr; + int16_t speechBuf[kCngMaxOutsizeOrder]; + + const size_t num_samples = speech.size(); + RTC_CHECK_LE(num_samples, kCngMaxOutsizeOrder); + + for (i = 0; i < num_samples; i++) { + speechBuf[i] = speech[i]; + } + + factor = num_samples; + + /* Calculate energy and a coefficients. */ + outEnergy = WebRtcSpl_Energy(speechBuf, num_samples, &outShifts); + while (outShifts > 0) { + /* We can only do 5 shifts without destroying accuracy in + * division factor. */ + if (outShifts > 5) { + outEnergy <<= (outShifts - 5); + outShifts = 5; + } else { + factor /= 2; + outShifts--; + } + } + outEnergy = WebRtcSpl_DivW32W16(outEnergy, (int16_t)factor); + + if (outEnergy > 1) { + /* Create Hanning Window. */ + WebRtcSpl_GetHanningWindow(hanningW, num_samples / 2); + for (i = 0; i < (num_samples / 2); i++) + hanningW[num_samples - i - 1] = hanningW[i]; + + WebRtcSpl_ElementwiseVectorMult(speechBuf, hanningW, speechBuf, num_samples, + 14); + + WebRtcSpl_AutoCorrelation(speechBuf, num_samples, enc_nrOfCoefs_, + corrVector, &acorrScale); + + if (*corrVector == 0) + *corrVector = WEBRTC_SPL_WORD16_MAX; + + /* Adds the bandwidth expansion. */ + aptr = WebRtcCng_kCorrWindow; + bptr = corrVector; + + /* (zzz) lpc16_1 = 17+1+820+2+2 = 842 (ordo2=700). */ + for (ind = 0; ind < enc_nrOfCoefs_; ind++) { + /* The below code multiplies the 16 b corrWindow values (Q15) with + * the 32 b corrvector (Q0) and shifts the result down 15 steps. */ + negate = *bptr < 0; + if (negate) + *bptr = -*bptr; + + blo = (int32_t) * aptr * (*bptr & 0xffff); + bhi = ((blo >> 16) & 0xffff) + + ((int32_t)(*aptr++) * ((*bptr >> 16) & 0xffff)); + blo = (blo & 0xffff) | ((bhi & 0xffff) << 16); + + *bptr = (((bhi >> 16) & 0x7fff) << 17) | ((uint32_t) blo >> 15); + if (negate) + *bptr = -*bptr; + bptr++; + } + /* End of bandwidth expansion. */ + + stab = WebRtcSpl_LevinsonDurbin(corrVector, arCoefs, refCs, + enc_nrOfCoefs_); + + if (!stab) { + /* Disregard from this frame */ + return 0; + } + + } else { + for (i = 0; i < enc_nrOfCoefs_; i++) + refCs[i] = 0; + } + + if (force_sid) { + /* Read instantaneous values instead of averaged. */ + for (i = 0; i < enc_nrOfCoefs_; i++) + enc_reflCoefs_[i] = refCs[i]; + enc_Energy_ = outEnergy; + } else { + /* Average history with new values. */ + for (i = 0; i < enc_nrOfCoefs_; i++) { + enc_reflCoefs_[i] = (int16_t) WEBRTC_SPL_MUL_16_16_RSFT( + enc_reflCoefs_[i], ReflBeta, 15); + enc_reflCoefs_[i] += + (int16_t) WEBRTC_SPL_MUL_16_16_RSFT(refCs[i], ReflBetaComp, 15); + } + enc_Energy_ = + (outEnergy >> 2) + (enc_Energy_ >> 1) + (enc_Energy_ >> 2); + } + + if (enc_Energy_ < 1) { + enc_Energy_ = 1; + } + + if ((enc_msSinceSid_ > (enc_interval_ - 1)) || force_sid) { + /* Search for best dbov value. */ + index = 0; + for (i = 1; i < 93; i++) { + /* Always round downwards. */ + if ((enc_Energy_ - WebRtcCng_kDbov[i]) > 0) { + index = i; + break; + } + } + if ((i == 93) && (index == 0)) + index = 94; + + const size_t output_coefs = enc_nrOfCoefs_ + 1; + output->AppendData(output_coefs, [&] (rtc::ArrayView output) { + output[0] = (uint8_t)index; + + /* Quantize coefficients with tweak for WebRtc implementation of + * RFC3389. */ + if (enc_nrOfCoefs_ == WEBRTC_CNG_MAX_LPC_ORDER) { + for (i = 0; i < enc_nrOfCoefs_; i++) { + /* Q15 to Q7 with rounding. */ + output[i + 1] = ((enc_reflCoefs_[i] + 128) >> 8); + } + } else { + for (i = 0; i < enc_nrOfCoefs_; i++) { + /* Q15 to Q7 with rounding. */ + output[i + 1] = (127 + ((enc_reflCoefs_[i] + 128) >> 8)); + } + } + + return output_coefs; + }); + + enc_msSinceSid_ = + static_cast((1000 * num_samples) / enc_sampfreq_); + return output_coefs; + } else { + enc_msSinceSid_ += + static_cast((1000 * num_samples) / enc_sampfreq_); + return 0; + } +} + +namespace { +/* Values in |k| are Q15, and |a| Q12. */ +void WebRtcCng_K2a16(int16_t* k, int useOrder, int16_t* a) { + int16_t any[WEBRTC_SPL_MAX_LPC_ORDER + 1]; + int16_t* aptr; + int16_t* aptr2; + int16_t* anyptr; + const int16_t* kptr; + int m, i; + + kptr = k; + *a = 4096; /* i.e., (Word16_MAX >> 3) + 1 */ + *any = *a; + a[1] = (*k + 4) >> 3; + for (m = 1; m < useOrder; m++) { + kptr++; + aptr = a; + aptr++; + aptr2 = &a[m]; + anyptr = any; + anyptr++; + + any[m + 1] = (*kptr + 4) >> 3; + for (i = 0; i < m; i++) { + *anyptr++ = + (*aptr++) + + (int16_t)((((int32_t)(*aptr2--) * (int32_t)*kptr) + 16384) >> 15); + } + + aptr = a; + anyptr = any; + for (i = 0; i < (m + 2); i++) { + *aptr++ = *anyptr++; + } + } +} + +} // namespace + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.h new file mode 100644 index 0000000000..5e21b8fd4f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/cng/webrtc_cng.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#ifndef MODULES_AUDIO_CODING_CODECS_CNG_WEBRTC_CNG_H_ +#define MODULES_AUDIO_CODING_CODECS_CNG_WEBRTC_CNG_H_ + +#include + +#include "api/array_view.h" +#include "rtc_base/buffer.h" +#include "typedefs.h" // NOLINT(build/include) + +#define WEBRTC_CNG_MAX_LPC_ORDER 12 + +namespace webrtc { + +class ComfortNoiseDecoder { + public: + ComfortNoiseDecoder(); + ~ComfortNoiseDecoder() = default; + + ComfortNoiseDecoder(const ComfortNoiseDecoder&) = delete; + ComfortNoiseDecoder& operator=(const ComfortNoiseDecoder&) = delete; + + void Reset(); + + // Updates the CN state when a new SID packet arrives. + // |sid| is a view of the SID packet without the headers. + void UpdateSid(rtc::ArrayView sid); + + // Generates comfort noise. + // |out_data| will be filled with samples - its size determines the number of + // samples generated. When |new_period| is true, CNG history will be reset + // before any audio is generated. Returns |false| if outData is too large - + // currently 640 bytes (equalling 10ms at 64kHz). + // TODO(ossu): Specify better limits for the size of out_data. Either let it + // be unbounded or limit to 10ms in the current sample rate. + bool Generate(rtc::ArrayView out_data, bool new_period); + + private: + uint32_t dec_seed_; + int32_t dec_target_energy_; + int32_t dec_used_energy_; + int16_t dec_target_reflCoefs_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t dec_used_reflCoefs_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t dec_filtstate_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t dec_filtstateLow_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + uint16_t dec_order_; + int16_t dec_target_scale_factor_; /* Q29 */ + int16_t dec_used_scale_factor_; /* Q29 */ +}; + +class ComfortNoiseEncoder { + public: + // Creates a comfort noise encoder. + // |fs| selects sample rate: 8000 for narrowband or 16000 for wideband. + // |interval| sets the interval at which to generate SID data (in ms). + // |quality| selects the number of refl. coeffs. Maximum allowed is 12. + ComfortNoiseEncoder(int fs, int interval, int quality); + ~ComfortNoiseEncoder() = default; + + ComfortNoiseEncoder(const ComfortNoiseEncoder&) = delete; + ComfortNoiseEncoder& operator=(const ComfortNoiseEncoder&) = delete; + + // Resets the comfort noise encoder to its initial state. + // Parameters are set as during construction. + void Reset(int fs, int interval, int quality); + + // Analyzes background noise from |speech| and appends coefficients to + // |output|. Returns the number of coefficients generated. If |force_sid| is + // true, a SID frame is forced and the internal sid interval counter is reset. + // Will fail if the input size is too large (> 640 samples, see + // ComfortNoiseDecoder::Generate). + size_t Encode(rtc::ArrayView speech, + bool force_sid, + rtc::Buffer* output); + + private: + size_t enc_nrOfCoefs_; + int enc_sampfreq_; + int16_t enc_interval_; + int16_t enc_msSinceSid_; + int32_t enc_Energy_; + int16_t enc_reflCoefs_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int32_t enc_corrVector_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + uint32_t enc_seed_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_CNG_WEBRTC_CNG_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/OWNERS new file mode 100644 index 0000000000..e69de29bb2 diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc new file mode 100644 index 0000000000..a620a3ed7b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" + +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "modules/audio_coding/codecs/g711/g711_interface.h" + +namespace webrtc { + +void AudioDecoderPcmU::Reset() {} + +std::vector AudioDecoderPcmU::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, 8 * num_channels_, 8); +} + +int AudioDecoderPcmU::SampleRateHz() const { + return 8000; +} + +size_t AudioDecoderPcmU::Channels() const { + return num_channels_; +} + +int AudioDecoderPcmU::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + int16_t temp_type = 1; // Default is speech. + size_t ret = WebRtcG711_DecodeU(encoded, encoded_len, decoded, &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return static_cast(ret); +} + +int AudioDecoderPcmU::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // One encoded byte per sample per channel. + return static_cast(encoded_len / Channels()); +} + +void AudioDecoderPcmA::Reset() {} + +std::vector AudioDecoderPcmA::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, 8 * num_channels_, 8); +} + +int AudioDecoderPcmA::SampleRateHz() const { + return 8000; +} + +size_t AudioDecoderPcmA::Channels() const { + return num_channels_; +} + +int AudioDecoderPcmA::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + int16_t temp_type = 1; // Default is speech. + size_t ret = WebRtcG711_DecodeA(encoded, encoded_len, decoded, &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return static_cast(ret); +} + +int AudioDecoderPcmA::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // One encoded byte per sample per channel. + return static_cast(encoded_len / Channels()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h new file mode 100644 index 0000000000..29e4fa6593 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G711_AUDIO_DECODER_PCM_H_ +#define MODULES_AUDIO_CODING_CODECS_G711_AUDIO_DECODER_PCM_H_ + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class AudioDecoderPcmU final : public AudioDecoder { + public: + explicit AudioDecoderPcmU(size_t num_channels) : num_channels_(num_channels) { + RTC_DCHECK_GE(num_channels, 1); + } + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + const size_t num_channels_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderPcmU); +}; + +class AudioDecoderPcmA final : public AudioDecoder { + public: + explicit AudioDecoderPcmA(size_t num_channels) : num_channels_(num_channels) { + RTC_DCHECK_GE(num_channels, 1); + } + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + const size_t num_channels_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderPcmA); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_G711_AUDIO_DECODER_PCM_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc new file mode 100644 index 0000000000..9fb94fd5c1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" + +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/g711/g711_interface.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { + +template +typename T::Config CreateConfig(const CodecInst& codec_inst) { + typename T::Config config; + config.frame_size_ms = codec_inst.pacsize / 8; + config.num_channels = codec_inst.channels; + config.payload_type = codec_inst.pltype; + return config; +} + +} // namespace + +bool AudioEncoderPcm::Config::IsOk() const { + return (frame_size_ms % 10 == 0) && (num_channels >= 1); +} + +AudioEncoderPcm::AudioEncoderPcm(const Config& config, int sample_rate_hz) + : sample_rate_hz_(sample_rate_hz), + num_channels_(config.num_channels), + payload_type_(config.payload_type), + num_10ms_frames_per_packet_( + static_cast(config.frame_size_ms / 10)), + full_frame_samples_( + config.num_channels * config.frame_size_ms * sample_rate_hz / 1000), + first_timestamp_in_buffer_(0) { + RTC_CHECK_GT(sample_rate_hz, 0) << "Sample rate must be larger than 0 Hz"; + RTC_CHECK_EQ(config.frame_size_ms % 10, 0) + << "Frame size must be an integer multiple of 10 ms."; + speech_buffer_.reserve(full_frame_samples_); +} + +AudioEncoderPcm::~AudioEncoderPcm() = default; + +int AudioEncoderPcm::SampleRateHz() const { + return sample_rate_hz_; +} + +size_t AudioEncoderPcm::NumChannels() const { + return num_channels_; +} + +size_t AudioEncoderPcm::Num10MsFramesInNextPacket() const { + return num_10ms_frames_per_packet_; +} + +size_t AudioEncoderPcm::Max10MsFramesInAPacket() const { + return num_10ms_frames_per_packet_; +} + +int AudioEncoderPcm::GetTargetBitrate() const { + return static_cast( + 8 * BytesPerSample() * SampleRateHz() * NumChannels()); +} + +AudioEncoder::EncodedInfo AudioEncoderPcm::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + if (speech_buffer_.empty()) { + first_timestamp_in_buffer_ = rtp_timestamp; + } + speech_buffer_.insert(speech_buffer_.end(), audio.begin(), audio.end()); + if (speech_buffer_.size() < full_frame_samples_) { + return EncodedInfo(); + } + RTC_CHECK_EQ(speech_buffer_.size(), full_frame_samples_); + EncodedInfo info; + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.encoded_bytes = + encoded->AppendData(full_frame_samples_ * BytesPerSample(), + [&] (rtc::ArrayView encoded) { + return EncodeCall(&speech_buffer_[0], + full_frame_samples_, + encoded.data()); + }); + speech_buffer_.clear(); + info.encoder_type = GetCodecType(); + return info; +} + +void AudioEncoderPcm::Reset() { + speech_buffer_.clear(); +} + +AudioEncoderPcmA::AudioEncoderPcmA(const CodecInst& codec_inst) + : AudioEncoderPcmA(CreateConfig(codec_inst)) {} + +size_t AudioEncoderPcmA::EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) { + return WebRtcG711_EncodeA(audio, input_len, encoded); +} + +size_t AudioEncoderPcmA::BytesPerSample() const { + return 1; +} + +AudioEncoder::CodecType AudioEncoderPcmA::GetCodecType() const { + return AudioEncoder::CodecType::kPcmA; +} + +AudioEncoderPcmU::AudioEncoderPcmU(const CodecInst& codec_inst) + : AudioEncoderPcmU(CreateConfig(codec_inst)) {} + +size_t AudioEncoderPcmU::EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) { + return WebRtcG711_EncodeU(audio, input_len, encoded); +} + +size_t AudioEncoderPcmU::BytesPerSample() const { + return 1; +} + +AudioEncoder::CodecType AudioEncoderPcmU::GetCodecType() const { + return AudioEncoder::CodecType::kPcmU; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h new file mode 100644 index 0000000000..37b67cf72b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G711_AUDIO_ENCODER_PCM_H_ +#define MODULES_AUDIO_CODING_CODECS_G711_AUDIO_ENCODER_PCM_H_ + +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class AudioEncoderPcm : public AudioEncoder { + public: + struct Config { + public: + bool IsOk() const; + + int frame_size_ms; + size_t num_channels; + int payload_type; + + protected: + explicit Config(int pt) + : frame_size_ms(20), num_channels(1), payload_type(pt) {} + }; + + ~AudioEncoderPcm() override; + + int SampleRateHz() const override; + size_t NumChannels() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + void Reset() override; + + protected: + AudioEncoderPcm(const Config& config, int sample_rate_hz); + + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + virtual size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) = 0; + + virtual size_t BytesPerSample() const = 0; + + // Used to set EncodedInfoLeaf::encoder_type in + // AudioEncoderPcm::EncodeImpl + virtual AudioEncoder::CodecType GetCodecType() const = 0; + + private: + const int sample_rate_hz_; + const size_t num_channels_; + const int payload_type_; + const size_t num_10ms_frames_per_packet_; + const size_t full_frame_samples_; + std::vector speech_buffer_; + uint32_t first_timestamp_in_buffer_; +}; + +struct CodecInst; + +class AudioEncoderPcmA final : public AudioEncoderPcm { + public: + struct Config : public AudioEncoderPcm::Config { + Config() : AudioEncoderPcm::Config(8) {} + }; + + explicit AudioEncoderPcmA(const Config& config) + : AudioEncoderPcm(config, kSampleRateHz) {} + explicit AudioEncoderPcmA(const CodecInst& codec_inst); + + protected: + size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) override; + + size_t BytesPerSample() const override; + + AudioEncoder::CodecType GetCodecType() const override; + + private: + static const int kSampleRateHz = 8000; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderPcmA); +}; + +class AudioEncoderPcmU final : public AudioEncoderPcm { + public: + struct Config : public AudioEncoderPcm::Config { + Config() : AudioEncoderPcm::Config(0) {} + }; + + explicit AudioEncoderPcmU(const Config& config) + : AudioEncoderPcm(config, kSampleRateHz) {} + explicit AudioEncoderPcmU(const CodecInst& codec_inst); + + protected: + size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) override; + + size_t BytesPerSample() const override; + + AudioEncoder::CodecType GetCodecType() const override; + + private: + static const int kSampleRateHz = 8000; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderPcmU); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_G711_AUDIO_ENCODER_PCM_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.c new file mode 100644 index 0000000000..2b1a428a3a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.c @@ -0,0 +1,73 @@ +/* + * SpanDSP - a series of DSP components for telephony + * + * g711.c - A-law and u-law transcoding routines + * + * Written by Steve Underwood + * + * Copyright (C) 2006 Steve Underwood + * + * Despite my general liking of the GPL, I place this code in the + * public domain for the benefit of all mankind - even the slimy + * ones who might try to proprietize my work and use it to my + * detriment. + * + * $Id: g711.c,v 1.1 2006/06/07 15:46:39 steveu Exp $ + * + * Modifications for WebRtc, 2011/04/28, by tlegrand: + * -Removed unused include files + * -Changed to use WebRtc types + * -Added option to run encoder bitexact with ITU-T reference implementation + */ + +#include "g711.h" +#include "typedefs.h" // NOLINT(build/include) + +/* Copied from the CCITT G.711 specification */ +static const uint8_t ulaw_to_alaw_table[256] = { + 42, 43, 40, 41, 46, 47, 44, 45, 34, 35, 32, 33, 38, 39, 36, + 37, 58, 59, 56, 57, 62, 63, 60, 61, 50, 51, 48, 49, 54, 55, + 52, 53, 10, 11, 8, 9, 14, 15, 12, 13, 2, 3, 0, 1, 6, + 7, 4, 26, 27, 24, 25, 30, 31, 28, 29, 18, 19, 16, 17, 22, + 23, 20, 21, 106, 104, 105, 110, 111, 108, 109, 98, 99, 96, 97, 102, + 103, 100, 101, 122, 120, 126, 127, 124, 125, 114, 115, 112, 113, 118, 119, + 116, 117, 75, 73, 79, 77, 66, 67, 64, 65, 70, 71, 68, 69, 90, + 91, 88, 89, 94, 95, 92, 93, 82, 82, 83, 83, 80, 80, 81, 81, + 86, 86, 87, 87, 84, 84, 85, 85, 170, 171, 168, 169, 174, 175, 172, + 173, 162, 163, 160, 161, 166, 167, 164, 165, 186, 187, 184, 185, 190, 191, + 188, 189, 178, 179, 176, 177, 182, 183, 180, 181, 138, 139, 136, 137, 142, + 143, 140, 141, 130, 131, 128, 129, 134, 135, 132, 154, 155, 152, 153, 158, + 159, 156, 157, 146, 147, 144, 145, 150, 151, 148, 149, 234, 232, 233, 238, + 239, 236, 237, 226, 227, 224, 225, 230, 231, 228, 229, 250, 248, 254, 255, + 252, 253, 242, 243, 240, 241, 246, 247, 244, 245, 203, 201, 207, 205, 194, + 195, 192, 193, 198, 199, 196, 197, 218, 219, 216, 217, 222, 223, 220, 221, + 210, 210, 211, 211, 208, 208, 209, 209, 214, 214, 215, 215, 212, 212, 213, + 213 +}; + +/* These transcoding tables are copied from the CCITT G.711 specification. To + achieve optimal results, do not change them. */ +static const uint8_t alaw_to_ulaw_table[256] = { + 42, 43, 40, 41, 46, 47, 44, 45, 34, 35, 32, 33, 38, 39, 36, + 37, 57, 58, 55, 56, 61, 62, 59, 60, 49, 50, 47, 48, 53, 54, + 51, 52, 10, 11, 8, 9, 14, 15, 12, 13, 2, 3, 0, 1, 6, + 7, 4, 5, 26, 27, 24, 25, 30, 31, 28, 29, 18, 19, 16, 17, + 22, 23, 20, 21, 98, 99, 96, 97, 102, 103, 100, 101, 93, 93, 92, + 92, 95, 95, 94, 94, 116, 118, 112, 114, 124, 126, 120, 122, 106, 107, + 104, 105, 110, 111, 108, 109, 72, 73, 70, 71, 76, 77, 74, 75, 64, + 65, 63, 63, 68, 69, 66, 67, 86, 87, 84, 85, 90, 91, 88, 89, + 79, 79, 78, 78, 82, 83, 80, 81, 170, 171, 168, 169, 174, 175, 172, + 173, 162, 163, 160, 161, 166, 167, 164, 165, 185, 186, 183, 184, 189, 190, + 187, 188, 177, 178, 175, 176, 181, 182, 179, 180, 138, 139, 136, 137, 142, + 143, 140, 141, 130, 131, 128, 129, 134, 135, 132, 133, 154, 155, 152, 153, + 158, 159, 156, 157, 146, 147, 144, 145, 150, 151, 148, 149, 226, 227, 224, + 225, 230, 231, 228, 229, 221, 221, 220, 220, 223, 223, 222, 222, 244, 246, + 240, 242, 252, 254, 248, 250, 234, 235, 232, 233, 238, 239, 236, 237, 200, + 201, 198, 199, 204, 205, 202, 203, 192, 193, 191, 191, 196, 197, 194, 195, + 214, 215, 212, 213, 218, 219, 216, 217, 207, 207, 206, 206, 210, 211, 208, + 209 +}; + +uint8_t alaw_to_ulaw(uint8_t alaw) { return alaw_to_ulaw_table[alaw]; } + +uint8_t ulaw_to_alaw(uint8_t ulaw) { return ulaw_to_alaw_table[ulaw]; } diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.h new file mode 100644 index 0000000000..8b1fc81ad1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.h @@ -0,0 +1,344 @@ +/* + * SpanDSP - a series of DSP components for telephony + * + * g711.h - In line A-law and u-law conversion routines + * + * Written by Steve Underwood + * + * Copyright (C) 2001 Steve Underwood + * + * Despite my general liking of the GPL, I place this code in the + * public domain for the benefit of all mankind - even the slimy + * ones who might try to proprietize my work and use it to my + * detriment. + * + * $Id: g711.h,v 1.1 2006/06/07 15:46:39 steveu Exp $ + * + * Modifications for WebRtc, 2011/04/28, by tlegrand: + * -Changed to use WebRtc types + * -Changed __inline__ to __inline + * -Two changes to make implementation bitexact with ITU-T reference implementation + */ + +/*! \page g711_page A-law and mu-law handling +Lookup tables for A-law and u-law look attractive, until you consider the impact +on the CPU cache. If it causes a substantial area of your processor cache to get +hit too often, cache sloshing will severely slow things down. The main reason +these routines are slow in C, is the lack of direct access to the CPU's "find +the first 1" instruction. A little in-line assembler fixes that, and the +conversion routines can be faster than lookup tables, in most real world usage. +A "find the first 1" instruction is available on most modern CPUs, and is a +much underused feature. + +If an assembly language method of bit searching is not available, these routines +revert to a method that can be a little slow, so the cache thrashing might not +seem so bad :( + +Feel free to submit patches to add fast "find the first 1" support for your own +favourite processor. + +Look up tables are used for transcoding between A-law and u-law, since it is +difficult to achieve the precise transcoding procedure laid down in the G.711 +specification by other means. +*/ + +#if !defined(_G711_H_) +#define _G711_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "typedefs.h" // NOLINT(build/include) + +#if defined(__i386__) +/*! \brief Find the bit position of the highest set bit in a word + \param bits The word to be searched + \return The bit number of the highest set bit, or -1 if the word is zero. */ +static __inline__ int top_bit(unsigned int bits) { + int res; + + __asm__ __volatile__(" movl $-1,%%edx;\n" + " bsrl %%eax,%%edx;\n" + : "=d" (res) + : "a" (bits)); + return res; +} + +/*! \brief Find the bit position of the lowest set bit in a word + \param bits The word to be searched + \return The bit number of the lowest set bit, or -1 if the word is zero. */ +static __inline__ int bottom_bit(unsigned int bits) { + int res; + + __asm__ __volatile__(" movl $-1,%%edx;\n" + " bsfl %%eax,%%edx;\n" + : "=d" (res) + : "a" (bits)); + return res; +} +#elif defined(__x86_64__) +static __inline__ int top_bit(unsigned int bits) { + int res; + + __asm__ __volatile__(" movq $-1,%%rdx;\n" + " bsrq %%rax,%%rdx;\n" + : "=d" (res) + : "a" (bits)); + return res; +} + +static __inline__ int bottom_bit(unsigned int bits) { + int res; + + __asm__ __volatile__(" movq $-1,%%rdx;\n" + " bsfq %%rax,%%rdx;\n" + : "=d" (res) + : "a" (bits)); + return res; +} +#else +static __inline int top_bit(unsigned int bits) { + int i; + + if (bits == 0) { + return -1; + } + i = 0; + if (bits & 0xFFFF0000) { + bits &= 0xFFFF0000; + i += 16; + } + if (bits & 0xFF00FF00) { + bits &= 0xFF00FF00; + i += 8; + } + if (bits & 0xF0F0F0F0) { + bits &= 0xF0F0F0F0; + i += 4; + } + if (bits & 0xCCCCCCCC) { + bits &= 0xCCCCCCCC; + i += 2; + } + if (bits & 0xAAAAAAAA) { + bits &= 0xAAAAAAAA; + i += 1; + } + return i; +} + +static __inline int bottom_bit(unsigned int bits) { + int i; + + if (bits == 0) { + return -1; + } + i = 32; + if (bits & 0x0000FFFF) { + bits &= 0x0000FFFF; + i -= 16; + } + if (bits & 0x00FF00FF) { + bits &= 0x00FF00FF; + i -= 8; + } + if (bits & 0x0F0F0F0F) { + bits &= 0x0F0F0F0F; + i -= 4; + } + if (bits & 0x33333333) { + bits &= 0x33333333; + i -= 2; + } + if (bits & 0x55555555) { + bits &= 0x55555555; + i -= 1; + } + return i; +} +#endif + +/* N.B. It is tempting to use look-up tables for A-law and u-law conversion. + * However, you should consider the cache footprint. + * + * A 64K byte table for linear to x-law and a 512 byte table for x-law to + * linear sound like peanuts these days, and shouldn't an array lookup be + * real fast? No! When the cache sloshes as badly as this one will, a tight + * calculation may be better. The messiest part is normally finding the + * segment, but a little inline assembly can fix that on an i386, x86_64 and + * many other modern processors. + */ + +/* + * Mu-law is basically as follows: + * + * Biased Linear Input Code Compressed Code + * ------------------------ --------------- + * 00000001wxyza 000wxyz + * 0000001wxyzab 001wxyz + * 000001wxyzabc 010wxyz + * 00001wxyzabcd 011wxyz + * 0001wxyzabcde 100wxyz + * 001wxyzabcdef 101wxyz + * 01wxyzabcdefg 110wxyz + * 1wxyzabcdefgh 111wxyz + * + * Each biased linear code has a leading 1 which identifies the segment + * number. The value of the segment number is equal to 7 minus the number + * of leading 0's. The quantization interval is directly available as the + * four bits wxyz. * The trailing bits (a - h) are ignored. + * + * Ordinarily the complement of the resulting code word is used for + * transmission, and so the code word is complemented before it is returned. + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ + +//#define ULAW_ZEROTRAP /* turn on the trap as per the MIL-STD */ +#define ULAW_BIAS 0x84 /* Bias for linear code. */ + +/*! \brief Encode a linear sample to u-law + \param linear The sample to encode. + \return The u-law value. +*/ +static __inline uint8_t linear_to_ulaw(int linear) { + uint8_t u_val; + int mask; + int seg; + + /* Get the sign and the magnitude of the value. */ + if (linear < 0) { + /* WebRtc, tlegrand: -1 added to get bitexact to reference implementation */ + linear = ULAW_BIAS - linear - 1; + mask = 0x7F; + } else { + linear = ULAW_BIAS + linear; + mask = 0xFF; + } + + seg = top_bit(linear | 0xFF) - 7; + + /* + * Combine the sign, segment, quantization bits, + * and complement the code word. + */ + if (seg >= 8) + u_val = (uint8_t)(0x7F ^ mask); + else + u_val = (uint8_t)(((seg << 4) | ((linear >> (seg + 3)) & 0xF)) ^ mask); +#ifdef ULAW_ZEROTRAP + /* Optional ITU trap */ + if (u_val == 0) + u_val = 0x02; +#endif + return u_val; +} + +/*! \brief Decode an u-law sample to a linear value. + \param ulaw The u-law sample to decode. + \return The linear value. +*/ +static __inline int16_t ulaw_to_linear(uint8_t ulaw) { + int t; + + /* Complement to obtain normal u-law value. */ + ulaw = ~ulaw; + /* + * Extract and bias the quantization bits. Then + * shift up by the segment number and subtract out the bias. + */ + t = (((ulaw & 0x0F) << 3) + ULAW_BIAS) << (((int) ulaw & 0x70) >> 4); + return (int16_t)((ulaw & 0x80) ? (ULAW_BIAS - t) : (t - ULAW_BIAS)); +} + +/* + * A-law is basically as follows: + * + * Linear Input Code Compressed Code + * ----------------- --------------- + * 0000000wxyza 000wxyz + * 0000001wxyza 001wxyz + * 000001wxyzab 010wxyz + * 00001wxyzabc 011wxyz + * 0001wxyzabcd 100wxyz + * 001wxyzabcde 101wxyz + * 01wxyzabcdef 110wxyz + * 1wxyzabcdefg 111wxyz + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ + +#define ALAW_AMI_MASK 0x55 + +/*! \brief Encode a linear sample to A-law + \param linear The sample to encode. + \return The A-law value. +*/ +static __inline uint8_t linear_to_alaw(int linear) { + int mask; + int seg; + + if (linear >= 0) { + /* Sign (bit 7) bit = 1 */ + mask = ALAW_AMI_MASK | 0x80; + } else { + /* Sign (bit 7) bit = 0 */ + mask = ALAW_AMI_MASK; + /* WebRtc, tlegrand: Changed from -8 to -1 to get bitexact to reference + * implementation */ + linear = -linear - 1; + } + + /* Convert the scaled magnitude to segment number. */ + seg = top_bit(linear | 0xFF) - 7; + if (seg >= 8) { + if (linear >= 0) { + /* Out of range. Return maximum value. */ + return (uint8_t)(0x7F ^ mask); + } + /* We must be just a tiny step below zero */ + return (uint8_t)(0x00 ^ mask); + } + /* Combine the sign, segment, and quantization bits. */ + return (uint8_t)(((seg << 4) | ((linear >> ((seg) ? (seg + 3) : 4)) & 0x0F)) ^ + mask); +} + +/*! \brief Decode an A-law sample to a linear value. + \param alaw The A-law sample to decode. + \return The linear value. +*/ +static __inline int16_t alaw_to_linear(uint8_t alaw) { + int i; + int seg; + + alaw ^= ALAW_AMI_MASK; + i = ((alaw & 0x0F) << 4); + seg = (((int) alaw & 0x70) >> 4); + if (seg) + i = (i + 0x108) << (seg - 1); + else + i += 8; + return (int16_t)((alaw & 0x80) ? i : -i); +} + +/*! \brief Transcode from A-law to u-law, using the procedure defined in G.711. + \param alaw The A-law sample to transcode. + \return The best matching u-law value. +*/ +uint8_t alaw_to_ulaw(uint8_t alaw); + +/*! \brief Transcode from u-law to A-law, using the procedure defined in G.711. + \param alaw The u-law sample to transcode. + \return The best matching A-law value. +*/ +uint8_t ulaw_to_alaw(uint8_t ulaw); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.c new file mode 100644 index 0000000000..b5dacd6894 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.c @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include +#include "g711.h" +#include "g711_interface.h" +#include "typedefs.h" // NOLINT(build/include) + +size_t WebRtcG711_EncodeA(const int16_t* speechIn, + size_t len, + uint8_t* encoded) { + size_t n; + for (n = 0; n < len; n++) + encoded[n] = linear_to_alaw(speechIn[n]); + return len; +} + +size_t WebRtcG711_EncodeU(const int16_t* speechIn, + size_t len, + uint8_t* encoded) { + size_t n; + for (n = 0; n < len; n++) + encoded[n] = linear_to_ulaw(speechIn[n]); + return len; +} + +size_t WebRtcG711_DecodeA(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) { + size_t n; + for (n = 0; n < len; n++) + decoded[n] = alaw_to_linear(encoded[n]); + *speechType = 1; + return len; +} + +size_t WebRtcG711_DecodeU(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) { + size_t n; + for (n = 0; n < len; n++) + decoded[n] = ulaw_to_linear(encoded[n]); + *speechType = 1; + return len; +} + +int16_t WebRtcG711_Version(char* version, int16_t lenBytes) { + strncpy(version, "2.0.0", lenBytes); + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.h new file mode 100644 index 0000000000..1f23da62de --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G711_G711_INTERFACE_H_ +#define MODULES_AUDIO_CODING_CODECS_G711_G711_INTERFACE_H_ + +#include "typedefs.h" // NOLINT(build/include) + +// Comfort noise constants +#define G711_WEBRTC_SPEECH 1 +#define G711_WEBRTC_CNG 2 + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * WebRtcG711_EncodeA(...) + * + * This function encodes a G711 A-law frame and inserts it into a packet. + * Input speech length has be of any length. + * + * Input: + * - speechIn : Input speech vector + * - len : Samples in speechIn + * + * Output: + * - encoded : The encoded data vector + * + * Return value : Length (in bytes) of coded data. + * Always equal to len input parameter. + */ + +size_t WebRtcG711_EncodeA(const int16_t* speechIn, + size_t len, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcG711_EncodeU(...) + * + * This function encodes a G711 U-law frame and inserts it into a packet. + * Input speech length has be of any length. + * + * Input: + * - speechIn : Input speech vector + * - len : Samples in speechIn + * + * Output: + * - encoded : The encoded data vector + * + * Return value : Length (in bytes) of coded data. + * Always equal to len input parameter. + */ + +size_t WebRtcG711_EncodeU(const int16_t* speechIn, + size_t len, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcG711_DecodeA(...) + * + * This function decodes a packet G711 A-law frame. + * + * Input: + * - encoded : Encoded data + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG (for G711 it should + * always return 1 since G711 does not have a + * built-in DTX/CNG scheme) + * + * Return value : >0 - Samples in decoded vector + * -1 - Error + */ + +size_t WebRtcG711_DecodeA(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + +/**************************************************************************** + * WebRtcG711_DecodeU(...) + * + * This function decodes a packet G711 U-law frame. + * + * Input: + * - encoded : Encoded data + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG (for G711 it should + * always return 1 since G711 does not have a + * built-in DTX/CNG scheme) + * + * Return value : >0 - Samples in decoded vector + * -1 - Error + */ + +size_t WebRtcG711_DecodeU(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + +/********************************************************************** +* WebRtcG711_Version(...) +* +* This function gives the version string of the G.711 codec. +* +* Input: +* - lenBytes: the size of Allocated space (in Bytes) where +* the version number is written to (in string format). +* +* Output: +* - version: Pointer to a buffer where the version number is +* written to. +* +*/ + +int16_t WebRtcG711_Version(char* version, int16_t lenBytes); + +#ifdef __cplusplus +} +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_G711_G711_INTERFACE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/test/testG711.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/test/testG711.cc new file mode 100644 index 0000000000..98f3925faa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/test/testG711.cc @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * testG711.cpp : Defines the entry point for the console application. + */ + +#include +#include +#include + +/* include API */ +#include "modules/audio_coding/codecs/g711/g711_interface.h" + +/* Runtime statistics */ +#include +#define CLOCKS_PER_SEC_G711 1000 + +/* function for reading audio data from PCM file */ +bool readframe(int16_t* data, FILE* inp, size_t length) { + size_t rlen = fread(data, sizeof(int16_t), length, inp); + if (rlen >= length) + return false; + memset(data + rlen, 0, (length - rlen) * sizeof(int16_t)); + return true; +} + +int main(int argc, char* argv[]) { + char inname[80], outname[40], bitname[40]; + FILE* inp; + FILE* outp; + FILE* bitp = NULL; + int framecnt; + bool endfile; + + size_t framelength = 80; + + /* Runtime statistics */ + double starttime; + double runtime; + double length_file; + + size_t stream_len = 0; + int16_t shortdata[480]; + int16_t decoded[480]; + uint8_t streamdata[1000]; + int16_t speechType[1]; + char law[2]; + char versionNumber[40]; + + /* handling wrong input arguments in the command line */ + if ((argc != 5) && (argc != 6)) { + printf("\n\nWrong number of arguments or flag values.\n\n"); + + printf("\n"); + printf("\nG.711 test application\n\n"); + printf("Usage:\n\n"); + printf("./testG711.exe framelength law infile outfile \n\n"); + printf("framelength: Framelength in samples.\n"); + printf("law : Coding law, A och u.\n"); + printf("infile : Normal speech input file\n"); + printf("outfile : Speech output file\n\n"); + printf("outbits : Output bitstream file [optional]\n\n"); + exit(0); + + } + + /* Get version and print */ + WebRtcG711_Version(versionNumber, 40); + + printf("-----------------------------------\n"); + printf("G.711 version: %s\n\n", versionNumber); + /* Get frame length */ + int framelength_int = atoi(argv[1]); + if (framelength_int < 0) { + printf(" G.722: Invalid framelength %d.\n", framelength_int); + exit(1); + } + framelength = static_cast(framelength_int); + + /* Get compression law */ + strcpy(law, argv[2]); + + /* Get Input and Output files */ + sscanf(argv[3], "%s", inname); + sscanf(argv[4], "%s", outname); + if (argc == 6) { + sscanf(argv[5], "%s", bitname); + if ((bitp = fopen(bitname, "wb")) == NULL) { + printf(" G.711: Cannot read file %s.\n", bitname); + exit(1); + } + } + + if ((inp = fopen(inname, "rb")) == NULL) { + printf(" G.711: Cannot read file %s.\n", inname); + exit(1); + } + if ((outp = fopen(outname, "wb")) == NULL) { + printf(" G.711: Cannot write file %s.\n", outname); + exit(1); + } + printf("\nInput: %s\nOutput: %s\n", inname, outname); + if (argc == 6) { + printf("\nBitfile: %s\n", bitname); + } + + starttime = clock() / (double) CLOCKS_PER_SEC_G711; /* Runtime statistics */ + + /* Initialize encoder and decoder */ + framecnt = 0; + endfile = false; + while (!endfile) { + framecnt++; + /* Read speech block */ + endfile = readframe(shortdata, inp, framelength); + + /* G.711 encoding */ + if (!strcmp(law, "A")) { + /* A-law encoding */ + stream_len = WebRtcG711_EncodeA(shortdata, framelength, streamdata); + if (argc == 6) { + /* Write bits to file */ + if (fwrite(streamdata, sizeof(unsigned char), stream_len, bitp) != + stream_len) { + return -1; + } + } + WebRtcG711_DecodeA(streamdata, stream_len, decoded, speechType); + } else if (!strcmp(law, "u")) { + /* u-law encoding */ + stream_len = WebRtcG711_EncodeU(shortdata, framelength, streamdata); + if (argc == 6) { + /* Write bits to file */ + if (fwrite(streamdata, sizeof(unsigned char), stream_len, bitp) != + stream_len) { + return -1; + } + } + WebRtcG711_DecodeU(streamdata, stream_len, decoded, speechType); + } else { + printf("Wrong law mode\n"); + exit(1); + } + /* Write coded speech to file */ + if (fwrite(decoded, sizeof(short), framelength, outp) != framelength) { + return -1; + } + } + + runtime = (double)(clock() / (double) CLOCKS_PER_SEC_G711 - starttime); + length_file = ((double) framecnt * (double) framelength / 8000); + printf("\n\nLength of speech file: %.1f s\n", length_file); + printf("Time to run G.711: %.2f s (%.2f %% of realtime)\n\n", + runtime, + (100 * runtime / length_file)); + printf("---------------------END----------------------\n"); + + fclose(inp); + fclose(outp); + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc new file mode 100644 index 0000000000..ea4a721b12 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g722/audio_decoder_g722.h" + +#include + +#include "modules/audio_coding/codecs/g722/g722_interface.h" +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioDecoderG722Impl::AudioDecoderG722Impl() { + WebRtcG722_CreateDecoder(&dec_state_); + WebRtcG722_DecoderInit(dec_state_); +} + +AudioDecoderG722Impl::~AudioDecoderG722Impl() { + WebRtcG722_FreeDecoder(dec_state_); +} + +bool AudioDecoderG722Impl::HasDecodePlc() const { + return false; +} + +int AudioDecoderG722Impl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + int16_t temp_type = 1; // Default is speech. + size_t ret = + WebRtcG722_Decode(dec_state_, encoded, encoded_len, decoded, &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return static_cast(ret); +} + +void AudioDecoderG722Impl::Reset() { + WebRtcG722_DecoderInit(dec_state_); +} + +std::vector AudioDecoderG722Impl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), + timestamp, 8, 16); +} + +int AudioDecoderG722Impl::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // 1/2 encoded byte per sample per channel. + return static_cast(2 * encoded_len / Channels()); +} + +int AudioDecoderG722Impl::SampleRateHz() const { + return 16000; +} + +size_t AudioDecoderG722Impl::Channels() const { + return 1; +} + +AudioDecoderG722StereoImpl::AudioDecoderG722StereoImpl() { + WebRtcG722_CreateDecoder(&dec_state_left_); + WebRtcG722_CreateDecoder(&dec_state_right_); + WebRtcG722_DecoderInit(dec_state_left_); + WebRtcG722_DecoderInit(dec_state_right_); +} + +AudioDecoderG722StereoImpl::~AudioDecoderG722StereoImpl() { + WebRtcG722_FreeDecoder(dec_state_left_); + WebRtcG722_FreeDecoder(dec_state_right_); +} + +int AudioDecoderG722StereoImpl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + int16_t temp_type = 1; // Default is speech. + // De-interleave the bit-stream into two separate payloads. + uint8_t* encoded_deinterleaved = new uint8_t[encoded_len]; + SplitStereoPacket(encoded, encoded_len, encoded_deinterleaved); + // Decode left and right. + size_t decoded_len = WebRtcG722_Decode(dec_state_left_, encoded_deinterleaved, + encoded_len / 2, decoded, &temp_type); + size_t ret = WebRtcG722_Decode( + dec_state_right_, &encoded_deinterleaved[encoded_len / 2], + encoded_len / 2, &decoded[decoded_len], &temp_type); + if (ret == decoded_len) { + ret += decoded_len; // Return total number of samples. + // Interleave output. + for (size_t k = ret / 2; k < ret; k++) { + int16_t temp = decoded[k]; + memmove(&decoded[2 * k - ret + 2], &decoded[2 * k - ret + 1], + (ret - k - 1) * sizeof(int16_t)); + decoded[2 * k - ret + 1] = temp; + } + } + *speech_type = ConvertSpeechType(temp_type); + delete[] encoded_deinterleaved; + return static_cast(ret); +} + +int AudioDecoderG722StereoImpl::SampleRateHz() const { + return 16000; +} + +size_t AudioDecoderG722StereoImpl::Channels() const { + return 2; +} + +void AudioDecoderG722StereoImpl::Reset() { + WebRtcG722_DecoderInit(dec_state_left_); + WebRtcG722_DecoderInit(dec_state_right_); +} + +std::vector AudioDecoderG722StereoImpl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), + timestamp, 2 * 8, 16); +} + +// Split the stereo packet and place left and right channel after each other +// in the output array. +void AudioDecoderG722StereoImpl::SplitStereoPacket( + const uint8_t* encoded, + size_t encoded_len, + uint8_t* encoded_deinterleaved) { + // Regroup the 4 bits/sample so |l1 l2| |r1 r2| |l3 l4| |r3 r4| ..., + // where "lx" is 4 bits representing left sample number x, and "rx" right + // sample. Two samples fit in one byte, represented with |...|. + for (size_t i = 0; i + 1 < encoded_len; i += 2) { + uint8_t right_byte = ((encoded[i] & 0x0F) << 4) + (encoded[i + 1] & 0x0F); + encoded_deinterleaved[i] = (encoded[i] & 0xF0) + (encoded[i + 1] >> 4); + encoded_deinterleaved[i + 1] = right_byte; + } + + // Move one byte representing right channel each loop, and place it at the + // end of the bytestream vector. After looping the data is reordered to: + // |l1 l2| |l3 l4| ... |l(N-1) lN| |r1 r2| |r3 r4| ... |r(N-1) r(N)|, + // where N is the total number of samples. + for (size_t i = 0; i < encoded_len / 2; i++) { + uint8_t right_byte = encoded_deinterleaved[i + 1]; + memmove(&encoded_deinterleaved[i + 1], &encoded_deinterleaved[i + 2], + encoded_len - i - 2); + encoded_deinterleaved[encoded_len - 1] = right_byte; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h new file mode 100644 index 0000000000..324044886d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G722_AUDIO_DECODER_G722_H_ +#define MODULES_AUDIO_CODING_CODECS_G722_AUDIO_DECODER_G722_H_ + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/constructormagic.h" + +typedef struct WebRtcG722DecInst G722DecInst; + +namespace webrtc { + +class AudioDecoderG722Impl final : public AudioDecoder { + public: + AudioDecoderG722Impl(); + ~AudioDecoderG722Impl() override; + bool HasDecodePlc() const override; + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + G722DecInst* dec_state_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderG722Impl); +}; + +class AudioDecoderG722StereoImpl final : public AudioDecoder { + public: + AudioDecoderG722StereoImpl(); + ~AudioDecoderG722StereoImpl() override; + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + // Splits the stereo-interleaved payload in |encoded| into separate payloads + // for left and right channels. The separated payloads are written to + // |encoded_deinterleaved|, which must hold at least |encoded_len| samples. + // The left channel starts at offset 0, while the right channel starts at + // offset encoded_len / 2 into |encoded_deinterleaved|. + void SplitStereoPacket(const uint8_t* encoded, + size_t encoded_len, + uint8_t* encoded_deinterleaved); + + G722DecInst* dec_state_left_; + G722DecInst* dec_state_right_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderG722StereoImpl); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_G722_AUDIO_DECODER_G722_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc new file mode 100644 index 0000000000..ec97ee3321 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g722/audio_encoder_g722.h" + +#include + +#include +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/g722/g722_interface.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { + +const size_t kSampleRateHz = 16000; + +AudioEncoderG722Config CreateConfig(const CodecInst& codec_inst) { + AudioEncoderG722Config config; + config.num_channels = rtc::dchecked_cast(codec_inst.channels); + config.frame_size_ms = codec_inst.pacsize / 16; + return config; +} + +} // namespace + +AudioEncoderG722Impl::AudioEncoderG722Impl(const AudioEncoderG722Config& config, + int payload_type) + : num_channels_(config.num_channels), + payload_type_(payload_type), + num_10ms_frames_per_packet_( + static_cast(config.frame_size_ms / 10)), + num_10ms_frames_buffered_(0), + first_timestamp_in_buffer_(0), + encoders_(new EncoderState[num_channels_]), + interleave_buffer_(2 * num_channels_) { + RTC_CHECK(config.IsOk()); + const size_t samples_per_channel = + kSampleRateHz / 100 * num_10ms_frames_per_packet_; + for (size_t i = 0; i < num_channels_; ++i) { + encoders_[i].speech_buffer.reset(new int16_t[samples_per_channel]); + encoders_[i].encoded_buffer.SetSize(samples_per_channel / 2); + } + Reset(); +} + +AudioEncoderG722Impl::AudioEncoderG722Impl(const CodecInst& codec_inst) + : AudioEncoderG722Impl(CreateConfig(codec_inst), codec_inst.pltype) {} + +AudioEncoderG722Impl::~AudioEncoderG722Impl() = default; + +int AudioEncoderG722Impl::SampleRateHz() const { + return kSampleRateHz; +} + +size_t AudioEncoderG722Impl::NumChannels() const { + return num_channels_; +} + +int AudioEncoderG722Impl::RtpTimestampRateHz() const { + // The RTP timestamp rate for G.722 is 8000 Hz, even though it is a 16 kHz + // codec. + return kSampleRateHz / 2; +} + +size_t AudioEncoderG722Impl::Num10MsFramesInNextPacket() const { + return num_10ms_frames_per_packet_; +} + +size_t AudioEncoderG722Impl::Max10MsFramesInAPacket() const { + return num_10ms_frames_per_packet_; +} + +int AudioEncoderG722Impl::GetTargetBitrate() const { + // 4 bits/sample, 16000 samples/s/channel. + return static_cast(64000 * NumChannels()); +} + +void AudioEncoderG722Impl::Reset() { + num_10ms_frames_buffered_ = 0; + for (size_t i = 0; i < num_channels_; ++i) + RTC_CHECK_EQ(0, WebRtcG722_EncoderInit(encoders_[i].encoder)); +} + +AudioEncoder::EncodedInfo AudioEncoderG722Impl::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + if (num_10ms_frames_buffered_ == 0) + first_timestamp_in_buffer_ = rtp_timestamp; + + // Deinterleave samples and save them in each channel's buffer. + const size_t start = kSampleRateHz / 100 * num_10ms_frames_buffered_; + for (size_t i = 0; i < kSampleRateHz / 100; ++i) + for (size_t j = 0; j < num_channels_; ++j) + encoders_[j].speech_buffer[start + i] = audio[i * num_channels_ + j]; + + // If we don't yet have enough samples for a packet, we're done for now. + if (++num_10ms_frames_buffered_ < num_10ms_frames_per_packet_) { + return EncodedInfo(); + } + + // Encode each channel separately. + RTC_CHECK_EQ(num_10ms_frames_buffered_, num_10ms_frames_per_packet_); + num_10ms_frames_buffered_ = 0; + const size_t samples_per_channel = SamplesPerChannel(); + for (size_t i = 0; i < num_channels_; ++i) { + const size_t bytes_encoded = WebRtcG722_Encode( + encoders_[i].encoder, encoders_[i].speech_buffer.get(), + samples_per_channel, encoders_[i].encoded_buffer.data()); + RTC_CHECK_EQ(bytes_encoded, samples_per_channel / 2); + } + + const size_t bytes_to_encode = samples_per_channel / 2 * num_channels_; + EncodedInfo info; + info.encoded_bytes = encoded->AppendData( + bytes_to_encode, [&] (rtc::ArrayView encoded) { + // Interleave the encoded bytes of the different channels. Each separate + // channel and the interleaved stream encodes two samples per byte, most + // significant half first. + for (size_t i = 0; i < samples_per_channel / 2; ++i) { + for (size_t j = 0; j < num_channels_; ++j) { + uint8_t two_samples = encoders_[j].encoded_buffer.data()[i]; + interleave_buffer_.data()[j] = two_samples >> 4; + interleave_buffer_.data()[num_channels_ + j] = two_samples & 0xf; + } + for (size_t j = 0; j < num_channels_; ++j) + encoded[i * num_channels_ + j] = + interleave_buffer_.data()[2 * j] << 4 | + interleave_buffer_.data()[2 * j + 1]; + } + + return bytes_to_encode; + }); + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.encoder_type = CodecType::kG722; + return info; +} + +AudioEncoderG722Impl::EncoderState::EncoderState() { + RTC_CHECK_EQ(0, WebRtcG722_CreateEncoder(&encoder)); +} + +AudioEncoderG722Impl::EncoderState::~EncoderState() { + RTC_CHECK_EQ(0, WebRtcG722_FreeEncoder(encoder)); +} + +size_t AudioEncoderG722Impl::SamplesPerChannel() const { + return kSampleRateHz / 100 * num_10ms_frames_per_packet_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h new file mode 100644 index 0000000000..1f4b943f66 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G722_AUDIO_ENCODER_G722_H_ +#define MODULES_AUDIO_CODING_CODECS_G722_AUDIO_ENCODER_G722_H_ + +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/g722/audio_encoder_g722_config.h" +#include "modules/audio_coding/codecs/g722/g722_interface.h" +#include "rtc_base/buffer.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +struct CodecInst; + +class AudioEncoderG722Impl final : public AudioEncoder { + public: + AudioEncoderG722Impl(const AudioEncoderG722Config& config, int payload_type); + explicit AudioEncoderG722Impl(const CodecInst& codec_inst); + ~AudioEncoderG722Impl() override; + + int SampleRateHz() const override; + size_t NumChannels() const override; + int RtpTimestampRateHz() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + void Reset() override; + + protected: + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + private: + // The encoder state for one channel. + struct EncoderState { + G722EncInst* encoder; + std::unique_ptr speech_buffer; // Queued up for encoding. + rtc::Buffer encoded_buffer; // Already encoded. + EncoderState(); + ~EncoderState(); + }; + + size_t SamplesPerChannel() const; + + const size_t num_channels_; + const int payload_type_; + const size_t num_10ms_frames_per_packet_; + size_t num_10ms_frames_buffered_; + uint32_t first_timestamp_in_buffer_; + const std::unique_ptr encoders_; + rtc::Buffer interleave_buffer_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderG722Impl); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_G722_AUDIO_ENCODER_G722_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_decode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_decode.c new file mode 100644 index 0000000000..84af155097 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_decode.c @@ -0,0 +1,400 @@ +/* + * SpanDSP - a series of DSP components for telephony + * + * g722_decode.c - The ITU G.722 codec, decode part. + * + * Written by Steve Underwood + * + * Copyright (C) 2005 Steve Underwood + * + * Despite my general liking of the GPL, I place my own contributions + * to this code in the public domain for the benefit of all mankind - + * even the slimy ones who might try to proprietize my work and use it + * to my detriment. + * + * Based in part on a single channel G.722 codec which is: + * + * Copyright (c) CMU 1993 + * Computer Science, Speech Group + * Chengxiang Lu and Alex Hauptmann + * + * $Id: g722_decode.c,v 1.15 2006/07/07 16:37:49 steveu Exp $ + * + * Modifications for WebRtc, 2011/04/28, by tlegrand: + * -Removed usage of inttypes.h and tgmath.h + * -Changed to use WebRtc types + * -Changed __inline__ to __inline + * -Added saturation check on output + */ + +/*! \file */ + + +#include +#include +#include + +#include "g722_enc_dec.h" +#include "typedefs.h" // NOLINT(build/include) + +#if !defined(FALSE) +#define FALSE 0 +#endif +#if !defined(TRUE) +#define TRUE (!FALSE) +#endif + +static __inline int16_t saturate(int32_t amp) +{ + int16_t amp16; + + /* Hopefully this is optimised for the common case - not clipping */ + amp16 = (int16_t) amp; + if (amp == amp16) + return amp16; + if (amp > WEBRTC_INT16_MAX) + return WEBRTC_INT16_MAX; + return WEBRTC_INT16_MIN; +} +/*- End of function --------------------------------------------------------*/ + +static void block4(G722DecoderState *s, int band, int d); + +static void block4(G722DecoderState *s, int band, int d) +{ + int wd1; + int wd2; + int wd3; + int i; + + /* Block 4, RECONS */ + s->band[band].d[0] = d; + s->band[band].r[0] = saturate(s->band[band].s + d); + + /* Block 4, PARREC */ + s->band[band].p[0] = saturate(s->band[band].sz + d); + + /* Block 4, UPPOL2 */ + for (i = 0; i < 3; i++) + s->band[band].sg[i] = s->band[band].p[i] >> 15; + wd1 = saturate(s->band[band].a[1] * 4); + + wd2 = (s->band[band].sg[0] == s->band[band].sg[1]) ? -wd1 : wd1; + if (wd2 > 32767) + wd2 = 32767; + wd3 = (s->band[band].sg[0] == s->band[band].sg[2]) ? 128 : -128; + wd3 += (wd2 >> 7); + wd3 += (s->band[band].a[2]*32512) >> 15; + if (wd3 > 12288) + wd3 = 12288; + else if (wd3 < -12288) + wd3 = -12288; + s->band[band].ap[2] = wd3; + + /* Block 4, UPPOL1 */ + s->band[band].sg[0] = s->band[band].p[0] >> 15; + s->band[band].sg[1] = s->band[band].p[1] >> 15; + wd1 = (s->band[band].sg[0] == s->band[band].sg[1]) ? 192 : -192; + wd2 = (s->band[band].a[1]*32640) >> 15; + + s->band[band].ap[1] = saturate(wd1 + wd2); + wd3 = saturate(15360 - s->band[band].ap[2]); + if (s->band[band].ap[1] > wd3) + s->band[band].ap[1] = wd3; + else if (s->band[band].ap[1] < -wd3) + s->band[band].ap[1] = -wd3; + + /* Block 4, UPZERO */ + wd1 = (d == 0) ? 0 : 128; + s->band[band].sg[0] = d >> 15; + for (i = 1; i < 7; i++) + { + s->band[band].sg[i] = s->band[band].d[i] >> 15; + wd2 = (s->band[band].sg[i] == s->band[band].sg[0]) ? wd1 : -wd1; + wd3 = (s->band[band].b[i]*32640) >> 15; + s->band[band].bp[i] = saturate(wd2 + wd3); + } + + /* Block 4, DELAYA */ + for (i = 6; i > 0; i--) + { + s->band[band].d[i] = s->band[band].d[i - 1]; + s->band[band].b[i] = s->band[band].bp[i]; + } + + for (i = 2; i > 0; i--) + { + s->band[band].r[i] = s->band[band].r[i - 1]; + s->band[band].p[i] = s->band[band].p[i - 1]; + s->band[band].a[i] = s->band[band].ap[i]; + } + + /* Block 4, FILTEP */ + wd1 = saturate(s->band[band].r[1] + s->band[band].r[1]); + wd1 = (s->band[band].a[1]*wd1) >> 15; + wd2 = saturate(s->band[band].r[2] + s->band[band].r[2]); + wd2 = (s->band[band].a[2]*wd2) >> 15; + s->band[band].sp = saturate(wd1 + wd2); + + /* Block 4, FILTEZ */ + s->band[band].sz = 0; + for (i = 6; i > 0; i--) + { + wd1 = saturate(s->band[band].d[i] + s->band[band].d[i]); + s->band[band].sz += (s->band[band].b[i]*wd1) >> 15; + } + s->band[band].sz = saturate(s->band[band].sz); + + /* Block 4, PREDIC */ + s->band[band].s = saturate(s->band[band].sp + s->band[band].sz); +} +/*- End of function --------------------------------------------------------*/ + +G722DecoderState* WebRtc_g722_decode_init(G722DecoderState* s, + int rate, + int options) { + s = s ? s : malloc(sizeof(*s)); + memset(s, 0, sizeof(*s)); + if (rate == 48000) + s->bits_per_sample = 6; + else if (rate == 56000) + s->bits_per_sample = 7; + else + s->bits_per_sample = 8; + if ((options & G722_SAMPLE_RATE_8000)) + s->eight_k = TRUE; + if ((options & G722_PACKED) && s->bits_per_sample != 8) + s->packed = TRUE; + else + s->packed = FALSE; + s->band[0].det = 32; + s->band[1].det = 8; + return s; +} +/*- End of function --------------------------------------------------------*/ + +int WebRtc_g722_decode_release(G722DecoderState *s) +{ + free(s); + return 0; +} +/*- End of function --------------------------------------------------------*/ + +size_t WebRtc_g722_decode(G722DecoderState *s, int16_t amp[], + const uint8_t g722_data[], size_t len) +{ + static const int wl[8] = {-60, -30, 58, 172, 334, 538, 1198, 3042 }; + static const int rl42[16] = {0, 7, 6, 5, 4, 3, 2, 1, + 7, 6, 5, 4, 3, 2, 1, 0 }; + static const int ilb[32] = + { + 2048, 2093, 2139, 2186, 2233, 2282, 2332, + 2383, 2435, 2489, 2543, 2599, 2656, 2714, + 2774, 2834, 2896, 2960, 3025, 3091, 3158, + 3228, 3298, 3371, 3444, 3520, 3597, 3676, + 3756, 3838, 3922, 4008 + }; + static const int wh[3] = {0, -214, 798}; + static const int rh2[4] = {2, 1, 2, 1}; + static const int qm2[4] = {-7408, -1616, 7408, 1616}; + static const int qm4[16] = + { + 0, -20456, -12896, -8968, + -6288, -4240, -2584, -1200, + 20456, 12896, 8968, 6288, + 4240, 2584, 1200, 0 + }; + static const int qm5[32] = + { + -280, -280, -23352, -17560, + -14120, -11664, -9752, -8184, + -6864, -5712, -4696, -3784, + -2960, -2208, -1520, -880, + 23352, 17560, 14120, 11664, + 9752, 8184, 6864, 5712, + 4696, 3784, 2960, 2208, + 1520, 880, 280, -280 + }; + static const int qm6[64] = + { + -136, -136, -136, -136, + -24808, -21904, -19008, -16704, + -14984, -13512, -12280, -11192, + -10232, -9360, -8576, -7856, + -7192, -6576, -6000, -5456, + -4944, -4464, -4008, -3576, + -3168, -2776, -2400, -2032, + -1688, -1360, -1040, -728, + 24808, 21904, 19008, 16704, + 14984, 13512, 12280, 11192, + 10232, 9360, 8576, 7856, + 7192, 6576, 6000, 5456, + 4944, 4464, 4008, 3576, + 3168, 2776, 2400, 2032, + 1688, 1360, 1040, 728, + 432, 136, -432, -136 + }; + static const int qmf_coeffs[12] = + { + 3, -11, 12, 32, -210, 951, 3876, -805, 362, -156, 53, -11, + }; + + int dlowt; + int rlow; + int ihigh; + int dhigh; + int rhigh; + int xout1; + int xout2; + int wd1; + int wd2; + int wd3; + int code; + size_t outlen; + int i; + size_t j; + + outlen = 0; + rhigh = 0; + for (j = 0; j < len; ) + { + if (s->packed) + { + /* Unpack the code bits */ + if (s->in_bits < s->bits_per_sample) + { + s->in_buffer |= (g722_data[j++] << s->in_bits); + s->in_bits += 8; + } + code = s->in_buffer & ((1 << s->bits_per_sample) - 1); + s->in_buffer >>= s->bits_per_sample; + s->in_bits -= s->bits_per_sample; + } + else + { + code = g722_data[j++]; + } + + switch (s->bits_per_sample) + { + default: + case 8: + wd1 = code & 0x3F; + ihigh = (code >> 6) & 0x03; + wd2 = qm6[wd1]; + wd1 >>= 2; + break; + case 7: + wd1 = code & 0x1F; + ihigh = (code >> 5) & 0x03; + wd2 = qm5[wd1]; + wd1 >>= 1; + break; + case 6: + wd1 = code & 0x0F; + ihigh = (code >> 4) & 0x03; + wd2 = qm4[wd1]; + break; + } + /* Block 5L, LOW BAND INVQBL */ + wd2 = (s->band[0].det*wd2) >> 15; + /* Block 5L, RECONS */ + rlow = s->band[0].s + wd2; + /* Block 6L, LIMIT */ + if (rlow > 16383) + rlow = 16383; + else if (rlow < -16384) + rlow = -16384; + + /* Block 2L, INVQAL */ + wd2 = qm4[wd1]; + dlowt = (s->band[0].det*wd2) >> 15; + + /* Block 3L, LOGSCL */ + wd2 = rl42[wd1]; + wd1 = (s->band[0].nb*127) >> 7; + wd1 += wl[wd2]; + if (wd1 < 0) + wd1 = 0; + else if (wd1 > 18432) + wd1 = 18432; + s->band[0].nb = wd1; + + /* Block 3L, SCALEL */ + wd1 = (s->band[0].nb >> 6) & 31; + wd2 = 8 - (s->band[0].nb >> 11); + wd3 = (wd2 < 0) ? (ilb[wd1] << -wd2) : (ilb[wd1] >> wd2); + s->band[0].det = wd3 << 2; + + block4(s, 0, dlowt); + + if (!s->eight_k) + { + /* Block 2H, INVQAH */ + wd2 = qm2[ihigh]; + dhigh = (s->band[1].det*wd2) >> 15; + /* Block 5H, RECONS */ + rhigh = dhigh + s->band[1].s; + /* Block 6H, LIMIT */ + if (rhigh > 16383) + rhigh = 16383; + else if (rhigh < -16384) + rhigh = -16384; + + /* Block 2H, INVQAH */ + wd2 = rh2[ihigh]; + wd1 = (s->band[1].nb*127) >> 7; + wd1 += wh[wd2]; + if (wd1 < 0) + wd1 = 0; + else if (wd1 > 22528) + wd1 = 22528; + s->band[1].nb = wd1; + + /* Block 3H, SCALEH */ + wd1 = (s->band[1].nb >> 6) & 31; + wd2 = 10 - (s->band[1].nb >> 11); + wd3 = (wd2 < 0) ? (ilb[wd1] << -wd2) : (ilb[wd1] >> wd2); + s->band[1].det = wd3 << 2; + + block4(s, 1, dhigh); + } + + if (s->itu_test_mode) + { + amp[outlen++] = (int16_t) (rlow << 1); + amp[outlen++] = (int16_t) (rhigh << 1); + } + else + { + if (s->eight_k) + { + amp[outlen++] = (int16_t) (rlow << 1); + } + else + { + /* Apply the receive QMF */ + for (i = 0; i < 22; i++) + s->x[i] = s->x[i + 2]; + s->x[22] = rlow + rhigh; + s->x[23] = rlow - rhigh; + + xout1 = 0; + xout2 = 0; + for (i = 0; i < 12; i++) + { + xout2 += s->x[2*i]*qmf_coeffs[i]; + xout1 += s->x[2*i + 1]*qmf_coeffs[11 - i]; + } + /* We shift by 12 to allow for the QMF filters (DC gain = 4096), less 1 + to allow for the 15 bit input to the G.722 algorithm. */ + /* WebRtc, tlegrand: added saturation */ + amp[outlen++] = saturate(xout1 >> 11); + amp[outlen++] = saturate(xout2 >> 11); + } + } + } + return outlen; +} +/*- End of function --------------------------------------------------------*/ +/*- End of file ------------------------------------------------------------*/ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_enc_dec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_enc_dec.h new file mode 100644 index 0000000000..ccda09baf7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_enc_dec.h @@ -0,0 +1,160 @@ +/* + * SpanDSP - a series of DSP components for telephony + * + * g722.h - The ITU G.722 codec. + * + * Written by Steve Underwood + * + * Copyright (C) 2005 Steve Underwood + * + * Despite my general liking of the GPL, I place my own contributions + * to this code in the public domain for the benefit of all mankind - + * even the slimy ones who might try to proprietize my work and use it + * to my detriment. + * + * Based on a single channel G.722 codec which is: + * + ***** Copyright (c) CMU 1993 ***** + * Computer Science, Speech Group + * Chengxiang Lu and Alex Hauptmann + * + * $Id: g722.h,v 1.10 2006/06/16 12:45:53 steveu Exp $ + * + * Modifications for WebRtc, 2011/04/28, by tlegrand: + * -Changed to use WebRtc types + * -Added new defines for minimum and maximum values of short int + */ + + +/*! \file */ + +#if !defined(_G722_ENC_DEC_H_) +#define _G722_ENC_DEC_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/*! \page g722_page G.722 encoding and decoding +\section g722_page_sec_1 What does it do? +The G.722 module is a bit exact implementation of the ITU G.722 specification for all three +specified bit rates - 64000bps, 56000bps and 48000bps. It passes the ITU tests. + +To allow fast and flexible interworking with narrow band telephony, the encoder and decoder +support an option for the linear audio to be an 8k samples/second stream. In this mode the +codec is considerably faster, and still fully compatible with wideband terminals using G.722. + +\section g722_page_sec_2 How does it work? +???. +*/ + +#define WEBRTC_INT16_MAX 32767 +#define WEBRTC_INT16_MIN -32768 + +enum +{ + G722_SAMPLE_RATE_8000 = 0x0001, + G722_PACKED = 0x0002 +}; + +typedef struct +{ + /*! TRUE if the operating in the special ITU test mode, with the band split filters + disabled. */ + int itu_test_mode; + /*! TRUE if the G.722 data is packed */ + int packed; + /*! TRUE if encode from 8k samples/second */ + int eight_k; + /*! 6 for 48000kbps, 7 for 56000kbps, or 8 for 64000kbps. */ + int bits_per_sample; + + /*! Signal history for the QMF */ + int x[24]; + + struct + { + int s; + int sp; + int sz; + int r[3]; + int a[3]; + int ap[3]; + int p[3]; + int d[7]; + int b[7]; + int bp[7]; + int sg[7]; + int nb; + int det; + } band[2]; + + unsigned int in_buffer; + int in_bits; + unsigned int out_buffer; + int out_bits; +} G722EncoderState; + +typedef struct +{ + /*! TRUE if the operating in the special ITU test mode, with the band split filters + disabled. */ + int itu_test_mode; + /*! TRUE if the G.722 data is packed */ + int packed; + /*! TRUE if decode to 8k samples/second */ + int eight_k; + /*! 6 for 48000kbps, 7 for 56000kbps, or 8 for 64000kbps. */ + int bits_per_sample; + + /*! Signal history for the QMF */ + int x[24]; + + struct + { + int s; + int sp; + int sz; + int r[3]; + int a[3]; + int ap[3]; + int p[3]; + int d[7]; + int b[7]; + int bp[7]; + int sg[7]; + int nb; + int det; + } band[2]; + + unsigned int in_buffer; + int in_bits; + unsigned int out_buffer; + int out_bits; +} G722DecoderState; + +#ifdef __cplusplus +extern "C" { +#endif + +G722EncoderState* WebRtc_g722_encode_init(G722EncoderState* s, + int rate, + int options); +int WebRtc_g722_encode_release(G722EncoderState *s); +size_t WebRtc_g722_encode(G722EncoderState *s, + uint8_t g722_data[], + const int16_t amp[], + size_t len); + +G722DecoderState* WebRtc_g722_decode_init(G722DecoderState* s, + int rate, + int options); +int WebRtc_g722_decode_release(G722DecoderState *s); +size_t WebRtc_g722_decode(G722DecoderState *s, + int16_t amp[], + const uint8_t g722_data[], + size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_encode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_encode.c new file mode 100644 index 0000000000..d060810bd9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_encode.c @@ -0,0 +1,430 @@ +/* + * SpanDSP - a series of DSP components for telephony + * + * g722_encode.c - The ITU G.722 codec, encode part. + * + * Written by Steve Underwood + * + * Copyright (C) 2005 Steve Underwood + * + * All rights reserved. + * + * Despite my general liking of the GPL, I place my own contributions + * to this code in the public domain for the benefit of all mankind - + * even the slimy ones who might try to proprietize my work and use it + * to my detriment. + * + * Based on a single channel 64kbps only G.722 codec which is: + * + ***** Copyright (c) CMU 1993 ***** + * Computer Science, Speech Group + * Chengxiang Lu and Alex Hauptmann + * + * $Id: g722_encode.c,v 1.14 2006/07/07 16:37:49 steveu Exp $ + * + * Modifications for WebRtc, 2011/04/28, by tlegrand: + * -Removed usage of inttypes.h and tgmath.h + * -Changed to use WebRtc types + * -Added option to run encoder bitexact with ITU-T reference implementation + */ + +/*! \file */ + +#include +#include +#include + +#include "g722_enc_dec.h" +#include "typedefs.h" // NOLINT(build/include) + +#if !defined(FALSE) +#define FALSE 0 +#endif +#if !defined(TRUE) +#define TRUE (!FALSE) +#endif + +static __inline int16_t saturate(int32_t amp) +{ + int16_t amp16; + + /* Hopefully this is optimised for the common case - not clipping */ + amp16 = (int16_t) amp; + if (amp == amp16) + return amp16; + if (amp > WEBRTC_INT16_MAX) + return WEBRTC_INT16_MAX; + return WEBRTC_INT16_MIN; +} +/*- End of function --------------------------------------------------------*/ + +static void block4(G722EncoderState *s, int band, int d) +{ + int wd1; + int wd2; + int wd3; + int i; + + /* Block 4, RECONS */ + s->band[band].d[0] = d; + s->band[band].r[0] = saturate(s->band[band].s + d); + + /* Block 4, PARREC */ + s->band[band].p[0] = saturate(s->band[band].sz + d); + + /* Block 4, UPPOL2 */ + for (i = 0; i < 3; i++) + s->band[band].sg[i] = s->band[band].p[i] >> 15; + wd1 = saturate(s->band[band].a[1] * 4); + + wd2 = (s->band[band].sg[0] == s->band[band].sg[1]) ? -wd1 : wd1; + if (wd2 > 32767) + wd2 = 32767; + wd3 = (wd2 >> 7) + ((s->band[band].sg[0] == s->band[band].sg[2]) ? 128 : -128); + wd3 += (s->band[band].a[2]*32512) >> 15; + if (wd3 > 12288) + wd3 = 12288; + else if (wd3 < -12288) + wd3 = -12288; + s->band[band].ap[2] = wd3; + + /* Block 4, UPPOL1 */ + s->band[band].sg[0] = s->band[band].p[0] >> 15; + s->band[band].sg[1] = s->band[band].p[1] >> 15; + wd1 = (s->band[band].sg[0] == s->band[band].sg[1]) ? 192 : -192; + wd2 = (s->band[band].a[1]*32640) >> 15; + + s->band[band].ap[1] = saturate(wd1 + wd2); + wd3 = saturate(15360 - s->band[band].ap[2]); + if (s->band[band].ap[1] > wd3) + s->band[band].ap[1] = wd3; + else if (s->band[band].ap[1] < -wd3) + s->band[band].ap[1] = -wd3; + + /* Block 4, UPZERO */ + wd1 = (d == 0) ? 0 : 128; + s->band[band].sg[0] = d >> 15; + for (i = 1; i < 7; i++) + { + s->band[band].sg[i] = s->band[band].d[i] >> 15; + wd2 = (s->band[band].sg[i] == s->band[band].sg[0]) ? wd1 : -wd1; + wd3 = (s->band[band].b[i]*32640) >> 15; + s->band[band].bp[i] = saturate(wd2 + wd3); + } + + /* Block 4, DELAYA */ + for (i = 6; i > 0; i--) + { + s->band[band].d[i] = s->band[band].d[i - 1]; + s->band[band].b[i] = s->band[band].bp[i]; + } + + for (i = 2; i > 0; i--) + { + s->band[band].r[i] = s->band[band].r[i - 1]; + s->band[band].p[i] = s->band[band].p[i - 1]; + s->band[band].a[i] = s->band[band].ap[i]; + } + + /* Block 4, FILTEP */ + wd1 = saturate(s->band[band].r[1] + s->band[band].r[1]); + wd1 = (s->band[band].a[1]*wd1) >> 15; + wd2 = saturate(s->band[band].r[2] + s->band[band].r[2]); + wd2 = (s->band[band].a[2]*wd2) >> 15; + s->band[band].sp = saturate(wd1 + wd2); + + /* Block 4, FILTEZ */ + s->band[band].sz = 0; + for (i = 6; i > 0; i--) + { + wd1 = saturate(s->band[band].d[i] + s->band[band].d[i]); + s->band[band].sz += (s->band[band].b[i]*wd1) >> 15; + } + s->band[band].sz = saturate(s->band[band].sz); + + /* Block 4, PREDIC */ + s->band[band].s = saturate(s->band[band].sp + s->band[band].sz); +} +/*- End of function --------------------------------------------------------*/ + +G722EncoderState* WebRtc_g722_encode_init(G722EncoderState* s, + int rate, + int options) { + if (s == NULL) + { + if ((s = (G722EncoderState *) malloc(sizeof(*s))) == NULL) + return NULL; + } + memset(s, 0, sizeof(*s)); + if (rate == 48000) + s->bits_per_sample = 6; + else if (rate == 56000) + s->bits_per_sample = 7; + else + s->bits_per_sample = 8; + if ((options & G722_SAMPLE_RATE_8000)) + s->eight_k = TRUE; + if ((options & G722_PACKED) && s->bits_per_sample != 8) + s->packed = TRUE; + else + s->packed = FALSE; + s->band[0].det = 32; + s->band[1].det = 8; + return s; +} +/*- End of function --------------------------------------------------------*/ + +int WebRtc_g722_encode_release(G722EncoderState *s) +{ + free(s); + return 0; +} +/*- End of function --------------------------------------------------------*/ + +/* WebRtc, tlegrand: + * Only define the following if bit-exactness with reference implementation + * is needed. Will only have any effect if input signal is saturated. + */ +//#define RUN_LIKE_REFERENCE_G722 +#ifdef RUN_LIKE_REFERENCE_G722 +int16_t limitValues (int16_t rl) +{ + + int16_t yl; + + yl = (rl > 16383) ? 16383 : ((rl < -16384) ? -16384 : rl); + + return (yl); +} +#endif + +size_t WebRtc_g722_encode(G722EncoderState *s, uint8_t g722_data[], + const int16_t amp[], size_t len) +{ + static const int q6[32] = + { + 0, 35, 72, 110, 150, 190, 233, 276, + 323, 370, 422, 473, 530, 587, 650, 714, + 786, 858, 940, 1023, 1121, 1219, 1339, 1458, + 1612, 1765, 1980, 2195, 2557, 2919, 0, 0 + }; + static const int iln[32] = + { + 0, 63, 62, 31, 30, 29, 28, 27, + 26, 25, 24, 23, 22, 21, 20, 19, + 18, 17, 16, 15, 14, 13, 12, 11, + 10, 9, 8, 7, 6, 5, 4, 0 + }; + static const int ilp[32] = + { + 0, 61, 60, 59, 58, 57, 56, 55, + 54, 53, 52, 51, 50, 49, 48, 47, + 46, 45, 44, 43, 42, 41, 40, 39, + 38, 37, 36, 35, 34, 33, 32, 0 + }; + static const int wl[8] = + { + -60, -30, 58, 172, 334, 538, 1198, 3042 + }; + static const int rl42[16] = + { + 0, 7, 6, 5, 4, 3, 2, 1, 7, 6, 5, 4, 3, 2, 1, 0 + }; + static const int ilb[32] = + { + 2048, 2093, 2139, 2186, 2233, 2282, 2332, + 2383, 2435, 2489, 2543, 2599, 2656, 2714, + 2774, 2834, 2896, 2960, 3025, 3091, 3158, + 3228, 3298, 3371, 3444, 3520, 3597, 3676, + 3756, 3838, 3922, 4008 + }; + static const int qm4[16] = + { + 0, -20456, -12896, -8968, + -6288, -4240, -2584, -1200, + 20456, 12896, 8968, 6288, + 4240, 2584, 1200, 0 + }; + static const int qm2[4] = + { + -7408, -1616, 7408, 1616 + }; + static const int qmf_coeffs[12] = + { + 3, -11, 12, 32, -210, 951, 3876, -805, 362, -156, 53, -11, + }; + static const int ihn[3] = {0, 1, 0}; + static const int ihp[3] = {0, 3, 2}; + static const int wh[3] = {0, -214, 798}; + static const int rh2[4] = {2, 1, 2, 1}; + + int dlow; + int dhigh; + int el; + int wd; + int wd1; + int ril; + int wd2; + int il4; + int ih2; + int wd3; + int eh; + int mih; + int i; + size_t j; + /* Low and high band PCM from the QMF */ + int xlow; + int xhigh; + size_t g722_bytes; + /* Even and odd tap accumulators */ + int sumeven; + int sumodd; + int ihigh; + int ilow; + int code; + + g722_bytes = 0; + xhigh = 0; + for (j = 0; j < len; ) + { + if (s->itu_test_mode) + { + xlow = + xhigh = amp[j++] >> 1; + } + else + { + if (s->eight_k) + { + /* We shift by 1 to allow for the 15 bit input to the G.722 algorithm. */ + xlow = amp[j++] >> 1; + } + else + { + /* Apply the transmit QMF */ + /* Shuffle the buffer down */ + for (i = 0; i < 22; i++) + s->x[i] = s->x[i + 2]; + s->x[22] = amp[j++]; + s->x[23] = amp[j++]; + + /* Discard every other QMF output */ + sumeven = 0; + sumodd = 0; + for (i = 0; i < 12; i++) + { + sumodd += s->x[2*i]*qmf_coeffs[i]; + sumeven += s->x[2*i + 1]*qmf_coeffs[11 - i]; + } + /* We shift by 12 to allow for the QMF filters (DC gain = 4096), plus 1 + to allow for us summing two filters, plus 1 to allow for the 15 bit + input to the G.722 algorithm. */ + xlow = (sumeven + sumodd) >> 14; + xhigh = (sumeven - sumodd) >> 14; + +#ifdef RUN_LIKE_REFERENCE_G722 + /* The following lines are only used to verify bit-exactness + * with reference implementation of G.722. Higher precision + * is achieved without limiting the values. + */ + xlow = limitValues(xlow); + xhigh = limitValues(xhigh); +#endif + } + } + /* Block 1L, SUBTRA */ + el = saturate(xlow - s->band[0].s); + + /* Block 1L, QUANTL */ + wd = (el >= 0) ? el : -(el + 1); + + for (i = 1; i < 30; i++) + { + wd1 = (q6[i]*s->band[0].det) >> 12; + if (wd < wd1) + break; + } + ilow = (el < 0) ? iln[i] : ilp[i]; + + /* Block 2L, INVQAL */ + ril = ilow >> 2; + wd2 = qm4[ril]; + dlow = (s->band[0].det*wd2) >> 15; + + /* Block 3L, LOGSCL */ + il4 = rl42[ril]; + wd = (s->band[0].nb*127) >> 7; + s->band[0].nb = wd + wl[il4]; + if (s->band[0].nb < 0) + s->band[0].nb = 0; + else if (s->band[0].nb > 18432) + s->band[0].nb = 18432; + + /* Block 3L, SCALEL */ + wd1 = (s->band[0].nb >> 6) & 31; + wd2 = 8 - (s->band[0].nb >> 11); + wd3 = (wd2 < 0) ? (ilb[wd1] << -wd2) : (ilb[wd1] >> wd2); + s->band[0].det = wd3 << 2; + + block4(s, 0, dlow); + + if (s->eight_k) + { + /* Just leave the high bits as zero */ + code = (0xC0 | ilow) >> (8 - s->bits_per_sample); + } + else + { + /* Block 1H, SUBTRA */ + eh = saturate(xhigh - s->band[1].s); + + /* Block 1H, QUANTH */ + wd = (eh >= 0) ? eh : -(eh + 1); + wd1 = (564*s->band[1].det) >> 12; + mih = (wd >= wd1) ? 2 : 1; + ihigh = (eh < 0) ? ihn[mih] : ihp[mih]; + + /* Block 2H, INVQAH */ + wd2 = qm2[ihigh]; + dhigh = (s->band[1].det*wd2) >> 15; + + /* Block 3H, LOGSCH */ + ih2 = rh2[ihigh]; + wd = (s->band[1].nb*127) >> 7; + s->band[1].nb = wd + wh[ih2]; + if (s->band[1].nb < 0) + s->band[1].nb = 0; + else if (s->band[1].nb > 22528) + s->band[1].nb = 22528; + + /* Block 3H, SCALEH */ + wd1 = (s->band[1].nb >> 6) & 31; + wd2 = 10 - (s->band[1].nb >> 11); + wd3 = (wd2 < 0) ? (ilb[wd1] << -wd2) : (ilb[wd1] >> wd2); + s->band[1].det = wd3 << 2; + + block4(s, 1, dhigh); + code = ((ihigh << 6) | ilow) >> (8 - s->bits_per_sample); + } + + if (s->packed) + { + /* Pack the code bits */ + s->out_buffer |= (code << s->out_bits); + s->out_bits += s->bits_per_sample; + if (s->out_bits >= 8) + { + g722_data[g722_bytes++] = (uint8_t) (s->out_buffer & 0xFF); + s->out_bits -= 8; + s->out_buffer >>= 8; + } + } + else + { + g722_data[g722_bytes++] = (uint8_t) code; + } + } + return g722_bytes; +} +/*- End of function --------------------------------------------------------*/ +/*- End of file ------------------------------------------------------------*/ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.c new file mode 100644 index 0000000000..b6dccb454f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + + +#include +#include +#include "g722_enc_dec.h" +#include "g722_interface.h" +#include "typedefs.h" // NOLINT(build/include) + +int16_t WebRtcG722_CreateEncoder(G722EncInst **G722enc_inst) +{ + *G722enc_inst=(G722EncInst*)malloc(sizeof(G722EncoderState)); + if (*G722enc_inst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcG722_EncoderInit(G722EncInst *G722enc_inst) +{ + // Create and/or reset the G.722 encoder + // Bitrate 64 kbps and wideband mode (2) + G722enc_inst = (G722EncInst *) WebRtc_g722_encode_init( + (G722EncoderState*) G722enc_inst, 64000, 2); + if (G722enc_inst == NULL) { + return -1; + } else { + return 0; + } +} + +int WebRtcG722_FreeEncoder(G722EncInst *G722enc_inst) +{ + // Free encoder memory + return WebRtc_g722_encode_release((G722EncoderState*) G722enc_inst); +} + +size_t WebRtcG722_Encode(G722EncInst *G722enc_inst, + const int16_t* speechIn, + size_t len, + uint8_t* encoded) +{ + unsigned char *codechar = (unsigned char*) encoded; + // Encode the input speech vector + return WebRtc_g722_encode((G722EncoderState*) G722enc_inst, codechar, + speechIn, len); +} + +int16_t WebRtcG722_CreateDecoder(G722DecInst **G722dec_inst) +{ + *G722dec_inst=(G722DecInst*)malloc(sizeof(G722DecoderState)); + if (*G722dec_inst!=NULL) { + return(0); + } else { + return(-1); + } +} + +void WebRtcG722_DecoderInit(G722DecInst* inst) { + // Create and/or reset the G.722 decoder + // Bitrate 64 kbps and wideband mode (2) + WebRtc_g722_decode_init((G722DecoderState*)inst, 64000, 2); +} + +int WebRtcG722_FreeDecoder(G722DecInst *G722dec_inst) +{ + // Free encoder memory + return WebRtc_g722_decode_release((G722DecoderState*) G722dec_inst); +} + +size_t WebRtcG722_Decode(G722DecInst *G722dec_inst, + const uint8_t *encoded, + size_t len, + int16_t *decoded, + int16_t *speechType) +{ + // Decode the G.722 encoder stream + *speechType=G722_WEBRTC_SPEECH; + return WebRtc_g722_decode((G722DecoderState*) G722dec_inst, decoded, + encoded, len); +} + +int16_t WebRtcG722_Version(char *versionStr, short len) +{ + // Get version string + char version[30] = "2.0.0\n"; + if (strlen(version) < (unsigned int)len) + { + strcpy(versionStr, version); + return 0; + } + else + { + return -1; + } +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.h new file mode 100644 index 0000000000..d95722314f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G722_G722_INTERFACE_H_ +#define MODULES_AUDIO_CODING_CODECS_G722_G722_INTERFACE_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/* + * Solution to support multiple instances + */ + +typedef struct WebRtcG722EncInst G722EncInst; +typedef struct WebRtcG722DecInst G722DecInst; + +/* + * Comfort noise constants + */ + +#define G722_WEBRTC_SPEECH 1 +#define G722_WEBRTC_CNG 2 + +#ifdef __cplusplus +extern "C" { +#endif + + +/**************************************************************************** + * WebRtcG722_CreateEncoder(...) + * + * Create memory used for G722 encoder + * + * Input: + * - G722enc_inst : G722 instance for encoder + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcG722_CreateEncoder(G722EncInst **G722enc_inst); + + +/**************************************************************************** + * WebRtcG722_EncoderInit(...) + * + * This function initializes a G722 instance + * + * Input: + * - G722enc_inst : G722 instance, i.e. the user that should receive + * be initialized + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcG722_EncoderInit(G722EncInst *G722enc_inst); + + +/**************************************************************************** + * WebRtcG722_FreeEncoder(...) + * + * Free the memory used for G722 encoder + * + * Input: + * - G722enc_inst : G722 instance for encoder + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcG722_FreeEncoder(G722EncInst *G722enc_inst); + + + +/**************************************************************************** + * WebRtcG722_Encode(...) + * + * This function encodes G722 encoded data. + * + * Input: + * - G722enc_inst : G722 instance, i.e. the user that should encode + * a packet + * - speechIn : Input speech vector + * - len : Samples in speechIn + * + * Output: + * - encoded : The encoded data vector + * + * Return value : Length (in bytes) of coded data + */ + +size_t WebRtcG722_Encode(G722EncInst* G722enc_inst, + const int16_t* speechIn, + size_t len, + uint8_t* encoded); + + +/**************************************************************************** + * WebRtcG722_CreateDecoder(...) + * + * Create memory used for G722 encoder + * + * Input: + * - G722dec_inst : G722 instance for decoder + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcG722_CreateDecoder(G722DecInst **G722dec_inst); + +/**************************************************************************** + * WebRtcG722_DecoderInit(...) + * + * This function initializes a G722 instance + * + * Input: + * - inst : G722 instance + */ + +void WebRtcG722_DecoderInit(G722DecInst* inst); + +/**************************************************************************** + * WebRtcG722_FreeDecoder(...) + * + * Free the memory used for G722 decoder + * + * Input: + * - G722dec_inst : G722 instance for decoder + * + * Return value : 0 - Ok + * -1 - Error + */ + +int WebRtcG722_FreeDecoder(G722DecInst *G722dec_inst); + + +/**************************************************************************** + * WebRtcG722_Decode(...) + * + * This function decodes a packet with G729 frame(s). Output speech length + * will be a multiple of 80 samples (80*frames/packet). + * + * Input: + * - G722dec_inst : G722 instance, i.e. the user that should decode + * a packet + * - encoded : Encoded G722 frame(s) + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG (Since G722 does not have its own + * DTX/CNG scheme it should always return 1) + * + * Return value : Samples in decoded vector + */ + +size_t WebRtcG722_Decode(G722DecInst *G722dec_inst, + const uint8_t* encoded, + size_t len, + int16_t *decoded, + int16_t *speechType); + +/**************************************************************************** + * WebRtcG722_Version(...) + * + * Get a string with the current version of the codec + */ + +int16_t WebRtcG722_Version(char *versionStr, short len); + + +#ifdef __cplusplus +} +#endif + + +#endif /* MODULES_AUDIO_CODING_CODECS_G722_G722_INTERFACE_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/test/testG722.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/test/testG722.cc new file mode 100644 index 0000000000..e0281f2b65 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/test/testG722.cc @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * testG722.cpp : Defines the entry point for the console application. + */ + +#include +#include +#include +#include "typedefs.h" // NOLINT(build/include) + +/* include API */ +#include "modules/audio_coding/codecs/g722/g722_interface.h" + +/* Runtime statistics */ +#include +#define CLOCKS_PER_SEC_G722 100000 + +// Forward declaration +typedef struct WebRtcG722EncInst G722EncInst; +typedef struct WebRtcG722DecInst G722DecInst; + +/* function for reading audio data from PCM file */ +bool readframe(int16_t *data, FILE *inp, size_t length) +{ + size_t rlen = fread(data, sizeof(int16_t), length, inp); + if (rlen >= length) + return false; + memset(data + rlen, 0, (length - rlen) * sizeof(int16_t)); + return true; +} + +int main(int argc, char* argv[]) +{ + char inname[60], outbit[40], outname[40]; + FILE *inp, *outbitp, *outp; + + int framecnt; + bool endfile; + size_t framelength = 160; + G722EncInst *G722enc_inst; + G722DecInst *G722dec_inst; + + /* Runtime statistics */ + double starttime; + double runtime = 0; + double length_file; + + size_t stream_len = 0; + int16_t shortdata[960]; + int16_t decoded[960]; + uint8_t streamdata[80 * 6]; + int16_t speechType[1]; + + /* handling wrong input arguments in the command line */ + if (argc!=5) { + printf("\n\nWrong number of arguments or flag values.\n\n"); + + printf("\n"); + printf("Usage:\n\n"); + printf("./testG722.exe framelength infile outbitfile outspeechfile \n\n"); + printf("with:\n"); + printf("framelength : Framelength in samples.\n\n"); + printf("infile : Normal speech input file\n\n"); + printf("outbitfile : Bitstream output file\n\n"); + printf("outspeechfile: Speech output file\n\n"); + exit(0); + + } + + /* Get frame length */ + int framelength_int = atoi(argv[1]); + if (framelength_int < 0) { + printf(" G.722: Invalid framelength %d.\n", framelength_int); + exit(1); + } + framelength = static_cast(framelength_int); + + /* Get Input and Output files */ + sscanf(argv[2], "%s", inname); + sscanf(argv[3], "%s", outbit); + sscanf(argv[4], "%s", outname); + + if ((inp = fopen(inname,"rb")) == NULL) { + printf(" G.722: Cannot read file %s.\n", inname); + exit(1); + } + if ((outbitp = fopen(outbit,"wb")) == NULL) { + printf(" G.722: Cannot write file %s.\n", outbit); + exit(1); + } + if ((outp = fopen(outname,"wb")) == NULL) { + printf(" G.722: Cannot write file %s.\n", outname); + exit(1); + } + printf("\nInput:%s\nOutput bitstream:%s\nOutput:%s\n", inname, outbit, outname); + + /* Create and init */ + WebRtcG722_CreateEncoder((G722EncInst **)&G722enc_inst); + WebRtcG722_CreateDecoder((G722DecInst **)&G722dec_inst); + WebRtcG722_EncoderInit((G722EncInst *)G722enc_inst); + WebRtcG722_DecoderInit((G722DecInst *)G722dec_inst); + + + /* Initialize encoder and decoder */ + framecnt = 0; + endfile = false; + while (!endfile) { + framecnt++; + + /* Read speech block */ + endfile = readframe(shortdata, inp, framelength); + + /* Start clock before call to encoder and decoder */ + starttime = clock()/(double)CLOCKS_PER_SEC_G722; + + /* G.722 encoding + decoding */ + stream_len = WebRtcG722_Encode((G722EncInst *)G722enc_inst, shortdata, framelength, streamdata); + WebRtcG722_Decode(G722dec_inst, streamdata, stream_len, decoded, + speechType); + + /* Stop clock after call to encoder and decoder */ + runtime += (double)((clock()/(double)CLOCKS_PER_SEC_G722)-starttime); + + /* Write coded bits to file */ + if (fwrite(streamdata, sizeof(short), stream_len / 2, outbitp) != + stream_len / 2) { + return -1; + } + /* Write coded speech to file */ + if (fwrite(decoded, sizeof(short), framelength, outp) != + framelength) { + return -1; + } + } + + WebRtcG722_FreeEncoder((G722EncInst *)G722enc_inst); + WebRtcG722_FreeDecoder((G722DecInst *)G722dec_inst); + + length_file = ((double)framecnt*(double)framelength/16000); + printf("\n\nLength of speech file: %.1f s\n", length_file); + printf("Time to run G.722: %.2f s (%.2f %% of realtime)\n\n", runtime, (100*runtime/length_file)); + printf("---------------------END----------------------\n"); + + fclose(inp); + fclose(outbitp); + fclose(outp); + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.c new file mode 100644 index 0000000000..263749ad2a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuant.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "abs_quant_loop.h" + + +/*----------------------------------------------------------------* + * predictive noise shaping encoding of scaled start state + * (subrutine for WebRtcIlbcfix_StateSearch) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AbsQuant( + IlbcEncoder *iLBCenc_inst, + /* (i) Encoder instance */ + iLBC_bits *iLBC_encbits, /* (i/o) Encoded bits (outputs idxForMax + and idxVec, uses state_first as + input) */ + int16_t *in, /* (i) vector to encode */ + int16_t *weightDenum /* (i) denominator of synthesis filter */ + ) { + int16_t *syntOut; + size_t quantLen[2]; + + /* Stack based */ + int16_t syntOutBuf[LPC_FILTERORDER+STATE_SHORT_LEN_30MS]; + int16_t in_weightedVec[STATE_SHORT_LEN_30MS+LPC_FILTERORDER]; + int16_t *in_weighted = &in_weightedVec[LPC_FILTERORDER]; + + /* Initialize the buffers */ + WebRtcSpl_MemSetW16(syntOutBuf, 0, LPC_FILTERORDER+STATE_SHORT_LEN_30MS); + syntOut = &syntOutBuf[LPC_FILTERORDER]; + /* Start with zero state */ + WebRtcSpl_MemSetW16(in_weightedVec, 0, LPC_FILTERORDER); + + /* Perform the quantization loop in two sections of length quantLen[i], + where the perceptual weighting filter is updated at the subframe + border */ + + if (iLBC_encbits->state_first) { + quantLen[0]=SUBL; + quantLen[1]=iLBCenc_inst->state_short_len-SUBL; + } else { + quantLen[0]=iLBCenc_inst->state_short_len-SUBL; + quantLen[1]=SUBL; + } + + /* Calculate the weighted residual, switch perceptual weighting + filter at the subframe border */ + WebRtcSpl_FilterARFastQ12( + in, in_weighted, + weightDenum, LPC_FILTERORDER+1, quantLen[0]); + WebRtcSpl_FilterARFastQ12( + &in[quantLen[0]], &in_weighted[quantLen[0]], + &weightDenum[LPC_FILTERORDER+1], LPC_FILTERORDER+1, quantLen[1]); + + WebRtcIlbcfix_AbsQuantLoop( + syntOut, + in_weighted, + weightDenum, + quantLen, + iLBC_encbits->idxVec); + +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.h new file mode 100644 index 0000000000..090ed73f12 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuant.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * predictive noise shaping encoding of scaled start state + * (subrutine for WebRtcIlbcfix_StateSearch) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AbsQuant( + IlbcEncoder *iLBCenc_inst, + /* (i) Encoder instance */ + iLBC_bits *iLBC_encbits, /* (i/o) Encoded bits (outputs idxForMax + and idxVec, uses state_first as + input) */ + int16_t *in, /* (i) vector to encode */ + int16_t *weightDenum /* (i) denominator of synthesis filter */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c new file mode 100644 index 0000000000..4b76453446 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuantLoop.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "sort_sq.h" + +void WebRtcIlbcfix_AbsQuantLoop(int16_t *syntOutIN, int16_t *in_weightedIN, + int16_t *weightDenumIN, size_t *quantLenIN, + int16_t *idxVecIN ) { + size_t k1, k2; + int16_t index; + int32_t toQW32; + int32_t toQ32; + int16_t tmp16a; + int16_t xq; + + int16_t *syntOut = syntOutIN; + int16_t *in_weighted = in_weightedIN; + int16_t *weightDenum = weightDenumIN; + size_t *quantLen = quantLenIN; + int16_t *idxVec = idxVecIN; + + for(k1=0;k1<2;k1++) { + for(k2=0;k2 32767) { + toQ32 = (int32_t) 32767; + } else if (toQ32 < -32768) { + toQ32 = (int32_t) -32768; + } + + /* Quantize the state */ + if (toQW32<(-7577)) { + /* To prevent negative overflow */ + index=0; + } else if (toQW32>8151) { + /* To prevent positive overflow */ + index=7; + } else { + /* Find the best quantization index + (state_sq3Tbl is in Q13 and toQ is in Q11) + */ + WebRtcIlbcfix_SortSq(&xq, &index, + (int16_t)toQ32, + WebRtcIlbcfix_kStateSq3, 8); + } + + /* Store selected index */ + (*idxVec++) = index; + + /* Compute decoded sample and update of the prediction filter */ + tmp16a = ((WebRtcIlbcfix_kStateSq3[index] + 2 ) >> 2); + + *syntOut = (int16_t) (tmp16a + (int32_t)(*in_weighted) - toQW32); + + syntOut++; in_weighted++; + } + /* Update perceptual weighting filter at subframe border */ + weightDenum += 11; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h new file mode 100644 index 0000000000..92f6fff847 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuantLoop.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_LOOP_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_LOOP_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * predictive noise shaping encoding of scaled start state + * (subrutine for WebRtcIlbcfix_StateSearch) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AbsQuantLoop(int16_t *syntOutIN, int16_t *in_weightedIN, + int16_t *weightDenumIN, size_t *quantLenIN, + int16_t *idxVecIN); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc new file mode 100644 index 0000000000..08d21f4202 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" + +#include + +#include "modules/audio_coding/codecs/ilbc/ilbc.h" +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +AudioDecoderIlbcImpl::AudioDecoderIlbcImpl() { + WebRtcIlbcfix_DecoderCreate(&dec_state_); + WebRtcIlbcfix_Decoderinit30Ms(dec_state_); +} + +AudioDecoderIlbcImpl::~AudioDecoderIlbcImpl() { + WebRtcIlbcfix_DecoderFree(dec_state_); +} + +bool AudioDecoderIlbcImpl::HasDecodePlc() const { + return true; +} + +int AudioDecoderIlbcImpl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz, 8000); + int16_t temp_type = 1; // Default is speech. + int ret = WebRtcIlbcfix_Decode(dec_state_, encoded, encoded_len, decoded, + &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +size_t AudioDecoderIlbcImpl::DecodePlc(size_t num_frames, int16_t* decoded) { + return WebRtcIlbcfix_NetEqPlc(dec_state_, decoded, num_frames); +} + +void AudioDecoderIlbcImpl::Reset() { + WebRtcIlbcfix_Decoderinit30Ms(dec_state_); +} + +std::vector AudioDecoderIlbcImpl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + size_t bytes_per_frame; + int timestamps_per_frame; + if (payload.size() >= 950) { + RTC_LOG(LS_WARNING) + << "AudioDecoderIlbcImpl::ParsePayload: Payload too large"; + return results; + } + if (payload.size() % 38 == 0) { + // 20 ms frames. + bytes_per_frame = 38; + timestamps_per_frame = 160; + } else if (payload.size() % 50 == 0) { + // 30 ms frames. + bytes_per_frame = 50; + timestamps_per_frame = 240; + } else { + RTC_LOG(LS_WARNING) + << "AudioDecoderIlbcImpl::ParsePayload: Invalid payload"; + return results; + } + + RTC_DCHECK_EQ(0, payload.size() % bytes_per_frame); + if (payload.size() == bytes_per_frame) { + std::unique_ptr frame( + new LegacyEncodedAudioFrame(this, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); + } else { + size_t byte_offset; + uint32_t timestamp_offset; + for (byte_offset = 0, timestamp_offset = 0; + byte_offset < payload.size(); + byte_offset += bytes_per_frame, + timestamp_offset += timestamps_per_frame) { + std::unique_ptr frame(new LegacyEncodedAudioFrame( + this, rtc::Buffer(payload.data() + byte_offset, bytes_per_frame))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); + } + } + + return results; +} + +int AudioDecoderIlbcImpl::SampleRateHz() const { + return 8000; +} + +size_t AudioDecoderIlbcImpl::Channels() const { + return 1; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h new file mode 100644 index 0000000000..edb65d09c6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_DECODER_ILBC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_DECODER_ILBC_H_ + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/constructormagic.h" + +typedef struct iLBC_decinst_t_ IlbcDecoderInstance; + +namespace webrtc { + +class AudioDecoderIlbcImpl final : public AudioDecoder { + public: + AudioDecoderIlbcImpl(); + ~AudioDecoderIlbcImpl() override; + bool HasDecodePlc() const override; + size_t DecodePlc(size_t num_frames, int16_t* decoded) override; + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + IlbcDecoderInstance* dec_state_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderIlbcImpl); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_DECODER_ILBC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc new file mode 100644 index 0000000000..6ddc078f09 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" + +#include +#include +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/ilbc/ilbc.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { + +const int kSampleRateHz = 8000; + +AudioEncoderIlbcConfig CreateConfig(const CodecInst& codec_inst) { + AudioEncoderIlbcConfig config; + config.frame_size_ms = codec_inst.pacsize / 8; + return config; +} + +int GetIlbcBitrate(int ptime) { + switch (ptime) { + case 20: + case 40: + // 38 bytes per frame of 20 ms => 15200 bits/s. + return 15200; + case 30: + case 60: + // 50 bytes per frame of 30 ms => (approx) 13333 bits/s. + return 13333; + default: + FATAL(); + } +} + +} // namespace + +AudioEncoderIlbcImpl::AudioEncoderIlbcImpl(const AudioEncoderIlbcConfig& config, + int payload_type) + : frame_size_ms_(config.frame_size_ms), + payload_type_(payload_type), + num_10ms_frames_per_packet_( + static_cast(config.frame_size_ms / 10)), + encoder_(nullptr) { + RTC_CHECK(config.IsOk()); + Reset(); +} + +AudioEncoderIlbcImpl::AudioEncoderIlbcImpl(const CodecInst& codec_inst) + : AudioEncoderIlbcImpl(CreateConfig(codec_inst), codec_inst.pltype) {} + +AudioEncoderIlbcImpl::~AudioEncoderIlbcImpl() { + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderFree(encoder_)); +} + +int AudioEncoderIlbcImpl::SampleRateHz() const { + return kSampleRateHz; +} + +size_t AudioEncoderIlbcImpl::NumChannels() const { + return 1; +} + +size_t AudioEncoderIlbcImpl::Num10MsFramesInNextPacket() const { + return num_10ms_frames_per_packet_; +} + +size_t AudioEncoderIlbcImpl::Max10MsFramesInAPacket() const { + return num_10ms_frames_per_packet_; +} + +int AudioEncoderIlbcImpl::GetTargetBitrate() const { + return GetIlbcBitrate(rtc::dchecked_cast(num_10ms_frames_per_packet_) * + 10); +} + +AudioEncoder::EncodedInfo AudioEncoderIlbcImpl::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + + // Save timestamp if starting a new packet. + if (num_10ms_frames_buffered_ == 0) + first_timestamp_in_buffer_ = rtp_timestamp; + + // Buffer input. + std::copy(audio.cbegin(), audio.cend(), + input_buffer_ + kSampleRateHz / 100 * num_10ms_frames_buffered_); + + // If we don't yet have enough buffered input for a whole packet, we're done + // for now. + if (++num_10ms_frames_buffered_ < num_10ms_frames_per_packet_) { + return EncodedInfo(); + } + + // Encode buffered input. + RTC_DCHECK_EQ(num_10ms_frames_buffered_, num_10ms_frames_per_packet_); + num_10ms_frames_buffered_ = 0; + size_t encoded_bytes = + encoded->AppendData( + RequiredOutputSizeBytes(), + [&] (rtc::ArrayView encoded) { + const int r = WebRtcIlbcfix_Encode( + encoder_, + input_buffer_, + kSampleRateHz / 100 * num_10ms_frames_per_packet_, + encoded.data()); + RTC_CHECK_GE(r, 0); + + return static_cast(r); + }); + + RTC_DCHECK_EQ(encoded_bytes, RequiredOutputSizeBytes()); + + EncodedInfo info; + info.encoded_bytes = encoded_bytes; + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.encoder_type = CodecType::kIlbc; + return info; +} + +void AudioEncoderIlbcImpl::Reset() { + if (encoder_) + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderFree(encoder_)); + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderCreate(&encoder_)); + const int encoder_frame_size_ms = frame_size_ms_ > 30 + ? frame_size_ms_ / 2 + : frame_size_ms_; + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderInit(encoder_, encoder_frame_size_ms)); + num_10ms_frames_buffered_ = 0; +} + +size_t AudioEncoderIlbcImpl::RequiredOutputSizeBytes() const { + switch (num_10ms_frames_per_packet_) { + case 2: return 38; + case 3: return 50; + case 4: return 2 * 38; + case 6: return 2 * 50; + default: FATAL(); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h new file mode 100644 index 0000000000..a238689aca --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_ENCODER_ILBC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_ENCODER_ILBC_H_ + +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/ilbc/audio_encoder_ilbc_config.h" +#include "modules/audio_coding/codecs/ilbc/ilbc.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +struct CodecInst; + +class AudioEncoderIlbcImpl final : public AudioEncoder { + public: + AudioEncoderIlbcImpl(const AudioEncoderIlbcConfig& config, int payload_type); + explicit AudioEncoderIlbcImpl(const CodecInst& codec_inst); + ~AudioEncoderIlbcImpl() override; + + int SampleRateHz() const override; + size_t NumChannels() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + void Reset() override; + + private: + size_t RequiredOutputSizeBytes() const; + + static constexpr size_t kMaxSamplesPerPacket = 480; + const int frame_size_ms_; + const int payload_type_; + const size_t num_10ms_frames_per_packet_; + size_t num_10ms_frames_buffered_; + uint32_t first_timestamp_in_buffer_; + int16_t input_buffer_[kMaxSamplesPerPacket]; + IlbcEncoderInstance* encoder_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderIlbcImpl); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_ENCODER_ILBC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c new file mode 100644 index 0000000000..1a3735fc3d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AugmentedCbCorr.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "augmented_cb_corr.h" + +void WebRtcIlbcfix_AugmentedCbCorr( + int16_t *target, /* (i) Target vector */ + int16_t *buffer, /* (i) Memory buffer */ + int16_t *interpSamples, /* (i) buffer with + interpolated samples */ + int32_t *crossDot, /* (o) The cross correlation between + the target and the Augmented + vector */ + size_t low, /* (i) Lag to start from (typically + 20) */ + size_t high, /* (i) Lag to end at (typically 39) */ + int scale) /* (i) Scale factor to use for + the crossDot */ +{ + size_t lagcount; + size_t ilow; + int16_t *targetPtr; + int32_t *crossDotPtr; + int16_t *iSPtr=interpSamples; + + /* Calculate the correlation between the target and the + interpolated codebook. The correlation is calculated in + 3 sections with the interpolated part in the middle */ + crossDotPtr=crossDot; + for (lagcount=low; lagcount<=high; lagcount++) { + + ilow = lagcount - 4; + + /* Compute dot product for the first (lagcount-4) samples */ + (*crossDotPtr) = WebRtcSpl_DotProductWithScale(target, buffer-lagcount, ilow, scale); + + /* Compute dot product on the interpolated samples */ + (*crossDotPtr) += WebRtcSpl_DotProductWithScale(target+ilow, iSPtr, 4, scale); + targetPtr = target + lagcount; + iSPtr += lagcount-ilow; + + /* Compute dot product for the remaining samples */ + (*crossDotPtr) += WebRtcSpl_DotProductWithScale(targetPtr, buffer-lagcount, SUBL-lagcount, scale); + crossDotPtr++; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h new file mode 100644 index 0000000000..e8eb0696d2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AugmentedCbCorr.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_AUGMENTED_CB_CORR_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_AUGMENTED_CB_CORR_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Calculate correlation between target and Augmented codebooks + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AugmentedCbCorr( + int16_t *target, /* (i) Target vector */ + int16_t *buffer, /* (i) Memory buffer */ + int16_t *interpSamples, /* (i) buffer with + interpolated samples */ + int32_t *crossDot, /* (o) The cross correlation between + the target and the Augmented + vector */ + size_t low, /* (i) Lag to start from (typically + 20) */ + size_t high, /* (i) Lag to end at (typically 39 */ + int scale); /* (i) Scale factor to use for the crossDot */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.c new file mode 100644 index 0000000000..b02abab3f2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_BwExpand.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * lpc bandwidth expansion + *---------------------------------------------------------------*/ + +/* The output is in the same domain as the input */ +void WebRtcIlbcfix_BwExpand( + int16_t *out, /* (o) the bandwidth expanded lpc coefficients */ + int16_t *in, /* (i) the lpc coefficients before bandwidth + expansion */ + int16_t *coef, /* (i) the bandwidth expansion factor Q15 */ + int16_t length /* (i) the length of lpc coefficient vectors */ + ) { + int i; + + out[0] = in[0]; + for (i = 1; i < length; i++) { + /* out[i] = coef[i] * in[i] with rounding. + in[] and out[] are in Q12 and coef[] is in Q15 + */ + out[i] = (int16_t)((coef[i] * in[i] + 16384) >> 15); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.h new file mode 100644 index 0000000000..6268baa9fa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_BwExpand.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_BW_EXPAND_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_BW_EXPAND_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * lpc bandwidth expansion + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_BwExpand( + int16_t *out, /* (o) the bandwidth expanded lpc coefficients */ + int16_t *in, /* (i) the lpc coefficients before bandwidth + expansion */ + int16_t *coef, /* (i) the bandwidth expansion factor Q15 */ + int16_t length /* (i) the length of lpc coefficient vectors */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.c new file mode 100644 index 0000000000..c19818ee37 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbConstruct.c + +******************************************************************/ + +#include "cb_construct.h" + +#include "defines.h" +#include "gain_dequant.h" +#include "get_cd_vec.h" + +/*----------------------------------------------------------------* + * Construct decoded vector from codebook and gains. + *---------------------------------------------------------------*/ + +bool WebRtcIlbcfix_CbConstruct( + int16_t* decvector, /* (o) Decoded vector */ + const int16_t* index, /* (i) Codebook indices */ + const int16_t* gain_index, /* (i) Gain quantization indices */ + int16_t* mem, /* (i) Buffer for codevector construction */ + size_t lMem, /* (i) Length of buffer */ + size_t veclen) { /* (i) Length of vector */ + size_t j; + int16_t gain[CB_NSTAGES]; + /* Stack based */ + int16_t cbvec0[SUBL]; + int16_t cbvec1[SUBL]; + int16_t cbvec2[SUBL]; + int32_t a32; + int16_t *gainPtr; + + /* gain de-quantization */ + + gain[0] = WebRtcIlbcfix_GainDequant(gain_index[0], 16384, 0); + gain[1] = WebRtcIlbcfix_GainDequant(gain_index[1], gain[0], 1); + gain[2] = WebRtcIlbcfix_GainDequant(gain_index[2], gain[1], 2); + + /* codebook vector construction and construction of total vector */ + + /* Stack based */ + if (!WebRtcIlbcfix_GetCbVec(cbvec0, mem, (size_t)index[0], lMem, veclen)) + return false; // Failure. + if (!WebRtcIlbcfix_GetCbVec(cbvec1, mem, (size_t)index[1], lMem, veclen)) + return false; // Failure. + if (!WebRtcIlbcfix_GetCbVec(cbvec2, mem, (size_t)index[2], lMem, veclen)) + return false; // Failure. + + gainPtr = &gain[0]; + for (j=0;j> 14); + } + + return true; // Success. +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.h new file mode 100644 index 0000000000..aeb00da5f1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbConstruct.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_CONSTRUCT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_CONSTRUCT_H_ + +#include +#include "defines.h" + +/*----------------------------------------------------------------* + * Construct decoded vector from codebook and gains. + *---------------------------------------------------------------*/ + +// Returns true on success, false on failure. +bool WebRtcIlbcfix_CbConstruct( + int16_t* decvector, /* (o) Decoded vector */ + const int16_t* index, /* (i) Codebook indices */ + const int16_t* gain_index, /* (i) Gain quantization indices */ + int16_t* mem, /* (i) Buffer for codevector construction */ + size_t lMem, /* (i) Length of buffer */ + size_t veclen /* (i) Length of vector */ + ) RTC_WARN_UNUSED_RESULT; + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c new file mode 100644 index 0000000000..6ad2f8eb7a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergy.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "cb_mem_energy_calc.h" + +/*----------------------------------------------------------------* + * Function WebRtcIlbcfix_CbMemEnergy computes the energy of all + * the vectors in the codebook memory that will be used in the + * following search for the best match. + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CbMemEnergy( + size_t range, + int16_t *CB, /* (i) The CB memory (1:st section) */ + int16_t *filteredCB, /* (i) The filtered CB memory (2:nd section) */ + size_t lMem, /* (i) Length of the CB memory */ + size_t lTarget, /* (i) Length of the target vector */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ) { + int16_t *ppi, *ppo, *pp; + int32_t energy, tmp32; + + /* Compute the energy and store it in a vector. Also the + * corresponding shift values are stored. The energy values + * are reused in all three stages. */ + + /* Calculate the energy in the first block of 'lTarget' sampels. */ + ppi = CB+lMem-lTarget-1; + ppo = CB+lMem-1; + + pp=CB+lMem-lTarget; + energy = WebRtcSpl_DotProductWithScale( pp, pp, lTarget, scale); + + /* Normalize the energy and store the number of shifts */ + energyShifts[0] = (int16_t)WebRtcSpl_NormW32(energy); + tmp32 = energy << energyShifts[0]; + energyW16[0] = (int16_t)(tmp32 >> 16); + + /* Compute the energy of the rest of the cb memory + * by step wise adding and subtracting the next + * sample and the last sample respectively. */ + WebRtcIlbcfix_CbMemEnergyCalc(energy, range, ppi, ppo, energyW16, energyShifts, scale, 0); + + /* Next, precompute the energy values for the filtered cb section */ + energy=0; + pp=filteredCB+lMem-lTarget; + + energy = WebRtcSpl_DotProductWithScale( pp, pp, lTarget, scale); + + /* Normalize the energy and store the number of shifts */ + energyShifts[base_size] = (int16_t)WebRtcSpl_NormW32(energy); + tmp32 = energy << energyShifts[base_size]; + energyW16[base_size] = (int16_t)(tmp32 >> 16); + + ppi = filteredCB + lMem - 1 - lTarget; + ppo = filteredCB + lMem - 1; + + WebRtcIlbcfix_CbMemEnergyCalc(energy, range, ppi, ppo, energyW16, energyShifts, scale, base_size); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h new file mode 100644 index 0000000000..e8e2fe979a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergy.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_H_ + +void WebRtcIlbcfix_CbMemEnergy( + size_t range, + int16_t *CB, /* (i) The CB memory (1:st section) */ + int16_t *filteredCB, /* (i) The filtered CB memory (2:nd section) */ + size_t lMem, /* (i) Length of the CB memory */ + size_t lTarget, /* (i) Length of the target vector */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c new file mode 100644 index 0000000000..acd6b9c118 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyAugmentation.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +void WebRtcIlbcfix_CbMemEnergyAugmentation( + int16_t *interpSamples, /* (i) The interpolated samples */ + int16_t *CBmem, /* (i) The CB memory */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size, /* (i) Index to where energy values should be stored */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts /* (o) Shift value of the energy */ + ){ + int32_t energy, tmp32; + int16_t *ppe, *pp, *interpSamplesPtr; + int16_t *CBmemPtr; + size_t lagcount; + int16_t *enPtr=&energyW16[base_size-20]; + int16_t *enShPtr=&energyShifts[base_size-20]; + int32_t nrjRecursive; + + CBmemPtr = CBmem+147; + interpSamplesPtr = interpSamples; + + /* Compute the energy for the first (low-5) noninterpolated samples */ + nrjRecursive = WebRtcSpl_DotProductWithScale( CBmemPtr-19, CBmemPtr-19, 15, scale); + ppe = CBmemPtr - 20; + + for (lagcount=20; lagcount<=39; lagcount++) { + + /* Update the energy recursively to save complexity */ + nrjRecursive += (*ppe * *ppe) >> scale; + ppe--; + energy = nrjRecursive; + + /* interpolation */ + energy += WebRtcSpl_DotProductWithScale(interpSamplesPtr, interpSamplesPtr, 4, scale); + interpSamplesPtr += 4; + + /* Compute energy for the remaining samples */ + pp = CBmemPtr - lagcount; + energy += WebRtcSpl_DotProductWithScale(pp, pp, SUBL-lagcount, scale); + + /* Normalize the energy and store the number of shifts */ + (*enShPtr) = (int16_t)WebRtcSpl_NormW32(energy); + tmp32 = energy << *enShPtr; + *enPtr = (int16_t)(tmp32 >> 16); + enShPtr++; + enPtr++; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h new file mode 100644 index 0000000000..00eb0179f1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyAugmentation.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_AUGMENTATION_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_AUGMENTATION_H_ + +void WebRtcIlbcfix_CbMemEnergyAugmentation( + int16_t *interpSamples, /* (i) The interpolated samples */ + int16_t *CBmem, /* (i) The CB memory */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size, /* (i) Index to where energy values should be stored */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts /* (o) Shift value of the energy */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c new file mode 100644 index 0000000000..f2415febc0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyCalc.c + +******************************************************************/ + +#include "defines.h" + +/* Compute the energy of the rest of the cb memory + * by step wise adding and subtracting the next + * sample and the last sample respectively */ +void WebRtcIlbcfix_CbMemEnergyCalc( + int32_t energy, /* (i) input start energy */ + size_t range, /* (i) number of iterations */ + int16_t *ppi, /* (i) input pointer 1 */ + int16_t *ppo, /* (i) input pointer 2 */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ) +{ + size_t j; + int16_t shft; + int32_t tmp; + int16_t *eSh_ptr; + int16_t *eW16_ptr; + + + eSh_ptr = &energyShifts[1+base_size]; + eW16_ptr = &energyW16[1+base_size]; + + for (j = 0; j + 1 < range; j++) { + + /* Calculate next energy by a +/- + operation on the edge samples */ + tmp = (*ppi) * (*ppi) - (*ppo) * (*ppo); + energy += tmp >> scale; + energy = WEBRTC_SPL_MAX(energy, 0); + + ppi--; + ppo--; + + /* Normalize the energy into a int16_t and store + the number of shifts */ + + shft = (int16_t)WebRtcSpl_NormW32(energy); + *eSh_ptr++ = shft; + + tmp = energy << shft; + *eW16_ptr++ = (int16_t)(tmp >> 16); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h new file mode 100644 index 0000000000..af8e658a02 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyCalc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_CALC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_CALC_H_ + +void WebRtcIlbcfix_CbMemEnergyCalc( + int32_t energy, /* (i) input start energy */ + size_t range, /* (i) number of iterations */ + int16_t *ppi, /* (i) input pointer 1 */ + int16_t *ppo, /* (i) input pointer 2 */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.c new file mode 100644 index 0000000000..be94951449 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.c @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearch.c + +******************************************************************/ + +#include "defines.h" +#include "gain_quant.h" +#include "filtered_cb_vecs.h" +#include "constants.h" +#include "cb_mem_energy.h" +#include "interpolate_samples.h" +#include "cb_mem_energy_augmentation.h" +#include "cb_search_core.h" +#include "energy_inverse.h" +#include "augmented_cb_corr.h" +#include "cb_update_best_index.h" +#include "create_augmented_vec.h" + +/*----------------------------------------------------------------* + * Search routine for codebook encoding and gain quantization. + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CbSearch( + IlbcEncoder *iLBCenc_inst, + /* (i) the encoder state structure */ + int16_t *index, /* (o) Codebook indices */ + int16_t *gain_index, /* (o) Gain quantization indices */ + int16_t *intarget, /* (i) Target vector for encoding */ + int16_t *decResidual,/* (i) Decoded residual for codebook construction */ + size_t lMem, /* (i) Length of buffer */ + size_t lTarget, /* (i) Length of vector */ + int16_t *weightDenum,/* (i) weighting filter coefficients in Q12 */ + size_t block /* (i) the subblock number */ + ) { + size_t i, range; + int16_t ii, j, stage; + int16_t *pp; + int16_t tmp; + int scale; + int16_t bits, temp1, temp2; + size_t base_size; + int32_t codedEner, targetEner; + int16_t gains[CB_NSTAGES+1]; + int16_t *cb_vecPtr; + size_t indexOffset, sInd, eInd; + int32_t CritMax=0; + int16_t shTotMax=WEBRTC_SPL_WORD16_MIN; + size_t bestIndex=0; + int16_t bestGain=0; + size_t indexNew; + int16_t CritNewSh; + int32_t CritNew; + int32_t *cDotPtr; + size_t noOfZeros; + int16_t *gainPtr; + int32_t t32, tmpW32; + int16_t *WebRtcIlbcfix_kGainSq5_ptr; + /* Stack based */ + int16_t CBbuf[CB_MEML+LPC_FILTERORDER+CB_HALFFILTERLEN]; + int32_t cDot[128]; + int32_t Crit[128]; + int16_t targetVec[SUBL+LPC_FILTERORDER]; + int16_t cbvectors[CB_MEML + 1]; /* Adding one extra position for + Coverity warnings. */ + int16_t codedVec[SUBL]; + int16_t interpSamples[20*4]; + int16_t interpSamplesFilt[20*4]; + int16_t energyW16[CB_EXPAND*128]; + int16_t energyShifts[CB_EXPAND*128]; + int16_t *inverseEnergy=energyW16; /* Reuse memory */ + int16_t *inverseEnergyShifts=energyShifts; /* Reuse memory */ + int16_t *buf = &CBbuf[LPC_FILTERORDER]; + int16_t *target = &targetVec[LPC_FILTERORDER]; + int16_t *aug_vec = (int16_t*)cDot; /* length [SUBL], reuse memory */ + + /* Determine size of codebook sections */ + + base_size=lMem-lTarget+1; + if (lTarget==SUBL) { + base_size=lMem-19; + } + + /* weighting of the CB memory */ + noOfZeros=lMem-WebRtcIlbcfix_kFilterRange[block]; + WebRtcSpl_MemSetW16(&buf[-LPC_FILTERORDER], 0, noOfZeros+LPC_FILTERORDER); + WebRtcSpl_FilterARFastQ12( + decResidual+noOfZeros, buf+noOfZeros, + weightDenum, LPC_FILTERORDER+1, WebRtcIlbcfix_kFilterRange[block]); + + /* weighting of the target vector */ + WEBRTC_SPL_MEMCPY_W16(&target[-LPC_FILTERORDER], buf+noOfZeros+WebRtcIlbcfix_kFilterRange[block]-LPC_FILTERORDER, LPC_FILTERORDER); + WebRtcSpl_FilterARFastQ12( + intarget, target, + weightDenum, LPC_FILTERORDER+1, lTarget); + + /* Store target, towards the end codedVec is calculated as + the initial target minus the remaining target */ + WEBRTC_SPL_MEMCPY_W16(codedVec, target, lTarget); + + /* Find the highest absolute value to calculate proper + vector scale factor (so that it uses 12 bits) */ + temp1 = WebRtcSpl_MaxAbsValueW16(buf, lMem); + temp2 = WebRtcSpl_MaxAbsValueW16(target, lTarget); + + if ((temp1>0)&&(temp2>0)) { + temp1 = WEBRTC_SPL_MAX(temp1, temp2); + scale = WebRtcSpl_GetSizeInBits((uint32_t)(temp1 * temp1)); + } else { + /* temp1 or temp2 is negative (maximum was -32768) */ + scale = 30; + } + + /* Scale to so that a mul-add 40 times does not overflow */ + scale = scale - 25; + scale = WEBRTC_SPL_MAX(0, scale); + + /* Compute energy of the original target */ + targetEner = WebRtcSpl_DotProductWithScale(target, target, lTarget, scale); + + /* Prepare search over one more codebook section. This section + is created by filtering the original buffer with a filter. */ + WebRtcIlbcfix_FilteredCbVecs(cbvectors, buf, lMem, WebRtcIlbcfix_kFilterRange[block]); + + range = WebRtcIlbcfix_kSearchRange[block][0]; + + if(lTarget == SUBL) { + /* Create the interpolated samples and store them for use in all stages */ + + /* First section, non-filtered half of the cb */ + WebRtcIlbcfix_InterpolateSamples(interpSamples, buf, lMem); + + /* Second section, filtered half of the cb */ + WebRtcIlbcfix_InterpolateSamples(interpSamplesFilt, cbvectors, lMem); + + /* Compute the CB vectors' energies for the first cb section (non-filtered) */ + WebRtcIlbcfix_CbMemEnergyAugmentation(interpSamples, buf, + scale, 20, energyW16, energyShifts); + + /* Compute the CB vectors' energies for the second cb section (filtered cb) */ + WebRtcIlbcfix_CbMemEnergyAugmentation(interpSamplesFilt, cbvectors, scale, + base_size + 20, energyW16, + energyShifts); + + /* Compute the CB vectors' energies and store them in the vector + * energyW16. Also the corresponding shift values are stored. The + * energy values are used in all three stages. */ + WebRtcIlbcfix_CbMemEnergy(range, buf, cbvectors, lMem, + lTarget, energyW16+20, energyShifts+20, scale, base_size); + + } else { + /* Compute the CB vectors' energies and store them in the vector + * energyW16. Also the corresponding shift values are stored. The + * energy values are used in all three stages. */ + WebRtcIlbcfix_CbMemEnergy(range, buf, cbvectors, lMem, + lTarget, energyW16, energyShifts, scale, base_size); + + /* Set the energy positions 58-63 and 122-127 to zero + (otherwise they are uninitialized) */ + WebRtcSpl_MemSetW16(energyW16+range, 0, (base_size-range)); + WebRtcSpl_MemSetW16(energyW16+range+base_size, 0, (base_size-range)); + } + + /* Calculate Inverse Energy (energyW16 is already normalized + and will contain the inverse energy in Q29 after this call */ + WebRtcIlbcfix_EnergyInverse(energyW16, base_size*CB_EXPAND); + + /* The gain value computed in the previous stage is used + * as an upper limit to what the next stage gain value + * is allowed to be. In stage 0, 16384 (1.0 in Q14) is used as + * the upper limit. */ + gains[0] = 16384; + + for (stage=0; stage> 1) > bestIndex) ? + 0 : (bestIndex - (CB_RESRANGE >> 1)); + eInd=sInd+CB_RESRANGE; + if (eInd>=range) { + eInd=range-1; + sInd=eInd-CB_RESRANGE; + } + + range = WebRtcIlbcfix_kSearchRange[block][stage]; + + if (lTarget==SUBL) { + i=sInd; + if (sInd<20) { + WebRtcIlbcfix_AugmentedCbCorr(target, cbvectors + lMem, + interpSamplesFilt, cDot, sInd + 20, + WEBRTC_SPL_MIN(39, (eInd + 20)), scale); + i=20; + cDotPtr = &cDot[20 - sInd]; + } else { + cDotPtr = cDot; + } + + cb_vecPtr = cbvectors+lMem-20-i; + + /* Calculate the cross correlations (main part of the filtered CB) */ + WebRtcSpl_CrossCorrelation(cDotPtr, target, cb_vecPtr, lTarget, + eInd - i + 1, scale, -1); + + } else { + cDotPtr = cDot; + cb_vecPtr = cbvectors+lMem-lTarget-sInd; + + /* Calculate the cross correlations (main part of the filtered CB) */ + WebRtcSpl_CrossCorrelation(cDotPtr, target, cb_vecPtr, lTarget, + eInd - sInd + 1, scale, -1); + + } + + /* Adjust the search range for the augmented vectors */ + indexOffset=base_size+sInd; + + /* Search for best index in this part of the vector */ + WebRtcIlbcfix_CbSearchCore( + cDot, eInd-sInd+1, stage, inverseEnergy+indexOffset, + inverseEnergyShifts+indexOffset, Crit, + &indexNew, &CritNew, &CritNewSh); + + /* Update the global best index and the corresponding gain */ + WebRtcIlbcfix_CbUpdateBestIndex( + CritNew, CritNewSh, indexNew+indexOffset, cDot[indexNew], + inverseEnergy[indexNew+indexOffset], inverseEnergyShifts[indexNew+indexOffset], + &CritMax, &shTotMax, &bestIndex, &bestGain); + + index[stage] = (int16_t)bestIndex; + + + bestGain = WebRtcIlbcfix_GainQuant(bestGain, + (int16_t)WEBRTC_SPL_ABS_W16(gains[stage]), stage, &gain_index[stage]); + + /* Extract the best (according to measure) codebook vector + Also adjust the index, so that the augmented vectors are last. + Above these vectors were first... + */ + + if(lTarget==(STATE_LEN-iLBCenc_inst->state_short_len)) { + + if((size_t)index[stage]=20) { + /* Adjust index and extract vector */ + index[stage]-=20; + pp=buf+lMem-lTarget-index[stage]; + } else { + /* Adjust index and extract vector */ + index[stage]+=(int16_t)(base_size-20); + + WebRtcIlbcfix_CreateAugmentedVec(index[stage]-base_size+40, + buf+lMem, aug_vec); + pp = aug_vec; + + } + } else { + + if ((index[stage] - base_size) >= 20) { + /* Adjust index and extract vector */ + index[stage]-=20; + pp=cbvectors+lMem-lTarget- + index[stage]+base_size; + } else { + /* Adjust index and extract vector */ + index[stage]+=(int16_t)(base_size-20); + WebRtcIlbcfix_CreateAugmentedVec(index[stage]-2*base_size+40, + cbvectors+lMem, aug_vec); + pp = aug_vec; + } + } + } + + /* Subtract the best codebook vector, according + to measure, from the target vector */ + + WebRtcSpl_AddAffineVectorToVector(target, pp, (int16_t)(-bestGain), + (int32_t)8192, (int16_t)14, lTarget); + + /* record quantized gain */ + gains[stage+1] = bestGain; + + } /* end of Main Loop. for (stage=0;... */ + + /* Calculte the coded vector (original target - what's left) */ + for (i=0;i> 14); + + targetEner = (int16_t)WEBRTC_SPL_SHIFT_W32(targetEner, -bits) * tmp; + + tmpW32 = ((int32_t)(gains[1]-1))<<1; + + /* Pointer to the table that contains + gain_sq5TblFIX * gain_sq5TblFIX in Q14 */ + gainPtr=(int16_t*)WebRtcIlbcfix_kGainSq5Sq+gain_index[0]; + temp1 = (int16_t)WEBRTC_SPL_SHIFT_W32(codedEner, -bits); + + WebRtcIlbcfix_kGainSq5_ptr = (int16_t*)&WebRtcIlbcfix_kGainSq5[j]; + + /* targetEner and codedEner are in Q(-2*scale) */ + for (ii=gain_index[0];ii<32;ii++) { + + /* Change the index if + (codedEnergy*gainTbl[i]*gainTbl[i])<(targetEn*gain[0]*gain[0]) AND + gainTbl[i] < 2*gain[0] + */ + + t32 = temp1 * *gainPtr; + t32 = t32 - targetEner; + if (t32 < 0) { + if ((*WebRtcIlbcfix_kGainSq5_ptr) < tmpW32) { + j=ii; + WebRtcIlbcfix_kGainSq5_ptr = (int16_t*)&WebRtcIlbcfix_kGainSq5[ii]; + } + } + gainPtr++; + } + gain_index[0]=j; + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.h new file mode 100644 index 0000000000..c8626c5153 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearch.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_H_ + +void WebRtcIlbcfix_CbSearch( + IlbcEncoder *iLBCenc_inst, + /* (i) the encoder state structure */ + int16_t *index, /* (o) Codebook indices */ + int16_t *gain_index, /* (o) Gain quantization indices */ + int16_t *intarget, /* (i) Target vector for encoding */ + int16_t *decResidual,/* (i) Decoded residual for codebook construction */ + size_t lMem, /* (i) Length of buffer */ + size_t lTarget, /* (i) Length of vector */ + int16_t *weightDenum,/* (i) weighting filter coefficients in Q12 */ + size_t block /* (i) the subblock number */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c new file mode 100644 index 0000000000..fafa39f69b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearchCore.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +void WebRtcIlbcfix_CbSearchCore( + int32_t *cDot, /* (i) Cross Correlation */ + size_t range, /* (i) Search range */ + int16_t stage, /* (i) Stage of this search */ + int16_t *inverseEnergy, /* (i) Inversed energy */ + int16_t *inverseEnergyShift, /* (i) Shifts of inversed energy + with the offset 2*16-29 */ + int32_t *Crit, /* (o) The criteria */ + size_t *bestIndex, /* (o) Index that corresponds to + maximum criteria (in this + vector) */ + int32_t *bestCrit, /* (o) Value of critera for the + chosen index */ + int16_t *bestCritSh) /* (o) The domain of the chosen + criteria */ +{ + int32_t maxW32, tmp32; + int16_t max, sh, tmp16; + size_t i; + int32_t *cDotPtr; + int16_t cDotSqW16; + int16_t *inverseEnergyPtr; + int32_t *critPtr; + int16_t *inverseEnergyShiftPtr; + + /* Don't allow negative values for stage 0 */ + if (stage==0) { + cDotPtr=cDot; + for (i=0;i> 16); + cDotSqW16 = (int16_t)(((int32_t)(tmp16)*(tmp16))>>16); + + /* Calculate the criteria (cDot*cDot/energy) */ + *critPtr = cDotSqW16 * *inverseEnergyPtr; + + /* Extract the maximum shift value under the constraint + that the criteria is not zero */ + if ((*critPtr)!=0) { + max = WEBRTC_SPL_MAX((*inverseEnergyShiftPtr), max); + } + + inverseEnergyPtr++; + inverseEnergyShiftPtr++; + critPtr++; + cDotPtr++; + } + + /* If no max shifts still at initialization value, set shift to zero */ + if (max==WEBRTC_SPL_WORD16_MIN) { + max = 0; + } + + /* Modify the criterias, so that all of them use the same Q domain */ + critPtr=Crit; + inverseEnergyShiftPtr=inverseEnergyShift; + for (i=0;i31) */ + tmp16 = WEBRTC_SPL_MIN(16, max-(*inverseEnergyShiftPtr)); + + (*critPtr)=WEBRTC_SPL_SHIFT_W32((*critPtr),-tmp16); + critPtr++; + inverseEnergyShiftPtr++; + } + + /* Find the index of the best value */ + *bestIndex = WebRtcSpl_MaxIndexW32(Crit, range); + *bestCrit = Crit[*bestIndex]; + + /* Calculate total shifts of this criteria */ + *bestCritSh = 32 - 2*sh + max; + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h new file mode 100644 index 0000000000..1db7f70c54 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearchCore.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_CORE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_CORE_H_ + +#include "defines.h" + +void WebRtcIlbcfix_CbSearchCore( + int32_t *cDot, /* (i) Cross Correlation */ + size_t range, /* (i) Search range */ + int16_t stage, /* (i) Stage of this search */ + int16_t *inverseEnergy, /* (i) Inversed energy */ + int16_t *inverseEnergyShift, /* (i) Shifts of inversed energy + with the offset 2*16-29 */ + int32_t *Crit, /* (o) The criteria */ + size_t *bestIndex, /* (o) Index that corresponds to + maximum criteria (in this + vector) */ + int32_t *bestCrit, /* (o) Value of critera for the + chosen index */ + int16_t *bestCritSh); /* (o) The domain of the chosen + criteria */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c new file mode 100644 index 0000000000..fc27ea9f6c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbUpdateBestIndex.c + +******************************************************************/ + +#include "defines.h" +#include "cb_update_best_index.h" +#include "constants.h" + +void WebRtcIlbcfix_CbUpdateBestIndex( + int32_t CritNew, /* (i) New Potentially best Criteria */ + int16_t CritNewSh, /* (i) Shift value of above Criteria */ + size_t IndexNew, /* (i) Index of new Criteria */ + int32_t cDotNew, /* (i) Cross dot of new index */ + int16_t invEnergyNew, /* (i) Inversed energy new index */ + int16_t energyShiftNew, /* (i) Energy shifts of new index */ + int32_t *CritMax, /* (i/o) Maximum Criteria (so far) */ + int16_t *shTotMax, /* (i/o) Shifts of maximum criteria */ + size_t *bestIndex, /* (i/o) Index that corresponds to + maximum criteria */ + int16_t *bestGain) /* (i/o) Gain in Q14 that corresponds + to maximum criteria */ +{ + int16_t shOld, shNew, tmp16; + int16_t scaleTmp; + int32_t gainW32; + + /* Normalize the new and old Criteria to the same domain */ + if (CritNewSh>(*shTotMax)) { + shOld=WEBRTC_SPL_MIN(31,CritNewSh-(*shTotMax)); + shNew=0; + } else { + shOld=0; + shNew=WEBRTC_SPL_MIN(31,(*shTotMax)-CritNewSh); + } + + /* Compare the two criterias. If the new one is better, + calculate the gain and store this index as the new best one + */ + + if ((CritNew >> shNew) > (*CritMax >> shOld)) { + + tmp16 = (int16_t)WebRtcSpl_NormW32(cDotNew); + tmp16 = 16 - tmp16; + + /* Calculate the gain in Q14 + Compensate for inverseEnergyshift in Q29 and that the energy + value was stored in a int16_t (shifted down 16 steps) + => 29-14+16 = 31 */ + + scaleTmp = -energyShiftNew-tmp16+31; + scaleTmp = WEBRTC_SPL_MIN(31, scaleTmp); + + gainW32 = ((int16_t)WEBRTC_SPL_SHIFT_W32(cDotNew, -tmp16) * invEnergyNew) >> + scaleTmp; + + /* Check if criteria satisfies Gain criteria (max 1.3) + if it is larger set the gain to 1.3 + (slightly different from FLP version) + */ + if (gainW32>21299) { + *bestGain=21299; + } else if (gainW32<-21299) { + *bestGain=-21299; + } else { + *bestGain=(int16_t)gainW32; + } + + *CritMax=CritNew; + *shTotMax=CritNewSh; + *bestIndex = IndexNew; + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h new file mode 100644 index 0000000000..4929b640c4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbUpdateBestIndex.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_UPDATE_BEST_INDEX_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_UPDATE_BEST_INDEX_H_ + +#include "defines.h" + +void WebRtcIlbcfix_CbUpdateBestIndex( + int32_t CritNew, /* (i) New Potentially best Criteria */ + int16_t CritNewSh, /* (i) Shift value of above Criteria */ + size_t IndexNew, /* (i) Index of new Criteria */ + int32_t cDotNew, /* (i) Cross dot of new index */ + int16_t invEnergyNew, /* (i) Inversed energy new index */ + int16_t energyShiftNew, /* (i) Energy shifts of new index */ + int32_t *CritMax, /* (i/o) Maximum Criteria (so far) */ + int16_t *shTotMax, /* (i/o) Shifts of maximum criteria */ + size_t *bestIndex, /* (i/o) Index that corresponds to + maximum criteria */ + int16_t *bestGain); /* (i/o) Gain in Q14 that corresponds + to maximum criteria */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.c new file mode 100644 index 0000000000..21a8f40df6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.c @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Chebyshev.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*------------------------------------------------------------------* + * Calculate the Chevyshev polynomial series + * F(w) = 2*exp(-j5w)*C(x) + * C(x) = (T_0(x) + f(1)T_1(x) + ... + f(4)T_1(x) + f(5)/2) + * T_i(x) is the i:th order Chebyshev polynomial + *------------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_Chebyshev( + /* (o) Result of C(x) */ + int16_t x, /* (i) Value to the Chevyshev polynomial */ + int16_t *f /* (i) The coefficients in the polynomial */ + ) { + int16_t b1_high, b1_low; /* Use the high, low format to increase the accuracy */ + int32_t b2; + int32_t tmp1W32; + int32_t tmp2W32; + int i; + + b2 = (int32_t)0x1000000; /* b2 = 1.0 (Q23) */ + /* Calculate b1 = 2*x + f[1] */ + tmp1W32 = (x << 10) + (f[1] << 14); + + for (i = 2; i < 5; i++) { + tmp2W32 = tmp1W32; + + /* Split b1 (in tmp1W32) into a high and low part */ + b1_high = (int16_t)(tmp1W32 >> 16); + b1_low = (int16_t)((tmp1W32 - ((int32_t)b1_high << 16)) >> 1); + + /* Calculate 2*x*b1-b2+f[i] */ + tmp1W32 = ((b1_high * x + ((b1_low * x) >> 15)) << 2) - b2 + (f[i] << 14); + + /* Update b2 for next round */ + b2 = tmp2W32; + } + + /* Split b1 (in tmp1W32) into a high and low part */ + b1_high = (int16_t)(tmp1W32 >> 16); + b1_low = (int16_t)((tmp1W32 - ((int32_t)b1_high << 16)) >> 1); + + /* tmp1W32 = x*b1 - b2 + f[i]/2 */ + tmp1W32 = ((b1_high * x) << 1) + (((b1_low * x) >> 15) << 1) - + b2 + (f[i] << 13); + + /* Handle overflows and set to maximum or minimum int16_t instead */ + if (tmp1W32>((int32_t)33553408)) { + return(WEBRTC_SPL_WORD16_MAX); + } else if (tmp1W32<((int32_t)-33554432)) { + return(WEBRTC_SPL_WORD16_MIN); + } else { + return (int16_t)(tmp1W32 >> 10); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.h new file mode 100644 index 0000000000..f990d0b9da --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Chebyshev.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CHEBYSHEV_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CHEBYSHEV_H_ + +#include "defines.h" + +/*------------------------------------------------------------------* + * Calculate the Chevyshev polynomial series + * F(w) = 2*exp(-j5w)*C(x) + * C(x) = (T_0(x) + f(1)T_1(x) + ... + f(4)T_1(x) + f(5)/2) + * T_i(x) is the i:th order Chebyshev polynomial + *------------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_Chebyshev( + /* (o) Result of C(x) */ + int16_t x, /* (i) Value to the Chevyshev polynomial */ + int16_t *f /* (i) The coefficients in the polynomial */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.c new file mode 100644 index 0000000000..7653cb0c25 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CompCorr.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Compute cross correlation and pitch gain for pitch prediction + * of last subframe at given lag. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_CompCorr( + int32_t *corr, /* (o) cross correlation */ + int32_t *ener, /* (o) energy */ + int16_t *buffer, /* (i) signal buffer */ + size_t lag, /* (i) pitch lag */ + size_t bLen, /* (i) length of buffer */ + size_t sRange, /* (i) correlation search length */ + int16_t scale /* (i) number of rightshifts to use */ + ){ + int16_t *w16ptr; + + w16ptr=&buffer[bLen-sRange-lag]; + + /* Calculate correlation and energy */ + (*corr)=WebRtcSpl_DotProductWithScale(&buffer[bLen-sRange], w16ptr, sRange, scale); + (*ener)=WebRtcSpl_DotProductWithScale(w16ptr, w16ptr, sRange, scale); + + /* For zero energy set the energy to 0 in order to avoid potential + problems for coming divisions */ + if (*ener == 0) { + *corr = 0; + *ener = 1; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.h new file mode 100644 index 0000000000..6f6c5ec11c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CompCorr.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_COMP_CORR_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_COMP_CORR_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Compute cross correlation and pitch gain for pitch prediction + * of last subframe at given lag. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_CompCorr( + int32_t *corr, /* (o) cross correlation */ + int32_t *ener, /* (o) energy */ + int16_t *buffer, /* (i) signal buffer */ + size_t lag, /* (i) pitch lag */ + size_t bLen, /* (i) length of buffer */ + size_t sRange, /* (i) correlation search length */ + int16_t scale /* (i) number of rightshifts to use */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m new file mode 100644 index 0000000000..5c39e3615c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m @@ -0,0 +1,59 @@ +% +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +clear; +pack; +% +% Enter the path to YOUR executable and remember to define the perprocessor +% variable PRINT_MIPS te get the instructions printed to the screen. +% +command = '!iLBCtest.exe 30 speechAndBGnoise.pcm out1.bit out1.pcm tlm10_30ms.dat'; +cout=' > st.txt'; %saves to matlab variable 'st' +eval(strcat(command,cout)); +if(length(cout)>3) + load st.txt +else + disp('No cout file to load') +end + +% initialize vector to zero +index = find(st(1:end,1)==-1); +indexnonzero = find(st(1:end,1)>0); +frames = length(index)-indexnonzero(1)+1; +start = indexnonzero(1) - 1; +functionOrder=max(st(:,2)); +new=zeros(frames,functionOrder); + +for i = 1:frames, + for j = index(start-1+i)+1:(index(start+i)-1), + new(i,st(j,2)) = new(i,st(j,2)) + st(j,1); + end +end + +result=zeros(functionOrder,3); +for i=1:functionOrder + nonzeroelements = find(new(1:end,i)>0); + result(i,1)=i; + + % Compute each function's mean complexity + % result(i,2)=(sum(new(nonzeroelements,i))/(length(nonzeroelements)*0.03))/1000000; + + % Compute each function's maximum complexity in encoding + % and decoding respectively and then add it together: + % result(i,3)=(max(new(1:end,i))/0.03)/1000000; + result(i,3)=(max(new(1:size(new,1)/2,i))/0.03)/1000000 + (max(new(size(new,1)/2+1:end,i))/0.03)/1000000; +end + +result + +% Compute maximum complexity for a single frame (enc/dec separately and together) +maxEncComplexityInAFrame = (max(sum(new(1:size(new,1)/2,:),2))/0.03)/1000000 +maxDecComplexityInAFrame = (max(sum(new(size(new,1)/2+1:end,:),2))/0.03)/1000000 +totalComplexity = maxEncComplexityInAFrame + maxDecComplexityInAFrame \ No newline at end of file diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.c new file mode 100644 index 0000000000..9e341942e6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.c @@ -0,0 +1,666 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + constants.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/* HP Filters {b[0] b[1] b[2] -a[1] -a[2]} */ + +const int16_t WebRtcIlbcfix_kHpInCoefs[5] = {3798, -7596, 3798, 7807, -3733}; +const int16_t WebRtcIlbcfix_kHpOutCoefs[5] = {3849, -7699, 3849, 7918, -3833}; + +/* Window in Q11 to window the energies of the 5 choises (3 for 20ms) in the choise for + the 80 sample start state +*/ +const int16_t WebRtcIlbcfix_kStartSequenceEnrgWin[NSUB_MAX-1]= { + 1638, 1843, 2048, 1843, 1638 +}; + +/* LP Filter coeffs used for downsampling */ +const int16_t WebRtcIlbcfix_kLpFiltCoefs[FILTERORDER_DS_PLUS1]= { + -273, 512, 1297, 1696, 1297, 512, -273 +}; + +/* Constants used in the LPC calculations */ + +/* Hanning LPC window (in Q15) */ +const int16_t WebRtcIlbcfix_kLpcWin[BLOCKL_MAX] = { + 6, 22, 50, 89, 139, 200, 272, 355, 449, 554, 669, 795, + 932, 1079, 1237, 1405, 1583, 1771, 1969, 2177, 2395, 2622, 2858, 3104, + 3359, 3622, 3894, 4175, 4464, 4761, 5066, 5379, 5699, 6026, 6361, 6702, + 7050, 7404, 7764, 8130, 8502, 8879, 9262, 9649, 10040, 10436, 10836, 11240, + 11647, 12058, 12471, 12887, 13306, 13726, 14148, 14572, 14997, 15423, 15850, 16277, + 16704, 17131, 17558, 17983, 18408, 18831, 19252, 19672, 20089, 20504, 20916, 21325, + 21730, 22132, 22530, 22924, 23314, 23698, 24078, 24452, 24821, 25185, 25542, 25893, + 26238, 26575, 26906, 27230, 27547, 27855, 28156, 28450, 28734, 29011, 29279, 29538, + 29788, 30029, 30261, 30483, 30696, 30899, 31092, 31275, 31448, 31611, 31764, 31906, + 32037, 32158, 32268, 32367, 32456, 32533, 32600, 32655, 32700, 32733, 32755, 32767, + 32767, 32755, 32733, 32700, 32655, 32600, 32533, 32456, 32367, 32268, 32158, 32037, + 31906, 31764, 31611, 31448, 31275, 31092, 30899, 30696, 30483, 30261, 30029, 29788, + 29538, 29279, 29011, 28734, 28450, 28156, 27855, 27547, 27230, 26906, 26575, 26238, + 25893, 25542, 25185, 24821, 24452, 24078, 23698, 23314, 22924, 22530, 22132, 21730, + 21325, 20916, 20504, 20089, 19672, 19252, 18831, 18408, 17983, 17558, 17131, 16704, + 16277, 15850, 15423, 14997, 14572, 14148, 13726, 13306, 12887, 12471, 12058, 11647, + 11240, 10836, 10436, 10040, 9649, 9262, 8879, 8502, 8130, 7764, 7404, 7050, + 6702, 6361, 6026, 5699, 5379, 5066, 4761, 4464, 4175, 3894, 3622, 3359, + 3104, 2858, 2622, 2395, 2177, 1969, 1771, 1583, 1405, 1237, 1079, 932, + 795, 669, 554, 449, 355, 272, 200, 139, 89, 50, 22, 6 +}; + +/* Asymmetric LPC window (in Q15)*/ +const int16_t WebRtcIlbcfix_kLpcAsymWin[BLOCKL_MAX] = { + 2, 7, 15, 27, 42, 60, 81, 106, 135, 166, 201, 239, + 280, 325, 373, 424, 478, 536, 597, 661, 728, 798, 872, 949, + 1028, 1111, 1197, 1287, 1379, 1474, 1572, 1674, 1778, 1885, 1995, 2108, + 2224, 2343, 2465, 2589, 2717, 2847, 2980, 3115, 3254, 3395, 3538, 3684, + 3833, 3984, 4138, 4295, 4453, 4615, 4778, 4944, 5112, 5283, 5456, 5631, + 5808, 5987, 6169, 6352, 6538, 6725, 6915, 7106, 7300, 7495, 7692, 7891, + 8091, 8293, 8497, 8702, 8909, 9118, 9328, 9539, 9752, 9966, 10182, 10398, + 10616, 10835, 11055, 11277, 11499, 11722, 11947, 12172, 12398, 12625, 12852, 13080, + 13309, 13539, 13769, 14000, 14231, 14463, 14695, 14927, 15160, 15393, 15626, 15859, + 16092, 16326, 16559, 16792, 17026, 17259, 17492, 17725, 17957, 18189, 18421, 18653, + 18884, 19114, 19344, 19573, 19802, 20030, 20257, 20483, 20709, 20934, 21157, 21380, + 21602, 21823, 22042, 22261, 22478, 22694, 22909, 23123, 23335, 23545, 23755, 23962, + 24168, 24373, 24576, 24777, 24977, 25175, 25371, 25565, 25758, 25948, 26137, 26323, + 26508, 26690, 26871, 27049, 27225, 27399, 27571, 27740, 27907, 28072, 28234, 28394, + 28552, 28707, 28860, 29010, 29157, 29302, 29444, 29584, 29721, 29855, 29987, 30115, + 30241, 30364, 30485, 30602, 30717, 30828, 30937, 31043, 31145, 31245, 31342, 31436, + 31526, 31614, 31699, 31780, 31858, 31933, 32005, 32074, 32140, 32202, 32261, 32317, + 32370, 32420, 32466, 32509, 32549, 32585, 32618, 32648, 32675, 32698, 32718, 32734, + 32748, 32758, 32764, 32767, 32767, 32667, 32365, 31863, 31164, 30274, 29197, 27939, + 26510, 24917, 23170, 21281, 19261, 17121, 14876, 12540, 10126, 7650, 5126, 2571 +}; + +/* Lag window for LPC (Q31) */ +const int32_t WebRtcIlbcfix_kLpcLagWin[LPC_FILTERORDER + 1]={ + 2147483647, 2144885453, 2137754373, 2125918626, 2109459810, + 2088483140, 2063130336, 2033564590, 1999977009, 1962580174, + 1921610283}; + +/* WebRtcIlbcfix_kLpcChirpSyntDenum vector in Q15 corresponding + * floating point vector {1 0.9025 0.9025^2 0.9025^3 ...} + */ +const int16_t WebRtcIlbcfix_kLpcChirpSyntDenum[LPC_FILTERORDER + 1] = { + 32767, 29573, 26690, 24087, + 21739, 19619, 17707, 15980, + 14422, 13016, 11747}; + +/* WebRtcIlbcfix_kLpcChirpWeightDenum in Q15 corresponding to + * floating point vector {1 0.4222 0.4222^2... } + */ +const int16_t WebRtcIlbcfix_kLpcChirpWeightDenum[LPC_FILTERORDER + 1] = { + 32767, 13835, 5841, 2466, 1041, 440, + 186, 78, 33, 14, 6}; + +/* LSF quantization Q13 domain */ +const int16_t WebRtcIlbcfix_kLsfCb[64 * 3 + 128 * 3 + 128 * 4] = { + 1273, 2238, 3696, + 3199, 5309, 8209, + 3606, 5671, 7829, + 2815, 5262, 8778, + 2608, 4027, 5493, + 1582, 3076, 5945, + 2983, 4181, 5396, + 2437, 4322, 6902, + 1861, 2998, 4613, + 2007, 3250, 5214, + 1388, 2459, 4262, + 2563, 3805, 5269, + 2036, 3522, 5129, + 1935, 4025, 6694, + 2744, 5121, 7338, + 2810, 4248, 5723, + 3054, 5405, 7745, + 1449, 2593, 4763, + 3411, 5128, 6596, + 2484, 4659, 7496, + 1668, 2879, 4818, + 1812, 3072, 5036, + 1638, 2649, 3900, + 2464, 3550, 4644, + 1853, 2900, 4158, + 2458, 4163, 5830, + 2556, 4036, 6254, + 2703, 4432, 6519, + 3062, 4953, 7609, + 1725, 3703, 6187, + 2221, 3877, 5427, + 2339, 3579, 5197, + 2021, 4633, 7037, + 2216, 3328, 4535, + 2961, 4739, 6667, + 2807, 3955, 5099, + 2788, 4501, 6088, + 1642, 2755, 4431, + 3341, 5282, 7333, + 2414, 3726, 5727, + 1582, 2822, 5269, + 2259, 3447, 4905, + 3117, 4986, 7054, + 1825, 3491, 5542, + 3338, 5736, 8627, + 1789, 3090, 5488, + 2566, 3720, 4923, + 2846, 4682, 7161, + 1950, 3321, 5976, + 1834, 3383, 6734, + 3238, 4769, 6094, + 2031, 3978, 5903, + 1877, 4068, 7436, + 2131, 4644, 8296, + 2764, 5010, 8013, + 2194, 3667, 6302, + 2053, 3127, 4342, + 3523, 6595, 10010, + 3134, 4457, 5748, + 3142, 5819, 9414, + 2223, 4334, 6353, + 2022, 3224, 4822, + 2186, 3458, 5544, + 2552, 4757, 6870, + 10905, 12917, 14578, + 9503, 11485, 14485, + 9518, 12494, 14052, + 6222, 7487, 9174, + 7759, 9186, 10506, + 8315, 12755, 14786, + 9609, 11486, 13866, + 8909, 12077, 13643, + 7369, 9054, 11520, + 9408, 12163, 14715, + 6436, 9911, 12843, + 7109, 9556, 11884, + 7557, 10075, 11640, + 6482, 9202, 11547, + 6463, 7914, 10980, + 8611, 10427, 12752, + 7101, 9676, 12606, + 7428, 11252, 13172, + 10197, 12955, 15842, + 7487, 10955, 12613, + 5575, 7858, 13621, + 7268, 11719, 14752, + 7476, 11744, 13795, + 7049, 8686, 11922, + 8234, 11314, 13983, + 6560, 11173, 14984, + 6405, 9211, 12337, + 8222, 12054, 13801, + 8039, 10728, 13255, + 10066, 12733, 14389, + 6016, 7338, 10040, + 6896, 8648, 10234, + 7538, 9170, 12175, + 7327, 12608, 14983, + 10516, 12643, 15223, + 5538, 7644, 12213, + 6728, 12221, 14253, + 7563, 9377, 12948, + 8661, 11023, 13401, + 7280, 8806, 11085, + 7723, 9793, 12333, + 12225, 14648, 16709, + 8768, 13389, 15245, + 10267, 12197, 13812, + 5301, 7078, 11484, + 7100, 10280, 11906, + 8716, 12555, 14183, + 9567, 12464, 15434, + 7832, 12305, 14300, + 7608, 10556, 12121, + 8913, 11311, 12868, + 7414, 9722, 11239, + 8666, 11641, 13250, + 9079, 10752, 12300, + 8024, 11608, 13306, + 10453, 13607, 16449, + 8135, 9573, 10909, + 6375, 7741, 10125, + 10025, 12217, 14874, + 6985, 11063, 14109, + 9296, 13051, 14642, + 8613, 10975, 12542, + 6583, 10414, 13534, + 6191, 9368, 13430, + 5742, 6859, 9260, + 7723, 9813, 13679, + 8137, 11291, 12833, + 6562, 8973, 10641, + 6062, 8462, 11335, + 6928, 8784, 12647, + 7501, 8784, 10031, + 8372, 10045, 12135, + 8191, 9864, 12746, + 5917, 7487, 10979, + 5516, 6848, 10318, + 6819, 9899, 11421, + 7882, 12912, 15670, + 9558, 11230, 12753, + 7752, 9327, 11472, + 8479, 9980, 11358, + 11418, 14072, 16386, + 7968, 10330, 14423, + 8423, 10555, 12162, + 6337, 10306, 14391, + 8850, 10879, 14276, + 6750, 11885, 15710, + 7037, 8328, 9764, + 6914, 9266, 13476, + 9746, 13949, 15519, + 11032, 14444, 16925, + 8032, 10271, 11810, + 10962, 13451, 15833, + 10021, 11667, 13324, + 6273, 8226, 12936, + 8543, 10397, 13496, + 7936, 10302, 12745, + 6769, 8138, 10446, + 6081, 7786, 11719, + 8637, 11795, 14975, + 8790, 10336, 11812, + 7040, 8490, 10771, + 7338, 10381, 13153, + 6598, 7888, 9358, + 6518, 8237, 12030, + 9055, 10763, 12983, + 6490, 10009, 12007, + 9589, 12023, 13632, + 6867, 9447, 10995, + 7930, 9816, 11397, + 10241, 13300, 14939, + 5830, 8670, 12387, + 9870, 11915, 14247, + 9318, 11647, 13272, + 6721, 10836, 12929, + 6543, 8233, 9944, + 8034, 10854, 12394, + 9112, 11787, 14218, + 9302, 11114, 13400, + 9022, 11366, 13816, + 6962, 10461, 12480, + 11288, 13333, 15222, + 7249, 8974, 10547, + 10566, 12336, 14390, + 6697, 11339, 13521, + 11851, 13944, 15826, + 6847, 8381, 11349, + 7509, 9331, 10939, + 8029, 9618, 11909, + 13973, 17644, 19647, 22474, + 14722, 16522, 20035, 22134, + 16305, 18179, 21106, 23048, + 15150, 17948, 21394, 23225, + 13582, 15191, 17687, 22333, + 11778, 15546, 18458, 21753, + 16619, 18410, 20827, 23559, + 14229, 15746, 17907, 22474, + 12465, 15327, 20700, 22831, + 15085, 16799, 20182, 23410, + 13026, 16935, 19890, 22892, + 14310, 16854, 19007, 22944, + 14210, 15897, 18891, 23154, + 14633, 18059, 20132, 22899, + 15246, 17781, 19780, 22640, + 16396, 18904, 20912, 23035, + 14618, 17401, 19510, 21672, + 15473, 17497, 19813, 23439, + 18851, 20736, 22323, 23864, + 15055, 16804, 18530, 20916, + 16490, 18196, 19990, 21939, + 11711, 15223, 21154, 23312, + 13294, 15546, 19393, 21472, + 12956, 16060, 20610, 22417, + 11628, 15843, 19617, 22501, + 14106, 16872, 19839, 22689, + 15655, 18192, 20161, 22452, + 12953, 15244, 20619, 23549, + 15322, 17193, 19926, 21762, + 16873, 18676, 20444, 22359, + 14874, 17871, 20083, 21959, + 11534, 14486, 19194, 21857, + 17766, 19617, 21338, 23178, + 13404, 15284, 19080, 23136, + 15392, 17527, 19470, 21953, + 14462, 16153, 17985, 21192, + 17734, 19750, 21903, 23783, + 16973, 19096, 21675, 23815, + 16597, 18936, 21257, 23461, + 15966, 17865, 20602, 22920, + 15416, 17456, 20301, 22972, + 18335, 20093, 21732, 23497, + 15548, 17217, 20679, 23594, + 15208, 16995, 20816, 22870, + 13890, 18015, 20531, 22468, + 13211, 15377, 19951, 22388, + 12852, 14635, 17978, 22680, + 16002, 17732, 20373, 23544, + 11373, 14134, 19534, 22707, + 17329, 19151, 21241, 23462, + 15612, 17296, 19362, 22850, + 15422, 19104, 21285, 23164, + 13792, 17111, 19349, 21370, + 15352, 17876, 20776, 22667, + 15253, 16961, 18921, 22123, + 14108, 17264, 20294, 23246, + 15785, 17897, 20010, 21822, + 17399, 19147, 20915, 22753, + 13010, 15659, 18127, 20840, + 16826, 19422, 22218, 24084, + 18108, 20641, 22695, 24237, + 18018, 20273, 22268, 23920, + 16057, 17821, 21365, 23665, + 16005, 17901, 19892, 23016, + 13232, 16683, 21107, 23221, + 13280, 16615, 19915, 21829, + 14950, 18575, 20599, 22511, + 16337, 18261, 20277, 23216, + 14306, 16477, 21203, 23158, + 12803, 17498, 20248, 22014, + 14327, 17068, 20160, 22006, + 14402, 17461, 21599, 23688, + 16968, 18834, 20896, 23055, + 15070, 17157, 20451, 22315, + 15419, 17107, 21601, 23946, + 16039, 17639, 19533, 21424, + 16326, 19261, 21745, 23673, + 16489, 18534, 21658, 23782, + 16594, 18471, 20549, 22807, + 18973, 21212, 22890, 24278, + 14264, 18674, 21123, 23071, + 15117, 16841, 19239, 23118, + 13762, 15782, 20478, 23230, + 14111, 15949, 20058, 22354, + 14990, 16738, 21139, 23492, + 13735, 16971, 19026, 22158, + 14676, 17314, 20232, 22807, + 16196, 18146, 20459, 22339, + 14747, 17258, 19315, 22437, + 14973, 17778, 20692, 23367, + 15715, 17472, 20385, 22349, + 15702, 18228, 20829, 23410, + 14428, 16188, 20541, 23630, + 16824, 19394, 21365, 23246, + 13069, 16392, 18900, 21121, + 12047, 16640, 19463, 21689, + 14757, 17433, 19659, 23125, + 15185, 16930, 19900, 22540, + 16026, 17725, 19618, 22399, + 16086, 18643, 21179, 23472, + 15462, 17248, 19102, 21196, + 17368, 20016, 22396, 24096, + 12340, 14475, 19665, 23362, + 13636, 16229, 19462, 22728, + 14096, 16211, 19591, 21635, + 12152, 14867, 19943, 22301, + 14492, 17503, 21002, 22728, + 14834, 16788, 19447, 21411, + 14650, 16433, 19326, 22308, + 14624, 16328, 19659, 23204, + 13888, 16572, 20665, 22488, + 12977, 16102, 18841, 22246, + 15523, 18431, 21757, 23738, + 14095, 16349, 18837, 20947, + 13266, 17809, 21088, 22839, + 15427, 18190, 20270, 23143, + 11859, 16753, 20935, 22486, + 12310, 17667, 21736, 23319, + 14021, 15926, 18702, 22002, + 12286, 15299, 19178, 21126, + 15703, 17491, 21039, 23151, + 12272, 14018, 18213, 22570, + 14817, 16364, 18485, 22598, + 17109, 19683, 21851, 23677, + 12657, 14903, 19039, 22061, + 14713, 16487, 20527, 22814, + 14635, 16726, 18763, 21715, + 15878, 18550, 20718, 22906 +}; + +const int16_t WebRtcIlbcfix_kLsfDimCb[LSF_NSPLIT] = {3, 3, 4}; +const int16_t WebRtcIlbcfix_kLsfSizeCb[LSF_NSPLIT] = {64,128,128}; + +const int16_t WebRtcIlbcfix_kLsfMean[LPC_FILTERORDER] = { + 2308, 3652, 5434, 7885, + 10255, 12559, 15160, 17513, + 20328, 22752}; + +const int16_t WebRtcIlbcfix_kLspMean[LPC_FILTERORDER] = { + 31476, 29565, 25819, 18725, 10276, + 1236, -9049, -17600, -25884, -30618 +}; + +/* Q14 */ +const int16_t WebRtcIlbcfix_kLsfWeight20ms[4] = {12288, 8192, 4096, 0}; +const int16_t WebRtcIlbcfix_kLsfWeight30ms[6] = {8192, 16384, 10923, 5461, 0, 0}; + +/* + cos(x) in Q15 + WebRtcIlbcfix_kCos[i] = cos(pi*i/64.0) + used in WebRtcIlbcfix_Lsp2Lsf() +*/ + +const int16_t WebRtcIlbcfix_kCos[64] = { + 32767, 32729, 32610, 32413, 32138, 31786, 31357, 30853, + 30274, 29622, 28899, 28106, 27246, 26320, 25330, 24279, + 23170, 22006, 20788, 19520, 18205, 16846, 15447, 14010, + 12540, 11039, 9512, 7962, 6393, 4808, 3212, 1608, + 0, -1608, -3212, -4808, -6393, -7962, -9512, -11039, + -12540, -14010, -15447, -16846, -18205, -19520, -20788, -22006, + -23170, -24279, -25330, -26320, -27246, -28106, -28899, -29622, + -30274, -30853, -31357, -31786, -32138, -32413, -32610, -32729 +}; + +/* + Derivative in Q19, used to interpolate between the + WebRtcIlbcfix_kCos[] values to get a more exact y = cos(x) +*/ +const int16_t WebRtcIlbcfix_kCosDerivative[64] = { + -632, -1893, -3150, -4399, -5638, -6863, -8072, -9261, + -10428, -11570, -12684, -13767, -14817, -15832, -16808, -17744, + -18637, -19486, -20287, -21039, -21741, -22390, -22986, -23526, + -24009, -24435, -24801, -25108, -25354, -25540, -25664, -25726, + -25726, -25664, -25540, -25354, -25108, -24801, -24435, -24009, + -23526, -22986, -22390, -21741, -21039, -20287, -19486, -18637, + -17744, -16808, -15832, -14817, -13767, -12684, -11570, -10428, + -9261, -8072, -6863, -5638, -4399, -3150, -1893, -632}; + +/* + Table in Q15, used for a2lsf conversion + WebRtcIlbcfix_kCosGrid[i] = cos((2*pi*i)/(float)(2*COS_GRID_POINTS)); +*/ + +const int16_t WebRtcIlbcfix_kCosGrid[COS_GRID_POINTS + 1] = { + 32760, 32723, 32588, 32364, 32051, 31651, 31164, 30591, + 29935, 29196, 28377, 27481, 26509, 25465, 24351, 23170, + 21926, 20621, 19260, 17846, 16384, 14876, 13327, 11743, + 10125, 8480, 6812, 5126, 3425, 1714, 0, -1714, -3425, + -5126, -6812, -8480, -10125, -11743, -13327, -14876, + -16384, -17846, -19260, -20621, -21926, -23170, -24351, + -25465, -26509, -27481, -28377, -29196, -29935, -30591, + -31164, -31651, -32051, -32364, -32588, -32723, -32760 +}; + +/* + Derivative of y = acos(x) in Q12 + used in WebRtcIlbcfix_Lsp2Lsf() +*/ + +const int16_t WebRtcIlbcfix_kAcosDerivative[64] = { + -26887, -8812, -5323, -3813, -2979, -2444, -2081, -1811, + -1608, -1450, -1322, -1219, -1132, -1059, -998, -946, + -901, -861, -827, -797, -772, -750, -730, -713, + -699, -687, -677, -668, -662, -657, -654, -652, + -652, -654, -657, -662, -668, -677, -687, -699, + -713, -730, -750, -772, -797, -827, -861, -901, + -946, -998, -1059, -1132, -1219, -1322, -1450, -1608, + -1811, -2081, -2444, -2979, -3813, -5323, -8812, -26887 +}; + + +/* Tables for quantization of start state */ + +/* State quantization tables */ +const int16_t WebRtcIlbcfix_kStateSq3[8] = { /* Values in Q13 */ + -30473, -17838, -9257, -2537, + 3639, 10893, 19958, 32636 +}; + +/* This table defines the limits for the selection of the freqg + less or equal than value 0 => index = 0 + less or equal than value k => index = k +*/ +const int32_t WebRtcIlbcfix_kChooseFrgQuant[64] = { + 118, 163, 222, 305, 425, 604, + 851, 1174, 1617, 2222, 3080, 4191, + 5525, 7215, 9193, 11540, 14397, 17604, + 21204, 25209, 29863, 35720, 42531, 50375, + 59162, 68845, 80108, 93754, 110326, 129488, + 150654, 174328, 201962, 233195, 267843, 308239, + 354503, 405988, 464251, 531550, 608652, 697516, + 802526, 928793, 1080145, 1258120, 1481106, 1760881, + 2111111, 2546619, 3078825, 3748642, 4563142, 5573115, + 6887601, 8582108, 10797296, 14014513, 18625760, 25529599, + 37302935, 58819185, 109782723, WEBRTC_SPL_WORD32_MAX +}; + +const int16_t WebRtcIlbcfix_kScale[64] = { + /* Values in Q16 */ + 29485, 25003, 21345, 18316, 15578, 13128, 10973, 9310, 7955, + 6762, 5789, 4877, 4255, 3699, 3258, 2904, 2595, 2328, + 2123, 1932, 1785, 1631, 1493, 1370, 1260, 1167, 1083, + /* Values in Q21 */ + 32081, 29611, 27262, 25229, 23432, 21803, 20226, 18883, 17609, + 16408, 15311, 14327, 13390, 12513, 11693, 10919, 10163, 9435, + 8739, 8100, 7424, 6813, 6192, 5648, 5122, 4639, 4207, 3798, + 3404, 3048, 2706, 2348, 2036, 1713, 1393, 1087, 747 +}; + +/*frgq in fixpoint, but already computed like this: + for(i=0; i<64; i++){ + a = (pow(10,frgq[i])/4.5); + WebRtcIlbcfix_kFrgQuantMod[i] = round(a); + } + + Value 0 :36 in Q8 + 37:58 in Q5 + 59:63 in Q3 +*/ +const int16_t WebRtcIlbcfix_kFrgQuantMod[64] = { + /* First 37 values in Q8 */ + 569, 671, 786, 916, 1077, 1278, + 1529, 1802, 2109, 2481, 2898, 3440, + 3943, 4535, 5149, 5778, 6464, 7208, + 7904, 8682, 9397, 10285, 11240, 12246, + 13313, 14382, 15492, 16735, 18131, 19693, + 21280, 22912, 24624, 26544, 28432, 30488, + 32720, + /* 22 values in Q5 */ + 4383, 4684, 5012, 5363, 5739, 6146, + 6603, 7113, 7679, 8285, 9040, 9850, + 10838, 11882, 13103, 14467, 15950, 17669, + 19712, 22016, 24800, 28576, + /* 5 values in Q3 */ + 8240, 9792, 12040, 15440, 22472 +}; + +/* Constants for codebook search and creation */ + +/* Expansion filter to get additional cb section. + * Q12 and reversed compared to flp + */ +const int16_t WebRtcIlbcfix_kCbFiltersRev[CB_FILTERLEN]={ + -140, 446, -755, 3302, 2922, -590, 343, -138}; + +/* Weighting coefficients for short lags. + * [0.2 0.4 0.6 0.8] in Q15 */ +const int16_t WebRtcIlbcfix_kAlpha[4]={ + 6554, 13107, 19661, 26214}; + +/* Ranges for search and filters at different subframes */ + +const size_t WebRtcIlbcfix_kSearchRange[5][CB_NSTAGES]={ + {58,58,58}, {108,44,44}, {108,108,108}, {108,108,108}, {108,108,108}}; + +const size_t WebRtcIlbcfix_kFilterRange[5]={63, 85, 125, 147, 147}; + +/* Gain Quantization for the codebook gains of the 3 stages */ + +/* Q14 (one extra value (max int16_t) to simplify for the search) */ +const int16_t WebRtcIlbcfix_kGainSq3[9]={ + -16384, -10813, -5407, 0, 4096, 8192, + 12288, 16384, 32767}; + +/* Q14 (one extra value (max int16_t) to simplify for the search) */ +const int16_t WebRtcIlbcfix_kGainSq4[17]={ + -17203, -14746, -12288, -9830, -7373, -4915, + -2458, 0, 2458, 4915, 7373, 9830, + 12288, 14746, 17203, 19661, 32767}; + +/* Q14 (one extra value (max int16_t) to simplify for the search) */ +const int16_t WebRtcIlbcfix_kGainSq5[33]={ + 614, 1229, 1843, 2458, 3072, 3686, + 4301, 4915, 5530, 6144, 6758, 7373, + 7987, 8602, 9216, 9830, 10445, 11059, + 11674, 12288, 12902, 13517, 14131, 14746, + 15360, 15974, 16589, 17203, 17818, 18432, + 19046, 19661, 32767}; + +/* Q14 gain_sq5Tbl squared in Q14 */ +const int16_t WebRtcIlbcfix_kGainSq5Sq[32] = { + 23, 92, 207, 368, 576, 829, + 1129, 1474, 1866, 2304, 2787, 3317, + 3893, 4516, 5184, 5897, 6658, 7464, + 8318, 9216, 10160, 11151, 12187, 13271, + 14400, 15574, 16796, 18062, 19377, 20736, + 22140, 23593 +}; + +const int16_t* const WebRtcIlbcfix_kGain[3] = +{WebRtcIlbcfix_kGainSq5, WebRtcIlbcfix_kGainSq4, WebRtcIlbcfix_kGainSq3}; + + +/* Tables for the Enhancer, using upsamling factor 4 (ENH_UPS0 = 4) */ + +const int16_t WebRtcIlbcfix_kEnhPolyPhaser[ENH_UPS0][ENH_FLO_MULT2_PLUS1]={ + {0, 0, 0, 4096, 0, 0, 0}, + {64, -315, 1181, 3531, -436, 77, -64}, + {97, -509, 2464, 2464, -509, 97, -97}, + {77, -436, 3531, 1181, -315, 64, -77} +}; + +const int16_t WebRtcIlbcfix_kEnhWt[3] = { + 4800, 16384, 27968 /* Q16 */ +}; + +const size_t WebRtcIlbcfix_kEnhPlocs[ENH_NBLOCKS_TOT] = { + 160, 480, 800, 1120, 1440, 1760, 2080, 2400 /* Q(-2) */ +}; + +/* PLC table */ + +const int16_t WebRtcIlbcfix_kPlcPerSqr[6] = { /* Grid points for square of periodiciy in Q15 */ + 839, 1343, 2048, 2998, 4247, 5849 +}; + +const int16_t WebRtcIlbcfix_kPlcPitchFact[6] = { /* Value of y=(x^4-0.4)/(0.7-0.4) in grid points in Q15 */ + 0, 5462, 10922, 16384, 21846, 27306 +}; + +const int16_t WebRtcIlbcfix_kPlcPfSlope[6] = { /* Slope of y=(x^4-0.4)/(0.7-0.4) in Q11 */ + 26667, 18729, 13653, 10258, 7901, 6214 +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.h new file mode 100644 index 0000000000..506b588f6a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + constants.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CONSTANTS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CONSTANTS_H_ + +#include "defines.h" +#include "typedefs.h" // NOLINT(build/include) + +/* high pass filters */ + +extern const int16_t WebRtcIlbcfix_kHpInCoefs[]; +extern const int16_t WebRtcIlbcfix_kHpOutCoefs[]; + +/* Window for start state decision */ +extern const int16_t WebRtcIlbcfix_kStartSequenceEnrgWin[]; + +/* low pass filter used for downsampling */ +extern const int16_t WebRtcIlbcfix_kLpFiltCoefs[]; + +/* LPC analysis and quantization */ + +extern const int16_t WebRtcIlbcfix_kLpcWin[]; +extern const int16_t WebRtcIlbcfix_kLpcAsymWin[]; +extern const int32_t WebRtcIlbcfix_kLpcLagWin[]; +extern const int16_t WebRtcIlbcfix_kLpcChirpSyntDenum[]; +extern const int16_t WebRtcIlbcfix_kLpcChirpWeightDenum[]; +extern const int16_t WebRtcIlbcfix_kLsfDimCb[]; +extern const int16_t WebRtcIlbcfix_kLsfSizeCb[]; +extern const int16_t WebRtcIlbcfix_kLsfCb[]; +extern const int16_t WebRtcIlbcfix_kLsfWeight20ms[]; +extern const int16_t WebRtcIlbcfix_kLsfWeight30ms[]; +extern const int16_t WebRtcIlbcfix_kLsfMean[]; +extern const int16_t WebRtcIlbcfix_kLspMean[]; +extern const int16_t WebRtcIlbcfix_kCos[]; +extern const int16_t WebRtcIlbcfix_kCosDerivative[]; +extern const int16_t WebRtcIlbcfix_kCosGrid[]; +extern const int16_t WebRtcIlbcfix_kAcosDerivative[]; + +/* state quantization tables */ + +extern const int16_t WebRtcIlbcfix_kStateSq3[]; +extern const int32_t WebRtcIlbcfix_kChooseFrgQuant[]; +extern const int16_t WebRtcIlbcfix_kScale[]; +extern const int16_t WebRtcIlbcfix_kFrgQuantMod[]; + +/* Ranges for search and filters at different subframes */ + +extern const size_t WebRtcIlbcfix_kSearchRange[5][CB_NSTAGES]; +extern const size_t WebRtcIlbcfix_kFilterRange[]; + +/* gain quantization tables */ + +extern const int16_t WebRtcIlbcfix_kGainSq3[]; +extern const int16_t WebRtcIlbcfix_kGainSq4[]; +extern const int16_t WebRtcIlbcfix_kGainSq5[]; +extern const int16_t WebRtcIlbcfix_kGainSq5Sq[]; +extern const int16_t* const WebRtcIlbcfix_kGain[]; + +/* adaptive codebook definitions */ + +extern const int16_t WebRtcIlbcfix_kCbFiltersRev[]; +extern const int16_t WebRtcIlbcfix_kAlpha[]; + +/* enhancer definitions */ + +extern const int16_t WebRtcIlbcfix_kEnhPolyPhaser[ENH_UPS0][ENH_FLO_MULT2_PLUS1]; +extern const int16_t WebRtcIlbcfix_kEnhWt[]; +extern const size_t WebRtcIlbcfix_kEnhPlocs[]; + +/* PLC tables */ + +extern const int16_t WebRtcIlbcfix_kPlcPerSqr[]; +extern const int16_t WebRtcIlbcfix_kPlcPitchFact[]; +extern const int16_t WebRtcIlbcfix_kPlcPfSlope[]; + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c new file mode 100644 index 0000000000..adc428df7b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CreateAugmentedVec.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "rtc_base/sanitizer.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" + +/*----------------------------------------------------------------* + * Recreate a specific codebook vector from the augmented part. + * + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CreateAugmentedVec( + size_t index, /* (i) Index for the augmented vector to be + created */ + const int16_t* buffer, /* (i) Pointer to the end of the codebook memory + that is used for creation of the augmented + codebook */ + int16_t* cbVec) { /* (o) The constructed codebook vector */ + size_t ilow; + const int16_t *ppo, *ppi; + int16_t cbVecTmp[4]; + /* Interpolation starts 4 elements before cbVec+index, but must not start + outside |cbVec|; clamping interp_len to stay within |cbVec|. + */ + size_t interp_len = WEBRTC_SPL_MIN(index, 4); + + rtc_MsanCheckInitialized(buffer - index - interp_len, sizeof(buffer[0]), + index + interp_len); + + ilow = index - interp_len; + + /* copy the first noninterpolated part */ + ppo = buffer-index; + WEBRTC_SPL_MEMCPY_W16(cbVec, ppo, index); + + /* interpolation */ + ppo = buffer - interp_len; + ppi = buffer - index - interp_len; + + /* perform cbVec[ilow+k] = ((ppi[k]*alphaTbl[k])>>15) + + ((ppo[k]*alphaTbl[interp_len-1-k])>>15); + for k = 0..interp_len-1 + */ + WebRtcSpl_ElementwiseVectorMult(&cbVec[ilow], ppi, WebRtcIlbcfix_kAlpha, + interp_len, 15); + WebRtcSpl_ReverseOrderMultArrayElements( + cbVecTmp, ppo, &WebRtcIlbcfix_kAlpha[interp_len - 1], interp_len, 15); + WebRtcSpl_AddVectorsAndShift(&cbVec[ilow], &cbVec[ilow], cbVecTmp, interp_len, + 0); + + /* copy the second noninterpolated part */ + ppo = buffer - index; + /* |tempbuff2| is declared in WebRtcIlbcfix_GetCbVec and is SUBL+5 elements + long. |buffer| points one element past the end of that vector, i.e., at + tempbuff2+SUBL+5. Since ppo=buffer-index, we cannot read any more than + |index| elements from |ppo|. + + |cbVec| is declared to be SUBL elements long in WebRtcIlbcfix_CbConstruct. + Therefore, we can only write SUBL-index elements to cbVec+index. + + These two conditions limit the number of elements to copy. + */ + WEBRTC_SPL_MEMCPY_W16(cbVec+index, ppo, WEBRTC_SPL_MIN(SUBL-index, index)); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h new file mode 100644 index 0000000000..0616e2e0b2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CreateAugmentedVec.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CREATE_AUGMENTED_VEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CREATE_AUGMENTED_VEC_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Recreate a specific codebook vector from the augmented part. + * + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CreateAugmentedVec( + size_t index, /* (i) Index for the augmented vector to be + created */ + const int16_t* buffer, /* (i) Pointer to the end of the codebook memory + that is used for creation of the augmented + codebook */ + int16_t* cbVec); /* (o) The construced codebook vector */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.c new file mode 100644 index 0000000000..a6e045dc67 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.c @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Decode.c + +******************************************************************/ + +#include "defines.h" +#include "simple_lsf_dequant.h" +#include "decoder_interpolate_lsf.h" +#include "index_conv_dec.h" +#include "do_plc.h" +#include "constants.h" +#include "enhancer_interface.h" +#include "xcorr_coef.h" +#include "lsf_check.h" +#include "decode_residual.h" +#include "unpack_bits.h" +#include "hp_output.h" +#include "init_decode.h" +#ifndef WEBRTC_ARCH_BIG_ENDIAN +#include "swap_bytes.h" +#endif + +/*----------------------------------------------------------------* + * main decoder function + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_DecodeImpl( + int16_t *decblock, /* (o) decoded signal block */ + const uint16_t *bytes, /* (i) encoded signal bits */ + IlbcDecoder *iLBCdec_inst, /* (i/o) the decoder state + structure */ + int16_t mode /* (i) 0: bad packet, PLC, + 1: normal */ + ) { + const int old_mode = iLBCdec_inst->mode; + const int old_use_enhancer = iLBCdec_inst->use_enhancer; + + size_t i; + int16_t order_plus_one; + + int16_t last_bit; + int16_t *data; + /* Stack based */ + int16_t decresidual[BLOCKL_MAX]; + int16_t PLCresidual[BLOCKL_MAX + LPC_FILTERORDER]; + int16_t syntdenum[NSUB_MAX*(LPC_FILTERORDER+1)]; + int16_t PLClpc[LPC_FILTERORDER + 1]; +#ifndef WEBRTC_ARCH_BIG_ENDIAN + uint16_t swapped[NO_OF_WORDS_30MS]; +#endif + iLBC_bits *iLBCbits_inst = (iLBC_bits*)PLCresidual; + + /* Reuse some buffers that are non overlapping in order to save stack memory */ + data = &PLCresidual[LPC_FILTERORDER]; + + if (mode) { /* the data are good */ + + /* decode data */ + + /* Unpacketize bits into parameters */ + +#ifndef WEBRTC_ARCH_BIG_ENDIAN + WebRtcIlbcfix_SwapBytes(bytes, iLBCdec_inst->no_of_words, swapped); + last_bit = WebRtcIlbcfix_UnpackBits(swapped, iLBCbits_inst, iLBCdec_inst->mode); +#else + last_bit = WebRtcIlbcfix_UnpackBits(bytes, iLBCbits_inst, iLBCdec_inst->mode); +#endif + + /* Check for bit errors */ + if (iLBCbits_inst->startIdx<1) + mode = 0; + if ((iLBCdec_inst->mode==20) && (iLBCbits_inst->startIdx>3)) + mode = 0; + if ((iLBCdec_inst->mode==30) && (iLBCbits_inst->startIdx>5)) + mode = 0; + if (last_bit==1) + mode = 0; + + if (mode) { /* No bit errors was detected, continue decoding */ + /* Stack based */ + int16_t lsfdeq[LPC_FILTERORDER*LPC_N_MAX]; + int16_t weightdenum[(LPC_FILTERORDER + 1)*NSUB_MAX]; + + /* adjust index */ + WebRtcIlbcfix_IndexConvDec(iLBCbits_inst->cb_index); + + /* decode the lsf */ + WebRtcIlbcfix_SimpleLsfDeQ(lsfdeq, (int16_t*)(iLBCbits_inst->lsf), iLBCdec_inst->lpc_n); + WebRtcIlbcfix_LsfCheck(lsfdeq, LPC_FILTERORDER, iLBCdec_inst->lpc_n); + WebRtcIlbcfix_DecoderInterpolateLsp(syntdenum, weightdenum, + lsfdeq, LPC_FILTERORDER, iLBCdec_inst); + + /* Decode the residual using the cb and gain indexes */ + if (!WebRtcIlbcfix_DecodeResidual(iLBCdec_inst, iLBCbits_inst, + decresidual, syntdenum)) + goto error; + + /* preparing the plc for a future loss! */ + WebRtcIlbcfix_DoThePlc( + PLCresidual, PLClpc, 0, decresidual, + syntdenum + (LPC_FILTERORDER + 1) * (iLBCdec_inst->nsub - 1), + iLBCdec_inst->last_lag, iLBCdec_inst); + + /* Use the output from doThePLC */ + WEBRTC_SPL_MEMCPY_W16(decresidual, PLCresidual, iLBCdec_inst->blockl); + } + + } + + if (mode == 0) { + /* the data is bad (either a PLC call + * was made or a bit error was detected) + */ + + /* packet loss conceal */ + + WebRtcIlbcfix_DoThePlc(PLCresidual, PLClpc, 1, decresidual, syntdenum, + iLBCdec_inst->last_lag, iLBCdec_inst); + + WEBRTC_SPL_MEMCPY_W16(decresidual, PLCresidual, iLBCdec_inst->blockl); + + order_plus_one = LPC_FILTERORDER + 1; + + for (i = 0; i < iLBCdec_inst->nsub; i++) { + WEBRTC_SPL_MEMCPY_W16(syntdenum+(i*order_plus_one), + PLClpc, order_plus_one); + } + } + + if ((*iLBCdec_inst).use_enhancer == 1) { /* Enhancer activated */ + + /* Update the filter and filter coefficients if there was a packet loss */ + if (iLBCdec_inst->prev_enh_pl==2) { + for (i=0;insub;i++) { + WEBRTC_SPL_MEMCPY_W16(&(iLBCdec_inst->old_syntdenum[i*(LPC_FILTERORDER+1)]), + syntdenum, (LPC_FILTERORDER+1)); + } + } + + /* post filtering */ + (*iLBCdec_inst).last_lag = + WebRtcIlbcfix_EnhancerInterface(data, decresidual, iLBCdec_inst); + + /* synthesis filtering */ + + /* Set up the filter state */ + WEBRTC_SPL_MEMCPY_W16(&data[-LPC_FILTERORDER], iLBCdec_inst->syntMem, LPC_FILTERORDER); + + if (iLBCdec_inst->mode==20) { + /* Enhancer has 40 samples delay */ + i=0; + WebRtcSpl_FilterARFastQ12( + data, data, + iLBCdec_inst->old_syntdenum + (i+iLBCdec_inst->nsub-1)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + + for (i=1; i < iLBCdec_inst->nsub; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + syntdenum+(i-1)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + + } else if (iLBCdec_inst->mode==30) { + /* Enhancer has 80 samples delay */ + for (i=0; i < 2; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + iLBCdec_inst->old_syntdenum + (i+4)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + for (i=2; i < iLBCdec_inst->nsub; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + syntdenum+(i-2)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + } + + /* Save the filter state */ + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->syntMem, &data[iLBCdec_inst->blockl-LPC_FILTERORDER], LPC_FILTERORDER); + + } else { /* Enhancer not activated */ + size_t lag; + + /* Find last lag (since the enhancer is not called to give this info) */ + lag = 20; + if (iLBCdec_inst->mode==20) { + lag = WebRtcIlbcfix_XcorrCoef( + &decresidual[iLBCdec_inst->blockl-60], + &decresidual[iLBCdec_inst->blockl-60-lag], + 60, + 80, lag, -1); + } else { + lag = WebRtcIlbcfix_XcorrCoef( + &decresidual[iLBCdec_inst->blockl-ENH_BLOCKL], + &decresidual[iLBCdec_inst->blockl-ENH_BLOCKL-lag], + ENH_BLOCKL, + 100, lag, -1); + } + + /* Store lag (it is needed if next packet is lost) */ + (*iLBCdec_inst).last_lag = lag; + + /* copy data and run synthesis filter */ + WEBRTC_SPL_MEMCPY_W16(data, decresidual, iLBCdec_inst->blockl); + + /* Set up the filter state */ + WEBRTC_SPL_MEMCPY_W16(&data[-LPC_FILTERORDER], iLBCdec_inst->syntMem, LPC_FILTERORDER); + + for (i=0; i < iLBCdec_inst->nsub; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + syntdenum + i*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + + /* Save the filter state */ + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->syntMem, &data[iLBCdec_inst->blockl-LPC_FILTERORDER], LPC_FILTERORDER); + } + + WEBRTC_SPL_MEMCPY_W16(decblock,data,iLBCdec_inst->blockl); + + /* High pass filter the signal (with upscaling a factor 2 and saturation) */ + WebRtcIlbcfix_HpOutput(decblock, (int16_t*)WebRtcIlbcfix_kHpOutCoefs, + iLBCdec_inst->hpimemy, iLBCdec_inst->hpimemx, + iLBCdec_inst->blockl); + + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->old_syntdenum, + syntdenum, iLBCdec_inst->nsub*(LPC_FILTERORDER+1)); + + iLBCdec_inst->prev_enh_pl=0; + + if (mode==0) { /* PLC was used */ + iLBCdec_inst->prev_enh_pl=1; + } + + return 0; // Success. + +error: + // The decoder got sick from eating that data. Reset it and return. + WebRtcIlbcfix_InitDecode(iLBCdec_inst, old_mode, old_use_enhancer); + return -1; // Error +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.h new file mode 100644 index 0000000000..b35be969b9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Decode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * main decoder function + *---------------------------------------------------------------*/ + +// Returns 0 on success, -1 on error. +int WebRtcIlbcfix_DecodeImpl( + int16_t* decblock, /* (o) decoded signal block */ + const uint16_t* bytes, /* (i) encoded signal bits */ + IlbcDecoder* iLBCdec_inst, /* (i/o) the decoder state + structure */ + int16_t mode /* (i) 0: bad packet, PLC, + 1: normal */ + ) RTC_WARN_UNUSED_RESULT; + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.c new file mode 100644 index 0000000000..dd5c353cb5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.c @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecodeResidual.c + +******************************************************************/ + +#include "decode_residual.h" + +#include + +#include "defines.h" +#include "state_construct.h" +#include "cb_construct.h" +#include "index_conv_dec.h" +#include "do_plc.h" +#include "constants.h" +#include "enhancer_interface.h" +#include "xcorr_coef.h" +#include "lsf_check.h" + +/*----------------------------------------------------------------* + * frame residual decoder function (subrutine to iLBC_decode) + *---------------------------------------------------------------*/ + +bool WebRtcIlbcfix_DecodeResidual( + IlbcDecoder *iLBCdec_inst, + /* (i/o) the decoder state structure */ + iLBC_bits *iLBC_encbits, /* (i/o) Encoded bits, which are used + for the decoding */ + int16_t *decresidual, /* (o) decoded residual frame */ + int16_t *syntdenum /* (i) the decoded synthesis filter + coefficients */ + ) { + size_t meml_gotten, diff, start_pos; + size_t subcount, subframe; + int16_t *reverseDecresidual = iLBCdec_inst->enh_buf; /* Reversed decoded data, used for decoding backwards in time (reuse memory in state) */ + int16_t *memVec = iLBCdec_inst->prevResidual; /* Memory for codebook and filter state (reuse memory in state) */ + int16_t *mem = &memVec[CB_HALFFILTERLEN]; /* Memory for codebook */ + + diff = STATE_LEN - iLBCdec_inst->state_short_len; + + if (iLBC_encbits->state_first == 1) { + start_pos = (iLBC_encbits->startIdx-1)*SUBL; + } else { + start_pos = (iLBC_encbits->startIdx-1)*SUBL + diff; + } + + /* decode scalar part of start state */ + + WebRtcIlbcfix_StateConstruct(iLBC_encbits->idxForMax, + iLBC_encbits->idxVec, &syntdenum[(iLBC_encbits->startIdx-1)*(LPC_FILTERORDER+1)], + &decresidual[start_pos], iLBCdec_inst->state_short_len + ); + + if (iLBC_encbits->state_first) { /* put adaptive part in the end */ + + /* setup memory */ + + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - iLBCdec_inst->state_short_len); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-iLBCdec_inst->state_short_len, decresidual+start_pos, + iLBCdec_inst->state_short_len); + + /* construct decoded vector */ + + if (!WebRtcIlbcfix_CbConstruct( + &decresidual[start_pos + iLBCdec_inst->state_short_len], + iLBC_encbits->cb_index, iLBC_encbits->gain_index, + mem + CB_MEML - ST_MEM_L_TBL, ST_MEM_L_TBL, diff)) + return false; // Error. + + } + else {/* put adaptive part in the beginning */ + + /* setup memory */ + + meml_gotten = iLBCdec_inst->state_short_len; + WebRtcSpl_MemCpyReversedOrder(mem+CB_MEML-1, + decresidual+start_pos, meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - meml_gotten); + + /* construct decoded vector */ + + if (!WebRtcIlbcfix_CbConstruct(reverseDecresidual, iLBC_encbits->cb_index, + iLBC_encbits->gain_index, + mem + CB_MEML - ST_MEM_L_TBL, ST_MEM_L_TBL, + diff)) + return false; // Error. + + /* get decoded residual from reversed vector */ + + WebRtcSpl_MemCpyReversedOrder(&decresidual[start_pos-1], + reverseDecresidual, diff); + } + + /* counter for predicted subframes */ + + subcount=1; + + /* forward prediction of subframes */ + + if (iLBCdec_inst->nsub > iLBC_encbits->startIdx + 1) { + + /* setup memory */ + WebRtcSpl_MemSetW16(mem, 0, CB_MEML-STATE_LEN); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-STATE_LEN, + decresidual+(iLBC_encbits->startIdx-1)*SUBL, STATE_LEN); + + /* loop over subframes to encode */ + + size_t Nfor = iLBCdec_inst->nsub - iLBC_encbits->startIdx - 1; + for (subframe=0; subframestartIdx + 1 + subframe) * SUBL], + iLBC_encbits->cb_index + subcount * CB_NSTAGES, + iLBC_encbits->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)) + return false; // Error; + + /* update memory */ + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &decresidual[(iLBC_encbits->startIdx+1+subframe)*SUBL], SUBL); + + subcount++; + } + + } + + /* backward prediction of subframes */ + + if (iLBC_encbits->startIdx > 1) { + + /* setup memory */ + + meml_gotten = SUBL*(iLBCdec_inst->nsub+1-iLBC_encbits->startIdx); + if( meml_gotten > CB_MEML ) { + meml_gotten=CB_MEML; + } + + WebRtcSpl_MemCpyReversedOrder(mem+CB_MEML-1, + decresidual+(iLBC_encbits->startIdx-1)*SUBL, meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - meml_gotten); + + /* loop over subframes to decode */ + + size_t Nback = iLBC_encbits->startIdx - 1; + for (subframe=0; subframecb_index + subcount * CB_NSTAGES, + iLBC_encbits->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)) + return false; // Error. + + /* update memory */ + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &reverseDecresidual[subframe*SUBL], SUBL); + + subcount++; + } + + /* get decoded residual from reversed vector */ + WebRtcSpl_MemCpyReversedOrder(decresidual+SUBL*Nback-1, + reverseDecresidual, SUBL*Nback); + } + + return true; // Success. +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.h new file mode 100644 index 0000000000..048671e0d6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecodeResidual.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_RESIDUAL_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_RESIDUAL_H_ + +#include +#include "defines.h" + +/*----------------------------------------------------------------* + * frame residual decoder function (subrutine to iLBC_decode) + *---------------------------------------------------------------*/ + +// Returns true on success, false on failure. In case of failure, the decoder +// state may be corrupted and needs resetting. +bool WebRtcIlbcfix_DecodeResidual( + IlbcDecoder* iLBCdec_inst, /* (i/o) the decoder state structure */ + iLBC_bits* iLBC_encbits, /* (i/o) Encoded bits, which are used + for the decoding */ + int16_t* decresidual, /* (o) decoded residual frame */ + int16_t* syntdenum /* (i) the decoded synthesis filter + coefficients */ + ) RTC_WARN_UNUSED_RESULT; + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c new file mode 100644 index 0000000000..06ab2e7190 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecoderInterpolateLsp.c + +******************************************************************/ + +#include "lsf_interpolate_to_poly_dec.h" +#include "bw_expand.h" +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * obtain synthesis and weighting filters form lsf coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DecoderInterpolateLsp( + int16_t *syntdenum, /* (o) synthesis filter coefficients */ + int16_t *weightdenum, /* (o) weighting denumerator + coefficients */ + int16_t *lsfdeq, /* (i) dequantized lsf coefficients */ + int16_t length, /* (i) length of lsf coefficient vector */ + IlbcDecoder *iLBCdec_inst + /* (i) the decoder state structure */ + ){ + size_t i; + int pos, lp_length; + int16_t lp[LPC_FILTERORDER + 1], *lsfdeq2; + + lsfdeq2 = lsfdeq + length; + lp_length = length + 1; + + if (iLBCdec_inst->mode==30) { + /* subframe 1: Interpolation between old and first LSF */ + + WebRtcIlbcfix_LspInterpolate2PolyDec(lp, (*iLBCdec_inst).lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight30ms[0], length); + WEBRTC_SPL_MEMCPY_W16(syntdenum,lp,lp_length); + WebRtcIlbcfix_BwExpand(weightdenum, lp, (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, (int16_t)lp_length); + + /* subframes 2 to 6: interpolation between first and last LSF */ + + pos = lp_length; + for (i = 1; i < 6; i++) { + WebRtcIlbcfix_LspInterpolate2PolyDec(lp, lsfdeq, lsfdeq2, + WebRtcIlbcfix_kLsfWeight30ms[i], length); + WEBRTC_SPL_MEMCPY_W16(syntdenum + pos,lp,lp_length); + WebRtcIlbcfix_BwExpand(weightdenum + pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, (int16_t)lp_length); + pos += lp_length; + } + } else { /* iLBCdec_inst->mode=20 */ + /* subframes 1 to 4: interpolation between old and new LSF */ + pos = 0; + for (i = 0; i < iLBCdec_inst->nsub; i++) { + WebRtcIlbcfix_LspInterpolate2PolyDec(lp, iLBCdec_inst->lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight20ms[i], length); + WEBRTC_SPL_MEMCPY_W16(syntdenum+pos,lp,lp_length); + WebRtcIlbcfix_BwExpand(weightdenum+pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, (int16_t)lp_length); + pos += lp_length; + } + } + + /* update memory */ + + if (iLBCdec_inst->mode==30) { + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->lsfdeqold, lsfdeq2, length); + } else { + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->lsfdeqold, lsfdeq, length); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h new file mode 100644 index 0000000000..d83ad47538 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecoderInterpolateLsp.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODER_INTERPOLATE_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODER_INTERPOLATE_LSF_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * obtain synthesis and weighting filters form lsf coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DecoderInterpolateLsp( + int16_t *syntdenum, /* (o) synthesis filter coefficients */ + int16_t *weightdenum, /* (o) weighting denumerator + coefficients */ + int16_t *lsfdeq, /* (i) dequantized lsf coefficients */ + int16_t length, /* (i) length of lsf coefficient vector */ + IlbcDecoder *iLBCdec_inst + /* (i) the decoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/defines.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/defines.h new file mode 100644 index 0000000000..604e337aa6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/defines.h @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + define.h + +******************************************************************/ +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DEFINES_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DEFINES_H_ + +#include +#include "signal_processing_library.h" +#include "typedefs.h" // NOLINT(build/include) + +/* general codec settings */ + +#define FS 8000 +#define BLOCKL_20MS 160 +#define BLOCKL_30MS 240 +#define BLOCKL_MAX 240 +#define NSUB_20MS 4 +#define NSUB_30MS 6 +#define NSUB_MAX 6 +#define NASUB_20MS 2 +#define NASUB_30MS 4 +#define NASUB_MAX 4 +#define SUBL 40 +#define STATE_LEN 80 +#define STATE_SHORT_LEN_30MS 58 +#define STATE_SHORT_LEN_20MS 57 + +/* LPC settings */ + +#define LPC_FILTERORDER 10 +#define LPC_LOOKBACK 60 +#define LPC_N_20MS 1 +#define LPC_N_30MS 2 +#define LPC_N_MAX 2 +#define LPC_ASYMDIFF 20 +#define LSF_NSPLIT 3 +#define LSF_NUMBER_OF_STEPS 4 +#define LPC_HALFORDER 5 +#define COS_GRID_POINTS 60 + +/* cb settings */ + +#define CB_NSTAGES 3 +#define CB_EXPAND 2 +#define CB_MEML 147 +#define CB_FILTERLEN (2*4) +#define CB_HALFFILTERLEN 4 +#define CB_RESRANGE 34 +#define CB_MAXGAIN_FIXQ6 83 /* error = -0.24% */ +#define CB_MAXGAIN_FIXQ14 21299 + +/* enhancer */ + +#define ENH_BLOCKL 80 /* block length */ +#define ENH_BLOCKL_HALF (ENH_BLOCKL/2) +#define ENH_HL 3 /* 2*ENH_HL+1 is number blocks + in said second sequence */ +#define ENH_SLOP 2 /* max difference estimated and + correct pitch period */ +#define ENH_PLOCSL 8 /* pitch-estimates and + pitch-locations buffer length */ +#define ENH_OVERHANG 2 +#define ENH_UPS0 4 /* upsampling rate */ +#define ENH_FL0 3 /* 2*FLO+1 is the length of each filter */ +#define ENH_FLO_MULT2_PLUS1 7 +#define ENH_VECTL (ENH_BLOCKL+2*ENH_FL0) +#define ENH_CORRDIM (2*ENH_SLOP+1) +#define ENH_NBLOCKS (BLOCKL/ENH_BLOCKL) +#define ENH_NBLOCKS_EXTRA 5 +#define ENH_NBLOCKS_TOT 8 /* ENH_NBLOCKS+ENH_NBLOCKS_EXTRA */ +#define ENH_BUFL (ENH_NBLOCKS_TOT)*ENH_BLOCKL +#define ENH_BUFL_FILTEROVERHEAD 3 +#define ENH_A0 819 /* Q14 */ +#define ENH_A0_MINUS_A0A0DIV4 848256041 /* Q34 */ +#define ENH_A0DIV2 26843546 /* Q30 */ + +/* PLC */ + +/* Down sampling */ + +#define FILTERORDER_DS_PLUS1 7 +#define DELAY_DS 3 +#define FACTOR_DS 2 + +/* bit stream defs */ + +#define NO_OF_BYTES_20MS 38 +#define NO_OF_BYTES_30MS 50 +#define NO_OF_WORDS_20MS 19 +#define NO_OF_WORDS_30MS 25 +#define STATE_BITS 3 +#define BYTE_LEN 8 +#define ULP_CLASSES 3 + +/* help parameters */ + +#define TWO_PI_FIX 25736 /* Q12 */ + +/* Constants for codebook search and creation */ + +#define ST_MEM_L_TBL 85 +#define MEM_LF_TBL 147 + + +/* Struct for the bits */ +typedef struct iLBC_bits_t_ { + int16_t lsf[LSF_NSPLIT*LPC_N_MAX]; + int16_t cb_index[CB_NSTAGES*(NASUB_MAX+1)]; /* First CB_NSTAGES values contains extra CB index */ + int16_t gain_index[CB_NSTAGES*(NASUB_MAX+1)]; /* First CB_NSTAGES values contains extra CB gain */ + size_t idxForMax; + int16_t state_first; + int16_t idxVec[STATE_SHORT_LEN_30MS]; + int16_t firstbits; + size_t startIdx; +} iLBC_bits; + +/* type definition encoder instance */ +typedef struct IlbcEncoder_ { + + /* flag for frame size mode */ + int16_t mode; + + /* basic parameters for different frame sizes */ + size_t blockl; + size_t nsub; + int16_t nasub; + size_t no_of_bytes, no_of_words; + int16_t lpc_n; + size_t state_short_len; + + /* analysis filter state */ + int16_t anaMem[LPC_FILTERORDER]; + + /* Fix-point old lsf parameters for interpolation */ + int16_t lsfold[LPC_FILTERORDER]; + int16_t lsfdeqold[LPC_FILTERORDER]; + + /* signal buffer for LP analysis */ + int16_t lpc_buffer[LPC_LOOKBACK + BLOCKL_MAX]; + + /* state of input HP filter */ + int16_t hpimemx[2]; + int16_t hpimemy[4]; + +#ifdef SPLIT_10MS + int16_t weightdenumbuf[66]; + int16_t past_samples[160]; + uint16_t bytes[25]; + int16_t section; + int16_t Nfor_flag; + int16_t Nback_flag; + int16_t start_pos; + size_t diff; +#endif + +} IlbcEncoder; + +/* type definition decoder instance */ +typedef struct IlbcDecoder_ { + + /* flag for frame size mode */ + int16_t mode; + + /* basic parameters for different frame sizes */ + size_t blockl; + size_t nsub; + int16_t nasub; + size_t no_of_bytes, no_of_words; + int16_t lpc_n; + size_t state_short_len; + + /* synthesis filter state */ + int16_t syntMem[LPC_FILTERORDER]; + + /* old LSF for interpolation */ + int16_t lsfdeqold[LPC_FILTERORDER]; + + /* pitch lag estimated in enhancer and used in PLC */ + size_t last_lag; + + /* PLC state information */ + int consPLICount, prev_enh_pl; + int16_t perSquare; + + int16_t prevScale, prevPLI; + size_t prevLag; + int16_t prevLpc[LPC_FILTERORDER+1]; + int16_t prevResidual[NSUB_MAX*SUBL]; + int16_t seed; + + /* previous synthesis filter parameters */ + + int16_t old_syntdenum[(LPC_FILTERORDER + 1)*NSUB_MAX]; + + /* state of output HP filter */ + int16_t hpimemx[2]; + int16_t hpimemy[4]; + + /* enhancer state information */ + int use_enhancer; + int16_t enh_buf[ENH_BUFL+ENH_BUFL_FILTEROVERHEAD]; + size_t enh_period[ENH_NBLOCKS_TOT]; + +} IlbcDecoder; + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.c new file mode 100644 index 0000000000..ebddec3f5e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.c @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DoThePlc.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "comp_corr.h" +#include "bw_expand.h" + +/*----------------------------------------------------------------* + * Packet loss concealment routine. Conceals a residual signal + * and LP parameters. If no packet loss, update state. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DoThePlc( + int16_t *PLCresidual, /* (o) concealed residual */ + int16_t *PLClpc, /* (o) concealed LP parameters */ + int16_t PLI, /* (i) packet loss indicator + 0 - no PL, 1 = PL */ + int16_t *decresidual, /* (i) decoded residual */ + int16_t *lpc, /* (i) decoded LPC (only used for no PL) */ + size_t inlag, /* (i) pitch lag */ + IlbcDecoder *iLBCdec_inst + /* (i/o) decoder instance */ + ){ + size_t i; + int32_t cross, ener, cross_comp, ener_comp = 0; + int32_t measure, maxMeasure, energy; + int16_t max, crossSquareMax, crossSquare; + size_t j, lag, randlag; + int16_t tmp1, tmp2; + int16_t shift1, shift2, shift3, shiftMax; + int16_t scale3; + size_t corrLen; + int32_t tmpW32, tmp2W32; + int16_t use_gain; + int16_t tot_gain; + int16_t max_perSquare; + int16_t scale1, scale2; + int16_t totscale; + int32_t nom; + int16_t denom; + int16_t pitchfact; + size_t use_lag; + int ind; + int16_t randvec[BLOCKL_MAX]; + + /* Packet Loss */ + if (PLI == 1) { + + (*iLBCdec_inst).consPLICount += 1; + + /* if previous frame not lost, + determine pitch pred. gain */ + + if (iLBCdec_inst->prevPLI != 1) { + + /* Maximum 60 samples are correlated, preserve as high accuracy + as possible without getting overflow */ + max = WebRtcSpl_MaxAbsValueW16((*iLBCdec_inst).prevResidual, + iLBCdec_inst->blockl); + scale3 = (WebRtcSpl_GetSizeInBits(max)<<1) - 25; + if (scale3 < 0) { + scale3 = 0; + } + + /* Store scale for use when interpolating between the + * concealment and the received packet */ + iLBCdec_inst->prevScale = scale3; + + /* Search around the previous lag +/-3 to find the + best pitch period */ + lag = inlag - 3; + + /* Guard against getting outside the frame */ + corrLen = (size_t)WEBRTC_SPL_MIN(60, iLBCdec_inst->blockl-(inlag+3)); + + WebRtcIlbcfix_CompCorr( &cross, &ener, + iLBCdec_inst->prevResidual, lag, iLBCdec_inst->blockl, corrLen, scale3); + + /* Normalize and store cross^2 and the number of shifts */ + shiftMax = WebRtcSpl_GetSizeInBits(WEBRTC_SPL_ABS_W32(cross))-15; + crossSquareMax = (int16_t)(( + (int16_t)WEBRTC_SPL_SHIFT_W32(cross, -shiftMax) * + (int16_t)WEBRTC_SPL_SHIFT_W32(cross, -shiftMax)) >> 15); + + for (j=inlag-2;j<=inlag+3;j++) { + WebRtcIlbcfix_CompCorr( &cross_comp, &ener_comp, + iLBCdec_inst->prevResidual, j, iLBCdec_inst->blockl, corrLen, scale3); + + /* Use the criteria (corr*corr)/energy to compare if + this lag is better or not. To avoid the division, + do a cross multiplication */ + shift1 = WebRtcSpl_GetSizeInBits(WEBRTC_SPL_ABS_W32(cross_comp))-15; + crossSquare = (int16_t)(( + (int16_t)WEBRTC_SPL_SHIFT_W32(cross_comp, -shift1) * + (int16_t)WEBRTC_SPL_SHIFT_W32(cross_comp, -shift1)) >> 15); + + shift2 = WebRtcSpl_GetSizeInBits(ener)-15; + measure = (int16_t)WEBRTC_SPL_SHIFT_W32(ener, -shift2) * crossSquare; + + shift3 = WebRtcSpl_GetSizeInBits(ener_comp)-15; + maxMeasure = (int16_t)WEBRTC_SPL_SHIFT_W32(ener_comp, -shift3) * + crossSquareMax; + + /* Calculate shift value, so that the two measures can + be put in the same Q domain */ + if(2 * shiftMax + shift3 > 2 * shift1 + shift2) { + tmp1 = + WEBRTC_SPL_MIN(31, 2 * shiftMax + shift3 - 2 * shift1 - shift2); + tmp2 = 0; + } else { + tmp1 = 0; + tmp2 = + WEBRTC_SPL_MIN(31, 2 * shift1 + shift2 - 2 * shiftMax - shift3); + } + + if ((measure>>tmp1) > (maxMeasure>>tmp2)) { + /* New lag is better => record lag, measure and domain */ + lag = j; + crossSquareMax = crossSquare; + cross = cross_comp; + shiftMax = shift1; + ener = ener_comp; + } + } + + /* Calculate the periodicity for the lag with the maximum correlation. + + Definition of the periodicity: + abs(corr(vec1, vec2))/(sqrt(energy(vec1))*sqrt(energy(vec2))) + + Work in the Square domain to simplify the calculations + max_perSquare is less than 1 (in Q15) + */ + tmp2W32=WebRtcSpl_DotProductWithScale(&iLBCdec_inst->prevResidual[iLBCdec_inst->blockl-corrLen], + &iLBCdec_inst->prevResidual[iLBCdec_inst->blockl-corrLen], + corrLen, scale3); + + if ((tmp2W32>0)&&(ener_comp>0)) { + /* norm energies to int16_t, compute the product of the energies and + use the upper int16_t as the denominator */ + + scale1=(int16_t)WebRtcSpl_NormW32(tmp2W32)-16; + tmp1=(int16_t)WEBRTC_SPL_SHIFT_W32(tmp2W32, scale1); + + scale2=(int16_t)WebRtcSpl_NormW32(ener)-16; + tmp2=(int16_t)WEBRTC_SPL_SHIFT_W32(ener, scale2); + denom = (int16_t)((tmp1 * tmp2) >> 16); /* in Q(scale1+scale2-16) */ + + /* Square the cross correlation and norm it such that max_perSquare + will be in Q15 after the division */ + + totscale = scale1+scale2-1; + tmp1 = (int16_t)WEBRTC_SPL_SHIFT_W32(cross, (totscale>>1)); + tmp2 = (int16_t)WEBRTC_SPL_SHIFT_W32(cross, totscale-(totscale>>1)); + + nom = tmp1 * tmp2; + max_perSquare = (int16_t)WebRtcSpl_DivW32W16(nom, denom); + + } else { + max_perSquare = 0; + } + } + + /* previous frame lost, use recorded lag and gain */ + + else { + lag = iLBCdec_inst->prevLag; + max_perSquare = iLBCdec_inst->perSquare; + } + + /* Attenuate signal and scale down pitch pred gain if + several frames lost consecutively */ + + use_gain = 32767; /* 1.0 in Q15 */ + + if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>320) { + use_gain = 29491; /* 0.9 in Q15 */ + } else if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>640) { + use_gain = 22938; /* 0.7 in Q15 */ + } else if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>960) { + use_gain = 16384; /* 0.5 in Q15 */ + } else if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>1280) { + use_gain = 0; /* 0.0 in Q15 */ + } + + /* Compute mixing factor of picth repeatition and noise: + for max_per>0.7 set periodicity to 1.0 + 0.47868) { /* periodicity > 0.7 (0.7^4=0.2401 in Q15) */ + pitchfact = 32767; + } else if (max_perSquare>839) { /* 0.4 < periodicity < 0.7 (0.4^4=0.0256 in Q15) */ + /* find best index and interpolate from that */ + ind = 5; + while ((max_perSquare0)) { + ind--; + } + /* pitch fact is approximated by first order */ + tmpW32 = (int32_t)WebRtcIlbcfix_kPlcPitchFact[ind] + + ((WebRtcIlbcfix_kPlcPfSlope[ind] * + (max_perSquare - WebRtcIlbcfix_kPlcPerSqr[ind])) >> 11); + + pitchfact = (int16_t)WEBRTC_SPL_MIN(tmpW32, 32767); /* guard against overflow */ + + } else { /* periodicity < 0.4 */ + pitchfact = 0; + } + + /* avoid repetition of same pitch cycle (buzzyness) */ + use_lag = lag; + if (lag<80) { + use_lag = 2*lag; + } + + /* compute concealed residual */ + energy = 0; + + for (i=0; iblockl; i++) { + + /* noise component - 52 < randlagFIX < 117 */ + iLBCdec_inst->seed = (int16_t)(iLBCdec_inst->seed * 31821 + 13849); + randlag = 53 + (iLBCdec_inst->seed & 63); + if (randlag > i) { + randvec[i] = + iLBCdec_inst->prevResidual[iLBCdec_inst->blockl + i - randlag]; + } else { + randvec[i] = iLBCdec_inst->prevResidual[i - randlag]; + } + + /* pitch repeatition component */ + if (use_lag > i) { + PLCresidual[i] = + iLBCdec_inst->prevResidual[iLBCdec_inst->blockl + i - use_lag]; + } else { + PLCresidual[i] = PLCresidual[i - use_lag]; + } + + /* Attinuate total gain for each 10 ms */ + if (i<80) { + tot_gain=use_gain; + } else if (i<160) { + tot_gain = (int16_t)((31130 * use_gain) >> 15); /* 0.95*use_gain */ + } else { + tot_gain = (int16_t)((29491 * use_gain) >> 15); /* 0.9*use_gain */ + } + + + /* mix noise and pitch repeatition */ + PLCresidual[i] = (int16_t)((tot_gain * + ((pitchfact * PLCresidual[i] + (32767 - pitchfact) * randvec[i] + + 16384) >> 15)) >> 15); + + /* Shifting down the result one step extra to ensure that no overflow + will occur */ + energy += (PLCresidual[i] * PLCresidual[i]) >> + (iLBCdec_inst->prevScale + 1); + } + + /* less than 30 dB, use only noise */ + if (energy < (WEBRTC_SPL_SHIFT_W32(((int32_t)iLBCdec_inst->blockl*900),-(iLBCdec_inst->prevScale+1)))) { + energy = 0; + for (i=0; iblockl; i++) { + PLCresidual[i] = randvec[i]; + } + } + + /* use the old LPC */ + WEBRTC_SPL_MEMCPY_W16(PLClpc, (*iLBCdec_inst).prevLpc, LPC_FILTERORDER+1); + + /* Update state in case there are multiple frame losses */ + iLBCdec_inst->prevLag = lag; + iLBCdec_inst->perSquare = max_perSquare; + } + + /* no packet loss, copy input */ + + else { + WEBRTC_SPL_MEMCPY_W16(PLCresidual, decresidual, iLBCdec_inst->blockl); + WEBRTC_SPL_MEMCPY_W16(PLClpc, lpc, (LPC_FILTERORDER+1)); + iLBCdec_inst->consPLICount = 0; + } + + /* update state */ + iLBCdec_inst->prevPLI = PLI; + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->prevLpc, PLClpc, (LPC_FILTERORDER+1)); + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->prevResidual, PLCresidual, iLBCdec_inst->blockl); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.h new file mode 100644 index 0000000000..76a598a9c6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DoThePlc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DO_PLC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DO_PLC_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Packet loss concealment routine. Conceals a residual signal + * and LP parameters. If no packet loss, update state. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DoThePlc( + int16_t *PLCresidual, /* (o) concealed residual */ + int16_t *PLClpc, /* (o) concealed LP parameters */ + int16_t PLI, /* (i) packet loss indicator + 0 - no PL, 1 = PL */ + int16_t *decresidual, /* (i) decoded residual */ + int16_t *lpc, /* (i) decoded LPC (only used for no PL) */ + size_t inlag, /* (i) pitch lag */ + IlbcDecoder *iLBCdec_inst + /* (i/o) decoder instance */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.c new file mode 100644 index 0000000000..72543555be --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.c @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Encode.c + +******************************************************************/ + +#include + +#include "defines.h" +#include "lpc_encode.h" +#include "frame_classify.h" +#include "state_search.h" +#include "state_construct.h" +#include "constants.h" +#include "cb_search.h" +#include "cb_construct.h" +#include "index_conv_enc.h" +#include "pack_bits.h" +#include "hp_input.h" +#include "rtc_base/checks.h" + +#ifdef SPLIT_10MS +#include "unpack_bits.h" +#include "index_conv_dec.h" +#endif +#ifndef WEBRTC_ARCH_BIG_ENDIAN +#include "swap_bytes.h" +#endif + +/*----------------------------------------------------------------* + * main encoder function + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_EncodeImpl( + uint16_t *bytes, /* (o) encoded data bits iLBC */ + const int16_t *block, /* (i) speech vector to encode */ + IlbcEncoder *iLBCenc_inst /* (i/o) the general encoder + state */ + ){ + size_t n, meml_gotten, Nfor; + size_t diff, start_pos; + size_t index; + size_t subcount, subframe; + size_t start_count, end_count; + int16_t *residual; + int32_t en1, en2; + int16_t scale, max; + int16_t *syntdenum; + int16_t *decresidual; + int16_t *reverseResidual; + int16_t *reverseDecresidual; + /* Stack based */ + int16_t weightdenum[(LPC_FILTERORDER + 1)*NSUB_MAX]; + int16_t dataVec[BLOCKL_MAX + LPC_FILTERORDER]; + int16_t memVec[CB_MEML+CB_FILTERLEN]; + int16_t bitsMemory[sizeof(iLBC_bits)/sizeof(int16_t)]; + iLBC_bits *iLBCbits_inst = (iLBC_bits*)bitsMemory; + + +#ifdef SPLIT_10MS + int16_t *weightdenumbuf = iLBCenc_inst->weightdenumbuf; + int16_t last_bit; +#endif + + int16_t *data = &dataVec[LPC_FILTERORDER]; + int16_t *mem = &memVec[CB_HALFFILTERLEN]; + + /* Reuse som buffers to save stack memory */ + residual = &iLBCenc_inst->lpc_buffer[LPC_LOOKBACK+BLOCKL_MAX-iLBCenc_inst->blockl]; + syntdenum = mem; /* syntdenum[(LPC_FILTERORDER + 1)*NSUB_MAX] and mem are used non overlapping in the code */ + decresidual = residual; /* Already encoded residual is overwritten by the decoded version */ + reverseResidual = data; /* data and reverseResidual are used non overlapping in the code */ + reverseDecresidual = reverseResidual; /* Already encoded residual is overwritten by the decoded version */ + +#ifdef SPLIT_10MS + + WebRtcSpl_MemSetW16 ( (int16_t *) iLBCbits_inst, 0, + sizeof(iLBC_bits) / sizeof(int16_t) ); + + start_pos = iLBCenc_inst->start_pos; + diff = iLBCenc_inst->diff; + + if (iLBCenc_inst->section != 0){ + WEBRTC_SPL_MEMCPY_W16 (weightdenum, weightdenumbuf, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + /* Un-Packetize the frame into parameters */ + last_bit = WebRtcIlbcfix_UnpackBits (iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + if (last_bit) + return; + /* adjust index */ + WebRtcIlbcfix_IndexConvDec (iLBCbits_inst->cb_index); + + if (iLBCenc_inst->section == 1){ + /* Save first 80 samples of a 160/240 sample frame for 20/30msec */ + WEBRTC_SPL_MEMCPY_W16 (iLBCenc_inst->past_samples, block, 80); + } + else{ // iLBCenc_inst->section == 2 AND mode = 30ms + /* Save second 80 samples of a 240 sample frame for 30msec */ + WEBRTC_SPL_MEMCPY_W16 (iLBCenc_inst->past_samples + 80, block, 80); + } + } + else{ // iLBCenc_inst->section == 0 + /* form a complete frame of 160/240 for 20msec/30msec mode */ + WEBRTC_SPL_MEMCPY_W16 (data + (iLBCenc_inst->mode * 8) - 80, block, 80); + WEBRTC_SPL_MEMCPY_W16 (data, iLBCenc_inst->past_samples, + (iLBCenc_inst->mode * 8) - 80); + iLBCenc_inst->Nfor_flag = 0; + iLBCenc_inst->Nback_flag = 0; +#else + /* copy input block to data*/ + WEBRTC_SPL_MEMCPY_W16(data,block,iLBCenc_inst->blockl); +#endif + + /* high pass filtering of input signal and scale down the residual (*0.5) */ + WebRtcIlbcfix_HpInput(data, (int16_t*)WebRtcIlbcfix_kHpInCoefs, + iLBCenc_inst->hpimemy, iLBCenc_inst->hpimemx, + iLBCenc_inst->blockl); + + /* LPC of hp filtered input data */ + WebRtcIlbcfix_LpcEncode(syntdenum, weightdenum, iLBCbits_inst->lsf, data, + iLBCenc_inst); + + /* Set up state */ + WEBRTC_SPL_MEMCPY_W16(dataVec, iLBCenc_inst->anaMem, LPC_FILTERORDER); + + /* inverse filter to get residual */ + for (n=0; nnsub; n++ ) { + WebRtcSpl_FilterMAFastQ12( + &data[n*SUBL], &residual[n*SUBL], + &syntdenum[n*(LPC_FILTERORDER+1)], + LPC_FILTERORDER+1, SUBL); + } + + /* Copy the state for next frame */ + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->anaMem, &data[iLBCenc_inst->blockl-LPC_FILTERORDER], LPC_FILTERORDER); + + /* find state location */ + + iLBCbits_inst->startIdx = WebRtcIlbcfix_FrameClassify(iLBCenc_inst,residual); + + /* check if state should be in first or last part of the + two subframes */ + + index = (iLBCbits_inst->startIdx-1)*SUBL; + max=WebRtcSpl_MaxAbsValueW16(&residual[index], 2*SUBL); + scale = WebRtcSpl_GetSizeInBits((uint32_t)(max * max)); + + /* Scale to maximum 25 bits so that the MAC won't cause overflow */ + scale = scale - 25; + if(scale < 0) { + scale = 0; + } + + diff = STATE_LEN - iLBCenc_inst->state_short_len; + en1=WebRtcSpl_DotProductWithScale(&residual[index], &residual[index], + iLBCenc_inst->state_short_len, scale); + index += diff; + en2=WebRtcSpl_DotProductWithScale(&residual[index], &residual[index], + iLBCenc_inst->state_short_len, scale); + if (en1 > en2) { + iLBCbits_inst->state_first = 1; + start_pos = (iLBCbits_inst->startIdx-1)*SUBL; + } else { + iLBCbits_inst->state_first = 0; + start_pos = (iLBCbits_inst->startIdx-1)*SUBL + diff; + } + + /* scalar quantization of state */ + + WebRtcIlbcfix_StateSearch(iLBCenc_inst, iLBCbits_inst, &residual[start_pos], + &syntdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)], + &weightdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)]); + + WebRtcIlbcfix_StateConstruct(iLBCbits_inst->idxForMax, iLBCbits_inst->idxVec, + &syntdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)], + &decresidual[start_pos], iLBCenc_inst->state_short_len + ); + + /* predictive quantization in state */ + + if (iLBCbits_inst->state_first) { /* put adaptive part in the end */ + + /* setup memory */ + + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - iLBCenc_inst->state_short_len); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-iLBCenc_inst->state_short_len, + decresidual+start_pos, iLBCenc_inst->state_short_len); + + /* encode subframes */ + + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index, iLBCbits_inst->gain_index, + &residual[start_pos+iLBCenc_inst->state_short_len], + mem+CB_MEML-ST_MEM_L_TBL, ST_MEM_L_TBL, diff, + &weightdenum[iLBCbits_inst->startIdx*(LPC_FILTERORDER+1)], 0); + + /* construct decoded vector */ + + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + &decresidual[start_pos + iLBCenc_inst->state_short_len], + iLBCbits_inst->cb_index, iLBCbits_inst->gain_index, + mem + CB_MEML - ST_MEM_L_TBL, ST_MEM_L_TBL, diff)); + + } + else { /* put adaptive part in the beginning */ + + /* create reversed vectors for prediction */ + + WebRtcSpl_MemCpyReversedOrder(&reverseResidual[diff-1], + &residual[(iLBCbits_inst->startIdx+1)*SUBL-STATE_LEN], diff); + + /* setup memory */ + + meml_gotten = iLBCenc_inst->state_short_len; + WebRtcSpl_MemCpyReversedOrder(&mem[CB_MEML-1], &decresidual[start_pos], meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - iLBCenc_inst->state_short_len); + + /* encode subframes */ + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index, iLBCbits_inst->gain_index, + reverseResidual, mem+CB_MEML-ST_MEM_L_TBL, ST_MEM_L_TBL, diff, + &weightdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)], + 0); + + /* construct decoded vector */ + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + reverseDecresidual, iLBCbits_inst->cb_index, + iLBCbits_inst->gain_index, mem + CB_MEML - ST_MEM_L_TBL, + ST_MEM_L_TBL, diff)); + + /* get decoded residual from reversed vector */ + + WebRtcSpl_MemCpyReversedOrder(&decresidual[start_pos-1], reverseDecresidual, diff); + } + +#ifdef SPLIT_10MS + iLBCenc_inst->start_pos = start_pos; + iLBCenc_inst->diff = diff; + iLBCenc_inst->section++; + /* adjust index */ + WebRtcIlbcfix_IndexConvEnc (iLBCbits_inst->cb_index); + /* Packetize the parameters into the frame */ + WebRtcIlbcfix_PackBits (iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + WEBRTC_SPL_MEMCPY_W16 (weightdenumbuf, weightdenum, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + return; + } +#endif + + /* forward prediction of subframes */ + + Nfor = iLBCenc_inst->nsub-iLBCbits_inst->startIdx-1; + + /* counter for predicted subframes */ +#ifdef SPLIT_10MS + if (iLBCenc_inst->mode == 20) + { + subcount = 1; + } + if (iLBCenc_inst->mode == 30) + { + if (iLBCenc_inst->section == 1) + { + subcount = 1; + } + if (iLBCenc_inst->section == 2) + { + subcount = 3; + } + } +#else + subcount=1; +#endif + + if( Nfor > 0 ){ + + /* setup memory */ + + WebRtcSpl_MemSetW16(mem, 0, CB_MEML-STATE_LEN); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-STATE_LEN, + decresidual+(iLBCbits_inst->startIdx-1)*SUBL, STATE_LEN); + +#ifdef SPLIT_10MS + if (iLBCenc_inst->Nfor_flag > 0) + { + for (subframe = 0; subframe < WEBRTC_SPL_MIN (Nfor, 2); subframe++) + { + /* update memory */ + WEBRTC_SPL_MEMCPY_W16 (mem, mem + SUBL, (CB_MEML - SUBL)); + WEBRTC_SPL_MEMCPY_W16 (mem + CB_MEML - SUBL, + &decresidual[(iLBCbits_inst->startIdx + 1 + + subframe) * SUBL], SUBL); + } + } + + iLBCenc_inst->Nfor_flag++; + + if (iLBCenc_inst->mode == 20) + { + start_count = 0; + end_count = Nfor; + } + if (iLBCenc_inst->mode == 30) + { + if (iLBCenc_inst->section == 1) + { + start_count = 0; + end_count = WEBRTC_SPL_MIN (Nfor, (size_t)2); + } + if (iLBCenc_inst->section == 2) + { + start_count = WEBRTC_SPL_MIN (Nfor, (size_t)2); + end_count = Nfor; + } + } +#else + start_count = 0; + end_count = Nfor; +#endif + + /* loop over subframes to encode */ + + for (subframe = start_count; subframe < end_count; subframe++){ + + /* encode subframe */ + + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index+subcount*CB_NSTAGES, + iLBCbits_inst->gain_index+subcount*CB_NSTAGES, + &residual[(iLBCbits_inst->startIdx+1+subframe)*SUBL], + mem, MEM_LF_TBL, SUBL, + &weightdenum[(iLBCbits_inst->startIdx+1+subframe)*(LPC_FILTERORDER+1)], + subcount); + + /* construct decoded vector */ + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + &decresidual[(iLBCbits_inst->startIdx + 1 + subframe) * SUBL], + iLBCbits_inst->cb_index + subcount * CB_NSTAGES, + iLBCbits_inst->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)); + + /* update memory */ + + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &decresidual[(iLBCbits_inst->startIdx+1+subframe)*SUBL], SUBL); + + subcount++; + } + } + +#ifdef SPLIT_10MS + if ((iLBCenc_inst->section == 1) && + (iLBCenc_inst->mode == 30) && (Nfor > 0) && (end_count == 2)) + { + iLBCenc_inst->section++; + /* adjust index */ + WebRtcIlbcfix_IndexConvEnc (iLBCbits_inst->cb_index); + /* Packetize the parameters into the frame */ + WebRtcIlbcfix_PackBits (iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + WEBRTC_SPL_MEMCPY_W16 (weightdenumbuf, weightdenum, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + return; + } +#endif + + /* backward prediction of subframes */ + + if (iLBCbits_inst->startIdx > 1) { + + /* create reverse order vectors + (The decresidual does not need to be copied since it is + contained in the same vector as the residual) + */ + + size_t Nback = iLBCbits_inst->startIdx - 1; + WebRtcSpl_MemCpyReversedOrder(&reverseResidual[Nback*SUBL-1], residual, Nback*SUBL); + + /* setup memory */ + + meml_gotten = SUBL*(iLBCenc_inst->nsub+1-iLBCbits_inst->startIdx); + if( meml_gotten > CB_MEML ) { + meml_gotten=CB_MEML; + } + + WebRtcSpl_MemCpyReversedOrder(&mem[CB_MEML-1], &decresidual[Nback*SUBL], meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - meml_gotten); + +#ifdef SPLIT_10MS + if (iLBCenc_inst->Nback_flag > 0) + { + for (subframe = 0; subframe < WEBRTC_SPL_MAX (2 - Nfor, 0); subframe++) + { + /* update memory */ + WEBRTC_SPL_MEMCPY_W16 (mem, mem + SUBL, (CB_MEML - SUBL)); + WEBRTC_SPL_MEMCPY_W16 (mem + CB_MEML - SUBL, + &reverseDecresidual[subframe * SUBL], SUBL); + } + } + + iLBCenc_inst->Nback_flag++; + + + if (iLBCenc_inst->mode == 20) + { + start_count = 0; + end_count = Nback; + } + if (iLBCenc_inst->mode == 30) + { + if (iLBCenc_inst->section == 1) + { + start_count = 0; + end_count = (Nfor >= 2) ? 0 : (2 - NFor); + } + if (iLBCenc_inst->section == 2) + { + start_count = (Nfor >= 2) ? 0 : (2 - NFor); + end_count = Nback; + } + } +#else + start_count = 0; + end_count = Nback; +#endif + + /* loop over subframes to encode */ + + for (subframe = start_count; subframe < end_count; subframe++){ + + /* encode subframe */ + + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index+subcount*CB_NSTAGES, + iLBCbits_inst->gain_index+subcount*CB_NSTAGES, &reverseResidual[subframe*SUBL], + mem, MEM_LF_TBL, SUBL, + &weightdenum[(iLBCbits_inst->startIdx-2-subframe)*(LPC_FILTERORDER+1)], + subcount); + + /* construct decoded vector */ + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + &reverseDecresidual[subframe * SUBL], + iLBCbits_inst->cb_index + subcount * CB_NSTAGES, + iLBCbits_inst->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)); + + /* update memory */ + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &reverseDecresidual[subframe*SUBL], SUBL); + + subcount++; + + } + + /* get decoded residual from reversed vector */ + + WebRtcSpl_MemCpyReversedOrder(&decresidual[SUBL*Nback-1], reverseDecresidual, SUBL*Nback); + } + /* end encoding part */ + + /* adjust index */ + + WebRtcIlbcfix_IndexConvEnc(iLBCbits_inst->cb_index); + + /* Packetize the parameters into the frame */ + +#ifdef SPLIT_10MS + if( (iLBCenc_inst->mode==30) && (iLBCenc_inst->section==1) ){ + WebRtcIlbcfix_PackBits(iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + } + else{ + WebRtcIlbcfix_PackBits(bytes, iLBCbits_inst, iLBCenc_inst->mode); + } +#else + WebRtcIlbcfix_PackBits(bytes, iLBCbits_inst, iLBCenc_inst->mode); +#endif + +#ifndef WEBRTC_ARCH_BIG_ENDIAN + /* Swap bytes for LITTLE ENDIAN since the packbits() + function assumes BIG_ENDIAN machine */ +#ifdef SPLIT_10MS + if (( (iLBCenc_inst->section == 1) && (iLBCenc_inst->mode == 20) ) || + ( (iLBCenc_inst->section == 2) && (iLBCenc_inst->mode == 30) )){ + WebRtcIlbcfix_SwapBytes(bytes, iLBCenc_inst->no_of_words, bytes); + } +#else + WebRtcIlbcfix_SwapBytes(bytes, iLBCenc_inst->no_of_words, bytes); +#endif +#endif + +#ifdef SPLIT_10MS + if (subcount == (iLBCenc_inst->nsub - 1)) + { + iLBCenc_inst->section = 0; + } + else + { + iLBCenc_inst->section++; + WEBRTC_SPL_MEMCPY_W16 (weightdenumbuf, weightdenum, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + } +#endif + +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.h new file mode 100644 index 0000000000..0692a34568 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Encode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ENCODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ENCODE_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * main encoder function + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_EncodeImpl( + uint16_t *bytes, /* (o) encoded data bits iLBC */ + const int16_t *block, /* (i) speech vector to encode */ + IlbcEncoder *iLBCenc_inst /* (i/o) the general encoder + state */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c new file mode 100644 index 0000000000..b2bdcfffc3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_EnergyInverse.c + +******************************************************************/ + +/* Inverses the in vector in into Q29 domain */ + +#include "energy_inverse.h" + +void WebRtcIlbcfix_EnergyInverse( + int16_t *energy, /* (i/o) Energy and inverse + energy (in Q29) */ + size_t noOfEnergies) /* (i) The length of the energy + vector */ +{ + int32_t Nom=(int32_t)0x1FFFFFFF; + int16_t *energyPtr; + size_t i; + + /* Set the minimum energy value to 16384 to avoid overflow */ + energyPtr=energy; + for (i=0; i + +#include "defines.h" +#include "constants.h" +#include "xcorr_coef.h" +#include "enhancer.h" +#include "hp_output.h" + + + +/*----------------------------------------------------------------* + * interface for enhancer + *---------------------------------------------------------------*/ + +size_t // (o) Estimated lag in end of in[] + WebRtcIlbcfix_EnhancerInterface( + int16_t* out, // (o) enhanced signal + const int16_t* in, // (i) unenhanced signal + IlbcDecoder* iLBCdec_inst) { // (i) buffers etc + size_t iblock; + size_t lag=20, tlag=20; + size_t inLen=iLBCdec_inst->blockl+120; + int16_t scale, scale1; + size_t plc_blockl; + int16_t *enh_buf; + size_t *enh_period; + int32_t tmp1, tmp2, max; + size_t new_blocks; + int16_t *enh_bufPtr1; + size_t i; + size_t k; + int16_t EnChange; + int16_t SqrtEnChange; + int16_t inc; + int16_t win; + int16_t *tmpW16ptr; + size_t startPos; + int16_t *plc_pred; + const int16_t *target, *regressor; + int16_t max16; + int shifts; + int32_t ener; + int16_t enerSh; + int16_t corrSh; + size_t ind; + int16_t sh; + size_t start, stop; + /* Stack based */ + int16_t totsh[3]; + int16_t downsampled[(BLOCKL_MAX+120)>>1]; /* length 180 */ + int32_t corr32[50]; + int32_t corrmax[3]; + int16_t corr16[3]; + int16_t en16[3]; + size_t lagmax[3]; + + plc_pred = downsampled; /* Reuse memory since plc_pred[ENH_BLOCKL] and + downsampled are non overlapping */ + enh_buf=iLBCdec_inst->enh_buf; + enh_period=iLBCdec_inst->enh_period; + + /* Copy in the new data into the enhancer buffer */ + memmove(enh_buf, &enh_buf[iLBCdec_inst->blockl], + (ENH_BUFL - iLBCdec_inst->blockl) * sizeof(*enh_buf)); + + WEBRTC_SPL_MEMCPY_W16(&enh_buf[ENH_BUFL-iLBCdec_inst->blockl], in, + iLBCdec_inst->blockl); + + /* Set variables that are dependent on frame size */ + if (iLBCdec_inst->mode==30) { + plc_blockl=ENH_BLOCKL; + new_blocks=3; + startPos=320; /* Start position for enhancement + (640-new_blocks*ENH_BLOCKL-80) */ + } else { + plc_blockl=40; + new_blocks=2; + startPos=440; /* Start position for enhancement + (640-new_blocks*ENH_BLOCKL-40) */ + } + + /* Update the pitch prediction for each enhancer block, move the old ones */ + memmove(enh_period, &enh_period[new_blocks], + (ENH_NBLOCKS_TOT - new_blocks) * sizeof(*enh_period)); + + WebRtcSpl_DownsampleFast( + enh_buf+ENH_BUFL-inLen, /* Input samples */ + inLen + ENH_BUFL_FILTEROVERHEAD, + downsampled, + inLen / 2, + (int16_t*)WebRtcIlbcfix_kLpFiltCoefs, /* Coefficients in Q12 */ + FILTERORDER_DS_PLUS1, /* Length of filter (order-1) */ + FACTOR_DS, + DELAY_DS); + + /* Estimate the pitch in the down sampled domain. */ + for(iblock = 0; iblock> 16); + en16[i] = (int16_t)WEBRTC_SPL_SHIFT_W32(ener, enerSh); + totsh[i] = enerSh - 2 * corrSh; + } + + /* Compare lagmax[0..3] for the (corr^2)/ener criteria */ + ind = 0; + for (i=1; i<3; i++) { + if (totsh[ind] > totsh[i]) { + sh = WEBRTC_SPL_MIN(31, totsh[ind]-totsh[i]); + if (corr16[ind] * en16[i] < (corr16[i] * en16[ind]) >> sh) { + ind = i; + } + } else { + sh = WEBRTC_SPL_MIN(31, totsh[i]-totsh[ind]); + if ((corr16[ind] * en16[i]) >> sh < corr16[i] * en16[ind]) { + ind = i; + } + } + } + + lag = lagmax[ind] + 10; + + /* Store the estimated lag in the non-downsampled domain */ + enh_period[ENH_NBLOCKS_TOT - new_blocks + iblock] = lag * 8; + + /* Store the estimated lag for backward PLC */ + if (iLBCdec_inst->prev_enh_pl==1) { + if (!iblock) { + tlag = lag * 2; + } + } else { + if (iblock==1) { + tlag = lag * 2; + } + } + + lag *= 2; + } + + if ((iLBCdec_inst->prev_enh_pl==1)||(iLBCdec_inst->prev_enh_pl==2)) { + + /* Calculate the best lag of the new frame + This is used to interpolate backwards and mix with the PLC'd data + */ + + /* references */ + target=in; + regressor=in+tlag-1; + + /* scaling */ + max16 = WebRtcSpl_MaxAbsValueW16(regressor, plc_blockl + 3 - 1); + if (max16>5000) + shifts=2; + else + shifts=0; + + /* compute cross correlation */ + WebRtcSpl_CrossCorrelation(corr32, target, regressor, plc_blockl, 3, shifts, + 1); + + /* find lag */ + lag=WebRtcSpl_MaxIndexW32(corr32, 3); + lag+=tlag-1; + + /* Copy the backward PLC to plc_pred */ + + if (iLBCdec_inst->prev_enh_pl==1) { + if (lag>plc_blockl) { + WEBRTC_SPL_MEMCPY_W16(plc_pred, &in[lag-plc_blockl], plc_blockl); + } else { + WEBRTC_SPL_MEMCPY_W16(&plc_pred[plc_blockl-lag], in, lag); + WEBRTC_SPL_MEMCPY_W16( + plc_pred, &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl+lag], + (plc_blockl-lag)); + } + } else { + size_t pos; + + pos = plc_blockl; + + while (lagprev_enh_pl==1) { + /* limit energy change + if energy in backward PLC is more than 4 times higher than the forward + PLC, then reduce the energy in the backward PLC vector: + sample 1...len-16 set energy of the to 4 times forward PLC + sample len-15..len interpolate between 4 times fw PLC and bw PLC energy + + Note: Compared to floating point code there is a slight change, + the window is 16 samples long instead of 10 samples to simplify the + calculations + */ + + max=WebRtcSpl_MaxAbsValueW16( + &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl], plc_blockl); + max16=WebRtcSpl_MaxAbsValueW16(plc_pred, plc_blockl); + max = WEBRTC_SPL_MAX(max, max16); + scale=22-(int16_t)WebRtcSpl_NormW32(max); + scale=WEBRTC_SPL_MAX(scale,0); + + tmp2 = WebRtcSpl_DotProductWithScale( + &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl], + &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl], + plc_blockl, scale); + tmp1 = WebRtcSpl_DotProductWithScale(plc_pred, plc_pred, + plc_blockl, scale); + + /* Check the energy difference */ + if ((tmp1>0)&&((tmp1>>2)>tmp2)) { + /* EnChange is now guaranteed to be <0.5 + Calculate EnChange=tmp2/tmp1 in Q16 + */ + + scale1=(int16_t)WebRtcSpl_NormW32(tmp1); + tmp1=WEBRTC_SPL_SHIFT_W32(tmp1, (scale1-16)); /* using 15 bits */ + + tmp2=WEBRTC_SPL_SHIFT_W32(tmp2, (scale1)); + EnChange = (int16_t)WebRtcSpl_DivW32W16(tmp2, + (int16_t)tmp1); + + /* Calculate the Sqrt of the energy in Q15 ((14+16)/2) */ + SqrtEnChange = (int16_t)WebRtcSpl_SqrtFloor(EnChange << 14); + + + /* Multiply first part of vector with 2*SqrtEnChange */ + WebRtcSpl_ScaleVector(plc_pred, plc_pred, SqrtEnChange, plc_blockl-16, + 14); + + /* Calculate increase parameter for window part (16 last samples) */ + /* (1-2*SqrtEnChange)/16 in Q15 */ + inc = 2048 - (SqrtEnChange >> 3); + + win=0; + tmpW16ptr=&plc_pred[plc_blockl-16]; + + for (i=16;i>0;i--) { + *tmpW16ptr = (int16_t)( + (*tmpW16ptr * (SqrtEnChange + (win >> 1))) >> 14); + /* multiply by (2.0*SqrtEnChange+win) */ + + win += inc; + tmpW16ptr++; + } + } + + /* Make the linear interpolation between the forward PLC'd data + and the backward PLC'd data (from the new frame) + */ + + if (plc_blockl==40) { + inc=400; /* 1/41 in Q14 */ + } else { /* plc_blockl==80 */ + inc=202; /* 1/81 in Q14 */ + } + win=0; + enh_bufPtr1=&enh_buf[ENH_BUFL-1-iLBCdec_inst->blockl]; + for (i=0; i> 14); + *enh_bufPtr1 += (int16_t)( + ((16384 - win) * plc_pred[plc_blockl - 1 - i]) >> 14); + enh_bufPtr1--; + } + } else { + int16_t *synt = &downsampled[LPC_FILTERORDER]; + + enh_bufPtr1=&enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl]; + WEBRTC_SPL_MEMCPY_W16(enh_bufPtr1, plc_pred, plc_blockl); + + /* Clear fileter memory */ + WebRtcSpl_MemSetW16(iLBCdec_inst->syntMem, 0, LPC_FILTERORDER); + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemy, 0, 4); + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemx, 0, 2); + + /* Initialize filter memory by filtering through 2 lags */ + WEBRTC_SPL_MEMCPY_W16(&synt[-LPC_FILTERORDER], iLBCdec_inst->syntMem, + LPC_FILTERORDER); + WebRtcSpl_FilterARFastQ12( + enh_bufPtr1, + synt, + &iLBCdec_inst->old_syntdenum[ + (iLBCdec_inst->nsub-1)*(LPC_FILTERORDER+1)], + LPC_FILTERORDER+1, lag); + + WEBRTC_SPL_MEMCPY_W16(&synt[-LPC_FILTERORDER], &synt[lag-LPC_FILTERORDER], + LPC_FILTERORDER); + WebRtcIlbcfix_HpOutput(synt, (int16_t*)WebRtcIlbcfix_kHpOutCoefs, + iLBCdec_inst->hpimemy, iLBCdec_inst->hpimemx, + lag); + WebRtcSpl_FilterARFastQ12( + enh_bufPtr1, synt, + &iLBCdec_inst->old_syntdenum[ + (iLBCdec_inst->nsub-1)*(LPC_FILTERORDER+1)], + LPC_FILTERORDER+1, lag); + + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->syntMem, &synt[lag-LPC_FILTERORDER], + LPC_FILTERORDER); + WebRtcIlbcfix_HpOutput(synt, (int16_t*)WebRtcIlbcfix_kHpOutCoefs, + iLBCdec_inst->hpimemy, iLBCdec_inst->hpimemx, + lag); + } + } + + + /* Perform enhancement block by block */ + + for (iblock = 0; iblockblockl); + scale = WebRtcSpl_GetSizeInBits((uint32_t)(max * max)); + + /* Scale to maximum 24 bits so that it won't overflow for 76 samples */ + scale = scale-24; + scale1 = WEBRTC_SPL_MAX(0, scale); + + /* Calculate energies */ + ssqPtr=residualFIX + 2; + seqEnPtr=ssqEn; + for (n=(iLBCenc_inst->nsub-1); n>0; n--) { + (*seqEnPtr) = WebRtcSpl_DotProductWithScale(ssqPtr, ssqPtr, 76, scale1); + ssqPtr += 40; + seqEnPtr++; + } + + /* Scale to maximum 20 bits in order to allow for the 11 bit window */ + maxW32 = WebRtcSpl_MaxValueW32(ssqEn, iLBCenc_inst->nsub - 1); + scale = WebRtcSpl_GetSizeInBits(maxW32) - 20; + scale1 = WEBRTC_SPL_MAX(0, scale); + + /* Window each 80 block with the ssqEn_winTbl window to give higher probability for + the blocks in the middle + */ + seqEnPtr=ssqEn; + if (iLBCenc_inst->mode==20) { + ssqPtr=(int16_t*)WebRtcIlbcfix_kStartSequenceEnrgWin+1; + } else { + ssqPtr=(int16_t*)WebRtcIlbcfix_kStartSequenceEnrgWin; + } + for (n=(iLBCenc_inst->nsub-1); n>0; n--) { + (*seqEnPtr)=WEBRTC_SPL_MUL(((*seqEnPtr)>>scale1), (*ssqPtr)); + seqEnPtr++; + ssqPtr++; + } + + /* Extract the best choise of start state */ + pos = WebRtcSpl_MaxIndexW32(ssqEn, iLBCenc_inst->nsub - 1) + 1; + + return(pos); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/frame_classify.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/frame_classify.h new file mode 100644 index 0000000000..60b3249505 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/frame_classify.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_FrameClassify.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_FRAME_CLASSIFY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_FRAME_CLASSIFY_H_ + +size_t WebRtcIlbcfix_FrameClassify( + /* (o) Index to the max-energy sub frame */ + IlbcEncoder *iLBCenc_inst, + /* (i/o) the encoder state structure */ + int16_t *residualFIX /* (i) lpc residual signal */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c new file mode 100644 index 0000000000..07e09bb887 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainDequant.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * decoder for quantized gains in the gain-shape coding of + * residual + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_GainDequant( + /* (o) quantized gain value (Q14) */ + int16_t index, /* (i) quantization index */ + int16_t maxIn, /* (i) maximum of unquantized gain (Q14) */ + int16_t stage /* (i) The stage of the search */ + ){ + int16_t scale; + const int16_t *gain; + + /* obtain correct scale factor */ + + scale=WEBRTC_SPL_ABS_W16(maxIn); + scale = WEBRTC_SPL_MAX(1638, scale); /* if lower than 0.1, set it to 0.1 */ + + /* select the quantization table and return the decoded value */ + gain = WebRtcIlbcfix_kGain[stage]; + + return (int16_t)((scale * gain[index] + 8192) >> 14); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h new file mode 100644 index 0000000000..efa3c7a32e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainDequant.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_DEQUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_DEQUANT_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * decoder for quantized gains in the gain-shape coding of + * residual + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_GainDequant( + /* (o) quantized gain value (Q14) */ + int16_t index, /* (i) quantization index */ + int16_t maxIn, /* (i) maximum of unquantized gain (Q14) */ + int16_t stage /* (i) The stage of the search */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.c new file mode 100644 index 0000000000..90b4114d3b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainQuant.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * quantizer for the gain in the gain-shape coding of residual + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_GainQuant( /* (o) quantized gain value */ + int16_t gain, /* (i) gain value Q14 */ + int16_t maxIn, /* (i) maximum of gain value Q14 */ + int16_t stage, /* (i) The stage of the search */ + int16_t *index /* (o) quantization index */ + ) { + + int16_t scale, cblen; + int32_t gainW32, measure1, measure2; + const int16_t *cbPtr, *cb; + int loc, noMoves, noChecks, i; + + /* ensure a lower bound (0.1) on the scaling factor */ + + scale = WEBRTC_SPL_MAX(1638, maxIn); + + /* select the quantization table and calculate + the length of the table and the number of + steps in the binary search that are needed */ + cb = WebRtcIlbcfix_kGain[stage]; + cblen = 32>>stage; + noChecks = 4-stage; + + /* Multiply the gain with 2^14 to make the comparison + easier and with higher precision */ + gainW32 = gain << 14; + + /* Do a binary search, starting in the middle of the CB + loc - defines the current position in the table + noMoves - defines the number of steps to move in the CB in order + to get next CB location + */ + + loc = cblen>>1; + noMoves = loc; + cbPtr = cb + loc; /* Centre of CB */ + + for (i=noChecks;i>0;i--) { + noMoves>>=1; + measure1 = scale * *cbPtr; + + /* Move up if gain is larger, otherwise move down in table */ + measure1 = measure1 - gainW32; + + if (0>measure1) { + cbPtr+=noMoves; + loc+=noMoves; + } else { + cbPtr-=noMoves; + loc-=noMoves; + } + } + + /* Check which value is the closest one: loc-1, loc or loc+1 */ + + measure1 = scale * *cbPtr; + if (gainW32>measure1) { + /* Check against value above loc */ + measure2 = scale * cbPtr[1]; + if ((measure2-gainW32)<(gainW32-measure1)) { + loc+=1; + } + } else { + /* Check against value below loc */ + measure2 = scale * cbPtr[-1]; + if ((gainW32-measure2)<=(measure1-gainW32)) { + loc-=1; + } + } + + /* Guard against getting outside the table. The calculation above can give a location + which is one above the maximum value (in very rare cases) */ + loc=WEBRTC_SPL_MIN(loc, (cblen-1)); + *index=loc; + + /* Calculate and return the quantized gain value (in Q14) */ + return (int16_t)((scale * cb[loc] + 8192) >> 14); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.h new file mode 100644 index 0000000000..5780bee366 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainQuant.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_QUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_QUANT_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * quantizer for the gain in the gain-shape coding of residual + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_GainQuant( /* (o) quantized gain value */ + int16_t gain, /* (i) gain value Q14 */ + int16_t maxIn, /* (i) maximum of gain value Q14 */ + int16_t stage, /* (i) The stage of the search */ + int16_t *index /* (o) quantization index */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c new file mode 100644 index 0000000000..272b49a71c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetCbVec.c + +******************************************************************/ + +#include "get_cd_vec.h" + +#include "defines.h" +#include "constants.h" +#include "create_augmented_vec.h" + +/*----------------------------------------------------------------* + * Construct codebook vector for given index. + *---------------------------------------------------------------*/ + +bool WebRtcIlbcfix_GetCbVec( + int16_t *cbvec, /* (o) Constructed codebook vector */ + int16_t *mem, /* (i) Codebook buffer */ + size_t index, /* (i) Codebook index */ + size_t lMem, /* (i) Length of codebook buffer */ + size_t cbveclen /* (i) Codebook vector length */ + ){ + size_t k, base_size; + size_t lag; + /* Stack based */ + int16_t tempbuff2[SUBL+5]; + + /* Determine size of codebook sections */ + + base_size=lMem-cbveclen+1; + + if (cbveclen==SUBL) { + base_size += cbveclen / 2; + } + + /* No filter -> First codebook section */ + + if (index + +#include "defines.h" + +// Returns true on success, false on failure. In case of failure, the decoder +// state may be corrupted and needs resetting. +bool WebRtcIlbcfix_GetCbVec( + int16_t* cbvec, /* (o) Constructed codebook vector */ + int16_t* mem, /* (i) Codebook buffer */ + size_t index, /* (i) Codebook index */ + size_t lMem, /* (i) Length of codebook buffer */ + size_t cbveclen /* (i) Codebook vector length */ + ) RTC_WARN_UNUSED_RESULT; + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c new file mode 100644 index 0000000000..a8375afb60 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetLspPoly.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Construct the polynomials F1(z) and F2(z) from the LSP + * (Computations are done in Q24) + * + * The expansion is performed using the following recursion: + * + * f[0] = 1; + * tmp = -2.0 * lsp[0]; + * f[1] = tmp; + * for (i=2; i<=5; i++) { + * b = -2.0 * lsp[2*i-2]; + * f[i] = tmp*f[i-1] + 2.0*f[i-2]; + * for (j=i; j>=2; j--) { + * f[j] = f[j] + tmp*f[j-1] + f[j-2]; + * } + * f[i] = f[i] + tmp; + * } + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetLspPoly( + int16_t *lsp, /* (i) LSP in Q15 */ + int32_t *f) /* (o) polonymial in Q24 */ +{ + int32_t tmpW32; + int i, j; + int16_t high, low; + int16_t *lspPtr; + int32_t *fPtr; + + lspPtr = lsp; + fPtr = f; + /* f[0] = 1.0 (Q24) */ + (*fPtr) = (int32_t)16777216; + fPtr++; + + (*fPtr) = WEBRTC_SPL_MUL((*lspPtr), -1024); + fPtr++; + lspPtr+=2; + + for(i=2; i<=5; i++) + { + (*fPtr) = fPtr[-2]; + + for(j=i; j>1; j--) + { + /* Compute f[j] = f[j] + tmp*f[j-1] + f[j-2]; */ + high = (int16_t)(fPtr[-1] >> 16); + low = (int16_t)((fPtr[-1] & 0xffff) >> 1); + + tmpW32 = 4 * high * *lspPtr + 4 * ((low * *lspPtr) >> 15); + + (*fPtr) += fPtr[-2]; + (*fPtr) -= tmpW32; + fPtr--; + } + *fPtr -= *lspPtr * (1 << 10); + + fPtr+=i; + lspPtr+=2; + } + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h new file mode 100644 index 0000000000..537b771506 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetLspPoly.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_LSP_POLY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_LSP_POLY_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Construct the polynomials F1(z) and F2(z) from the LSP + * (Computations are done in Q24) + * + * The expansion is performed using the following recursion: + * + * f[0] = 1; + * tmp = -2.0 * lsp[0]; + * f[1] = tmp; + * for (i=2; i<=5; i++) { + * b = -2.0 * lsp[2*i-2]; + * f[i] = tmp*f[i-1] + 2.0*f[i-2]; + * for (j=i; j>=2; j--) { + * f[j] = f[j] + tmp*f[j-1] + f[j-2]; + * } + * f[i] = f[i] + tmp; + * } + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetLspPoly( + int16_t *lsp, /* (i) LSP in Q15 */ + int32_t *f); /* (o) polonymial in Q24 */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c new file mode 100644 index 0000000000..a98a96cdf1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetSyncSeq.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "refiner.h" +#include "nearest_neighbor.h" + +/*----------------------------------------------------------------* + * get the pitch-synchronous sample sequence + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetSyncSeq( + int16_t *idata, /* (i) original data */ + size_t idatal, /* (i) dimension of data */ + size_t centerStartPos, /* (i) where current block starts */ + size_t *period, /* (i) rough-pitch-period array (Q-2) */ + const size_t *plocs, /* (i) where periods of period array are taken (Q-2) */ + size_t periodl, /* (i) dimension period array */ + size_t hl, /* (i) 2*hl+1 is the number of sequences */ + int16_t *surround /* (i/o) The contribution from this sequence + summed with earlier contributions */ + ){ + size_t i, centerEndPos, q; + /* Stack based */ + size_t lagBlock[2 * ENH_HL + 1]; + size_t blockStartPos[2 * ENH_HL + 1]; /* The position to search around (Q2) */ + size_t plocs2[ENH_PLOCSL]; + + centerEndPos = centerStartPos + ENH_BLOCKL - 1; + + /* present (find predicted lag from this position) */ + + WebRtcIlbcfix_NearestNeighbor(lagBlock + hl, + plocs, + 2 * (centerStartPos + centerEndPos), + periodl); + + blockStartPos[hl] = 4 * centerStartPos; + + /* past (find predicted position and perform a refined + search to find the best sequence) */ + + for (q = hl; q > 0; q--) { + size_t qq = q - 1; + size_t period_q = period[lagBlock[q]]; + /* Stop if this sequence would be outside the buffer; that means all + further-past sequences would also be outside the buffer. */ + if (blockStartPos[q] < period_q + (4 * ENH_OVERHANG)) + break; + blockStartPos[qq] = blockStartPos[q] - period_q; + + size_t value = blockStartPos[qq] + 4 * ENH_BLOCKL_HALF; + value = (value > period_q) ? (value - period_q) : 0; + WebRtcIlbcfix_NearestNeighbor(lagBlock + qq, plocs, value, periodl); + + /* Find the best possible sequence in the 4 times upsampled + domain around blockStartPos+q */ + WebRtcIlbcfix_Refiner(blockStartPos + qq, idata, idatal, centerStartPos, + blockStartPos[qq], surround, + WebRtcIlbcfix_kEnhWt[qq]); + } + + /* future (find predicted position and perform a refined + search to find the best sequence) */ + + for (i = 0; i < periodl; i++) { + plocs2[i] = plocs[i] - period[i]; + } + + for (q = hl + 1; q <= (2 * hl); q++) { + + WebRtcIlbcfix_NearestNeighbor( + lagBlock + q, + plocs2, + blockStartPos[q - 1] + 4 * ENH_BLOCKL_HALF, + periodl); + + blockStartPos[q]=blockStartPos[q-1]+period[lagBlock[q]]; + + if (blockStartPos[q] + 4 * (ENH_BLOCKL + ENH_OVERHANG) < 4 * idatal) { + + /* Find the best possible sequence in the 4 times upsampled + domain around blockStartPos+q */ + WebRtcIlbcfix_Refiner(blockStartPos + q, idata, idatal, centerStartPos, + blockStartPos[q], surround, + WebRtcIlbcfix_kEnhWt[2 * hl - q]); + + } else { + /* Don't add anything since this sequence would + be outside the buffer */ + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h new file mode 100644 index 0000000000..9e97e60359 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetSyncSeq.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_SYNC_SEQ_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_SYNC_SEQ_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * get the pitch-synchronous sample sequence + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetSyncSeq( + int16_t *idata, /* (i) original data */ + size_t idatal, /* (i) dimension of data */ + size_t centerStartPos, /* (i) where current block starts */ + size_t *period, /* (i) rough-pitch-period array (Q-2) */ + const size_t *plocs, /* (i) where periods of period array are taken (Q-2) */ + size_t periodl, /* (i) dimension period array */ + size_t hl, /* (i) 2*hl+1 is the number of sequences */ + int16_t *surround /* (i/o) The contribution from this sequence + summed with earlier contributions */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.c new file mode 100644 index 0000000000..5d8a86050e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpInput.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * high-pass filter of input with *0.5 and saturation + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_HpInput( + int16_t *signal, /* (i/o) signal vector */ + int16_t *ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} a[0] + is assumed to be 1.0 */ + int16_t *y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t *x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len) /* (i) Number of samples to filter */ +{ + size_t i; + int32_t tmpW32; + int32_t tmpW32b; + + for (i=0; i>15); + tmpW32 += y[0] * ba[3]; /* (-a[1])*y[i-1] (high part) */ + tmpW32 += y[2] * ba[4]; /* (-a[2])*y[i-2] (high part) */ + tmpW32 = (tmpW32<<1); + + tmpW32 += signal[i] * ba[0]; /* b[0]*x[0] */ + tmpW32 += x[0] * ba[1]; /* b[1]*x[i-1] */ + tmpW32 += x[1] * ba[2]; /* b[2]*x[i-2] */ + + /* Update state (input part) */ + x[1] = x[0]; + x[0] = signal[i]; + + /* Rounding in Q(12+1), i.e. add 2^12 */ + tmpW32b = tmpW32 + 4096; + + /* Saturate (to 2^28) so that the HP filtered signal does not overflow */ + tmpW32b = WEBRTC_SPL_SAT((int32_t)268435455, tmpW32b, (int32_t)-268435456); + + /* Convert back to Q0 and multiply with 0.5 */ + signal[i] = (int16_t)(tmpW32b >> 13); + + /* Update state (filtered part) */ + y[2] = y[0]; + y[3] = y[1]; + + /* upshift tmpW32 by 3 with saturation */ + if (tmpW32>268435455) { + tmpW32 = WEBRTC_SPL_WORD32_MAX; + } else if (tmpW32<-268435456) { + tmpW32 = WEBRTC_SPL_WORD32_MIN; + } else { + tmpW32 <<= 3; + } + + y[0] = (int16_t)(tmpW32 >> 16); + y[1] = (int16_t)((tmpW32 - (y[0] << 16)) >> 1); + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.h new file mode 100644 index 0000000000..fdc0c37497 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpInput.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_INPUT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_INPUT_H_ + +#include "defines.h" + +void WebRtcIlbcfix_HpInput( + int16_t *signal, /* (i/o) signal vector */ + int16_t *ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} a[0] + is assumed to be 1.0 */ + int16_t *y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t *x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len); /* (i) Number of samples to filter */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.c new file mode 100644 index 0000000000..8b18c047b9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpOutput.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * high-pass filter of output and *2 with saturation + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_HpOutput( + int16_t *signal, /* (i/o) signal vector */ + int16_t *ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} a[0] + is assumed to be 1.0 */ + int16_t *y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t *x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len) /* (i) Number of samples to filter */ +{ + size_t i; + int32_t tmpW32; + int32_t tmpW32b; + + for (i=0; i>15); + tmpW32 += y[0] * ba[3]; /* (-a[1])*y[i-1] (high part) */ + tmpW32 += y[2] * ba[4]; /* (-a[2])*y[i-2] (high part) */ + tmpW32 *= 2; + + tmpW32 += signal[i] * ba[0]; /* b[0]*x[0] */ + tmpW32 += x[0] * ba[1]; /* b[1]*x[i-1] */ + tmpW32 += x[1] * ba[2]; /* b[2]*x[i-2] */ + + /* Update state (input part) */ + x[1] = x[0]; + x[0] = signal[i]; + + /* Rounding in Q(12-1), i.e. add 2^10 */ + tmpW32b = tmpW32 + 1024; + + /* Saturate (to 2^26) so that the HP filtered signal does not overflow */ + tmpW32b = WEBRTC_SPL_SAT((int32_t)67108863, tmpW32b, (int32_t)-67108864); + + /* Convert back to Q0 and multiply with 2 */ + signal[i] = (int16_t)(tmpW32b >> 11); + + /* Update state (filtered part) */ + y[2] = y[0]; + y[3] = y[1]; + + /* upshift tmpW32 by 3 with saturation */ + if (tmpW32>268435455) { + tmpW32 = WEBRTC_SPL_WORD32_MAX; + } else if (tmpW32<-268435456) { + tmpW32 = WEBRTC_SPL_WORD32_MIN; + } else { + tmpW32 *= 8; + } + + y[0] = (int16_t)(tmpW32 >> 16); + y[1] = (int16_t)((tmpW32 & 0xffff) >> 1); + + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.h new file mode 100644 index 0000000000..6f6a56dd19 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpOutput.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_OUTPUT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_OUTPUT_H_ + +#include "defines.h" + +void WebRtcIlbcfix_HpOutput( + int16_t *signal, /* (i/o) signal vector */ + int16_t *ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} a[0] + is assumed to be 1.0 */ + int16_t *y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t *x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len); /* (i) Number of samples to filter */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.c new file mode 100644 index 0000000000..08677d9b85 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.c @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + iLBCInterface.c + +******************************************************************/ + +#include "ilbc.h" +#include "defines.h" +#include "init_encode.h" +#include "encode.h" +#include "init_decode.h" +#include "decode.h" +#include "rtc_base/checks.h" +#include + +int16_t WebRtcIlbcfix_EncoderAssign(IlbcEncoderInstance** iLBC_encinst, + int16_t* ILBCENC_inst_Addr, + int16_t* size) { + *iLBC_encinst=(IlbcEncoderInstance*)ILBCENC_inst_Addr; + *size=sizeof(IlbcEncoder)/sizeof(int16_t); + if (*iLBC_encinst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_DecoderAssign(IlbcDecoderInstance** iLBC_decinst, + int16_t* ILBCDEC_inst_Addr, + int16_t* size) { + *iLBC_decinst=(IlbcDecoderInstance*)ILBCDEC_inst_Addr; + *size=sizeof(IlbcDecoder)/sizeof(int16_t); + if (*iLBC_decinst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_EncoderCreate(IlbcEncoderInstance **iLBC_encinst) { + *iLBC_encinst=(IlbcEncoderInstance*)malloc(sizeof(IlbcEncoder)); + if (*iLBC_encinst!=NULL) { + WebRtcSpl_Init(); + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_DecoderCreate(IlbcDecoderInstance **iLBC_decinst) { + *iLBC_decinst=(IlbcDecoderInstance*)malloc(sizeof(IlbcDecoder)); + if (*iLBC_decinst!=NULL) { + WebRtcSpl_Init(); + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_EncoderFree(IlbcEncoderInstance *iLBC_encinst) { + free(iLBC_encinst); + return(0); +} + +int16_t WebRtcIlbcfix_DecoderFree(IlbcDecoderInstance *iLBC_decinst) { + free(iLBC_decinst); + return(0); +} + +int16_t WebRtcIlbcfix_EncoderInit(IlbcEncoderInstance* iLBCenc_inst, + int16_t mode) { + if ((mode==20)||(mode==30)) { + WebRtcIlbcfix_InitEncode((IlbcEncoder*) iLBCenc_inst, mode); + return(0); + } else { + return(-1); + } +} + +int WebRtcIlbcfix_Encode(IlbcEncoderInstance* iLBCenc_inst, + const int16_t* speechIn, + size_t len, + uint8_t* encoded) { + size_t pos = 0; + size_t encpos = 0; + + if ((len != ((IlbcEncoder*)iLBCenc_inst)->blockl) && +#ifdef SPLIT_10MS + (len != 80) && +#endif + (len != 2*((IlbcEncoder*)iLBCenc_inst)->blockl) && + (len != 3*((IlbcEncoder*)iLBCenc_inst)->blockl)) + { + /* A maximum of 3 frames/packet is allowed */ + return(-1); + } else { + + /* call encoder */ + while (possection == 0) +#else + pos += ((IlbcEncoder*)iLBCenc_inst)->blockl; +#endif + encpos += ((IlbcEncoder*)iLBCenc_inst)->no_of_words; + } + return (int)(encpos*2); + } +} + +int16_t WebRtcIlbcfix_DecoderInit(IlbcDecoderInstance* iLBCdec_inst, + int16_t mode) { + if ((mode==20)||(mode==30)) { + WebRtcIlbcfix_InitDecode((IlbcDecoder*) iLBCdec_inst, mode, 1); + return(0); + } else { + return(-1); + } +} +void WebRtcIlbcfix_DecoderInit20Ms(IlbcDecoderInstance* iLBCdec_inst) { + WebRtcIlbcfix_InitDecode((IlbcDecoder*) iLBCdec_inst, 20, 1); +} +void WebRtcIlbcfix_Decoderinit30Ms(IlbcDecoderInstance* iLBCdec_inst) { + WebRtcIlbcfix_InitDecode((IlbcDecoder*) iLBCdec_inst, 30, 1); +} + + +int WebRtcIlbcfix_Decode(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + size_t i=0; + /* Allow for automatic switching between the frame sizes + (although you do get some discontinuity) */ + if ((len==((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==2*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==3*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)) { + /* ok, do nothing */ + } else { + /* Test if the mode has changed */ + if (((IlbcDecoder*)iLBCdec_inst)->mode==20) { + if ((len==NO_OF_BYTES_30MS)|| + (len==2*NO_OF_BYTES_30MS)|| + (len==3*NO_OF_BYTES_30MS)) { + WebRtcIlbcfix_InitDecode( + ((IlbcDecoder*)iLBCdec_inst), 30, + ((IlbcDecoder*)iLBCdec_inst)->use_enhancer); + } else { + /* Unsupported frame length */ + return(-1); + } + } else { + if ((len==NO_OF_BYTES_20MS)|| + (len==2*NO_OF_BYTES_20MS)|| + (len==3*NO_OF_BYTES_20MS)) { + WebRtcIlbcfix_InitDecode( + ((IlbcDecoder*)iLBCdec_inst), 20, + ((IlbcDecoder*)iLBCdec_inst)->use_enhancer); + } else { + /* Unsupported frame length */ + return(-1); + } + } + } + + while ((i*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)blockl], + (const uint16_t*)&encoded + [2 * i * ((IlbcDecoder*)iLBCdec_inst)->no_of_words], + (IlbcDecoder*)iLBCdec_inst, 1) == -1) + return -1; + i++; + } + /* iLBC does not support VAD/CNG yet */ + *speechType=1; + return (int)(i*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +int WebRtcIlbcfix_Decode20Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + size_t i=0; + if ((len==((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==2*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==3*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)) { + /* ok, do nothing */ + } else { + return(-1); + } + + while ((i*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)blockl], + (const uint16_t*)&encoded + [2 * i * ((IlbcDecoder*)iLBCdec_inst)->no_of_words], + (IlbcDecoder*)iLBCdec_inst, 1)) + return -1; + i++; + } + /* iLBC does not support VAD/CNG yet */ + *speechType=1; + return (int)(i*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +int WebRtcIlbcfix_Decode30Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + size_t i=0; + if ((len==((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==2*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==3*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)) { + /* ok, do nothing */ + } else { + return(-1); + } + + while ((i*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)blockl], + (const uint16_t*)&encoded + [2 * i * ((IlbcDecoder*)iLBCdec_inst)->no_of_words], + (IlbcDecoder*)iLBCdec_inst, 1)) + return -1; + i++; + } + /* iLBC does not support VAD/CNG yet */ + *speechType=1; + return (int)(i*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +size_t WebRtcIlbcfix_DecodePlc(IlbcDecoderInstance* iLBCdec_inst, + int16_t* decoded, + size_t noOfLostFrames) { + size_t i; + uint16_t dummy; + + for (i=0;iblockl], &dummy, + (IlbcDecoder*)iLBCdec_inst, 0)); + } + return (noOfLostFrames*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +size_t WebRtcIlbcfix_NetEqPlc(IlbcDecoderInstance* iLBCdec_inst, + int16_t* decoded, + size_t noOfLostFrames) { + /* Two input parameters not used, but needed for function pointers in NetEQ */ + (void)(decoded = NULL); + (void)(noOfLostFrames = 0); + + WebRtcSpl_MemSetW16(((IlbcDecoder*)iLBCdec_inst)->enh_buf, 0, ENH_BUFL); + ((IlbcDecoder*)iLBCdec_inst)->prev_enh_pl = 2; + + return (0); +} + +void WebRtcIlbcfix_version(char *version) +{ + strcpy((char*)version, "1.1.1"); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.h new file mode 100644 index 0000000000..7836489082 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.h @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * ilbc.h + * + * This header file contains all of the API's for iLBC. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_ILBC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_ILBC_H_ + +#include + +/* + * Define the fixpoint numeric formats + */ + +#include "typedefs.h" // NOLINT(build/include) + +/* + * Solution to support multiple instances + * Customer has to cast instance to proper type + */ + +typedef struct iLBC_encinst_t_ IlbcEncoderInstance; + +typedef struct iLBC_decinst_t_ IlbcDecoderInstance; + +/* + * Comfort noise constants + */ + +#define ILBC_SPEECH 1 +#define ILBC_CNG 2 + +#ifdef __cplusplus +extern "C" { +#endif + + /**************************************************************************** + * WebRtcIlbcfix_XxxAssign(...) + * + * These functions assigns the encoder/decoder instance to the specified + * memory location + * + * Input: + * - XXX_xxxinst : Pointer to created instance that should be + * assigned + * - ILBCXXX_inst_Addr : Pointer to the desired memory space + * - size : The size that this structure occupies (in Word16) + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIlbcfix_EncoderAssign(IlbcEncoderInstance **iLBC_encinst, + int16_t *ILBCENC_inst_Addr, + int16_t *size); + int16_t WebRtcIlbcfix_DecoderAssign(IlbcDecoderInstance **iLBC_decinst, + int16_t *ILBCDEC_inst_Addr, + int16_t *size); + + + /**************************************************************************** + * WebRtcIlbcfix_XxxAssign(...) + * + * These functions create a instance to the specified structure + * + * Input: + * - XXX_inst : Pointer to created instance that should be created + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIlbcfix_EncoderCreate(IlbcEncoderInstance **iLBC_encinst); + int16_t WebRtcIlbcfix_DecoderCreate(IlbcDecoderInstance **iLBC_decinst); + + /**************************************************************************** + * WebRtcIlbcfix_XxxFree(...) + * + * These functions frees the dynamic memory of a specified instance + * + * Input: + * - XXX_inst : Pointer to created instance that should be freed + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIlbcfix_EncoderFree(IlbcEncoderInstance *iLBC_encinst); + int16_t WebRtcIlbcfix_DecoderFree(IlbcDecoderInstance *iLBC_decinst); + + + /**************************************************************************** + * WebRtcIlbcfix_EncoderInit(...) + * + * This function initializes a iLBC instance + * + * Input: + * - iLBCenc_inst : iLBC instance, i.e. the user that should receive + * be initialized + * - frameLen : The frame length of the codec 20/30 (ms) + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIlbcfix_EncoderInit(IlbcEncoderInstance *iLBCenc_inst, + int16_t frameLen); + + /**************************************************************************** + * WebRtcIlbcfix_Encode(...) + * + * This function encodes one iLBC frame. Input speech length has be a + * multiple of the frame length. + * + * Input: + * - iLBCenc_inst : iLBC instance, i.e. the user that should encode + * a package + * - speechIn : Input speech vector + * - len : Samples in speechIn (160, 240, 320 or 480) + * + * Output: + * - encoded : The encoded data vector + * + * Return value : >0 - Length (in bytes) of coded data + * -1 - Error + */ + + int WebRtcIlbcfix_Encode(IlbcEncoderInstance *iLBCenc_inst, + const int16_t *speechIn, + size_t len, + uint8_t* encoded); + + /**************************************************************************** + * WebRtcIlbcfix_DecoderInit(...) + * + * This function initializes a iLBC instance with either 20 or 30 ms frames + * Alternatively the WebRtcIlbcfix_DecoderInit_XXms can be used. Then it's + * not needed to specify the frame length with a variable. + * + * Input: + * - IlbcDecoderInstance : iLBC decoder instance + * - frameLen : The frame length of the codec 20/30 (ms) + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIlbcfix_DecoderInit(IlbcDecoderInstance *iLBCdec_inst, + int16_t frameLen); + void WebRtcIlbcfix_DecoderInit20Ms(IlbcDecoderInstance* iLBCdec_inst); + void WebRtcIlbcfix_Decoderinit30Ms(IlbcDecoderInstance* iLBCdec_inst); + + /**************************************************************************** + * WebRtcIlbcfix_Decode(...) + * + * This function decodes a packet with iLBC frame(s). Output speech length + * will be a multiple of 160 or 240 samples ((160 or 240)*frames/packet). + * + * Input: + * - iLBCdec_inst : iLBC instance, i.e. the user that should decode + * a packet + * - encoded : Encoded iLBC frame(s) + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG + * + * Return value : >0 - Samples in decoded vector + * -1 - Error + */ + + int WebRtcIlbcfix_Decode(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + int WebRtcIlbcfix_Decode20Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + int WebRtcIlbcfix_Decode30Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + + /**************************************************************************** + * WebRtcIlbcfix_DecodePlc(...) + * + * This function conducts PLC for iLBC frame(s). Output speech length + * will be a multiple of 160 or 240 samples. + * + * Input: + * - iLBCdec_inst : iLBC instance, i.e. the user that should perform + * a PLC + * - noOfLostFrames : Number of PLC frames to produce + * + * Output: + * - decoded : The "decoded" vector + * + * Return value : Samples in decoded PLC vector + */ + + size_t WebRtcIlbcfix_DecodePlc(IlbcDecoderInstance *iLBCdec_inst, + int16_t *decoded, + size_t noOfLostFrames); + + /**************************************************************************** + * WebRtcIlbcfix_NetEqPlc(...) + * + * This function updates the decoder when a packet loss has occured, but it + * does not produce any PLC data. Function can be used if another PLC method + * is used (i.e NetEq). + * + * Input: + * - iLBCdec_inst : iLBC instance that should be updated + * - noOfLostFrames : Number of lost frames + * + * Output: + * - decoded : The "decoded" vector (nothing in this case) + * + * Return value : Samples in decoded PLC vector + */ + + size_t WebRtcIlbcfix_NetEqPlc(IlbcDecoderInstance *iLBCdec_inst, + int16_t *decoded, + size_t noOfLostFrames); + + /**************************************************************************** + * WebRtcIlbcfix_version(...) + * + * This function returns the version number of iLBC + * + * Output: + * - version : Version number of iLBC (maximum 20 char) + */ + + void WebRtcIlbcfix_version(char *version); + +#ifdef __cplusplus +} +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_ILBC_ILBC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc new file mode 100644 index 0000000000..b8d3c7c55a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(IlbcTest, BadPacket) { + // Get a good packet. + AudioEncoderIlbcConfig config; + config.frame_size_ms = 20; // We need 20 ms rather than the default 30 ms; + // otherwise, all possible values of cb_index[2] + // are valid. + AudioEncoderIlbcImpl encoder(config, 102); + std::vector samples(encoder.SampleRateHz() / 100, 4711); + rtc::Buffer packet; + int num_10ms_chunks = 0; + while (packet.size() == 0) { + encoder.Encode(0, samples, &packet); + num_10ms_chunks += 1; + } + + // Break the packet by setting all bits of the unsigned 7-bit number + // cb_index[2] to 1, giving it a value of 127. For a 20 ms packet, this is + // too large. + EXPECT_EQ(38u, packet.size()); + rtc::Buffer bad_packet(packet.data(), packet.size()); + bad_packet[29] |= 0x3f; // Bits 1-6. + bad_packet[30] |= 0x80; // Bit 0. + + // Decode the bad packet. We expect the decoder to respond by returning -1. + AudioDecoderIlbcImpl decoder; + std::vector decoded_samples(num_10ms_chunks * samples.size()); + AudioDecoder::SpeechType speech_type; + EXPECT_EQ(-1, decoder.Decode(bad_packet.data(), bad_packet.size(), + encoder.SampleRateHz(), + sizeof(int16_t) * decoded_samples.size(), + decoded_samples.data(), &speech_type)); + + // Decode the good packet. This should work, because the failed decoding + // should not have left the decoder in a broken state. + EXPECT_EQ(static_cast(decoded_samples.size()), + decoder.Decode(packet.data(), packet.size(), encoder.SampleRateHz(), + sizeof(int16_t) * decoded_samples.size(), + decoded_samples.data(), &speech_type)); +} + +class SplitIlbcTest : public ::testing::TestWithParam > { + protected: + virtual void SetUp() { + const std::pair parameters = GetParam(); + num_frames_ = parameters.first; + frame_length_ms_ = parameters.second; + frame_length_bytes_ = (frame_length_ms_ == 20) ? 38 : 50; + } + size_t num_frames_; + int frame_length_ms_; + size_t frame_length_bytes_; +}; + +TEST_P(SplitIlbcTest, NumFrames) { + AudioDecoderIlbcImpl decoder; + const size_t frame_length_samples = frame_length_ms_ * 8; + const auto generate_payload = [] (size_t payload_length_bytes) { + rtc::Buffer payload(payload_length_bytes); + // Fill payload with increasing integers {0, 1, 2, ...}. + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i); + } + return payload; + }; + + const auto results = decoder.ParsePayload( + generate_payload(frame_length_bytes_ * num_frames_), 0); + EXPECT_EQ(num_frames_, results.size()); + + size_t frame_num = 0; + uint8_t payload_value = 0; + for (const auto& result : results) { + EXPECT_EQ(frame_length_samples * frame_num, result.timestamp); + const LegacyEncodedAudioFrame* frame = + static_cast(result.frame.get()); + const rtc::Buffer& payload = frame->payload(); + EXPECT_EQ(frame_length_bytes_, payload.size()); + for (size_t i = 0; i < payload.size(); ++i, ++payload_value) { + EXPECT_EQ(payload_value, payload[i]); + } + ++frame_num; + } +} + +// Test 1 through 5 frames of 20 and 30 ms size. +// Also test the maximum number of frames in one packet for 20 and 30 ms. +// The maximum is defined by the largest payload length that can be uniquely +// resolved to a frame size of either 38 bytes (20 ms) or 50 bytes (30 ms). +INSTANTIATE_TEST_CASE_P( + IlbcTest, SplitIlbcTest, + ::testing::Values(std::pair(1, 20), // 1 frame, 20 ms. + std::pair(2, 20), // 2 frames, 20 ms. + std::pair(3, 20), // And so on. + std::pair(4, 20), + std::pair(5, 20), + std::pair(24, 20), + std::pair(1, 30), + std::pair(2, 30), + std::pair(3, 30), + std::pair(4, 30), + std::pair(5, 30), + std::pair(18, 30))); + +// Test too large payload size. +TEST(IlbcTest, SplitTooLargePayload) { + AudioDecoderIlbcImpl decoder; + constexpr size_t kPayloadLengthBytes = 950; + const auto results = + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); + EXPECT_TRUE(results.empty()); +} + +// Payload not an integer number of frames. +TEST(IlbcTest, SplitUnevenPayload) { + AudioDecoderIlbcImpl decoder; + constexpr size_t kPayloadLengthBytes = 39; // Not an even number of frames. + const auto results = + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); + EXPECT_TRUE(results.empty()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c new file mode 100644 index 0000000000..033a0a3584 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvDec.c + +******************************************************************/ + +#include "defines.h" + +void WebRtcIlbcfix_IndexConvDec( + int16_t *index /* (i/o) Codebook indexes */ + ){ + int k; + + for (k=4;k<6;k++) { + /* Readjust the second and third codebook index for the first 40 sample + so that they look the same as the first (in terms of lag) + */ + if ((index[k]>=44)&&(index[k]<108)) { + index[k]+=64; + } else if ((index[k]>=108)&&(index[k]<128)) { + index[k]+=128; + } else { + /* ERROR */ + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h new file mode 100644 index 0000000000..4b61d31617 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvDec.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_DEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_DEC_H_ + +#include "defines.h" + +void WebRtcIlbcfix_IndexConvDec( + int16_t *index /* (i/o) Codebook indexes */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c new file mode 100644 index 0000000000..0e7701b4f6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + IiLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvEnc.c + +******************************************************************/ + +#include "defines.h" +/*----------------------------------------------------------------* + * Convert the codebook indexes to make the search easier + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_IndexConvEnc( + int16_t *index /* (i/o) Codebook indexes */ + ){ + int k; + + for (k=4;k<6;k++) { + /* Readjust the second and third codebook index so that it is + packetized into 7 bits (before it was put in lag-wise the same + way as for the first codebook which uses 8 bits) + */ + if ((index[k]>=108)&&(index[k]<172)) { + index[k]-=64; + } else if (index[k]>=236) { + index[k]-=128; + } else { + /* ERROR */ + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h new file mode 100644 index 0000000000..935928a55c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvEnc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_ENC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_ENC_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Convert the codebook indexes to make the search easier + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_IndexConvEnc( + int16_t *index /* (i/o) Codebook indexes */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.c new file mode 100644 index 0000000000..1f92480d9f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitDecode.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * Initiation of decoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitDecode( /* (o) Number of decoded samples */ + IlbcDecoder *iLBCdec_inst, /* (i/o) Decoder instance */ + int16_t mode, /* (i) frame size mode */ + int use_enhancer) { /* (i) 1: use enhancer, 0: no enhancer */ + int i; + + iLBCdec_inst->mode = mode; + + /* Set all the variables that are dependent on the frame size mode */ + if (mode==30) { + iLBCdec_inst->blockl = BLOCKL_30MS; + iLBCdec_inst->nsub = NSUB_30MS; + iLBCdec_inst->nasub = NASUB_30MS; + iLBCdec_inst->lpc_n = LPC_N_30MS; + iLBCdec_inst->no_of_bytes = NO_OF_BYTES_30MS; + iLBCdec_inst->no_of_words = NO_OF_WORDS_30MS; + iLBCdec_inst->state_short_len=STATE_SHORT_LEN_30MS; + } + else if (mode==20) { + iLBCdec_inst->blockl = BLOCKL_20MS; + iLBCdec_inst->nsub = NSUB_20MS; + iLBCdec_inst->nasub = NASUB_20MS; + iLBCdec_inst->lpc_n = LPC_N_20MS; + iLBCdec_inst->no_of_bytes = NO_OF_BYTES_20MS; + iLBCdec_inst->no_of_words = NO_OF_WORDS_20MS; + iLBCdec_inst->state_short_len=STATE_SHORT_LEN_20MS; + } + else { + return(-1); + } + + /* Reset all the previous LSF to mean LSF */ + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->lsfdeqold, WebRtcIlbcfix_kLsfMean, LPC_FILTERORDER); + + /* Clear the synthesis filter memory */ + WebRtcSpl_MemSetW16(iLBCdec_inst->syntMem, 0, LPC_FILTERORDER); + + /* Set the old synthesis filter to {1.0 0.0 ... 0.0} */ + WebRtcSpl_MemSetW16(iLBCdec_inst->old_syntdenum, 0, ((LPC_FILTERORDER + 1)*NSUB_MAX)); + for (i=0; iold_syntdenum[i*(LPC_FILTERORDER+1)] = 4096; + } + + /* Clear the variables that are used for the PLC */ + iLBCdec_inst->last_lag = 20; + iLBCdec_inst->consPLICount = 0; + iLBCdec_inst->prevPLI = 0; + iLBCdec_inst->perSquare = 0; + iLBCdec_inst->prevLag = 120; + iLBCdec_inst->prevLpc[0] = 4096; + WebRtcSpl_MemSetW16(iLBCdec_inst->prevLpc+1, 0, LPC_FILTERORDER); + WebRtcSpl_MemSetW16(iLBCdec_inst->prevResidual, 0, BLOCKL_MAX); + + /* Initialize the seed for the random number generator */ + iLBCdec_inst->seed = 777; + + /* Set the filter state of the HP filter to 0 */ + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemx, 0, 2); + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemy, 0, 4); + + /* Set the variables that are used in the ehnahcer */ + iLBCdec_inst->use_enhancer = use_enhancer; + WebRtcSpl_MemSetW16(iLBCdec_inst->enh_buf, 0, (ENH_BUFL+ENH_BUFL_FILTEROVERHEAD)); + for (i=0;ienh_period[i]=160; /* Q(-4) */ + } + + iLBCdec_inst->prev_enh_pl = 0; + + return (int)(iLBCdec_inst->blockl); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.h new file mode 100644 index 0000000000..6f3a17becb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitDecode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_DECODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_DECODE_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Initiation of decoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitDecode( /* (o) Number of decoded samples */ + IlbcDecoder *iLBCdec_inst, /* (i/o) Decoder instance */ + int16_t mode, /* (i) frame size mode */ + int use_enhancer /* (i) 1 to use enhancer + 0 to run without enhancer */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.c new file mode 100644 index 0000000000..f559d8441f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitEncode.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * Initiation of encoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitEncode( /* (o) Number of bytes encoded */ + IlbcEncoder *iLBCenc_inst, /* (i/o) Encoder instance */ + int16_t mode) { /* (i) frame size mode */ + iLBCenc_inst->mode = mode; + + /* Set all the variables that are dependent on the frame size mode */ + if (mode==30) { + iLBCenc_inst->blockl = BLOCKL_30MS; + iLBCenc_inst->nsub = NSUB_30MS; + iLBCenc_inst->nasub = NASUB_30MS; + iLBCenc_inst->lpc_n = LPC_N_30MS; + iLBCenc_inst->no_of_bytes = NO_OF_BYTES_30MS; + iLBCenc_inst->no_of_words = NO_OF_WORDS_30MS; + iLBCenc_inst->state_short_len=STATE_SHORT_LEN_30MS; + } + else if (mode==20) { + iLBCenc_inst->blockl = BLOCKL_20MS; + iLBCenc_inst->nsub = NSUB_20MS; + iLBCenc_inst->nasub = NASUB_20MS; + iLBCenc_inst->lpc_n = LPC_N_20MS; + iLBCenc_inst->no_of_bytes = NO_OF_BYTES_20MS; + iLBCenc_inst->no_of_words = NO_OF_WORDS_20MS; + iLBCenc_inst->state_short_len=STATE_SHORT_LEN_20MS; + } + else { + return(-1); + } + + /* Clear the buffers and set the previous LSF and LSP to the mean value */ + WebRtcSpl_MemSetW16(iLBCenc_inst->anaMem, 0, LPC_FILTERORDER); + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lsfold, WebRtcIlbcfix_kLsfMean, LPC_FILTERORDER); + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lsfdeqold, WebRtcIlbcfix_kLsfMean, LPC_FILTERORDER); + WebRtcSpl_MemSetW16(iLBCenc_inst->lpc_buffer, 0, LPC_LOOKBACK + BLOCKL_MAX); + + /* Set the filter state of the HP filter to 0 */ + WebRtcSpl_MemSetW16(iLBCenc_inst->hpimemx, 0, 2); + WebRtcSpl_MemSetW16(iLBCenc_inst->hpimemy, 0, 4); + +#ifdef SPLIT_10MS + /*Zeroing the past samples for 10msec Split*/ + WebRtcSpl_MemSetW16(iLBCenc_inst->past_samples,0,160); + iLBCenc_inst->section = 0; +#endif + + return (int)(iLBCenc_inst->no_of_bytes); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.h new file mode 100644 index 0000000000..fc11cc7d20 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitEncode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_ENCODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_ENCODE_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Initiation of encoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitEncode( /* (o) Number of bytes encoded */ + IlbcEncoder *iLBCenc_inst, /* (i/o) Encoder instance */ + int16_t mode /* (i) frame size mode */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.c new file mode 100644 index 0000000000..d0869a5dc2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Interpolate.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * interpolation between vectors + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Interpolate( + int16_t *out, /* (o) output vector */ + int16_t *in1, /* (i) first input vector */ + int16_t *in2, /* (i) second input vector */ + int16_t coef, /* (i) weight coefficient in Q14 */ + int16_t length) /* (i) number of sample is vectors */ +{ + int i; + int16_t invcoef; + + /* + Performs the operation out[i] = in[i]*coef + (1-coef)*in2[i] (with rounding) + */ + + invcoef = 16384 - coef; /* 16384 = 1.0 (Q14)*/ + for (i = 0; i < length; i++) { + out[i] = (int16_t)((coef * in1[i] + invcoef * in2[i] + 8192) >> 14); + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.h new file mode 100644 index 0000000000..befe2416f4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Interpolate.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * interpolation between vectors + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Interpolate( + int16_t *out, /* (o) output vector */ + int16_t *in1, /* (i) first input vector */ + int16_t *in2, /* (i) second input vector */ + int16_t coef, /* (i) weight coefficient in Q14 */ + int16_t length); /* (i) number of sample is vectors */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c new file mode 100644 index 0000000000..376dbbb668 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InterpolateSamples.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +void WebRtcIlbcfix_InterpolateSamples( + int16_t *interpSamples, /* (o) The interpolated samples */ + int16_t *CBmem, /* (i) The CB memory */ + size_t lMem /* (i) Length of the CB memory */ + ) { + int16_t *ppi, *ppo, i, j, temp1, temp2; + int16_t *tmpPtr; + + /* Calculate the 20 vectors of interpolated samples (4 samples each) + that are used in the codebooks for lag 20 to 39 */ + tmpPtr = interpSamples; + for (j=0; j<20; j++) { + temp1 = 0; + temp2 = 3; + ppo = CBmem+lMem-4; + ppi = CBmem+lMem-j-24; + for (i=0; i<4; i++) { + + *tmpPtr++ = (int16_t)((WebRtcIlbcfix_kAlpha[temp2] * *ppo) >> 15) + + (int16_t)((WebRtcIlbcfix_kAlpha[temp1] * *ppi) >> 15); + + ppo++; + ppi++; + temp1++; + temp2--; + } + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h new file mode 100644 index 0000000000..8a2827385f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InterpolateSamples.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_SAMPLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_SAMPLES_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Construct the interpolated samples for the Augmented CB + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_InterpolateSamples( + int16_t *interpSamples, /* (o) The interpolated samples */ + int16_t *CBmem, /* (i) The CB memory */ + size_t lMem /* (i) Length of the CB memory */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c new file mode 100644 index 0000000000..258b6b1552 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LpcEncode.c + +******************************************************************/ + +#include "defines.h" +#include "simple_lpc_analysis.h" +#include "simple_interpolate_lsf.h" +#include "simple_lsf_quant.h" +#include "lsf_check.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * lpc encoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LpcEncode( + int16_t *syntdenum, /* (i/o) synthesis filter coefficients + before/after encoding */ + int16_t *weightdenum, /* (i/o) weighting denumerator coefficients + before/after encoding */ + int16_t *lsf_index, /* (o) lsf quantization index */ + int16_t *data, /* (i) Speech to do LPC analysis on */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ) { + /* Stack based */ + int16_t lsf[LPC_FILTERORDER * LPC_N_MAX]; + int16_t lsfdeq[LPC_FILTERORDER * LPC_N_MAX]; + + /* Calculate LSF's from the input speech */ + WebRtcIlbcfix_SimpleLpcAnalysis(lsf, data, iLBCenc_inst); + + /* Quantize the LSF's */ + WebRtcIlbcfix_SimpleLsfQ(lsfdeq, lsf_index, lsf, iLBCenc_inst->lpc_n); + + /* Stableize the LSF's if needed */ + WebRtcIlbcfix_LsfCheck(lsfdeq, LPC_FILTERORDER, iLBCenc_inst->lpc_n); + + /* Calculate the synthesis and weighting filter coefficients from + the optimal LSF and the dequantized LSF */ + WebRtcIlbcfix_SimpleInterpolateLsf(syntdenum, weightdenum, + lsf, lsfdeq, iLBCenc_inst->lsfold, + iLBCenc_inst->lsfdeqold, LPC_FILTERORDER, iLBCenc_inst); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h new file mode 100644 index 0000000000..776b3a14cd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LpcEncode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LPC_ENCODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LPC_ENCODE_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * lpc encoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LpcEncode( + int16_t *syntdenum, /* (i/o) synthesis filter coefficients + before/after encoding */ + int16_t *weightdenum, /* (i/o) weighting denumerator coefficients + before/after encoding */ + int16_t *lsf_index, /* (o) lsf quantization index */ + int16_t *data, /* (i) Speech to do LPC analysis on */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.c new file mode 100644 index 0000000000..0e3bd8485a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfCheck.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * check for stability of lsf coefficients + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_LsfCheck( + int16_t *lsf, /* LSF parameters */ + int dim, /* dimension of LSF */ + int NoAn) /* No of analysis per frame */ +{ + int k,n,m, Nit=2, change=0,pos; + const int16_t eps=319; /* 0.039 in Q13 (50 Hz)*/ + const int16_t eps2=160; /* eps/2.0 in Q13;*/ + const int16_t maxlsf=25723; /* 3.14; (4000 Hz)*/ + const int16_t minlsf=82; /* 0.01; (0 Hz)*/ + + /* LSF separation check*/ + for (n=0;nmaxlsf) { + lsf[pos]=maxlsf; + change=1; + } + } + } + } + + return change; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.h new file mode 100644 index 0000000000..bfcea083aa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfCheck.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_CHECK_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_CHECK_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * check for stability of lsf coefficients + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_LsfCheck( + int16_t *lsf, /* LSF parameters */ + int dim, /* dimension of LSF */ + int NoAn); /* No of analysis per frame */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c new file mode 100644 index 0000000000..66bbde8e7f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LspInterpolate2PolyDec.c + +******************************************************************/ + +#include "interpolate.h" +#include "lsf_to_poly.h" +#include "defines.h" + +/*----------------------------------------------------------------* + * interpolation of lsf coefficients for the decoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LspInterpolate2PolyDec( + int16_t *a, /* (o) lpc coefficients Q12 */ + int16_t *lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t *lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ){ + int16_t lsftmp[LPC_FILTERORDER]; + + /* interpolate LSF */ + WebRtcIlbcfix_Interpolate(lsftmp, lsf1, lsf2, coef, length); + + /* Compute the filter coefficients from the LSF */ + WebRtcIlbcfix_Lsf2Poly(a, lsftmp); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h new file mode 100644 index 0000000000..4530528497 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LspInterpolate2PolyDec.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_DEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_DEC_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * interpolation of lsf coefficients for the decoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LspInterpolate2PolyDec( + int16_t *a, /* (o) lpc coefficients Q12 */ + int16_t *lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t *lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c new file mode 100644 index 0000000000..cf67ecc3be --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfInterpolate2PloyEnc.c + +******************************************************************/ + +#include "defines.h" +#include "interpolate.h" +#include "lsf_to_poly.h" + +/*----------------------------------------------------------------* + * lsf interpolator and conversion from lsf to a coefficients + * (subrutine to SimpleInterpolateLSF) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LsfInterpolate2PloyEnc( + int16_t *a, /* (o) lpc coefficients Q12 */ + int16_t *lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t *lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ) { + /* Stack based */ + int16_t lsftmp[LPC_FILTERORDER]; + + /* interpolate LSF */ + WebRtcIlbcfix_Interpolate(lsftmp, lsf1, lsf2, coef, length); + + /* Compute the filter coefficients from the LSF */ + WebRtcIlbcfix_Lsf2Poly(a, lsftmp); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h new file mode 100644 index 0000000000..a2d2db61a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfInterpolate2PloyEnc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_ENC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_ENC_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * lsf interpolator and conversion from lsf to a coefficients + * (subrutine to SimpleInterpolateLSF) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LsfInterpolate2PloyEnc( + int16_t *a, /* (o) lpc coefficients Q12 */ + int16_t *lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t *lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c new file mode 100644 index 0000000000..cfab013ab7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Lsp.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * conversion from lsf to lsp coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsf2Lsp( + int16_t *lsf, /* (i) lsf in Q13 values between 0 and pi */ + int16_t *lsp, /* (o) lsp in Q15 values between -1 and 1 */ + int16_t m /* (i) number of coefficients */ + ) { + int16_t i, k; + int16_t diff; /* difference, which is used for the + linear approximation (Q8) */ + int16_t freq; /* normalized frequency in Q15 (0..1) */ + int32_t tmpW32; + + for(i=0; i> 15); + /* 20861: 1.0/(2.0*PI) in Q17 */ + /* + Upper 8 bits give the index k and + Lower 8 bits give the difference, which needs + to be approximated linearly + */ + k = freq >> 8; + diff = (freq&0x00ff); + + /* Guard against getting outside table */ + + if (k>63) { + k = 63; + } + + /* Calculate linear approximation */ + tmpW32 = WebRtcIlbcfix_kCosDerivative[k] * diff; + lsp[i] = WebRtcIlbcfix_kCos[k] + (int16_t)(tmpW32 >> 12); + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h new file mode 100644 index 0000000000..ace29dec00 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Lsp.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_LSP_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_LSP_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * conversion from lsf to lsp coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsf2Lsp( + int16_t *lsf, /* (i) lsf in Q13 values between 0 and pi */ + int16_t *lsp, /* (o) lsp in Q15 values between -1 and 1 */ + int16_t m /* (i) number of coefficients */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c new file mode 100644 index 0000000000..c3a34cab00 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Poly.c + +******************************************************************/ + +#include "defines.h" +#include "lsf_to_lsp.h" +#include "get_lsp_poly.h" +#include "constants.h" + +void WebRtcIlbcfix_Lsf2Poly( + int16_t *a, /* (o) predictor coefficients (order = 10) in Q12 */ + int16_t *lsf /* (i) line spectral frequencies in Q13 */ + ) { + int32_t f[2][6]; /* f[0][] and f[1][] corresponds to + F1(z) and F2(z) respectivly */ + int32_t *f1ptr, *f2ptr; + int16_t *a1ptr, *a2ptr; + int32_t tmpW32; + int16_t lsp[10]; + int i; + + /* Convert lsf to lsp */ + WebRtcIlbcfix_Lsf2Lsp(lsf, lsp, LPC_FILTERORDER); + + /* Get F1(z) and F2(z) from the lsp */ + f1ptr=f[0]; + f2ptr=f[1]; + WebRtcIlbcfix_GetLspPoly(&lsp[0],f1ptr); + WebRtcIlbcfix_GetLspPoly(&lsp[1],f2ptr); + + /* for i = 5 down to 1 + Compute f1[i] += f1[i-1]; + and f2[i] += f2[i-1]; + */ + f1ptr=&f[0][5]; + f2ptr=&f[1][5]; + for (i=5; i>0; i--) + { + (*f1ptr) += (*(f1ptr-1)); + (*f2ptr) -= (*(f2ptr-1)); + f1ptr--; + f2ptr--; + } + + /* Get the A(z) coefficients + a[0] = 1.0 + for i = 1 to 5 + a[i] = (f1[i] + f2[i] + round)>>13; + for i = 1 to 5 + a[11-i] = (f1[i] - f2[i] + round)>>13; + */ + a[0]=4096; + a1ptr=&a[1]; + a2ptr=&a[10]; + f1ptr=&f[0][1]; + f2ptr=&f[1][1]; + for (i=5; i>0; i--) + { + tmpW32 = (*f1ptr) + (*f2ptr); + *a1ptr = (int16_t)((tmpW32 + 4096) >> 13); + + tmpW32 = (*f1ptr) - (*f2ptr); + *a2ptr = (int16_t)((tmpW32 + 4096) >> 13); + + a1ptr++; + a2ptr--; + f1ptr++; + f2ptr++; + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h new file mode 100644 index 0000000000..782ef9f6c3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Poly.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_POLY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_POLY_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Convert from LSF coefficients to A coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsf2Poly( + int16_t *a, /* (o) predictor coefficients (order = 10) in Q12 */ + int16_t *lsf /* (i) line spectral frequencies in Q13 */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c new file mode 100644 index 0000000000..be95de7a2f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsp2Lsf.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * conversion from LSP coefficients to LSF coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsp2Lsf( + int16_t *lsp, /* (i) lsp vector -1...+1 in Q15 */ + int16_t *lsf, /* (o) Lsf vector 0...Pi in Q13 + (ordered, so that lsf[i]=0; i--) + { + /* + locate value in the table, which is just above lsp[i], + basically an approximation to acos(x) + */ + while( (((int32_t)(*cosTblPtr)-(*lspPtr)) < 0)&&(k>0) ) + { + k-=1; + cosTblPtr--; + } + + /* Calculate diff, which is used in the linear approximation of acos(x) */ + diff = (*lspPtr)-(*cosTblPtr); + + /* + The linear approximation of acos(lsp[i]) : + acos(lsp[i])= k*512 + (WebRtcIlbcfix_kAcosDerivative[ind]*offset >> 11) + */ + + /* tmp (linear offset) in Q16 */ + tmp = (int16_t)((WebRtcIlbcfix_kAcosDerivative[k] * diff) >> 11); + + /* freq in Q16 */ + freq = (k << 9) + tmp; + + /* lsf = freq*2*pi */ + (*lsfPtr) = (int16_t)(((int32_t)freq*25736)>>15); + + lsfPtr--; + lspPtr--; + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h new file mode 100644 index 0000000000..11739853fb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsp2Lsf.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSP_TO_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSP_TO_LSF_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * conversion from LSP coefficients to LSF coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsp2Lsf( + int16_t *lsp, /* (i) lsp vector -1...+1 in Q15 */ + int16_t *lsf, /* (o) Lsf vector 0...Pi in Q13 + (ordered, so that lsf[i]lsf[0])<<10; /* Bit 0..5 */ + (*bitstreamPtr) |= (enc_bits->lsf[1])<<3; /* Bit 6..12 */ + (*bitstreamPtr) |= (enc_bits->lsf[2]&0x70)>>4; /* Bit 13..15 */ + bitstreamPtr++; + /* Second int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->lsf[2]&0xF)<<12; /* Bit 0..3 */ + + if (mode==20) { + (*bitstreamPtr) |= (enc_bits->startIdx)<<10; /* Bit 4..5 */ + (*bitstreamPtr) |= (enc_bits->state_first)<<9; /* Bit 6 */ + (*bitstreamPtr) |= (enc_bits->idxForMax)<<3; /* Bit 7..12 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[0])&0x70)>>4; /* Bit 13..15 */ + bitstreamPtr++; + /* Third int16_t */ + (*bitstreamPtr) = ((enc_bits->cb_index[0])&0xE)<<12; /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[0])&0x18)<<8; /* Bit 3..4 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[1])&0x8)<<7; /* Bit 5 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[3])&0xFE)<<2; /* Bit 6..12 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[3])&0x10)>>2; /* Bit 13 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[4])&0x8)>>2; /* Bit 14 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[6])&0x10)>>4; /* Bit 15 */ + } else { /* mode==30 */ + (*bitstreamPtr) |= (enc_bits->lsf[3])<<6; /* Bit 4..9 */ + (*bitstreamPtr) |= (enc_bits->lsf[4]&0x7E)>>1; /* Bit 10..15 */ + bitstreamPtr++; + /* Third int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->lsf[4]&0x1)<<15; /* Bit 0 */ + (*bitstreamPtr) |= (enc_bits->lsf[5])<<8; /* Bit 1..7 */ + (*bitstreamPtr) |= (enc_bits->startIdx)<<5; /* Bit 8..10 */ + (*bitstreamPtr) |= (enc_bits->state_first)<<4; /* Bit 11 */ + (*bitstreamPtr) |= ((enc_bits->idxForMax)&0x3C)>>2; /* Bit 12..15 */ + bitstreamPtr++; + /* 4:th int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->idxForMax&0x3)<<14; /* Bit 0..1 */ + (*bitstreamPtr) |= (enc_bits->cb_index[0]&0x78)<<7; /* Bit 2..5 */ + (*bitstreamPtr) |= (enc_bits->gain_index[0]&0x10)<<5; /* Bit 6 */ + (*bitstreamPtr) |= (enc_bits->gain_index[1]&0x8)<<5; /* Bit 7 */ + (*bitstreamPtr) |= (enc_bits->cb_index[3]&0xFC); /* Bit 8..13 */ + (*bitstreamPtr) |= (enc_bits->gain_index[3]&0x10)>>3; /* Bit 14 */ + (*bitstreamPtr) |= (enc_bits->gain_index[4]&0x8)>>3; /* Bit 15 */ + } + /* Class 2 bits of ULP */ + /* 4:th to 6:th int16_t for 20 ms case + 5:th to 7:th int16_t for 30 ms case */ + bitstreamPtr++; + tmpPtr=enc_bits->idxVec; + for (k=0; k<3; k++) { + (*bitstreamPtr) = 0; + for (i=15; i>=0; i--) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x4)>>2)<6; i--) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x4)>>2)<gain_index[1]&0x4)<<4; /* Bit 9 */ + (*bitstreamPtr) |= (enc_bits->gain_index[3]&0xC)<<2; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[4]&0x4)<<1; /* Bit 12 */ + (*bitstreamPtr) |= (enc_bits->gain_index[6]&0x8)>>1; /* Bit 13 */ + (*bitstreamPtr) |= (enc_bits->gain_index[7]&0xC)>>2; /* Bit 14..15 */ + + } else { /* mode==30 */ + /* 8:th int16_t */ + (*bitstreamPtr) = 0; + for (i=15; i>5; i--) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x4)>>2)<cb_index[0]&0x6)<<3; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[0]&0x8); /* Bit 12 */ + (*bitstreamPtr) |= (enc_bits->gain_index[1]&0x4); /* Bit 13 */ + (*bitstreamPtr) |= (enc_bits->cb_index[3]&0x2); /* Bit 14 */ + (*bitstreamPtr) |= (enc_bits->cb_index[6]&0x80)>>7; /* Bit 15 */ + bitstreamPtr++; + /* 9:th int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->cb_index[6]&0x7E)<<9;/* Bit 0..5 */ + (*bitstreamPtr) |= (enc_bits->cb_index[9]&0xFE)<<2; /* Bit 6..12 */ + (*bitstreamPtr) |= (enc_bits->cb_index[12]&0xE0)>>5; /* Bit 13..15 */ + bitstreamPtr++; + /* 10:th int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->cb_index[12]&0x1E)<<11;/* Bit 0..3 */ + (*bitstreamPtr) |= (enc_bits->gain_index[3]&0xC)<<8; /* Bit 4..5 */ + (*bitstreamPtr) |= (enc_bits->gain_index[4]&0x6)<<7; /* Bit 6..7 */ + (*bitstreamPtr) |= (enc_bits->gain_index[6]&0x18)<<3; /* Bit 8..9 */ + (*bitstreamPtr) |= (enc_bits->gain_index[7]&0xC)<<2; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[9]&0x10)>>1; /* Bit 12 */ + (*bitstreamPtr) |= (enc_bits->gain_index[10]&0x8)>>1; /* Bit 13 */ + (*bitstreamPtr) |= (enc_bits->gain_index[12]&0x10)>>3; /* Bit 14 */ + (*bitstreamPtr) |= (enc_bits->gain_index[13]&0x8)>>3; /* Bit 15 */ + } + bitstreamPtr++; + /* Class 3 bits of ULP */ + /* 8:th to 14:th int16_t for 20 ms case + 11:th to 17:th int16_t for 30 ms case */ + tmpPtr=enc_bits->idxVec; + for (k=0; k<7; k++) { + (*bitstreamPtr) = 0; + for (i=14; i>=0; i-=2) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x3))<idxVec[56])&0x3))<<14;/* Bit 0..1 */ + (*bitstreamPtr) |= (((enc_bits->cb_index[0])&1))<<13; /* Bit 2 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[1]))<<6; /* Bit 3..9 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[2])&0x7E)>>1; /* Bit 10..15 */ + bitstreamPtr++; + /* 16:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->cb_index[2])&0x1))<<15; + /* Bit 0 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[0])&0x7)<<12; /* Bit 1..3 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[1])&0x3)<<10; /* Bit 4..5 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[2]))<<7; /* Bit 6..8 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[3])&0x1)<<6; /* Bit 9 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[4])&0x7E)>>1; /* Bit 10..15 */ + bitstreamPtr++; + /* 17:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->cb_index[4])&0x1))<<15; + /* Bit 0 */ + (*bitstreamPtr) |= (enc_bits->cb_index[5])<<8; /* Bit 1..7 */ + (*bitstreamPtr) |= (enc_bits->cb_index[6]); /* Bit 8..15 */ + bitstreamPtr++; + /* 18:th int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[7]))<<8; /* Bit 0..7 */ + (*bitstreamPtr) |= (enc_bits->cb_index[8]); /* Bit 8..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->gain_index[3])&0x3))<<14; + /* Bit 0..1 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[4])&0x3)<<12; /* Bit 2..3 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[5]))<<9; /* Bit 4..6 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[6])&0x7)<<6; /* Bit 7..9 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[7])&0x3)<<4; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[8])<<1; /* Bit 12..14 */ + } else { /* mode==30 */ + /* 18:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->idxVec[56])&0x3))<<14;/* Bit 0..1 */ + (*bitstreamPtr) |= (((enc_bits->idxVec[57])&0x3))<<12; /* Bit 2..3 */ + (*bitstreamPtr) |= (((enc_bits->cb_index[0])&1))<<11; /* Bit 4 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[1]))<<4; /* Bit 5..11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[2])&0x78)>>3; /* Bit 12..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[2])&0x7)<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[0])&0x7)<<10; /* Bit 3..5 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[1])&0x3)<<8; /* Bit 6..7 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[2])&0x7)<<5; /* Bit 8..10 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[3])&0x1)<<4; /* Bit 11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[4])&0x78)>>3; /* Bit 12..15 */ + bitstreamPtr++; + /* 20:th int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[4])&0x7)<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[5]))<<6; /* Bit 3..9 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[6])&0x1)<<5; /* Bit 10 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[7])&0xF8)>>3; /* Bit 11..15 */ + bitstreamPtr++; + /* 21:st int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[7])&0x7)<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[8]))<<5; /* Bit 3..10 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[9])&0x1)<<4; /* Bit 11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[10])&0xF0)>>4; /* Bit 12..15 */ + bitstreamPtr++; + /* 22:nd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[10])&0xF)<<12; + /* Bit 0..3 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[11]))<<4; /* Bit 4..11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[12])&0x1)<<3; /* Bit 12 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[13])&0xE0)>>5; /* Bit 13..15 */ + bitstreamPtr++; + /* 23:rd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[13])&0x1F)<<11; + /* Bit 0..4 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[14]))<<3; /* Bit 5..12 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[3])&0x3)<<1; /* Bit 13..14 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[4])&0x1); /* Bit 15 */ + bitstreamPtr++; + /* 24:rd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->gain_index[5]))<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[6])&0x7)<<10; /* Bit 3..5 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[7])&0x3)<<8; /* Bit 6..7 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[8]))<<5; /* Bit 8..10 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[9])&0xF)<<1; /* Bit 11..14 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[10])&0x4)>>2; /* Bit 15 */ + bitstreamPtr++; + /* 25:rd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->gain_index[10])&0x3)<<14; + /* Bit 0..1 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[11]))<<11; /* Bit 2..4 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[12])&0xF)<<7; /* Bit 5..8 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[13])&0x7)<<4; /* Bit 9..11 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[14]))<<1; /* Bit 12..14 */ + } + /* Last bit is automatically zero */ + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/pack_bits.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/pack_bits.h new file mode 100644 index 0000000000..66df7fc803 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/pack_bits.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_PackBits.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_PACK_BITS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_PACK_BITS_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * unpacking of bits from bitstream, i.e., vector of bytes + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_PackBits( + uint16_t *bitstream, /* (o) The packetized bitstream */ + iLBC_bits *enc_bits, /* (i) Encoded bits */ + int16_t mode /* (i) Codec mode (20 or 30) */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c new file mode 100644 index 0000000000..df8a78b1ed --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsf.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "poly_to_lsp.h" +#include "lsp_to_lsf.h" + +void WebRtcIlbcfix_Poly2Lsf( + int16_t *lsf, /* (o) lsf coefficients (Q13) */ + int16_t *a /* (i) A coefficients (Q12) */ + ) { + int16_t lsp[10]; + WebRtcIlbcfix_Poly2Lsp(a, lsp, (int16_t*)WebRtcIlbcfix_kLspMean); + WebRtcIlbcfix_Lsp2Lsf(lsp, lsf, 10); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h new file mode 100644 index 0000000000..3f2a2a0115 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsf.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSF_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * conversion from lpc coefficients to lsf coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Poly2Lsf( + int16_t *lsf, /* (o) lsf coefficients (Q13) */ + int16_t *a /* (i) A coefficients (Q12) */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c new file mode 100644 index 0000000000..ca376b34de --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsp.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "chebyshev.h" + +/*----------------------------------------------------------------* + * conversion from lpc coefficients to lsp coefficients + * function is only for 10:th order LPC + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Poly2Lsp( + int16_t *a, /* (o) A coefficients in Q12 */ + int16_t *lsp, /* (i) LSP coefficients in Q15 */ + int16_t *old_lsp /* (i) old LSP coefficients that are used if the new + coefficients turn out to be unstable */ + ) { + int16_t f[2][6]; /* f[0][] represents f1 and f[1][] represents f2 */ + int16_t *a_i_ptr, *a_10mi_ptr; + int16_t *f1ptr, *f2ptr; + int32_t tmpW32; + int16_t x, y, xlow, ylow, xmid, ymid, xhigh, yhigh, xint; + int16_t shifts, sign; + int i, j; + int foundFreqs; + int fi_select; + + /* + Calculate the two polynomials f1(z) and f2(z) + (the sum and the diff polynomial) + f1[0] = f2[0] = 1.0; + f1[i+1] = a[i+1] + a[10-i] - f1[i]; + f2[i+1] = a[i+1] - a[10-i] - f1[i]; + */ + + a_i_ptr = a + 1; + a_10mi_ptr = a + 10; + f1ptr = f[0]; + f2ptr = f[1]; + (*f1ptr) = 1024; /* 1.0 in Q10 */ + (*f2ptr) = 1024; /* 1.0 in Q10 */ + for (i = 0; i < 5; i++) { + *(f1ptr + 1) = + (int16_t)((((int32_t)(*a_i_ptr) + *a_10mi_ptr) >> 2) - *f1ptr); + *(f2ptr + 1) = + (int16_t)((((int32_t)(*a_i_ptr) - *a_10mi_ptr) >> 2) + *f2ptr); + a_i_ptr++; + a_10mi_ptr--; + f1ptr++; + f2ptr++; + } + + /* + find the LSPs using the Chebychev pol. evaluation + */ + + fi_select = 0; /* selector between f1 and f2, start with f1 */ + + foundFreqs = 0; + + xlow = WebRtcIlbcfix_kCosGrid[0]; + ylow = WebRtcIlbcfix_Chebyshev(xlow, f[fi_select]); + + /* + Iterate until all the 10 LSP's have been found or + all the grid points have been tried. If the 10 LSP's can + not be found, set the LSP vector to previous LSP + */ + + for (j = 1; j < COS_GRID_POINTS && foundFreqs < 10; j++) { + xhigh = xlow; + yhigh = ylow; + xlow = WebRtcIlbcfix_kCosGrid[j]; + ylow = WebRtcIlbcfix_Chebyshev(xlow, f[fi_select]); + + if (ylow * yhigh <= 0) { + /* Run 4 times to reduce the interval */ + for (i = 0; i < 4; i++) { + /* xmid =(xlow + xhigh)/2 */ + xmid = (xlow >> 1) + (xhigh >> 1); + ymid = WebRtcIlbcfix_Chebyshev(xmid, f[fi_select]); + + if (ylow * ymid <= 0) { + yhigh = ymid; + xhigh = xmid; + } else { + ylow = ymid; + xlow = xmid; + } + } + + /* + Calculater xint by linear interpolation: + xint = xlow - ylow*(xhigh-xlow)/(yhigh-ylow); + */ + + x = xhigh - xlow; + y = yhigh - ylow; + + if (y == 0) { + xint = xlow; + } else { + sign = y; + y = WEBRTC_SPL_ABS_W16(y); + shifts = (int16_t)WebRtcSpl_NormW32(y)-16; + y <<= shifts; + y = (int16_t)WebRtcSpl_DivW32W16(536838144, y); /* 1/(yhigh-ylow) */ + + tmpW32 = (x * y) >> (19 - shifts); + + /* y=(xhigh-xlow)/(yhigh-ylow) */ + y = (int16_t)(tmpW32&0xFFFF); + + if (sign < 0) { + y = -y; + } + /* tmpW32 = ylow*(xhigh-xlow)/(yhigh-ylow) */ + tmpW32 = (ylow * y) >> 10; + xint = xlow-(int16_t)(tmpW32&0xFFFF); + } + + /* Store the calculated lsp */ + lsp[foundFreqs] = (int16_t)xint; + foundFreqs++; + + /* if needed, set xlow and ylow for next recursion */ + if (foundFreqs<10) { + xlow = xint; + /* Swap between f1 and f2 (f[0][] and f[1][]) */ + fi_select = ((fi_select+1)&0x1); + + ylow = WebRtcIlbcfix_Chebyshev(xlow, f[fi_select]); + } + } + } + + /* Check if M roots found, if not then use the old LSP */ + if (foundFreqs < 10) { + WEBRTC_SPL_MEMCPY_W16(lsp, old_lsp, 10); + } + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h new file mode 100644 index 0000000000..cc275076fa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsp.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSP_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSP_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * conversion from lpc coefficients to lsp coefficients + * function is only for 10:th order LPC + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Poly2Lsp( + int16_t *a, /* (o) A coefficients in Q12 */ + int16_t *lsp, /* (i) LSP coefficients in Q15 */ + int16_t *old_lsp /* (i) old LSP coefficients that are used if the new + coefficients turn out to be unstable */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.c new file mode 100644 index 0000000000..3c1265e5d5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Refiner.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "enh_upsample.h" +#include "my_corr.h" + +/*----------------------------------------------------------------* + * find segment starting near idata+estSegPos that has highest + * correlation with idata+centerStartPos through + * idata+centerStartPos+ENH_BLOCKL-1 segment is found at a + * resolution of ENH_UPSO times the original of the original + * sampling rate + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Refiner( + size_t *updStartPos, /* (o) updated start point (Q-2) */ + int16_t *idata, /* (i) original data buffer */ + size_t idatal, /* (i) dimension of idata */ + size_t centerStartPos, /* (i) beginning center segment */ + size_t estSegPos, /* (i) estimated beginning other segment (Q-2) */ + int16_t *surround, /* (i/o) The contribution from this sequence + summed with earlier contributions */ + int16_t gain /* (i) Gain to use for this sequence */ + ){ + size_t estSegPosRounded, searchSegStartPos, searchSegEndPos, corrdim; + size_t tloc, tloc2, i; + + int32_t maxtemp, scalefact; + int16_t *filtStatePtr, *polyPtr; + /* Stack based */ + int16_t filt[7]; + int32_t corrVecUps[ENH_CORRDIM*ENH_UPS0]; + int32_t corrVecTemp[ENH_CORRDIM]; + int16_t vect[ENH_VECTL]; + int16_t corrVec[ENH_CORRDIM]; + + /* defining array bounds */ + + estSegPosRounded = (estSegPos - 2) >> 2; + + searchSegStartPos = + (estSegPosRounded < ENH_SLOP) ? 0 : (estSegPosRounded - ENH_SLOP); + + searchSegEndPos = estSegPosRounded + ENH_SLOP; + if ((searchSegEndPos + ENH_BLOCKL) >= idatal) { + searchSegEndPos = idatal - ENH_BLOCKL - 1; + } + + corrdim = searchSegEndPos + 1 - searchSegStartPos; + + /* compute upsampled correlation and find + location of max */ + + WebRtcIlbcfix_MyCorr(corrVecTemp, idata + searchSegStartPos, + corrdim + ENH_BLOCKL - 1, idata + centerStartPos, + ENH_BLOCKL); + + /* Calculate the rescaling factor for the correlation in order to + put the correlation in a int16_t vector instead */ + maxtemp = WebRtcSpl_MaxAbsValueW32(corrVecTemp, corrdim); + + scalefact = WebRtcSpl_GetSizeInBits(maxtemp) - 15; + + if (scalefact > 0) { + for (i = 0; i < corrdim; i++) { + corrVec[i] = (int16_t)(corrVecTemp[i] >> scalefact); + } + } else { + for (i = 0; i < corrdim; i++) { + corrVec[i] = (int16_t)corrVecTemp[i]; + } + } + /* In order to guarantee that all values are initialized */ + for (i = corrdim; i < ENH_CORRDIM; i++) { + corrVec[i] = 0; + } + + /* Upsample the correlation */ + WebRtcIlbcfix_EnhUpsample(corrVecUps, corrVec); + + /* Find maximum */ + tloc = WebRtcSpl_MaxIndexW32(corrVecUps, ENH_UPS0 * corrdim); + + /* make vector can be upsampled without ever running outside + bounds */ + *updStartPos = searchSegStartPos * 4 + tloc + 4; + + tloc2 = (tloc + 3) >> 2; + + /* initialize the vector to be filtered, stuff with zeros + when data is outside idata buffer */ + if (ENH_FL0 > (searchSegStartPos + tloc2)) { + const size_t st = ENH_FL0 - searchSegStartPos - tloc2; + WebRtcSpl_MemSetW16(vect, 0, st); + WEBRTC_SPL_MEMCPY_W16(&vect[st], idata, ENH_VECTL - st); + } else { + const size_t st = searchSegStartPos + tloc2 - ENH_FL0; + if ((st + ENH_VECTL) > idatal) { + const size_t en = st + ENH_VECTL - idatal; + WEBRTC_SPL_MEMCPY_W16(vect, &idata[st], ENH_VECTL - en); + WebRtcSpl_MemSetW16(&vect[ENH_VECTL - en], 0, en); + } else { + WEBRTC_SPL_MEMCPY_W16(vect, &idata[st], ENH_VECTL); + } + } + + /* compute the segment (this is actually a convolution) */ + filtStatePtr = filt + 6; + polyPtr = (int16_t*)WebRtcIlbcfix_kEnhPolyPhaser[tloc2 * ENH_UPS0 - tloc]; + for (i = 0; i < 7; i++) { + *filtStatePtr-- = *polyPtr++; + } + + WebRtcSpl_FilterMAFastQ12(&vect[6], vect, filt, ENH_FLO_MULT2_PLUS1, + ENH_BLOCKL); + + /* Add the contribution from this vector (scaled with gain) to the total + surround vector */ + WebRtcSpl_AddAffineVectorToVector(surround, vect, gain, 32768, 16, + ENH_BLOCKL); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.h new file mode 100644 index 0000000000..92b81be1ca --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Refiner.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_REFINER_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_REFINER_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * find segment starting near idata+estSegPos that has highest + * correlation with idata+centerStartPos through + * idata+centerStartPos+ENH_BLOCKL-1 segment is found at a + * resolution of ENH_UPSO times the original of the original + * sampling rate + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Refiner( + size_t *updStartPos, /* (o) updated start point (Q-2) */ + int16_t *idata, /* (i) original data buffer */ + size_t idatal, /* (i) dimension of idata */ + size_t centerStartPos, /* (i) beginning center segment */ + size_t estSegPos, /* (i) estimated beginning other segment (Q-2) */ + int16_t *surround, /* (i/o) The contribution from this sequence + summed with earlier contributions */ + int16_t gain /* (i) Gain to use for this sequence */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c new file mode 100644 index 0000000000..e63dda8c8f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleInterpolateLsf.c + +******************************************************************/ + +#include "defines.h" +#include "lsf_interpolate_to_poly_enc.h" +#include "bw_expand.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * lsf interpolator (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleInterpolateLsf( + int16_t *syntdenum, /* (o) the synthesis filter denominator + resulting from the quantized + interpolated lsf Q12 */ + int16_t *weightdenum, /* (o) the weighting filter denominator + resulting from the unquantized + interpolated lsf Q12 */ + int16_t *lsf, /* (i) the unquantized lsf coefficients Q13 */ + int16_t *lsfdeq, /* (i) the dequantized lsf coefficients Q13 */ + int16_t *lsfold, /* (i) the unquantized lsf coefficients of + the previous signal frame Q13 */ + int16_t *lsfdeqold, /* (i) the dequantized lsf coefficients of the + previous signal frame Q13 */ + int16_t length, /* (i) should equate FILTERORDER */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ) { + size_t i; + int pos, lp_length; + + int16_t *lsf2, *lsfdeq2; + /* Stack based */ + int16_t lp[LPC_FILTERORDER + 1]; + + lsf2 = lsf + length; + lsfdeq2 = lsfdeq + length; + lp_length = length + 1; + + if (iLBCenc_inst->mode==30) { + /* subframe 1: Interpolation between old and first set of + lsf coefficients */ + + /* Calculate Analysis/Syntehsis filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight30ms[0], + length); + WEBRTC_SPL_MEMCPY_W16(syntdenum, lp, lp_length); + + /* Calculate Weighting filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfold, lsf, + WebRtcIlbcfix_kLsfWeight30ms[0], + length); + WebRtcIlbcfix_BwExpand(weightdenum, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpWeightDenum, + (int16_t)lp_length); + + /* subframe 2 to 6: Interpolation between first and second + set of lsf coefficients */ + + pos = lp_length; + for (i = 1; i < iLBCenc_inst->nsub; i++) { + + /* Calculate Analysis/Syntehsis filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfdeq, lsfdeq2, + WebRtcIlbcfix_kLsfWeight30ms[i], + length); + WEBRTC_SPL_MEMCPY_W16(syntdenum + pos, lp, lp_length); + + /* Calculate Weighting filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsf, lsf2, + WebRtcIlbcfix_kLsfWeight30ms[i], + length); + WebRtcIlbcfix_BwExpand(weightdenum + pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpWeightDenum, + (int16_t)lp_length); + + pos += lp_length; + } + + /* update memory */ + + WEBRTC_SPL_MEMCPY_W16(lsfold, lsf2, length); + WEBRTC_SPL_MEMCPY_W16(lsfdeqold, lsfdeq2, length); + + } else { /* iLBCenc_inst->mode==20 */ + pos = 0; + for (i = 0; i < iLBCenc_inst->nsub; i++) { + + /* Calculate Analysis/Syntehsis filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight20ms[i], + length); + WEBRTC_SPL_MEMCPY_W16(syntdenum + pos, lp, lp_length); + + /* Calculate Weighting filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfold, lsf, + WebRtcIlbcfix_kLsfWeight20ms[i], + length); + WebRtcIlbcfix_BwExpand(weightdenum+pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpWeightDenum, + (int16_t)lp_length); + + pos += lp_length; + } + + /* update memory */ + + WEBRTC_SPL_MEMCPY_W16(lsfold, lsf, length); + WEBRTC_SPL_MEMCPY_W16(lsfdeqold, lsfdeq, length); + + } + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h new file mode 100644 index 0000000000..ccf66d3295 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleInterpolateLsf.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_INTERPOLATE_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_INTERPOLATE_LSF_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * lsf interpolator (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleInterpolateLsf( + int16_t *syntdenum, /* (o) the synthesis filter denominator + resulting from the quantized + interpolated lsf Q12 */ + int16_t *weightdenum, /* (o) the weighting filter denominator + resulting from the unquantized + interpolated lsf Q12 */ + int16_t *lsf, /* (i) the unquantized lsf coefficients Q13 */ + int16_t *lsfdeq, /* (i) the dequantized lsf coefficients Q13 */ + int16_t *lsfold, /* (i) the unquantized lsf coefficients of + the previous signal frame Q13 */ + int16_t *lsfdeqold, /* (i) the dequantized lsf coefficients of the + previous signal frame Q13 */ + int16_t length, /* (i) should equate FILTERORDER */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c new file mode 100644 index 0000000000..72d80e0430 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLpcAnalysis.c + +******************************************************************/ + +#include "defines.h" +#include "window32_w32.h" +#include "bw_expand.h" +#include "poly_to_lsf.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * lpc analysis (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLpcAnalysis( + int16_t *lsf, /* (o) lsf coefficients */ + int16_t *data, /* (i) new block of speech */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ) { + int k; + int scale; + size_t is; + int16_t stability; + /* Stack based */ + int16_t A[LPC_FILTERORDER + 1]; + int32_t R[LPC_FILTERORDER + 1]; + int16_t windowedData[BLOCKL_MAX]; + int16_t rc[LPC_FILTERORDER]; + + is=LPC_LOOKBACK+BLOCKL_MAX-iLBCenc_inst->blockl; + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lpc_buffer+is,data,iLBCenc_inst->blockl); + + /* No lookahead, last window is asymmetric */ + + for (k = 0; k < iLBCenc_inst->lpc_n; k++) { + + is = LPC_LOOKBACK; + + if (k < (iLBCenc_inst->lpc_n - 1)) { + + /* Hanning table WebRtcIlbcfix_kLpcWin[] is in Q15-domain so the output is right-shifted 15 */ + WebRtcSpl_ElementwiseVectorMult(windowedData, iLBCenc_inst->lpc_buffer, WebRtcIlbcfix_kLpcWin, BLOCKL_MAX, 15); + } else { + + /* Hanning table WebRtcIlbcfix_kLpcAsymWin[] is in Q15-domain so the output is right-shifted 15 */ + WebRtcSpl_ElementwiseVectorMult(windowedData, iLBCenc_inst->lpc_buffer+is, WebRtcIlbcfix_kLpcAsymWin, BLOCKL_MAX, 15); + } + + /* Compute autocorrelation */ + WebRtcSpl_AutoCorrelation(windowedData, BLOCKL_MAX, LPC_FILTERORDER, R, &scale); + + /* Window autocorrelation vector */ + WebRtcIlbcfix_Window32W32(R, R, WebRtcIlbcfix_kLpcLagWin, LPC_FILTERORDER + 1 ); + + /* Calculate the A coefficients from the Autocorrelation using Levinson Durbin algorithm */ + stability=WebRtcSpl_LevinsonDurbin(R, A, rc, LPC_FILTERORDER); + + /* + Set the filter to {1.0, 0.0, 0.0,...} if filter from Levinson Durbin algorithm is unstable + This should basically never happen... + */ + if (stability!=1) { + A[0]=4096; + WebRtcSpl_MemSetW16(&A[1], 0, LPC_FILTERORDER); + } + + /* Bandwidth expand the filter coefficients */ + WebRtcIlbcfix_BwExpand(A, A, (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, LPC_FILTERORDER+1); + + /* Convert from A to LSF representation */ + WebRtcIlbcfix_Poly2Lsf(lsf + k*LPC_FILTERORDER, A); + } + + is=LPC_LOOKBACK+BLOCKL_MAX-iLBCenc_inst->blockl; + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lpc_buffer, + iLBCenc_inst->lpc_buffer+LPC_LOOKBACK+BLOCKL_MAX-is, is); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h new file mode 100644 index 0000000000..eb5b80302e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLpcAnalysis.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LPC_ANALYSIS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LPC_ANALYSIS_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * lpc analysis (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLpcAnalysis( + int16_t *lsf, /* (o) lsf coefficients */ + int16_t *data, /* (i) new block of speech */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c new file mode 100644 index 0000000000..9fa0d61ded --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfDeQ.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * obtain dequantized lsf coefficients from quantization index + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfDeQ( + int16_t *lsfdeq, /* (o) dequantized lsf coefficients */ + int16_t *index, /* (i) quantization index */ + int16_t lpc_n /* (i) number of LPCs */ + ){ + int i, j, pos, cb_pos; + + /* decode first LSF */ + + pos = 0; + cb_pos = 0; + for (i = 0; i < LSF_NSPLIT; i++) { + for (j = 0; j < WebRtcIlbcfix_kLsfDimCb[i]; j++) { + lsfdeq[pos + j] = WebRtcIlbcfix_kLsfCb[cb_pos + j + index[i] * + WebRtcIlbcfix_kLsfDimCb[i]]; + } + pos += WebRtcIlbcfix_kLsfDimCb[i]; + cb_pos += WebRtcIlbcfix_kLsfSizeCb[i] * WebRtcIlbcfix_kLsfDimCb[i]; + } + + if (lpc_n>1) { + /* decode last LSF */ + pos = 0; + cb_pos = 0; + for (i = 0; i < LSF_NSPLIT; i++) { + for (j = 0; j < WebRtcIlbcfix_kLsfDimCb[i]; j++) { + lsfdeq[LPC_FILTERORDER + pos + j] = WebRtcIlbcfix_kLsfCb[ + cb_pos + index[LSF_NSPLIT + i] * WebRtcIlbcfix_kLsfDimCb[i] + j]; + } + pos += WebRtcIlbcfix_kLsfDimCb[i]; + cb_pos += WebRtcIlbcfix_kLsfSizeCb[i] * WebRtcIlbcfix_kLsfDimCb[i]; + } + } + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h new file mode 100644 index 0000000000..660f8fe15d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfDeQ.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_DEQUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_DEQUANT_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * obtain dequantized lsf coefficients from quantization index + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfDeQ( + int16_t *lsfdeq, /* (o) dequantized lsf coefficients */ + int16_t *index, /* (i) quantization index */ + int16_t lpc_n /* (i) number of LPCs */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c new file mode 100644 index 0000000000..8daba50150 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfQ.c + +******************************************************************/ + +#include "defines.h" +#include "split_vq.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * lsf quantizer (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfQ( + int16_t *lsfdeq, /* (o) dequantized lsf coefficients + (dimension FILTERORDER) Q13 */ + int16_t *index, /* (o) quantization index */ + int16_t *lsf, /* (i) the lsf coefficient vector to be + quantized (dimension FILTERORDER) Q13 */ + int16_t lpc_n /* (i) number of lsf sets to quantize */ + ){ + + /* Quantize first LSF with memoryless split VQ */ + WebRtcIlbcfix_SplitVq( lsfdeq, index, lsf, + (int16_t*)WebRtcIlbcfix_kLsfCb, (int16_t*)WebRtcIlbcfix_kLsfDimCb, (int16_t*)WebRtcIlbcfix_kLsfSizeCb); + + if (lpc_n==2) { + /* Quantize second LSF with memoryless split VQ */ + WebRtcIlbcfix_SplitVq( lsfdeq + LPC_FILTERORDER, index + LSF_NSPLIT, + lsf + LPC_FILTERORDER, (int16_t*)WebRtcIlbcfix_kLsfCb, + (int16_t*)WebRtcIlbcfix_kLsfDimCb, (int16_t*)WebRtcIlbcfix_kLsfSizeCb); + } + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h new file mode 100644 index 0000000000..6dc57196bf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfQ.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_QUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_QUANT_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * lsf quantizer (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfQ( + int16_t *lsfdeq, /* (o) dequantized lsf coefficients + (dimension FILTERORDER) Q13 */ + int16_t *index, /* (o) quantization index */ + int16_t *lsf, /* (i) the lsf coefficient vector to be + quantized (dimension FILTERORDER) Q13 */ + int16_t lpc_n /* (i) number of lsf sets to quantize */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.c new file mode 100644 index 0000000000..ed879327c0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "smooth_out_data.h" + +/*----------------------------------------------------------------* + * find the smoothed output data + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Smooth( + int16_t *odata, /* (o) smoothed output */ + int16_t *current, /* (i) the un enhanced residual for + this block */ + int16_t *surround /* (i) The approximation from the + surrounding sequences */ + ) { + int16_t scale, scale1, scale2; + int16_t A, B, C, denomW16; + int32_t B_W32, denom, num; + int32_t errs; + int32_t w00,w10,w11, endiff, crit; + int32_t w00prim, w10prim, w11_div_w00; + int16_t w11prim; + int16_t bitsw00, bitsw10, bitsw11; + int32_t w11w00, w10w10, w00w00; + uint32_t max1, max2, max12; + + /* compute some inner products (ensure no overflow by first calculating proper scale factor) */ + + w00 = w10 = w11 = 0; + + // Calculate a right shift that will let us sum ENH_BLOCKL pairwise products + // of values from the two sequences without overflowing an int32_t. (The +1 + // in max1 and max2 are because WebRtcSpl_MaxAbsValueW16 will return 2**15 - + // 1 if the input array contains -2**15.) + max1 = WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL) + 1; + max2 = WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL) + 1; + max12 = WEBRTC_SPL_MAX(max1, max2); + scale = (64 - 31) - + WebRtcSpl_CountLeadingZeros64((max12 * max12) * (uint64_t)ENH_BLOCKL); + scale=WEBRTC_SPL_MAX(0, scale); + + w00=WebRtcSpl_DotProductWithScale(current,current,ENH_BLOCKL,scale); + w11=WebRtcSpl_DotProductWithScale(surround,surround,ENH_BLOCKL,scale); + w10=WebRtcSpl_DotProductWithScale(surround,current,ENH_BLOCKL,scale); + + if (w00<0) w00 = WEBRTC_SPL_WORD32_MAX; + if (w11<0) w11 = WEBRTC_SPL_WORD32_MAX; + + /* Rescale w00 and w11 to w00prim and w11prim, so that w00prim/w11prim + is in Q16 */ + + bitsw00 = WebRtcSpl_GetSizeInBits(w00); + bitsw11 = WebRtcSpl_GetSizeInBits(w11); + bitsw10 = WebRtcSpl_GetSizeInBits(WEBRTC_SPL_ABS_W32(w10)); + scale1 = 31 - bitsw00; + scale2 = 15 - bitsw11; + + if (scale2>(scale1-16)) { + scale2 = scale1 - 16; + } else { + scale1 = scale2 + 16; + } + + w00prim = w00 << scale1; + w11prim = (int16_t) WEBRTC_SPL_SHIFT_W32(w11, scale2); + + /* Perform C = sqrt(w11/w00) (C is in Q11 since (16+6)/2=11) */ + if (w11prim>64) { + endiff = WebRtcSpl_DivW32W16(w00prim, w11prim) << 6; + C = (int16_t)WebRtcSpl_SqrtFloor(endiff); /* C is in Q11 */ + } else { + C = 1; + } + + /* first try enhancement without power-constraint */ + + errs = WebRtcIlbcfix_Smooth_odata(odata, current, surround, C); + + + + /* if constraint violated by first try, add constraint */ + + if ( (6-scale+scale1) > 31) { + crit=0; + } else { + /* crit = 0.05 * w00 (Result in Q-6) */ + crit = WEBRTC_SPL_SHIFT_W32( + WEBRTC_SPL_MUL(ENH_A0, w00prim >> 14), + -(6-scale+scale1)); + } + + if (errs > crit) { + + if( w00 < 1) { + w00=1; + } + + /* Calculate w11*w00, w10*w10 and w00*w00 in the same Q domain */ + + scale1 = bitsw00-15; + scale2 = bitsw11-15; + + if (scale2>scale1) { + scale = scale2; + } else { + scale = scale1; + } + + w11w00 = (int16_t)WEBRTC_SPL_SHIFT_W32(w11, -scale) * + (int16_t)WEBRTC_SPL_SHIFT_W32(w00, -scale); + + w10w10 = (int16_t)WEBRTC_SPL_SHIFT_W32(w10, -scale) * + (int16_t)WEBRTC_SPL_SHIFT_W32(w10, -scale); + + w00w00 = (int16_t)WEBRTC_SPL_SHIFT_W32(w00, -scale) * + (int16_t)WEBRTC_SPL_SHIFT_W32(w00, -scale); + + /* Calculate (w11*w00-w10*w10)/(w00*w00) in Q16 */ + if (w00w00>65536) { + endiff = (w11w00-w10w10); + endiff = WEBRTC_SPL_MAX(0, endiff); + /* denom is in Q16 */ + denom = WebRtcSpl_DivW32W16(endiff, (int16_t)(w00w00 >> 16)); + } else { + denom = 65536; + } + + if( denom > 7){ /* eliminates numerical problems + for if smooth */ + + scale=WebRtcSpl_GetSizeInBits(denom)-15; + + if (scale>0) { + /* denomW16 is in Q(16+scale) */ + denomW16 = (int16_t)(denom >> scale); + + /* num in Q(34-scale) */ + num = ENH_A0_MINUS_A0A0DIV4 >> scale; + } else { + /* denomW16 is in Q16 */ + denomW16=(int16_t)denom; + + /* num in Q34 */ + num=ENH_A0_MINUS_A0A0DIV4; + } + + /* A sqrt( (ENH_A0-(ENH_A0^2)/4)*(w00*w00)/(w11*w00 + w10*w10) ) in Q9 */ + A = (int16_t)WebRtcSpl_SqrtFloor(WebRtcSpl_DivW32W16(num, denomW16)); + + /* B_W32 is in Q30 ( B = 1 - ENH_A0/2 - A * w10/w00 ) */ + scale1 = 31-bitsw10; + scale2 = 21-scale1; + w10prim = w10 == 0 ? 0 : w10 * (1 << scale1); + w00prim = WEBRTC_SPL_SHIFT_W32(w00, -scale2); + scale = bitsw00-scale2-15; + + if (scale>0) { + w10prim >>= scale; + w00prim >>= scale; + } + + if ((w00prim>0)&&(w10prim>0)) { + w11_div_w00=WebRtcSpl_DivW32W16(w10prim, (int16_t)w00prim); + + if (WebRtcSpl_GetSizeInBits(w11_div_w00)+WebRtcSpl_GetSizeInBits(A)>31) { + B_W32 = 0; + } else { + B_W32 = (int32_t)1073741824 - (int32_t)ENH_A0DIV2 - + WEBRTC_SPL_MUL(A, w11_div_w00); + } + B = (int16_t)(B_W32 >> 16); /* B in Q14. */ + } else { + /* No smoothing */ + A = 0; + B = 16384; /* 1 in Q14 */ + } + } + else{ /* essentially no difference between cycles; + smoothing not needed */ + + A = 0; + B = 16384; /* 1 in Q14 */ + } + + /* create smoothed sequence */ + + WebRtcSpl_ScaleAndAddVectors(surround, A, 9, + current, B, 14, + odata, ENH_BLOCKL); + } + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.h new file mode 100644 index 0000000000..351543229a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * find the smoothed output data + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Smooth( + int16_t *odata, /* (o) smoothed output */ + int16_t *current, /* (i) the un enhanced residual for + this block */ + int16_t *surround /* (i) The approximation from the + surrounding sequences */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c new file mode 100644 index 0000000000..9c41e0422b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth_odata.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +int32_t WebRtcIlbcfix_Smooth_odata( + int16_t *odata, + int16_t *psseq, + int16_t *surround, + int16_t C) +{ + int i; + + int16_t err; + int32_t errs; + + for(i=0;i<80;i++) { + odata[i]= (int16_t)((C * surround[i] + 1024) >> 11); + } + + errs=0; + for(i=0;i<80;i++) { + err = (psseq[i] - odata[i]) >> 3; + errs += err * err; /* errs in Q-6 */ + } + + return errs; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h new file mode 100644 index 0000000000..7ed1a055c9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth_odata.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_OUT_DATA_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_OUT_DATA_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * help function to WebRtcIlbcfix_Smooth() + *---------------------------------------------------------------*/ + +int32_t WebRtcIlbcfix_Smooth_odata( + int16_t *odata, + int16_t *psseq, + int16_t *surround, + int16_t C); + + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.c new file mode 100644 index 0000000000..c51bf6dc63 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SortSq.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * scalar quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SortSq( + int16_t *xq, /* (o) the quantized value */ + int16_t *index, /* (o) the quantization index */ + int16_t x, /* (i) the value to quantize */ + const int16_t *cb, /* (i) the quantization codebook */ + int16_t cb_size /* (i) the size of the quantization codebook */ + ){ + int i; + + if (x <= cb[0]) { + *index = 0; + *xq = cb[0]; + } else { + i = 0; + while ((x > cb[i]) && (i < (cb_size-1))) { + i++; + } + + if (x > (((int32_t)cb[i] + cb[i - 1] + 1) >> 1)) { + *index = i; + *xq = cb[i]; + } else { + *index = i - 1; + *xq = cb[i - 1]; + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.h new file mode 100644 index 0000000000..436e1185c0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SortSq.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SORT_SQ_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SORT_SQ_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * scalar quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SortSq( + int16_t *xq, /* (o) the quantized value */ + int16_t *index, /* (o) the quantization index */ + int16_t x, /* (i) the value to quantize */ + const int16_t *cb, /* (i) the quantization codebook */ + int16_t cb_size /* (i) the size of the quantization codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.c new file mode 100644 index 0000000000..39b6e1b56a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SplitVq.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" +#include "vq3.h" +#include "vq4.h" + +/*----------------------------------------------------------------* + * split vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SplitVq( + int16_t *qX, /* (o) the quantized vector in Q13 */ + int16_t *index, /* (o) a vector of indexes for all vector + codebooks in the split */ + int16_t *X, /* (i) the vector to quantize */ + int16_t *CB, /* (i) the quantizer codebook in Q13 */ + int16_t *dim, /* (i) the dimension of X and qX */ + int16_t *cbsize /* (i) the number of vectors in the codebook */ + ) { + + int16_t *qXPtr, *indexPtr, *CBPtr, *XPtr; + + /* Quantize X with the 3 vectror quantization tables */ + + qXPtr=qX; + indexPtr=index; + CBPtr=CB; + XPtr=X; + WebRtcIlbcfix_Vq3(qXPtr, indexPtr, CBPtr, XPtr, cbsize[0]); + + qXPtr+=3; + indexPtr+=1; + CBPtr+=(dim[0]*cbsize[0]); + XPtr+=3; + WebRtcIlbcfix_Vq3(qXPtr, indexPtr, CBPtr, XPtr, cbsize[1]); + + qXPtr+=3; + indexPtr+=1; + CBPtr+=(dim[1]*cbsize[1]); + XPtr+=3; + WebRtcIlbcfix_Vq4(qXPtr, indexPtr, CBPtr, XPtr, cbsize[2]); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.h new file mode 100644 index 0000000000..3b303bd156 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SplitVq.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SPLIT_VQ_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SPLIT_VQ_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * split vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SplitVq( + int16_t *qX, /* (o) the quantized vector in Q13 */ + int16_t *index, /* (o) a vector of indexes for all vector + codebooks in the split */ + int16_t *X, /* (i) the vector to quantize */ + int16_t *CB, /* (i) the quantizer codebook in Q13 */ + int16_t *dim, /* (i) the dimension of X and qX */ + int16_t *cbsize /* (i) the number of vectors in the codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_construct.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_construct.c new file mode 100644 index 0000000000..29fe91b87e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_construct.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_StateConstruct.c + +******************************************************************/ + +#include "defines.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * decoding of the start state + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_StateConstruct( + size_t idxForMax, /* (i) 6-bit index for the quantization of + max amplitude */ + int16_t *idxVec, /* (i) vector of quantization indexes */ + int16_t *syntDenum, /* (i) synthesis filter denumerator */ + int16_t *Out_fix, /* (o) the decoded state vector */ + size_t len /* (i) length of a state vector */ + ) { + size_t k; + int16_t maxVal; + int16_t *tmp1, *tmp2, *tmp3; + /* Stack based */ + int16_t numerator[1+LPC_FILTERORDER]; + int16_t sampleValVec[2*STATE_SHORT_LEN_30MS+LPC_FILTERORDER]; + int16_t sampleMaVec[2*STATE_SHORT_LEN_30MS+LPC_FILTERORDER]; + int16_t *sampleVal = &sampleValVec[LPC_FILTERORDER]; + int16_t *sampleMa = &sampleMaVec[LPC_FILTERORDER]; + int16_t *sampleAr = &sampleValVec[LPC_FILTERORDER]; + + /* initialization of coefficients */ + + for (k=0; k> + 22); + tmp1++; + tmp2--; + } + } else if (idxForMax<59) { + for(k=0; k> + 19); + tmp1++; + tmp2--; + } + } else { + for(k=0; k> + 17); + tmp1++; + tmp2--; + } + } + + /* Set the rest of the data to zero */ + WebRtcSpl_MemSetW16(&sampleVal[len], 0, len); + + /* circular convolution with all-pass filter */ + + /* Set the state to zero */ + WebRtcSpl_MemSetW16(sampleValVec, 0, (LPC_FILTERORDER)); + + /* Run MA filter + AR filter */ + WebRtcSpl_FilterMAFastQ12( + sampleVal, sampleMa, + numerator, LPC_FILTERORDER+1, len + LPC_FILTERORDER); + WebRtcSpl_MemSetW16(&sampleMa[len + LPC_FILTERORDER], 0, (len - LPC_FILTERORDER)); + WebRtcSpl_FilterARFastQ12( + sampleMa, sampleAr, + syntDenum, LPC_FILTERORDER+1, 2 * len); + + tmp1 = &sampleAr[len-1]; + tmp2 = &sampleAr[2*len-1]; + tmp3 = Out_fix; + for(k=0;kstate_short_len); + scaleRes = WebRtcSpl_GetSizeInBits(max)-12; + scaleRes = WEBRTC_SPL_MAX(0, scaleRes); + /* Set up the filter coefficients for the circular convolution */ + for (i=0; i>scaleRes); + } + + /* Copy the residual to a temporary buffer that we can filter + * and set the remaining samples to zero. + */ + WEBRTC_SPL_MEMCPY_W16(residualLong, residual, iLBCenc_inst->state_short_len); + WebRtcSpl_MemSetW16(residualLong + iLBCenc_inst->state_short_len, 0, iLBCenc_inst->state_short_len); + + /* Run the Zero-Pole filter (Ciurcular convolution) */ + WebRtcSpl_MemSetW16(residualLongVec, 0, LPC_FILTERORDER); + WebRtcSpl_FilterMAFastQ12(residualLong, sampleMa, numerator, + LPC_FILTERORDER + 1, + iLBCenc_inst->state_short_len + LPC_FILTERORDER); + WebRtcSpl_MemSetW16(&sampleMa[iLBCenc_inst->state_short_len + LPC_FILTERORDER], 0, iLBCenc_inst->state_short_len - LPC_FILTERORDER); + + WebRtcSpl_FilterARFastQ12( + sampleMa, sampleAr, + syntDenum, LPC_FILTERORDER+1, 2 * iLBCenc_inst->state_short_len); + + for(k=0;kstate_short_len;k++){ + sampleAr[k] += sampleAr[k+iLBCenc_inst->state_short_len]; + } + + /* Find maximum absolute value in the vector */ + maxVal=WebRtcSpl_MaxAbsValueW16(sampleAr, iLBCenc_inst->state_short_len); + + /* Find the best index */ + + if ((((int32_t)maxVal)<=WebRtcIlbcfix_kChooseFrgQuant[i]) { + index=i+1; + } else { + i=63; + } + } + iLBC_encbits->idxForMax=index; + + /* Rescale the vector before quantization */ + scale=WebRtcIlbcfix_kScale[index]; + + if (index<27) { /* scale table is in Q16, fout[] is in Q(-1) and we want the result to be in Q11 */ + shift=4; + } else { /* scale table is in Q21, fout[] is in Q(-1) and we want the result to be in Q11 */ + shift=9; + } + + /* Set up vectors for AbsQuant and rescale it with the scale factor */ + WebRtcSpl_ScaleVectorWithSat(sampleAr, sampleAr, scale, + iLBCenc_inst->state_short_len, (int16_t)(shift-scaleRes)); + + /* Quantize the values in fout[] */ + WebRtcIlbcfix_AbsQuant(iLBCenc_inst, iLBC_encbits, sampleAr, weightDenum); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_search.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_search.h new file mode 100644 index 0000000000..94a9d09266 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_search.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_StateSearch.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_STATE_SEARCH_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_STATE_SEARCH_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * encoding of start state + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_StateSearch( + IlbcEncoder *iLBCenc_inst, + /* (i) Encoder instance */ + iLBC_bits *iLBC_encbits,/* (i/o) Encoded bits (output idxForMax + and idxVec, input state_first) */ + int16_t *residual, /* (i) target residual vector */ + int16_t *syntDenum, /* (i) lpc synthesis filter */ + int16_t *weightDenum /* (i) weighting filter denuminator */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c new file mode 100644 index 0000000000..b795e56ac4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SwapBytes.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Swap bytes (to simplify operations on Little Endian machines) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SwapBytes( + const uint16_t* input, /* (i) the sequence to swap */ + size_t wordLength, /* (i) number or uint16_t to swap */ + uint16_t* output /* (o) the swapped sequence */ + ) { + size_t k; + for (k = wordLength; k > 0; k--) { + *output++ = (*input >> 8)|(*input << 8); + input++; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h new file mode 100644 index 0000000000..95182993ca --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SwapBytes.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SWAP_BYTES_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SWAP_BYTES_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * Swap bytes (to simplify operations on Little Endian machines) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SwapBytes( + const uint16_t* input, /* (i) the sequence to swap */ + size_t wordLength, /* (i) number or uint16_t to swap */ + uint16_t* output /* (o) the swapped sequence */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/empty.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/empty.cc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c new file mode 100644 index 0000000000..4dbc18513a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + iLBC_test.c + +******************************************************************/ + +#include +#include +#include +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +/*---------------------------------------------------------------* + * Main program to test iLBC encoding and decoding + * + * Usage: + * exefile_name.exe + * + * : Input file, speech for encoder (16-bit pcm file) + * : Bit stream output from the encoder + * : Output file, decoded speech (16-bit pcm file) + * : Bit error file, optional (16-bit) + * 1 - Packet received correctly + * 0 - Packet Lost + * + *--------------------------------------------------------------*/ + +#define BLOCKL_MAX 240 +#define ILBCNOOFWORDS_MAX 25 + + +int main(int argc, char* argv[]) +{ + + FILE *ifileid,*efileid,*ofileid, *cfileid; + int16_t data[BLOCKL_MAX]; + uint8_t encoded_data[2 * ILBCNOOFWORDS_MAX]; + int16_t decoded_data[BLOCKL_MAX]; + int len_int, mode; + short pli; + int blockcount = 0; + int packetlosscount = 0; + size_t frameLen, len, len_i16s; + int16_t speechType; + IlbcEncoderInstance *Enc_Inst; + IlbcDecoderInstance *Dec_Inst; + +#ifdef __ILBC_WITH_40BITACC + /* Doublecheck that long long exists */ + if (sizeof(long)>=sizeof(long long)) { + fprintf(stderr, "40-bit simulation is not be supported on this platform\n"); + exit(0); + } +#endif + + /* get arguments and open files */ + + if ((argc!=5) && (argc!=6)) { + fprintf(stderr, + "\n*-----------------------------------------------*\n"); + fprintf(stderr, + " %s <20,30> input encoded decoded (channel)\n\n", + argv[0]); + fprintf(stderr, + " mode : Frame size for the encoding/decoding\n"); + fprintf(stderr, + " 20 - 20 ms\n"); + fprintf(stderr, + " 30 - 30 ms\n"); + fprintf(stderr, + " input : Speech for encoder (16-bit pcm file)\n"); + fprintf(stderr, + " encoded : Encoded bit stream\n"); + fprintf(stderr, + " decoded : Decoded speech (16-bit pcm file)\n"); + fprintf(stderr, + " channel : Packet loss pattern, optional (16-bit)\n"); + fprintf(stderr, + " 1 - Packet received correctly\n"); + fprintf(stderr, + " 0 - Packet Lost\n"); + fprintf(stderr, + "*-----------------------------------------------*\n\n"); + exit(1); + } + mode=atoi(argv[1]); + if (mode != 20 && mode != 30) { + fprintf(stderr,"Wrong mode %s, must be 20, or 30\n", + argv[1]); + exit(2); + } + if ( (ifileid=fopen(argv[2],"rb")) == NULL) { + fprintf(stderr,"Cannot open input file %s\n", argv[2]); + exit(2);} + if ( (efileid=fopen(argv[3],"wb")) == NULL) { + fprintf(stderr, "Cannot open encoded file file %s\n", + argv[3]); exit(1);} + if ( (ofileid=fopen(argv[4],"wb")) == NULL) { + fprintf(stderr, "Cannot open decoded file %s\n", + argv[4]); exit(1);} + if (argc==6) { + if( (cfileid=fopen(argv[5],"rb")) == NULL) { + fprintf(stderr, "Cannot open channel file %s\n", + argv[5]); + exit(1); + } + } else { + cfileid=NULL; + } + + /* print info */ + + fprintf(stderr, "\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* iLBC test program *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr,"\nMode : %2d ms\n", mode); + fprintf(stderr,"Input file : %s\n", argv[2]); + fprintf(stderr,"Encoded file : %s\n", argv[3]); + fprintf(stderr,"Output file : %s\n", argv[4]); + if (argc==6) { + fprintf(stderr,"Channel file : %s\n", argv[5]); + } + fprintf(stderr,"\n"); + + /* Create structs */ + WebRtcIlbcfix_EncoderCreate(&Enc_Inst); + WebRtcIlbcfix_DecoderCreate(&Dec_Inst); + + + /* Initialization */ + + WebRtcIlbcfix_EncoderInit(Enc_Inst, mode); + WebRtcIlbcfix_DecoderInit(Dec_Inst, mode); + frameLen = (size_t)(mode*8); + + /* loop over input blocks */ + + while (fread(data,sizeof(int16_t),frameLen,ifileid) == frameLen) { + + blockcount++; + + /* encoding */ + + fprintf(stderr, "--- Encoding block %i --- ",blockcount); + len_int = WebRtcIlbcfix_Encode(Enc_Inst, data, frameLen, encoded_data); + if (len_int < 0) { + fprintf(stderr, "Error encoding\n"); + exit(0); + } + len = (size_t)len_int; + fprintf(stderr, "\r"); + + /* write byte file */ + + len_i16s = (len + 1) / sizeof(int16_t); + if (fwrite(encoded_data, sizeof(int16_t), len_i16s, efileid) != len_i16s) { + return -1; + } + + /* get channel data if provided */ + if (argc==6) { + if (fread(&pli, sizeof(int16_t), 1, cfileid)) { + if ((pli!=0)&&(pli!=1)) { + fprintf(stderr, "Error in channel file\n"); + exit(0); + } + if (pli==0) { + /* Packet loss -> remove info from frame */ + memset(encoded_data, 0, + sizeof(int16_t)*ILBCNOOFWORDS_MAX); + packetlosscount++; + } + } else { + fprintf(stderr, "Error. Channel file too short\n"); + exit(0); + } + } else { + pli=1; + } + + /* decoding */ + + fprintf(stderr, "--- Decoding block %i --- ",blockcount); + if (pli==1) { + len_int=WebRtcIlbcfix_Decode(Dec_Inst, encoded_data, + len, decoded_data,&speechType); + if (len_int < 0) { + fprintf(stderr, "Error decoding\n"); + exit(0); + } + len = (size_t)len_int; + } else { + len=WebRtcIlbcfix_DecodePlc(Dec_Inst, decoded_data, 1); + } + fprintf(stderr, "\r"); + + /* write output file */ + + if (fwrite(decoded_data, sizeof(int16_t), len, ofileid) != len) { + return -1; + } + } + + /* close files */ + + fclose(ifileid); fclose(efileid); fclose(ofileid); + if (argc==6) { + fclose(cfileid); + } + + /* Free structs */ + WebRtcIlbcfix_EncoderFree(Enc_Inst); + WebRtcIlbcfix_DecoderFree(Dec_Inst); + + + printf("\nDone with simulation\n\n"); + + return(0); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c new file mode 100644 index 0000000000..132f3bdb37 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + +iLBC Speech Coder ANSI-C Source Code + +iLBC_test.c + +******************************************************************/ + +#include +#include +#include +#include +#include +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +//#define JUNK_DATA +#ifdef JUNK_DATA +#define SEED_FILE "randseed.txt" +#endif + + +/*----------------------------------------------------------------* +* Main program to test iLBC encoding and decoding +* +* Usage: +* exefile_name.exe +* +*---------------------------------------------------------------*/ + +int main(int argc, char* argv[]) +{ + FILE *ifileid,*efileid,*ofileid, *chfileid; + short encoded_data[55], data[240], speechType; + int len_int, mode; + short pli; + size_t len, readlen; + int blockcount = 0; + + IlbcEncoderInstance *Enc_Inst; + IlbcDecoderInstance *Dec_Inst; +#ifdef JUNK_DATA + size_t i; + FILE *seedfile; + unsigned int random_seed = (unsigned int) time(NULL);//1196764538 +#endif + + /* Create structs */ + WebRtcIlbcfix_EncoderCreate(&Enc_Inst); + WebRtcIlbcfix_DecoderCreate(&Dec_Inst); + + /* get arguments and open files */ + + if (argc != 6 ) { + fprintf(stderr, "%s mode inputfile bytefile outputfile channelfile\n", + argv[0]); + fprintf(stderr, "Example:\n"); + fprintf(stderr, "%s <30,20> in.pcm byte.dat out.pcm T30.0.dat\n", argv[0]); + exit(1); + } + mode=atoi(argv[1]); + if (mode != 20 && mode != 30) { + fprintf(stderr,"Wrong mode %s, must be 20, or 30\n", argv[1]); + exit(2); + } + if ( (ifileid=fopen(argv[2],"rb")) == NULL) { + fprintf(stderr,"Cannot open input file %s\n", argv[2]); + exit(2);} + if ( (efileid=fopen(argv[3],"wb")) == NULL) { + fprintf(stderr, "Cannot open channelfile file %s\n", + argv[3]); exit(3);} + if( (ofileid=fopen(argv[4],"wb")) == NULL) { + fprintf(stderr, "Cannot open output file %s\n", + argv[4]); exit(3);} + if ( (chfileid=fopen(argv[5],"rb")) == NULL) { + fprintf(stderr,"Cannot open channel file file %s\n", argv[5]); + exit(2); + } + /* print info */ + fprintf(stderr, "\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* iLBCtest *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); +#ifdef SPLIT_10MS + fprintf(stderr,"\n10ms split with raw mode: %2d ms\n", mode); +#else + fprintf(stderr,"\nMode : %2d ms\n", mode); +#endif + fprintf(stderr,"\nInput file : %s\n", argv[2]); + fprintf(stderr,"Coded file : %s\n", argv[3]); + fprintf(stderr,"Output file : %s\n\n", argv[4]); + fprintf(stderr,"Channel file : %s\n\n", argv[5]); + +#ifdef JUNK_DATA + srand(random_seed); + + if ( (seedfile = fopen(SEED_FILE, "a+t") ) == NULL ) { + fprintf(stderr, "Error: Could not open file %s\n", SEED_FILE); + } + else { + fprintf(seedfile, "%u\n", random_seed); + fclose(seedfile); + } +#endif + + /* Initialization */ + WebRtcIlbcfix_EncoderInit(Enc_Inst, mode); + WebRtcIlbcfix_DecoderInit(Dec_Inst, mode); + + /* loop over input blocks */ +#ifdef SPLIT_10MS + readlen = 80; +#else + readlen = (size_t)(mode << 3); +#endif + while(fread(data, sizeof(short), readlen, ifileid) == readlen) { + blockcount++; + + /* encoding */ + fprintf(stderr, "--- Encoding block %i --- ",blockcount); + len_int=WebRtcIlbcfix_Encode(Enc_Inst, data, readlen, encoded_data); + if (len_int < 0) { + fprintf(stderr, "Error encoding\n"); + exit(0); + } + len = (size_t)len_int; + fprintf(stderr, "\r"); + +#ifdef JUNK_DATA + for ( i = 0; i < len; i++) { + encoded_data[i] = (short) (encoded_data[i] + (short) rand()); + } +#endif + /* write byte file */ + if(len != 0){ //len may be 0 in 10ms split case + fwrite(encoded_data,1,len,efileid); + + /* get channel data if provided */ + if (argc==6) { + if (fread(&pli, sizeof(int16_t), 1, chfileid)) { + if ((pli!=0)&&(pli!=1)) { + fprintf(stderr, "Error in channel file\n"); + exit(0); + } + if (pli==0) { + /* Packet loss -> remove info from frame */ + memset(encoded_data, 0, sizeof(int16_t)*25); + } + } else { + fprintf(stderr, "Error. Channel file too short\n"); + exit(0); + } + } else { + pli=1; + } + + /* decoding */ + fprintf(stderr, "--- Decoding block %i --- ",blockcount); + if (pli==1) { + len_int = WebRtcIlbcfix_Decode(Dec_Inst, encoded_data, len, data, + &speechType); + if (len_int < 0) { + fprintf(stderr, "Error decoding\n"); + exit(0); + } + len = (size_t)len_int; + } else { + len=WebRtcIlbcfix_DecodePlc(Dec_Inst, data, 1); + } + fprintf(stderr, "\r"); + + /* write output file */ + fwrite(data,sizeof(short),len,ofileid); + } + } + +#ifdef JUNK_DATA + if ( (seedfile = fopen(SEED_FILE, "a+t") ) == NULL ) { + fprintf(stderr, "Error: Could not open file %s\n", SEED_FILE); + } + else { + fprintf(seedfile, "ok\n\n"); + fclose(seedfile); + } +#endif + + /* free structs */ + WebRtcIlbcfix_EncoderFree(Enc_Inst); + WebRtcIlbcfix_DecoderFree(Dec_Inst); + + /* close files */ + fclose(ifileid); + fclose(efileid); + fclose(ofileid); + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c new file mode 100644 index 0000000000..a62a42edf6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + iLBC_test.c + +******************************************************************/ + +#include +#include +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/nit_encode.h" +#include "modules/audio_coding/codecs/ilbc/encode.h" +#include "modules/audio_coding/codecs/ilbc/init_decode.h" +#include "modules/audio_coding/codecs/ilbc/decode.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +#define ILBCNOOFWORDS_MAX (NO_OF_BYTES_30MS)/2 + +/* Runtime statistics */ +#include +/* #define CLOCKS_PER_SEC 1000 */ + +/*----------------------------------------------------------------* + * Encoder interface function + *---------------------------------------------------------------*/ + +short encode( /* (o) Number of bytes encoded */ + IlbcEncoder *iLBCenc_inst, /* (i/o) Encoder instance */ + int16_t *encoded_data, /* (o) The encoded bytes */ + int16_t *data /* (i) The signal block to encode */ + ){ + + /* do the actual encoding */ + WebRtcIlbcfix_Encode((uint16_t *)encoded_data, data, iLBCenc_inst); + + return (iLBCenc_inst->no_of_bytes); +} + +/*----------------------------------------------------------------* + * Decoder interface function + *---------------------------------------------------------------*/ + +short decode( /* (o) Number of decoded samples */ + IlbcDecoder *iLBCdec_inst, /* (i/o) Decoder instance */ + short *decoded_data, /* (o) Decoded signal block */ + short *encoded_data, /* (i) Encoded bytes */ + short mode /* (i) 0=PL, 1=Normal */ + ){ + + /* check if mode is valid */ + + if (mode<0 || mode>1) { + printf("\nERROR - Wrong mode - 0, 1 allowed\n"); exit(3);} + + /* do actual decoding of block */ + + WebRtcIlbcfix_Decode(decoded_data, (uint16_t *)encoded_data, + iLBCdec_inst, mode); + + return (iLBCdec_inst->blockl); +} + +/*----------------------------------------------------------------* + * Main program to test iLBC encoding and decoding + * + * Usage: + * exefile_name.exe + * + *---------------------------------------------------------------*/ + +#define MAXFRAMES 10000 +#define MAXFILELEN (BLOCKL_MAX*MAXFRAMES) + +int main(int argc, char* argv[]) +{ + + /* Runtime statistics */ + + float starttime1, starttime2; + float runtime1, runtime2; + float outtime; + + FILE *ifileid,*efileid,*ofileid, *chfileid; + short *inputdata, *encodeddata, *decodeddata; + short *channeldata; + int blockcount = 0, noOfBlocks=0, i, noOfLostBlocks=0; + short mode; + IlbcEncoder Enc_Inst; + IlbcDecoder Dec_Inst; + + short frameLen; + short count; +#ifdef SPLIT_10MS + short size; +#endif + + inputdata=(short*) malloc(MAXFILELEN*sizeof(short)); + if (inputdata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + exit(0); + } + encodeddata=(short*) malloc(ILBCNOOFWORDS_MAX*MAXFRAMES*sizeof(short)); + if (encodeddata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + free(inputdata); + exit(0); + } + decodeddata=(short*) malloc(MAXFILELEN*sizeof(short)); + if (decodeddata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + free(inputdata); + free(encodeddata); + exit(0); + } + channeldata=(short*) malloc(MAXFRAMES*sizeof(short)); + if (channeldata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + free(inputdata); + free(encodeddata); + free(decodeddata); + exit(0); + } + + /* get arguments and open files */ + + if (argc != 6 ) { + fprintf(stderr, "%s mode inputfile bytefile outputfile channelfile\n", + argv[0]); + fprintf(stderr, "Example:\n"); + fprintf(stderr, "%s <30,20> in.pcm byte.dat out.pcm T30.0.dat\n", argv[0]); + exit(1); + } + mode=atoi(argv[1]); + if (mode != 20 && mode != 30) { + fprintf(stderr,"Wrong mode %s, must be 20, or 30\n", argv[1]); + exit(2); + } + if ( (ifileid=fopen(argv[2],"rb")) == NULL) { + fprintf(stderr,"Cannot open input file %s\n", argv[2]); + exit(2);} + if ( (efileid=fopen(argv[3],"wb")) == NULL) { + fprintf(stderr, "Cannot open channelfile file %s\n", + argv[3]); exit(3);} + if( (ofileid=fopen(argv[4],"wb")) == NULL) { + fprintf(stderr, "Cannot open output file %s\n", + argv[4]); exit(3);} + if ( (chfileid=fopen(argv[5],"rb")) == NULL) { + fprintf(stderr,"Cannot open channel file file %s\n", argv[5]); + exit(2);} + + + /* print info */ +#ifndef PRINT_MIPS + fprintf(stderr, "\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* iLBCtest *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); +#ifdef SPLIT_10MS + fprintf(stderr,"\n10ms split with raw mode: %2d ms\n", mode); +#else + fprintf(stderr,"\nMode : %2d ms\n", mode); +#endif + fprintf(stderr,"\nInput file : %s\n", argv[2]); + fprintf(stderr,"Coded file : %s\n", argv[3]); + fprintf(stderr,"Output file : %s\n\n", argv[4]); + fprintf(stderr,"Channel file : %s\n\n", argv[5]); +#endif + + /* Initialization */ + + WebRtcIlbcfix_EncoderInit(&Enc_Inst, mode); + WebRtcIlbcfix_DecoderInit(&Dec_Inst, mode, 1); + + /* extract the input file and channel file */ + +#ifdef SPLIT_10MS + frameLen = (mode==20)? 80:160; + fread(Enc_Inst.past_samples, sizeof(short), frameLen, ifileid); + Enc_Inst.section = 0; + + while( fread(&inputdata[noOfBlocks*80], sizeof(short), + 80, ifileid) == 80 ) { + noOfBlocks++; + } + + noOfBlocks += frameLen/80; + frameLen = 80; +#else + frameLen = Enc_Inst.blockl; + + while( fread(&inputdata[noOfBlocks*Enc_Inst.blockl],sizeof(short), + Enc_Inst.blockl,ifileid)==(uint16_t)Enc_Inst.blockl){ + noOfBlocks++; + } +#endif + + + while ((fread(&channeldata[blockcount],sizeof(short), 1,chfileid)==1) + && ( blockcount < noOfBlocks/(Enc_Inst.blockl/frameLen) )) { + blockcount++; + } + + if ( blockcount < noOfBlocks/(Enc_Inst.blockl/frameLen) ) { + fprintf(stderr,"Channel file %s is too short\n", argv[4]); + free(inputdata); + free(encodeddata); + free(decodeddata); + free(channeldata); + exit(0); + } + + count=0; + + /* Runtime statistics */ + + starttime1 = clock()/(float)CLOCKS_PER_SEC; + + /* Encoding loop */ +#ifdef PRINT_MIPS + printf("-1 -1\n"); +#endif + +#ifdef SPLIT_10MS + /* "Enc_Inst.section != 0" is to make sure we run through full + lengths of all vectors for 10ms split mode. + */ + // while( (count < noOfBlocks) || (Enc_Inst.section != 0) ) { + while( count < blockcount * (Enc_Inst.blockl/frameLen) ) { + + encode(&Enc_Inst, &encodeddata[Enc_Inst.no_of_words * + (count/(Enc_Inst.nsub/2))], + &inputdata[frameLen * count] ); +#else + while (count < noOfBlocks) { + encode( &Enc_Inst, &encodeddata[Enc_Inst.no_of_words * count], + &inputdata[frameLen * count] ); +#endif + +#ifdef PRINT_MIPS + printf("-1 -1\n"); +#endif + + count++; + } + + count=0; + + /* Runtime statistics */ + + starttime2=clock()/(float)CLOCKS_PER_SEC; + runtime1 = (float)(starttime2-starttime1); + + /* Decoding loop */ + + while (count < blockcount) { + if (channeldata[count]==1) { + /* Normal decoding */ + decode(&Dec_Inst, &decodeddata[count * Dec_Inst.blockl], + &encodeddata[Dec_Inst.no_of_words * count], 1); + } else if (channeldata[count]==0) { + /* PLC */ + short emptydata[ILBCNOOFWORDS_MAX]; + memset(emptydata, 0, Dec_Inst.no_of_words*sizeof(short)); + decode(&Dec_Inst, &decodeddata[count*Dec_Inst.blockl], + emptydata, 0); + noOfLostBlocks++; + } else { + printf("Error in channel file (values have to be either 1 or 0)\n"); + exit(0); + } +#ifdef PRINT_MIPS + printf("-1 -1\n"); +#endif + + count++; + } + + /* Runtime statistics */ + + runtime2 = (float)(clock()/(float)CLOCKS_PER_SEC-starttime2); + + outtime = (float)((float)blockcount* + (float)mode/1000.0); + +#ifndef PRINT_MIPS + printf("\nLength of speech file: %.1f s\n", outtime); + printf("Lost frames : %.1f%%\n\n", 100*(float)noOfLostBlocks/(float)blockcount); + + printf("Time to run iLBC_encode+iLBC_decode:"); + printf(" %.1f s (%.1f%% of realtime)\n", runtime1+runtime2, + (100*(runtime1+runtime2)/outtime)); + + printf("Time in iLBC_encode :"); + printf(" %.1f s (%.1f%% of total runtime)\n", + runtime1, 100.0*runtime1/(runtime1+runtime2)); + + printf("Time in iLBC_decode :"); + printf(" %.1f s (%.1f%% of total runtime)\n\n", + runtime2, 100.0*runtime2/(runtime1+runtime2)); +#endif + + /* Write data to files */ + for (i=0; i/dev/null && set -o igncr; # force bash to ignore \r character + +# +# This script can be used to verify the bit exactness of iLBC fixed-point version 1.0.6 +# + +INP=../../../../../../resources/audio_coding +EXEP=../../../../../../../out/Release +OUTP=./GeneratedFiles +mkdir ./GeneratedFiles + +$EXEP/iLBCtest 20 $INP/F00.INP $OUTP/F00.BIT20 $OUTP/F00.OUT20 $INP/clean.chn +$EXEP/iLBCtest 20 $INP/F01.INP $OUTP/F01.BIT20 $OUTP/F01.OUT20 $INP/clean.chn +$EXEP/iLBCtest 20 $INP/F02.INP $OUTP/F02.BIT20 $OUTP/F02.OUT20 $INP/clean.chn +$EXEP/iLBCtest 20 $INP/F03.INP $OUTP/F03.BIT20 $OUTP/F03.OUT20 $INP/clean.chn +$EXEP/iLBCtest 20 $INP/F04.INP $OUTP/F04.BIT20 $OUTP/F04.OUT20 $INP/clean.chn +$EXEP/iLBCtest 20 $INP/F05.INP $OUTP/F05.BIT20 $OUTP/F05.OUT20 $INP/clean.chn +$EXEP/iLBCtest 20 $INP/F06.INP $OUTP/F06.BIT20 $OUTP/F06.OUT20 $INP/clean.chn + +$EXEP/iLBCtest 30 $INP/F00.INP $OUTP/F00.BIT30 $OUTP/F00.OUT30 $INP/clean.chn +$EXEP/iLBCtest 30 $INP/F01.INP $OUTP/F01.BIT30 $OUTP/F01.OUT30 $INP/clean.chn +$EXEP/iLBCtest 30 $INP/F02.INP $OUTP/F02.BIT30 $OUTP/F02.OUT30 $INP/clean.chn +$EXEP/iLBCtest 30 $INP/F03.INP $OUTP/F03.BIT30 $OUTP/F03.OUT30 $INP/clean.chn +$EXEP/iLBCtest 30 $INP/F04.INP $OUTP/F04.BIT30 $OUTP/F04.OUT30 $INP/clean.chn +$EXEP/iLBCtest 30 $INP/F05.INP $OUTP/F05.BIT30 $OUTP/F05.OUT30 $INP/clean.chn +$EXEP/iLBCtest 30 $INP/F06.INP $OUTP/F06.BIT30 $OUTP/F06.OUT30 $INP/clean.chn + +$EXEP/iLBCtest 20 $INP/F00.INP $OUTP/F00.BIT20 $OUTP/F00_tlm10.OUT20 $INP/tlm10.chn +$EXEP/iLBCtest 20 $INP/F01.INP $OUTP/F01.BIT20 $OUTP/F01_tlm10.OUT20 $INP/tlm10.chn +$EXEP/iLBCtest 20 $INP/F02.INP $OUTP/F02.BIT20 $OUTP/F02_tlm10.OUT20 $INP/tlm10.chn +$EXEP/iLBCtest 30 $INP/F00.INP $OUTP/F00.BIT30 $OUTP/F00_tlm10.OUT30 $INP/tlm10.chn +$EXEP/iLBCtest 30 $INP/F01.INP $OUTP/F01.BIT30 $OUTP/F01_tlm10.OUT30 $INP/tlm10.chn +$EXEP/iLBCtest 30 $INP/F02.INP $OUTP/F02.BIT30 $OUTP/F02_tlm10.OUT30 $INP/tlm10.chn + + +diff $OUTP/F00.BIT20 $INP/F00.BIT20 +diff $OUTP/F01.BIT20 $INP/F01.BIT20 +diff $OUTP/F02.BIT20 $INP/F02.BIT20 +diff $OUTP/F03.BIT20 $INP/F03.BIT20 +diff $OUTP/F04.BIT20 $INP/F04.BIT20 +diff $OUTP/F05.BIT20 $INP/F05.BIT20 +diff $OUTP/F06.BIT20 $INP/F06.BIT20 +diff $OUTP/F00.OUT20 $INP/F00.OUT20 +diff $OUTP/F01.OUT20 $INP/F01.OUT20 +diff $OUTP/F02.OUT20 $INP/F02.OUT20 +diff $OUTP/F03.OUT20 $INP/F03.OUT20 +diff $OUTP/F04.OUT20 $INP/F04.OUT20 +diff $OUTP/F05.OUT20 $INP/F05.OUT20 +diff $OUTP/F06.OUT20 $INP/F06.OUT20 + +diff $OUTP/F00.BIT30 $INP/F00.BIT30 +diff $OUTP/F01.BIT30 $INP/F01.BIT30 +diff $OUTP/F02.BIT30 $INP/F02.BIT30 +diff $OUTP/F03.BIT30 $INP/F03.BIT30 +diff $OUTP/F04.BIT30 $INP/F04.BIT30 +diff $OUTP/F05.BIT30 $INP/F05.BIT30 +diff $OUTP/F06.BIT30 $INP/F06.BIT30 +diff $OUTP/F00.OUT30 $INP/F00.OUT30 +diff $OUTP/F01.OUT30 $INP/F01.OUT30 +diff $OUTP/F02.OUT30 $INP/F02.OUT30 +diff $OUTP/F03.OUT30 $INP/F03.OUT30 +diff $OUTP/F04.OUT30 $INP/F04.OUT30 +diff $OUTP/F05.OUT30 $INP/F05.OUT30 +diff $OUTP/F06.OUT30 $INP/F06.OUT30 + +diff $OUTP/F00_tlm10.OUT20 $INP/F00_tlm10.OUT20 +diff $OUTP/F01_tlm10.OUT20 $INP/F01_tlm10.OUT20 +diff $OUTP/F02_tlm10.OUT20 $INP/F02_tlm10.OUT20 +diff $OUTP/F00_tlm10.OUT30 $INP/F00_tlm10.OUT30 +diff $OUTP/F01_tlm10.OUT30 $INP/F01_tlm10.OUT30 +diff $OUTP/F02_tlm10.OUT30 $INP/F02_tlm10.OUT30 + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.c new file mode 100644 index 0000000000..fae06f6339 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.c @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_UnpackBits.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * unpacking of bits from bitstream, i.e., vector of bytes + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_UnpackBits( /* (o) "Empty" frame indicator */ + const uint16_t *bitstream, /* (i) The packatized bitstream */ + iLBC_bits *enc_bits, /* (o) Paramerers from bitstream */ + int16_t mode /* (i) Codec mode (20 or 30) */ + ) { + const uint16_t *bitstreamPtr; + int i, k; + int16_t *tmpPtr; + + bitstreamPtr=bitstream; + + /* First int16_t */ + enc_bits->lsf[0] = (*bitstreamPtr)>>10; /* Bit 0..5 */ + enc_bits->lsf[1] = ((*bitstreamPtr)>>3)&0x7F; /* Bit 6..12 */ + enc_bits->lsf[2] = ((*bitstreamPtr)&0x7)<<4; /* Bit 13..15 */ + bitstreamPtr++; + /* Second int16_t */ + enc_bits->lsf[2] |= ((*bitstreamPtr)>>12)&0xF; /* Bit 0..3 */ + + if (mode==20) { + enc_bits->startIdx = ((*bitstreamPtr)>>10)&0x3; /* Bit 4..5 */ + enc_bits->state_first = ((*bitstreamPtr)>>9)&0x1; /* Bit 6 */ + enc_bits->idxForMax = ((*bitstreamPtr)>>3)&0x3F; /* Bit 7..12 */ + enc_bits->cb_index[0] = ((*bitstreamPtr)&0x7)<<4; /* Bit 13..15 */ + bitstreamPtr++; + /* Third int16_t */ + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>12)&0xE; /* Bit 0..2 */ + enc_bits->gain_index[0] = ((*bitstreamPtr)>>8)&0x18; /* Bit 3..4 */ + enc_bits->gain_index[1] = ((*bitstreamPtr)>>7)&0x8; /* Bit 5 */ + enc_bits->cb_index[3] = ((*bitstreamPtr)>>2)&0xFE; /* Bit 6..12 */ + enc_bits->gain_index[3] = ((*bitstreamPtr)<<2)&0x10; /* Bit 13 */ + enc_bits->gain_index[4] = ((*bitstreamPtr)<<2)&0x8; /* Bit 14 */ + enc_bits->gain_index[6] = ((*bitstreamPtr)<<4)&0x10; /* Bit 15 */ + } else { /* mode==30 */ + enc_bits->lsf[3] = ((*bitstreamPtr)>>6)&0x3F; /* Bit 4..9 */ + enc_bits->lsf[4] = ((*bitstreamPtr)<<1)&0x7E; /* Bit 10..15 */ + bitstreamPtr++; + /* Third int16_t */ + enc_bits->lsf[4] |= ((*bitstreamPtr)>>15)&0x1; /* Bit 0 */ + enc_bits->lsf[5] = ((*bitstreamPtr)>>8)&0x7F; /* Bit 1..7 */ + enc_bits->startIdx = ((*bitstreamPtr)>>5)&0x7; /* Bit 8..10 */ + enc_bits->state_first = ((*bitstreamPtr)>>4)&0x1; /* Bit 11 */ + enc_bits->idxForMax = ((*bitstreamPtr)<<2)&0x3C; /* Bit 12..15 */ + bitstreamPtr++; + /* 4:th int16_t */ + enc_bits->idxForMax |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->cb_index[0] = ((*bitstreamPtr)>>7)&0x78; /* Bit 2..5 */ + enc_bits->gain_index[0] = ((*bitstreamPtr)>>5)&0x10; /* Bit 6 */ + enc_bits->gain_index[1] = ((*bitstreamPtr)>>5)&0x8; /* Bit 7 */ + enc_bits->cb_index[3] = ((*bitstreamPtr))&0xFC; /* Bit 8..13 */ + enc_bits->gain_index[3] = ((*bitstreamPtr)<<3)&0x10; /* Bit 14 */ + enc_bits->gain_index[4] = ((*bitstreamPtr)<<3)&0x8; /* Bit 15 */ + } + /* Class 2 bits of ULP */ + /* 4:th to 6:th int16_t for 20 ms case + 5:th to 7:th int16_t for 30 ms case */ + bitstreamPtr++; + tmpPtr=enc_bits->idxVec; + for (k=0; k<3; k++) { + for (i=15; i>=0; i--) { + (*tmpPtr) = (((*bitstreamPtr)>>i)<<2)&0x4; + /* Bit 15-i */ + tmpPtr++; + } + bitstreamPtr++; + } + + if (mode==20) { + /* 7:th int16_t */ + for (i=15; i>6; i--) { + (*tmpPtr) = (((*bitstreamPtr)>>i)<<2)&0x4; + /* Bit 15-i */ + tmpPtr++; + } + enc_bits->gain_index[1] |= ((*bitstreamPtr)>>4)&0x4; /* Bit 9 */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>2)&0xC; /* Bit 10..11 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)>>1)&0x4; /* Bit 12 */ + enc_bits->gain_index[6] |= ((*bitstreamPtr)<<1)&0x8; /* Bit 13 */ + enc_bits->gain_index[7] = ((*bitstreamPtr)<<2)&0xC; /* Bit 14..15 */ + + } else { /* mode==30 */ + /* 8:th int16_t */ + for (i=15; i>5; i--) { + (*tmpPtr) = (((*bitstreamPtr)>>i)<<2)&0x4; + /* Bit 15-i */ + tmpPtr++; + } + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>3)&0x6; /* Bit 10..11 */ + enc_bits->gain_index[0] |= ((*bitstreamPtr))&0x8; /* Bit 12 */ + enc_bits->gain_index[1] |= ((*bitstreamPtr))&0x4; /* Bit 13 */ + enc_bits->cb_index[3] |= ((*bitstreamPtr))&0x2; /* Bit 14 */ + enc_bits->cb_index[6] = ((*bitstreamPtr)<<7)&0x80; /* Bit 15 */ + bitstreamPtr++; + /* 9:th int16_t */ + enc_bits->cb_index[6] |= ((*bitstreamPtr)>>9)&0x7E; /* Bit 0..5 */ + enc_bits->cb_index[9] = ((*bitstreamPtr)>>2)&0xFE; /* Bit 6..12 */ + enc_bits->cb_index[12] = ((*bitstreamPtr)<<5)&0xE0; /* Bit 13..15 */ + bitstreamPtr++; + /* 10:th int16_t */ + enc_bits->cb_index[12] |= ((*bitstreamPtr)>>11)&0x1E;/* Bit 0..3 */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>8)&0xC; /* Bit 4..5 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)>>7)&0x6; /* Bit 6..7 */ + enc_bits->gain_index[6] = ((*bitstreamPtr)>>3)&0x18; /* Bit 8..9 */ + enc_bits->gain_index[7] = ((*bitstreamPtr)>>2)&0xC; /* Bit 10..11 */ + enc_bits->gain_index[9] = ((*bitstreamPtr)<<1)&0x10; /* Bit 12 */ + enc_bits->gain_index[10] = ((*bitstreamPtr)<<1)&0x8; /* Bit 13 */ + enc_bits->gain_index[12] = ((*bitstreamPtr)<<3)&0x10; /* Bit 14 */ + enc_bits->gain_index[13] = ((*bitstreamPtr)<<3)&0x8; /* Bit 15 */ + } + bitstreamPtr++; + /* Class 3 bits of ULP */ + /* 8:th to 14:th int16_t for 20 ms case + 11:th to 17:th int16_t for 30 ms case */ + tmpPtr=enc_bits->idxVec; + for (k=0; k<7; k++) { + for (i=14; i>=0; i-=2) { + (*tmpPtr) |= ((*bitstreamPtr)>>i)&0x3; /* Bit 15-i..14-i*/ + tmpPtr++; + } + bitstreamPtr++; + } + + if (mode==20) { + /* 15:th int16_t */ + enc_bits->idxVec[56] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>13)&0x1; /* Bit 2 */ + enc_bits->cb_index[1] = ((*bitstreamPtr)>>6)&0x7F; /* Bit 3..9 */ + enc_bits->cb_index[2] = ((*bitstreamPtr)<<1)&0x7E; /* Bit 10..15 */ + bitstreamPtr++; + /* 16:th int16_t */ + enc_bits->cb_index[2] |= ((*bitstreamPtr)>>15)&0x1; /* Bit 0 */ + enc_bits->gain_index[0] |= ((*bitstreamPtr)>>12)&0x7; /* Bit 1..3 */ + enc_bits->gain_index[1] |= ((*bitstreamPtr)>>10)&0x3; /* Bit 4..5 */ + enc_bits->gain_index[2] = ((*bitstreamPtr)>>7)&0x7; /* Bit 6..8 */ + enc_bits->cb_index[3] |= ((*bitstreamPtr)>>6)&0x1; /* Bit 9 */ + enc_bits->cb_index[4] = ((*bitstreamPtr)<<1)&0x7E; /* Bit 10..15 */ + bitstreamPtr++; + /* 17:th int16_t */ + enc_bits->cb_index[4] |= ((*bitstreamPtr)>>15)&0x1; /* Bit 0 */ + enc_bits->cb_index[5] = ((*bitstreamPtr)>>8)&0x7F; /* Bit 1..7 */ + enc_bits->cb_index[6] = ((*bitstreamPtr))&0xFF; /* Bit 8..15 */ + bitstreamPtr++; + /* 18:th int16_t */ + enc_bits->cb_index[7] = (*bitstreamPtr)>>8; /* Bit 0..7 */ + enc_bits->cb_index[8] = (*bitstreamPtr)&0xFF; /* Bit 8..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)>>12)&0x3; /* Bit 2..3 */ + enc_bits->gain_index[5] = ((*bitstreamPtr)>>9)&0x7; /* Bit 4..6 */ + enc_bits->gain_index[6] |= ((*bitstreamPtr)>>6)&0x7; /* Bit 7..9 */ + enc_bits->gain_index[7] |= ((*bitstreamPtr)>>4)&0x3; /* Bit 10..11 */ + enc_bits->gain_index[8] = ((*bitstreamPtr)>>1)&0x7; /* Bit 12..14 */ + } else { /* mode==30 */ + /* 18:th int16_t */ + enc_bits->idxVec[56] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->idxVec[57] |= ((*bitstreamPtr)>>12)&0x3; /* Bit 2..3 */ + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>11)&1; /* Bit 4 */ + enc_bits->cb_index[1] = ((*bitstreamPtr)>>4)&0x7F; /* Bit 5..11 */ + enc_bits->cb_index[2] = ((*bitstreamPtr)<<3)&0x78; /* Bit 12..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + enc_bits->cb_index[2] |= ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->gain_index[0] |= ((*bitstreamPtr)>>10)&0x7; /* Bit 3..5 */ + enc_bits->gain_index[1] |= ((*bitstreamPtr)>>8)&0x3; /* Bit 6..7 */ + enc_bits->gain_index[2] = ((*bitstreamPtr)>>5)&0x7; /* Bit 8..10 */ + enc_bits->cb_index[3] |= ((*bitstreamPtr)>>4)&0x1; /* Bit 11 */ + enc_bits->cb_index[4] = ((*bitstreamPtr)<<3)&0x78; /* Bit 12..15 */ + bitstreamPtr++; + /* 20:th int16_t */ + enc_bits->cb_index[4] |= ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->cb_index[5] = ((*bitstreamPtr)>>6)&0x7F; /* Bit 3..9 */ + enc_bits->cb_index[6] |= ((*bitstreamPtr)>>5)&0x1; /* Bit 10 */ + enc_bits->cb_index[7] = ((*bitstreamPtr)<<3)&0xF8; /* Bit 11..15 */ + bitstreamPtr++; + /* 21:st int16_t */ + enc_bits->cb_index[7] |= ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->cb_index[8] = ((*bitstreamPtr)>>5)&0xFF; /* Bit 3..10 */ + enc_bits->cb_index[9] |= ((*bitstreamPtr)>>4)&0x1; /* Bit 11 */ + enc_bits->cb_index[10] = ((*bitstreamPtr)<<4)&0xF0; /* Bit 12..15 */ + bitstreamPtr++; + /* 22:nd int16_t */ + enc_bits->cb_index[10] |= ((*bitstreamPtr)>>12)&0xF; /* Bit 0..3 */ + enc_bits->cb_index[11] = ((*bitstreamPtr)>>4)&0xFF; /* Bit 4..11 */ + enc_bits->cb_index[12] |= ((*bitstreamPtr)>>3)&0x1; /* Bit 12 */ + enc_bits->cb_index[13] = ((*bitstreamPtr)<<5)&0xE0; /* Bit 13..15 */ + bitstreamPtr++; + /* 23:rd int16_t */ + enc_bits->cb_index[13] |= ((*bitstreamPtr)>>11)&0x1F;/* Bit 0..4 */ + enc_bits->cb_index[14] = ((*bitstreamPtr)>>3)&0xFF; /* Bit 5..12 */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>1)&0x3; /* Bit 13..14 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)&0x1); /* Bit 15 */ + bitstreamPtr++; + /* 24:rd int16_t */ + enc_bits->gain_index[5] = ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->gain_index[6] |= ((*bitstreamPtr)>>10)&0x7; /* Bit 3..5 */ + enc_bits->gain_index[7] |= ((*bitstreamPtr)>>8)&0x3; /* Bit 6..7 */ + enc_bits->gain_index[8] = ((*bitstreamPtr)>>5)&0x7; /* Bit 8..10 */ + enc_bits->gain_index[9] |= ((*bitstreamPtr)>>1)&0xF; /* Bit 11..14 */ + enc_bits->gain_index[10] |= ((*bitstreamPtr)<<2)&0x4; /* Bit 15 */ + bitstreamPtr++; + /* 25:rd int16_t */ + enc_bits->gain_index[10] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->gain_index[11] = ((*bitstreamPtr)>>11)&0x7; /* Bit 2..4 */ + enc_bits->gain_index[12] |= ((*bitstreamPtr)>>7)&0xF; /* Bit 5..8 */ + enc_bits->gain_index[13] |= ((*bitstreamPtr)>>4)&0x7; /* Bit 9..11 */ + enc_bits->gain_index[14] = ((*bitstreamPtr)>>1)&0x7; /* Bit 12..14 */ + } + /* Last bit should be zero, otherwise it's an "empty" frame */ + if (((*bitstreamPtr)&0x1) == 1) { + return(1); + } else { + return(0); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h new file mode 100644 index 0000000000..01cf398004 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_UnpackBits.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_UNPACK_BITS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_UNPACK_BITS_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * unpacking of bits from bitstream, i.e., vector of bytes + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_UnpackBits( /* (o) "Empty" frame indicator */ + const uint16_t *bitstream, /* (i) The packatized bitstream */ + iLBC_bits *enc_bits, /* (o) Paramerers from bitstream */ + int16_t mode /* (i) Codec mode (20 or 30) */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.c new file mode 100644 index 0000000000..ee3b26e039 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq3.c + +******************************************************************/ + +#include "vq3.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq3( + int16_t *Xq, /* quantized vector (Q13) */ + int16_t *index, + int16_t *CB, /* codebook in Q13 */ + int16_t *X, /* vector to quantize (Q13) */ + int16_t n_cb + ){ + int16_t i, j; + int16_t pos, minindex=0; + int16_t tmp; + int32_t dist, mindist; + + pos = 0; + mindist = WEBRTC_SPL_WORD32_MAX; /* start value */ + + /* Find the codebook with the lowest square distance */ + for (j = 0; j < n_cb; j++) { + tmp = X[0] - CB[pos]; + dist = tmp * tmp; + for (i = 1; i < 3; i++) { + tmp = X[i] - CB[pos + i]; + dist += tmp * tmp; + } + + if (dist < mindist) { + mindist = dist; + minindex = j; + } + pos += 3; + } + + /* Store the quantized codebook and the index */ + for (i = 0; i < 3; i++) { + Xq[i] = CB[minindex*3 + i]; + } + *index = minindex; + +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.h new file mode 100644 index 0000000000..6d3dc3ac62 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq3.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ3_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ3_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/*----------------------------------------------------------------* + * Vector quantization of order 3 (based on MSE) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq3( + int16_t *Xq, /* (o) the quantized vector (Q13) */ + int16_t *index, /* (o) the quantization index */ + int16_t *CB, /* (i) the vector quantization codebook (Q13) */ + int16_t *X, /* (i) the vector to quantize (Q13) */ + int16_t n_cb /* (i) the number of vectors in the codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.c new file mode 100644 index 0000000000..bc20952132 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq4.c + +******************************************************************/ + +#include "vq4.h" +#include "constants.h" + +/*----------------------------------------------------------------* + * vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq4( + int16_t *Xq, /* quantized vector (Q13) */ + int16_t *index, + int16_t *CB, /* codebook in Q13 */ + int16_t *X, /* vector to quantize (Q13) */ + int16_t n_cb + ){ + int16_t i, j; + int16_t pos, minindex=0; + int16_t tmp; + int32_t dist, mindist; + + pos = 0; + mindist = WEBRTC_SPL_WORD32_MAX; /* start value */ + + /* Find the codebook with the lowest square distance */ + for (j = 0; j < n_cb; j++) { + tmp = X[0] - CB[pos]; + dist = tmp * tmp; + for (i = 1; i < 4; i++) { + tmp = X[i] - CB[pos + i]; + dist += tmp * tmp; + } + + if (dist < mindist) { + mindist = dist; + minindex = j; + } + pos += 4; + } + + /* Store the quantized codebook and the index */ + for (i = 0; i < 4; i++) { + Xq[i] = CB[minindex*4 + i]; + } + *index = minindex; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.h new file mode 100644 index 0000000000..c7f5271fe5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq4.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ4_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ4_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/*----------------------------------------------------------------* + * Vector quantization of order 4 (based on MSE) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq4( + int16_t *Xq, /* (o) the quantized vector (Q13) */ + int16_t *index, /* (o) the quantization index */ + int16_t *CB, /* (i) the vector quantization codebook (Q13) */ + int16_t *X, /* (i) the vector to quantize (Q13) */ + int16_t n_cb /* (i) the number of vectors in the codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.c new file mode 100644 index 0000000000..dc12a5a7c4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Window32W32.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * window multiplication + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Window32W32( + int32_t *z, /* Output */ + int32_t *x, /* Input (same domain as Output)*/ + const int32_t *y, /* Q31 Window */ + size_t N /* length to process */ + ) { + size_t i; + int16_t x_low, x_hi, y_low, y_hi; + int16_t left_shifts; + int32_t temp; + + left_shifts = (int16_t)WebRtcSpl_NormW32(x[0]); + WebRtcSpl_VectorBitShiftW32(x, N, x, (int16_t)(-left_shifts)); + + + /* The double precision numbers use a special representation: + * w32 = hi<<16 + lo<<1 + */ + for (i = 0; i < N; i++) { + /* Extract higher bytes */ + x_hi = (int16_t)(x[i] >> 16); + y_hi = (int16_t)(y[i] >> 16); + + /* Extract lower bytes, defined as (w32 - hi<<16)>>1 */ + x_low = (int16_t)((x[i] - (x_hi << 16)) >> 1); + + y_low = (int16_t)((y[i] - (y_hi << 16)) >> 1); + + /* Calculate z by a 32 bit multiplication using both low and high from x and y */ + temp = ((x_hi * y_hi) << 1) + ((x_hi * y_low) >> 14); + + z[i] = temp + ((x_low * y_hi) >> 14); + } + + WebRtcSpl_VectorBitShiftW32(z, N, z, left_shifts); + + return; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.h new file mode 100644 index 0000000000..f53592e87b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Window32W32.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_WINDOW32_W32_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_WINDOW32_W32_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * window multiplication + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Window32W32( + int32_t *z, /* Output */ + int32_t *x, /* Input (same domain as Output)*/ + const int32_t *y, /* Q31 Window */ + size_t N /* length to process */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c new file mode 100644 index 0000000000..0d898c54a4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_XcorrCoef.c + +******************************************************************/ + +#include "defines.h" + +/*----------------------------------------------------------------* + * cross correlation which finds the optimal lag for the + * crossCorr*crossCorr/(energy) criteria + *---------------------------------------------------------------*/ + +size_t WebRtcIlbcfix_XcorrCoef( + int16_t *target, /* (i) first array */ + int16_t *regressor, /* (i) second array */ + size_t subl, /* (i) dimension arrays */ + size_t searchLen, /* (i) the search lenght */ + size_t offset, /* (i) samples offset between arrays */ + int16_t step /* (i) +1 or -1 */ + ){ + size_t k; + size_t maxlag; + int16_t pos; + int16_t max; + int16_t crossCorrScale, Energyscale; + int16_t crossCorrSqMod, crossCorrSqMod_Max; + int32_t crossCorr, Energy; + int16_t crossCorrmod, EnergyMod, EnergyMod_Max; + int16_t *tp, *rp; + int16_t *rp_beg, *rp_end; + int16_t totscale, totscale_max; + int16_t scalediff; + int32_t newCrit, maxCrit; + int shifts; + + /* Initializations, to make sure that the first one is selected */ + crossCorrSqMod_Max=0; + EnergyMod_Max=WEBRTC_SPL_WORD16_MAX; + totscale_max=-500; + maxlag=0; + pos=0; + + /* Find scale value and start position */ + if (step==1) { + max=WebRtcSpl_MaxAbsValueW16(regressor, subl + searchLen - 1); + rp_beg = regressor; + rp_end = regressor + subl; + } else { /* step==-1 */ + max = WebRtcSpl_MaxAbsValueW16(regressor - searchLen, subl + searchLen - 1); + rp_beg = regressor - 1; + rp_end = regressor + subl - 1; + } + + /* Introduce a scale factor on the Energy in int32_t in + order to make sure that the calculation does not + overflow */ + + if (max>5000) { + shifts=2; + } else { + shifts=0; + } + + /* Calculate the first energy, then do a +/- to get the other energies */ + Energy=WebRtcSpl_DotProductWithScale(regressor, regressor, subl, shifts); + + for (k=0;k0)&&(crossCorr>0)) { + + /* Put cross correlation and energy on 16 bit word */ + crossCorrScale=(int16_t)WebRtcSpl_NormW32(crossCorr)-16; + crossCorrmod=(int16_t)WEBRTC_SPL_SHIFT_W32(crossCorr, crossCorrScale); + Energyscale=(int16_t)WebRtcSpl_NormW32(Energy)-16; + EnergyMod=(int16_t)WEBRTC_SPL_SHIFT_W32(Energy, Energyscale); + + /* Square cross correlation and store upper int16_t */ + crossCorrSqMod = (int16_t)((crossCorrmod * crossCorrmod) >> 16); + + /* Calculate the total number of (dynamic) right shifts that have + been performed on (crossCorr*crossCorr)/energy + */ + totscale=Energyscale-(crossCorrScale<<1); + + /* Calculate the shift difference in order to be able to compare the two + (crossCorr*crossCorr)/energy in the same domain + */ + scalediff=totscale-totscale_max; + scalediff=WEBRTC_SPL_MIN(scalediff,31); + scalediff=WEBRTC_SPL_MAX(scalediff,-31); + + /* Compute the cross multiplication between the old best criteria + and the new one to be able to compare them without using a + division */ + + if (scalediff<0) { + newCrit = ((int32_t)crossCorrSqMod*EnergyMod_Max)>>(-scalediff); + maxCrit = ((int32_t)crossCorrSqMod_Max*EnergyMod); + } else { + newCrit = ((int32_t)crossCorrSqMod*EnergyMod_Max); + maxCrit = ((int32_t)crossCorrSqMod_Max*EnergyMod)>>scalediff; + } + + /* Store the new lag value if the new criteria is larger + than previous largest criteria */ + + if (newCrit > maxCrit) { + crossCorrSqMod_Max = crossCorrSqMod; + EnergyMod_Max = EnergyMod; + totscale_max = totscale; + maxlag = k; + } + } + pos+=step; + + /* Do a +/- to get the next energy */ + Energy += step * ((*rp_end * *rp_end - *rp_beg * *rp_beg) >> shifts); + rp_beg+=step; + rp_end+=step; + } + + return(maxlag+offset); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h new file mode 100644 index 0000000000..e2d511a6f9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_XcorrCoef.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_XCORR_COEF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_XCORR_COEF_H_ + +#include "defines.h" + +/*----------------------------------------------------------------* + * cross correlation which finds the optimal lag for the + * crossCorr*crossCorr/(energy) criteria + *---------------------------------------------------------------*/ + +size_t WebRtcIlbcfix_XcorrCoef( + int16_t *target, /* (i) first array */ + int16_t *regressor, /* (i) second array */ + size_t subl, /* (i) dimension arrays */ + size_t searchLen, /* (i) the search lenght */ + size_t offset, /* (i) samples offset between arrays */ + int16_t step /* (i) +1 or -1 */ + ); + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t.h new file mode 100644 index 0000000000..eda1cfa545 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_DECODER_ISAC_T_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_DECODER_ISAC_T_H_ + +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "api/optional.h" +#include "modules/audio_coding/codecs/isac/locked_bandwidth_info.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/scoped_ref_ptr.h" + +namespace webrtc { + +template +class AudioDecoderIsacT final : public AudioDecoder { + public: + explicit AudioDecoderIsacT(int sample_rate_hz); + AudioDecoderIsacT(int sample_rate_hz, + const rtc::scoped_refptr& bwinfo); + ~AudioDecoderIsacT() override; + + bool HasDecodePlc() const override; + size_t DecodePlc(size_t num_frames, int16_t* decoded) override; + void Reset() override; + int IncomingPacket(const uint8_t* payload, + size_t payload_len, + uint16_t rtp_sequence_number, + uint32_t rtp_timestamp, + uint32_t arrival_timestamp) override; + int ErrorCode() override; + int SampleRateHz() const override; + size_t Channels() const override; + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + typename T::instance_type* isac_state_; + int sample_rate_hz_; + rtc::scoped_refptr bwinfo_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderIsacT); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_DECODER_ISAC_T_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t_impl.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t_impl.h new file mode 100644 index 0000000000..90f3e8d46a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_decoder_isac_t_impl.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_DECODER_ISAC_T_IMPL_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_DECODER_ISAC_T_IMPL_H_ + +#include "modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +template +AudioDecoderIsacT::AudioDecoderIsacT(int sample_rate_hz) + : AudioDecoderIsacT(sample_rate_hz, nullptr) {} + +template +AudioDecoderIsacT::AudioDecoderIsacT( + int sample_rate_hz, + const rtc::scoped_refptr& bwinfo) + : sample_rate_hz_(sample_rate_hz), bwinfo_(bwinfo) { + RTC_CHECK(sample_rate_hz == 16000 || sample_rate_hz == 32000) + << "Unsupported sample rate " << sample_rate_hz; + RTC_CHECK_EQ(0, T::Create(&isac_state_)); + T::DecoderInit(isac_state_); + if (bwinfo_) { + IsacBandwidthInfo bi; + T::GetBandwidthInfo(isac_state_, &bi); + bwinfo_->Set(bi); + } + RTC_CHECK_EQ(0, T::SetDecSampRate(isac_state_, sample_rate_hz_)); +} + +template +AudioDecoderIsacT::~AudioDecoderIsacT() { + RTC_CHECK_EQ(0, T::Free(isac_state_)); +} + +template +int AudioDecoderIsacT::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_CHECK_EQ(sample_rate_hz_, sample_rate_hz); + int16_t temp_type = 1; // Default is speech. + int ret = + T::DecodeInternal(isac_state_, encoded, encoded_len, decoded, &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +template +bool AudioDecoderIsacT::HasDecodePlc() const { + return false; +} + +template +size_t AudioDecoderIsacT::DecodePlc(size_t num_frames, int16_t* decoded) { + return T::DecodePlc(isac_state_, decoded, num_frames); +} + +template +void AudioDecoderIsacT::Reset() { + T::DecoderInit(isac_state_); +} + +template +int AudioDecoderIsacT::IncomingPacket(const uint8_t* payload, + size_t payload_len, + uint16_t rtp_sequence_number, + uint32_t rtp_timestamp, + uint32_t arrival_timestamp) { + int ret = T::UpdateBwEstimate(isac_state_, payload, payload_len, + rtp_sequence_number, rtp_timestamp, + arrival_timestamp); + if (bwinfo_) { + IsacBandwidthInfo bwinfo; + T::GetBandwidthInfo(isac_state_, &bwinfo); + bwinfo_->Set(bwinfo); + } + return ret; +} + +template +int AudioDecoderIsacT::ErrorCode() { + return T::GetErrorCode(isac_state_); +} + +template +int AudioDecoderIsacT::SampleRateHz() const { + return sample_rate_hz_; +} + +template +size_t AudioDecoderIsacT::Channels() const { + return 1; +} + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_DECODER_ISAC_T_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h new file mode 100644 index 0000000000..541b90c9f5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_ENCODER_ISAC_T_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_ENCODER_ISAC_T_H_ + +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "modules/audio_coding/codecs/isac/locked_bandwidth_info.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/scoped_ref_ptr.h" + +namespace webrtc { + +struct CodecInst; + +template +class AudioEncoderIsacT final : public AudioEncoder { + public: + // Allowed combinations of sample rate, frame size, and bit rate are + // - 16000 Hz, 30 ms, 10000-32000 bps + // - 16000 Hz, 60 ms, 10000-32000 bps + // - 32000 Hz, 30 ms, 10000-56000 bps (if T has super-wideband support) + struct Config { + bool IsOk() const; + + rtc::scoped_refptr bwinfo; + + int payload_type = 103; + int sample_rate_hz = 16000; + int frame_size_ms = 30; + int bit_rate = kDefaultBitRate; // Limit on the short-term average bit + // rate, in bits/s. + int max_payload_size_bytes = -1; + int max_bit_rate = -1; + + // If true, the encoder will dynamically adjust frame size and bit rate; + // the configured values are then merely the starting point. + bool adaptive_mode = false; + + // In adaptive mode, prevent adaptive changes to the frame size. (Not used + // in nonadaptive mode.) + bool enforce_frame_size = false; + }; + + explicit AudioEncoderIsacT(const Config& config); + explicit AudioEncoderIsacT( + const CodecInst& codec_inst, + const rtc::scoped_refptr& bwinfo); + ~AudioEncoderIsacT() override; + + int SampleRateHz() const override; + size_t NumChannels() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + void Reset() override; + + private: + // This value is taken from STREAM_SIZE_MAX_60 for iSAC float (60 ms) and + // STREAM_MAXW16_60MS for iSAC fix (60 ms). + static const size_t kSufficientEncodeBufferSizeBytes = 400; + + static const int kDefaultBitRate = 32000; + + // Recreate the iSAC encoder instance with the given settings, and save them. + void RecreateEncoderInstance(const Config& config); + + Config config_; + typename T::instance_type* isac_state_ = nullptr; + rtc::scoped_refptr bwinfo_; + + // Have we accepted input but not yet emitted it in a packet? + bool packet_in_progress_ = false; + + // Timestamp of the first input of the currently in-progress packet. + uint32_t packet_timestamp_; + + // Timestamp of the previously encoded packet. + uint32_t last_encoded_timestamp_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderIsacT); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_ENCODER_ISAC_T_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h new file mode 100644 index 0000000000..696b799a41 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_ENCODER_ISAC_T_IMPL_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_ENCODER_ISAC_T_IMPL_H_ + +#include "common_types.h" // NOLINT(build/include) +#include "rtc_base/checks.h" + +namespace webrtc { + +template +typename AudioEncoderIsacT::Config CreateIsacConfig( + const CodecInst& codec_inst, + const rtc::scoped_refptr& bwinfo) { + typename AudioEncoderIsacT::Config config; + config.bwinfo = bwinfo; + config.payload_type = codec_inst.pltype; + config.sample_rate_hz = codec_inst.plfreq; + config.frame_size_ms = + rtc::CheckedDivExact(1000 * codec_inst.pacsize, config.sample_rate_hz); + config.adaptive_mode = (codec_inst.rate == -1); + if (codec_inst.rate != -1) + config.bit_rate = codec_inst.rate; + return config; +} + +template +bool AudioEncoderIsacT::Config::IsOk() const { + if (max_bit_rate < 32000 && max_bit_rate != -1) + return false; + if (max_payload_size_bytes < 120 && max_payload_size_bytes != -1) + return false; + if (adaptive_mode && !bwinfo) + return false; + switch (sample_rate_hz) { + case 16000: + if (max_bit_rate > 53400) + return false; + if (max_payload_size_bytes > 400) + return false; + return (frame_size_ms == 30 || frame_size_ms == 60) && + (bit_rate == 0 || (bit_rate >= 10000 && bit_rate <= 32000)); + case 32000: + if (max_bit_rate > 160000) + return false; + if (max_payload_size_bytes > 600) + return false; + return T::has_swb && + (frame_size_ms == 30 && + (bit_rate == 0 || (bit_rate >= 10000 && bit_rate <= 56000))); + default: + return false; + } +} + +template +AudioEncoderIsacT::AudioEncoderIsacT(const Config& config) { + RecreateEncoderInstance(config); +} + +template +AudioEncoderIsacT::AudioEncoderIsacT( + const CodecInst& codec_inst, + const rtc::scoped_refptr& bwinfo) + : AudioEncoderIsacT(CreateIsacConfig(codec_inst, bwinfo)) {} + +template +AudioEncoderIsacT::~AudioEncoderIsacT() { + RTC_CHECK_EQ(0, T::Free(isac_state_)); +} + +template +int AudioEncoderIsacT::SampleRateHz() const { + return T::EncSampRate(isac_state_); +} + +template +size_t AudioEncoderIsacT::NumChannels() const { + return 1; +} + +template +size_t AudioEncoderIsacT::Num10MsFramesInNextPacket() const { + const int samples_in_next_packet = T::GetNewFrameLen(isac_state_); + return static_cast( + rtc::CheckedDivExact(samples_in_next_packet, + rtc::CheckedDivExact(SampleRateHz(), 100))); +} + +template +size_t AudioEncoderIsacT::Max10MsFramesInAPacket() const { + return 6; // iSAC puts at most 60 ms in a packet. +} + +template +int AudioEncoderIsacT::GetTargetBitrate() const { + if (config_.adaptive_mode) + return -1; + return config_.bit_rate == 0 ? kDefaultBitRate : config_.bit_rate; +} + +template +AudioEncoder::EncodedInfo AudioEncoderIsacT::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + if (!packet_in_progress_) { + // Starting a new packet; remember the timestamp for later. + packet_in_progress_ = true; + packet_timestamp_ = rtp_timestamp; + } + if (bwinfo_) { + IsacBandwidthInfo bwinfo = bwinfo_->Get(); + T::SetBandwidthInfo(isac_state_, &bwinfo); + } + + size_t encoded_bytes = encoded->AppendData( + kSufficientEncodeBufferSizeBytes, + [&] (rtc::ArrayView encoded) { + int r = T::Encode(isac_state_, audio.data(), encoded.data()); + + RTC_CHECK_GE(r, 0) << "Encode failed (error code " + << T::GetErrorCode(isac_state_) << ")"; + + return static_cast(r); + }); + + if (encoded_bytes == 0) + return EncodedInfo(); + + // Got enough input to produce a packet. Return the saved timestamp from + // the first chunk of input that went into the packet. + packet_in_progress_ = false; + EncodedInfo info; + info.encoded_bytes = encoded_bytes; + info.encoded_timestamp = packet_timestamp_; + info.payload_type = config_.payload_type; + info.encoder_type = CodecType::kIsac; + return info; +} + +template +void AudioEncoderIsacT::Reset() { + RecreateEncoderInstance(config_); +} + +template +void AudioEncoderIsacT::RecreateEncoderInstance(const Config& config) { + RTC_CHECK(config.IsOk()); + packet_in_progress_ = false; + bwinfo_ = config.bwinfo; + if (isac_state_) + RTC_CHECK_EQ(0, T::Free(isac_state_)); + RTC_CHECK_EQ(0, T::Create(&isac_state_)); + RTC_CHECK_EQ(0, T::EncoderInit(isac_state_, config.adaptive_mode ? 0 : 1)); + RTC_CHECK_EQ(0, T::SetEncSampRate(isac_state_, config.sample_rate_hz)); + const int bit_rate = config.bit_rate == 0 ? kDefaultBitRate : config.bit_rate; + if (config.adaptive_mode) { + RTC_CHECK_EQ(0, T::ControlBwe(isac_state_, bit_rate, config.frame_size_ms, + config.enforce_frame_size)); + } else { + RTC_CHECK_EQ(0, T::Control(isac_state_, bit_rate, config.frame_size_ms)); + } + if (config.max_payload_size_bytes != -1) + RTC_CHECK_EQ( + 0, T::SetMaxPayloadSize(isac_state_, config.max_payload_size_bytes)); + if (config.max_bit_rate != -1) + RTC_CHECK_EQ(0, T::SetMaxRate(isac_state_, config.max_bit_rate)); + + // Set the decoder sample rate even though we just use the encoder. This + // doesn't appear to be necessary to produce a valid encoding, but without it + // we get an encoding that isn't bit-for-bit identical with what a combined + // encoder+decoder object produces. + RTC_CHECK_EQ(0, T::SetDecSampRate(isac_state_, config.sample_rate_hz)); + + config_ = config; +} + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_ENCODER_ISAC_T_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h new file mode 100644 index 0000000000..05397804f5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ + +#include "typedefs.h" // NOLINT(build/include) + +typedef struct { + int in_use; + int32_t send_bw_avg; + int32_t send_max_delay_avg; + int16_t bottleneck_idx; + int16_t jitter_info; +} IsacBandwidthInfo; + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/empty.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/empty.cc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_decoder_isacfix.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_decoder_isacfix.h new file mode 100644 index 0000000000..0b4eadd448 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_decoder_isacfix.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_AUDIO_DECODER_ISACFIX_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_AUDIO_DECODER_ISACFIX_H_ + +#include "modules/audio_coding/codecs/isac/audio_decoder_isac_t.h" +#include "modules/audio_coding/codecs/isac/fix/source/isac_fix_type.h" + +namespace webrtc { + +using AudioDecoderIsacFixImpl = AudioDecoderIsacT; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_AUDIO_DECODER_ISACFIX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h new file mode 100644 index 0000000000..f0cc038328 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_AUDIO_ENCODER_ISACFIX_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_AUDIO_ENCODER_ISACFIX_H_ + +#include "modules/audio_coding/codecs/isac/audio_encoder_isac_t.h" +#include "modules/audio_coding/codecs/isac/fix/source/isac_fix_type.h" + +namespace webrtc { + +using AudioEncoderIsacFixImpl = AudioEncoderIsacT; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_AUDIO_ENCODER_ISACFIX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/isacfix.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/isacfix.h new file mode 100644 index 0000000000..ef194ca528 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/isacfix.h @@ -0,0 +1,637 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_ISACFIX_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_ISACFIX_H_ + +#include + +#include "modules/audio_coding/codecs/isac/bandwidth_info.h" +#include "typedefs.h" // NOLINT(build/include) + +typedef struct { + void *dummy; +} ISACFIX_MainStruct; + + +#if defined(__cplusplus) +extern "C" { +#endif + + + /************************************************************************** + * WebRtcIsacfix_AssignSize(...) + * + * Functions used when malloc is not allowed + * Output the number of bytes needed to allocate for iSAC struct. + * + */ + + int16_t WebRtcIsacfix_AssignSize(int *sizeinbytes); + + /************************************************************************** + * WebRtcIsacfix_Assign(...) + * + * Functions used when malloc is not allowed, it + * places a struct at the given address. + * + * Input: + * - *ISAC_main_inst : a pointer to the coder instance. + * - ISACFIX_inst_Addr : address of the memory where a space is + * for iSAC structure. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_Assign(ISACFIX_MainStruct **inst, + void *ISACFIX_inst_Addr); + + /**************************************************************************** + * WebRtcIsacfix_Create(...) + * + * This function creates an ISAC instance, which will contain the state + * information for one coding/decoding channel. + * + * Input: + * - *ISAC_main_inst : a pointer to the coder instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_Create(ISACFIX_MainStruct **ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsacfix_Free(...) + * + * This function frees the ISAC instance created at the beginning. + * + * Input: + * - ISAC_main_inst : a ISAC instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_Free(ISACFIX_MainStruct *ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsacfix_EncoderInit(...) + * + * This function initializes an ISAC instance prior to the encoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - CodingMode : 0 - Bit rate and frame length are automatically + * adjusted to available bandwidth on + * transmission channel. + * 1 - User sets a frame length and a target bit + * rate which is taken as the maximum short-term + * average bit rate. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_EncoderInit(ISACFIX_MainStruct *ISAC_main_inst, + int16_t CodingMode); + + + /**************************************************************************** + * WebRtcIsacfix_Encode(...) + * + * This function encodes 10ms frame(s) and inserts it into a package. + * Input speech length has to be 160 samples (10ms). The encoder buffers those + * 10ms frames until it reaches the chosen Framesize (480 or 960 samples + * corresponding to 30 or 60 ms frames), and then proceeds to the encoding. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - speechIn : input speech vector. + * + * Output: + * - encoded : the encoded data vector + * + * Return value : >0 - Length (in bytes) of coded data + * 0 - The buffer didn't reach the chosen framesize + * so it keeps buffering speech samples. + * -1 - Error + */ + + int WebRtcIsacfix_Encode(ISACFIX_MainStruct *ISAC_main_inst, + const int16_t *speechIn, + uint8_t* encoded); + + + + /**************************************************************************** + * WebRtcIsacfix_EncodeNb(...) + * + * This function encodes 10ms narrow band (8 kHz sampling) frame(s) and inserts + * it into a package. Input speech length has to be 80 samples (10ms). The encoder + * interpolates into wide-band (16 kHz sampling) buffers those + * 10ms frames until it reaches the chosen Framesize (480 or 960 wide-band samples + * corresponding to 30 or 60 ms frames), and then proceeds to the encoding. + * + * The function is enabled if WEBRTC_ISAC_FIX_NB_CALLS_ENABLED is defined + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - speechIn : input speech vector. + * + * Output: + * - encoded : the encoded data vector + * + * Return value : >0 - Length (in bytes) of coded data + * 0 - The buffer didn't reach the chosen framesize + * so it keeps buffering speech samples. + * -1 - Error + */ + + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + int16_t WebRtcIsacfix_EncodeNb(ISACFIX_MainStruct *ISAC_main_inst, + const int16_t *speechIn, + int16_t *encoded); +#endif // WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + + + + /**************************************************************************** + * WebRtcIsacfix_DecoderInit(...) + * + * This function initializes an ISAC instance prior to the decoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + */ + + void WebRtcIsacfix_DecoderInit(ISACFIX_MainStruct* ISAC_main_inst); + + /**************************************************************************** + * WebRtcIsacfix_UpdateBwEstimate1(...) + * + * This function updates the estimate of the bandwidth. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s). + * - packet_size : size of the packet in bytes. + * - rtp_seq_number : the RTP number of the packet. + * - arr_ts : the arrival time of the packet (from NetEq) + * in samples. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_UpdateBwEstimate1(ISACFIX_MainStruct *ISAC_main_inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t arr_ts); + + /**************************************************************************** + * WebRtcIsacfix_UpdateBwEstimate(...) + * + * This function updates the estimate of the bandwidth. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s). + * - packet_size : size of the packet in bytes. + * - rtp_seq_number : the RTP number of the packet. + * - send_ts : the send time of the packet from RTP header, + * in samples. + * - arr_ts : the arrival time of the packet (from NetEq) + * in samples. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_UpdateBwEstimate(ISACFIX_MainStruct *ISAC_main_inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts); + + /**************************************************************************** + * WebRtcIsacfix_Decode(...) + * + * This function decodes an ISAC frame. Output speech length + * will be a multiple of 480 samples: 480 or 960 samples, + * depending on the framesize (30 or 60 ms). + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s) + * - len : bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded vector + * -1 - Error + */ + + int WebRtcIsacfix_Decode(ISACFIX_MainStruct *ISAC_main_inst, + const uint8_t* encoded, + size_t len, + int16_t *decoded, + int16_t *speechType); + + + /**************************************************************************** + * WebRtcIsacfix_DecodeNb(...) + * + * This function decodes a ISAC frame in narrow-band (8 kHz sampling). + * Output speech length will be a multiple of 240 samples: 240 or 480 samples, + * depending on the framesize (30 or 60 ms). + * + * The function is enabled if WEBRTC_ISAC_FIX_NB_CALLS_ENABLED is defined + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s) + * - len : bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded vector + * -1 - Error + */ + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + int WebRtcIsacfix_DecodeNb(ISACFIX_MainStruct *ISAC_main_inst, + const uint16_t *encoded, + size_t len, + int16_t *decoded, + int16_t *speechType); +#endif // WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + + + /**************************************************************************** + * WebRtcIsacfix_DecodePlcNb(...) + * + * This function conducts PLC for ISAC frame(s) in narrow-band (8kHz sampling). + * Output speech length will be "240*noOfLostFrames" samples + * that equevalent of "30*noOfLostFrames" millisecond. + * + * The function is enabled if WEBRTC_ISAC_FIX_NB_CALLS_ENABLED is defined + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - noOfLostFrames : Number of PLC frames (240 sample=30ms) to produce + * NOTE! Maximum number is 2 (480 samples = 60ms) + * + * Output: + * - decoded : The decoded vector + * + * Return value : Number of samples in decoded PLC vector + */ + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + size_t WebRtcIsacfix_DecodePlcNb(ISACFIX_MainStruct *ISAC_main_inst, + int16_t *decoded, + size_t noOfLostFrames); +#endif // WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + + + + + /**************************************************************************** + * WebRtcIsacfix_DecodePlc(...) + * + * This function conducts PLC for ISAC frame(s) in wide-band (16kHz sampling). + * Output speech length will be "480*noOfLostFrames" samples + * that is equevalent of "30*noOfLostFrames" millisecond. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - noOfLostFrames : Number of PLC frames (480sample = 30ms) + * to produce + * NOTE! Maximum number is 2 (960 samples = 60ms) + * + * Output: + * - decoded : The decoded vector + * + * Return value : Number of samples in decoded PLC vector + */ + + size_t WebRtcIsacfix_DecodePlc(ISACFIX_MainStruct *ISAC_main_inst, + int16_t *decoded, + size_t noOfLostFrames ); + + + /**************************************************************************** + * WebRtcIsacfix_ReadFrameLen(...) + * + * This function returns the length of the frame represented in the packet. + * + * Input: + * - encoded : Encoded bitstream + * - encoded_len_bytes : Length of the bitstream in bytes. + * + * Output: + * - frameLength : Length of frame in packet (in samples) + * + */ + + int16_t WebRtcIsacfix_ReadFrameLen(const uint8_t* encoded, + size_t encoded_len_bytes, + size_t* frameLength); + + /**************************************************************************** + * WebRtcIsacfix_Control(...) + * + * This function sets the limit on the short-term average bit rate and the + * frame length. Should be used only in Instantaneous mode. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - rate : limit on the short-term average bit rate, + * in bits/second (between 10000 and 32000) + * - framesize : number of milliseconds per frame (30 or 60) + * + * Return value : 0 - ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_Control(ISACFIX_MainStruct *ISAC_main_inst, + int16_t rate, + int framesize); + + void WebRtcIsacfix_SetInitialBweBottleneck(ISACFIX_MainStruct* ISAC_main_inst, + int bottleneck_bits_per_second); + + /**************************************************************************** + * WebRtcIsacfix_ControlBwe(...) + * + * This function sets the initial values of bottleneck and frame-size if + * iSAC is used in channel-adaptive mode. Through this API, users can + * enforce a frame-size for all values of bottleneck. Then iSAC will not + * automatically change the frame-size. + * + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - rateBPS : initial value of bottleneck in bits/second + * 10000 <= rateBPS <= 32000 is accepted + * - frameSizeMs : number of milliseconds per frame (30 or 60) + * - enforceFrameSize : 1 to enforce the given frame-size through out + * the adaptation process, 0 to let iSAC change + * the frame-size if required. + * + * Return value : 0 - ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_ControlBwe(ISACFIX_MainStruct *ISAC_main_inst, + int16_t rateBPS, + int frameSizeMs, + int16_t enforceFrameSize); + + + + /**************************************************************************** + * WebRtcIsacfix_version(...) + * + * This function returns the version number. + * + * Output: + * - version : Pointer to character string + * + */ + + void WebRtcIsacfix_version(char *version); + + + /**************************************************************************** + * WebRtcIsacfix_GetErrorCode(...) + * + * This function can be used to check the error code of an iSAC instance. When + * a function returns -1 a error code will be set for that instance. The + * function below extract the code of the last error that occured in the + * specified instance. + * + * Input: + * - ISAC_main_inst : ISAC instance + * + * Return value : Error code + */ + + int16_t WebRtcIsacfix_GetErrorCode(ISACFIX_MainStruct *ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsacfix_GetUplinkBw(...) + * + * This function return iSAC send bitrate + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Return value : <0 Error code + * else bitrate + */ + + int32_t WebRtcIsacfix_GetUplinkBw(ISACFIX_MainStruct *ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsacfix_SetMaxPayloadSize(...) + * + * This function sets a limit for the maximum payload size of iSAC. The same + * value is used both for 30 and 60 msec packets. + * The absolute max will be valid until next time the function is called. + * NOTE! This function may override the function WebRtcIsacfix_SetMaxRate() + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxPayloadBytes : maximum size of the payload in bytes + * valid values are between 100 and 400 bytes + * + * + * Return value : 0 if sucessful + * -1 if error happens + */ + + int16_t WebRtcIsacfix_SetMaxPayloadSize(ISACFIX_MainStruct *ISAC_main_inst, + int16_t maxPayloadBytes); + + + /**************************************************************************** + * WebRtcIsacfix_SetMaxRate(...) + * + * This function sets the maximum rate which the codec may not exceed for a + * singel packet. The maximum rate is set in bits per second. + * The codec has an absolute maximum rate of 53400 bits per second (200 bytes + * per 30 msec). + * It is possible to set a maximum rate between 32000 and 53400 bits per second. + * + * The rate limit is valid until next time the function is called. + * + * NOTE! Packet size will never go above the value set if calling + * WebRtcIsacfix_SetMaxPayloadSize() (default max packet size is 400 bytes). + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxRateInBytes : maximum rate in bits per second, + * valid values are 32000 to 53400 bits + * + * Return value : 0 if sucessful + * -1 if error happens + */ + + int16_t WebRtcIsacfix_SetMaxRate(ISACFIX_MainStruct *ISAC_main_inst, + int32_t maxRate); + + /**************************************************************************** + * WebRtcIsacfix_CreateInternal(...) + * + * This function creates the memory that is used to store data in the encoder + * + * Input: + * - *ISAC_main_inst : a pointer to the coder instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_CreateInternal(ISACFIX_MainStruct *ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsacfix_FreeInternal(...) + * + * This function frees the internal memory for storing encoder data. + * + * Input: + * - ISAC_main_inst : an ISAC instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsacfix_FreeInternal(ISACFIX_MainStruct *ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsacfix_GetNewBitStream(...) + * + * This function returns encoded data, with the recieved bwe-index in the + * stream. It should always return a complete packet, i.e. only called once + * even for 60 msec frames + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - bweIndex : index of bandwidth estimate to put in new bitstream + * - scale : factor for rate change (0.4 ~=> half the rate, 1 no change). + * + * Output: + * - encoded : the encoded data vector + * + * Return value : >0 - Length (in bytes) of coded data + * -1 - Error + */ + + int16_t WebRtcIsacfix_GetNewBitStream(ISACFIX_MainStruct *ISAC_main_inst, + int16_t bweIndex, + float scale, + uint8_t* encoded); + + + /**************************************************************************** + * WebRtcIsacfix_GetDownLinkBwIndex(...) + * + * This function returns index representing the Bandwidth estimate from + * other side to this side. + * + * Input: + * - ISAC_main_inst : iSAC struct + * + * Output: + * - rateIndex : Bandwidth estimate to transmit to other side. + * + */ + + int16_t WebRtcIsacfix_GetDownLinkBwIndex(ISACFIX_MainStruct* ISAC_main_inst, + int16_t* rateIndex); + + + /**************************************************************************** + * WebRtcIsacfix_UpdateUplinkBw(...) + * + * This function takes an index representing the Bandwidth estimate from + * this side to other side and updates BWE. + * + * Input: + * - ISAC_main_inst : iSAC struct + * - rateIndex : Bandwidth estimate from other side. + * + */ + + int16_t WebRtcIsacfix_UpdateUplinkBw(ISACFIX_MainStruct* ISAC_main_inst, + int16_t rateIndex); + + + /**************************************************************************** + * WebRtcIsacfix_ReadBwIndex(...) + * + * This function returns the index of the Bandwidth estimate from the bitstream. + * + * Input: + * - encoded : Encoded bitstream + * - encoded_len_bytes : Length of the bitstream in bytes. + * + * Output: + * - rateIndex : Bandwidth estimate in bitstream + * + */ + + int16_t WebRtcIsacfix_ReadBwIndex(const uint8_t* encoded, + size_t encoded_len_bytes, + int16_t* rateIndex); + + + /**************************************************************************** + * WebRtcIsacfix_GetNewFrameLen(...) + * + * This function return the next frame length (in samples) of iSAC. + * + * Input: + * -ISAC_main_inst : iSAC instance + * + * Return value : frame lenght in samples + */ + + int16_t WebRtcIsacfix_GetNewFrameLen(ISACFIX_MainStruct *ISAC_main_inst); + + /* Fills in an IsacBandwidthInfo struct. */ + void WebRtcIsacfix_GetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + IsacBandwidthInfo* bwinfo); + + /* Uses the values from an IsacBandwidthInfo struct. */ + void WebRtcIsacfix_SetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + const IsacBandwidthInfo* bwinfo); + +#if defined(__cplusplus) +} +#endif + + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INCLUDE_ISACFIX_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines.c new file mode 100644 index 0000000000..5925d68e35 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines.c @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * arith_routins.c + * + * This C file contains a function for finalizing the bitstream + * after arithmetic coding. + * + */ + +#include "arith_routins.h" + + +/**************************************************************************** + * WebRtcIsacfix_EncTerminate(...) + * + * Final call to the arithmetic coder for an encoder call. This function + * terminates and return byte stream. + * + * Input: + * - streamData : in-/output struct containing bitstream + * + * Return value : number of bytes in the stream + */ +int16_t WebRtcIsacfix_EncTerminate(Bitstr_enc *streamData) +{ + uint16_t *streamPtr; + uint16_t negCarry; + + /* point to the right place in the stream buffer */ + streamPtr = streamData->stream + streamData->stream_index; + + /* find minimum length (determined by current interval width) */ + if ( streamData->W_upper > 0x01FFFFFF ) + { + streamData->streamval += 0x01000000; + + /* if result is less than the added value we must take care of the carry */ + if (streamData->streamval < 0x01000000) + { + /* propagate carry */ + if (streamData->full == 0) { + /* Add value to current value */ + negCarry = *streamPtr; + negCarry += 0x0100; + *streamPtr = negCarry; + + /* if value is too big, propagate carry to next byte, and so on */ + while (!(negCarry)) + { + negCarry = *--streamPtr; + negCarry++; + *streamPtr = negCarry; + } + } else { + /* propagate carry by adding one to the previous byte in the + * stream if that byte is 0xFFFF we need to propagate the carry + * furhter back in the stream */ + while ( !(++(*--streamPtr)) ); + } + + /* put pointer back to the old value */ + streamPtr = streamData->stream + streamData->stream_index; + } + /* write remaining data to bitstream, if "full == 0" first byte has data */ + if (streamData->full == 0) { + *streamPtr++ += (uint16_t)(streamData->streamval >> 24); + streamData->full = 1; + } else { + *streamPtr = (uint16_t)((streamData->streamval >> 24) << 8); + streamData->full = 0; + } + } + else + { + streamData->streamval += 0x00010000; + + /* if result is less than the added value we must take care of the carry */ + if (streamData->streamval < 0x00010000) + { + /* propagate carry */ + if (streamData->full == 0) { + /* Add value to current value */ + negCarry = *streamPtr; + negCarry += 0x0100; + *streamPtr = negCarry; + + /* if value to big, propagate carry to next byte, and so on */ + while (!(negCarry)) + { + negCarry = *--streamPtr; + negCarry++; + *streamPtr = negCarry; + } + } else { + /* Add carry to previous byte */ + while ( !(++(*--streamPtr)) ); + } + + /* put pointer back to the old value */ + streamPtr = streamData->stream + streamData->stream_index; + } + /* write remaining data (2 bytes) to bitstream */ + if (streamData->full) { + *streamPtr++ = (uint16_t)(streamData->streamval >> 16); + } else { + *streamPtr++ |= (uint16_t)(streamData->streamval >> 24); + *streamPtr = (uint16_t)(streamData->streamval >> 8) & 0xFF00; + } + } + + /* calculate stream length in bytes */ + return (((streamPtr - streamData->stream)<<1) + !(streamData->full)); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_hist.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_hist.c new file mode 100644 index 0000000000..f1eb5ae8f7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_hist.c @@ -0,0 +1,401 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * arith_routinshist.c + * + * This C file contains arithmetic encoding and decoding. + * + */ + +#include "arith_routins.h" + + +/**************************************************************************** + * WebRtcIsacfix_EncHistMulti(...) + * + * Encode the histogram interval + * + * Input: + * - streamData : in-/output struct containing bitstream + * - data : data vector + * - cdf : array of cdf arrays + * - lenData : data vector length + * + * Return value : 0 if ok + * <0 if error detected + */ +int WebRtcIsacfix_EncHistMulti(Bitstr_enc *streamData, + const int16_t *data, + const uint16_t *const *cdf, + const int16_t lenData) +{ + uint32_t W_lower; + uint32_t W_upper; + uint32_t W_upper_LSB; + uint32_t W_upper_MSB; + uint16_t *streamPtr; + uint16_t negCarry; + uint16_t *maxStreamPtr; + uint16_t *streamPtrCarry; + uint32_t cdfLo; + uint32_t cdfHi; + int k; + + + /* point to beginning of stream buffer + * and set maximum streamPtr value */ + streamPtr = streamData->stream + streamData->stream_index; + maxStreamPtr = streamData->stream + STREAM_MAXW16_60MS - 1; + + W_upper = streamData->W_upper; + + for (k = lenData; k > 0; k--) + { + /* fetch cdf_lower and cdf_upper from cdf tables */ + cdfLo = (uint32_t) *(*cdf + (uint32_t)*data); + cdfHi = (uint32_t) *(*cdf++ + (uint32_t)*data++ + 1); + + /* update interval */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = W_upper >> 16; + W_lower = WEBRTC_SPL_UMUL(W_upper_MSB, cdfLo); + W_lower += ((W_upper_LSB * cdfLo) >> 16); + W_upper = WEBRTC_SPL_UMUL(W_upper_MSB, cdfHi); + W_upper += ((W_upper_LSB * cdfHi) >> 16); + + /* shift interval such that it begins at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamData->streamval += W_lower; + + /* handle carry */ + if (streamData->streamval < W_lower) + { + /* propagate carry */ + streamPtrCarry = streamPtr; + if (streamData->full == 0) { + negCarry = *streamPtrCarry; + negCarry += 0x0100; + *streamPtrCarry = negCarry; + while (!(negCarry)) + { + negCarry = *--streamPtrCarry; + negCarry++; + *streamPtrCarry = negCarry; + } + } else { + while ( !(++(*--streamPtrCarry)) ); + } + } + + /* renormalize interval, store most significant byte of streamval and update streamval + * W_upper < 2^24 */ + while ( !(W_upper & 0xFF000000) ) + { + W_upper <<= 8; + if (streamData->full == 0) { + *streamPtr++ += (uint16_t)(streamData->streamval >> 24); + streamData->full = 1; + } else { + *streamPtr = (uint16_t)((streamData->streamval >> 24) << 8); + streamData->full = 0; + } + + if( streamPtr > maxStreamPtr ) { + return -ISAC_DISALLOWED_BITSTREAM_LENGTH; + } + streamData->streamval <<= 8; + } + } + + /* calculate new stream_index */ + streamData->stream_index = streamPtr - streamData->stream; + streamData->W_upper = W_upper; + + return 0; +} + + +/**************************************************************************** + * WebRtcIsacfix_DecHistBisectMulti(...) + * + * Function to decode more symbols from the arithmetic bytestream, using + * method of bisection cdf tables should be of size 2^k-1 (which corresponds + * to an alphabet size of 2^k-2) + * + * Input: + * - streamData : in-/output struct containing bitstream + * - cdf : array of cdf arrays + * - cdfSize : array of cdf table sizes+1 (power of two: 2^k) + * - lenData : data vector length + * + * Output: + * - data : data vector + * + * Return value : number of bytes in the stream + * <0 if error detected + */ +int16_t WebRtcIsacfix_DecHistBisectMulti(int16_t *data, + Bitstr_dec *streamData, + const uint16_t *const *cdf, + const uint16_t *cdfSize, + const int16_t lenData) +{ + uint32_t W_lower = 0; + uint32_t W_upper; + uint32_t W_tmp; + uint32_t W_upper_LSB; + uint32_t W_upper_MSB; + uint32_t streamval; + const uint16_t *streamPtr; + const uint16_t *cdfPtr; + int16_t sizeTmp; + int k; + + + streamPtr = streamData->stream + streamData->stream_index; + W_upper = streamData->W_upper; + + /* Error check: should not be possible in normal operation */ + if (W_upper == 0) { + return -2; + } + + /* first time decoder is called for this stream */ + if (streamData->stream_index == 0) + { + /* read first word from bytestream */ + streamval = (uint32_t)*streamPtr++ << 16; + streamval |= *streamPtr++; + } else { + streamval = streamData->streamval; + } + + for (k = lenData; k > 0; k--) + { + /* find the integer *data for which streamval lies in [W_lower+1, W_upper] */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = W_upper >> 16; + + /* start halfway the cdf range */ + sizeTmp = *cdfSize++ / 2; + cdfPtr = *cdf + (sizeTmp - 1); + + /* method of bisection */ + for ( ;; ) + { + W_tmp = WEBRTC_SPL_UMUL_32_16(W_upper_MSB, *cdfPtr); + W_tmp += (W_upper_LSB * (*cdfPtr)) >> 16; + sizeTmp /= 2; + if (sizeTmp == 0) { + break; + } + + if (streamval > W_tmp) + { + W_lower = W_tmp; + cdfPtr += sizeTmp; + } else { + W_upper = W_tmp; + cdfPtr -= sizeTmp; + } + } + if (streamval > W_tmp) + { + W_lower = W_tmp; + *data++ = cdfPtr - *cdf++; + } else { + W_upper = W_tmp; + *data++ = cdfPtr - *cdf++ - 1; + } + + /* shift interval to start at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamval -= W_lower; + + /* renormalize interval and update streamval */ + /* W_upper < 2^24 */ + while ( !(W_upper & 0xFF000000) ) + { + /* read next byte from stream */ + if (streamData->full == 0) { + streamval = (streamval << 8) | (*streamPtr++ & 0x00FF); + streamData->full = 1; + } else { + streamval = (streamval << 8) | (*streamPtr >> 8); + streamData->full = 0; + } + W_upper <<= 8; + } + + + /* Error check: should not be possible in normal operation */ + if (W_upper == 0) { + return -2; + } + + } + + streamData->stream_index = streamPtr - streamData->stream; + streamData->W_upper = W_upper; + streamData->streamval = streamval; + + if ( W_upper > 0x01FFFFFF ) { + return (streamData->stream_index*2 - 3 + !streamData->full); + } else { + return (streamData->stream_index*2 - 2 + !streamData->full); + } +} + + +/**************************************************************************** + * WebRtcIsacfix_DecHistOneStepMulti(...) + * + * Function to decode more symbols from the arithmetic bytestream, taking + * single step up or down at a time. + * cdf tables can be of arbitrary size, but large tables may take a lot of + * iterations. + * + * Input: + * - streamData : in-/output struct containing bitstream + * - cdf : array of cdf arrays + * - initIndex : vector of initial cdf table search entries + * - lenData : data vector length + * + * Output: + * - data : data vector + * + * Return value : number of bytes in original stream + * <0 if error detected + */ +int16_t WebRtcIsacfix_DecHistOneStepMulti(int16_t *data, + Bitstr_dec *streamData, + const uint16_t *const *cdf, + const uint16_t *initIndex, + const int16_t lenData) +{ + uint32_t W_lower; + uint32_t W_upper; + uint32_t W_tmp; + uint32_t W_upper_LSB; + uint32_t W_upper_MSB; + uint32_t streamval; + const uint16_t *streamPtr; + const uint16_t *cdfPtr; + int k; + + + streamPtr = streamData->stream + streamData->stream_index; + W_upper = streamData->W_upper; + /* Error check: Should not be possible in normal operation */ + if (W_upper == 0) { + return -2; + } + + /* Check if it is the first time decoder is called for this stream */ + if (streamData->stream_index == 0) + { + /* read first word from bytestream */ + streamval = (uint32_t)(*streamPtr++) << 16; + streamval |= *streamPtr++; + } else { + streamval = streamData->streamval; + } + + for (k = lenData; k > 0; k--) + { + /* find the integer *data for which streamval lies in [W_lower+1, W_upper] */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = WEBRTC_SPL_RSHIFT_U32(W_upper, 16); + + /* start at the specified table entry */ + cdfPtr = *cdf + (*initIndex++); + W_tmp = WEBRTC_SPL_UMUL_32_16(W_upper_MSB, *cdfPtr); + W_tmp += (W_upper_LSB * (*cdfPtr)) >> 16; + + if (streamval > W_tmp) + { + for ( ;; ) + { + W_lower = W_tmp; + + /* range check */ + if (cdfPtr[0] == 65535) { + return -3; + } + + W_tmp = WEBRTC_SPL_UMUL_32_16(W_upper_MSB, *++cdfPtr); + W_tmp += (W_upper_LSB * (*cdfPtr)) >> 16; + + if (streamval <= W_tmp) { + break; + } + } + W_upper = W_tmp; + *data++ = cdfPtr - *cdf++ - 1; + } else { + for ( ;; ) + { + W_upper = W_tmp; + --cdfPtr; + + /* range check */ + if (cdfPtr < *cdf) { + return -3; + } + + W_tmp = WEBRTC_SPL_UMUL_32_16(W_upper_MSB, *cdfPtr); + W_tmp += (W_upper_LSB * (*cdfPtr)) >> 16; + + if (streamval > W_tmp) { + break; + } + } + W_lower = W_tmp; + *data++ = cdfPtr - *cdf++; + } + + /* shift interval to start at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamval -= W_lower; + + /* renormalize interval and update streamval */ + /* W_upper < 2^24 */ + while ( !(W_upper & 0xFF000000) ) + { + /* read next byte from stream */ + if (streamData->full == 0) { + streamval = (streamval << 8) | (*streamPtr++ & 0x00FF); + streamData->full = 1; + } else { + streamval = (streamval << 8) | (*streamPtr >> 8); + streamData->full = 0; + } + W_upper <<= 8; + } + } + + streamData->stream_index = streamPtr - streamData->stream; + streamData->W_upper = W_upper; + streamData->streamval = streamval; + + /* find number of bytes in original stream (determined by current interval width) */ + if ( W_upper > 0x01FFFFFF ) { + return (streamData->stream_index*2 - 3 + !streamData->full); + } else { + return (streamData->stream_index*2 - 2 + !streamData->full); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_logist.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_logist.c new file mode 100644 index 0000000000..808aeb7fd9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_logist.c @@ -0,0 +1,413 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * arith_routinslogist.c + * + * This C file contains arithmetic encode and decode logistic + * + */ + +#include "arith_routins.h" + +/* Tables for piecewise linear cdf functions: y = k*x */ + +/* x Points for function piecewise() in Q15 */ +static const int32_t kHistEdges[51] = { + -327680, -314573, -301466, -288359, -275252, -262144, -249037, -235930, -222823, -209716, + -196608, -183501, -170394, -157287, -144180, -131072, -117965, -104858, -91751, -78644, + -65536, -52429, -39322, -26215, -13108, 0, 13107, 26214, 39321, 52428, + 65536, 78643, 91750, 104857, 117964, 131072, 144179, 157286, 170393, 183500, + 196608, 209715, 222822, 235929, 249036, 262144, 275251, 288358, 301465, 314572, + 327680 +}; + + +/* k Points for function piecewise() in Q0 */ +static const uint16_t kCdfSlope[51] = { + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 13, 23, 47, 87, 154, 315, 700, 1088, + 2471, 6064, 14221, 21463, 36634, 36924, 19750, 13270, 5806, 2312, + 1095, 660, 316, 145, 86, 41, 32, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 2, + 0 +}; + +/* y Points for function piecewise() in Q0 */ +static const uint16_t kCdfLogistic[51] = { + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, + 20, 22, 24, 29, 38, 57, 92, 153, 279, 559, + 994, 1983, 4408, 10097, 18682, 33336, 48105, 56005, 61313, 63636, + 64560, 64998, 65262, 65389, 65447, 65481, 65497, 65510, 65512, 65514, + 65516, 65518, 65520, 65522, 65524, 65526, 65528, 65530, 65532, 65534, + 65535 +}; + + +/**************************************************************************** + * WebRtcIsacfix_Piecewise(...) + * + * Piecewise linear function + * + * Input: + * - xinQ15 : input value x in Q15 + * + * Return value : korresponding y-value in Q0 + */ + + +static __inline uint16_t WebRtcIsacfix_Piecewise(int32_t xinQ15) { + int32_t ind; + int32_t qtmp1; + uint16_t qtmp2; + + /* Find index for x-value */ + qtmp1 = WEBRTC_SPL_SAT(kHistEdges[50],xinQ15,kHistEdges[0]); + ind = WEBRTC_SPL_MUL(5, qtmp1 - kHistEdges[0]); + ind >>= 16; + + /* Calculate corresponding y-value ans return*/ + qtmp1 = qtmp1 - kHistEdges[ind]; + qtmp2 = (uint16_t)WEBRTC_SPL_RSHIFT_U32( + WEBRTC_SPL_UMUL_32_16(qtmp1,kCdfSlope[ind]), 15); + return (kCdfLogistic[ind] + qtmp2); +} + +/**************************************************************************** + * WebRtcIsacfix_EncLogisticMulti2(...) + * + * Arithmetic coding of spectrum. + * + * Input: + * - streamData : in-/output struct containing bitstream + * - dataQ7 : data vector in Q7 + * - envQ8 : side info vector defining the width of the pdf + * in Q8 + * - lenData : data vector length + * + * Return value : 0 if ok, + * <0 otherwise. + */ +int WebRtcIsacfix_EncLogisticMulti2(Bitstr_enc *streamData, + int16_t *dataQ7, + const uint16_t *envQ8, + const int16_t lenData) +{ + uint32_t W_lower; + uint32_t W_upper; + uint16_t W_upper_LSB; + uint16_t W_upper_MSB; + uint16_t *streamPtr; + uint16_t *maxStreamPtr; + uint16_t *streamPtrCarry; + uint16_t negcarry; + uint32_t cdfLo; + uint32_t cdfHi; + int k; + + /* point to beginning of stream buffer + * and set maximum streamPtr value */ + streamPtr = streamData->stream + streamData->stream_index; + maxStreamPtr = streamData->stream + STREAM_MAXW16_60MS - 1; + W_upper = streamData->W_upper; + + for (k = 0; k < lenData; k++) + { + /* compute cdf_lower and cdf_upper by evaluating the + * WebRtcIsacfix_Piecewise linear cdf */ + cdfLo = WebRtcIsacfix_Piecewise(WEBRTC_SPL_MUL_16_U16(*dataQ7 - 64, *envQ8)); + cdfHi = WebRtcIsacfix_Piecewise(WEBRTC_SPL_MUL_16_U16(*dataQ7 + 64, *envQ8)); + + /* test and clip if probability gets too small */ + while ((cdfLo + 1) >= cdfHi) { + /* clip */ + if (*dataQ7 > 0) { + *dataQ7 -= 128; + cdfHi = cdfLo; + cdfLo = WebRtcIsacfix_Piecewise( + WEBRTC_SPL_MUL_16_U16(*dataQ7 - 64, *envQ8)); + } else { + *dataQ7 += 128; + cdfLo = cdfHi; + cdfHi = WebRtcIsacfix_Piecewise( + WEBRTC_SPL_MUL_16_U16(*dataQ7 + 64, *envQ8)); + } + } + + dataQ7++; + /* increment only once per 4 iterations */ + envQ8 += (k & 1) & (k >> 1); + + + /* update interval */ + W_upper_LSB = (uint16_t)W_upper; + W_upper_MSB = (uint16_t)WEBRTC_SPL_RSHIFT_U32(W_upper, 16); + W_lower = WEBRTC_SPL_UMUL_32_16(cdfLo, W_upper_MSB); + W_lower += (cdfLo * W_upper_LSB) >> 16; + W_upper = WEBRTC_SPL_UMUL_32_16(cdfHi, W_upper_MSB); + W_upper += (cdfHi * W_upper_LSB) >> 16; + + /* shift interval such that it begins at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamData->streamval += W_lower; + + /* handle carry */ + if (streamData->streamval < W_lower) + { + /* propagate carry */ + streamPtrCarry = streamPtr; + if (streamData->full == 0) { + negcarry = *streamPtrCarry; + negcarry += 0x0100; + *streamPtrCarry = negcarry; + while (!(negcarry)) + { + negcarry = *--streamPtrCarry; + negcarry++; + *streamPtrCarry = negcarry; + } + } else { + while (!(++(*--streamPtrCarry))); + } + } + + /* renormalize interval, store most significant byte of streamval and update streamval + * W_upper < 2^24 */ + while ( !(W_upper & 0xFF000000) ) + { + W_upper <<= 8; + if (streamData->full == 0) { + *streamPtr++ += (uint16_t) WEBRTC_SPL_RSHIFT_U32( + streamData->streamval, 24); + streamData->full = 1; + } else { + *streamPtr = (uint16_t)((streamData->streamval >> 24) << 8); + streamData->full = 0; + } + + if( streamPtr > maxStreamPtr ) + return -ISAC_DISALLOWED_BITSTREAM_LENGTH; + + streamData->streamval <<= 8; + } + } + + /* calculate new stream_index */ + streamData->stream_index = streamPtr - streamData->stream; + streamData->W_upper = W_upper; + + return 0; +} + + +/**************************************************************************** + * WebRtcIsacfix_DecLogisticMulti2(...) + * + * Arithmetic decoding of spectrum. + * + * Input: + * - streamData : in-/output struct containing bitstream + * - envQ8 : side info vector defining the width of the pdf + * in Q8 + * - lenData : data vector length + * + * Input/Output: + * - dataQ7 : input: dither vector, output: data vector + * + * Return value : number of bytes in the stream so far + * -1 if error detected + */ +int WebRtcIsacfix_DecLogisticMulti2(int16_t *dataQ7, + Bitstr_dec *streamData, + const int32_t *envQ8, + const int16_t lenData) +{ + uint32_t W_lower; + uint32_t W_upper; + uint32_t W_tmp; + uint16_t W_upper_LSB; + uint16_t W_upper_MSB; + uint32_t streamVal; + uint16_t cdfTmp; + int32_t res; + int32_t inSqrt; + int32_t newRes; + const uint16_t *streamPtr; + int16_t candQ7; + int16_t envCount; + uint16_t tmpARSpecQ8 = 0; + int k, i; + int offset = 0; + + /* point to beginning of stream buffer */ + streamPtr = streamData->stream + streamData->stream_index; + W_upper = streamData->W_upper; + + /* Check if it is first time decoder is called for this stream */ + if (streamData->stream_index == 0) + { + /* read first word from bytestream */ + streamVal = (uint32_t)(*streamPtr++) << 16; + streamVal |= *streamPtr++; + + } else { + streamVal = streamData->streamval; + } + + + res = 1 << (WebRtcSpl_GetSizeInBits(envQ8[0]) >> 1); + envCount = 0; + + /* code assumes lenData%4 == 0 */ + for (k = 0; k < lenData; k += 4) + { + int k4; + + /* convert to magnitude spectrum, by doing square-roots (modified from SPLIB) */ + inSqrt = envQ8[envCount]; + i = 10; + + /* For safty reasons */ + if (inSqrt < 0) + inSqrt=-inSqrt; + + newRes = (inSqrt / res + res) >> 1; + do + { + res = newRes; + newRes = (inSqrt / res + res) >> 1; + } while (newRes != res && i-- > 0); + + tmpARSpecQ8 = (uint16_t)newRes; + + for(k4 = 0; k4 < 4; k4++) + { + /* find the integer *data for which streamVal lies in [W_lower+1, W_upper] */ + W_upper_LSB = (uint16_t) (W_upper & 0x0000FFFF); + W_upper_MSB = (uint16_t) WEBRTC_SPL_RSHIFT_U32(W_upper, 16); + + /* find first candidate by inverting the logistic cdf + * Input dither value collected from io-stream */ + candQ7 = - *dataQ7 + 64; + cdfTmp = WebRtcIsacfix_Piecewise(WEBRTC_SPL_MUL_16_U16(candQ7, tmpARSpecQ8)); + + W_tmp = (uint32_t)cdfTmp * W_upper_MSB; + W_tmp += ((uint32_t)cdfTmp * (uint32_t)W_upper_LSB) >> 16; + + if (streamVal > W_tmp) + { + W_lower = W_tmp; + candQ7 += 128; + cdfTmp = WebRtcIsacfix_Piecewise(WEBRTC_SPL_MUL_16_U16(candQ7, tmpARSpecQ8)); + + W_tmp = (uint32_t)cdfTmp * W_upper_MSB; + W_tmp += ((uint32_t)cdfTmp * (uint32_t)W_upper_LSB) >> 16; + + while (streamVal > W_tmp) + { + W_lower = W_tmp; + candQ7 += 128; + cdfTmp = WebRtcIsacfix_Piecewise( + WEBRTC_SPL_MUL_16_U16(candQ7, tmpARSpecQ8)); + + W_tmp = (uint32_t)cdfTmp * W_upper_MSB; + W_tmp += ((uint32_t)cdfTmp * (uint32_t)W_upper_LSB) >> 16; + + /* error check */ + if (W_lower == W_tmp) { + return -1; + } + } + W_upper = W_tmp; + + /* Output value put in dataQ7: another sample decoded */ + *dataQ7 = candQ7 - 64; + } + else + { + W_upper = W_tmp; + candQ7 -= 128; + cdfTmp = WebRtcIsacfix_Piecewise(WEBRTC_SPL_MUL_16_U16(candQ7, tmpARSpecQ8)); + + W_tmp = (uint32_t)cdfTmp * W_upper_MSB; + W_tmp += ((uint32_t)cdfTmp * (uint32_t)W_upper_LSB) >> 16; + + while ( !(streamVal > W_tmp) ) + { + W_upper = W_tmp; + candQ7 -= 128; + cdfTmp = WebRtcIsacfix_Piecewise( + WEBRTC_SPL_MUL_16_U16(candQ7, tmpARSpecQ8)); + + W_tmp = (uint32_t)cdfTmp * W_upper_MSB; + W_tmp += ((uint32_t)cdfTmp * (uint32_t)W_upper_LSB) >> 16; + + /* error check */ + if (W_upper == W_tmp){ + return -1; + } + } + W_lower = W_tmp; + + /* Output value put in dataQ7: another sample decoded */ + *dataQ7 = candQ7 + 64; + } + + dataQ7++; + + /* shift interval to start at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamVal -= W_lower; + + /* renormalize interval and update streamVal + * W_upper < 2^24 */ + while ( !(W_upper & 0xFF000000) ) + { + if (streamPtr < streamData->stream + streamData->stream_size) { + /* read next byte from stream */ + if (streamData->full == 0) { + streamVal = (streamVal << 8) | (*streamPtr++ & 0x00FF); + streamData->full = 1; + } else { + streamVal = (streamVal << 8) | (*streamPtr >> 8); + streamData->full = 0; + } + } else { + /* Intending to read outside the stream. This can happen for the last + * two or three bytes. It is how the algorithm is implemented. Do + * not read from the bit stream and insert zeros instead. */ + streamVal <<= 8; + if (streamData->full == 0) { + offset++; // We would have incremented the pointer in this case. + streamData->full = 1; + } else { + streamData->full = 0; + } + } + W_upper <<= 8; + } + } + envCount++; + } + + streamData->stream_index = streamPtr + offset - streamData->stream; + streamData->W_upper = W_upper; + streamData->streamval = streamVal; + + /* find number of bytes in original stream (determined by current interval width) */ + if ( W_upper > 0x01FFFFFF ) + return (streamData->stream_index*2 - 3 + !streamData->full); + else + return (streamData->stream_index*2 - 2 + !streamData->full); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routins.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routins.h new file mode 100644 index 0000000000..830c52fcaf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routins.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * arith_routins.h + * + * Functions for arithmetic coding. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ARITH_ROUTINS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ARITH_ROUTINS_H_ + +#include "structs.h" + + +/**************************************************************************** + * WebRtcIsacfix_EncLogisticMulti2(...) + * + * Arithmetic coding of spectrum. + * + * Input: + * - streamData : in-/output struct containing bitstream + * - dataQ7 : data vector in Q7 + * - envQ8 : side info vector defining the width of the pdf + * in Q8 + * - lenData : data vector length + * + * Return value : 0 if ok, + * <0 otherwise. + */ +int WebRtcIsacfix_EncLogisticMulti2( + Bitstr_enc *streamData, + int16_t *dataQ7, + const uint16_t *env, + const int16_t lenData); + + +/**************************************************************************** + * WebRtcIsacfix_EncTerminate(...) + * + * Final call to the arithmetic coder for an encoder call. This function + * terminates and return byte stream. + * + * Input: + * - streamData : in-/output struct containing bitstream + * + * Return value : number of bytes in the stream + */ +int16_t WebRtcIsacfix_EncTerminate(Bitstr_enc *streamData); + + +/**************************************************************************** + * WebRtcIsacfix_DecLogisticMulti2(...) + * + * Arithmetic decoding of spectrum. + * + * Input: + * - streamData : in-/output struct containing bitstream + * - envQ8 : side info vector defining the width of the pdf + * in Q8 + * - lenData : data vector length + * + * Input/Output: + * - dataQ7 : input: dither vector, output: data vector, in Q7 + * + * Return value : number of bytes in the stream so far + * <0 if error detected + */ +int WebRtcIsacfix_DecLogisticMulti2( + int16_t *data, + Bitstr_dec *streamData, + const int32_t *env, + const int16_t lenData); + + +/**************************************************************************** + * WebRtcIsacfix_EncHistMulti(...) + * + * Encode the histogram interval + * + * Input: + * - streamData : in-/output struct containing bitstream + * - data : data vector + * - cdf : array of cdf arrays + * - lenData : data vector length + * + * Return value : 0 if ok + * <0 if error detected + */ +int WebRtcIsacfix_EncHistMulti( + Bitstr_enc *streamData, + const int16_t *data, + const uint16_t *const *cdf, + const int16_t lenData); + + +/**************************************************************************** + * WebRtcIsacfix_DecHistBisectMulti(...) + * + * Function to decode more symbols from the arithmetic bytestream, using + * method of bisection. + * C df tables should be of size 2^k-1 (which corresponds to an + * alphabet size of 2^k-2) + * + * Input: + * - streamData : in-/output struct containing bitstream + * - cdf : array of cdf arrays + * - cdfSize : array of cdf table sizes+1 (power of two: 2^k) + * - lenData : data vector length + * + * Output: + * - data : data vector + * + * Return value : number of bytes in the stream + * <0 if error detected + */ +int16_t WebRtcIsacfix_DecHistBisectMulti( + int16_t *data, + Bitstr_dec *streamData, + const uint16_t *const *cdf, + const uint16_t *cdfSize, + const int16_t lenData); + + +/**************************************************************************** + * WebRtcIsacfix_DecHistOneStepMulti(...) + * + * Function to decode more symbols from the arithmetic bytestream, taking + * single step up or down at a time. + * cdf tables can be of arbitrary size, but large tables may take a lot of + * iterations. + * + * Input: + * - streamData : in-/output struct containing bitstream + * - cdf : array of cdf arrays + * - initIndex : vector of initial cdf table search entries + * - lenData : data vector length + * + * Output: + * - data : data vector + * + * Return value : number of bytes in original stream + * <0 if error detected + */ +int16_t WebRtcIsacfix_DecHistOneStepMulti( + int16_t *data, + Bitstr_dec *streamData, + const uint16_t *const *cdf, + const uint16_t *initIndex, + const int16_t lenData); + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ARITH_ROUTINS_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_decoder_isacfix.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_decoder_isacfix.cc new file mode 100644 index 0000000000..21259ee2e2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_decoder_isacfix.cc @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/include/audio_decoder_isacfix.h" + +#include "modules/audio_coding/codecs/isac/audio_decoder_isac_t_impl.h" + +namespace webrtc { + +// Explicit instantiation: +template class AudioDecoderIsacT; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_encoder_isacfix.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_encoder_isacfix.cc new file mode 100644 index 0000000000..0190ab91b6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_encoder_isacfix.cc @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h" + +#include "modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h" + +namespace webrtc { + +// Explicit instantiation: +template class AudioEncoderIsacT; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c new file mode 100644 index 0000000000..d926f47e3a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c @@ -0,0 +1,1037 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * bandwidth_estimator.c + * + * This file contains the code for the Bandwidth Estimator designed + * for iSAC. + * + * NOTE! Castings needed for C55, do not remove! + * + */ + +#include "bandwidth_estimator.h" + +#include "settings.h" +#include "rtc_base/checks.h" + +/* array of quantization levels for bottle neck info; Matlab code: */ +/* sprintf('%4.1ff, ', logspace(log10(5000), log10(40000), 12)) */ +static const int16_t kQRateTable[12] = { + 10000, 11115, 12355, 13733, 15265, 16967, + 18860, 20963, 23301, 25900, 28789, 32000 +}; + +/* 0.1 times the values in the table kQRateTable */ +/* values are in Q16 */ +static const int32_t KQRate01[12] = { + 65536000, 72843264, 80969728, 90000589, 100040704, 111194931, + 123600896, 137383117, 152705434, 169738240, 188671590, 209715200 +}; + +/* Bits per Bytes Seconds + * 8 bits/byte * 1000 msec/sec * 1/framelength (in msec)->bits/byte*sec + * frame length will either be 30 or 60 msec. 8738 is 1/60 in Q19 and 1/30 in Q18 + * The following number is either in Q15 or Q14 depending on the current frame length */ +static const int32_t kBitsByteSec = 4369000; + +/* Received header rate. First value is for 30 ms packets and second for 60 ms */ +static const int16_t kRecHeaderRate[2] = { + 9333, 4666 +}; + +/* Inverted minimum and maximum bandwidth in Q30. + minBwInv 30 ms, maxBwInv 30 ms, + minBwInv 60 ms, maxBwInv 69 ms +*/ +static const int32_t kInvBandwidth[4] = { + 55539, 25978, + 73213, 29284 +}; + +/* Number of samples in 25 msec */ +static const int32_t kSamplesIn25msec = 400; + + +/**************************************************************************** + * WebRtcIsacfix_InitBandwidthEstimator(...) + * + * This function initializes the struct for the bandwidth estimator + * + * Input/Output: + * - bweStr : Struct containing bandwidth information. + * + * Return value : 0 + */ +int32_t WebRtcIsacfix_InitBandwidthEstimator(BwEstimatorstr *bweStr) +{ + bweStr->prevFrameSizeMs = INIT_FRAME_LEN; + bweStr->prevRtpNumber = 0; + bweStr->prevSendTime = 0; + bweStr->prevArrivalTime = 0; + bweStr->prevRtpRate = 1; + bweStr->lastUpdate = 0; + bweStr->lastReduction = 0; + bweStr->countUpdates = -9; + + /* INIT_BN_EST = 20000 + * INIT_BN_EST_Q7 = 2560000 + * INIT_HDR_RATE = 4666 + * INIT_REC_BN_EST_Q5 = 789312 + * + * recBwInv = 1/(INIT_BN_EST + INIT_HDR_RATE) in Q30 + * recBwAvg = INIT_BN_EST + INIT_HDR_RATE in Q5 + */ + bweStr->recBwInv = 43531; + bweStr->recBw = INIT_BN_EST; + bweStr->recBwAvgQ = INIT_BN_EST_Q7; + bweStr->recBwAvg = INIT_REC_BN_EST_Q5; + bweStr->recJitter = (int32_t) 327680; /* 10 in Q15 */ + bweStr->recJitterShortTerm = 0; + bweStr->recJitterShortTermAbs = (int32_t) 40960; /* 5 in Q13 */ + bweStr->recMaxDelay = (int32_t) 10; + bweStr->recMaxDelayAvgQ = (int32_t) 5120; /* 10 in Q9 */ + bweStr->recHeaderRate = INIT_HDR_RATE; + bweStr->countRecPkts = 0; + bweStr->sendBwAvg = INIT_BN_EST_Q7; + bweStr->sendMaxDelayAvg = (int32_t) 5120; /* 10 in Q9 */ + + bweStr->countHighSpeedRec = 0; + bweStr->highSpeedRec = 0; + bweStr->countHighSpeedSent = 0; + bweStr->highSpeedSend = 0; + bweStr->inWaitPeriod = 0; + + /* Find the inverse of the max bw and min bw in Q30 + * (1 / (MAX_ISAC_BW + INIT_HDR_RATE) in Q30 + * (1 / (MIN_ISAC_BW + INIT_HDR_RATE) in Q30 + */ + bweStr->maxBwInv = kInvBandwidth[3]; + bweStr->minBwInv = kInvBandwidth[2]; + + bweStr->external_bw_info.in_use = 0; + + return 0; +} + +/**************************************************************************** + * WebRtcIsacfix_UpdateUplinkBwImpl(...) + * + * This function updates bottle neck rate received from other side in payload + * and calculates a new bottle neck to send to the other side. + * + * Input/Output: + * - bweStr : struct containing bandwidth information. + * - rtpNumber : value from RTP packet, from NetEq + * - frameSize : length of signal frame in ms, from iSAC decoder + * - sendTime : value in RTP header giving send time in samples + * - arrivalTime : value given by timeGetTime() time of arrival in + * samples of packet from NetEq + * - pksize : size of packet in bytes, from NetEq + * - Index : integer (range 0...23) indicating bottle neck & + * jitter as estimated by other side + * + * Return value : 0 if everything went fine, + * -1 otherwise + */ +int32_t WebRtcIsacfix_UpdateUplinkBwImpl(BwEstimatorstr *bweStr, + const uint16_t rtpNumber, + const int16_t frameSize, + const uint32_t sendTime, + const uint32_t arrivalTime, + const size_t pksize, + const uint16_t Index) +{ + uint16_t weight = 0; + uint32_t currBwInv = 0; + uint16_t recRtpRate; + uint32_t arrTimeProj; + int32_t arrTimeDiff; + int32_t arrTimeNoise; + int32_t arrTimeNoiseAbs; + int32_t sendTimeDiff; + + int32_t delayCorrFactor = DELAY_CORRECTION_MED; + int32_t lateDiff = 0; + int16_t immediateSet = 0; + int32_t frameSizeSampl; + + int32_t temp; + int32_t msec; + uint32_t exponent; + uint32_t reductionFactor; + uint32_t numBytesInv; + int32_t sign; + + uint32_t byteSecondsPerBit; + uint32_t tempLower; + uint32_t tempUpper; + int32_t recBwAvgInv; + int32_t numPktsExpected; + + int16_t errCode; + + RTC_DCHECK(!bweStr->external_bw_info.in_use); + + /* UPDATE ESTIMATES FROM OTHER SIDE */ + + /* The function also checks if Index has a valid value */ + errCode = WebRtcIsacfix_UpdateUplinkBwRec(bweStr, Index); + if (errCode <0) { + return(errCode); + } + + + /* UPDATE ESTIMATES ON THIS SIDE */ + + /* Bits per second per byte * 1/30 or 1/60 */ + if (frameSize == 60) { + /* If frameSize changed since last call, from 30 to 60, recalculate some values */ + if ( (frameSize != bweStr->prevFrameSizeMs) && (bweStr->countUpdates > 0)) { + bweStr->countUpdates = 10; + bweStr->recHeaderRate = kRecHeaderRate[1]; + + bweStr->maxBwInv = kInvBandwidth[3]; + bweStr->minBwInv = kInvBandwidth[2]; + bweStr->recBwInv = 1073741824 / (bweStr->recBw + bweStr->recHeaderRate); + } + + /* kBitsByteSec is in Q15 */ + recRtpRate = (int16_t)((kBitsByteSec * pksize) >> 15) + + bweStr->recHeaderRate; + + } else { + /* If frameSize changed since last call, from 60 to 30, recalculate some values */ + if ( (frameSize != bweStr->prevFrameSizeMs) && (bweStr->countUpdates > 0)) { + bweStr->countUpdates = 10; + bweStr->recHeaderRate = kRecHeaderRate[0]; + + bweStr->maxBwInv = kInvBandwidth[1]; + bweStr->minBwInv = kInvBandwidth[0]; + bweStr->recBwInv = 1073741824 / (bweStr->recBw + bweStr->recHeaderRate); + } + + /* kBitsByteSec is in Q14 */ + recRtpRate = (uint16_t)((kBitsByteSec * pksize) >> 14) + + bweStr->recHeaderRate; + } + + + /* Check for timer wrap-around */ + if (arrivalTime < bweStr->prevArrivalTime) { + bweStr->prevArrivalTime = arrivalTime; + bweStr->lastUpdate = arrivalTime; + bweStr->lastReduction = arrivalTime + FS3; + + bweStr->countRecPkts = 0; + + /* store frame size */ + bweStr->prevFrameSizeMs = frameSize; + + /* store far-side transmission rate */ + bweStr->prevRtpRate = recRtpRate; + + /* store far-side RTP time stamp */ + bweStr->prevRtpNumber = rtpNumber; + + return 0; + } + + bweStr->countRecPkts++; + + /* Calculate framesize in msec */ + frameSizeSampl = SAMPLES_PER_MSEC * frameSize; + + /* Check that it's not one of the first 9 packets */ + if ( bweStr->countUpdates > 0 ) { + + /* Stay in Wait Period for 1.5 seconds (no updates in wait period) */ + if(bweStr->inWaitPeriod) { + if ((arrivalTime - bweStr->startWaitPeriod)> FS_1_HALF) { + bweStr->inWaitPeriod = 0; + } + } + + /* If not been updated for a long time, reduce the BN estimate */ + + /* Check send time difference between this packet and previous received */ + sendTimeDiff = sendTime - bweStr->prevSendTime; + if (sendTimeDiff <= frameSizeSampl * 2) { + + /* Only update if 3 seconds has past since last update */ + if ((arrivalTime - bweStr->lastUpdate) > FS3) { + + /* Calculate expected number of received packets since last update */ + numPktsExpected = (arrivalTime - bweStr->lastUpdate) / frameSizeSampl; + + /* If received number of packets is more than 90% of expected (922 = 0.9 in Q10): */ + /* do the update, else not */ + if ((int32_t)bweStr->countRecPkts << 10 > 922 * numPktsExpected) { + /* Q4 chosen to approx dividing by 16 */ + msec = (arrivalTime - bweStr->lastReduction); + + /* the number below represents 13 seconds, highly unlikely + but to insure no overflow when reduction factor is multiplied by recBw inverse */ + if (msec > 208000) { + msec = 208000; + } + + /* Q20 2^(negative number: - 76/1048576) = .99995 + product is Q24 */ + exponent = WEBRTC_SPL_UMUL(0x0000004C, msec); + + /* do the approx with positive exponent so that value is actually rf^-1 + and multiply by bw inverse */ + reductionFactor = WEBRTC_SPL_RSHIFT_U32(0x01000000 | (exponent & 0x00FFFFFF), + WEBRTC_SPL_RSHIFT_U32(exponent, 24)); + + /* reductionFactor in Q13 */ + reductionFactor = WEBRTC_SPL_RSHIFT_U32(reductionFactor, 11); + + if ( reductionFactor != 0 ) { + bweStr->recBwInv = WEBRTC_SPL_MUL((int32_t)bweStr->recBwInv, (int32_t)reductionFactor); + bweStr->recBwInv = (int32_t)bweStr->recBwInv >> 13; + + } else { + static const uint32_t kInitRate = INIT_BN_EST + INIT_HDR_RATE; + /* recBwInv = 1 / kInitRate in Q26 (Q30??)*/ + bweStr->recBwInv = (1073741824 + kInitRate / 2) / kInitRate; + } + + /* reset time-since-update counter */ + bweStr->lastReduction = arrivalTime; + } else { + /* Delay last reduction with 3 seconds */ + bweStr->lastReduction = arrivalTime + FS3; + bweStr->lastUpdate = arrivalTime; + bweStr->countRecPkts = 0; + } + } + } else { + bweStr->lastReduction = arrivalTime + FS3; + bweStr->lastUpdate = arrivalTime; + bweStr->countRecPkts = 0; + } + + + /* update only if previous packet was not lost */ + if ( rtpNumber == bweStr->prevRtpNumber + 1 ) { + arrTimeDiff = arrivalTime - bweStr->prevArrivalTime; + + if (!(bweStr->highSpeedSend && bweStr->highSpeedRec)) { + if (arrTimeDiff > frameSizeSampl) { + if (sendTimeDiff > 0) { + lateDiff = arrTimeDiff - sendTimeDiff - frameSizeSampl * 2; + } else { + lateDiff = arrTimeDiff - frameSizeSampl; + } + + /* 8000 is 1/2 second (in samples at FS) */ + if (lateDiff > 8000) { + delayCorrFactor = (int32_t) DELAY_CORRECTION_MAX; + bweStr->inWaitPeriod = 1; + bweStr->startWaitPeriod = arrivalTime; + immediateSet = 1; + } else if (lateDiff > 5120) { + delayCorrFactor = (int32_t) DELAY_CORRECTION_MED; + immediateSet = 1; + bweStr->inWaitPeriod = 1; + bweStr->startWaitPeriod = arrivalTime; + } + } + } + + if ((bweStr->prevRtpRate > (int32_t)bweStr->recBwAvg >> 5) && + (recRtpRate > (int32_t)bweStr->recBwAvg >> 5) && + !bweStr->inWaitPeriod) { + + /* test if still in initiation period and increment counter */ + if (bweStr->countUpdates++ > 99) { + /* constant weight after initiation part, 0.01 in Q13 */ + weight = (uint16_t) 82; + } else { + /* weight decreases with number of updates, 1/countUpdates in Q13 */ + weight = (uint16_t) WebRtcSpl_DivW32W16( + 8192 + (bweStr->countUpdates >> 1), + (int16_t)bweStr->countUpdates); + } + + /* Bottle Neck Estimation */ + + /* limit outliers, if more than 25 ms too much */ + if (arrTimeDiff > frameSizeSampl + kSamplesIn25msec) { + arrTimeDiff = frameSizeSampl + kSamplesIn25msec; + } + + /* don't allow it to be less than frame rate - 10 ms */ + if (arrTimeDiff < frameSizeSampl - FRAMESAMPLES_10ms) { + arrTimeDiff = frameSizeSampl - FRAMESAMPLES_10ms; + } + + /* compute inverse receiving rate for last packet, in Q19 */ + numBytesInv = (uint16_t) WebRtcSpl_DivW32W16( + (int32_t)(524288 + ((pksize + HEADER_SIZE) >> 1)), + (int16_t)(pksize + HEADER_SIZE)); + + /* 8389 is ~ 1/128000 in Q30 */ + byteSecondsPerBit = (uint32_t)(arrTimeDiff * 8389); + + /* get upper N bits */ + tempUpper = WEBRTC_SPL_RSHIFT_U32(byteSecondsPerBit, 15); + + /* get lower 15 bits */ + tempLower = byteSecondsPerBit & 0x00007FFF; + + tempUpper = WEBRTC_SPL_MUL(tempUpper, numBytesInv); + tempLower = WEBRTC_SPL_MUL(tempLower, numBytesInv); + tempLower = WEBRTC_SPL_RSHIFT_U32(tempLower, 15); + + currBwInv = tempUpper + tempLower; + currBwInv = WEBRTC_SPL_RSHIFT_U32(currBwInv, 4); + + /* Limit inv rate. Note that minBwInv > maxBwInv! */ + if(currBwInv < bweStr->maxBwInv) { + currBwInv = bweStr->maxBwInv; + } else if(currBwInv > bweStr->minBwInv) { + currBwInv = bweStr->minBwInv; + } + + /* update bottle neck rate estimate */ + bweStr->recBwInv = WEBRTC_SPL_UMUL(weight, currBwInv) + + WEBRTC_SPL_UMUL((uint32_t) 8192 - weight, bweStr->recBwInv); + + /* Shift back to Q30 from Q40 (actual used bits shouldn't be more than 27 based on minBwInv) + up to 30 bits used with Q13 weight */ + bweStr->recBwInv = WEBRTC_SPL_RSHIFT_U32(bweStr->recBwInv, 13); + + /* reset time-since-update counter */ + bweStr->lastUpdate = arrivalTime; + bweStr->lastReduction = arrivalTime + FS3; + bweStr->countRecPkts = 0; + + /* to save resolution compute the inverse of recBwAvg in Q26 by left shifting numerator to 2^31 + and NOT right shifting recBwAvg 5 bits to an integer + At max 13 bits are used + shift to Q5 */ + recBwAvgInv = (0x80000000 + bweStr->recBwAvg / 2) / bweStr->recBwAvg; + + /* Calculate Projected arrival time difference */ + + /* The numerator of the quotient can be 22 bits so right shift inv by 4 to avoid overflow + result in Q22 */ + arrTimeProj = WEBRTC_SPL_MUL((int32_t)8000, recBwAvgInv); + /* shift to Q22 */ + arrTimeProj = WEBRTC_SPL_RSHIFT_U32(arrTimeProj, 4); + /* complete calulation */ + arrTimeProj = WEBRTC_SPL_MUL(((int32_t)pksize + HEADER_SIZE), arrTimeProj); + /* shift to Q10 */ + arrTimeProj = WEBRTC_SPL_RSHIFT_U32(arrTimeProj, 12); + + /* difference between projected and actual arrival time differences */ + /* Q9 (only shift arrTimeDiff by 5 to simulate divide by 16 (need to revisit if change sampling rate) DH */ + if ((arrTimeDiff << 6) > (int32_t)arrTimeProj) { + arrTimeNoise = (arrTimeDiff << 6) - arrTimeProj; + sign = 1; + } else { + arrTimeNoise = arrTimeProj - (arrTimeDiff << 6); + sign = -1; + } + + /* Q9 */ + arrTimeNoiseAbs = arrTimeNoise; + + /* long term averaged absolute jitter, Q15 */ + weight >>= 3; + bweStr->recJitter = weight * (arrTimeNoiseAbs << 5) + + (1024 - weight) * bweStr->recJitter; + + /* remove the fractional portion */ + bweStr->recJitter >>= 10; + + /* Maximum jitter is 10 msec in Q15 */ + if (bweStr->recJitter > (int32_t)327680) { + bweStr->recJitter = (int32_t)327680; + } + + /* short term averaged absolute jitter */ + /* Calculation in Q13 products in Q23 */ + bweStr->recJitterShortTermAbs = 51 * (arrTimeNoiseAbs << 3) + + WEBRTC_SPL_MUL(973, bweStr->recJitterShortTermAbs); + bweStr->recJitterShortTermAbs >>= 10; + + /* short term averaged jitter */ + /* Calculation in Q13 products in Q23 */ + bweStr->recJitterShortTerm = 205 * (arrTimeNoise << 3) * sign + + WEBRTC_SPL_MUL(3891, bweStr->recJitterShortTerm); + + if (bweStr->recJitterShortTerm < 0) { + temp = -bweStr->recJitterShortTerm; + temp >>= 12; + bweStr->recJitterShortTerm = -temp; + } else { + bweStr->recJitterShortTerm >>= 12; + } + } + } + } else { + /* reset time-since-update counter when receiving the first 9 packets */ + bweStr->lastUpdate = arrivalTime; + bweStr->lastReduction = arrivalTime + FS3; + bweStr->countRecPkts = 0; + bweStr->countUpdates++; + } + + /* Limit to minimum or maximum bottle neck rate (in Q30) */ + if (bweStr->recBwInv > bweStr->minBwInv) { + bweStr->recBwInv = bweStr->minBwInv; + } else if (bweStr->recBwInv < bweStr->maxBwInv) { + bweStr->recBwInv = bweStr->maxBwInv; + } + + + /* store frame length */ + bweStr->prevFrameSizeMs = frameSize; + + /* store far-side transmission rate */ + bweStr->prevRtpRate = recRtpRate; + + /* store far-side RTP time stamp */ + bweStr->prevRtpNumber = rtpNumber; + + /* Replace bweStr->recMaxDelay by the new value (atomic operation) */ + if (bweStr->prevArrivalTime != 0xffffffff) { + bweStr->recMaxDelay = WEBRTC_SPL_MUL(3, bweStr->recJitter); + } + + /* store arrival time stamp */ + bweStr->prevArrivalTime = arrivalTime; + bweStr->prevSendTime = sendTime; + + /* Replace bweStr->recBw by the new value */ + bweStr->recBw = 1073741824 / bweStr->recBwInv - bweStr->recHeaderRate; + + if (immediateSet) { + /* delay correction factor is in Q10 */ + bweStr->recBw = WEBRTC_SPL_UMUL(delayCorrFactor, bweStr->recBw); + bweStr->recBw = WEBRTC_SPL_RSHIFT_U32(bweStr->recBw, 10); + + if (bweStr->recBw < (int32_t) MIN_ISAC_BW) { + bweStr->recBw = (int32_t) MIN_ISAC_BW; + } + + bweStr->recBwAvg = (bweStr->recBw + bweStr->recHeaderRate) << 5; + + bweStr->recBwAvgQ = bweStr->recBw << 7; + + bweStr->recJitterShortTerm = 0; + + bweStr->recBwInv = 1073741824 / (bweStr->recBw + bweStr->recHeaderRate); + + immediateSet = 0; + } + + + return 0; +} + +/* This function updates the send bottle neck rate */ +/* Index - integer (range 0...23) indicating bottle neck & jitter as estimated by other side */ +/* returns 0 if everything went fine, -1 otherwise */ +int16_t WebRtcIsacfix_UpdateUplinkBwRec(BwEstimatorstr *bweStr, + const int16_t Index) +{ + uint16_t RateInd; + + RTC_DCHECK(!bweStr->external_bw_info.in_use); + + if ( (Index < 0) || (Index > 23) ) { + return -ISAC_RANGE_ERROR_BW_ESTIMATOR; + } + + /* UPDATE ESTIMATES FROM OTHER SIDE */ + + if ( Index > 11 ) { + RateInd = Index - 12; + /* compute the jitter estimate as decoded on the other side in Q9 */ + /* sendMaxDelayAvg = 0.9 * sendMaxDelayAvg + 0.1 * MAX_ISAC_MD */ + bweStr->sendMaxDelayAvg = WEBRTC_SPL_MUL(461, bweStr->sendMaxDelayAvg) + + 51 * (MAX_ISAC_MD << 9); + bweStr->sendMaxDelayAvg >>= 9; + + } else { + RateInd = Index; + /* compute the jitter estimate as decoded on the other side in Q9 */ + /* sendMaxDelayAvg = 0.9 * sendMaxDelayAvg + 0.1 * MIN_ISAC_MD */ + bweStr->sendMaxDelayAvg = WEBRTC_SPL_MUL(461, bweStr->sendMaxDelayAvg) + + 51 * (MIN_ISAC_MD << 9); + bweStr->sendMaxDelayAvg >>= 9; + + } + + + /* compute the BN estimate as decoded on the other side */ + /* sendBwAvg = 0.9 * sendBwAvg + 0.1 * kQRateTable[RateInd]; */ + bweStr->sendBwAvg = 461 * bweStr->sendBwAvg + + 51 * ((uint32_t)kQRateTable[RateInd] << 7); + bweStr->sendBwAvg = WEBRTC_SPL_RSHIFT_U32(bweStr->sendBwAvg, 9); + + + if (WEBRTC_SPL_RSHIFT_U32(bweStr->sendBwAvg, 7) > 28000 && !bweStr->highSpeedSend) { + bweStr->countHighSpeedSent++; + + /* approx 2 seconds with 30ms frames */ + if (bweStr->countHighSpeedSent >= 66) { + bweStr->highSpeedSend = 1; + } + } else if (!bweStr->highSpeedSend) { + bweStr->countHighSpeedSent = 0; + } + + return 0; +} + +/**************************************************************************** + * WebRtcIsacfix_GetDownlinkBwIndexImpl(...) + * + * This function calculates and returns the bandwidth/jitter estimation code + * (integer 0...23) to put in the sending iSAC payload. + * + * Input: + * - bweStr : BWE struct + * + * Return: + * bandwith and jitter index (0..23) + */ +uint16_t WebRtcIsacfix_GetDownlinkBwIndexImpl(BwEstimatorstr *bweStr) +{ + int32_t rate; + int32_t maxDelay; + uint16_t rateInd; + uint16_t maxDelayBit; + int32_t tempTerm1; + int32_t tempTerm2; + int32_t tempTermX; + int32_t tempTermY; + int32_t tempMin; + int32_t tempMax; + + if (bweStr->external_bw_info.in_use) + return bweStr->external_bw_info.bottleneck_idx; + + /* Get Rate Index */ + + /* Get unquantized rate. Always returns 10000 <= rate <= 32000 */ + rate = WebRtcIsacfix_GetDownlinkBandwidth(bweStr); + + /* Compute the averaged BN estimate on this side */ + + /* recBwAvg = 0.9 * recBwAvg + 0.1 * (rate + bweStr->recHeaderRate), 0.9 and 0.1 in Q9 */ + bweStr->recBwAvg = 922 * bweStr->recBwAvg + + 102 * (((uint32_t)rate + bweStr->recHeaderRate) << 5); + bweStr->recBwAvg = WEBRTC_SPL_RSHIFT_U32(bweStr->recBwAvg, 10); + + /* Find quantization index that gives the closest rate after averaging. + * Note that we don't need to check the last value, rate <= kQRateTable[11], + * because we will use rateInd = 11 even if rate > kQRateTable[11]. */ + for (rateInd = 1; rateInd < 11; rateInd++) { + if (rate <= kQRateTable[rateInd]){ + break; + } + } + + /* find closest quantization index, and update quantized average by taking: */ + /* 0.9*recBwAvgQ + 0.1*kQRateTable[rateInd] */ + + /* 0.9 times recBwAvgQ in Q16 */ + /* 461/512 - 25/65536 =0.900009 */ + tempTerm1 = WEBRTC_SPL_MUL(bweStr->recBwAvgQ, 25); + tempTerm1 >>= 7; + tempTermX = WEBRTC_SPL_UMUL(461, bweStr->recBwAvgQ) - tempTerm1; + + /* rate in Q16 */ + tempTermY = rate << 16; + + /* 0.1 * kQRateTable[rateInd] = KQRate01[rateInd] */ + tempTerm1 = tempTermX + KQRate01[rateInd] - tempTermY; + tempTerm2 = tempTermY - tempTermX - KQRate01[rateInd-1]; + + /* Compare (0.9 * recBwAvgQ + 0.1 * kQRateTable[rateInd] - rate) > + (rate - 0.9 * recBwAvgQ - 0.1 * kQRateTable[rateInd-1]) */ + if (tempTerm1 > tempTerm2) { + rateInd--; + } + + /* Update quantized average by taking: */ + /* 0.9*recBwAvgQ + 0.1*kQRateTable[rateInd] */ + + /* Add 0.1 times kQRateTable[rateInd], in Q16 */ + tempTermX += KQRate01[rateInd]; + + /* Shift back to Q7 */ + bweStr->recBwAvgQ = tempTermX >> 9; + + /* Count consecutive received bandwidth above 28000 kbps (28000 in Q7 = 3584000) */ + /* If 66 high estimates in a row, set highSpeedRec to one */ + /* 66 corresponds to ~2 seconds in 30 msec mode */ + if ((bweStr->recBwAvgQ > 3584000) && !bweStr->highSpeedRec) { + bweStr->countHighSpeedRec++; + if (bweStr->countHighSpeedRec >= 66) { + bweStr->highSpeedRec = 1; + } + } else if (!bweStr->highSpeedRec) { + bweStr->countHighSpeedRec = 0; + } + + /* Get Max Delay Bit */ + + /* get unquantized max delay */ + maxDelay = WebRtcIsacfix_GetDownlinkMaxDelay(bweStr); + + /* Update quantized max delay average */ + tempMax = 652800; /* MAX_ISAC_MD * 0.1 in Q18 */ + tempMin = 130560; /* MIN_ISAC_MD * 0.1 in Q18 */ + tempTermX = WEBRTC_SPL_MUL((int32_t)bweStr->recMaxDelayAvgQ, (int32_t)461); + tempTermY = maxDelay << 18; + + tempTerm1 = tempTermX + tempMax - tempTermY; + tempTerm2 = tempTermY - tempTermX - tempMin; + + if ( tempTerm1 > tempTerm2) { + maxDelayBit = 0; + tempTerm1 = tempTermX + tempMin; + + /* update quantized average, shift back to Q9 */ + bweStr->recMaxDelayAvgQ = tempTerm1 >> 9; + } else { + maxDelayBit = 12; + tempTerm1 = tempTermX + tempMax; + + /* update quantized average, shift back to Q9 */ + bweStr->recMaxDelayAvgQ = tempTerm1 >> 9; + } + + /* Return bandwitdh and jitter index (0..23) */ + return (uint16_t)(rateInd + maxDelayBit); +} + +/* get the bottle neck rate from far side to here, as estimated on this side */ +uint16_t WebRtcIsacfix_GetDownlinkBandwidth(const BwEstimatorstr *bweStr) +{ + uint32_t recBw; + int32_t jitter_sign; /* Q8 */ + int32_t bw_adjust; /* Q16 */ + int32_t rec_jitter_short_term_abs_inv; /* Q18 */ + int32_t temp; + + RTC_DCHECK(!bweStr->external_bw_info.in_use); + + /* Q18 rec jitter short term abs is in Q13, multiply it by 2^13 to save precision + 2^18 then needs to be shifted 13 bits to 2^31 */ + rec_jitter_short_term_abs_inv = 0x80000000u / bweStr->recJitterShortTermAbs; + + /* Q27 = 9 + 18 */ + jitter_sign = (bweStr->recJitterShortTerm >> 4) * + rec_jitter_short_term_abs_inv; + + if (jitter_sign < 0) { + temp = -jitter_sign; + temp >>= 19; + jitter_sign = -temp; + } else { + jitter_sign >>= 19; + } + + /* adjust bw proportionally to negative average jitter sign */ + //bw_adjust = 1.0f - jitter_sign * (0.15f + 0.15f * jitter_sign * jitter_sign); + //Q8 -> Q16 .15 +.15 * jitter^2 first term is .15 in Q16 latter term is Q8*Q8*Q8 + //38 in Q8 ~.15 9830 in Q16 ~.15 + temp = 9830 + ((38 * jitter_sign * jitter_sign) >> 8); + + if (jitter_sign < 0) { + temp = WEBRTC_SPL_MUL(jitter_sign, temp); + temp = -temp; + temp >>= 8; + bw_adjust = (uint32_t)65536 + temp; /* (1 << 16) + temp; */ + } else { + /* (1 << 16) - ((jitter_sign * temp) >> 8); */ + bw_adjust = 65536 - ((jitter_sign * temp) >> 8); + } + + //make sure following multiplication won't overflow + //bw adjust now Q14 + bw_adjust >>= 2; // See if good resolution is maintained. + + /* adjust Rate if jitter sign is mostly constant */ + recBw = WEBRTC_SPL_UMUL(bweStr->recBw, bw_adjust); + + recBw >>= 14; + + /* limit range of bottle neck rate */ + if (recBw < MIN_ISAC_BW) { + recBw = MIN_ISAC_BW; + } else if (recBw > MAX_ISAC_BW) { + recBw = MAX_ISAC_BW; + } + + return (uint16_t) recBw; +} + +/* Returns the mmax delay (in ms) */ +int16_t WebRtcIsacfix_GetDownlinkMaxDelay(const BwEstimatorstr *bweStr) +{ + int16_t recMaxDelay = (int16_t)(bweStr->recMaxDelay >> 15); + + RTC_DCHECK(!bweStr->external_bw_info.in_use); + + /* limit range of jitter estimate */ + if (recMaxDelay < MIN_ISAC_MD) { + recMaxDelay = MIN_ISAC_MD; + } else if (recMaxDelay > MAX_ISAC_MD) { + recMaxDelay = MAX_ISAC_MD; + } + + return recMaxDelay; +} + +/* Clamp val to the closed interval [min,max]. */ +static int16_t clamp(int16_t val, int16_t min, int16_t max) { + RTC_DCHECK_LE(min, max); + return val < min ? min : (val > max ? max : val); +} + +int16_t WebRtcIsacfix_GetUplinkBandwidth(const BwEstimatorstr* bweStr) { + return bweStr->external_bw_info.in_use + ? bweStr->external_bw_info.send_bw_avg + : clamp(bweStr->sendBwAvg >> 7, MIN_ISAC_BW, MAX_ISAC_BW); +} + +int16_t WebRtcIsacfix_GetUplinkMaxDelay(const BwEstimatorstr* bweStr) { + return bweStr->external_bw_info.in_use + ? bweStr->external_bw_info.send_max_delay_avg + : clamp(bweStr->sendMaxDelayAvg >> 9, MIN_ISAC_MD, MAX_ISAC_MD); +} + +void WebRtcIsacfixBw_GetBandwidthInfo(BwEstimatorstr* bweStr, + IsacBandwidthInfo* bwinfo) { + RTC_DCHECK(!bweStr->external_bw_info.in_use); + bwinfo->in_use = 1; + bwinfo->send_bw_avg = WebRtcIsacfix_GetUplinkBandwidth(bweStr); + bwinfo->send_max_delay_avg = WebRtcIsacfix_GetUplinkMaxDelay(bweStr); + bwinfo->bottleneck_idx = WebRtcIsacfix_GetDownlinkBwIndexImpl(bweStr); + bwinfo->jitter_info = 0; // Not used. +} + +void WebRtcIsacfixBw_SetBandwidthInfo(BwEstimatorstr* bweStr, + const IsacBandwidthInfo* bwinfo) { + memcpy(&bweStr->external_bw_info, bwinfo, + sizeof bweStr->external_bw_info); +} + +/* + * update long-term average bitrate and amount of data in buffer + * returns minimum payload size (bytes) + */ +uint16_t WebRtcIsacfix_GetMinBytes(RateModel *State, + int16_t StreamSize, /* bytes in bitstream */ + const int16_t FrameSamples, /* samples per frame */ + const int16_t BottleNeck, /* bottle neck rate; excl headers (bps) */ + const int16_t DelayBuildUp) /* max delay from bottle neck buffering (ms) */ +{ + int32_t MinRate = 0; + uint16_t MinBytes; + int16_t TransmissionTime; + int32_t inv_Q12; + int32_t den; + + + /* first 10 packets @ low rate, then INIT_BURST_LEN packets @ fixed rate of INIT_RATE bps */ + if (State->InitCounter > 0) { + if (State->InitCounter-- <= INIT_BURST_LEN) { + MinRate = INIT_RATE; + } else { + MinRate = 0; + } + } else { + /* handle burst */ + if (State->BurstCounter) { + if (State->StillBuffered < + (((512 - 512 / BURST_LEN) * DelayBuildUp) >> 9)) { + /* max bps derived from BottleNeck and DelayBuildUp values */ + inv_Q12 = 4096 / (BURST_LEN * FrameSamples); + MinRate = (512 + SAMPLES_PER_MSEC * ((DelayBuildUp * inv_Q12) >> 3)) * + BottleNeck; + } else { + /* max bps derived from StillBuffered and DelayBuildUp values */ + inv_Q12 = 4096 / FrameSamples; + if (DelayBuildUp > State->StillBuffered) { + MinRate = (512 + SAMPLES_PER_MSEC * (((DelayBuildUp - + State->StillBuffered) * inv_Q12) >> 3)) * BottleNeck; + } else if ((den = WEBRTC_SPL_MUL(SAMPLES_PER_MSEC, (State->StillBuffered - DelayBuildUp))) >= FrameSamples) { + /* MinRate will be negative here */ + MinRate = 0; + } else { + MinRate = (512 - ((den * inv_Q12) >> 3)) * BottleNeck; + } + //if (MinRate < 1.04 * BottleNeck) + // MinRate = 1.04 * BottleNeck; + //Q9 + if (MinRate < WEBRTC_SPL_MUL(532, BottleNeck)) { + MinRate += WEBRTC_SPL_MUL(22, BottleNeck); + } + } + + State->BurstCounter--; + } + } + + + /* convert rate from bits/second to bytes/packet */ + //round and shift before conversion + MinRate += 256; + MinRate >>= 9; + MinBytes = MinRate * FrameSamples / FS8; + + /* StreamSize will be adjusted if less than MinBytes */ + if (StreamSize < MinBytes) { + StreamSize = MinBytes; + } + + /* keep track of when bottle neck was last exceeded by at least 1% */ + //517/512 ~ 1.01 + if ((StreamSize * (int32_t)FS8) / FrameSamples > (517 * BottleNeck) >> 9) { + if (State->PrevExceed) { + /* bottle_neck exceded twice in a row, decrease ExceedAgo */ + State->ExceedAgo -= BURST_INTERVAL / (BURST_LEN - 1); + if (State->ExceedAgo < 0) { + State->ExceedAgo = 0; + } + } else { + State->ExceedAgo += FrameSamples / SAMPLES_PER_MSEC; /* ms */ + State->PrevExceed = 1; + } + } else { + State->PrevExceed = 0; + State->ExceedAgo += FrameSamples / SAMPLES_PER_MSEC; /* ms */ + } + + /* set burst flag if bottle neck not exceeded for long time */ + if ((State->ExceedAgo > BURST_INTERVAL) && (State->BurstCounter == 0)) { + if (State->PrevExceed) { + State->BurstCounter = BURST_LEN - 1; + } else { + State->BurstCounter = BURST_LEN; + } + } + + + /* Update buffer delay */ + TransmissionTime = (StreamSize * 8000) / BottleNeck; /* ms */ + State->StillBuffered += TransmissionTime; + State->StillBuffered -= FrameSamples / SAMPLES_PER_MSEC; /* ms */ + if (State->StillBuffered < 0) { + State->StillBuffered = 0; + } + + if (State->StillBuffered > 2000) { + State->StillBuffered = 2000; + } + + return MinBytes; +} + + +/* + * update long-term average bitrate and amount of data in buffer + */ +void WebRtcIsacfix_UpdateRateModel(RateModel *State, + int16_t StreamSize, /* bytes in bitstream */ + const int16_t FrameSamples, /* samples per frame */ + const int16_t BottleNeck) /* bottle neck rate; excl headers (bps) */ +{ + const int16_t TransmissionTime = (StreamSize * 8000) / BottleNeck; /* ms */ + + /* avoid the initial "high-rate" burst */ + State->InitCounter = 0; + + /* Update buffer delay */ + State->StillBuffered += TransmissionTime; + State->StillBuffered -= FrameSamples >> 4; /* ms */ + if (State->StillBuffered < 0) { + State->StillBuffered = 0; + } + +} + + +void WebRtcIsacfix_InitRateModel(RateModel *State) +{ + State->PrevExceed = 0; /* boolean */ + State->ExceedAgo = 0; /* ms */ + State->BurstCounter = 0; /* packets */ + State->InitCounter = INIT_BURST_LEN + 10; /* packets */ + State->StillBuffered = 1; /* ms */ +} + + + + + +int16_t WebRtcIsacfix_GetNewFrameLength(int16_t bottle_neck, int16_t current_framesamples) +{ + int16_t new_framesamples; + + new_framesamples = current_framesamples; + + /* find new framelength */ + switch(current_framesamples) { + case 480: + if (bottle_neck < Thld_30_60) { + new_framesamples = 960; + } + break; + case 960: + if (bottle_neck >= Thld_60_30) { + new_framesamples = 480; + } + break; + default: + new_framesamples = -1; /* Error */ + } + + return new_framesamples; +} + +int16_t WebRtcIsacfix_GetSnr(int16_t bottle_neck, int16_t framesamples) +{ + int16_t s2nr = 0; + + /* find new SNR value */ + //consider BottleNeck to be in Q10 ( * 1 in Q10) + switch(framesamples) { + // TODO(bjornv): The comments below confuses me. I don't know if there is a + // difference between frame lengths (in which case the implementation is + // wrong), or if it is frame length independent in which case we should + // correct the comment and simplify the implementation. + case 480: + /*s2nr = -1*(a_30 << 10) + ((b_30 * bottle_neck) >> 10);*/ + s2nr = -22500 + (int16_t)(500 * bottle_neck >> 10); + break; + case 960: + /*s2nr = -1*(a_60 << 10) + ((b_60 * bottle_neck) >> 10);*/ + s2nr = -22500 + (int16_t)(500 * bottle_neck >> 10); + break; + default: + s2nr = -1; /* Error */ + } + + return s2nr; //return in Q10 + +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h new file mode 100644 index 0000000000..f80a5876fa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * bandwidth_estimator.h + * + * This header file contains the API for the Bandwidth Estimator + * designed for iSAC. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_BANDWIDTH_ESTIMATOR_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_BANDWIDTH_ESTIMATOR_H_ + +#include "structs.h" + + +/**************************************************************************** + * WebRtcIsacfix_InitBandwidthEstimator(...) + * + * This function initializes the struct for the bandwidth estimator + * + * Input/Output: + * - bwest_str : Struct containing bandwidth information. + * + * Return value : 0 + */ + +int32_t WebRtcIsacfix_InitBandwidthEstimator(BwEstimatorstr *bwest_str); + + +/**************************************************************************** + * WebRtcIsacfix_UpdateUplinkBwImpl(...) + * + * This function updates bottle neck rate received from other side in payload + * and calculates a new bottle neck to send to the other side. + * + * Input/Output: + * - bweStr : struct containing bandwidth information. + * - rtpNumber : value from RTP packet, from NetEq + * - frameSize : length of signal frame in ms, from iSAC decoder + * - sendTime : value in RTP header giving send time in samples + * - arrivalTime : value given by timeGetTime() time of arrival in + * samples of packet from NetEq + * - pksize : size of packet in bytes, from NetEq + * - Index : integer (range 0...23) indicating bottle neck & + * jitter as estimated by other side + * + * Return value : 0 if everything went fine, + * -1 otherwise + */ + +int32_t WebRtcIsacfix_UpdateUplinkBwImpl(BwEstimatorstr *bwest_str, + const uint16_t rtp_number, + const int16_t frameSize, + const uint32_t send_ts, + const uint32_t arr_ts, + const size_t pksize, + const uint16_t Index); + +/* Update receiving estimates. Used when we only receive BWE index, no iSAC data packet. */ +int16_t WebRtcIsacfix_UpdateUplinkBwRec(BwEstimatorstr *bwest_str, + const int16_t Index); + +/**************************************************************************** + * WebRtcIsacfix_GetDownlinkBwIndexImpl(...) + * + * This function calculates and returns the bandwidth/jitter estimation code + * (integer 0...23) to put in the sending iSAC payload. + * + * Input: + * - bweStr : BWE struct + * + * Return: + * bandwith and jitter index (0..23) + */ +uint16_t WebRtcIsacfix_GetDownlinkBwIndexImpl(BwEstimatorstr *bwest_str); + +/* Returns the bandwidth estimation (in bps) */ +uint16_t WebRtcIsacfix_GetDownlinkBandwidth(const BwEstimatorstr *bwest_str); + +/* Returns the bandwidth that iSAC should send with in bps */ +int16_t WebRtcIsacfix_GetUplinkBandwidth(const BwEstimatorstr *bwest_str); + +/* Returns the max delay (in ms) */ +int16_t WebRtcIsacfix_GetDownlinkMaxDelay(const BwEstimatorstr *bwest_str); + +/* Returns the max delay value from the other side in ms */ +int16_t WebRtcIsacfix_GetUplinkMaxDelay(const BwEstimatorstr *bwest_str); + +/* Fills in an IsacExternalBandwidthInfo struct. */ +void WebRtcIsacfixBw_GetBandwidthInfo(BwEstimatorstr* bwest_str, + IsacBandwidthInfo* bwinfo); + +/* Uses the values from an IsacExternalBandwidthInfo struct. */ +void WebRtcIsacfixBw_SetBandwidthInfo(BwEstimatorstr* bwest_str, + const IsacBandwidthInfo* bwinfo); + +/* + * update amount of data in bottle neck buffer and burst handling + * returns minimum payload size (bytes) + */ +uint16_t WebRtcIsacfix_GetMinBytes(RateModel *State, + int16_t StreamSize, /* bytes in bitstream */ + const int16_t FrameLen, /* ms per frame */ + const int16_t BottleNeck, /* bottle neck rate; excl headers (bps) */ + const int16_t DelayBuildUp); /* max delay from bottle neck buffering (ms) */ + +/* + * update long-term average bitrate and amount of data in buffer + */ +void WebRtcIsacfix_UpdateRateModel(RateModel *State, + int16_t StreamSize, /* bytes in bitstream */ + const int16_t FrameSamples, /* samples per frame */ + const int16_t BottleNeck); /* bottle neck rate; excl headers (bps) */ + + +void WebRtcIsacfix_InitRateModel(RateModel *State); + +/* Returns the new framelength value (input argument: bottle_neck) */ +int16_t WebRtcIsacfix_GetNewFrameLength(int16_t bottle_neck, int16_t current_framelength); + +/* Returns the new SNR value (input argument: bottle_neck) */ +//returns snr in Q10 +int16_t WebRtcIsacfix_GetSnr(int16_t bottle_neck, int16_t framesamples); + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_BANDWIDTH_ESTIMATOR_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/codec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/codec.h new file mode 100644 index 0000000000..88d40551f2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/codec.h @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * codec.h + * + * This header file contains the calls to the internal encoder + * and decoder functions. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_CODEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_CODEC_H_ + +#include "structs.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int WebRtcIsacfix_EstimateBandwidth(BwEstimatorstr* bwest_str, + Bitstr_dec* streamdata, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts); + +int WebRtcIsacfix_DecodeImpl(int16_t* signal_out16, + IsacFixDecoderInstance* ISACdec_obj, + size_t* current_framesamples); + +void WebRtcIsacfix_DecodePlcImpl(int16_t* decoded, + IsacFixDecoderInstance* ISACdec_obj, + size_t* current_framesample ); + +int WebRtcIsacfix_EncodeImpl(int16_t* in, + IsacFixEncoderInstance* ISACenc_obj, + BwEstimatorstr* bw_estimatordata, + int16_t CodingMode); + +int WebRtcIsacfix_EncodeStoredData(IsacFixEncoderInstance* ISACenc_obj, + int BWnumber, + float scale); + +/* initialization functions */ + +void WebRtcIsacfix_InitMaskingEnc(MaskFiltstr_enc* maskdata); +void WebRtcIsacfix_InitMaskingDec(MaskFiltstr_dec* maskdata); + +void WebRtcIsacfix_InitPreFilterbank(PreFiltBankstr* prefiltdata); + +void WebRtcIsacfix_InitPostFilterbank(PostFiltBankstr* postfiltdata); + +void WebRtcIsacfix_InitPitchFilter(PitchFiltstr* pitchfiltdata); + +void WebRtcIsacfix_InitPitchAnalysis(PitchAnalysisStruct* State); + +void WebRtcIsacfix_InitPlc(PLCstr* State); + + +/* transform functions */ + +void WebRtcIsacfix_InitTransform(); + +typedef void (*Time2Spec)(int16_t* inre1Q9, + int16_t* inre2Q9, + int16_t* outre, + int16_t* outim); +typedef void (*Spec2Time)(int16_t* inreQ7, + int16_t* inimQ7, + int32_t* outre1Q16, + int32_t* outre2Q16); + +extern Time2Spec WebRtcIsacfix_Time2Spec; +extern Spec2Time WebRtcIsacfix_Spec2Time; + +void WebRtcIsacfix_Time2SpecC(int16_t* inre1Q9, + int16_t* inre2Q9, + int16_t* outre, + int16_t* outim); +void WebRtcIsacfix_Spec2TimeC(int16_t* inreQ7, + int16_t* inimQ7, + int32_t* outre1Q16, + int32_t* outre2Q16); + +#if defined(WEBRTC_HAS_NEON) +void WebRtcIsacfix_Time2SpecNeon(int16_t* inre1Q9, + int16_t* inre2Q9, + int16_t* outre, + int16_t* outim); +void WebRtcIsacfix_Spec2TimeNeon(int16_t* inreQ7, + int16_t* inimQ7, + int32_t* outre1Q16, + int32_t* outre2Q16); +#endif + +#if defined(MIPS32_LE) +void WebRtcIsacfix_Time2SpecMIPS(int16_t* inre1Q9, + int16_t* inre2Q9, + int16_t* outre, + int16_t* outim); +void WebRtcIsacfix_Spec2TimeMIPS(int16_t* inreQ7, + int16_t* inimQ7, + int32_t* outre1Q16, + int32_t* outre2Q16); +#endif + +/* filterbank functions */ + +void WebRtcIsacfix_SplitAndFilter1(int16_t* in, + int16_t* LP16, + int16_t* HP16, + PreFiltBankstr* prefiltdata); + +void WebRtcIsacfix_FilterAndCombine1(int16_t* tempin_ch1, + int16_t* tempin_ch2, + int16_t* out16, + PostFiltBankstr* postfiltdata); + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + +void WebRtcIsacfix_SplitAndFilter2(int16_t* in, + int16_t* LP16, + int16_t* HP16, + PreFiltBankstr* prefiltdata); + +void WebRtcIsacfix_FilterAndCombine2(int16_t* tempin_ch1, + int16_t* tempin_ch2, + int16_t* out16, + PostFiltBankstr* postfiltdata, + int16_t len); + +#endif + +/* normalized lattice filters */ + +void WebRtcIsacfix_NormLatticeFilterMa(size_t orderCoef, + int32_t* stateGQ15, + int16_t* lat_inQ0, + int16_t* filt_coefQ15, + int32_t* gain_lo_hiQ17, + int16_t lo_hi, + int16_t* lat_outQ9); + +void WebRtcIsacfix_NormLatticeFilterAr(size_t orderCoef, + int16_t* stateGQ0, + int32_t* lat_inQ25, + int16_t* filt_coefQ15, + int32_t* gain_lo_hiQ17, + int16_t lo_hi, + int16_t* lat_outQ0); + +/* TODO(kma): Remove the following functions into individual header files. */ + +/* Internal functions in both C and ARM Neon versions */ + +int WebRtcIsacfix_AutocorrC(int32_t* __restrict r, + const int16_t* __restrict x, + int16_t N, + int16_t order, + int16_t* __restrict scale); + +void WebRtcIsacfix_FilterMaLoopC(int16_t input0, + int16_t input1, + int32_t input2, + int32_t* ptr0, + int32_t* ptr1, + int32_t* ptr2); + +#if defined(WEBRTC_HAS_NEON) +int WebRtcIsacfix_AutocorrNeon(int32_t* __restrict r, + const int16_t* __restrict x, + int16_t N, + int16_t order, + int16_t* __restrict scale); + +void WebRtcIsacfix_FilterMaLoopNeon(int16_t input0, + int16_t input1, + int32_t input2, + int32_t* ptr0, + int32_t* ptr1, + int32_t* ptr2); +#endif + +#if defined(MIPS32_LE) +int WebRtcIsacfix_AutocorrMIPS(int32_t* __restrict r, + const int16_t* __restrict x, + int16_t N, + int16_t order, + int16_t* __restrict scale); + +void WebRtcIsacfix_FilterMaLoopMIPS(int16_t input0, + int16_t input1, + int32_t input2, + int32_t* ptr0, + int32_t* ptr1, + int32_t* ptr2); +#endif + +/* Function pointers associated with the above functions. */ + +typedef int (*AutocorrFix)(int32_t* __restrict r, + const int16_t* __restrict x, + int16_t N, + int16_t order, + int16_t* __restrict scale); +extern AutocorrFix WebRtcIsacfix_AutocorrFix; + +typedef void (*FilterMaLoopFix)(int16_t input0, + int16_t input1, + int32_t input2, + int32_t* ptr0, + int32_t* ptr1, + int32_t* ptr2); +extern FilterMaLoopFix WebRtcIsacfix_FilterMaLoopFix; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_CODEC_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode.c new file mode 100644 index 0000000000..cf2d507cb5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * decode.c + * + * This C file contains the internal decoding function. + * + */ + +#include + +#include "bandwidth_estimator.h" +#include "codec.h" +#include "entropy_coding.h" +#include "pitch_estimator.h" +#include "settings.h" +#include "structs.h" + + + + +int WebRtcIsacfix_DecodeImpl(int16_t* signal_out16, + IsacFixDecoderInstance* ISACdec_obj, + size_t* current_framesamples) +{ + int k; + int err; + int16_t BWno; + int len = 0; + + int16_t model; + + + int16_t Vector_Word16_1[FRAMESAMPLES/2]; + int16_t Vector_Word16_2[FRAMESAMPLES/2]; + + int32_t Vector_Word32_1[FRAMESAMPLES/2]; + int32_t Vector_Word32_2[FRAMESAMPLES/2]; + + int16_t lofilt_coefQ15[ORDERLO*SUBFRAMES]; //refl. coeffs + int16_t hifilt_coefQ15[ORDERHI*SUBFRAMES]; //refl. coeffs + int32_t gain_lo_hiQ17[2*SUBFRAMES]; + + int16_t PitchLags_Q7[PITCH_SUBFRAMES]; + int16_t PitchGains_Q12[PITCH_SUBFRAMES]; + int16_t AvgPitchGain_Q12; + + int16_t tmp_1, tmp_2; + int32_t tmp32a; + int16_t gainQ13; + + + size_t frame_nb; /* counter */ + size_t frame_mode; /* 0 for 30ms, 1 for 60ms */ + static const size_t kProcessedSamples = 480; /* 480 (for both 30, 60 ms) */ + + /* PLC */ + int16_t overlapWin[ 240 ]; + + (ISACdec_obj->bitstr_obj).W_upper = 0xFFFFFFFF; + (ISACdec_obj->bitstr_obj).streamval = 0; + (ISACdec_obj->bitstr_obj).stream_index = 0; + (ISACdec_obj->bitstr_obj).full = 1; + + + /* decode framelength and BW estimation - not used, only for stream pointer*/ + err = WebRtcIsacfix_DecodeFrameLen(&ISACdec_obj->bitstr_obj, current_framesamples); + if (err<0) // error check + return err; + + frame_mode = *current_framesamples / MAX_FRAMESAMPLES; /* 0, or 1 */ + + err = WebRtcIsacfix_DecodeSendBandwidth(&ISACdec_obj->bitstr_obj, &BWno); + if (err<0) // error check + return err; + + /* one loop if it's one frame (30ms), two loops if two frames bundled together + * (60ms) */ + for (frame_nb = 0; frame_nb <= frame_mode; frame_nb++) { + + /* decode & dequantize pitch parameters */ + err = WebRtcIsacfix_DecodePitchGain(&(ISACdec_obj->bitstr_obj), PitchGains_Q12); + if (err<0) // error check + return err; + + err = WebRtcIsacfix_DecodePitchLag(&ISACdec_obj->bitstr_obj, PitchGains_Q12, PitchLags_Q7); + if (err<0) // error check + return err; + + AvgPitchGain_Q12 = (int16_t)(((int32_t)PitchGains_Q12[0] + PitchGains_Q12[1] + PitchGains_Q12[2] + PitchGains_Q12[3])>>2); + + /* decode & dequantize FiltCoef */ + err = WebRtcIsacfix_DecodeLpc(gain_lo_hiQ17, lofilt_coefQ15, hifilt_coefQ15, + &ISACdec_obj->bitstr_obj, &model); + + if (err<0) // error check + return err; + + /* decode & dequantize spectrum */ + len = WebRtcIsacfix_DecodeSpec(&ISACdec_obj->bitstr_obj, Vector_Word16_1, Vector_Word16_2, AvgPitchGain_Q12); + if (len < 0) // error check + return len; + + // Why does this need Q16 in and out? /JS + WebRtcIsacfix_Spec2Time(Vector_Word16_1, Vector_Word16_2, Vector_Word32_1, Vector_Word32_2); + + for (k=0; k Q9. + Vector_Word16_1[k] = (int16_t)((Vector_Word32_1[k] + 64) >> 7); + } + + /* ---- If this is recovery frame ---- */ + if( (ISACdec_obj->plcstr_obj).used == PLC_WAS_USED ) + { + (ISACdec_obj->plcstr_obj).used = PLC_NOT_USED; + if( (ISACdec_obj->plcstr_obj).B < 1000 ) + { + (ISACdec_obj->plcstr_obj).decayCoeffPriodic = 4000; + } + + ISACdec_obj->plcstr_obj.decayCoeffPriodic = WEBRTC_SPL_WORD16_MAX; /* DECAY_RATE is in Q15 */ + ISACdec_obj->plcstr_obj.decayCoeffNoise = WEBRTC_SPL_WORD16_MAX; /* DECAY_RATE is in Q15 */ + ISACdec_obj->plcstr_obj.pitchCycles = 0; + + PitchGains_Q12[0] = (int16_t)(PitchGains_Q12[0] * 700 >> 10); + + /* ---- Add-overlap ---- */ + WebRtcSpl_GetHanningWindow( overlapWin, RECOVERY_OVERLAP ); + for( k = 0; k < RECOVERY_OVERLAP; k++ ) + Vector_Word16_1[k] = WebRtcSpl_AddSatW16( + (int16_t)(ISACdec_obj->plcstr_obj.overlapLP[k] * + overlapWin[RECOVERY_OVERLAP - k - 1] >> 14), + (int16_t)(Vector_Word16_1[k] * overlapWin[k] >> 14)); + + + + } + + /* --- Store side info --- */ + if( frame_nb == frame_mode ) + { + /* --- LPC info */ + WEBRTC_SPL_MEMCPY_W16( (ISACdec_obj->plcstr_obj).lofilt_coefQ15, &lofilt_coefQ15[(SUBFRAMES-1)*ORDERLO], ORDERLO ); + WEBRTC_SPL_MEMCPY_W16( (ISACdec_obj->plcstr_obj).hifilt_coefQ15, &hifilt_coefQ15[(SUBFRAMES-1)*ORDERHI], ORDERHI ); + (ISACdec_obj->plcstr_obj).gain_lo_hiQ17[0] = gain_lo_hiQ17[(SUBFRAMES-1) * 2]; + (ISACdec_obj->plcstr_obj).gain_lo_hiQ17[1] = gain_lo_hiQ17[(SUBFRAMES-1) * 2 + 1]; + + /* --- LTP info */ + (ISACdec_obj->plcstr_obj).AvgPitchGain_Q12 = PitchGains_Q12[3]; + (ISACdec_obj->plcstr_obj).lastPitchGain_Q12 = PitchGains_Q12[3]; + (ISACdec_obj->plcstr_obj).lastPitchLag_Q7 = PitchLags_Q7[3]; + + if( PitchLags_Q7[3] < 3000 ) + (ISACdec_obj->plcstr_obj).lastPitchLag_Q7 += PitchLags_Q7[3]; + + WEBRTC_SPL_MEMCPY_W16( (ISACdec_obj->plcstr_obj).prevPitchInvIn, Vector_Word16_1, FRAMESAMPLES/2 ); + + } + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ + + /* inverse pitch filter */ + WebRtcIsacfix_PitchFilter(Vector_Word16_1, Vector_Word16_2, &ISACdec_obj->pitchfiltstr_obj, PitchLags_Q7, PitchGains_Q12, 4); + + if( frame_nb == frame_mode ) + { + WEBRTC_SPL_MEMCPY_W16( (ISACdec_obj->plcstr_obj).prevPitchInvOut, &(Vector_Word16_2[FRAMESAMPLES/2 - (PITCH_MAX_LAG + 10)]), PITCH_MAX_LAG ); + } + + + /* reduce gain to compensate for pitch enhancer */ + /* gain = 1.0f - 0.45f * AvgPitchGain; */ + tmp32a = AvgPitchGain_Q12 * 29; // Q18 + gainQ13 = (int16_t)((262144 - tmp32a) >> 5); // Q18 -> Q13. + + for (k = 0; k < FRAMESAMPLES/2; k++) + { + Vector_Word32_1[k] = (Vector_Word16_2[k] * gainQ13) * (1 << 3); // Q25 + } + + + /* perceptual post-filtering (using normalized lattice filter) */ + WebRtcIsacfix_NormLatticeFilterAr(ORDERLO, (ISACdec_obj->maskfiltstr_obj).PostStateLoGQ0, + Vector_Word32_1, lofilt_coefQ15, gain_lo_hiQ17, 0, Vector_Word16_1); + + /* --- Store Highpass Residual --- */ + for (k = 0; k < FRAMESAMPLES/2; k++) + Vector_Word32_1[k] = Vector_Word32_2[k] * (1 << 9); // Q16 -> Q25 + + for( k = 0; k < PITCH_MAX_LAG + 10; k++ ) + (ISACdec_obj->plcstr_obj).prevHP[k] = Vector_Word32_1[FRAMESAMPLES/2 - (PITCH_MAX_LAG + 10) + k]; + + + WebRtcIsacfix_NormLatticeFilterAr(ORDERHI, (ISACdec_obj->maskfiltstr_obj).PostStateHiGQ0, + Vector_Word32_1, hifilt_coefQ15, gain_lo_hiQ17, 1, Vector_Word16_2); + + /* recombine the 2 bands */ + + /* Form the polyphase signals, and compensate for DC offset */ + for (k=0;kpostfiltbankstr_obj); + + } + return len; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_bwe.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_bwe.c new file mode 100644 index 0000000000..316f59a5e2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_bwe.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * decode_bwe.c + * + * This C file contains the internal decode bandwidth estimate function. + * + */ + + +#include "bandwidth_estimator.h" +#include "codec.h" +#include "entropy_coding.h" +#include "structs.h" + + + + +int WebRtcIsacfix_EstimateBandwidth(BwEstimatorstr *bwest_str, + Bitstr_dec *streamdata, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts) +{ + int16_t index; + size_t frame_samples; + int err; + + /* decode framelength */ + err = WebRtcIsacfix_DecodeFrameLen(streamdata, &frame_samples); + /* error check */ + if (err<0) { + return err; + } + + /* decode BW estimation */ + err = WebRtcIsacfix_DecodeSendBandwidth(streamdata, &index); + /* error check */ + if (err<0) { + return err; + } + + /* Update BWE with received data */ + err = WebRtcIsacfix_UpdateUplinkBwImpl( + bwest_str, + rtp_seq_number, + (int16_t)(frame_samples * 1000 / FS), + send_ts, + arr_ts, + packet_size, /* in bytes */ + index); + + /* error check */ + if (err<0) { + return err; + } + + /* Succesful */ + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_plc.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_plc.c new file mode 100644 index 0000000000..e907f2b6a6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_plc.c @@ -0,0 +1,805 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * decode_plc.c + * + * Packet Loss Concealment. + * + */ + +#include + +#include "settings.h" +#include "entropy_coding.h" +#include "pitch_estimator.h" +#include "bandwidth_estimator.h" +#include "structs.h" +#include "codec.h" + + +#define NO_OF_PRIMES 8 +#define NOISE_FILTER_LEN 30 + +/* + * function to decode the bitstream + * returns the total number of bytes in the stream + */ + +static int16_t plc_filterma_Fast( + int16_t *In, /* (i) Vector to be filtered. InOut[-orderCoef+1] + to InOut[-1] contains state */ + int16_t *Out, /* (o) Filtered vector */ + int16_t *B, /* (i) The filter coefficients (in Q0) */ + int16_t Blen, /* (i) Number of B coefficients */ + int16_t len, /* (i) Number of samples to be filtered */ + int16_t reduceDecay, + int16_t decay, + int16_t rshift ) +{ + int i, j; + int32_t o; + int32_t lim = (1 << (15 + rshift)) - 1; + + for (i = 0; i < len; i++) + { + const int16_t *b_ptr = &B[0]; + const int16_t *x_ptr = &In[i]; + + o = (int32_t)0; + + for (j = 0;j < Blen; j++) + { + o = WebRtcSpl_AddSatW32(o, *b_ptr * *x_ptr); + b_ptr++; + x_ptr--; + } + + /* to round off correctly */ + o = WebRtcSpl_AddSatW32(o, 1 << (rshift - 1)); + + /* saturate according to the domain of the filter coefficients */ + o = WEBRTC_SPL_SAT((int32_t)lim, o, (int32_t)-lim); + + /* o should be in the range of int16_t */ + o >>= rshift; + + /* decay the output signal; this is specific to plc */ + *Out++ = (int16_t)((int16_t)o * decay >> 15); + + /* change the decay */ + decay -= reduceDecay; + if( decay < 0 ) + decay = 0; + } + return( decay ); +} + + + + + + + + +static __inline int32_t log2_Q8_T( uint32_t x ) { + + int32_t zeros; + int16_t frac; + + zeros=WebRtcSpl_NormU32(x); + frac = (int16_t)(((x << zeros) & 0x7FFFFFFF) >> 23); + + /* log2(magn(i)) */ + return ((31 - zeros) << 8) + frac; +} + +static __inline int16_t exp2_Q10_T(int16_t x) { // Both in and out in Q10 + + int16_t tmp16_1, tmp16_2; + + tmp16_2=(int16_t)(0x0400|(x&0x03FF)); + tmp16_1 = -(x >> 10); + if(tmp16_1>0) + return tmp16_2 >> tmp16_1; + else + return tmp16_2 << -tmp16_1; + +} + + +/* + This is a fixed-point version of the above code with limLow = 700 and limHigh = 5000, + hard-coded. The values 700 and 5000 were experimentally obtained. + + The function implements membership values for two sets. The mebership functions are + of second orders corresponding to half-bell-shapped pulses. +*/ +static void MemshipValQ15( int16_t in, int16_t *A, int16_t *B ) +{ + int16_t x; + + in -= 700; /* translate the lowLim to 0, limHigh = 5000 - 700, M = 2150 */ + + if( in <= 2150 ) + { + if( in > 0 ) + { + /* b = in^2 / (2 * M^2), a = 1 - b in Q0. + We have to compute in Q15 */ + + /* x = in / 2150 {in Q15} = x * 15.2409 {in Q15} = + x*15 + (x*983)/(2^12); note that 983/2^12 = 0.23999 */ + + /* we are sure that x is in the range of int16_t */ + x = (int16_t)(in * 15 + (in * 983 >> 12)); + /* b = x^2 / 2 {in Q15} so a shift of 16 is required to + be in correct domain and one more for the division by 2 */ + *B = (int16_t)((x * x + 0x00010000) >> 17); + *A = WEBRTC_SPL_WORD16_MAX - *B; + } + else + { + *B = 0; + *A = WEBRTC_SPL_WORD16_MAX; + } + } + else + { + if( in < 4300 ) + { + /* This is a mirror case of the above */ + in = 4300 - in; + x = (int16_t)(in * 15 + (in * 983 >> 12)); + /* b = x^2 / 2 {in Q15} so a shift of 16 is required to + be in correct domain and one more for the division by 2 */ + *A = (int16_t)((x * x + 0x00010000) >> 17); + *B = WEBRTC_SPL_WORD16_MAX - *A; + + } + else + { + *A = 0; + *B = WEBRTC_SPL_WORD16_MAX; + } + } +} + + + + +static void LinearResampler(int16_t* in, + int16_t* out, + size_t lenIn, + size_t lenOut) +{ + size_t n = (lenIn - 1) * RESAMP_RES; + int16_t resOut, relativePos, diff; /* */ + size_t i, j; + uint16_t udiff; + + if( lenIn == lenOut ) + { + WEBRTC_SPL_MEMCPY_W16( out, in, lenIn ); + return; + } + + resOut = WebRtcSpl_DivW32W16ResW16( (int32_t)n, (int16_t)(lenOut-1) ); + + out[0] = in[0]; + for( i = 1, j = 0, relativePos = 0; i < lenOut; i++ ) + { + + relativePos += resOut; + while( relativePos > RESAMP_RES ) + { + j++; + relativePos -= RESAMP_RES; + } + + + /* an overflow may happen and the differce in sample values may + * require more than 16 bits. We like to avoid 32 bit arithmatic + * as much as possible */ + + if( (in[ j ] > 0) && (in[j + 1] < 0) ) + { + udiff = (uint16_t)(in[ j ] - in[j + 1]); + out[ i ] = in[ j ] - (uint16_t)( ((int32_t)( udiff * relativePos )) >> RESAMP_RES_BIT); + } + else + { + if( (in[j] < 0) && (in[j+1] > 0) ) + { + udiff = (uint16_t)( in[j + 1] - in[ j ] ); + out[ i ] = in[ j ] + (uint16_t)( ((int32_t)( udiff * relativePos )) >> RESAMP_RES_BIT); + } + else + { + diff = in[ j + 1 ] - in[ j ]; + out[i] = in[j] + (int16_t)(diff * relativePos >> RESAMP_RES_BIT); + } + } + } +} + + + + + +void WebRtcIsacfix_DecodePlcImpl(int16_t *signal_out16, + IsacFixDecoderInstance *ISACdec_obj, + size_t *current_framesamples ) +{ + int subframecnt; + + int16_t* Vector_Word16_1; + int16_t Vector_Word16_Extended_1[FRAMESAMPLES_HALF + NOISE_FILTER_LEN]; + int16_t* Vector_Word16_2; + int16_t Vector_Word16_Extended_2[FRAMESAMPLES_HALF + NOISE_FILTER_LEN]; + + int32_t Vector_Word32_1[FRAMESAMPLES_HALF]; + int32_t Vector_Word32_2[FRAMESAMPLES_HALF]; + + int16_t lofilt_coefQ15[ORDERLO*SUBFRAMES]; //refl. coeffs + int16_t hifilt_coefQ15[ORDERHI*SUBFRAMES]; //refl. coeffs + + int16_t pitchLags_Q7[PITCH_SUBFRAMES]; + int16_t pitchGains_Q12[PITCH_SUBFRAMES]; + + int16_t tmp_1, tmp_2; + int32_t tmp32a, tmp32b; + int16_t gainQ13; + + int16_t myDecayRate; + + /* ---------- PLC variables ------------ */ + size_t lag0, i, k; + int16_t noiseIndex; + int16_t stretchPitchLP[PITCH_MAX_LAG + 10], stretchPitchLP1[PITCH_MAX_LAG + 10]; + + int32_t gain_lo_hiQ17[2*SUBFRAMES]; + + int16_t nLP, pLP, wNoisyLP, wPriodicLP, tmp16; + size_t minIdx; + int32_t nHP, pHP, wNoisyHP, wPriodicHP, corr, minCorr, maxCoeff; + int16_t noise1, rshift; + + + int16_t ltpGain, pitchGain, myVoiceIndicator, myAbs, maxAbs; + int32_t varIn, varOut, logVarIn, logVarOut, Q, logMaxAbs; + int rightShiftIn, rightShiftOut; + + + /* ------------------------------------- */ + + + myDecayRate = (DECAY_RATE); + Vector_Word16_1 = &Vector_Word16_Extended_1[NOISE_FILTER_LEN]; + Vector_Word16_2 = &Vector_Word16_Extended_2[NOISE_FILTER_LEN]; + + + /* ----- Simply Copy Previous LPC parameters ------ */ + for( subframecnt = 0; subframecnt < SUBFRAMES; subframecnt++ ) + { + /* lower Band */ + WEBRTC_SPL_MEMCPY_W16(&lofilt_coefQ15[ subframecnt * ORDERLO ], + (ISACdec_obj->plcstr_obj).lofilt_coefQ15, ORDERLO); + gain_lo_hiQ17[2*subframecnt] = (ISACdec_obj->plcstr_obj).gain_lo_hiQ17[0]; + + /* Upper Band */ + WEBRTC_SPL_MEMCPY_W16(&hifilt_coefQ15[ subframecnt * ORDERHI ], + (ISACdec_obj->plcstr_obj).hifilt_coefQ15, ORDERHI); + gain_lo_hiQ17[2*subframecnt + 1] = (ISACdec_obj->plcstr_obj).gain_lo_hiQ17[1]; + } + + + + + lag0 = (size_t)(((ISACdec_obj->plcstr_obj.lastPitchLag_Q7 + 64) >> 7) + 1); + + + if( (ISACdec_obj->plcstr_obj).used != PLC_WAS_USED ) + { + (ISACdec_obj->plcstr_obj).pitchCycles = 0; + + (ISACdec_obj->plcstr_obj).lastPitchLP = + &((ISACdec_obj->plcstr_obj).prevPitchInvIn[FRAMESAMPLES_HALF - lag0]); + minCorr = WEBRTC_SPL_WORD32_MAX; + + if ((FRAMESAMPLES_HALF - 10) > 2 * lag0) + { + minIdx = 11; + for( i = 0; i < 21; i++ ) + { + corr = 0; + for( k = 0; k < lag0; k++ ) + { + corr = WebRtcSpl_AddSatW32(corr, WEBRTC_SPL_ABS_W32( + WebRtcSpl_SubSatW16( + (ISACdec_obj->plcstr_obj).lastPitchLP[k], + (ISACdec_obj->plcstr_obj).prevPitchInvIn[ + FRAMESAMPLES_HALF - 2*lag0 - 10 + i + k ] ) ) ); + } + if( corr < minCorr ) + { + minCorr = corr; + minIdx = i; + } + } + (ISACdec_obj->plcstr_obj).prevPitchLP = + &( (ISACdec_obj->plcstr_obj).prevPitchInvIn[ + FRAMESAMPLES_HALF - lag0*2 - 10 + minIdx] ); + } + else + { + (ISACdec_obj->plcstr_obj).prevPitchLP = + (ISACdec_obj->plcstr_obj).lastPitchLP; + } + pitchGain = (ISACdec_obj->plcstr_obj).lastPitchGain_Q12; + + WebRtcSpl_AutoCorrelation( + &(ISACdec_obj->plcstr_obj).prevPitchInvIn[FRAMESAMPLES_HALF - lag0], + lag0, 0, &varIn, &rightShiftIn); + WebRtcSpl_AutoCorrelation( + &(ISACdec_obj->plcstr_obj).prevPitchInvOut[PITCH_MAX_LAG + 10 - lag0], + lag0, 0, &varOut, &rightShiftOut); + + maxAbs = 0; + for( i = 0; i< lag0; i++) + { + myAbs = WEBRTC_SPL_ABS_W16( + (ISACdec_obj->plcstr_obj).prevPitchInvOut[ + PITCH_MAX_LAG + 10 - lag0 + i] ); + maxAbs = (myAbs > maxAbs)? myAbs:maxAbs; + } + logVarIn = log2_Q8_T( (uint32_t)( varIn ) ) + + (int32_t)(rightShiftIn << 8); + logVarOut = log2_Q8_T( (uint32_t)( varOut ) ) + + (int32_t)(rightShiftOut << 8); + logMaxAbs = log2_Q8_T( (uint32_t)( maxAbs ) ); + + ltpGain = (int16_t)(logVarOut - logVarIn); + Q = 2 * logMaxAbs - ( logVarOut - 1512 ); + + /* + * --- + * We are computing sqrt( (VarIn/lag0) / var( noise ) ) + * var( noise ) is almost 256. we have already computed log2( VarIn ) in Q8 + * so we actually compute 2^( 0.5*(log2( VarIn ) - log2( lag0 ) - log2( var(noise ) ) ). + * Note that put log function is in Q8 but the exponential function is in Q10. + * -- + */ + + logVarIn -= log2_Q8_T( (uint32_t)( lag0 ) ); + tmp16 = (int16_t)((logVarIn<<1) - (4<<10) ); + rightShiftIn = 0; + if( tmp16 > 4096 ) + { + tmp16 -= 4096; + tmp16 = exp2_Q10_T( tmp16 ); + tmp16 >>= 6; + } + else + tmp16 = exp2_Q10_T( tmp16 )>>10; + + (ISACdec_obj->plcstr_obj).std = tmp16 - 4; + + if( (ltpGain < 110) || (ltpGain > 230) ) + { + if( ltpGain < 100 && (pitchGain < 1800) ) + { + (ISACdec_obj->plcstr_obj).A = WEBRTC_SPL_WORD16_MAX; + } + else + { + (ISACdec_obj->plcstr_obj).A = ((ltpGain < 110) && (Q < 800) + )? WEBRTC_SPL_WORD16_MAX:0; + } + (ISACdec_obj->plcstr_obj).B = WEBRTC_SPL_WORD16_MAX - + (ISACdec_obj->plcstr_obj).A; + } + else + { + if( (pitchGain < 450) || (pitchGain > 1600) ) + { + (ISACdec_obj->plcstr_obj).A = ((pitchGain < 450) + )? WEBRTC_SPL_WORD16_MAX:0; + (ISACdec_obj->plcstr_obj).B = WEBRTC_SPL_WORD16_MAX - + (ISACdec_obj->plcstr_obj).A; + } + else + { + myVoiceIndicator = ltpGain * 2 + pitchGain; + MemshipValQ15( myVoiceIndicator, + &(ISACdec_obj->plcstr_obj).A, &(ISACdec_obj->plcstr_obj).B ); + } + } + + + + myVoiceIndicator = ltpGain * 16 + pitchGain * 2 + (pitchGain >> 8); + MemshipValQ15( myVoiceIndicator, + &(ISACdec_obj->plcstr_obj).A, &(ISACdec_obj->plcstr_obj).B ); + + + + (ISACdec_obj->plcstr_obj).stretchLag = lag0; + (ISACdec_obj->plcstr_obj).pitchIndex = 0; + + } + else + { + myDecayRate = (DECAY_RATE<<2); + } + + if( (ISACdec_obj->plcstr_obj).B < 1000 ) + { + myDecayRate += (DECAY_RATE<<3); + } + + /* ------------ reconstructing the residual signal ------------------ */ + + LinearResampler( (ISACdec_obj->plcstr_obj).lastPitchLP, + stretchPitchLP, lag0, (ISACdec_obj->plcstr_obj).stretchLag ); + /* inverse pitch filter */ + + pitchLags_Q7[0] = pitchLags_Q7[1] = pitchLags_Q7[2] = pitchLags_Q7[3] = + (int16_t)((ISACdec_obj->plcstr_obj).stretchLag<<7); + pitchGains_Q12[3] = ( (ISACdec_obj->plcstr_obj).lastPitchGain_Q12); + pitchGains_Q12[2] = (int16_t)(pitchGains_Q12[3] * 1010 >> 10); + pitchGains_Q12[1] = (int16_t)(pitchGains_Q12[2] * 1010 >> 10); + pitchGains_Q12[0] = (int16_t)(pitchGains_Q12[1] * 1010 >> 10); + + + /* most of the time either B or A are zero so seperating */ + if( (ISACdec_obj->plcstr_obj).B == 0 ) + { + for( i = 0; i < FRAMESAMPLES_HALF; i++ ) + { + /* --- Low Pass */ + (ISACdec_obj->plcstr_obj).seed = WEBRTC_SPL_RAND( + (ISACdec_obj->plcstr_obj).seed ); + Vector_Word16_1[i] = (ISACdec_obj->plcstr_obj.seed >> 10) - 16; + + /* --- Highpass */ + (ISACdec_obj->plcstr_obj).seed = WEBRTC_SPL_RAND( + (ISACdec_obj->plcstr_obj).seed ); + Vector_Word16_2[i] = (ISACdec_obj->plcstr_obj.seed >> 10) - 16; + + } + for( i = 1; i < NOISE_FILTER_LEN; i++ ) + { + (ISACdec_obj->plcstr_obj).seed = WEBRTC_SPL_RAND( + (ISACdec_obj->plcstr_obj).seed ); + Vector_Word16_Extended_1[i] = (ISACdec_obj->plcstr_obj.seed >> 10) - 16; + + (ISACdec_obj->plcstr_obj).seed = WEBRTC_SPL_RAND( + (ISACdec_obj->plcstr_obj).seed ); + Vector_Word16_Extended_2[i] = (ISACdec_obj->plcstr_obj.seed >> 10) - 16; + } + plc_filterma_Fast(Vector_Word16_1, Vector_Word16_Extended_1, + &(ISACdec_obj->plcstr_obj).prevPitchInvIn[FRAMESAMPLES_HALF - + NOISE_FILTER_LEN], (int16_t) NOISE_FILTER_LEN, + (int16_t) FRAMESAMPLES_HALF, (int16_t)(5), + (ISACdec_obj->plcstr_obj).decayCoeffNoise, (int16_t)(6)); + + maxCoeff = WebRtcSpl_MaxAbsValueW32( + &(ISACdec_obj->plcstr_obj).prevHP[ + PITCH_MAX_LAG + 10 - NOISE_FILTER_LEN], NOISE_FILTER_LEN ); + + rshift = 0; + while( maxCoeff > WEBRTC_SPL_WORD16_MAX ) + { + maxCoeff >>= 1; + rshift++; + } + for( i = 0; i < NOISE_FILTER_LEN; i++ ) { + Vector_Word16_1[FRAMESAMPLES_HALF - NOISE_FILTER_LEN + i] =(int16_t)( + ISACdec_obj->plcstr_obj.prevHP[PITCH_MAX_LAG + 10 - NOISE_FILTER_LEN + + i] >> rshift); + } + (ISACdec_obj->plcstr_obj).decayCoeffNoise = plc_filterma_Fast( + Vector_Word16_2, + Vector_Word16_Extended_2, + &Vector_Word16_1[FRAMESAMPLES_HALF - NOISE_FILTER_LEN], + (int16_t) NOISE_FILTER_LEN, + (int16_t) FRAMESAMPLES_HALF, + (int16_t) (5), + (ISACdec_obj->plcstr_obj).decayCoeffNoise, + (int16_t) (7) ); + + for( i = 0; i < FRAMESAMPLES_HALF; i++ ) + Vector_Word32_2[i] = Vector_Word16_Extended_2[i] << rshift; + + Vector_Word16_1 = Vector_Word16_Extended_1; + } + else + { + if( (ISACdec_obj->plcstr_obj).A == 0 ) + { + /* ------ Periodic Vector --- */ + for( i = 0, noiseIndex = 0; i < FRAMESAMPLES_HALF; i++, noiseIndex++ ) + { + /* --- Lowpass */ + pLP = (int16_t)(stretchPitchLP[ISACdec_obj->plcstr_obj.pitchIndex] * + ISACdec_obj->plcstr_obj.decayCoeffPriodic >> 15); + + /* --- Highpass */ + pHP = (int32_t)WEBRTC_SPL_MUL_16_32_RSFT15( + (ISACdec_obj->plcstr_obj).decayCoeffPriodic, + (ISACdec_obj->plcstr_obj).prevHP[PITCH_MAX_LAG + 10 - + (ISACdec_obj->plcstr_obj).stretchLag + + (ISACdec_obj->plcstr_obj).pitchIndex] ); + + /* --- lower the muliplier (more decay at next sample) --- */ + (ISACdec_obj->plcstr_obj).decayCoeffPriodic -= (myDecayRate); + if( (ISACdec_obj->plcstr_obj).decayCoeffPriodic < 0 ) + (ISACdec_obj->plcstr_obj).decayCoeffPriodic = 0; + + (ISACdec_obj->plcstr_obj).pitchIndex++; + + if( (ISACdec_obj->plcstr_obj).pitchIndex == + (ISACdec_obj->plcstr_obj).stretchLag ) + { + (ISACdec_obj->plcstr_obj).pitchIndex = 0; + (ISACdec_obj->plcstr_obj).pitchCycles++; + + if( (ISACdec_obj->plcstr_obj).stretchLag != (lag0 + 1) ) + { + (ISACdec_obj->plcstr_obj).stretchLag = lag0 + 1; + } + else + { + (ISACdec_obj->plcstr_obj).stretchLag = lag0; + } + + (ISACdec_obj->plcstr_obj).stretchLag = ( + (ISACdec_obj->plcstr_obj).stretchLag > PITCH_MAX_LAG + )? (PITCH_MAX_LAG):(ISACdec_obj->plcstr_obj).stretchLag; + + LinearResampler( (ISACdec_obj->plcstr_obj).lastPitchLP, + stretchPitchLP, lag0, (ISACdec_obj->plcstr_obj).stretchLag ); + + LinearResampler( (ISACdec_obj->plcstr_obj).prevPitchLP, + stretchPitchLP1, lag0, (ISACdec_obj->plcstr_obj).stretchLag ); + + switch( (ISACdec_obj->plcstr_obj).pitchCycles ) + { + case 1: + { + for( k=0; k<(ISACdec_obj->plcstr_obj).stretchLag; k++ ) + { + stretchPitchLP[k] = (int16_t)(( + (int32_t)stretchPitchLP[k]* 3 + + (int32_t)stretchPitchLP1[k])>>2); + } + break; + } + case 2: + { + for( k=0; k<(ISACdec_obj->plcstr_obj).stretchLag; k++ ) + { + stretchPitchLP[k] = (int16_t)(( + (int32_t)stretchPitchLP[k] + + (int32_t)stretchPitchLP1[k] )>>1); + } + break; + } + case 3: + { + for( k=0; k<(ISACdec_obj->plcstr_obj).stretchLag; k++ ) + { + stretchPitchLP[k] = (int16_t)((stretchPitchLP[k] + + (int32_t)stretchPitchLP1[k]*3 )>>2); + } + break; + } + } + + if( (ISACdec_obj->plcstr_obj).pitchCycles == 3 ) + { + myDecayRate += 35; //(myDecayRate>>1); + (ISACdec_obj->plcstr_obj).pitchCycles = 0; + } + + } + + /* ------ Sum the noisy and periodic signals ------ */ + Vector_Word16_1[i] = pLP; + Vector_Word32_2[i] = pHP; + } + } + else + { + for( i = 0, noiseIndex = 0; i < FRAMESAMPLES_HALF; i++, noiseIndex++ ) + { + + (ISACdec_obj->plcstr_obj).seed = WEBRTC_SPL_RAND( + (ISACdec_obj->plcstr_obj).seed ); + + noise1 = (ISACdec_obj->plcstr_obj.seed >> 10) - 16; + + nLP = (int16_t)((int16_t)(noise1 * ISACdec_obj->plcstr_obj.std) * + ISACdec_obj->plcstr_obj.decayCoeffNoise >> 15); + + /* --- Highpass */ + (ISACdec_obj->plcstr_obj).seed = WEBRTC_SPL_RAND( + (ISACdec_obj->plcstr_obj).seed ); + noise1 = (ISACdec_obj->plcstr_obj.seed >> 11) - 8; + + nHP = (int32_t)WEBRTC_SPL_MUL_16_32_RSFT15( + (ISACdec_obj->plcstr_obj).decayCoeffNoise, + (int32_t)(noise1*(ISACdec_obj->plcstr_obj).std) ); + + /* --- lower the muliplier (more decay at next sample) --- */ + (ISACdec_obj->plcstr_obj).decayCoeffNoise -= (myDecayRate); + if( (ISACdec_obj->plcstr_obj).decayCoeffNoise < 0 ) + (ISACdec_obj->plcstr_obj).decayCoeffNoise = 0; + + /* ------ Periodic Vector --- */ + /* --- Lowpass */ + pLP = (int16_t)(stretchPitchLP[ISACdec_obj->plcstr_obj.pitchIndex] * + ISACdec_obj->plcstr_obj.decayCoeffPriodic >> 15); + + /* --- Highpass */ + pHP = (int32_t)WEBRTC_SPL_MUL_16_32_RSFT15( + (ISACdec_obj->plcstr_obj).decayCoeffPriodic, + (ISACdec_obj->plcstr_obj).prevHP[PITCH_MAX_LAG + 10 - + (ISACdec_obj->plcstr_obj).stretchLag + + (ISACdec_obj->plcstr_obj).pitchIndex] ); + + /* --- lower the muliplier (more decay at next sample) --- */ + (ISACdec_obj->plcstr_obj).decayCoeffPriodic -= (myDecayRate); + if( (ISACdec_obj->plcstr_obj).decayCoeffPriodic < 0 ) + { + (ISACdec_obj->plcstr_obj).decayCoeffPriodic = 0; + } + + /* ------ Weighting the noisy and periodic vectors ------- */ + wNoisyLP = (int16_t)(ISACdec_obj->plcstr_obj.A * nLP >> 15); + wNoisyHP = (int32_t)(WEBRTC_SPL_MUL_16_32_RSFT15( + (ISACdec_obj->plcstr_obj).A, (nHP) ) ); + + wPriodicLP = (int16_t)(ISACdec_obj->plcstr_obj.B * pLP >> 15); + wPriodicHP = (int32_t)(WEBRTC_SPL_MUL_16_32_RSFT15( + (ISACdec_obj->plcstr_obj).B, pHP)); + + (ISACdec_obj->plcstr_obj).pitchIndex++; + + if((ISACdec_obj->plcstr_obj).pitchIndex == + (ISACdec_obj->plcstr_obj).stretchLag) + { + (ISACdec_obj->plcstr_obj).pitchIndex = 0; + (ISACdec_obj->plcstr_obj).pitchCycles++; + + if( (ISACdec_obj->plcstr_obj).stretchLag != (lag0 + 1) ) + (ISACdec_obj->plcstr_obj).stretchLag = lag0 + 1; + else + (ISACdec_obj->plcstr_obj).stretchLag = lag0; + + (ISACdec_obj->plcstr_obj).stretchLag = ( + (ISACdec_obj->plcstr_obj).stretchLag > PITCH_MAX_LAG + )? (PITCH_MAX_LAG):(ISACdec_obj->plcstr_obj).stretchLag; + LinearResampler( + (ISACdec_obj->plcstr_obj).lastPitchLP, + stretchPitchLP, lag0, (ISACdec_obj->plcstr_obj).stretchLag ); + + LinearResampler((ISACdec_obj->plcstr_obj).prevPitchLP, + stretchPitchLP1, lag0, (ISACdec_obj->plcstr_obj).stretchLag ); + + switch((ISACdec_obj->plcstr_obj).pitchCycles) + { + case 1: + { + for( k=0; k<(ISACdec_obj->plcstr_obj).stretchLag; k++ ) + { + stretchPitchLP[k] = (int16_t)(( + (int32_t)stretchPitchLP[k]* 3 + + (int32_t)stretchPitchLP1[k] )>>2); + } + break; + } + case 2: + { + for( k=0; k<(ISACdec_obj->plcstr_obj).stretchLag; k++ ) + { + stretchPitchLP[k] = (int16_t)(( + (int32_t)stretchPitchLP[k] + + (int32_t)stretchPitchLP1[k])>>1); + } + break; + } + case 3: + { + for( k=0; k<(ISACdec_obj->plcstr_obj).stretchLag; k++ ) + { + stretchPitchLP[k] = (int16_t)( + (stretchPitchLP[k] + + (int32_t)stretchPitchLP1[k]*3 )>>2); + } + break; + } + } + + if( (ISACdec_obj->plcstr_obj).pitchCycles == 3 ) + { + myDecayRate += 55; //(myDecayRate>>1); + (ISACdec_obj->plcstr_obj).pitchCycles = 0; + } + } + + /* ------ Sum the noisy and periodic signals ------ */ + Vector_Word16_1[i] = WebRtcSpl_AddSatW16(wNoisyLP, wPriodicLP); + Vector_Word32_2[i] = WebRtcSpl_AddSatW32(wNoisyHP, wPriodicHP); + } + } + } + /* ----------------- residual signal is reconstructed ------------------ */ + + k = (ISACdec_obj->plcstr_obj).pitchIndex; + /* --- Write one pitch cycle for recovery block --- */ + + for( i = 0; i < RECOVERY_OVERLAP; i++ ) + { + ISACdec_obj->plcstr_obj.overlapLP[i] = (int16_t)( + stretchPitchLP[k] * ISACdec_obj->plcstr_obj.decayCoeffPriodic >> 15); + k = ( k < ((ISACdec_obj->plcstr_obj).stretchLag - 1) )? (k+1):0; + } + + (ISACdec_obj->plcstr_obj).lastPitchLag_Q7 = + (int16_t)((ISACdec_obj->plcstr_obj).stretchLag << 7); + + + /* --- Inverse Pitch Filter --- */ + WebRtcIsacfix_PitchFilter(Vector_Word16_1, Vector_Word16_2, + &ISACdec_obj->pitchfiltstr_obj, pitchLags_Q7, pitchGains_Q12, 4); + + /* reduce gain to compensate for pitch enhancer */ + /* gain = 1.0f - 0.45f * AvgPitchGain; */ + tmp32a = ISACdec_obj->plcstr_obj.AvgPitchGain_Q12 * 29; // Q18 + tmp32b = 262144 - tmp32a; // Q18 + gainQ13 = (int16_t) (tmp32b >> 5); // Q13 + + /* perceptual post-filtering (using normalized lattice filter) */ + for (k = 0; k < FRAMESAMPLES_HALF; k++) + Vector_Word32_1[k] = (Vector_Word16_2[k] * gainQ13) << 3; // Q25 + + + WebRtcIsacfix_NormLatticeFilterAr(ORDERLO, + (ISACdec_obj->maskfiltstr_obj).PostStateLoGQ0, + Vector_Word32_1, lofilt_coefQ15, gain_lo_hiQ17, 0, Vector_Word16_1); + + WebRtcIsacfix_NormLatticeFilterAr(ORDERHI, + (ISACdec_obj->maskfiltstr_obj).PostStateHiGQ0, + Vector_Word32_2, hifilt_coefQ15, gain_lo_hiQ17, 1, Vector_Word16_2); + + /* recombine the 2 bands */ + + /* Form the polyphase signals, and compensate for DC offset */ + for (k=0;kpostfiltbankstr_obj); + + (ISACdec_obj->plcstr_obj).used = PLC_WAS_USED; + *current_framesamples = 480; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/encode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/encode.c new file mode 100644 index 0000000000..ef3e320e2c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/encode.c @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * encode.c + * + * Encoding function for the iSAC coder. + * + */ + +#include "rtc_base/checks.h" +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" + +#include + +#include "modules/audio_coding/codecs/isac/fix/source/arith_routins.h" +#include "modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h" +#include "modules/audio_coding/codecs/isac/fix/source/entropy_coding.h" +#include "modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h" +#include "modules/audio_coding/codecs/isac/fix/source/lpc_tables.h" +#include "modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h" +#include "modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.h" +#include "modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.h" +#include "modules/audio_coding/codecs/isac/fix/source/structs.h" + + +int WebRtcIsacfix_EncodeImpl(int16_t *in, + IsacFixEncoderInstance *ISACenc_obj, + BwEstimatorstr *bw_estimatordata, + int16_t CodingMode) +{ + int16_t stream_length = 0; + int16_t usefulstr_len = 0; + int k; + int16_t BWno; + + int16_t lofilt_coefQ15[(ORDERLO)*SUBFRAMES]; + int16_t hifilt_coefQ15[(ORDERHI)*SUBFRAMES]; + int32_t gain_lo_hiQ17[2*SUBFRAMES]; + + int16_t LPandHP[FRAMESAMPLES/2 + QLOOKAHEAD]; + int16_t LP16a[FRAMESAMPLES/2 + QLOOKAHEAD]; + int16_t HP16a[FRAMESAMPLES/2 + QLOOKAHEAD]; + + int16_t PitchLags_Q7[PITCH_SUBFRAMES]; + int16_t PitchGains_Q12[PITCH_SUBFRAMES]; + int16_t AvgPitchGain_Q12; + + int16_t frame_mode; /* 0 for 30ms, 1 for 60ms */ + int16_t processed_samples; + int status; + + int32_t bits_gainsQ11; + int16_t MinBytes; + int16_t bmodel; + + transcode_obj transcodingParam; + int16_t payloadLimitBytes; + int16_t arithLenBeforeEncodingDFT; + int16_t iterCntr; + + /* copy new frame length and bottle neck rate only for the first 10 ms data */ + if (ISACenc_obj->buffer_index == 0) { + /* set the framelength for the next packet */ + ISACenc_obj->current_framesamples = ISACenc_obj->new_framelength; + } + + frame_mode = ISACenc_obj->current_framesamples/MAX_FRAMESAMPLES; /* 0 (30 ms) or 1 (60 ms) */ + processed_samples = ISACenc_obj->current_framesamples/(frame_mode+1); /* 480 (30, 60 ms) */ + + /* buffer speech samples (by 10ms packet) until the framelength is reached (30 or 60 ms) */ + /**************************************************************************************/ + /* fill the buffer with 10ms input data */ + for(k=0; kdata_buffer_fix[k + ISACenc_obj->buffer_index] = in[k]; + } + /* if buffersize is not equal to current framesize, and end of file is not reached yet, */ + /* increase index and go back to main to get more speech samples */ + if (ISACenc_obj->buffer_index + FRAMESAMPLES_10ms != processed_samples) { + ISACenc_obj->buffer_index = ISACenc_obj->buffer_index + FRAMESAMPLES_10ms; + return 0; + } + /* if buffer reached the right size, reset index and continue with encoding the frame */ + ISACenc_obj->buffer_index = 0; + + /* end of buffer function */ + /**************************/ + + /* encoding */ + /************/ + + if (frame_mode == 0 || ISACenc_obj->frame_nb == 0 ) + { + /* reset bitstream */ + ISACenc_obj->bitstr_obj.W_upper = 0xFFFFFFFF; + ISACenc_obj->bitstr_obj.streamval = 0; + ISACenc_obj->bitstr_obj.stream_index = 0; + ISACenc_obj->bitstr_obj.full = 1; + + if (CodingMode == 0) { + ISACenc_obj->BottleNeck = WebRtcIsacfix_GetUplinkBandwidth(bw_estimatordata); + ISACenc_obj->MaxDelay = WebRtcIsacfix_GetUplinkMaxDelay(bw_estimatordata); + } + if (CodingMode == 0 && frame_mode == 0 && (ISACenc_obj->enforceFrameSize == 0)) { + ISACenc_obj->new_framelength = WebRtcIsacfix_GetNewFrameLength(ISACenc_obj->BottleNeck, + ISACenc_obj->current_framesamples); + } + + // multiply the bottleneck by 0.88 before computing SNR, 0.88 is tuned by experimenting on TIMIT + // 901/1024 is 0.87988281250000 + ISACenc_obj->s2nr = WebRtcIsacfix_GetSnr( + (int16_t)(ISACenc_obj->BottleNeck * 901 >> 10), + ISACenc_obj->current_framesamples); + + /* encode frame length */ + status = WebRtcIsacfix_EncodeFrameLen(ISACenc_obj->current_framesamples, &ISACenc_obj->bitstr_obj); + if (status < 0) + { + /* Wrong frame size */ + if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + // If this is the second 30ms of a 60ms frame reset this such that in the next call + // encoder starts fresh. + ISACenc_obj->frame_nb = 0; + } + return status; + } + + /* Save framelength for multiple packets memory */ + if (ISACenc_obj->SaveEnc_ptr != NULL) { + (ISACenc_obj->SaveEnc_ptr)->framelength=ISACenc_obj->current_framesamples; + } + + /* bandwidth estimation and coding */ + BWno = WebRtcIsacfix_GetDownlinkBwIndexImpl(bw_estimatordata); + status = WebRtcIsacfix_EncodeReceiveBandwidth(&BWno, &ISACenc_obj->bitstr_obj); + if (status < 0) + { + if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + // If this is the second 30ms of a 60ms frame reset this such that in the next call + // encoder starts fresh. + ISACenc_obj->frame_nb = 0; + } + return status; + } + } + + /* split signal in two bands */ + WebRtcIsacfix_SplitAndFilter1(ISACenc_obj->data_buffer_fix, LP16a, HP16a, &ISACenc_obj->prefiltbankstr_obj ); + + /* estimate pitch parameters and pitch-filter lookahead signal */ + WebRtcIsacfix_PitchAnalysis(LP16a+QLOOKAHEAD, LPandHP, + &ISACenc_obj->pitchanalysisstr_obj, PitchLags_Q7, PitchGains_Q12); /* LPandHP = LP_lookahead_pfQ0, */ + + /* Set where to store data in multiple packets memory */ + if (ISACenc_obj->SaveEnc_ptr != NULL) { + if (frame_mode == 0 || ISACenc_obj->frame_nb == 0) + { + (ISACenc_obj->SaveEnc_ptr)->startIdx = 0; + } + else + { + (ISACenc_obj->SaveEnc_ptr)->startIdx = 1; + } + } + + /* quantize & encode pitch parameters */ + status = WebRtcIsacfix_EncodePitchGain(PitchGains_Q12, &ISACenc_obj->bitstr_obj, ISACenc_obj->SaveEnc_ptr); + if (status < 0) + { + if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + // If this is the second 30ms of a 60ms frame reset this such that in the next call + // encoder starts fresh. + ISACenc_obj->frame_nb = 0; + } + return status; + } + status = WebRtcIsacfix_EncodePitchLag(PitchLags_Q7 , PitchGains_Q12, &ISACenc_obj->bitstr_obj, ISACenc_obj->SaveEnc_ptr); + if (status < 0) + { + if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + // If this is the second 30ms of a 60ms frame reset this such that in the next call + // encoder starts fresh. + ISACenc_obj->frame_nb = 0; + } + return status; + } + AvgPitchGain_Q12 = (PitchGains_Q12[0] + PitchGains_Q12[1] + + PitchGains_Q12[2] + PitchGains_Q12[3]) >> 2; + + /* find coefficients for perceptual pre-filters */ + WebRtcIsacfix_GetLpcCoef(LPandHP, HP16a+QLOOKAHEAD, &ISACenc_obj->maskfiltstr_obj, + ISACenc_obj->s2nr, PitchGains_Q12, + gain_lo_hiQ17, lofilt_coefQ15, hifilt_coefQ15); /*LPandHP = LP_lookahead_pfQ0*/ + + // record LPC Gains for possible bit-rate reduction + for(k = 0; k < KLT_ORDER_GAIN; k++) + { + transcodingParam.lpcGains[k] = gain_lo_hiQ17[k]; + } + + /* code LPC model and shape - gains not quantized yet */ + status = WebRtcIsacfix_EncodeLpc(gain_lo_hiQ17, lofilt_coefQ15, hifilt_coefQ15, + &bmodel, &bits_gainsQ11, &ISACenc_obj->bitstr_obj, ISACenc_obj->SaveEnc_ptr, &transcodingParam); + if (status < 0) + { + if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + // If this is the second 30ms of a 60ms frame reset this such that in the next call + // encoder starts fresh. + ISACenc_obj->frame_nb = 0; + } + return status; + } + arithLenBeforeEncodingDFT = (ISACenc_obj->bitstr_obj.stream_index << 1) + (1-ISACenc_obj->bitstr_obj.full); + + /* low-band filtering */ + WebRtcIsacfix_NormLatticeFilterMa(ORDERLO, ISACenc_obj->maskfiltstr_obj.PreStateLoGQ15, + LP16a, lofilt_coefQ15, gain_lo_hiQ17, 0, LPandHP);/* LPandHP = LP16b */ + + /* pitch filter */ + WebRtcIsacfix_PitchFilter(LPandHP, LP16a, &ISACenc_obj->pitchfiltstr_obj, PitchLags_Q7, PitchGains_Q12, 1);/* LPandHP = LP16b */ + + /* high-band filtering */ + WebRtcIsacfix_NormLatticeFilterMa(ORDERHI, ISACenc_obj->maskfiltstr_obj.PreStateHiGQ15, + HP16a, hifilt_coefQ15, gain_lo_hiQ17, 1, LPandHP);/*LPandHP = HP16b*/ + + /* transform */ + WebRtcIsacfix_Time2Spec(LP16a, LPandHP, LP16a, LPandHP); /*LPandHP = HP16b*/ + + /* Save data for multiple packets memory */ + if (ISACenc_obj->SaveEnc_ptr != NULL) { + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + (ISACenc_obj->SaveEnc_ptr)->fre[k + (ISACenc_obj->SaveEnc_ptr)->startIdx*FRAMESAMPLES_HALF] = LP16a[k]; + (ISACenc_obj->SaveEnc_ptr)->fim[k + (ISACenc_obj->SaveEnc_ptr)->startIdx*FRAMESAMPLES_HALF] = LPandHP[k]; + } + (ISACenc_obj->SaveEnc_ptr)->AvgPitchGain[(ISACenc_obj->SaveEnc_ptr)->startIdx] = AvgPitchGain_Q12; + } + + /* quantization and lossless coding */ + status = WebRtcIsacfix_EncodeSpec(LP16a, LPandHP, &ISACenc_obj->bitstr_obj, AvgPitchGain_Q12); + if((status <= -1) && (status != -ISAC_DISALLOWED_BITSTREAM_LENGTH)) /*LPandHP = HP16b*/ + { + if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + // If this is the second 30ms of a 60ms frame reset this such that in the next call + // encoder starts fresh. + ISACenc_obj->frame_nb = 0; + } + return status; + } + + if((frame_mode == 1) && (ISACenc_obj->frame_nb == 0)) + { + // it is a 60ms and we are in the first 30ms + // then the limit at this point should be half of the assigned value + payloadLimitBytes = ISACenc_obj->payloadLimitBytes60 >> 1; + } + else if (frame_mode == 0) + { + // it is a 30ms frame + payloadLimitBytes = (ISACenc_obj->payloadLimitBytes30) - 3; + } + else + { + // this is the second half of a 60ms frame. + payloadLimitBytes = ISACenc_obj->payloadLimitBytes60 - 3; // subract 3 because termination process may add 3 bytes + } + + iterCntr = 0; + while((((ISACenc_obj->bitstr_obj.stream_index) << 1) > payloadLimitBytes) || + (status == -ISAC_DISALLOWED_BITSTREAM_LENGTH)) + { + int16_t arithLenDFTByte; + int16_t bytesLeftQ5; + int16_t ratioQ5[8] = {0, 6, 9, 12, 16, 19, 22, 25}; + + // According to experiments on TIMIT the following is proper for audio, but it is not agressive enough for tonal inputs + // such as DTMF, sweep-sine, ... + // + // (0.55 - (0.8 - ratio[i]/32) * 5 / 6) * 2^14 + // int16_t scaleQ14[8] = {0, 648, 1928, 3208, 4915, 6195, 7475, 8755}; + + + // This is a supper-agressive scaling passed the tests (tonal inputs) tone with one iteration for payload limit + // of 120 (32kbps bottleneck), number of frames needed a rate-reduction was 58403 + // + int16_t scaleQ14[8] = {0, 348, 828, 1408, 2015, 3195, 3500, 3500}; + int16_t idx; + + if(iterCntr >= MAX_PAYLOAD_LIMIT_ITERATION) + { + // We were not able to limit the payload size + + if((frame_mode == 1) && (ISACenc_obj->frame_nb == 0)) + { + // This was the first 30ms of a 60ms frame. Although the payload is larger than it + // should be but we let the second 30ms be encoded. Maybe togetehr we won't exceed + // the limit. + ISACenc_obj->frame_nb = 1; + return 0; + } + else if((frame_mode == 1) && (ISACenc_obj->frame_nb == 1)) + { + ISACenc_obj->frame_nb = 0; + } + + if(status != -ISAC_DISALLOWED_BITSTREAM_LENGTH) + { + return -ISAC_PAYLOAD_LARGER_THAN_LIMIT; + } + else + { + return status; + } + } + if(status != -ISAC_DISALLOWED_BITSTREAM_LENGTH) + { + arithLenDFTByte = (ISACenc_obj->bitstr_obj.stream_index << 1) + (1-ISACenc_obj->bitstr_obj.full) - arithLenBeforeEncodingDFT; + bytesLeftQ5 = (payloadLimitBytes - arithLenBeforeEncodingDFT) << 5; + + // bytesLeft / arithLenDFTBytes indicates how much scaling is required a rough estimate (agressive) + // scale = 0.55 - (0.8 - bytesLeft / arithLenDFTBytes) * 5 / 6 + // bytesLeft / arithLenDFTBytes below 0.2 will have a scale of zero and above 0.8 are treated as 0.8 + // to avoid division we do more simplification. + // + // values of (bytesLeft / arithLenDFTBytes)*32 between ratioQ5[i] and ratioQ5[i+1] are rounded to ratioQ5[i] + // and the corresponding scale is chosen + + // we compare bytesLeftQ5 with ratioQ5[]*arithLenDFTByte; + idx = 4; + idx += (bytesLeftQ5 >= ratioQ5[idx] * arithLenDFTByte) ? 2 : -2; + idx += (bytesLeftQ5 >= ratioQ5[idx] * arithLenDFTByte) ? 1 : -1; + idx += (bytesLeftQ5 >= ratioQ5[idx] * arithLenDFTByte) ? 0 : -1; + } + else + { + // we are here because the bit-stream did not fit into the buffer, in this case, the stream_index is not + // trustable, especially if the is the first 30ms of a packet. Thereforem, we will go for the most agressive + // case. + idx = 0; + } + // scale FFT coefficients to reduce the bit-rate + for(k = 0; k < FRAMESAMPLES_HALF; k++) + { + LP16a[k] = (int16_t)(LP16a[k] * scaleQ14[idx] >> 14); + LPandHP[k] = (int16_t)(LPandHP[k] * scaleQ14[idx] >> 14); + } + + // Save data for multiple packets memory + if (ISACenc_obj->SaveEnc_ptr != NULL) + { + for(k = 0; k < FRAMESAMPLES_HALF; k++) + { + (ISACenc_obj->SaveEnc_ptr)->fre[k + (ISACenc_obj->SaveEnc_ptr)->startIdx*FRAMESAMPLES_HALF] = LP16a[k]; + (ISACenc_obj->SaveEnc_ptr)->fim[k + (ISACenc_obj->SaveEnc_ptr)->startIdx*FRAMESAMPLES_HALF] = LPandHP[k]; + } + } + + // scale the unquantized LPC gains and save the scaled version for the future use + for(k = 0; k < KLT_ORDER_GAIN; k++) + { + gain_lo_hiQ17[k] = WEBRTC_SPL_MUL_16_32_RSFT14(scaleQ14[idx], transcodingParam.lpcGains[k]);//transcodingParam.lpcGains[k]; // + transcodingParam.lpcGains[k] = gain_lo_hiQ17[k]; + } + + // reset the bit-stream object to the state which it had before encoding LPC Gains + ISACenc_obj->bitstr_obj.full = transcodingParam.full; + ISACenc_obj->bitstr_obj.stream_index = transcodingParam.stream_index; + ISACenc_obj->bitstr_obj.streamval = transcodingParam.streamval; + ISACenc_obj->bitstr_obj.W_upper = transcodingParam.W_upper; + ISACenc_obj->bitstr_obj.stream[transcodingParam.stream_index-1] = transcodingParam.beforeLastWord; + ISACenc_obj->bitstr_obj.stream[transcodingParam.stream_index] = transcodingParam.lastWord; + + + // quantize and encode LPC gain + WebRtcIsacfix_EstCodeLpcGain(gain_lo_hiQ17, &ISACenc_obj->bitstr_obj, ISACenc_obj->SaveEnc_ptr); + arithLenBeforeEncodingDFT = (ISACenc_obj->bitstr_obj.stream_index << 1) + (1-ISACenc_obj->bitstr_obj.full); + status = WebRtcIsacfix_EncodeSpec(LP16a, LPandHP, &ISACenc_obj->bitstr_obj, AvgPitchGain_Q12); + if((status <= -1) && (status != -ISAC_DISALLOWED_BITSTREAM_LENGTH)) /*LPandHP = HP16b*/ + { + if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + // If this is the second 30ms of a 60ms frame reset this such that in the next call + // encoder starts fresh. + ISACenc_obj->frame_nb = 0; + } + return status; + } + iterCntr++; + } + + if (frame_mode == 1 && ISACenc_obj->frame_nb == 0) + /* i.e. 60 ms framesize and just processed the first 30ms, */ + /* go back to main function to buffer the other 30ms speech frame */ + { + ISACenc_obj->frame_nb = 1; + return 0; + } + else if (frame_mode == 1 && ISACenc_obj->frame_nb == 1) + { + ISACenc_obj->frame_nb = 0; + /* also update the framelength for next packet, in Adaptive mode only */ + if (CodingMode == 0 && (ISACenc_obj->enforceFrameSize == 0)) { + ISACenc_obj->new_framelength = WebRtcIsacfix_GetNewFrameLength(ISACenc_obj->BottleNeck, + ISACenc_obj->current_framesamples); + } + } + + + /* complete arithmetic coding */ + stream_length = WebRtcIsacfix_EncTerminate(&ISACenc_obj->bitstr_obj); + /* can this be negative? */ + + if(CodingMode == 0) + { + + /* update rate model and get minimum number of bytes in this packet */ + MinBytes = WebRtcIsacfix_GetMinBytes(&ISACenc_obj->rate_data_obj, (int16_t) stream_length, + ISACenc_obj->current_framesamples, ISACenc_obj->BottleNeck, ISACenc_obj->MaxDelay); + + /* if bitstream is too short, add garbage at the end */ + + /* Store length of coded data */ + usefulstr_len = stream_length; + + /* Make sure MinBytes does not exceed packet size limit */ + if ((ISACenc_obj->frame_nb == 0) && (MinBytes > ISACenc_obj->payloadLimitBytes30)) { + MinBytes = ISACenc_obj->payloadLimitBytes30; + } else if ((ISACenc_obj->frame_nb == 1) && (MinBytes > ISACenc_obj->payloadLimitBytes60)) { + MinBytes = ISACenc_obj->payloadLimitBytes60; + } + + /* Make sure we don't allow more than 255 bytes of garbage data. + We store the length of the garbage data in 8 bits in the bitstream, + 255 is the max garbage lenght we can signal using 8 bits. */ + if( MinBytes > usefulstr_len + 255 ) { + MinBytes = usefulstr_len + 255; + } + + /* Save data for creation of multiple bitstreams */ + if (ISACenc_obj->SaveEnc_ptr != NULL) { + (ISACenc_obj->SaveEnc_ptr)->minBytes = MinBytes; + } + + while (stream_length < MinBytes) + { + RTC_DCHECK_GE(stream_length, 0); + if (stream_length & 0x0001){ + ISACenc_obj->bitstr_seed = WEBRTC_SPL_RAND( ISACenc_obj->bitstr_seed ); + ISACenc_obj->bitstr_obj.stream[stream_length / 2] |= + (uint16_t)(ISACenc_obj->bitstr_seed & 0xFF); + } else { + ISACenc_obj->bitstr_seed = WEBRTC_SPL_RAND( ISACenc_obj->bitstr_seed ); + ISACenc_obj->bitstr_obj.stream[stream_length / 2] = + ((uint16_t)ISACenc_obj->bitstr_seed << 8); + } + stream_length++; + } + + /* to get the real stream_length, without garbage */ + if (usefulstr_len & 0x0001) { + ISACenc_obj->bitstr_obj.stream[usefulstr_len>>1] &= 0xFF00; + ISACenc_obj->bitstr_obj.stream[usefulstr_len>>1] += (MinBytes - usefulstr_len) & 0x00FF; + } + else { + ISACenc_obj->bitstr_obj.stream[usefulstr_len>>1] &= 0x00FF; + ISACenc_obj->bitstr_obj.stream[usefulstr_len >> 1] += + ((uint16_t)((MinBytes - usefulstr_len) & 0x00FF) << 8); + } + } + else + { + /* update rate model */ + WebRtcIsacfix_UpdateRateModel(&ISACenc_obj->rate_data_obj, (int16_t) stream_length, + ISACenc_obj->current_framesamples, ISACenc_obj->BottleNeck); + } + return stream_length; +} + +/* This function is used to create a new bitstream with new BWE. + The same data as previously encoded with the fucntion WebRtcIsacfix_EncodeImpl() + is used. The data needed is taken from the struct, where it was stored + when calling the encoder. */ +int WebRtcIsacfix_EncodeStoredData(IsacFixEncoderInstance *ISACenc_obj, + int BWnumber, + float scale) +{ + int ii; + int status; + int16_t BWno = (int16_t)BWnumber; + int stream_length = 0; + + int16_t model; + const uint16_t *Q_PitchGain_cdf_ptr[1]; + const uint16_t **cdf; + const IsacSaveEncoderData *SaveEnc_str; + int32_t tmpLPCcoeffs_g[KLT_ORDER_GAIN<<1]; + int16_t tmpLPCindex_g[KLT_ORDER_GAIN<<1]; + int16_t tmp_fre[FRAMESAMPLES]; + int16_t tmp_fim[FRAMESAMPLES]; + + SaveEnc_str = ISACenc_obj->SaveEnc_ptr; + + /* Check if SaveEnc memory exists */ + if (SaveEnc_str == NULL) { + return (-1); + } + + /* Sanity Check - possible values for BWnumber is 0 - 23 */ + if ((BWnumber < 0) || (BWnumber > 23)) { + return -ISAC_RANGE_ERROR_BW_ESTIMATOR; + } + + /* reset bitstream */ + ISACenc_obj->bitstr_obj.W_upper = 0xFFFFFFFF; + ISACenc_obj->bitstr_obj.streamval = 0; + ISACenc_obj->bitstr_obj.stream_index = 0; + ISACenc_obj->bitstr_obj.full = 1; + + /* encode frame length */ + status = WebRtcIsacfix_EncodeFrameLen(SaveEnc_str->framelength, &ISACenc_obj->bitstr_obj); + if (status < 0) { + /* Wrong frame size */ + return status; + } + + /* encode bandwidth estimate */ + status = WebRtcIsacfix_EncodeReceiveBandwidth(&BWno, &ISACenc_obj->bitstr_obj); + if (status < 0) { + return status; + } + + /* Transcoding */ + /* If scale < 1, rescale data to produce lower bitrate signal */ + if ((0.0 < scale) && (scale < 1.0)) { + /* Compensate LPC gain */ + for (ii = 0; ii < (KLT_ORDER_GAIN*(1+SaveEnc_str->startIdx)); ii++) { + tmpLPCcoeffs_g[ii] = (int32_t) ((scale) * (float) SaveEnc_str->LPCcoeffs_g[ii]); + } + + /* Scale DFT */ + for (ii = 0; ii < (FRAMESAMPLES_HALF*(1+SaveEnc_str->startIdx)); ii++) { + tmp_fre[ii] = (int16_t) ((scale) * (float) SaveEnc_str->fre[ii]) ; + tmp_fim[ii] = (int16_t) ((scale) * (float) SaveEnc_str->fim[ii]) ; + } + } else { + for (ii = 0; ii < (KLT_ORDER_GAIN*(1+SaveEnc_str->startIdx)); ii++) { + tmpLPCindex_g[ii] = SaveEnc_str->LPCindex_g[ii]; + } + + for (ii = 0; ii < (FRAMESAMPLES_HALF*(1+SaveEnc_str->startIdx)); ii++) { + tmp_fre[ii] = SaveEnc_str->fre[ii]; + tmp_fim[ii] = SaveEnc_str->fim[ii]; + } + } + + /* Loop over number of 30 msec */ + for (ii = 0; ii <= SaveEnc_str->startIdx; ii++) + { + + /* encode pitch gains */ + *Q_PitchGain_cdf_ptr = WebRtcIsacfix_kPitchGainCdf; + status = WebRtcIsacfix_EncHistMulti(&ISACenc_obj->bitstr_obj, &SaveEnc_str->pitchGain_index[ii], + Q_PitchGain_cdf_ptr, 1); + if (status < 0) { + return status; + } + + /* entropy coding of quantization pitch lags */ + /* voicing classificiation */ + if (SaveEnc_str->meanGain[ii] <= 819) { + cdf = WebRtcIsacfix_kPitchLagPtrLo; + } else if (SaveEnc_str->meanGain[ii] <= 1638) { + cdf = WebRtcIsacfix_kPitchLagPtrMid; + } else { + cdf = WebRtcIsacfix_kPitchLagPtrHi; + } + status = WebRtcIsacfix_EncHistMulti(&ISACenc_obj->bitstr_obj, + &SaveEnc_str->pitchIndex[PITCH_SUBFRAMES*ii], cdf, PITCH_SUBFRAMES); + if (status < 0) { + return status; + } + + /* LPC */ + /* entropy coding of model number */ + model = 0; + status = WebRtcIsacfix_EncHistMulti(&ISACenc_obj->bitstr_obj, &model, + WebRtcIsacfix_kModelCdfPtr, 1); + if (status < 0) { + return status; + } + + /* entropy coding of quantization indices - LPC shape only */ + status = WebRtcIsacfix_EncHistMulti(&ISACenc_obj->bitstr_obj, &SaveEnc_str->LPCindex_s[KLT_ORDER_SHAPE*ii], + WebRtcIsacfix_kCdfShapePtr[0], KLT_ORDER_SHAPE); + if (status < 0) { + return status; + } + + /* If transcoding, get new LPC gain indices */ + if (scale < 1.0) { + WebRtcIsacfix_TranscodeLpcCoef(&tmpLPCcoeffs_g[KLT_ORDER_GAIN*ii], &tmpLPCindex_g[KLT_ORDER_GAIN*ii]); + } + + /* entropy coding of quantization indices - LPC gain */ + status = WebRtcIsacfix_EncHistMulti(&ISACenc_obj->bitstr_obj, &tmpLPCindex_g[KLT_ORDER_GAIN*ii], + WebRtcIsacfix_kCdfGainPtr[0], KLT_ORDER_GAIN); + if (status < 0) { + return status; + } + + /* quantization and lossless coding */ + status = WebRtcIsacfix_EncodeSpec(&tmp_fre[ii*FRAMESAMPLES_HALF], &tmp_fim[ii*FRAMESAMPLES_HALF], + &ISACenc_obj->bitstr_obj, SaveEnc_str->AvgPitchGain[ii]); + if (status < 0) { + return status; + } + } + + /* complete arithmetic coding */ + stream_length = WebRtcIsacfix_EncTerminate(&ISACenc_obj->bitstr_obj); + + return stream_length; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.c new file mode 100644 index 0000000000..ed816c0ad7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.c @@ -0,0 +1,2056 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * entropy_coding.c + * + * This file contains all functions used to arithmetically + * encode the iSAC bistream. + * + */ + +#include + +#include "arith_routins.h" +#include "spectrum_ar_model_tables.h" +#include "pitch_gain_tables.h" +#include "pitch_lag_tables.h" +#include "entropy_coding.h" +#include "lpc_tables.h" +#include "settings.h" +#include "signal_processing_library.h" +#include "rtc_base/sanitizer.h" + +/* + * Eenumerations for arguments to functions WebRtcIsacfix_MatrixProduct1() + * and WebRtcIsacfix_MatrixProduct2(). +*/ + +enum matrix_index_factor { + kTIndexFactor1 = 1, + kTIndexFactor2 = 2, + kTIndexFactor3 = SUBFRAMES, + kTIndexFactor4 = LPC_SHAPE_ORDER +}; + +enum matrix_index_step { + kTIndexStep1 = 1, + kTIndexStep2 = SUBFRAMES, + kTIndexStep3 = LPC_SHAPE_ORDER +}; + +enum matrixprod_loop_count { + kTLoopCount1 = SUBFRAMES, + kTLoopCount2 = 2, + kTLoopCount3 = LPC_SHAPE_ORDER +}; + +enum matrix1_shift_value { + kTMatrix1_shift0 = 0, + kTMatrix1_shift1 = 1, + kTMatrix1_shift5 = 5 +}; + +enum matrixprod_init_case { + kTInitCase0 = 0, + kTInitCase1 = 1 +}; + +/* + This function implements the fix-point correspondant function to lrint. + + FLP: (int32_t)floor(flt+.499999999999) + FIP: (fixVal+roundVal)>>qDomain + + where roundVal = 2^(qDomain-1) = 1<<(qDomain-1) + +*/ +static __inline int32_t CalcLrIntQ(int32_t fixVal, int16_t qDomain) { + return (fixVal + (1 << (qDomain - 1))) >> qDomain; +} + +/* + __inline uint32_t stepwise(int32_t dinQ10) { + + int32_t ind, diQ10, dtQ10; + + diQ10 = dinQ10; + if (diQ10 < DPMIN_Q10) + diQ10 = DPMIN_Q10; + if (diQ10 >= DPMAX_Q10) + diQ10 = DPMAX_Q10 - 1; + + dtQ10 = diQ10 - DPMIN_Q10;*/ /* Q10 + Q10 = Q10 */ +/* ind = (dtQ10 * 5) >> 10; */ /* 2^10 / 5 = 0.2 in Q10 */ +/* Q10 -> Q0 */ + +/* return rpointsFIX_Q10[ind]; + + } +*/ + +/* logN(x) = logN(2)*log2(x) = 0.6931*log2(x). Output in Q8. */ +/* The input argument X to logN(X) is 2^17 times higher than the + input floating point argument Y to log(Y), since the X value + is a Q17 value. This can be compensated for after the call, by + subraction a value Z for each Q-step. One Q-step means that + X gets 2 thimes higher, i.e. Z = logN(2)*256 = 0.693147180559*256 = + 177.445678 should be subtracted (since logN() returns a Q8 value). + For a X value in Q17, the value 177.445678*17 = 3017 should be + subtracted */ +static int16_t CalcLogN(int32_t arg) { + int16_t zeros, log2, frac, logN; + + zeros=WebRtcSpl_NormU32(arg); + frac = (int16_t)((uint32_t)((arg << zeros) & 0x7FFFFFFF) >> 23); + log2 = (int16_t)(((31 - zeros) << 8) + frac); // log2(x) in Q8 + logN = (int16_t)(log2 * 22713 >> 15); // log(2) = 0.693147 = 22713 in Q15 + logN=logN+11; //Scalar compensation which minimizes the (log(x)-logN(x))^2 error over all x. + + return logN; +} + + +/* + expN(x) = 2^(a*x), where a = log2(e) ~= 1.442695 + + Input: Q8 (int16_t) + Output: Q17 (int32_t) + + a = log2(e) = log2(exp(1)) ~= 1.442695 ==> a = 23637 in Q14 (1.442688) + To this value, 700 is added or subtracted in order to get an average error + nearer zero, instead of always same-sign. +*/ + +static int32_t CalcExpN(int16_t x) { + int16_t axINT, axFRAC; + int16_t exp16; + int32_t exp; + int16_t ax = (int16_t)(x * 23637 >> 14); // Q8 + + if (x>=0) { + axINT = ax >> 8; //Q0 + axFRAC = ax&0x00FF; + exp16 = 1 << axINT; // Q0 + axFRAC = axFRAC+256; //Q8 + exp = exp16 * axFRAC; // Q0*Q8 = Q8 + exp <<= 9; // Q17 + } else { + ax = -ax; + axINT = 1 + (ax >> 8); //Q0 + axFRAC = 0x00FF - (ax&0x00FF); + exp16 = (int16_t)(32768 >> axINT); // Q15 + axFRAC = axFRAC+256; //Q8 + exp = exp16 * axFRAC; // Q15*Q8 = Q23 + exp >>= 6; // Q17 + } + + return exp; +} + + +/* compute correlation from power spectrum */ +static void CalcCorrelation(int32_t *PSpecQ12, int32_t *CorrQ7) +{ + int32_t summ[FRAMESAMPLES/8]; + int32_t diff[FRAMESAMPLES/8]; + int32_t sum; + int k, n; + + for (k = 0; k < FRAMESAMPLES/8; k++) { + summ[k] = (PSpecQ12[k] + PSpecQ12[FRAMESAMPLES / 4 - 1 - k] + 16) >> 5; + diff[k] = (PSpecQ12[k] - PSpecQ12[FRAMESAMPLES / 4 - 1 - k] + 16) >> 5; + } + + sum = 2; + for (n = 0; n < FRAMESAMPLES/8; n++) + sum += summ[n]; + CorrQ7[0] = sum; + + for (k = 0; k < AR_ORDER; k += 2) { + sum = 0; + for (n = 0; n < FRAMESAMPLES/8; n++) + sum += (WebRtcIsacfix_kCos[k][n] * diff[n] + 256) >> 9; + CorrQ7[k+1] = sum; + } + + for (k=1; k> 9; + CorrQ7[k+1] = sum; + } +} + +// Some arithmetic operations that are allowed to overflow. (It's still +// undefined behavior, so not a good idea; this just makes UBSan ignore the +// violations, so that our old code can continue to do what it's always been +// doing.) +static inline int32_t RTC_NO_SANITIZE("signed-integer-overflow") + OverflowingMulS16S32ToS32(int16_t a, int32_t b) { + return a * b; +} +static inline int32_t RTC_NO_SANITIZE("signed-integer-overflow") + OverflowingAddS32S32ToS32(int32_t a, int32_t b) { + return a + b; +} +static inline int32_t RTC_NO_SANITIZE("signed-integer-overflow") + OverflowingSubS32S32ToS32(int32_t a, int32_t b) { + return a - b; +} + +/* compute inverse AR power spectrum */ +static void CalcInvArSpec(const int16_t *ARCoefQ12, + const int32_t gainQ10, + int32_t *CurveQ16) +{ + int32_t CorrQ11[AR_ORDER+1]; + int32_t sum, tmpGain; + int32_t diffQ16[FRAMESAMPLES/8]; + const int16_t *CS_ptrQ9; + int k, n; + int16_t round, shftVal = 0, sh; + + sum = 0; + for (n = 0; n < AR_ORDER+1; n++) + sum += WEBRTC_SPL_MUL(ARCoefQ12[n], ARCoefQ12[n]); /* Q24 */ + sum = ((sum >> 6) * 65 + 32768) >> 16; /* Result in Q8. */ + CorrQ11[0] = (sum * gainQ10 + 256) >> 9; + + /* To avoid overflow, we shift down gainQ10 if it is large. We will not lose any precision */ + if(gainQ10>400000){ + tmpGain = gainQ10 >> 3; + round = 32; + shftVal = 6; + } else { + tmpGain = gainQ10; + round = 256; + shftVal = 9; + } + + for (k = 1; k < AR_ORDER+1; k++) { + sum = 16384; + for (n = k; n < AR_ORDER+1; n++) + sum += WEBRTC_SPL_MUL(ARCoefQ12[n-k], ARCoefQ12[n]); /* Q24 */ + sum >>= 15; + CorrQ11[k] = (sum * tmpGain + round) >> shftVal; + } + sum = CorrQ11[0] << 7; + for (n = 0; n < FRAMESAMPLES/8; n++) + CurveQ16[n] = sum; + + for (k = 1; k < AR_ORDER; k += 2) { + for (n = 0; n < FRAMESAMPLES/8; n++) + CurveQ16[n] += + (OverflowingMulS16S32ToS32(WebRtcIsacfix_kCos[k][n], CorrQ11[k + 1]) + + 2) >> + 2; + } + + CS_ptrQ9 = WebRtcIsacfix_kCos[0]; + + /* If CorrQ11[1] too large we avoid getting overflow in the calculation by shifting */ + sh=WebRtcSpl_NormW32(CorrQ11[1]); + if (CorrQ11[1]==0) /* Use next correlation */ + sh=WebRtcSpl_NormW32(CorrQ11[2]); + + if (sh<9) + shftVal = 9 - sh; + else + shftVal = 0; + + for (n = 0; n < FRAMESAMPLES/8; n++) + diffQ16[n] = (CS_ptrQ9[n] * (CorrQ11[1] >> shftVal) + 2) >> 2; + for (k = 2; k < AR_ORDER; k += 2) { + CS_ptrQ9 = WebRtcIsacfix_kCos[k]; + for (n = 0; n < FRAMESAMPLES/8; n++) + diffQ16[n] += (CS_ptrQ9[n] * (CorrQ11[k + 1] >> shftVal) + 2) >> 2; + } + + for (k=0; k> 6) * 65 + 32768) >> 16; /* Result in Q8. */ + CorrQ11[0] = (sum * gainQ10 + 256) >> 9; + + /* To avoid overflow, we shift down gainQ10 if it is large. We will not lose any precision */ + if(gainQ10>400000){ + tmpGain = gainQ10 >> 3; + round = 32; + shftVal = 6; + } else { + tmpGain = gainQ10; + round = 256; + shftVal = 9; + } + + for (k = 1; k < AR_ORDER+1; k++) { + sum = 16384; + for (n = k; n < AR_ORDER+1; n++) + sum += WEBRTC_SPL_MUL(ARCoefQ12[n-k], ARCoefQ12[n]); /* Q24 */ + sum >>= 15; + CorrQ11[k] = (sum * tmpGain + round) >> shftVal; + } + sum = CorrQ11[0] << 7; + for (n = 0; n < FRAMESAMPLES/8; n++) + summQ16[n] = sum; + + for (k = 1; k < (AR_ORDER); k += 2) { + for (n = 0; n < FRAMESAMPLES/8; n++) + summQ16[n] += ((CorrQ11[k + 1] * WebRtcIsacfix_kCos[k][n]) + 2) >> 2; + } + + CS_ptrQ9 = WebRtcIsacfix_kCos[0]; + + /* If CorrQ11[1] too large we avoid getting overflow in the calculation by shifting */ + sh=WebRtcSpl_NormW32(CorrQ11[1]); + if (CorrQ11[1]==0) /* Use next correlation */ + sh=WebRtcSpl_NormW32(CorrQ11[2]); + + if (sh<9) + shftVal = 9 - sh; + else + shftVal = 0; + + for (n = 0; n < FRAMESAMPLES/8; n++) + diffQ16[n] = (CS_ptrQ9[n] * (CorrQ11[1] >> shftVal) + 2) >> 2; + for (k = 2; k < AR_ORDER; k += 2) { + CS_ptrQ9 = WebRtcIsacfix_kCos[k]; + for (n = 0; n < FRAMESAMPLES/8; n++) + diffQ16[n] += (CS_ptrQ9[n] * (CorrQ11[k + 1] >> shftVal) + 2) >> 2; + } + + in_sqrt = summQ16[0] + (diffQ16[0] << shftVal); + + /* convert to magnitude spectrum, by doing square-roots (modified from SPLIB) */ + res = 1 << (WebRtcSpl_GetSizeInBits(in_sqrt) >> 1); + + for (k = 0; k < FRAMESAMPLES/8; k++) + { + in_sqrt = summQ16[k] + (diffQ16[k] << shftVal); + i = 10; + + /* make in_sqrt positive to prohibit sqrt of negative values */ + if(in_sqrt<0) + in_sqrt=-in_sqrt; + + newRes = (in_sqrt / res + res) >> 1; + do + { + res = newRes; + newRes = (in_sqrt / res + res) >> 1; + } while (newRes != res && i-- > 0); + + CurveQ8[k] = (int16_t)newRes; + } + for (k = FRAMESAMPLES/8; k < FRAMESAMPLES/4; k++) { + + in_sqrt = summQ16[FRAMESAMPLES / 4 - 1 - k] - + (diffQ16[FRAMESAMPLES / 4 - 1 - k] << shftVal); + i = 10; + + /* make in_sqrt positive to prohibit sqrt of negative values */ + if(in_sqrt<0) + in_sqrt=-in_sqrt; + + newRes = (in_sqrt / res + res) >> 1; + do + { + res = newRes; + newRes = (in_sqrt / res + res) >> 1; + } while (newRes != res && i-- > 0); + + CurveQ8[k] = (int16_t)newRes; + } + +} + + + +/* generate array of dither samples in Q7 */ +static void GenerateDitherQ7(int16_t *bufQ7, + uint32_t seed, + int16_t length, + int16_t AvgPitchGain_Q12) +{ + int k; + int16_t dither1_Q7, dither2_Q7, dither_gain_Q14, shft; + + if (AvgPitchGain_Q12 < 614) /* this threshold should be equal to that in decode_spec() */ + { + for (k = 0; k < length-2; k += 3) + { + /* new random unsigned int32_t */ + seed = WEBRTC_SPL_UMUL(seed, 196314165) + 907633515; + + /* fixed-point dither sample between -64 and 64 (Q7) */ + dither1_Q7 = (int16_t)(((int32_t)(seed + 16777216)) >> 25); + + /* new random unsigned int32_t */ + seed = WEBRTC_SPL_UMUL(seed, 196314165) + 907633515; + + /* fixed-point dither sample between -64 and 64 */ + dither2_Q7 = (int16_t)(((int32_t)(seed + 16777216)) >> 25); + + shft = (int16_t)(WEBRTC_SPL_RSHIFT_U32(seed, 25) & 15); + if (shft < 5) + { + bufQ7[k] = dither1_Q7; + bufQ7[k+1] = dither2_Q7; + bufQ7[k+2] = 0; + } + else if (shft < 10) + { + bufQ7[k] = dither1_Q7; + bufQ7[k+1] = 0; + bufQ7[k+2] = dither2_Q7; + } + else + { + bufQ7[k] = 0; + bufQ7[k+1] = dither1_Q7; + bufQ7[k+2] = dither2_Q7; + } + } + } + else + { + dither_gain_Q14 = (int16_t)(22528 - WEBRTC_SPL_MUL(10, AvgPitchGain_Q12)); + + /* dither on half of the coefficients */ + for (k = 0; k < length-1; k += 2) + { + /* new random unsigned int32_t */ + seed = WEBRTC_SPL_UMUL(seed, 196314165) + 907633515; + + /* fixed-point dither sample between -64 and 64 */ + dither1_Q7 = (int16_t)(((int32_t)(seed + 16777216)) >> 25); + + /* dither sample is placed in either even or odd index */ + shft = (int16_t)(WEBRTC_SPL_RSHIFT_U32(seed, 25) & 1); /* either 0 or 1 */ + + bufQ7[k + shft] = (int16_t)((dither_gain_Q14 * dither1_Q7 + 8192) >> 14); + bufQ7[k + 1 - shft] = 0; + } + } +} + + + + +/* + * function to decode the complex spectrum from the bitstream + * returns the total number of bytes in the stream + */ +int WebRtcIsacfix_DecodeSpec(Bitstr_dec *streamdata, + int16_t *frQ7, + int16_t *fiQ7, + int16_t AvgPitchGain_Q12) +{ + int16_t data[FRAMESAMPLES]; + int32_t invARSpec2_Q16[FRAMESAMPLES/4]; + int16_t ARCoefQ12[AR_ORDER+1]; + int16_t RCQ15[AR_ORDER]; + int16_t gainQ10; + int32_t gain2_Q10; + int len; + int k; + + /* create dither signal */ + GenerateDitherQ7(data, streamdata->W_upper, FRAMESAMPLES, AvgPitchGain_Q12); /* Dither is output in vector 'Data' */ + + /* decode model parameters */ + if (WebRtcIsacfix_DecodeRcCoef(streamdata, RCQ15) < 0) + return -ISAC_RANGE_ERROR_DECODE_SPECTRUM; + + + WebRtcSpl_ReflCoefToLpc(RCQ15, AR_ORDER, ARCoefQ12); + + if (WebRtcIsacfix_DecodeGain2(streamdata, &gain2_Q10) < 0) + return -ISAC_RANGE_ERROR_DECODE_SPECTRUM; + + /* compute inverse AR power spectrum */ + CalcInvArSpec(ARCoefQ12, gain2_Q10, invARSpec2_Q16); + + /* arithmetic decoding of spectrum */ + /* 'data' input and output. Input = Dither */ + len = WebRtcIsacfix_DecLogisticMulti2(data, streamdata, invARSpec2_Q16, (int16_t)FRAMESAMPLES); + + if (len<1) + return -ISAC_RANGE_ERROR_DECODE_SPECTRUM; + + /* subtract dither and scale down spectral samples with low SNR */ + if (AvgPitchGain_Q12 <= 614) + { + for (k = 0; k < FRAMESAMPLES; k += 4) + { + gainQ10 = WebRtcSpl_DivW32W16ResW16(30 << 10, + (int16_t)((uint32_t)(invARSpec2_Q16[k >> 2] + 2195456) >> 16)); + *frQ7++ = (int16_t)((data[k] * gainQ10 + 512) >> 10); + *fiQ7++ = (int16_t)((data[k + 1] * gainQ10 + 512) >> 10); + *frQ7++ = (int16_t)((data[k + 2] * gainQ10 + 512) >> 10); + *fiQ7++ = (int16_t)((data[k + 3] * gainQ10 + 512) >> 10); + } + } + else + { + for (k = 0; k < FRAMESAMPLES; k += 4) + { + gainQ10 = WebRtcSpl_DivW32W16ResW16(36 << 10, + (int16_t)((uint32_t)(invARSpec2_Q16[k >> 2] + 2654208) >> 16)); + *frQ7++ = (int16_t)((data[k] * gainQ10 + 512) >> 10); + *fiQ7++ = (int16_t)((data[k + 1] * gainQ10 + 512) >> 10); + *frQ7++ = (int16_t)((data[k + 2] * gainQ10 + 512) >> 10); + *fiQ7++ = (int16_t)((data[k + 3] * gainQ10 + 512) >> 10); + } + } + + return len; +} + + +int WebRtcIsacfix_EncodeSpec(const int16_t *fr, + const int16_t *fi, + Bitstr_enc *streamdata, + int16_t AvgPitchGain_Q12) +{ + int16_t dataQ7[FRAMESAMPLES]; + int32_t PSpec[FRAMESAMPLES/4]; + uint16_t invARSpecQ8[FRAMESAMPLES/4]; + int32_t CorrQ7[AR_ORDER+1]; + int32_t CorrQ7_norm[AR_ORDER+1]; + int16_t RCQ15[AR_ORDER]; + int16_t ARCoefQ12[AR_ORDER+1]; + int32_t gain2_Q10; + int16_t val; + int32_t nrg; + uint32_t sum; + int16_t lft_shft; + int16_t status; + int k, n, j; + + + /* create dither_float signal */ + GenerateDitherQ7(dataQ7, streamdata->W_upper, FRAMESAMPLES, AvgPitchGain_Q12); + + /* add dither and quantize, and compute power spectrum */ + /* Vector dataQ7 contains Dither in Q7 */ + for (k = 0; k < FRAMESAMPLES; k += 4) + { + val = ((*fr++ + dataQ7[k] + 64) & 0xFF80) - dataQ7[k]; /* Data = Dither */ + dataQ7[k] = val; /* New value in Data */ + sum = WEBRTC_SPL_UMUL(val, val); + + val = ((*fi++ + dataQ7[k+1] + 64) & 0xFF80) - dataQ7[k+1]; /* Data = Dither */ + dataQ7[k+1] = val; /* New value in Data */ + sum += WEBRTC_SPL_UMUL(val, val); + + val = ((*fr++ + dataQ7[k+2] + 64) & 0xFF80) - dataQ7[k+2]; /* Data = Dither */ + dataQ7[k+2] = val; /* New value in Data */ + sum += WEBRTC_SPL_UMUL(val, val); + + val = ((*fi++ + dataQ7[k+3] + 64) & 0xFF80) - dataQ7[k+3]; /* Data = Dither */ + dataQ7[k+3] = val; /* New value in Data */ + sum += WEBRTC_SPL_UMUL(val, val); + + PSpec[k>>2] = WEBRTC_SPL_RSHIFT_U32(sum, 2); + } + + /* compute correlation from power spectrum */ + CalcCorrelation(PSpec, CorrQ7); + + + /* find AR coefficients */ + /* number of bit shifts to 14-bit normalize CorrQ7[0] (leaving room for sign) */ + lft_shft = WebRtcSpl_NormW32(CorrQ7[0]) - 18; + + if (lft_shft > 0) { + for (k=0; k> -lft_shft; + } + + /* find RC coefficients */ + WebRtcSpl_AutoCorrToReflCoef(CorrQ7_norm, AR_ORDER, RCQ15); + + /* quantize & code RC Coef */ + status = WebRtcIsacfix_EncodeRcCoef(RCQ15, streamdata); + if (status < 0) { + return status; + } + + /* RC -> AR coefficients */ + WebRtcSpl_ReflCoefToLpc(RCQ15, AR_ORDER, ARCoefQ12); + + /* compute ARCoef' * Corr * ARCoef in Q19 */ + nrg = 0; + for (j = 0; j <= AR_ORDER; j++) { + for (n = 0; n <= j; n++) + nrg += (ARCoefQ12[j] * ((CorrQ7_norm[j - n] * ARCoefQ12[n] + 256) >> 9) + + 4) >> 3; + for (n = j+1; n <= AR_ORDER; n++) + nrg += (ARCoefQ12[j] * ((CorrQ7_norm[n - j] * ARCoefQ12[n] + 256) >> 9) + + 4) >> 3; + } + + if (lft_shft > 0) + nrg >>= lft_shft; + else + nrg <<= -lft_shft; + + if(nrg>131072) + gain2_Q10 = WebRtcSpl_DivResultInQ31(FRAMESAMPLES >> 2, nrg); /* also shifts 31 bits to the left! */ + else + gain2_Q10 = FRAMESAMPLES >> 2; + + /* quantize & code gain2_Q10 */ + if (WebRtcIsacfix_EncodeGain2(&gain2_Q10, streamdata)) + return -1; + + /* compute inverse AR magnitude spectrum */ + CalcRootInvArSpec(ARCoefQ12, gain2_Q10, invARSpecQ8); + + + /* arithmetic coding of spectrum */ + status = WebRtcIsacfix_EncLogisticMulti2(streamdata, dataQ7, invARSpecQ8, (int16_t)FRAMESAMPLES); + if ( status ) + return( status ); + + return 0; +} + + +/* Matlab's LAR definition */ +static void Rc2LarFix(const int16_t *rcQ15, int32_t *larQ17, int16_t order) { + + /* + + This is a piece-wise implemenetation of a rc2lar-function (all values in the comment + are Q15 values and are based on [0 24956/32768 30000/32768 32500/32768], i.e. + [0.76159667968750 0.91552734375000 0.99182128906250] + + x0 x1 a k x0(again) b + ================================================================================== + 0.00 0.76: 0 2.625997508581 0 0 + 0.76 0.91: 2.000012018559 7.284502668663 0.761596679688 -3.547841027073 + 0.91 0.99: 3.121320351712 31.115835041229 0.915527343750 -25.366077452148 + 0.99 1.00: 5.495270168700 686.663805654056 0.991821289063 -675.552510708011 + + The implementation is y(x)= a + (x-x0)*k, but this can be simplified to + + y(x) = a-x0*k + x*k = b + x*k, where b = a-x0*k + + akx=[0 2.625997508581 0 + 2.000012018559 7.284502668663 0.761596679688 + 3.121320351712 31.115835041229 0.915527343750 + 5.495270168700 686.663805654056 0.991821289063]; + + b = akx(:,1) - akx(:,3).*akx(:,2) + + [ 0.0 + -3.547841027073 + -25.366077452148 + -675.552510708011] + + */ + + int k; + int16_t rc; + int32_t larAbsQ17; + + for (k = 0; k < order; k++) { + + rc = WEBRTC_SPL_ABS_W16(rcQ15[k]); //Q15 + + /* Calculate larAbsQ17 in Q17 from rc in Q15 */ + + if (rc<24956) { //0.7615966 in Q15 + // (Q15*Q13)>>11 = Q17 + larAbsQ17 = rc * 21512 >> 11; + } else if (rc<30000) { //0.91552734375 in Q15 + // Q17 + (Q15*Q12)>>10 = Q17 + larAbsQ17 = -465024 + (rc * 29837 >> 10); + } else if (rc<32500) { //0.99182128906250 in Q15 + // Q17 + (Q15*Q10)>>8 = Q17 + larAbsQ17 = -3324784 + (rc * 31863 >> 8); + } else { + // Q17 + (Q15*Q5)>>3 = Q17 + larAbsQ17 = -88546020 + (rc * 21973 >> 3); + } + + if (rcQ15[k]>0) { + larQ17[k] = larAbsQ17; + } else { + larQ17[k] = -larAbsQ17; + } + } +} + + +static void Lar2RcFix(const int32_t *larQ17, int16_t *rcQ15, int16_t order) { + + /* + This is a piece-wise implemenetation of a lar2rc-function + See comment in Rc2LarFix() about details. + */ + + int k; + int16_t larAbsQ11; + int32_t rc; + + for (k = 0; k < order; k++) { + + larAbsQ11 = (int16_t)WEBRTC_SPL_ABS_W32((larQ17[k] + 32) >> 6); // Q11 + + if (larAbsQ11<4097) { //2.000012018559 in Q11 + // Q11*Q16>>12 = Q15 + rc = larAbsQ11 * 24957 >> 12; + } else if (larAbsQ11<6393) { //3.121320351712 in Q11 + // (Q11*Q17 + Q13)>>13 = Q15 + rc = (larAbsQ11 * 17993 + 130738688) >> 13; + } else if (larAbsQ11<11255) { //5.495270168700 in Q11 + // (Q11*Q19 + Q30)>>15 = Q15 + rc = (larAbsQ11 * 16850 + 875329820) >> 15; + } else { + // (Q11*Q24>>16 + Q19)>>4 = Q15 + rc = (((larAbsQ11 * 24433) >> 16) + 515804) >> 4; + } + + if (larQ17[k]<=0) { + rc = -rc; + } + + rcQ15[k] = (int16_t) rc; // Q15 + } +} + +static void Poly2LarFix(int16_t *lowbandQ15, + int16_t orderLo, + int16_t *hibandQ15, + int16_t orderHi, + int16_t Nsub, + int32_t *larsQ17) { + + int k, n; + int32_t *outpQ17; + int16_t orderTot; + int32_t larQ17[MAX_ORDER]; // Size 7+6 is enough + + orderTot = (orderLo + orderHi); + outpQ17 = larsQ17; + for (k = 0; k < Nsub; k++) { + + Rc2LarFix(lowbandQ15, larQ17, orderLo); + + for (n = 0; n < orderLo; n++) + outpQ17[n] = larQ17[n]; //Q17 + + Rc2LarFix(hibandQ15, larQ17, orderHi); + + for (n = 0; n < orderHi; n++) + outpQ17[n + orderLo] = larQ17[n]; //Q17; + + outpQ17 += orderTot; + lowbandQ15 += orderLo; + hibandQ15 += orderHi; + } +} + + +static void Lar2polyFix(int32_t *larsQ17, + int16_t *lowbandQ15, + int16_t orderLo, + int16_t *hibandQ15, + int16_t orderHi, + int16_t Nsub) { + + int k, n; + int16_t orderTot; + int16_t *outplQ15, *outphQ15; + int32_t *inpQ17; + int16_t rcQ15[7+6]; + + orderTot = (orderLo + orderHi); + outplQ15 = lowbandQ15; + outphQ15 = hibandQ15; + inpQ17 = larsQ17; + for (k = 0; k < Nsub; k++) { + + /* gains not handled here as in the FLP version */ + + /* Low band */ + Lar2RcFix(&inpQ17[0], rcQ15, orderLo); + for (n = 0; n < orderLo; n++) + outplQ15[n] = rcQ15[n]; // Refl. coeffs + + /* High band */ + Lar2RcFix(&inpQ17[orderLo], rcQ15, orderHi); + for (n = 0; n < orderHi; n++) + outphQ15[n] = rcQ15[n]; // Refl. coeffs + + inpQ17 += orderTot; + outplQ15 += orderLo; + outphQ15 += orderHi; + } +} + +/* +Function WebRtcIsacfix_MatrixProduct1C() does one form of matrix multiplication. +It first shifts input data of one matrix, determines the right indexes for the +two matrixes, multiply them, and write the results into an output buffer. + +Note that two factors (or, multipliers) determine the initialization values of +the variable |matrix1_index| in the code. The relationship is +|matrix1_index| = |matrix1_index_factor1| * |matrix1_index_factor2|, where +|matrix1_index_factor1| is given by the argument while |matrix1_index_factor2| +is determined by the value of argument |matrix1_index_init_case|; +|matrix1_index_factor2| is the value of the outmost loop counter j (when +|matrix1_index_init_case| is 0), or the value of the middle loop counter k (when +|matrix1_index_init_case| is non-zero). + +|matrix0_index| is determined the same way. + +Arguments: + matrix0[]: matrix0 data in Q15 domain. + matrix1[]: matrix1 data. + matrix_product[]: output data (matrix product). + matrix1_index_factor1: The first of two factors determining the + initialization value of matrix1_index. + matrix0_index_factor1: The first of two factors determining the + initialization value of matrix0_index. + matrix1_index_init_case: Case number for selecting the second of two + factors determining the initialization value + of matrix1_index and matrix0_index. + matrix1_index_step: Incremental step for matrix1_index. + matrix0_index_step: Incremental step for matrix0_index. + inner_loop_count: Maximum count of the inner loop. + mid_loop_count: Maximum count of the intermediate loop. + shift: Left shift value for matrix1. +*/ +void WebRtcIsacfix_MatrixProduct1C(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix1_index_factor1, + const int matrix0_index_factor1, + const int matrix1_index_init_case, + const int matrix1_index_step, + const int matrix0_index_step, + const int inner_loop_count, + const int mid_loop_count, + const int shift) { + int j = 0, k = 0, n = 0; + int matrix0_index = 0, matrix1_index = 0, matrix_prod_index = 0; + int* matrix0_index_factor2 = &k; + int* matrix1_index_factor2 = &j; + if (matrix1_index_init_case != 0) { + matrix0_index_factor2 = &j; + matrix1_index_factor2 = &k; + } + + for (j = 0; j < SUBFRAMES; j++) { + matrix_prod_index = mid_loop_count * j; + for (k = 0; k < mid_loop_count; k++) { + int32_t sum32 = 0; + matrix0_index = matrix0_index_factor1 * (*matrix0_index_factor2); + matrix1_index = matrix1_index_factor1 * (*matrix1_index_factor2); + for (n = 0; n < inner_loop_count; n++) { + sum32 += WEBRTC_SPL_MUL_16_32_RSFT16( + matrix0[matrix0_index], matrix1[matrix1_index] * (1 << shift)); + matrix0_index += matrix0_index_step; + matrix1_index += matrix1_index_step; + } + matrix_product[matrix_prod_index] = sum32; + matrix_prod_index++; + } + } +} + +/* +Function WebRtcIsacfix_MatrixProduct2C() returns the product of two matrixes, +one of which has two columns. It first has to determine the correct index of +the first matrix before doing the actual element multiplication. + +Arguments: + matrix0[]: A matrix in Q15 domain. + matrix1[]: A matrix in Q21 domain. + matrix_product[]: Output data in Q17 domain. + matrix0_index_factor: A factor determining the initialization value + of matrix0_index. + matrix0_index_step: Incremental step for matrix0_index. +*/ +void WebRtcIsacfix_MatrixProduct2C(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix0_index_factor, + const int matrix0_index_step) { + int j = 0, n = 0; + int matrix1_index = 0, matrix0_index = 0, matrix_prod_index = 0; + for (j = 0; j < SUBFRAMES; j++) { + int32_t sum32 = 0, sum32_2 = 0; + matrix1_index = 0; + matrix0_index = matrix0_index_factor * j; + for (n = SUBFRAMES; n > 0; n--) { + sum32 += (WEBRTC_SPL_MUL_16_32_RSFT16(matrix0[matrix0_index], + matrix1[matrix1_index])); + sum32_2 += (WEBRTC_SPL_MUL_16_32_RSFT16(matrix0[matrix0_index], + matrix1[matrix1_index + 1])); + matrix1_index += 2; + matrix0_index += matrix0_index_step; + } + matrix_product[matrix_prod_index] = sum32 >> 3; + matrix_product[matrix_prod_index + 1] = sum32_2 >> 3; + matrix_prod_index += 2; + } +} + +int WebRtcIsacfix_DecodeLpc(int32_t *gain_lo_hiQ17, + int16_t *LPCCoef_loQ15, + int16_t *LPCCoef_hiQ15, + Bitstr_dec *streamdata, + int16_t *outmodel) { + + int32_t larsQ17[KLT_ORDER_SHAPE]; // KLT_ORDER_GAIN+KLT_ORDER_SHAPE == (ORDERLO+ORDERHI)*SUBFRAMES + int err; + + err = WebRtcIsacfix_DecodeLpcCoef(streamdata, larsQ17, gain_lo_hiQ17, outmodel); + if (err<0) // error check + return -ISAC_RANGE_ERROR_DECODE_LPC; + + Lar2polyFix(larsQ17, LPCCoef_loQ15, ORDERLO, LPCCoef_hiQ15, ORDERHI, SUBFRAMES); + + return 0; +} + +/* decode & dequantize LPC Coef */ +int WebRtcIsacfix_DecodeLpcCoef(Bitstr_dec *streamdata, + int32_t *LPCCoefQ17, + int32_t *gain_lo_hiQ17, + int16_t *outmodel) +{ + int j, k, n; + int err; + int16_t pos, pos2, posg, poss; + int16_t gainpos; + int16_t model; + int16_t index_QQ[KLT_ORDER_SHAPE]; + int32_t tmpcoeffs_gQ17[KLT_ORDER_GAIN]; + int32_t tmpcoeffs2_gQ21[KLT_ORDER_GAIN]; + int16_t tmpcoeffs_sQ10[KLT_ORDER_SHAPE]; + int32_t tmpcoeffs_sQ17[KLT_ORDER_SHAPE]; + int32_t tmpcoeffs2_sQ18[KLT_ORDER_SHAPE]; + int32_t sumQQ; + int16_t sumQQ16; + int32_t tmp32; + + + + /* entropy decoding of model number */ + err = WebRtcIsacfix_DecHistOneStepMulti(&model, streamdata, WebRtcIsacfix_kModelCdfPtr, WebRtcIsacfix_kModelInitIndex, 1); + if (err<0) // error check + return err; + + /* entropy decoding of quantization indices */ + err = WebRtcIsacfix_DecHistOneStepMulti(index_QQ, streamdata, WebRtcIsacfix_kCdfShapePtr[model], WebRtcIsacfix_kInitIndexShape[model], KLT_ORDER_SHAPE); + if (err<0) // error check + return err; + /* find quantization levels for coefficients */ + for (k=0; k> 7; // (Q10*Q15)>>7 = Q18 + pos++; + pos2++; + } + tmpcoeffs2_sQ18[poss] = sumQQ; //Q18 + poss++; + } + } + + /* right transform */ // Transpose matrix + WebRtcIsacfix_MatrixProduct2(WebRtcIsacfix_kT2GainQ15[0], tmpcoeffs2_gQ21, + tmpcoeffs_gQ17, kTIndexFactor1, kTIndexStep2); + WebRtcIsacfix_MatrixProduct1(WebRtcIsacfix_kT2ShapeQ15[model], + tmpcoeffs2_sQ18, tmpcoeffs_sQ17, kTIndexFactor1, kTIndexFactor1, + kTInitCase1, kTIndexStep3, kTIndexStep2, kTLoopCount1, kTLoopCount3, + kTMatrix1_shift0); + + /* scaling, mean addition, and gain restoration */ + gainpos = 0; + posg = 0;poss = 0;pos=0; + for (k=0; k> 11); + sumQQ16 += WebRtcIsacfix_kMeansGainQ8[model][posg]; + sumQQ = CalcExpN(sumQQ16); // Q8 in and Q17 out + gain_lo_hiQ17[gainpos] = sumQQ; //Q17 + gainpos++; + posg++; + + // Divide by 4 and get Q17 to Q8, i.e. shift 2+9. + sumQQ16 = (int16_t)(tmpcoeffs_gQ17[posg] >> 11); + sumQQ16 += WebRtcIsacfix_kMeansGainQ8[model][posg]; + sumQQ = CalcExpN(sumQQ16); // Q8 in and Q17 out + gain_lo_hiQ17[gainpos] = sumQQ; //Q17 + gainpos++; + posg++; + + /* lo band LAR coeffs */ + for (n=0; n>16 = Q17, with 1/2.1 = 0.47619047619 ~= 31208 in Q16 + tmp32 = tmp32 + WebRtcIsacfix_kMeansShapeQ17[model][poss]; // Q17+Q17 = Q17 + LPCCoefQ17[pos] = tmp32; + } + + /* hi band LAR coeffs */ + for (n=0; n>16)<<3 = Q17, with 1/0.45 = 2.222222222222 ~= 18204 in Q13 + tmp32 = + WEBRTC_SPL_MUL_16_32_RSFT16(18204, tmpcoeffs_sQ17[poss]) * (1 << 3); + tmp32 = tmp32 + WebRtcIsacfix_kMeansShapeQ17[model][poss]; // Q17+Q17 = Q17 + LPCCoefQ17[pos] = tmp32; + } + } + + + *outmodel=model; + + return 0; +} + +/* estimate codel length of LPC Coef */ +static int EstCodeLpcCoef(int32_t *LPCCoefQ17, + int32_t *gain_lo_hiQ17, + int16_t *model, + int32_t *sizeQ11, + Bitstr_enc *streamdata, + IsacSaveEncoderData* encData, + transcode_obj *transcodingParam) { + int j, k, n; + int16_t posQQ, pos2QQ, gainpos; + int16_t pos, poss, posg, offsg; + int16_t index_gQQ[KLT_ORDER_GAIN], index_sQQ[KLT_ORDER_SHAPE]; + int16_t index_ovr_gQQ[KLT_ORDER_GAIN], index_ovr_sQQ[KLT_ORDER_SHAPE]; + int32_t BitsQQ; + + int16_t tmpcoeffs_gQ6[KLT_ORDER_GAIN]; + int32_t tmpcoeffs_gQ17[KLT_ORDER_GAIN]; + int32_t tmpcoeffs_sQ17[KLT_ORDER_SHAPE]; + int32_t tmpcoeffs2_gQ21[KLT_ORDER_GAIN]; + int32_t tmpcoeffs2_sQ17[KLT_ORDER_SHAPE]; + int32_t sumQQ; + int32_t tmp32; + int16_t sumQQ16; + int status = 0; + + /* write LAR coefficients to statistics file */ + /* Save data for creation of multiple bitstreams (and transcoding) */ + if (encData != NULL) { + for (k=0; kLPCcoeffs_g[KLT_ORDER_GAIN*encData->startIdx + k] = gain_lo_hiQ17[k]; + } + } + + /* log gains, mean removal and scaling */ + posg = 0;poss = 0;pos=0; gainpos=0; + + for (k=0; k WebRtcIsacfix_kMaxIndGain[k]) { + index_gQQ[k] = WebRtcIsacfix_kMaxIndGain[k]; + } + index_ovr_gQQ[k] = WebRtcIsacfix_kOffsetGain[0][k]+index_gQQ[k]; + posQQ = WebRtcIsacfix_kOfLevelsGain[0] + index_ovr_gQQ[k]; + + /* Save data for creation of multiple bitstreams */ + if (encData != NULL) { + encData->LPCindex_g[KLT_ORDER_GAIN*encData->startIdx + k] = index_gQQ[k]; + } + + /* determine number of bits */ + sumQQ = WebRtcIsacfix_kCodeLenGainQ11[posQQ]; //Q11 + BitsQQ += sumQQ; + } + + for (k=0; k WebRtcIsacfix_kMaxIndShape[k]) + index_sQQ[k] = WebRtcIsacfix_kMaxIndShape[k]; + index_ovr_sQQ[k] = WebRtcIsacfix_kOffsetShape[0][k]+index_sQQ[k]; + + posQQ = WebRtcIsacfix_kOfLevelsShape[0] + index_ovr_sQQ[k]; + sumQQ = WebRtcIsacfix_kCodeLenShapeQ11[posQQ]; //Q11 + BitsQQ += sumQQ; + } + + + + *model = 0; + *sizeQ11=BitsQQ; + + /* entropy coding of model number */ + status = WebRtcIsacfix_EncHistMulti(streamdata, model, WebRtcIsacfix_kModelCdfPtr, 1); + if (status < 0) { + return status; + } + + /* entropy coding of quantization indices - shape only */ + status = WebRtcIsacfix_EncHistMulti(streamdata, index_sQQ, WebRtcIsacfix_kCdfShapePtr[0], KLT_ORDER_SHAPE); + if (status < 0) { + return status; + } + + /* Save data for creation of multiple bitstreams */ + if (encData != NULL) { + for (k=0; kLPCindex_s[KLT_ORDER_SHAPE*encData->startIdx + k] = index_sQQ[k]; + } + } + /* save the state of the bitstream object 'streamdata' for the possible bit-rate reduction */ + transcodingParam->full = streamdata->full; + transcodingParam->stream_index = streamdata->stream_index; + transcodingParam->streamval = streamdata->streamval; + transcodingParam->W_upper = streamdata->W_upper; + transcodingParam->beforeLastWord = streamdata->stream[streamdata->stream_index-1]; + transcodingParam->lastWord = streamdata->stream[streamdata->stream_index]; + + /* entropy coding of index */ + status = WebRtcIsacfix_EncHistMulti(streamdata, index_gQQ, WebRtcIsacfix_kCdfGainPtr[0], KLT_ORDER_GAIN); + if (status < 0) { + return status; + } + + /* find quantization levels for shape coefficients */ + for (k=0; k>16 = Q17, with 1/2.1 = 0.47619047619 ~= 31208 in Q16 + tmp32 = tmp32 + WebRtcIsacfix_kMeansShapeQ17[0][poss]; // Q17+Q17 = Q17 + LPCCoefQ17[pos] = tmp32; + } + + /* hi band LAR coeffs */ + for (n=0; n>16)<<3 = Q17, with 1/0.45 = 2.222222222222 ~= 18204 in Q13 + tmp32 = WEBRTC_SPL_MUL_16_32_RSFT16(18204, tmpcoeffs_sQ17[poss]) << 3; + tmp32 = tmp32 + WebRtcIsacfix_kMeansShapeQ17[0][poss]; // Q17+Q17 = Q17 + LPCCoefQ17[pos] = tmp32; + } + + } + + //to update tmpcoeffs_gQ17 to the proper state + for (k=0; k> (16 - 1) = Q17; Q17 << 4 = Q21. + sumQQ = (WEBRTC_SPL_MUL_16_32_RSFT16(WebRtcIsacfix_kT1GainQ15[0][0], + tmpcoeffs_gQ17[offsg]) << 1); + sumQQ += (WEBRTC_SPL_MUL_16_32_RSFT16(WebRtcIsacfix_kT1GainQ15[0][1], + tmpcoeffs_gQ17[offsg + 1]) << 1); + tmpcoeffs2_gQ21[posg] = sumQQ << 4; + posg++; + + sumQQ = (WEBRTC_SPL_MUL_16_32_RSFT16(WebRtcIsacfix_kT1GainQ15[0][2], + tmpcoeffs_gQ17[offsg]) << 1); + sumQQ += (WEBRTC_SPL_MUL_16_32_RSFT16(WebRtcIsacfix_kT1GainQ15[0][3], + tmpcoeffs_gQ17[offsg + 1]) << 1); + tmpcoeffs2_gQ21[posg] = sumQQ << 4; + posg++; + offsg += 2; + } + + /* right transform */ // Transpose matrix + WebRtcIsacfix_MatrixProduct2(WebRtcIsacfix_kT2GainQ15[0], tmpcoeffs2_gQ21, + tmpcoeffs_gQ17, kTIndexFactor1, kTIndexStep2); + + /* scaling, mean addition, and gain restoration */ + posg = 0; + gainpos = 0; + for (k=0; k<2*SUBFRAMES; k++) { + + // Divide by 4 and get Q17 to Q8, i.e. shift 2+9. + sumQQ16 = (int16_t)(tmpcoeffs_gQ17[posg] >> 11); + sumQQ16 += WebRtcIsacfix_kMeansGainQ8[0][posg]; + sumQQ = CalcExpN(sumQQ16); // Q8 in and Q17 out + gain_lo_hiQ17[gainpos] = sumQQ; //Q17 + + gainpos++; + pos++;posg++; + } + + return 0; +} + +int WebRtcIsacfix_EstCodeLpcGain(int32_t *gain_lo_hiQ17, + Bitstr_enc *streamdata, + IsacSaveEncoderData* encData) { + int j, k; + int16_t posQQ, pos2QQ, gainpos; + int16_t posg; + int16_t index_gQQ[KLT_ORDER_GAIN]; + + int16_t tmpcoeffs_gQ6[KLT_ORDER_GAIN]; + int32_t tmpcoeffs_gQ17[KLT_ORDER_GAIN]; + int32_t tmpcoeffs2_gQ21[KLT_ORDER_GAIN]; + int32_t sumQQ; + int status = 0; + + /* write LAR coefficients to statistics file */ + /* Save data for creation of multiple bitstreams (and transcoding) */ + if (encData != NULL) { + for (k=0; kLPCcoeffs_g[KLT_ORDER_GAIN*encData->startIdx + k] = gain_lo_hiQ17[k]; + } + } + + /* log gains, mean removal and scaling */ + posg = 0; gainpos = 0; + + for (k=0; k WebRtcIsacfix_kMaxIndGain[k]) { + index_gQQ[k] = WebRtcIsacfix_kMaxIndGain[k]; + } + + /* Save data for creation of multiple bitstreams */ + if (encData != NULL) { + encData->LPCindex_g[KLT_ORDER_GAIN*encData->startIdx + k] = index_gQQ[k]; + } + } + + /* entropy coding of index */ + status = WebRtcIsacfix_EncHistMulti(streamdata, index_gQQ, WebRtcIsacfix_kCdfGainPtr[0], KLT_ORDER_GAIN); + if (status < 0) { + return status; + } + + return 0; +} + + +int WebRtcIsacfix_EncodeLpc(int32_t *gain_lo_hiQ17, + int16_t *LPCCoef_loQ15, + int16_t *LPCCoef_hiQ15, + int16_t *model, + int32_t *sizeQ11, + Bitstr_enc *streamdata, + IsacSaveEncoderData* encData, + transcode_obj *transcodeParam) +{ + int status = 0; + int32_t larsQ17[KLT_ORDER_SHAPE]; // KLT_ORDER_SHAPE == (ORDERLO+ORDERHI)*SUBFRAMES + // = (6+12)*6 == 108 + + Poly2LarFix(LPCCoef_loQ15, ORDERLO, LPCCoef_hiQ15, ORDERHI, SUBFRAMES, larsQ17); + + status = EstCodeLpcCoef(larsQ17, gain_lo_hiQ17, model, sizeQ11, + streamdata, encData, transcodeParam); + if (status < 0) { + return (status); + } + + Lar2polyFix(larsQ17, LPCCoef_loQ15, ORDERLO, LPCCoef_hiQ15, ORDERHI, SUBFRAMES); + + return 0; +} + + +/* decode & dequantize RC */ +int WebRtcIsacfix_DecodeRcCoef(Bitstr_dec *streamdata, int16_t *RCQ15) +{ + int k, err; + int16_t index[AR_ORDER]; + + /* entropy decoding of quantization indices */ + err = WebRtcIsacfix_DecHistOneStepMulti(index, streamdata, WebRtcIsacfix_kRcCdfPtr, WebRtcIsacfix_kRcInitInd, AR_ORDER); + if (err<0) // error check + return err; + + /* find quantization levels for reflection coefficients */ + for (k=0; k WebRtcIsacfix_kRcBound[index[k]]) + { + while (RCQ15[k] > WebRtcIsacfix_kRcBound[index[k] + 1]) + index[k]++; + } + else + { + while (RCQ15[k] < WebRtcIsacfix_kRcBound[--index[k]]) ; + } + + RCQ15[k] = *(WebRtcIsacfix_kRcLevPtr[k] + index[k]); + } + + + /* entropy coding of quantization indices */ + status = WebRtcIsacfix_EncHistMulti(streamdata, index, WebRtcIsacfix_kRcCdfPtr, AR_ORDER); + + /* If error in WebRtcIsacfix_EncHistMulti(), status will be negative, otherwise 0 */ + return status; +} + + +/* decode & dequantize squared Gain */ +int WebRtcIsacfix_DecodeGain2(Bitstr_dec *streamdata, int32_t *gainQ10) +{ + int err; + int16_t index; + + /* entropy decoding of quantization index */ + err = WebRtcIsacfix_DecHistOneStepMulti( + &index, + streamdata, + WebRtcIsacfix_kGainPtr, + WebRtcIsacfix_kGainInitInd, + 1); + /* error check */ + if (err<0) { + return err; + } + + /* find quantization level */ + *gainQ10 = WebRtcIsacfix_kGain2Lev[index]; + + return 0; +} + + + +/* quantize & code squared Gain */ +int WebRtcIsacfix_EncodeGain2(int32_t *gainQ10, Bitstr_enc *streamdata) +{ + int16_t index; + int status = 0; + + /* find quantization index */ + index = WebRtcIsacfix_kGainInitInd[0]; + if (*gainQ10 > WebRtcIsacfix_kGain2Bound[index]) + { + while (*gainQ10 > WebRtcIsacfix_kGain2Bound[index + 1]) + index++; + } + else + { + while (*gainQ10 < WebRtcIsacfix_kGain2Bound[--index]) ; + } + + /* dequantize */ + *gainQ10 = WebRtcIsacfix_kGain2Lev[index]; + + /* entropy coding of quantization index */ + status = WebRtcIsacfix_EncHistMulti(streamdata, &index, WebRtcIsacfix_kGainPtr, 1); + + /* If error in WebRtcIsacfix_EncHistMulti(), status will be negative, otherwise 0 */ + return status; +} + + +/* code and decode Pitch Gains and Lags functions */ + +/* decode & dequantize Pitch Gains */ +int WebRtcIsacfix_DecodePitchGain(Bitstr_dec *streamdata, int16_t *PitchGains_Q12) +{ + int err; + int16_t index_comb; + const uint16_t *pitch_gain_cdf_ptr[1]; + + /* entropy decoding of quantization indices */ + *pitch_gain_cdf_ptr = WebRtcIsacfix_kPitchGainCdf; + err = WebRtcIsacfix_DecHistBisectMulti(&index_comb, streamdata, pitch_gain_cdf_ptr, WebRtcIsacfix_kCdfTableSizeGain, 1); + /* error check, Q_mean_Gain.. tables are of size 144 */ + if ((err < 0) || (index_comb < 0) || (index_comb >= 144)) + return -ISAC_RANGE_ERROR_DECODE_PITCH_GAIN; + + /* unquantize back to pitch gains by table look-up */ + PitchGains_Q12[0] = WebRtcIsacfix_kPitchGain1[index_comb]; + PitchGains_Q12[1] = WebRtcIsacfix_kPitchGain2[index_comb]; + PitchGains_Q12[2] = WebRtcIsacfix_kPitchGain3[index_comb]; + PitchGains_Q12[3] = WebRtcIsacfix_kPitchGain4[index_comb]; + + return 0; +} + + +/* quantize & code Pitch Gains */ +int WebRtcIsacfix_EncodePitchGain(int16_t* PitchGains_Q12, + Bitstr_enc* streamdata, + IsacSaveEncoderData* encData) { + int k,j; + int16_t SQ15[PITCH_SUBFRAMES]; + int16_t index[3]; + int16_t index_comb; + const uint16_t *pitch_gain_cdf_ptr[1]; + int32_t CQ17; + int status = 0; + + + /* get the approximate arcsine (almost linear)*/ + for (k=0; k> 2); // Q15 + + + /* find quantization index; only for the first three transform coefficients */ + for (k=0; k<3; k++) + { + /* transform */ + CQ17=0; + for (j=0; j> 10; // Q17 + } + + index[k] = (int16_t)((CQ17 + 8192)>>14); // Rounding and scaling with stepsize (=1/0.125=8) + + /* check that the index is not outside the boundaries of the table */ + if (index[k] < WebRtcIsacfix_kLowerlimiGain[k]) index[k] = WebRtcIsacfix_kLowerlimiGain[k]; + else if (index[k] > WebRtcIsacfix_kUpperlimitGain[k]) index[k] = WebRtcIsacfix_kUpperlimitGain[k]; + index[k] -= WebRtcIsacfix_kLowerlimiGain[k]; + } + + /* calculate unique overall index */ + index_comb = (int16_t)(WEBRTC_SPL_MUL(WebRtcIsacfix_kMultsGain[0], index[0]) + + WEBRTC_SPL_MUL(WebRtcIsacfix_kMultsGain[1], index[1]) + index[2]); + + /* unquantize back to pitch gains by table look-up */ + // (Y) + PitchGains_Q12[0] = WebRtcIsacfix_kPitchGain1[index_comb]; + PitchGains_Q12[1] = WebRtcIsacfix_kPitchGain2[index_comb]; + PitchGains_Q12[2] = WebRtcIsacfix_kPitchGain3[index_comb]; + PitchGains_Q12[3] = WebRtcIsacfix_kPitchGain4[index_comb]; + + + /* entropy coding of quantization pitch gains */ + *pitch_gain_cdf_ptr = WebRtcIsacfix_kPitchGainCdf; + status = WebRtcIsacfix_EncHistMulti(streamdata, &index_comb, pitch_gain_cdf_ptr, 1); + if (status < 0) { + return status; + } + + /* Save data for creation of multiple bitstreams */ + if (encData != NULL) { + encData->pitchGain_index[encData->startIdx] = index_comb; + } + + return 0; +} + + + +/* Pitch LAG */ + + +/* decode & dequantize Pitch Lags */ +int WebRtcIsacfix_DecodePitchLag(Bitstr_dec *streamdata, + int16_t *PitchGain_Q12, + int16_t *PitchLags_Q7) +{ + int k, err; + int16_t index[PITCH_SUBFRAMES]; + const int16_t *mean_val2Q10, *mean_val4Q10; + + const int16_t *lower_limit; + const uint16_t *init_index; + const uint16_t *cdf_size; + const uint16_t **cdf; + + int32_t meangainQ12; + int32_t CQ11, CQ10,tmp32a,tmp32b; + int16_t shft; + + meangainQ12=0; + for (k = 0; k < 4; k++) + meangainQ12 += PitchGain_Q12[k]; + + meangainQ12 >>= 2; // Get average. + + /* voicing classificiation */ + if (meangainQ12 <= 819) { // mean_gain < 0.2 + shft = -1; // StepSize=2.0; + cdf = WebRtcIsacfix_kPitchLagPtrLo; + cdf_size = WebRtcIsacfix_kPitchLagSizeLo; + mean_val2Q10 = WebRtcIsacfix_kMeanLag2Lo; + mean_val4Q10 = WebRtcIsacfix_kMeanLag4Lo; + lower_limit = WebRtcIsacfix_kLowerLimitLo; + init_index = WebRtcIsacfix_kInitIndLo; + } else if (meangainQ12 <= 1638) { // mean_gain < 0.4 + shft = 0; // StepSize=1.0; + cdf = WebRtcIsacfix_kPitchLagPtrMid; + cdf_size = WebRtcIsacfix_kPitchLagSizeMid; + mean_val2Q10 = WebRtcIsacfix_kMeanLag2Mid; + mean_val4Q10 = WebRtcIsacfix_kMeanLag4Mid; + lower_limit = WebRtcIsacfix_kLowerLimitMid; + init_index = WebRtcIsacfix_kInitIndMid; + } else { + shft = 1; // StepSize=0.5; + cdf = WebRtcIsacfix_kPitchLagPtrHi; + cdf_size = WebRtcIsacfix_kPitchLagSizeHi; + mean_val2Q10 = WebRtcIsacfix_kMeanLag2Hi; + mean_val4Q10 = WebRtcIsacfix_kMeanLag4Hi; + lower_limit = WebRtcIsacfix_kLowerLimitHi; + init_index = WebRtcIsacfix_kInitIndHi; + } + + /* entropy decoding of quantization indices */ + err = WebRtcIsacfix_DecHistBisectMulti(index, streamdata, cdf, cdf_size, 1); + if ((err<0) || (index[0]<0)) // error check + return -ISAC_RANGE_ERROR_DECODE_PITCH_LAG; + + err = WebRtcIsacfix_DecHistOneStepMulti(index+1, streamdata, cdf+1, init_index, 3); + if (err<0) // error check + return -ISAC_RANGE_ERROR_DECODE_PITCH_LAG; + + + /* unquantize back to transform coefficients and do the inverse transform: S = T'*C */ + CQ11 = ((int32_t)index[0] + lower_limit[0]); // Q0 + CQ11 = WEBRTC_SPL_SHIFT_W32(CQ11,11-shft); // Scale with StepSize, Q11 + for (k=0; k> 5); + } + + CQ10 = mean_val2Q10[index[1]]; + for (k=0; k> 10; + PitchLags_Q7[k] += (int16_t)(tmp32b >> 5); + } + + CQ10 = mean_val4Q10[index[3]]; + for (k=0; k> 10; + PitchLags_Q7[k] += (int16_t)(tmp32b >> 5); + } + + return 0; +} + + + +/* quantize & code Pitch Lags */ +int WebRtcIsacfix_EncodePitchLag(int16_t* PitchLagsQ7, + int16_t* PitchGain_Q12, + Bitstr_enc* streamdata, + IsacSaveEncoderData* encData) { + int k, j; + int16_t index[PITCH_SUBFRAMES]; + int32_t meangainQ12, CQ17; + int32_t CQ11, CQ10,tmp32a; + + const int16_t *mean_val2Q10,*mean_val4Q10; + const int16_t *lower_limit, *upper_limit; + const uint16_t **cdf; + int16_t shft, tmp16b; + int32_t tmp32b; + int status = 0; + + /* compute mean pitch gain */ + meangainQ12=0; + for (k = 0; k < 4; k++) + meangainQ12 += PitchGain_Q12[k]; + + meangainQ12 >>= 2; + + /* Save data for creation of multiple bitstreams */ + if (encData != NULL) { + encData->meanGain[encData->startIdx] = meangainQ12; + } + + /* voicing classificiation */ + if (meangainQ12 <= 819) { // mean_gain < 0.2 + shft = -1; // StepSize=2.0; + cdf = WebRtcIsacfix_kPitchLagPtrLo; + mean_val2Q10 = WebRtcIsacfix_kMeanLag2Lo; + mean_val4Q10 = WebRtcIsacfix_kMeanLag4Lo; + lower_limit = WebRtcIsacfix_kLowerLimitLo; + upper_limit = WebRtcIsacfix_kUpperLimitLo; + } else if (meangainQ12 <= 1638) { // mean_gain < 0.4 + shft = 0; // StepSize=1.0; + cdf = WebRtcIsacfix_kPitchLagPtrMid; + mean_val2Q10 = WebRtcIsacfix_kMeanLag2Mid; + mean_val4Q10 = WebRtcIsacfix_kMeanLag4Mid; + lower_limit = WebRtcIsacfix_kLowerLimitMid; + upper_limit = WebRtcIsacfix_kUpperLimitMid; + } else { + shft = 1; // StepSize=0.5; + cdf = WebRtcIsacfix_kPitchLagPtrHi; + mean_val2Q10 = WebRtcIsacfix_kMeanLag2Hi; + mean_val4Q10 = WebRtcIsacfix_kMeanLag4Hi; + lower_limit = WebRtcIsacfix_kLowerLimitHi; + upper_limit = WebRtcIsacfix_kUpperLimitHi; + } + + /* find quantization index */ + for (k=0; k<4; k++) + { + /* transform */ + CQ17=0; + for (j=0; j> 2; // Q17 + + CQ17 = WEBRTC_SPL_SHIFT_W32(CQ17,shft); // Scale with StepSize + + /* quantize */ + tmp16b = (int16_t)((CQ17 + 65536) >> 17); + index[k] = tmp16b; + + /* check that the index is not outside the boundaries of the table */ + if (index[k] < lower_limit[k]) index[k] = lower_limit[k]; + else if (index[k] > upper_limit[k]) index[k] = upper_limit[k]; + index[k] -= lower_limit[k]; + + /* Save data for creation of multiple bitstreams */ + if(encData != NULL) { + encData->pitchIndex[PITCH_SUBFRAMES*encData->startIdx + k] = index[k]; + } + } + + /* unquantize back to transform coefficients and do the inverse transform: S = T'*C */ + CQ11 = (index[0] + lower_limit[0]); // Q0 + CQ11 = WEBRTC_SPL_SHIFT_W32(CQ11,11-shft); // Scale with StepSize, Q11 + + for (k=0; k> 5); // Q7. + } + + CQ10 = mean_val2Q10[index[1]]; + for (k=0; k> 10; + PitchLagsQ7[k] += (int16_t)(tmp32b >> 5); // Q7. + } + + CQ10 = mean_val4Q10[index[3]]; + for (k=0; k> 10; + PitchLagsQ7[k] += (int16_t)(tmp32b >> 5); // Q7. + } + + /* entropy coding of quantization pitch lags */ + status = WebRtcIsacfix_EncHistMulti(streamdata, index, cdf, PITCH_SUBFRAMES); + + /* If error in WebRtcIsacfix_EncHistMulti(), status will be negative, otherwise 0 */ + return status; +} + + + +/* Routines for inband signaling of bandwitdh estimation */ +/* Histograms based on uniform distribution of indices */ +/* Move global variables later! */ + + +/* cdf array for frame length indicator */ +const uint16_t kFrameLenCdf[4] = { + 0, 21845, 43690, 65535}; + +/* pointer to cdf array for frame length indicator */ +const uint16_t * const kFrameLenCdfPtr[1] = {kFrameLenCdf}; + +/* initial cdf index for decoder of frame length indicator */ +const uint16_t kFrameLenInitIndex[1] = {1}; + + +int WebRtcIsacfix_DecodeFrameLen(Bitstr_dec *streamdata, + size_t *framesamples) +{ + + int err; + int16_t frame_mode; + + err = 0; + /* entropy decoding of frame length [1:30ms,2:60ms] */ + err = WebRtcIsacfix_DecHistOneStepMulti(&frame_mode, streamdata, kFrameLenCdfPtr, kFrameLenInitIndex, 1); + if (err<0) // error check + return -ISAC_RANGE_ERROR_DECODE_FRAME_LENGTH; + + switch(frame_mode) { + case 1: + *framesamples = 480; /* 30ms */ + break; + case 2: + *framesamples = 960; /* 60ms */ + break; + default: + err = -ISAC_DISALLOWED_FRAME_MODE_DECODER; + } + + return err; +} + + +int WebRtcIsacfix_EncodeFrameLen(int16_t framesamples, Bitstr_enc *streamdata) { + + int status; + int16_t frame_mode; + + status = 0; + frame_mode = 0; + /* entropy coding of frame length [1:480 samples,2:960 samples] */ + switch(framesamples) { + case 480: + frame_mode = 1; + break; + case 960: + frame_mode = 2; + break; + default: + status = - ISAC_DISALLOWED_FRAME_MODE_ENCODER; + } + + if (status < 0) + return status; + + status = WebRtcIsacfix_EncHistMulti(streamdata, &frame_mode, kFrameLenCdfPtr, 1); + + return status; +} + +/* cdf array for estimated bandwidth */ +const uint16_t kBwCdf[25] = { + 0, 2731, 5461, 8192, 10923, 13653, 16384, 19114, 21845, 24576, 27306, 30037, + 32768, 35498, 38229, 40959, 43690, 46421, 49151, 51882, 54613, 57343, 60074, + 62804, 65535}; + +/* pointer to cdf array for estimated bandwidth */ +const uint16_t * const kBwCdfPtr[1] = {kBwCdf}; + +/* initial cdf index for decoder of estimated bandwidth*/ +const uint16_t kBwInitIndex[1] = {7}; + + +int WebRtcIsacfix_DecodeSendBandwidth(Bitstr_dec *streamdata, int16_t *BWno) { + + int err; + int16_t BWno32; + + /* entropy decoding of sender's BW estimation [0..23] */ + err = WebRtcIsacfix_DecHistOneStepMulti(&BWno32, streamdata, kBwCdfPtr, kBwInitIndex, 1); + if (err<0) // error check + return -ISAC_RANGE_ERROR_DECODE_BANDWIDTH; + *BWno = (int16_t)BWno32; + return err; + +} + + +int WebRtcIsacfix_EncodeReceiveBandwidth(int16_t *BWno, Bitstr_enc *streamdata) +{ + int status = 0; + /* entropy encoding of receiver's BW estimation [0..23] */ + status = WebRtcIsacfix_EncHistMulti(streamdata, BWno, kBwCdfPtr, 1); + + return status; +} + +/* estimate codel length of LPC Coef */ +void WebRtcIsacfix_TranscodeLpcCoef(int32_t *gain_lo_hiQ17, + int16_t *index_gQQ) { + int j, k; + int16_t posQQ, pos2QQ; + int16_t posg, offsg, gainpos; + int32_t tmpcoeffs_gQ6[KLT_ORDER_GAIN]; + int32_t tmpcoeffs_gQ17[KLT_ORDER_GAIN]; + int32_t tmpcoeffs2_gQ21[KLT_ORDER_GAIN]; + int32_t sumQQ; + + + /* log gains, mean removal and scaling */ + posg = 0; gainpos=0; + + for (k=0; k WebRtcIsacfix_kMaxIndGain[k]) { + index_gQQ[k] = WebRtcIsacfix_kMaxIndGain[k]; + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.h new file mode 100644 index 0000000000..31f044de0b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * entropy_coding.h + * + * This header file contains all of the functions used to arithmetically + * encode the iSAC bistream + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ENTROPY_CODING_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ENTROPY_CODING_H_ + +#include "structs.h" + +/* decode complex spectrum (return number of bytes in stream) */ +int WebRtcIsacfix_DecodeSpec(Bitstr_dec *streamdata, + int16_t *frQ7, + int16_t *fiQ7, + int16_t AvgPitchGain_Q12); + +/* encode complex spectrum */ +int WebRtcIsacfix_EncodeSpec(const int16_t *fr, + const int16_t *fi, + Bitstr_enc *streamdata, + int16_t AvgPitchGain_Q12); + + +/* decode & dequantize LPC Coef */ +int WebRtcIsacfix_DecodeLpcCoef(Bitstr_dec *streamdata, + int32_t *LPCCoefQ17, + int32_t *gain_lo_hiQ17, + int16_t *outmodel); + +int WebRtcIsacfix_DecodeLpc(int32_t *gain_lo_hiQ17, + int16_t *LPCCoef_loQ15, + int16_t *LPCCoef_hiQ15, + Bitstr_dec *streamdata, + int16_t *outmodel); + +/* quantize & code LPC Coef */ +int WebRtcIsacfix_EncodeLpc(int32_t *gain_lo_hiQ17, + int16_t *LPCCoef_loQ15, + int16_t *LPCCoef_hiQ15, + int16_t *model, + int32_t *sizeQ11, + Bitstr_enc *streamdata, + IsacSaveEncoderData* encData, + transcode_obj *transcodeParam); + +int WebRtcIsacfix_EstCodeLpcGain(int32_t *gain_lo_hiQ17, + Bitstr_enc *streamdata, + IsacSaveEncoderData* encData); +/* decode & dequantize RC */ +int WebRtcIsacfix_DecodeRcCoef(Bitstr_dec *streamdata, + int16_t *RCQ15); + +/* quantize & code RC */ +int WebRtcIsacfix_EncodeRcCoef(int16_t *RCQ15, + Bitstr_enc *streamdata); + +/* decode & dequantize squared Gain */ +int WebRtcIsacfix_DecodeGain2(Bitstr_dec *streamdata, + int32_t *Gain2); + +/* quantize & code squared Gain (input is squared gain) */ +int WebRtcIsacfix_EncodeGain2(int32_t *gain2, + Bitstr_enc *streamdata); + +int WebRtcIsacfix_EncodePitchGain(int16_t *PitchGains_Q12, + Bitstr_enc *streamdata, + IsacSaveEncoderData* encData); + +int WebRtcIsacfix_EncodePitchLag(int16_t *PitchLagQ7, + int16_t *PitchGain_Q12, + Bitstr_enc *streamdata, + IsacSaveEncoderData* encData); + +int WebRtcIsacfix_DecodePitchGain(Bitstr_dec *streamdata, + int16_t *PitchGain_Q12); + +int WebRtcIsacfix_DecodePitchLag(Bitstr_dec *streamdata, + int16_t *PitchGain_Q12, + int16_t *PitchLagQ7); + +int WebRtcIsacfix_DecodeFrameLen(Bitstr_dec *streamdata, + size_t *framelength); + + +int WebRtcIsacfix_EncodeFrameLen(int16_t framelength, + Bitstr_enc *streamdata); + +int WebRtcIsacfix_DecodeSendBandwidth(Bitstr_dec *streamdata, + int16_t *BWno); + + +int WebRtcIsacfix_EncodeReceiveBandwidth(int16_t *BWno, + Bitstr_enc *streamdata); + +void WebRtcIsacfix_TranscodeLpcCoef(int32_t *tmpcoeffs_gQ6, + int16_t *index_gQQ); + +// Pointer functions for LPC transforms. + +typedef void (*MatrixProduct1)(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix1_index_factor1, + const int matrix0_index_factor1, + const int matrix1_index_init_case, + const int matrix1_index_step, + const int matrix0_index_step, + const int inner_loop_count, + const int mid_loop_count, + const int shift); +typedef void (*MatrixProduct2)(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix0_index_factor, + const int matrix0_index_step); + +extern MatrixProduct1 WebRtcIsacfix_MatrixProduct1; +extern MatrixProduct2 WebRtcIsacfix_MatrixProduct2; + +void WebRtcIsacfix_MatrixProduct1C(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix1_index_factor1, + const int matrix0_index_factor1, + const int matrix1_index_init_case, + const int matrix1_index_step, + const int matrix0_index_step, + const int inner_loop_count, + const int mid_loop_count, + const int shift); +void WebRtcIsacfix_MatrixProduct2C(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix0_index_factor, + const int matrix0_index_step); + +#if defined(WEBRTC_HAS_NEON) +void WebRtcIsacfix_MatrixProduct1Neon(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix1_index_factor1, + const int matrix0_index_factor1, + const int matrix1_index_init_case, + const int matrix1_index_step, + const int matrix0_index_step, + const int inner_loop_count, + const int mid_loop_count, + const int shift); +void WebRtcIsacfix_MatrixProduct2Neon(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix0_index_factor, + const int matrix0_index_step); +#endif + +#if defined(MIPS32_LE) +void WebRtcIsacfix_MatrixProduct1MIPS(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix1_index_factor1, + const int matrix0_index_factor1, + const int matrix1_index_init_case, + const int matrix1_index_step, + const int matrix0_index_step, + const int inner_loop_count, + const int mid_loop_count, + const int shift); + +void WebRtcIsacfix_MatrixProduct2MIPS(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix0_index_factor, + const int matrix0_index_step); +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ENTROPY_CODING_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_mips.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_mips.c new file mode 100644 index 0000000000..a66a43ef99 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_mips.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/entropy_coding.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" + +// MIPS optimization of the function WebRtcIsacfix_MatrixProduct1. +// Bit-exact with the function WebRtcIsacfix_MatrixProduct1C from +// entropy_coding.c file. +void WebRtcIsacfix_MatrixProduct1MIPS(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix1_index_factor1, + const int matrix0_index_factor1, + const int matrix1_index_init_case, + const int matrix1_index_step, + const int matrix0_index_step, + const int inner_loop_count, + const int mid_loop_count, + const int shift) { + if (matrix1_index_init_case != 0) { + int j = SUBFRAMES, k = 0, n = 0; + int32_t r0, r1, r2, sum32; + int32_t* product_start = matrix_product; + int32_t* product_ptr; + const uint32_t product_step = 4 * mid_loop_count; + const uint32_t matrix0_step = 2 * matrix0_index_step; + const uint32_t matrix1_step = 4 * matrix1_index_step; + const uint32_t matrix0_step2 = 2 * matrix0_index_factor1; + const uint32_t matrix1_step2 = 4 * matrix1_index_factor1; + const int16_t* matrix0_start = matrix0; + const int32_t* matrix1_start = matrix1; + int16_t* matrix0_ptr; + int32_t* matrix1_ptr; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "addu %[product_ptr], %[product_start], $0 \n\t" + "addu %[k], %[product_step], $0 \n\t" + "addiu %[j], %[j], -1 \n\t" + "addu %[matrix1_start], %[matrix1], $0 \n\t" + "2: \n\t" + "addu %[matrix1_ptr], %[matrix1_start], $0 \n\t" + "addu %[matrix0_ptr], %[matrix0_start], $0 \n\t" + "addu %[n], %[inner_loop_count], $0 \n\t" + "mul %[sum32], $0, $0 \n\t" + "3: \n\t" + "lw %[r0], 0(%[matrix1_ptr]) \n\t" + "lh %[r1], 0(%[matrix0_ptr]) \n\t" + "addu %[matrix1_ptr], %[matrix1_ptr], %[matrix1_step] \n\t" + "sllv %[r0], %[r0], %[shift] \n\t" + "andi %[r2], %[r0], 0xffff \n\t" + "sra %[r2], %[r2], 1 \n\t" + "mul %[r2], %[r2], %[r1] \n\t" + "sra %[r0], %[r0], 16 \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "addu %[matrix0_ptr], %[matrix0_ptr], %[matrix0_step] \n\t" + "addiu %[n], %[n], -1 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r2], %[r2], 15 \n\t" +#else + "addiu %[r2], %[r2], 0x4000 \n\t" + "sra %[r2], %[r2], 15 \n\t" +#endif + "addu %[sum32], %[sum32], %[r2] \n\t" + "bgtz %[n], 3b \n\t" + " addu %[sum32], %[sum32], %[r0] \n\t" + "addiu %[k], %[k], -4 \n\t" + "addu %[matrix1_start], %[matrix1_start], %[matrix1_step2] \n\t" + "sw %[sum32], 0(%[product_ptr]) \n\t" + "bgtz %[k], 2b \n\t" + " addiu %[product_ptr], %[product_ptr], 4 \n\t" + "addu %[matrix0_start], %[matrix0_start], %[matrix0_step2] \n\t" + "bgtz %[j], 1b \n\t" + " addu %[product_start], %[product_start], %[product_step] \n\t" + ".set pop \n\t" + : [product_ptr] "=&r" (product_ptr), [product_start] "+r" (product_start), + [k] "=&r" (k), [j] "+r" (j), [matrix1_start] "=&r"(matrix1_start), + [matrix1_ptr] "=&r" (matrix1_ptr), [matrix0_ptr] "=&r" (matrix0_ptr), + [matrix0_start] "+r" (matrix0_start), [n] "=&r" (n), [r0] "=&r" (r0), + [sum32] "=&r" (sum32), [r1] "=&r" (r1),[r2] "=&r" (r2) + : [product_step] "r" (product_step), [matrix1] "r" (matrix1), + [inner_loop_count] "r" (inner_loop_count), + [matrix1_step] "r" (matrix1_step), [shift] "r" (shift), + [matrix0_step] "r" (matrix0_step), [matrix1_step2] "r" (matrix1_step2), + [matrix0_step2] "r" (matrix0_step2) + : "hi", "lo", "memory" + ); + } else { + int j = SUBFRAMES, k = 0, n = 0; + int32_t r0, r1, r2, sum32; + int32_t* product_start = matrix_product; + int32_t* product_ptr; + const uint32_t product_step = 4 * mid_loop_count; + const uint32_t matrix0_step = 2 * matrix0_index_step; + const uint32_t matrix1_step = 4 * matrix1_index_step; + const uint32_t matrix0_step2 = 2 * matrix0_index_factor1; + const uint32_t matrix1_step2 = 4 * matrix1_index_factor1; + const int16_t* matrix0_start = matrix0; + const int32_t* matrix1_start = matrix1; + int16_t* matrix0_ptr; + int32_t* matrix1_ptr; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "addu %[product_ptr], %[product_start], $0 \n\t" + "addu %[k], %[product_step], $0 \n\t" + "addiu %[j], %[j], -1 \n\t" + "addu %[matrix0_start], %[matrix0], $0 \n\t" + "2: \n\t" + "addu %[matrix1_ptr], %[matrix1_start], $0 \n\t" + "addu %[matrix0_ptr], %[matrix0_start], $0 \n\t" + "addu %[n], %[inner_loop_count], $0 \n\t" + "mul %[sum32], $0, $0 \n\t" + "3: \n\t" + "lw %[r0], 0(%[matrix1_ptr]) \n\t" + "lh %[r1], 0(%[matrix0_ptr]) \n\t" + "addu %[matrix1_ptr], %[matrix1_ptr], %[matrix1_step] \n\t" + "sllv %[r0], %[r0], %[shift] \n\t" + "andi %[r2], %[r0], 0xffff \n\t" + "sra %[r2], %[r2], 1 \n\t" + "mul %[r2], %[r2], %[r1] \n\t" + "sra %[r0], %[r0], 16 \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "addu %[matrix0_ptr], %[matrix0_ptr], %[matrix0_step] \n\t" + "addiu %[n], %[n], -1 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r2], %[r2], 15 \n\t" +#else + "addiu %[r2], %[r2], 0x4000 \n\t" + "sra %[r2], %[r2], 15 \n\t" +#endif + "addu %[sum32], %[sum32], %[r2] \n\t" + "bgtz %[n], 3b \n\t" + " addu %[sum32], %[sum32], %[r0] \n\t" + "addiu %[k], %[k], -4 \n\t" + "addu %[matrix0_start], %[matrix0_start], %[matrix0_step2] \n\t" + "sw %[sum32], 0(%[product_ptr]) \n\t" + "bgtz %[k], 2b \n\t" + " addiu %[product_ptr], %[product_ptr], 4 \n\t" + "addu %[matrix1_start], %[matrix1_start], %[matrix1_step2] \n\t" + "bgtz %[j], 1b \n\t" + " addu %[product_start], %[product_start], %[product_step] \n\t" + ".set pop \n\t" + : [product_ptr] "=&r" (product_ptr), [product_start] "+r" (product_start), + [k] "=&r" (k), [j] "+r" (j), [matrix1_start] "+r"(matrix1_start), + [matrix1_ptr] "=&r" (matrix1_ptr), [matrix0_ptr] "=&r" (matrix0_ptr), + [matrix0_start] "=&r" (matrix0_start), [n] "=&r" (n), [r0] "=&r" (r0), + [sum32] "=&r" (sum32), [r1] "=&r" (r1),[r2] "=&r" (r2) + : [product_step] "r" (product_step), [matrix0] "r" (matrix0), + [inner_loop_count] "r" (inner_loop_count), + [matrix1_step] "r" (matrix1_step), [shift] "r" (shift), + [matrix0_step] "r" (matrix0_step), [matrix1_step2] "r" (matrix1_step2), + [matrix0_step2] "r" (matrix0_step2) + : "hi", "lo", "memory" + ); + } +} + +// MIPS optimization of the function WebRtcIsacfix_MatrixProduct2. +// Bit-exact with the function WebRtcIsacfix_MatrixProduct2C from +// entropy_coding.c file. +void WebRtcIsacfix_MatrixProduct2MIPS(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix0_index_factor, + const int matrix0_index_step) { + int j = 0, n = 0; + int loop_count = SUBFRAMES; + const int16_t* matrix0_ptr; + const int32_t* matrix1_ptr; + const int16_t* matrix0_start = matrix0; + const int matrix0_step = 2 * matrix0_index_step; + const int matrix0_step2 = 2 * matrix0_index_factor; + int32_t r0, r1, r2, r3, r4, sum32, sum32_2; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addu %[j], %[loop_count], $0 \n\t" + "addu %[matrix0_start], %[matrix0], $0 \n\t" + "1: \n\t" + "addu %[matrix1_ptr], %[matrix1], $0 \n\t" + "addu %[matrix0_ptr], %[matrix0_start], $0 \n\t" + "addu %[n], %[loop_count], $0 \n\t" + "mul %[sum32], $0, $0 \n\t" + "mul %[sum32_2], $0, $0 \n\t" + "2: \n\t" + "lw %[r0], 0(%[matrix1_ptr]) \n\t" + "lw %[r1], 4(%[matrix1_ptr]) \n\t" + "lh %[r2], 0(%[matrix0_ptr]) \n\t" + "andi %[r3], %[r0], 0xffff \n\t" + "sra %[r3], %[r3], 1 \n\t" + "mul %[r3], %[r3], %[r2] \n\t" + "andi %[r4], %[r1], 0xffff \n\t" + "sra %[r4], %[r4], 1 \n\t" + "mul %[r4], %[r4], %[r2] \n\t" + "sra %[r0], %[r0], 16 \n\t" + "mul %[r0], %[r0], %[r2] \n\t" + "sra %[r1], %[r1], 16 \n\t" + "mul %[r1], %[r1], %[r2] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r3], %[r3], 15 \n\t" + "shra_r.w %[r4], %[r4], 15 \n\t" +#else + "addiu %[r3], %[r3], 0x4000 \n\t" + "sra %[r3], %[r3], 15 \n\t" + "addiu %[r4], %[r4], 0x4000 \n\t" + "sra %[r4], %[r4], 15 \n\t" +#endif + "addiu %[matrix1_ptr], %[matrix1_ptr], 8 \n\t" + "addu %[matrix0_ptr], %[matrix0_ptr], %[matrix0_step] \n\t" + "addiu %[n], %[n], -1 \n\t" + "addu %[sum32], %[sum32], %[r3] \n\t" + "addu %[sum32_2], %[sum32_2], %[r4] \n\t" + "addu %[sum32], %[sum32], %[r0] \n\t" + "bgtz %[n], 2b \n\t" + " addu %[sum32_2], %[sum32_2], %[r1] \n\t" + "sra %[sum32], %[sum32], 3 \n\t" + "sra %[sum32_2], %[sum32_2], 3 \n\t" + "addiu %[j], %[j], -1 \n\t" + "addu %[matrix0_start], %[matrix0_start], %[matrix0_step2] \n\t" + "sw %[sum32], 0(%[matrix_product]) \n\t" + "sw %[sum32_2], 4(%[matrix_product]) \n\t" + "bgtz %[j], 1b \n\t" + " addiu %[matrix_product], %[matrix_product], 8 \n\t" + ".set pop \n\t" + : [j] "=&r" (j), [matrix0_start] "=&r" (matrix0_start), + [matrix1_ptr] "=&r" (matrix1_ptr), [matrix0_ptr] "=&r" (matrix0_ptr), + [n] "=&r" (n), [sum32] "=&r" (sum32), [sum32_2] "=&r" (sum32_2), + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), + [r4] "=&r" (r4), [matrix_product] "+r" (matrix_product) + : [loop_count] "r" (loop_count), [matrix0] "r" (matrix0), + [matrix1] "r" (matrix1), [matrix0_step] "r" (matrix0_step), + [matrix0_step2] "r" (matrix0_step2) + : "hi", "lo", "memory" + ); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_neon.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_neon.c new file mode 100644 index 0000000000..2eccdfaa61 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_neon.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* This file contains WebRtcIsacfix_MatrixProduct1Neon() and + * WebRtcIsacfix_MatrixProduct2Neon() for ARM Neon platform. API's are in + * entropy_coding.c. Results are bit exact with the c code for + * generic platforms. + */ + +#include "entropy_coding.h" + +#include +#include + +#include "signal_processing_library.h" +#include "rtc_base/checks.h" + +void WebRtcIsacfix_MatrixProduct1Neon(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix1_index_factor1, + const int matrix0_index_factor1, + const int matrix1_index_init_case, + const int matrix1_index_step, + const int matrix0_index_step, + const int inner_loop_count, + const int mid_loop_count, + const int shift) { + int j = 0, k = 0, n = 0; + int matrix1_index = 0, matrix0_index = 0, matrix_prod_index = 0; + int* matrix1_index_factor2 = &j; + int* matrix0_index_factor2 = &k; + if (matrix1_index_init_case != 0) { + matrix1_index_factor2 = &k; + matrix0_index_factor2 = &j; + } + int32x4_t shift32x4 = vdupq_n_s32(shift); + int32x2_t shift32x2 = vdup_n_s32(shift); + int32x4_t sum_32x4 = vdupq_n_s32(0); + int32x2_t sum_32x2 = vdup_n_s32(0); + + RTC_DCHECK_EQ(0, inner_loop_count % 2); + RTC_DCHECK_EQ(0, mid_loop_count % 2); + + if (matrix1_index_init_case != 0 && matrix1_index_factor1 == 1) { + for (j = 0; j < SUBFRAMES; j++) { + matrix_prod_index = mid_loop_count * j; + for (k = 0; k < (mid_loop_count >> 2) << 2; k += 4) { + sum_32x4 = veorq_s32(sum_32x4, sum_32x4); // Initialize to zeros. + matrix1_index = k; + matrix0_index = matrix0_index_factor1 * j; + for (n = 0; n < inner_loop_count; n++) { + int32x4_t matrix0_32x4 = + vdupq_n_s32((int32_t)(matrix0[matrix0_index]) << 15); + int32x4_t matrix1_32x4 = + vshlq_s32(vld1q_s32(&matrix1[matrix1_index]), shift32x4); + int32x4_t multi_32x4 = vqdmulhq_s32(matrix0_32x4, matrix1_32x4); + sum_32x4 = vqaddq_s32(sum_32x4, multi_32x4); + matrix1_index += matrix1_index_step; + matrix0_index += matrix0_index_step; + } + vst1q_s32(&matrix_product[matrix_prod_index], sum_32x4); + matrix_prod_index += 4; + } + if (mid_loop_count % 4 > 1) { + sum_32x2 = veor_s32(sum_32x2, sum_32x2); // Initialize to zeros. + matrix1_index = k; + k += 2; + matrix0_index = matrix0_index_factor1 * j; + for (n = 0; n < inner_loop_count; n++) { + int32x2_t matrix0_32x2 = + vdup_n_s32((int32_t)(matrix0[matrix0_index]) << 15); + int32x2_t matrix1_32x2 = + vshl_s32(vld1_s32(&matrix1[matrix1_index]), shift32x2); + int32x2_t multi_32x2 = vqdmulh_s32(matrix0_32x2, matrix1_32x2); + sum_32x2 = vqadd_s32(sum_32x2, multi_32x2); + matrix1_index += matrix1_index_step; + matrix0_index += matrix0_index_step; + } + vst1_s32(&matrix_product[matrix_prod_index], sum_32x2); + matrix_prod_index += 2; + } + } + } + else if (matrix1_index_init_case == 0 && matrix0_index_factor1 == 1) { + int32x2_t multi_32x2 = vdup_n_s32(0); + int32x2_t matrix0_32x2 = vdup_n_s32(0); + for (j = 0; j < SUBFRAMES; j++) { + matrix_prod_index = mid_loop_count * j; + for (k = 0; k < (mid_loop_count >> 2) << 2; k += 4) { + sum_32x4 = veorq_s32(sum_32x4, sum_32x4); // Initialize to zeros. + matrix1_index = matrix1_index_factor1 * j; + matrix0_index = k; + for (n = 0; n < inner_loop_count; n++) { + int32x4_t matrix1_32x4 = vdupq_n_s32(matrix1[matrix1_index] << shift); + int32x4_t matrix0_32x4 = + vshll_n_s16(vld1_s16(&matrix0[matrix0_index]), 15); + int32x4_t multi_32x4 = vqdmulhq_s32(matrix0_32x4, matrix1_32x4); + sum_32x4 = vqaddq_s32(sum_32x4, multi_32x4); + matrix1_index += matrix1_index_step; + matrix0_index += matrix0_index_step; + } + vst1q_s32(&matrix_product[matrix_prod_index], sum_32x4); + matrix_prod_index += 4; + } + if (mid_loop_count % 4 > 1) { + sum_32x2 = veor_s32(sum_32x2, sum_32x2); // Initialize to zeros. + matrix1_index = matrix1_index_factor1 * j; + matrix0_index = k; + for (n = 0; n < inner_loop_count; n++) { + int32x2_t matrix1_32x2 = vdup_n_s32(matrix1[matrix1_index] << shift); + matrix0_32x2 = + vset_lane_s32((int32_t)matrix0[matrix0_index], matrix0_32x2, 0); + matrix0_32x2 = vset_lane_s32((int32_t)matrix0[matrix0_index + 1], + matrix0_32x2, 1); + matrix0_32x2 = vshl_n_s32(matrix0_32x2, 15); + multi_32x2 = vqdmulh_s32(matrix1_32x2, matrix0_32x2); + sum_32x2 = vqadd_s32(sum_32x2, multi_32x2); + matrix1_index += matrix1_index_step; + matrix0_index += matrix0_index_step; + } + vst1_s32(&matrix_product[matrix_prod_index], sum_32x2); + matrix_prod_index += 2; + } + } + } + else if (matrix1_index_init_case == 0 && + matrix1_index_step == 1 && + matrix0_index_step == 1) { + int32x2_t multi_32x2 = vdup_n_s32(0); + int32x2_t matrix0_32x2 = vdup_n_s32(0); + for (j = 0; j < SUBFRAMES; j++) { + matrix_prod_index = mid_loop_count * j; + for (k = 0; k < mid_loop_count; k++) { + sum_32x4 = veorq_s32(sum_32x4, sum_32x4); // Initialize to zeros. + matrix1_index = matrix1_index_factor1 * j; + matrix0_index = matrix0_index_factor1 * k; + for (n = 0; n < (inner_loop_count >> 2) << 2; n += 4) { + int32x4_t matrix1_32x4 = + vshlq_s32(vld1q_s32(&matrix1[matrix1_index]), shift32x4); + int32x4_t matrix0_32x4 = + vshll_n_s16(vld1_s16(&matrix0[matrix0_index]), 15); + int32x4_t multi_32x4 = vqdmulhq_s32(matrix0_32x4, matrix1_32x4); + sum_32x4 = vqaddq_s32(sum_32x4, multi_32x4); + matrix1_index += 4; + matrix0_index += 4; + } + sum_32x2 = vqadd_s32(vget_low_s32(sum_32x4), vget_high_s32(sum_32x4)); + if (inner_loop_count % 4 > 1) { + int32x2_t matrix1_32x2 = + vshl_s32(vld1_s32(&matrix1[matrix1_index]), shift32x2); + matrix0_32x2 = + vset_lane_s32((int32_t)matrix0[matrix0_index], matrix0_32x2, 0); + matrix0_32x2 = vset_lane_s32((int32_t)matrix0[matrix0_index + 1], + matrix0_32x2, 1); + matrix0_32x2 = vshl_n_s32(matrix0_32x2, 15); + multi_32x2 = vqdmulh_s32(matrix1_32x2, matrix0_32x2); + sum_32x2 = vqadd_s32(sum_32x2, multi_32x2); + } + sum_32x2 = vpadd_s32(sum_32x2, sum_32x2); + vst1_lane_s32(&matrix_product[matrix_prod_index], sum_32x2, 0); + matrix_prod_index++; + } + } + } + else { + for (j = 0; j < SUBFRAMES; j++) { + matrix_prod_index = mid_loop_count * j; + for (k=0; k < mid_loop_count; k++) { + int32_t sum32 = 0; + matrix1_index = matrix1_index_factor1 * (*matrix1_index_factor2); + matrix0_index = matrix0_index_factor1 * (*matrix0_index_factor2); + for (n = 0; n < inner_loop_count; n++) { + sum32 += (WEBRTC_SPL_MUL_16_32_RSFT16(matrix0[matrix0_index], + matrix1[matrix1_index] << shift)); + matrix1_index += matrix1_index_step; + matrix0_index += matrix0_index_step; + } + matrix_product[matrix_prod_index] = sum32; + matrix_prod_index++; + } + } + } +} + +void WebRtcIsacfix_MatrixProduct2Neon(const int16_t matrix0[], + const int32_t matrix1[], + int32_t matrix_product[], + const int matrix0_index_factor, + const int matrix0_index_step) { + int j = 0, n = 0; + int matrix1_index = 0, matrix0_index = 0, matrix_prod_index = 0; + int32x2_t sum_32x2 = vdup_n_s32(0); + for (j = 0; j < SUBFRAMES; j++) { + sum_32x2 = veor_s32(sum_32x2, sum_32x2); // Initialize to zeros. + matrix1_index = 0; + matrix0_index = matrix0_index_factor * j; + for (n = SUBFRAMES; n > 0; n--) { + int32x2_t matrix0_32x2 = + vdup_n_s32((int32_t)(matrix0[matrix0_index]) << 15); + int32x2_t matrix1_32x2 = vld1_s32(&matrix1[matrix1_index]); + int32x2_t multi_32x2 = vqdmulh_s32(matrix0_32x2, matrix1_32x2); + sum_32x2 = vqadd_s32(sum_32x2, multi_32x2); + matrix1_index += 2; + matrix0_index += matrix0_index_step; + } + sum_32x2 = vshr_n_s32(sum_32x2, 3); + vst1_s32(&matrix_product[matrix_prod_index], sum_32x2); + matrix_prod_index += 2; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.c new file mode 100644 index 0000000000..21566f1718 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.c @@ -0,0 +1,415 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * fft.c + * + * Fast Fourier Transform + * + */ + + +#include "fft.h" + +static const int16_t kSortTabFft[240] = { + 0, 60, 120, 180, 20, 80, 140, 200, 40, 100, 160, 220, + 4, 64, 124, 184, 24, 84, 144, 204, 44, 104, 164, 224, + 8, 68, 128, 188, 28, 88, 148, 208, 48, 108, 168, 228, + 12, 72, 132, 192, 32, 92, 152, 212, 52, 112, 172, 232, + 16, 76, 136, 196, 36, 96, 156, 216, 56, 116, 176, 236, + 1, 61, 121, 181, 21, 81, 141, 201, 41, 101, 161, 221, + 5, 65, 125, 185, 25, 85, 145, 205, 45, 105, 165, 225, + 9, 69, 129, 189, 29, 89, 149, 209, 49, 109, 169, 229, + 13, 73, 133, 193, 33, 93, 153, 213, 53, 113, 173, 233, + 17, 77, 137, 197, 37, 97, 157, 217, 57, 117, 177, 237, + 2, 62, 122, 182, 22, 82, 142, 202, 42, 102, 162, 222, + 6, 66, 126, 186, 26, 86, 146, 206, 46, 106, 166, 226, + 10, 70, 130, 190, 30, 90, 150, 210, 50, 110, 170, 230, + 14, 74, 134, 194, 34, 94, 154, 214, 54, 114, 174, 234, + 18, 78, 138, 198, 38, 98, 158, 218, 58, 118, 178, 238, + 3, 63, 123, 183, 23, 83, 143, 203, 43, 103, 163, 223, + 7, 67, 127, 187, 27, 87, 147, 207, 47, 107, 167, 227, + 11, 71, 131, 191, 31, 91, 151, 211, 51, 111, 171, 231, + 15, 75, 135, 195, 35, 95, 155, 215, 55, 115, 175, 235, + 19, 79, 139, 199, 39, 99, 159, 219, 59, 119, 179, 239 +}; + +/* Cosine table in Q14 */ +static const int16_t kCosTabFfftQ14[240] = { + 16384, 16378, 16362, 16333, 16294, 16244, 16182, 16110, 16026, 15931, 15826, 15709, + 15582, 15444, 15296, 15137, 14968, 14788, 14598, 14399, 14189, 13970, 13741, 13502, + 13255, 12998, 12733, 12458, 12176, 11885, 11585, 11278, 10963, 10641, 10311, 9974, + 9630, 9280, 8923, 8561, 8192, 7818, 7438, 7053, 6664, 6270, 5872, 5469, + 5063, 4653, 4240, 3825, 3406, 2986, 2563, 2139, 1713, 1285, 857, 429, + 0, -429, -857, -1285, -1713, -2139, -2563, -2986, -3406, -3825, -4240, -4653, + -5063, -5469, -5872, -6270, -6664, -7053, -7438, -7818, -8192, -8561, -8923, -9280, + -9630, -9974, -10311, -10641, -10963, -11278, -11585, -11885, -12176, -12458, -12733, -12998, + -13255, -13502, -13741, -13970, -14189, -14399, -14598, -14788, -14968, -15137, -15296, -15444, + -15582, -15709, -15826, -15931, -16026, -16110, -16182, -16244, -16294, -16333, -16362, -16378, + -16384, -16378, -16362, -16333, -16294, -16244, -16182, -16110, -16026, -15931, -15826, -15709, + -15582, -15444, -15296, -15137, -14968, -14788, -14598, -14399, -14189, -13970, -13741, -13502, + -13255, -12998, -12733, -12458, -12176, -11885, -11585, -11278, -10963, -10641, -10311, -9974, + -9630, -9280, -8923, -8561, -8192, -7818, -7438, -7053, -6664, -6270, -5872, -5469, + -5063, -4653, -4240, -3825, -3406, -2986, -2563, -2139, -1713, -1285, -857, -429, + 0, 429, 857, 1285, 1713, 2139, 2563, 2986, 3406, 3825, 4240, 4653, + 5063, 5469, 5872, 6270, 6664, 7053, 7438, 7818, 8192, 8561, 8923, 9280, + 9630, 9974, 10311, 10641, 10963, 11278, 11585, 11885, 12176, 12458, 12733, 12998, + 13255, 13502, 13741, 13970, 14189, 14399, 14598, 14788, 14968, 15137, 15296, 15444, + 15582, 15709, 15826, 15931, 16026, 16110, 16182, 16244, 16294, 16333, 16362, 16378 +}; + + + +/* Uses 16x16 mul, without rounding, which is faster. Uses WEBRTC_SPL_MUL_16_16_RSFT */ +int16_t WebRtcIsacfix_FftRadix16Fastest(int16_t RexQx[], int16_t ImxQx[], int16_t iSign) { + + int16_t dd, ee, ff, gg, hh, ii; + int16_t k0, k1, k2, k3, k4, kk; + int16_t tmp116, tmp216; + + int16_t ccc1Q14, ccc2Q14, ccc3Q14, sss1Q14, sss2Q14, sss3Q14; + int16_t sss60Q14, ccc72Q14, sss72Q14; + int16_t aaQx, ajQx, akQx, ajmQx, ajpQx, akmQx, akpQx; + int16_t bbQx, bjQx, bkQx, bjmQx, bjpQx, bkmQx, bkpQx; + + int16_t ReDATAQx[240], ImDATAQx[240]; + + sss60Q14 = kCosTabFfftQ14[20]; + ccc72Q14 = kCosTabFfftQ14[48]; + sss72Q14 = kCosTabFfftQ14[12]; + + if (iSign < 0) { + sss72Q14 = -sss72Q14; + sss60Q14 = -sss60Q14; + } + /* Complexity is: 10 cycles */ + + /* compute fourier transform */ + + // transform for factor of 4 + for (kk=0; kk<60; kk++) { + k0 = kk; + k1 = k0 + 60; + k2 = k1 + 60; + k3 = k2 + 60; + + akpQx = RexQx[k0] + RexQx[k2]; + akmQx = RexQx[k0] - RexQx[k2]; + ajpQx = RexQx[k1] + RexQx[k3]; + ajmQx = RexQx[k1] - RexQx[k3]; + bkpQx = ImxQx[k0] + ImxQx[k2]; + bkmQx = ImxQx[k0] - ImxQx[k2]; + bjpQx = ImxQx[k1] + ImxQx[k3]; + bjmQx = ImxQx[k1] - ImxQx[k3]; + + RexQx[k0] = akpQx + ajpQx; + ImxQx[k0] = bkpQx + bjpQx; + ajpQx = akpQx - ajpQx; + bjpQx = bkpQx - bjpQx; + if (iSign < 0) { + akpQx = akmQx + bjmQx; + bkpQx = bkmQx - ajmQx; + akmQx -= bjmQx; + bkmQx += ajmQx; + } else { + akpQx = akmQx - bjmQx; + bkpQx = bkmQx + ajmQx; + akmQx += bjmQx; + bkmQx -= ajmQx; + } + + ccc1Q14 = kCosTabFfftQ14[kk]; + ccc2Q14 = kCosTabFfftQ14[2 * kk]; + ccc3Q14 = kCosTabFfftQ14[3 * kk]; + sss1Q14 = kCosTabFfftQ14[kk + 60]; + sss2Q14 = kCosTabFfftQ14[2 * kk + 60]; + sss3Q14 = kCosTabFfftQ14[3 * kk + 60]; + if (iSign==1) { + sss1Q14 = -sss1Q14; + sss2Q14 = -sss2Q14; + sss3Q14 = -sss3Q14; + } + + //Do several multiplications like Q14*Q16>>14 = Q16 + // RexQ16[k1] = akpQ16 * ccc1Q14 - bkpQ16 * sss1Q14; + // RexQ16[k2] = ajpQ16 * ccc2Q14 - bjpQ16 * sss2Q14; + // RexQ16[k3] = akmQ16 * ccc3Q14 - bkmQ16 * sss3Q14; + // ImxQ16[k1] = akpQ16 * sss1Q14 + bkpQ16 * ccc1Q14; + // ImxQ16[k2] = ajpQ16 * sss2Q14 + bjpQ16 * ccc2Q14; + // ImxQ16[k3] = akmQ16 * sss3Q14 + bkmQ16 * ccc3Q14; + + RexQx[k1] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc1Q14, akpQx, 14) - + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss1Q14, bkpQx, 14); // 6 non-mul + 2 mul cycles, i.e. 8 cycles (6+2*7=20 cycles if 16x32mul) + RexQx[k2] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, ajpQx, 14) - + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, bjpQx, 14); + RexQx[k3] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc3Q14, akmQx, 14) - + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss3Q14, bkmQx, 14); + ImxQx[k1] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss1Q14, akpQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc1Q14, bkpQx, 14); + ImxQx[k2] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, ajpQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, bjpQx, 14); + ImxQx[k3] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss3Q14, akmQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc3Q14, bkmQx, 14); + //This mul segment needs 6*8 = 48 cycles for 16x16 muls, but 6*20 = 120 cycles for 16x32 muls + + + } + /* Complexity is: 51+48 = 99 cycles for 16x16 muls, but 51+120 = 171 cycles for 16x32 muls*/ + + // transform for factor of 3 + kk=0; + k1=20; + k2=40; + + for (hh=0; hh<4; hh++) { + for (ii=0; ii<20; ii++) { + akQx = RexQx[kk]; + bkQx = ImxQx[kk]; + ajQx = RexQx[k1] + RexQx[k2]; + bjQx = ImxQx[k1] + ImxQx[k2]; + RexQx[kk] = akQx + ajQx; + ImxQx[kk] = bkQx + bjQx; + tmp116 = ajQx >> 1; + tmp216 = bjQx >> 1; + akQx = akQx - tmp116; + bkQx = bkQx - tmp216; + tmp116 = RexQx[k1] - RexQx[k2]; + tmp216 = ImxQx[k1] - ImxQx[k2]; + + ajQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss60Q14, tmp116, 14); // Q14*Qx>>14 = Qx + bjQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss60Q14, tmp216, 14); // Q14*Qx>>14 = Qx + RexQx[k1] = akQx - bjQx; + RexQx[k2] = akQx + bjQx; + ImxQx[k1] = bkQx + ajQx; + ImxQx[k2] = bkQx - ajQx; + + kk++; + k1++; + k2++; + } + /* Complexity : (31+6)*20 = 740 cycles for 16x16 muls, but (31+18)*20 = 980 cycles for 16x32 muls*/ + kk=kk+40; + k1=k1+40; + k2=k2+40; + } + /* Complexity : 4*(740+3) = 2972 cycles for 16x16 muls, but 4*(980+3) = 3932 cycles for 16x32 muls*/ + + /* multiply by rotation factor for odd factor 3 or 5 (not for 4) + Same code (duplicated) for both ii=2 and ii=3 */ + kk = 1; + ee = 0; + ff = 0; + + for (gg=0; gg<19; gg++) { + kk += 20; + ff = ff+4; + for (hh=0; hh<2; hh++) { + ee = ff + hh * ff; + dd = ee + 60; + ccc2Q14 = kCosTabFfftQ14[ee]; + sss2Q14 = kCosTabFfftQ14[dd]; + if (iSign==1) { + sss2Q14 = -sss2Q14; + } + for (ii=0; ii<4; ii++) { + akQx = RexQx[kk]; + bkQx = ImxQx[kk]; + RexQx[kk] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, akQx, 14) - // Q14*Qx>>14 = Qx + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, bkQx, 14); + ImxQx[kk] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, akQx, 14) + // Q14*Qx>>14 = Qx + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, bkQx, 14); + + + kk += 60; + } + kk = kk - 220; + } + // Complexity: 2*(13+5+4*13+2) = 144 for 16x16 muls, but 2*(13+5+4*33+2) = 304 cycles for 16x32 muls + kk = kk - 59; + } + // Complexity: 19*144 = 2736 for 16x16 muls, but 19*304 = 5776 cycles for 16x32 muls + + // transform for factor of 5 + kk = 0; + ccc2Q14 = kCosTabFfftQ14[96]; + sss2Q14 = kCosTabFfftQ14[84]; + if (iSign==1) { + sss2Q14 = -sss2Q14; + } + + for (hh=0; hh<4; hh++) { + for (ii=0; ii<12; ii++) { + k1 = kk + 4; + k2 = k1 + 4; + k3 = k2 + 4; + k4 = k3 + 4; + + akpQx = RexQx[k1] + RexQx[k4]; + akmQx = RexQx[k1] - RexQx[k4]; + bkpQx = ImxQx[k1] + ImxQx[k4]; + bkmQx = ImxQx[k1] - ImxQx[k4]; + ajpQx = RexQx[k2] + RexQx[k3]; + ajmQx = RexQx[k2] - RexQx[k3]; + bjpQx = ImxQx[k2] + ImxQx[k3]; + bjmQx = ImxQx[k2] - ImxQx[k3]; + aaQx = RexQx[kk]; + bbQx = ImxQx[kk]; + RexQx[kk] = aaQx + akpQx + ajpQx; + ImxQx[kk] = bbQx + bkpQx + bjpQx; + + akQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc72Q14, akpQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, ajpQx, 14) + aaQx; + bkQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc72Q14, bkpQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, bjpQx, 14) + bbQx; + ajQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss72Q14, akmQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, ajmQx, 14); + bjQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss72Q14, bkmQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, bjmQx, 14); + // 32+4*8=64 or 32+4*20=112 + + RexQx[k1] = akQx - bjQx; + RexQx[k4] = akQx + bjQx; + ImxQx[k1] = bkQx + ajQx; + ImxQx[k4] = bkQx - ajQx; + + akQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, akpQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc72Q14, ajpQx, 14) + aaQx; + bkQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, bkpQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc72Q14, bjpQx, 14) + bbQx; + ajQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, akmQx, 14) - + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss72Q14, ajmQx, 14); + bjQx = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, bkmQx, 14) - + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss72Q14, bjmQx, 14); + // 8+4*8=40 or 8+4*20=88 + + RexQx[k2] = akQx - bjQx; + RexQx[k3] = akQx + bjQx; + ImxQx[k2] = bkQx + ajQx; + ImxQx[k3] = bkQx - ajQx; + + kk = k4 + 4; + } + // Complexity: 12*(64+40+10) = 1368 for 16x16 muls, but 12*(112+88+10) = 2520 cycles for 16x32 muls + kk -= 239; + } + // Complexity: 4*1368 = 5472 for 16x16 muls, but 4*2520 = 10080 cycles for 16x32 muls + + /* multiply by rotation factor for odd factor 3 or 5 (not for 4) + Same code (duplicated) for both ii=2 and ii=3 */ + kk = 1; + ee=0; + + for (gg=0; gg<3; gg++) { + kk += 4; + dd = 12 + 12 * gg; + ff = 0; + for (hh=0; hh<4; hh++) { + ff = ff+dd; + ee = ff+60; + for (ii=0; ii<12; ii++) { + akQx = RexQx[kk]; + bkQx = ImxQx[kk]; + + ccc2Q14 = kCosTabFfftQ14[ff]; + sss2Q14 = kCosTabFfftQ14[ee]; + + if (iSign==1) { + sss2Q14 = -sss2Q14; + } + + RexQx[kk] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, akQx, 14) - + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, bkQx, 14); + ImxQx[kk] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(sss2Q14, akQx, 14) + + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(ccc2Q14, bkQx, 14); + + kk += 20; + } + kk = kk - 236; + // Complexity: 12*(12+12) = 288 for 16x16 muls, but 12*(12+32) = 528 cycles for 16x32 muls + } + kk = kk - 19; + // Complexity: 4*288+6 for 16x16 muls, but 4*528+6 cycles for 16x32 muls + } + // Complexity: 3*4*288+6 = 3462 for 16x16 muls, but 3*4*528+6 = 6342 cycles for 16x32 muls + + + // last transform for factor of 4 */ + for (kk=0; kk<240; kk=kk+4) { + k1 = kk + 1; + k2 = k1 + 1; + k3 = k2 + 1; + + akpQx = RexQx[kk] + RexQx[k2]; + akmQx = RexQx[kk] - RexQx[k2]; + ajpQx = RexQx[k1] + RexQx[k3]; + ajmQx = RexQx[k1] - RexQx[k3]; + bkpQx = ImxQx[kk] + ImxQx[k2]; + bkmQx = ImxQx[kk] - ImxQx[k2]; + bjpQx = ImxQx[k1] + ImxQx[k3]; + bjmQx = ImxQx[k1] - ImxQx[k3]; + RexQx[kk] = akpQx + ajpQx; + ImxQx[kk] = bkpQx + bjpQx; + ajpQx = akpQx - ajpQx; + bjpQx = bkpQx - bjpQx; + if (iSign < 0) { + akpQx = akmQx + bjmQx; + bkpQx = bkmQx - ajmQx; + akmQx -= bjmQx; + bkmQx += ajmQx; + } else { + akpQx = akmQx - bjmQx; + bkpQx = bkmQx + ajmQx; + akmQx += bjmQx; + bkmQx -= ajmQx; + } + RexQx[k1] = akpQx; + RexQx[k2] = ajpQx; + RexQx[k3] = akmQx; + ImxQx[k1] = bkpQx; + ImxQx[k2] = bjpQx; + ImxQx[k3] = bkmQx; + } + // Complexity: 60*45 = 2700 for 16x16 muls, but 60*45 = 2700 cycles for 16x32 muls + + /* permute the results to normal order */ + for (ii=0; ii<240; ii++) { + ReDATAQx[ii]=RexQx[ii]; + ImDATAQx[ii]=ImxQx[ii]; + } + // Complexity: 240*2=480 cycles + + for (ii=0; ii<240; ii++) { + RexQx[ii]=ReDATAQx[kSortTabFft[ii]]; + ImxQx[ii]=ImDATAQx[kSortTabFft[ii]]; + } + // Complexity: 240*2*2=960 cycles + + // Total complexity: + // 16x16 16x32 + // Complexity: 10 10 + // Complexity: 99 171 + // Complexity: 2972 3932 + // Complexity: 2736 5776 + // Complexity: 5472 10080 + // Complexity: 3462 6342 + // Complexity: 2700 2700 + // Complexity: 480 480 + // Complexity: 960 960 + // ======================= + // 18891 30451 + // + // If this FFT is called 2 time each frame, i.e. 67 times per second, it will correspond to + // a C54 complexity of 67*18891/1000000 = 1.27 MIPS with 16x16-muls, and 67*30451/1000000 = + // = 2.04 MIPS with 16x32-muls. Note that this routine somtimes is called 6 times during the + // encoding of a frame, i.e. the max complexity would be 7/2*1.27 = 4.4 MIPS for the 16x16 mul case. + + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.h new file mode 100644 index 0000000000..83062eafd4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/*--------------------------------*-C-*---------------------------------* + * File: + * fft.h + * ---------------------------------------------------------------------* + * Re[]: real value array + * Im[]: imaginary value array + * nTotal: total number of complex values + * nPass: number of elements involved in this pass of transform + * nSpan: nspan/nPass = number of bytes to increment pointer + * in Re[] and Im[] + * isign: exponent: +1 = forward -1 = reverse + * scaling: normalizing constant by which the final result is *divided* + * scaling == -1, normalize by total dimension of the transform + * scaling < -1, normalize by the square-root of the total dimension + * + * ---------------------------------------------------------------------- + * See the comments in the code for correct usage! + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FFT_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FFT_H_ + + +#include "structs.h" + + +int16_t WebRtcIsacfix_FftRadix16Fastest(int16_t RexQx[], int16_t ImxQx[], int16_t iSign); + + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FFT_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_internal.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_internal.h new file mode 100644 index 0000000000..1c349697dd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_internal.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FILTERBANK_INTERNAL_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FILTERBANK_INTERNAL_H_ + +#include "typedefs.h" // NOLINT(build/include) + +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + +/* Arguments: + * io: Input/output, in Q0. + * len: Input, sample length. + * coefficient: Input. + * state: Input/output, filter state, in Q4. + */ +typedef void (*HighpassFilterFixDec32)(int16_t* io, + int16_t len, + const int16_t* coefficient, + int32_t* state); +extern HighpassFilterFixDec32 WebRtcIsacfix_HighpassFilterFixDec32; + +void WebRtcIsacfix_HighpassFilterFixDec32C(int16_t* io, + int16_t len, + const int16_t* coefficient, + int32_t* state); + +#if defined(MIPS_DSP_R1_LE) +void WebRtcIsacfix_HighpassFilterFixDec32MIPS(int16_t* io, + int16_t len, + const int16_t* coefficient, + int32_t* state); +#endif + +typedef void (*AllpassFilter2FixDec16)( + int16_t *data_ch1, // Input and output in channel 1, in Q0 + int16_t *data_ch2, // Input and output in channel 2, in Q0 + const int16_t *factor_ch1, // Scaling factor for channel 1, in Q15 + const int16_t *factor_ch2, // Scaling factor for channel 2, in Q15 + const int length, // Length of the data buffers + int32_t *filter_state_ch1, // Filter state for channel 1, in Q16 + int32_t *filter_state_ch2); // Filter state for channel 2, in Q16 +extern AllpassFilter2FixDec16 WebRtcIsacfix_AllpassFilter2FixDec16; + +void WebRtcIsacfix_AllpassFilter2FixDec16C( + int16_t *data_ch1, + int16_t *data_ch2, + const int16_t *factor_ch1, + const int16_t *factor_ch2, + const int length, + int32_t *filter_state_ch1, + int32_t *filter_state_ch2); + +#if defined(WEBRTC_HAS_NEON) +void WebRtcIsacfix_AllpassFilter2FixDec16Neon( + int16_t *data_ch1, + int16_t *data_ch2, + const int16_t *factor_ch1, + const int16_t *factor_ch2, + const int length, + int32_t *filter_state_ch1, + int32_t *filter_state_ch2); +#endif + +#if defined(MIPS_DSP_R1_LE) +void WebRtcIsacfix_AllpassFilter2FixDec16MIPS( + int16_t *data_ch1, + int16_t *data_ch2, + const int16_t *factor_ch1, + const int16_t *factor_ch2, + const int length, + int32_t *filter_state_ch1, + int32_t *filter_state_ch2); +#endif + +#if defined(__cplusplus) || defined(c_plusplus) +} +#endif + +#endif +/* WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FILTERBANK_INTERNAL_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.c new file mode 100644 index 0000000000..732611b643 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * filterbank_tables.c + * + * This file contains variables that are used in + * filterbanks.c + * + */ + +#include "filterbank_tables.h" + +/* HPstcoeff_in_Q14 = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; + * In float, they are: {-1.94895953203325f, 0.94984516000000f, + * -0.05101826139794f, 0.05015484000000f}; + */ +const int16_t WebRtcIsacfix_kHpStCoeffInQ30[8] = { + 16189, -31932, /* Q30 lo/hi pair */ + 17243, 15562, /* Q30 lo/hi pair */ + -17186, -26748, /* Q35 lo/hi pair */ + -27476, 26296 /* Q35 lo/hi pair */ +}; + +/* HPstcoeff_out_1_Q14 = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; + * In float, they are: {-1.99701049409000f, 0.99714204490000f, + * 0.01701049409000f, -0.01704204490000f}; + */ +const int16_t WebRtcIsacfix_kHPStCoeffOut1Q30[8] = { + -1306, -32719, /* Q30 lo/hi pair */ + 11486, 16337, /* Q30 lo/hi pair */ + 26078, 8918, /* Q35 lo/hi pair */ + 3956, -8935 /* Q35 lo/hi pair */ +}; + +/* HPstcoeff_out_2_Q14 = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; + * In float, they are: {-1.98645294509837f, 0.98672435560000f, + * 0.00645294509837f, -0.00662435560000f}; + */ +const int16_t WebRtcIsacfix_kHPStCoeffOut2Q30[8] = { + -2953, -32546, /* Q30 lo/hi pair */ + 32233, 16166, /* Q30 lo/hi pair */ + 13217, 3383, /* Q35 lo/hi pair */ + -4597, -3473 /* Q35 lo/hi pair */ +}; + +/* The upper channel all-pass filter factors */ +const int16_t WebRtcIsacfix_kUpperApFactorsQ15[2] = { + 1137, 12537 +}; + +/* The lower channel all-pass filter factors */ +const int16_t WebRtcIsacfix_kLowerApFactorsQ15[2] = { + 5059, 24379 +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.h new file mode 100644 index 0000000000..55e1f44a01 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * filterbank_tables.h + * + * Header file for variables that are defined in + * filterbank_tables.c. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FILTERBANK_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FILTERBANK_TABLES_H_ + +#include "typedefs.h" // NOLINT(build/include) + +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + +/********************* Coefficient Tables ************************/ + +/* HPstcoeff_in_Q14 = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; */ +/* [Q30lo Q30hi Q30lo Q30hi Q35lo Q35hi Q35lo Q35hi] */ +extern const int16_t WebRtcIsacfix_kHpStCoeffInQ30[8]; + +/* HPstcoeff_out_1_Q14 = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; */ +/* [Q30lo Q30hi Q30lo Q30hi Q35lo Q35hi Q35lo Q35hi] */ +extern const int16_t WebRtcIsacfix_kHPStCoeffOut1Q30[8]; + +/* HPstcoeff_out_2_Q14 = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; */ +/* [Q30lo Q30hi Q30lo Q30hi Q35lo Q35hi Q35lo Q35hi] */ +extern const int16_t WebRtcIsacfix_kHPStCoeffOut2Q30[8]; + +/* The upper channel all-pass filter factors */ +extern const int16_t WebRtcIsacfix_kUpperApFactorsQ15[2]; + +/* The lower channel all-pass filter factors */ +extern const int16_t WebRtcIsacfix_kLowerApFactorsQ15[2]; + +#if defined(__cplusplus) || defined(c_plusplus) +} +#endif + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_FILTERBANK_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks.c new file mode 100644 index 0000000000..b4583e95d1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks.c @@ -0,0 +1,421 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * filterbanks.c + * + * This file contains function + * WebRtcIsacfix_SplitAndFilter, and WebRtcIsacfix_FilterAndCombine + * which implement filterbanks that produce decimated lowpass and + * highpass versions of a signal, and performs reconstruction. + * + */ + +#include "filterbank_internal.h" + +#include "codec.h" +#include "filterbank_tables.h" +#include "settings.h" +#include "rtc_base/checks.h" + +// Declare a function pointer. +AllpassFilter2FixDec16 WebRtcIsacfix_AllpassFilter2FixDec16; + +void WebRtcIsacfix_AllpassFilter2FixDec16C( + int16_t *data_ch1, // Input and output in channel 1, in Q0 + int16_t *data_ch2, // Input and output in channel 2, in Q0 + const int16_t *factor_ch1, // Scaling factor for channel 1, in Q15 + const int16_t *factor_ch2, // Scaling factor for channel 2, in Q15 + const int length, // Length of the data buffers + int32_t *filter_state_ch1, // Filter state for channel 1, in Q16 + int32_t *filter_state_ch2) { // Filter state for channel 2, in Q16 + int n = 0; + int32_t state0_ch1 = filter_state_ch1[0], state1_ch1 = filter_state_ch1[1]; + int32_t state0_ch2 = filter_state_ch2[0], state1_ch2 = filter_state_ch2[1]; + int16_t in_out = 0; + int32_t a = 0, b = 0; + + // Assembly file assumption. + RTC_DCHECK_EQ(0, length % 2); + + for (n = 0; n < length; n++) { + // Process channel 1: + in_out = data_ch1[n]; + a = factor_ch1[0] * in_out; // Q15 * Q0 = Q15 + a *= 1 << 1; // Q15 -> Q16 + b = WebRtcSpl_AddSatW32(a, state0_ch1); + a = -factor_ch1[0] * (int16_t)(b >> 16); // Q15 + state0_ch1 = + WebRtcSpl_AddSatW32(a * (1 << 1), (int32_t)in_out * (1 << 16)); // Q16 + in_out = (int16_t) (b >> 16); // Save as Q0 + + a = factor_ch1[1] * in_out; // Q15 * Q0 = Q15 + a *= 1 << 1; // Q15 -> Q16 + b = WebRtcSpl_AddSatW32(a, state1_ch1); // Q16 + a = -factor_ch1[1] * (int16_t)(b >> 16); // Q15 + state1_ch1 = + WebRtcSpl_AddSatW32(a * (1 << 1), (int32_t)in_out * (1 << 16)); // Q16 + data_ch1[n] = (int16_t) (b >> 16); // Save as Q0 + + // Process channel 2: + in_out = data_ch2[n]; + a = factor_ch2[0] * in_out; // Q15 * Q0 = Q15 + a *= 1 << 1; // Q15 -> Q16 + b = WebRtcSpl_AddSatW32(a, state0_ch2); // Q16 + a = -factor_ch2[0] * (int16_t)(b >> 16); // Q15 + state0_ch2 = + WebRtcSpl_AddSatW32(a * (1 << 1), (int32_t)in_out * (1 << 16)); // Q16 + in_out = (int16_t) (b >> 16); // Save as Q0 + + a = factor_ch2[1] * in_out; // Q15 * Q0 = Q15 + a *= (1 << 1); // Q15 -> Q16 + b = WebRtcSpl_AddSatW32(a, state1_ch2); // Q16 + a = -factor_ch2[1] * (int16_t)(b >> 16); // Q15 + state1_ch2 = + WebRtcSpl_AddSatW32(a * (1 << 1), (int32_t)in_out * (1 << 16)); // Q16 + data_ch2[n] = (int16_t) (b >> 16); // Save as Q0 + } + + filter_state_ch1[0] = state0_ch1; + filter_state_ch1[1] = state1_ch1; + filter_state_ch2[0] = state0_ch2; + filter_state_ch2[1] = state1_ch2; +} + +// Declare a function pointer. +HighpassFilterFixDec32 WebRtcIsacfix_HighpassFilterFixDec32; + +void WebRtcIsacfix_HighpassFilterFixDec32C(int16_t *io, + int16_t len, + const int16_t *coefficient, + int32_t *state) +{ + int k; + int32_t a1 = 0, b1 = 0, c = 0, in = 0; + int32_t a2 = 0, b2 = 0; + int32_t state0 = state[0]; + int32_t state1 = state[1]; + + for (k=0; k Q7 */ + a1 = WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[5], state0) + + (WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[4], state0) >> 16); + b1 = WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[7], state1) + + (WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[6], state1) >> 16); + + /* Q30 * Q4 = Q34 ; shift 32 bit => Q2 */ + a2 = WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[1], state0) + + (WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[0], state0) >> 16); + b2 = WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[3], state1) + + (WEBRTC_SPL_MUL_16_32_RSFT16(coefficient[2], state1) >> 16); +#endif + + c = in + ((a1 + b1) >> 7); // Q0. + io[k] = (int16_t)WebRtcSpl_SatW32ToW16(c); // Write output as Q0. + + c = in * (1 << 2) - a2 - b2; // In Q2. + c = (int32_t)WEBRTC_SPL_SAT(536870911, c, -536870912); + + state1 = state0; + state0 = c * (1 << 2); // Write state as Q4 + } + state[0] = state0; + state[1] = state1; +} + + +void WebRtcIsacfix_SplitAndFilter1(int16_t *pin, + int16_t *LP16, + int16_t *HP16, + PreFiltBankstr *prefiltdata) +{ + /* Function WebRtcIsacfix_SplitAndFilter */ + /* This function creates low-pass and high-pass decimated versions of part of + the input signal, and part of the signal in the input 'lookahead buffer'. */ + + int k; + + int16_t tempin_ch1[FRAMESAMPLES/2 + QLOOKAHEAD]; + int16_t tempin_ch2[FRAMESAMPLES/2 + QLOOKAHEAD]; + int32_t tmpState_ch1[2 * (QORDER-1)]; /* 4 */ + int32_t tmpState_ch2[2 * (QORDER-1)]; /* 4 */ + + /* High pass filter */ + WebRtcIsacfix_HighpassFilterFixDec32(pin, FRAMESAMPLES, WebRtcIsacfix_kHpStCoeffInQ30, prefiltdata->HPstates_fix); + + + /* First Channel */ + for (k=0;kINLABUF1_fix[k]; + prefiltdata->INLABUF1_fix[k] = pin[FRAMESAMPLES + 1 - 2 * (QLOOKAHEAD - k)]; + } + + /* Second Channel. This is exactly like the first channel, except that the + even samples are now filtered instead (lower channel). */ + for (k=0;kINLABUF2_fix[k]; + prefiltdata->INLABUF2_fix[k] = pin[FRAMESAMPLES - 2 * (QLOOKAHEAD - k)]; + } + + + /*obtain polyphase components by forward all-pass filtering through each channel */ + /* The all pass filtering automatically updates the filter states which are exported in the + prefiltdata structure */ + WebRtcIsacfix_AllpassFilter2FixDec16(tempin_ch1, + tempin_ch2, + WebRtcIsacfix_kUpperApFactorsQ15, + WebRtcIsacfix_kLowerApFactorsQ15, + FRAMESAMPLES/2, + prefiltdata->INSTAT1_fix, + prefiltdata->INSTAT2_fix); + + for (k = 0; k < 2 * (QORDER - 1); k++) { + tmpState_ch1[k] = prefiltdata->INSTAT1_fix[k]; + tmpState_ch2[k] = prefiltdata->INSTAT2_fix[k]; + } + WebRtcIsacfix_AllpassFilter2FixDec16(tempin_ch1 + FRAMESAMPLES/2, + tempin_ch2 + FRAMESAMPLES/2, + WebRtcIsacfix_kUpperApFactorsQ15, + WebRtcIsacfix_kLowerApFactorsQ15, + QLOOKAHEAD, + tmpState_ch1, + tmpState_ch2); + + /* Now Construct low-pass and high-pass signals as combinations of polyphase components */ + for (k=0; k Q0 + tmp2 = (int32_t)tempin_ch2[k]; // Q0 -> Q0 + tmp3 = (tmp1 + tmp2) >> 1; /* Low pass signal. */ + LP16[k] = (int16_t)WebRtcSpl_SatW32ToW16(tmp3); /*low pass */ + tmp3 = (tmp1 - tmp2) >> 1; /* High pass signal. */ + HP16[k] = (int16_t)WebRtcSpl_SatW32ToW16(tmp3); /*high pass */ + } + +}/*end of WebRtcIsacfix_SplitAndFilter */ + + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + +/* Without lookahead */ +void WebRtcIsacfix_SplitAndFilter2(int16_t *pin, + int16_t *LP16, + int16_t *HP16, + PreFiltBankstr *prefiltdata) +{ + /* Function WebRtcIsacfix_SplitAndFilter2 */ + /* This function creates low-pass and high-pass decimated versions of part of + the input signal. */ + + int k; + + int16_t tempin_ch1[FRAMESAMPLES/2]; + int16_t tempin_ch2[FRAMESAMPLES/2]; + + + /* High pass filter */ + WebRtcIsacfix_HighpassFilterFixDec32(pin, FRAMESAMPLES, WebRtcIsacfix_kHpStCoeffInQ30, prefiltdata->HPstates_fix); + + + /* First Channel */ + for (k=0;kINSTAT1_fix, + prefiltdata->INSTAT2_fix); + + /* Now Construct low-pass and high-pass signals as combinations of polyphase components */ + for (k=0; k Q0 + tmp2 = (int32_t)tempin_ch2[k]; // Q0 -> Q0 + tmp3 = (tmp1 + tmp2) >> 1; /* Low pass signal. */ + LP16[k] = (int16_t)WebRtcSpl_SatW32ToW16(tmp3); /*low pass */ + tmp3 = (tmp1 - tmp2) >> 1; /* High pass signal. */ + HP16[k] = (int16_t)WebRtcSpl_SatW32ToW16(tmp3); /*high pass */ + } + +}/*end of WebRtcIsacfix_SplitAndFilter */ + +#endif + + + +////////////////////////////////////////////////////////// +////////// Combining +/* Function WebRtcIsacfix_FilterAndCombine */ +/* This is a decoder function that takes the decimated + length FRAMESAMPLES/2 input low-pass and + high-pass signals and creates a reconstructed fullband + output signal of length FRAMESAMPLES. WebRtcIsacfix_FilterAndCombine + is the sibling function of WebRtcIsacfix_SplitAndFilter */ +/* INPUTS: + inLP: a length FRAMESAMPLES/2 array of input low-pass + samples. + inHP: a length FRAMESAMPLES/2 array of input high-pass + samples. + postfiltdata: input data structure containing the filterbank + states from the previous decoding iteration. + OUTPUTS: + Out: a length FRAMESAMPLES array of output reconstructed + samples (fullband) based on the input low-pass and + high-pass signals. + postfiltdata: the input data structure containing the filterbank + states is updated for the next decoding iteration */ +void WebRtcIsacfix_FilterAndCombine1(int16_t *tempin_ch1, + int16_t *tempin_ch2, + int16_t *out16, + PostFiltBankstr *postfiltdata) +{ + int k; + int16_t in[FRAMESAMPLES]; + + /* all-pass filter the new upper and lower channel signal. + For upper channel, use the all-pass filter factors that were used as a + lower channel at the encoding side. So at the decoder, the corresponding + all-pass filter factors for each channel are swapped. + For lower channel signal, since all-pass filter factors at the decoder are + swapped from the ones at the encoder, the 'upper' channel all-pass filter + factors (kUpperApFactors) are used to filter this new lower channel signal. + */ + WebRtcIsacfix_AllpassFilter2FixDec16(tempin_ch1, + tempin_ch2, + WebRtcIsacfix_kLowerApFactorsQ15, + WebRtcIsacfix_kUpperApFactorsQ15, + FRAMESAMPLES/2, + postfiltdata->STATE_0_UPPER_fix, + postfiltdata->STATE_0_LOWER_fix); + + /* Merge outputs to form the full length output signal.*/ + for (k=0;kHPstates1_fix); + WebRtcIsacfix_HighpassFilterFixDec32(in, FRAMESAMPLES, WebRtcIsacfix_kHPStCoeffOut2Q30, postfiltdata->HPstates2_fix); + + for (k=0;kSTATE_0_UPPER_fix, + postfiltdata->STATE_0_LOWER_fix); + + /* Merge outputs to form the full length output signal.*/ + for (k=0;kHPstates1_fix); + WebRtcIsacfix_HighpassFilterFixDec32(in, len, WebRtcIsacfix_kHPStCoeffOut2Q30, postfiltdata->HPstates2_fix); + + for (k=0;k + +#include "rtc_base/checks.h" + +void WebRtcIsacfix_AllpassFilter2FixDec16Neon( + int16_t* data_ch1, // Input and output in channel 1, in Q0 + int16_t* data_ch2, // Input and output in channel 2, in Q0 + const int16_t* factor_ch1, // Scaling factor for channel 1, in Q15 + const int16_t* factor_ch2, // Scaling factor for channel 2, in Q15 + const int length, // Length of the data buffers + int32_t* filter_state_ch1, // Filter state for channel 1, in Q16 + int32_t* filter_state_ch2) { // Filter state for channel 2, in Q16 + RTC_DCHECK_EQ(0, length % 2); + int n = 0; + int16x4_t factorv; + int16x4_t datav; + int32x4_t statev; + + // Load factor_ch1 and factor_ch2. + factorv = vld1_dup_s16(factor_ch1); + factorv = vld1_lane_s16(factor_ch1 + 1, factorv, 1); + factorv = vld1_lane_s16(factor_ch2, factorv, 2); + factorv = vld1_lane_s16(factor_ch2 + 1, factorv, 3); + + // Load filter_state_ch1[0] and filter_state_ch2[0]. + statev = vld1q_dup_s32(filter_state_ch1); + statev = vld1q_lane_s32(filter_state_ch2, statev, 2); + + // Loop unrolling preprocessing. + int32x4_t a; + int16x4_t tmp1, tmp2; + + // Load data_ch1[0] and data_ch2[0]. + datav = vld1_dup_s16(data_ch1); + datav = vld1_lane_s16(data_ch2, datav, 2); + + a = vqdmlal_s16(statev, datav, factorv); + tmp1 = vshrn_n_s32(a, 16); + + // Update filter_state_ch1[0] and filter_state_ch2[0]. + statev = vqdmlsl_s16(vshll_n_s16(datav, 16), tmp1, factorv); + + // Load filter_state_ch1[1] and filter_state_ch2[1]. + statev = vld1q_lane_s32(filter_state_ch1 + 1, statev, 1); + statev = vld1q_lane_s32(filter_state_ch2 + 1, statev, 3); + + // Load data_ch1[1] and data_ch2[1]. + tmp1 = vld1_lane_s16(data_ch1 + 1, tmp1, 1); + tmp1 = vld1_lane_s16(data_ch2 + 1, tmp1, 3); + datav = vrev32_s16(tmp1); + + // Loop unrolling processing. + for (n = 0; n < length - 2; n += 2) { + a = vqdmlal_s16(statev, datav, factorv); + tmp1 = vshrn_n_s32(a, 16); + // Store data_ch1[n] and data_ch2[n]. + vst1_lane_s16(data_ch1 + n, tmp1, 1); + vst1_lane_s16(data_ch2 + n, tmp1, 3); + + // Update filter_state_ch1[0], filter_state_ch1[1] + // and filter_state_ch2[0], filter_state_ch2[1]. + statev = vqdmlsl_s16(vshll_n_s16(datav, 16), tmp1, factorv); + + // Load data_ch1[n + 2] and data_ch2[n + 2]. + tmp1 = vld1_lane_s16(data_ch1 + n + 2, tmp1, 1); + tmp1 = vld1_lane_s16(data_ch2 + n + 2, tmp1, 3); + datav = vrev32_s16(tmp1); + + a = vqdmlal_s16(statev, datav, factorv); + tmp2 = vshrn_n_s32(a, 16); + // Store data_ch1[n + 1] and data_ch2[n + 1]. + vst1_lane_s16(data_ch1 + n + 1, tmp2, 1); + vst1_lane_s16(data_ch2 + n + 1, tmp2, 3); + + // Update filter_state_ch1[0], filter_state_ch1[1] + // and filter_state_ch2[0], filter_state_ch2[1]. + statev = vqdmlsl_s16(vshll_n_s16(datav, 16), tmp2, factorv); + + // Load data_ch1[n + 3] and data_ch2[n + 3]. + tmp2 = vld1_lane_s16(data_ch1 + n + 3, tmp2, 1); + tmp2 = vld1_lane_s16(data_ch2 + n + 3, tmp2, 3); + datav = vrev32_s16(tmp2); + } + + // Loop unrolling post-processing. + a = vqdmlal_s16(statev, datav, factorv); + tmp1 = vshrn_n_s32(a, 16); + // Store data_ch1[n] and data_ch2[n]. + vst1_lane_s16(data_ch1 + n, tmp1, 1); + vst1_lane_s16(data_ch2 + n, tmp1, 3); + + // Update filter_state_ch1[0], filter_state_ch1[1] + // and filter_state_ch2[0], filter_state_ch2[1]. + statev = vqdmlsl_s16(vshll_n_s16(datav, 16), tmp1, factorv); + // Store filter_state_ch1[0] and filter_state_ch2[0]. + vst1q_lane_s32(filter_state_ch1, statev, 0); + vst1q_lane_s32(filter_state_ch2, statev, 2); + + datav = vrev32_s16(tmp1); + a = vqdmlal_s16(statev, datav, factorv); + tmp2 = vshrn_n_s32(a, 16); + // Store data_ch1[n + 1] and data_ch2[n + 1]. + vst1_lane_s16(data_ch1 + n + 1, tmp2, 1); + vst1_lane_s16(data_ch2 + n + 1, tmp2, 3); + + // Update filter_state_ch1[1] and filter_state_ch2[1]. + statev = vqdmlsl_s16(vshll_n_s16(datav, 16), tmp2, factorv); + // Store filter_state_ch1[1] and filter_state_ch2[1]. + vst1q_lane_s32(filter_state_ch1 + 1, statev, 1); + vst1q_lane_s32(filter_state_ch2 + 1, statev, 3); +} + +// This function is the prototype for above neon optimized function. +//void AllpassFilter2FixDec16BothChannels( +// int16_t *data_ch1, // Input and output in channel 1, in Q0 +// int16_t *data_ch2, // Input and output in channel 2, in Q0 +// const int16_t *factor_ch1, // Scaling factor for channel 1, in Q15 +// const int16_t *factor_ch2, // Scaling factor for channel 2, in Q15 +// const int length, // Length of the data buffers +// int32_t *filter_state_ch1, // Filter state for channel 1, in Q16 +// int32_t *filter_state_ch2) { // Filter state for channel 2, in Q16 +// int n = 0; +// int32_t state0_ch1 = filter_state_ch1[0], state1_ch1 = filter_state_ch1[1]; +// int32_t state0_ch2 = filter_state_ch2[0], state1_ch2 = filter_state_ch2[1]; +// int16_t sample0_ch1 = 0, sample0_ch2 = 0; +// int16_t sample1_ch1 = 0, sample1_ch2 = 0; +// int32_t a0_ch1 = 0, a0_ch2 = 0; +// int32_t b0_ch1 = 0, b0_ch2 = 0; +// +// int32_t a1_ch1 = 0, a1_ch2 = 0; +// int32_t b1_ch1 = 0, b1_ch2 = 0; +// int32_t b2_ch1 = 0, b2_ch2 = 0; +// +// // Loop unrolling preprocessing. +// +// sample0_ch1 = data_ch1[n]; +// sample0_ch2 = data_ch2[n]; +// +// a0_ch1 = (factor_ch1[0] * sample0_ch1) << 1; +// a0_ch2 = (factor_ch2[0] * sample0_ch2) << 1; +// +// b0_ch1 = WebRtcSpl_AddSatW32(a0_ch1, state0_ch1); +// b0_ch2 = WebRtcSpl_AddSatW32(a0_ch2, state0_ch2); //Q16+Q16=Q16 +// +// a0_ch1 = -factor_ch1[0] * (int16_t)(b0_ch1 >> 16); +// a0_ch2 = -factor_ch2[0] * (int16_t)(b0_ch2 >> 16); +// +// state0_ch1 = WebRtcSpl_AddSatW32(a0_ch1 <<1, (uint32_t)sample0_ch1 << 16); +// state0_ch2 = WebRtcSpl_AddSatW32(a0_ch2 <<1, (uint32_t)sample0_ch2 << 16); +// +// sample1_ch1 = data_ch1[n + 1]; +// sample0_ch1 = (int16_t) (b0_ch1 >> 16); //Save as Q0 +// sample1_ch2 = data_ch2[n + 1]; +// sample0_ch2 = (int16_t) (b0_ch2 >> 16); //Save as Q0 +// +// +// for (n = 0; n < length - 2; n += 2) { +// a1_ch1 = (factor_ch1[0] * sample1_ch1) << 1; +// a0_ch1 = (factor_ch1[1] * sample0_ch1) << 1; +// a1_ch2 = (factor_ch2[0] * sample1_ch2) << 1; +// a0_ch2 = (factor_ch2[1] * sample0_ch2) << 1; +// +// b1_ch1 = WebRtcSpl_AddSatW32(a1_ch1, state0_ch1); +// b0_ch1 = WebRtcSpl_AddSatW32(a0_ch1, state1_ch1); //Q16+Q16=Q16 +// b1_ch2 = WebRtcSpl_AddSatW32(a1_ch2, state0_ch2); //Q16+Q16=Q16 +// b0_ch2 = WebRtcSpl_AddSatW32(a0_ch2, state1_ch2); //Q16+Q16=Q16 +// +// a1_ch1 = -factor_ch1[0] * (int16_t)(b1_ch1 >> 16); +// a0_ch1 = -factor_ch1[1] * (int16_t)(b0_ch1 >> 16); +// a1_ch2 = -factor_ch2[0] * (int16_t)(b1_ch2 >> 16); +// a0_ch2 = -factor_ch2[1] * (int16_t)(b0_ch2 >> 16); +// +// state0_ch1 = WebRtcSpl_AddSatW32(a1_ch1<<1, (uint32_t)sample1_ch1 <<16); +// state1_ch1 = WebRtcSpl_AddSatW32(a0_ch1<<1, (uint32_t)sample0_ch1 <<16); +// state0_ch2 = WebRtcSpl_AddSatW32(a1_ch2<<1, (uint32_t)sample1_ch2 <<16); +// state1_ch2 = WebRtcSpl_AddSatW32(a0_ch2<<1, (uint32_t)sample0_ch2 <<16); +// +// sample0_ch1 = data_ch1[n + 2]; +// sample1_ch1 = (int16_t) (b1_ch1 >> 16); //Save as Q0 +// sample0_ch2 = data_ch2[n + 2]; +// sample1_ch2 = (int16_t) (b1_ch2 >> 16); //Save as Q0 +// +// a0_ch1 = (factor_ch1[0] * sample0_ch1) << 1; +// a1_ch1 = (factor_ch1[1] * sample1_ch1) << 1; +// a0_ch2 = (factor_ch2[0] * sample0_ch2) << 1; +// a1_ch2 = (factor_ch2[1] * sample1_ch2) << 1; +// +// b2_ch1 = WebRtcSpl_AddSatW32(a0_ch1, state0_ch1); +// b1_ch1 = WebRtcSpl_AddSatW32(a1_ch1, state1_ch1); //Q16+Q16=Q16 +// b2_ch2 = WebRtcSpl_AddSatW32(a0_ch2, state0_ch2); //Q16+Q16=Q16 +// b1_ch2 = WebRtcSpl_AddSatW32(a1_ch2, state1_ch2); //Q16+Q16=Q16 +// +// a0_ch1 = -factor_ch1[0] * (int16_t)(b2_ch1 >> 16); +// a1_ch1 = -factor_ch1[1] * (int16_t)(b1_ch1 >> 16); +// a0_ch2 = -factor_ch2[0] * (int16_t)(b2_ch2 >> 16); +// a1_ch2 = -factor_ch2[1] * (int16_t)(b1_ch2 >> 16); +// +// state0_ch1 = WebRtcSpl_AddSatW32(a0_ch1<<1, (uint32_t)sample0_ch1<<16); +// state1_ch1 = WebRtcSpl_AddSatW32(a1_ch1<<1, (uint32_t)sample1_ch1<<16); +// state0_ch2 = WebRtcSpl_AddSatW32(a0_ch2<<1, (uint32_t)sample0_ch2<<16); +// state1_ch2 = WebRtcSpl_AddSatW32(a1_ch2<<1, (uint32_t)sample1_ch2<<16); +// +// +// sample1_ch1 = data_ch1[n + 3]; +// sample0_ch1 = (int16_t) (b2_ch1 >> 16); //Save as Q0 +// sample1_ch2 = data_ch2[n + 3]; +// sample0_ch2 = (int16_t) (b2_ch2 >> 16); //Save as Q0 +// +// data_ch1[n] = (int16_t) (b0_ch1 >> 16); //Save as Q0 +// data_ch1[n + 1] = (int16_t) (b1_ch1 >> 16); //Save as Q0 +// data_ch2[n] = (int16_t) (b0_ch2 >> 16); +// data_ch2[n + 1] = (int16_t) (b1_ch2 >> 16); +// } +// +// // Loop unrolling post-processing. +// +// a1_ch1 = (factor_ch1[0] * sample1_ch1) << 1; +// a0_ch1 = (factor_ch1[1] * sample0_ch1) << 1; +// a1_ch2 = (factor_ch2[0] * sample1_ch2) << 1; +// a0_ch2 = (factor_ch2[1] * sample0_ch2) << 1; +// +// b1_ch1 = WebRtcSpl_AddSatW32(a1_ch1, state0_ch1); +// b0_ch1 = WebRtcSpl_AddSatW32(a0_ch1, state1_ch1); +// b1_ch2 = WebRtcSpl_AddSatW32(a1_ch2, state0_ch2); +// b0_ch2 = WebRtcSpl_AddSatW32(a0_ch2, state1_ch2); +// +// a1_ch1 = -factor_ch1[0] * (int16_t)(b1_ch1 >> 16); +// a0_ch1 = -factor_ch1[1] * (int16_t)(b0_ch1 >> 16); +// a1_ch2 = -factor_ch2[0] * (int16_t)(b1_ch2 >> 16); +// a0_ch2 = -factor_ch2[1] * (int16_t)(b0_ch2 >> 16); +// +// state0_ch1 = WebRtcSpl_AddSatW32(a1_ch1<<1, (uint32_t)sample1_ch1 << 16); +// state1_ch1 = WebRtcSpl_AddSatW32(a0_ch1<<1, (uint32_t)sample0_ch1 << 16); +// state0_ch2 = WebRtcSpl_AddSatW32(a1_ch2<<1, (uint32_t)sample1_ch2 << 16); +// state1_ch2 = WebRtcSpl_AddSatW32(a0_ch2<<1, (uint32_t)sample0_ch2 << 16); +// +// data_ch1[n] = (int16_t) (b0_ch1 >> 16); //Save as Q0 +// data_ch2[n] = (int16_t) (b0_ch2 >> 16); +// +// sample1_ch1 = (int16_t) (b1_ch1 >> 16); //Save as Q0 +// sample1_ch2 = (int16_t) (b1_ch2 >> 16); //Save as Q0 +// +// a1_ch1 = (factor_ch1[1] * sample1_ch1) << 1; +// a1_ch2 = (factor_ch2[1] * sample1_ch2) << 1; +// +// b1_ch1 = WebRtcSpl_AddSatW32(a1_ch1, state1_ch1); //Q16+Q16=Q16 +// b1_ch2 = WebRtcSpl_AddSatW32(a1_ch2, state1_ch2); //Q16+Q16=Q16 +// +// a1_ch1 = -factor_ch1[1] * (int16_t)(b1_ch1 >> 16); +// a1_ch2 = -factor_ch2[1] * (int16_t)(b1_ch2 >> 16); +// +// state1_ch1 = WebRtcSpl_AddSatW32(a1_ch1<<1, (uint32_t)sample1_ch1<<16); +// state1_ch2 = WebRtcSpl_AddSatW32(a1_ch2<<1, (uint32_t)sample1_ch2<<16); +// +// data_ch1[n + 1] = (int16_t) (b1_ch1 >> 16); //Save as Q0 +// data_ch2[n + 1] = (int16_t) (b1_ch2 >> 16); +// +// filter_state_ch1[0] = state0_ch1; +// filter_state_ch1[1] = state1_ch1; +// filter_state_ch2[0] = state0_ch2; +// filter_state_ch2[1] = state1_ch2; +//} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks_unittest.cc new file mode 100644 index 0000000000..d17f4a5f9d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks_unittest.cc @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/codecs/isac/fix/source/filterbank_internal.h" +#include "modules/audio_coding/codecs/isac/fix/source/filterbank_tables.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" +#include "rtc_base/sanitizer.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +class FilterBanksTest : public testing::Test { + protected: + // Pass a function pointer to the Tester function. + void RTC_NO_SANITIZE("signed-integer-overflow") // bugs.webrtc.org/5513 + CalculateResidualEnergyTester(AllpassFilter2FixDec16 + AllpassFilter2FixDec16Function) { + const int kSamples = QLOOKAHEAD; + const int kState = 2; + int16_t data_ch1[kSamples] = {0}; + int16_t data_ch2[kSamples] = {0}; + int32_t state_ch1[kState] = {0}; + int32_t state_ch2[kState] = {0}; + const int32_t out_state_ch1[kState] = {-809122714, 1645972152}; + const int32_t out_state_ch2[kState] = {428019288, 1057309936}; + const int32_t out_data_ch1[kSamples] = {0, 0, 347, 10618, 16718, -7089, + 32767, 16913, 27042, 8377, -22973, -28372, -27603, -14804, 398, -25332, + -11200, 18044, 25223, -6839, 1116, -23984, 32717, 7364}; + const int32_t out_data_ch2[kSamples] = {0, 0, 3010, 22351, 21106, 16969, + -2095, -664, 3513, -30980, 32767, -23839, 13335, 20289, -6831, 339, + -17207, 32767, 4959, 6177, 32767, 16599, -4747, 20504}; + int sign = 1; + + for (int i = 0; i < kSamples; i++) { + sign *= -1; + data_ch1[i] = sign * WEBRTC_SPL_WORD32_MAX / (i * i + 1); + data_ch2[i] = sign * WEBRTC_SPL_WORD32_MIN / (i * i + 1); + // UBSan: -1 * -2147483648 cannot be represented in type 'int' + }; + + AllpassFilter2FixDec16Function(data_ch1, + data_ch2, + WebRtcIsacfix_kUpperApFactorsQ15, + WebRtcIsacfix_kLowerApFactorsQ15, + kSamples, + state_ch1, + state_ch2); + + for (int i = 0; i < kSamples; i++) { + EXPECT_EQ(out_data_ch1[i], data_ch1[i]); + EXPECT_EQ(out_data_ch2[i], data_ch2[i]); + } + for (int i = 0; i < kState; i++) { + EXPECT_EQ(out_state_ch1[i], state_ch1[i]); + EXPECT_EQ(out_state_ch2[i], state_ch2[i]); + } + } +}; + +TEST_F(FilterBanksTest, AllpassFilter2FixDec16Test) { + CalculateResidualEnergyTester(WebRtcIsacfix_AllpassFilter2FixDec16C); +#if defined(WEBRTC_HAS_NEON) + CalculateResidualEnergyTester(WebRtcIsacfix_AllpassFilter2FixDec16Neon); +#endif +} + +TEST_F(FilterBanksTest, HighpassFilterFixDec32Test) { + const int kSamples = 20; + int16_t in[kSamples]; + int32_t state[2] = {12345, 987654}; +#ifdef WEBRTC_ARCH_ARM_V7 + int32_t out[kSamples] = {-1040, -1035, -22875, -1397, -27604, 20018, 7917, + -1279, -8552, -14494, -7558, -23537, -27258, -30554, -32768, -3432, -32768, + 25215, -27536, 22436}; +#else + int32_t out[kSamples] = {-1040, -1035, -22875, -1397, -27604, 20017, 7915, + -1280, -8554, -14496, -7561, -23541, -27263, -30560, -32768, -3441, -32768, + 25203, -27550, 22419}; +#endif + HighpassFilterFixDec32 WebRtcIsacfix_HighpassFilterFixDec32; +#if defined(MIPS_DSP_R1_LE) + WebRtcIsacfix_HighpassFilterFixDec32 = + WebRtcIsacfix_HighpassFilterFixDec32MIPS; +#else + WebRtcIsacfix_HighpassFilterFixDec32 = WebRtcIsacfix_HighpassFilterFixDec32C; +#endif + + for (int i = 0; i < kSamples; i++) { + in[i] = WEBRTC_SPL_WORD32_MAX / (i + 1); + } + + WebRtcIsacfix_HighpassFilterFixDec32(in, kSamples, + WebRtcIsacfix_kHPStCoeffOut1Q30, state); + + for (int i = 0; i < kSamples; i++) { + EXPECT_EQ(out[i], in[i]); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters.c new file mode 100644 index 0000000000..85860f72f4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/checks.h" +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" + +// Autocorrelation function in fixed point. +// NOTE! Different from SPLIB-version in how it scales the signal. +int WebRtcIsacfix_AutocorrC(int32_t* __restrict r, + const int16_t* __restrict x, + int16_t N, + int16_t order, + int16_t* __restrict scale) { + int i = 0; + int j = 0; + int16_t scaling = 0; + int32_t sum = 0; + uint32_t temp = 0; + int64_t prod = 0; + + // The ARM assembly code assumptoins. + RTC_DCHECK_EQ(0, N % 4); + RTC_DCHECK_GE(N, 8); + + // Calculate r[0]. + for (i = 0; i < N; i++) { + prod += x[i] * x[i]; + } + + // Calculate scaling (the value of shifting). + temp = (uint32_t)(prod >> 31); + if(temp == 0) { + scaling = 0; + } else { + scaling = 32 - WebRtcSpl_NormU32(temp); + } + r[0] = (int32_t)(prod >> scaling); + + // Perform the actual correlation calculation. + for (i = 1; i < order + 1; i++) { + prod = 0; + for (j = 0; j < N - i; j++) { + prod += x[j] * x[i + j]; + } + sum = (int32_t)(prod >> scaling); + r[i] = sum; + } + + *scale = scaling; + + return(order + 1); +} + +static const int32_t kApUpperQ15[ALLPASSSECTIONS] = { 1137, 12537 }; +static const int32_t kApLowerQ15[ALLPASSSECTIONS] = { 5059, 24379 }; + + +static void AllpassFilterForDec32(int16_t *InOut16, //Q0 + const int32_t *APSectionFactors, //Q15 + int16_t lengthInOut, + int32_t *FilterState) //Q16 +{ + int n, j; + int32_t a, b; + + for (j=0; j Q16 + b = WebRtcSpl_AddSatW32(a, FilterState[j]); //Q16+Q16=Q16 + // |a| in Q15 (Q0*Q31=Q31 shifted 16 gives Q15). + a = WEBRTC_SPL_MUL_16_32_RSFT16(b >> 16, -APSectionFactors[j]); + // FilterState[j]: Q15<<1 + Q0<<16 = Q16 + Q16 = Q16 + FilterState[j] = WebRtcSpl_AddSatW32(a << 1, (uint32_t)InOut16[n] << 16); + InOut16[n] = (int16_t)(b >> 16); // Save as Q0. + } + } +} + + + + +void WebRtcIsacfix_DecimateAllpass32(const int16_t *in, + int32_t *state_in, /* array of size: 2*ALLPASSSECTIONS+1 */ + int16_t N, /* number of input samples */ + int16_t *out) /* array of size N/2 */ +{ + int n; + int16_t data_vec[PITCH_FRAME_LEN]; + + /* copy input */ + memcpy(data_vec + 1, in, sizeof(int16_t) * (N - 1)); + + data_vec[0] = (int16_t)(state_in[2 * ALLPASSSECTIONS] >> 16); // z^-1 state. + state_in[2 * ALLPASSSECTIONS] = (uint32_t)in[N - 1] << 16; + + + + AllpassFilterForDec32(data_vec+1, kApUpperQ15, N, state_in); + AllpassFilterForDec32(data_vec, kApLowerQ15, N, state_in+ALLPASSSECTIONS); + + for (n=0;n> 3); + int count = (int)(N & 7); + // Declare temporary variables used as registry values. + int32_t r0, r1, r2, r3; +#if !defined(MIPS_DSP_R2_LE) + // For non-DSPR2 optimizations 4 more registers are used. + int32_t r4, r5, r6, r7; +#endif + + // Calculate r[0] and scaling needed. + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "mult $0, $0 \n\t" + // Loop is unrolled 8 times, set accumulator to zero in branch delay slot. + "beqz %[loop_size], 2f \n\t" + " mult $0, $0 \n\t" + "1: \n\t" + // Load 8 samples per loop iteration. +#if defined(MIPS_DSP_R2_LE) + "ulw %[r0], 0(%[in]) \n\t" + "ulw %[r1], 4(%[in]) \n\t" + "ulw %[r2], 8(%[in]) \n\t" + "ulw %[r3], 12(%[in]) \n\t" +#else + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 2(%[in]) \n\t" + "lh %[r2], 4(%[in]) \n\t" + "lh %[r3], 6(%[in]) \n\t" + "lh %[r4], 8(%[in]) \n\t" + "lh %[r5], 10(%[in]) \n\t" + "lh %[r6], 12(%[in]) \n\t" + "lh %[r7], 14(%[in]) \n\t" +#endif + "addiu %[loop_size], %[loop_size], -1 \n\t" + // Multiply and accumulate. +#if defined(MIPS_DSP_R2_LE) + "dpa.w.ph $ac0, %[r0], %[r0] \n\t" + "dpa.w.ph $ac0, %[r1], %[r1] \n\t" + "dpa.w.ph $ac0, %[r2], %[r2] \n\t" + "dpa.w.ph $ac0, %[r3], %[r3] \n\t" +#else + "madd %[r0], %[r0] \n\t" + "madd %[r1], %[r1] \n\t" + "madd %[r2], %[r2] \n\t" + "madd %[r3], %[r3] \n\t" + "madd %[r4], %[r4] \n\t" + "madd %[r5], %[r5] \n\t" + "madd %[r6], %[r6] \n\t" + "madd %[r7], %[r7] \n\t" +#endif + "bnez %[loop_size], 1b \n\t" + " addiu %[in], %[in], 16 \n\t" + "2: \n\t" + "beqz %[count], 4f \n\t" +#if defined(MIPS_DSP_R1_LE) + " extr.w %[r0], $ac0, 31 \n\t" +#else + " mfhi %[r2] \n\t" +#endif + // Process remaining samples (if any). + "3: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "addiu %[count], %[count], -1 \n\t" + "madd %[r0], %[r0] \n\t" + "bnez %[count], 3b \n\t" + " addiu %[in], %[in], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "extr.w %[r0], $ac0, 31 \n\t" +#else + "mfhi %[r2] \n\t" +#endif + "4: \n\t" +#if !defined(MIPS_DSP_R1_LE) + "mflo %[r3] \n\t" + "sll %[r0], %[r2], 1 \n\t" + "srl %[r1], %[r3], 31 \n\t" + "addu %[r0], %[r0], %[r1] \n\t" +#endif + // Calculate scaling (the value of shifting). + "clz %[r1], %[r0] \n\t" + "addiu %[r1], %[r1], -32 \n\t" + "subu %[scaling], $0, %[r1] \n\t" + "slti %[r1], %[r0], 0x1 \n\t" + "movn %[scaling], $0, %[r1] \n\t" +#if defined(MIPS_DSP_R1_LE) + "extrv.w %[r0], $ac0, %[scaling] \n\t" + "mfhi %[r2], $ac0 \n\t" +#else + "addiu %[r1], %[scaling], -32 \n\t" + "subu %[r1], $0, %[r1] \n\t" + "sllv %[r1], %[r2], %[r1] \n\t" + "srlv %[r0], %[r3], %[scaling] \n\t" + "addu %[r0], %[r0], %[r1] \n\t" +#endif + "slti %[r1], %[scaling], 32 \n\t" + "movz %[r0], %[r2], %[r1] \n\t" + ".set pop \n\t" + : [loop_size] "+r" (loop_size), [in] "+r" (in), [r0] "=&r" (r0), + [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), +#if !defined(MIPS_DSP_R2_LE) + [r4] "=&r" (r4), [r5] "=&r" (r5), [r6] "=&r" (r6), [r7] "=&r" (r7), +#endif + [count] "+r" (count), [scaling] "=r" (scaling) + : [N] "r" (N) + : "memory", "hi", "lo" + ); + r[0] = r0; + + // Correlation calculation is divided in 3 cases depending on the scaling + // value (different accumulator manipulation needed). Three slightly different + // loops are written in order to avoid branches inside the loop. + if (scaling == 0) { + // In this case, the result will be in low part of the accumulator. + for (i = 1; i < order + 1; i++) { + in = (int16_t*)x; + int16_t* in1 = (int16_t*)x + i; + count = N - i; + loop_size = (count) >> 2; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "mult $0, $0 \n\t" + "beqz %[loop_size], 2f \n\t" + " andi %[count], %[count], 0x3 \n\t" + // Loop processing 4 pairs of samples per iteration. + "1: \n\t" +#if defined(MIPS_DSP_R2_LE) + "ulw %[r0], 0(%[in]) \n\t" + "ulw %[r1], 0(%[in1]) \n\t" + "ulw %[r2], 4(%[in]) \n\t" + "ulw %[r3], 4(%[in1]) \n\t" +#else + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 0(%[in1]) \n\t" + "lh %[r2], 2(%[in]) \n\t" + "lh %[r3], 2(%[in1]) \n\t" + "lh %[r4], 4(%[in]) \n\t" + "lh %[r5], 4(%[in1]) \n\t" + "lh %[r6], 6(%[in]) \n\t" + "lh %[r7], 6(%[in1]) \n\t" +#endif + "addiu %[loop_size], %[loop_size], -1 \n\t" +#if defined(MIPS_DSP_R2_LE) + "dpa.w.ph $ac0, %[r0], %[r1] \n\t" + "dpa.w.ph $ac0, %[r2], %[r3] \n\t" +#else + "madd %[r0], %[r1] \n\t" + "madd %[r2], %[r3] \n\t" + "madd %[r4], %[r5] \n\t" + "madd %[r6], %[r7] \n\t" +#endif + "addiu %[in], %[in], 8 \n\t" + "bnez %[loop_size], 1b \n\t" + " addiu %[in1], %[in1], 8 \n\t" + "2: \n\t" + "beqz %[count], 4f \n\t" + " mflo %[r0] \n\t" + // Process remaining samples (if any). + "3: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 0(%[in1]) \n\t" + "addiu %[count], %[count], -1 \n\t" + "addiu %[in], %[in], 2 \n\t" + "madd %[r0], %[r1] \n\t" + "bnez %[count], 3b \n\t" + " addiu %[in1], %[in1], 2 \n\t" + "mflo %[r0] \n\t" + "4: \n\t" + ".set pop \n\t" + : [loop_size] "+r" (loop_size), [in] "+r" (in), [in1] "+r" (in1), +#if !defined(MIPS_DSP_R2_LE) + [r4] "=&r" (r4), [r5] "=&r" (r5), [r6] "=&r" (r6), [r7] "=&r" (r7), +#endif + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), + [count] "+r" (count) + : + : "memory", "hi", "lo" + ); + r[i] = r0; + } + } else if (scaling == 32) { + // In this case, the result will be high part of the accumulator. + for (i = 1; i < order + 1; i++) { + in = (int16_t*)x; + int16_t* in1 = (int16_t*)x + i; + count = N - i; + loop_size = (count) >> 2; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "mult $0, $0 \n\t" + "beqz %[loop_size], 2f \n\t" + " andi %[count], %[count], 0x3 \n\t" + // Loop processing 4 pairs of samples per iteration. + "1: \n\t" +#if defined(MIPS_DSP_R2_LE) + "ulw %[r0], 0(%[in]) \n\t" + "ulw %[r1], 0(%[in1]) \n\t" + "ulw %[r2], 4(%[in]) \n\t" + "ulw %[r3], 4(%[in1]) \n\t" +#else + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 0(%[in1]) \n\t" + "lh %[r2], 2(%[in]) \n\t" + "lh %[r3], 2(%[in1]) \n\t" + "lh %[r4], 4(%[in]) \n\t" + "lh %[r5], 4(%[in1]) \n\t" + "lh %[r6], 6(%[in]) \n\t" + "lh %[r7], 6(%[in1]) \n\t" +#endif + "addiu %[loop_size], %[loop_size], -1 \n\t" +#if defined(MIPS_DSP_R2_LE) + "dpa.w.ph $ac0, %[r0], %[r1] \n\t" + "dpa.w.ph $ac0, %[r2], %[r3] \n\t" +#else + "madd %[r0], %[r1] \n\t" + "madd %[r2], %[r3] \n\t" + "madd %[r4], %[r5] \n\t" + "madd %[r6], %[r7] \n\t" +#endif + "addiu %[in], %[in], 8 \n\t" + "bnez %[loop_size], 1b \n\t" + " addiu %[in1], %[in1], 8 \n\t" + "2: \n\t" + "beqz %[count], 4f \n\t" + " mfhi %[r0] \n\t" + // Process remaining samples (if any). + "3: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 0(%[in1]) \n\t" + "addiu %[count], %[count], -1 \n\t" + "addiu %[in], %[in], 2 \n\t" + "madd %[r0], %[r1] \n\t" + "bnez %[count], 3b \n\t" + " addiu %[in1], %[in1], 2 \n\t" + "mfhi %[r0] \n\t" + "4: \n\t" + ".set pop \n\t" + : [loop_size] "+r" (loop_size), [in] "+r" (in), [in1] "+r" (in1), +#if !defined(MIPS_DSP_R2_LE) + [r4] "=&r" (r4), [r5] "=&r" (r5), [r6] "=&r" (r6), [r7] "=&r" (r7), +#endif + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), + [count] "+r" (count) + : + : "memory", "hi", "lo" + ); + r[i] = r0; + } + } else { + // In this case, the result is obtained by combining low and high parts + // of the accumulator. +#if !defined(MIPS_DSP_R1_LE) + int32_t tmp_shift = 32 - scaling; +#endif + for (i = 1; i < order + 1; i++) { + in = (int16_t*)x; + int16_t* in1 = (int16_t*)x + i; + count = N - i; + loop_size = (count) >> 2; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "mult $0, $0 \n\t" + "beqz %[loop_size], 2f \n\t" + " andi %[count], %[count], 0x3 \n\t" + "1: \n\t" +#if defined(MIPS_DSP_R2_LE) + "ulw %[r0], 0(%[in]) \n\t" + "ulw %[r1], 0(%[in1]) \n\t" + "ulw %[r2], 4(%[in]) \n\t" + "ulw %[r3], 4(%[in1]) \n\t" +#else + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 0(%[in1]) \n\t" + "lh %[r2], 2(%[in]) \n\t" + "lh %[r3], 2(%[in1]) \n\t" + "lh %[r4], 4(%[in]) \n\t" + "lh %[r5], 4(%[in1]) \n\t" + "lh %[r6], 6(%[in]) \n\t" + "lh %[r7], 6(%[in1]) \n\t" +#endif + "addiu %[loop_size], %[loop_size], -1 \n\t" +#if defined(MIPS_DSP_R2_LE) + "dpa.w.ph $ac0, %[r0], %[r1] \n\t" + "dpa.w.ph $ac0, %[r2], %[r3] \n\t" +#else + "madd %[r0], %[r1] \n\t" + "madd %[r2], %[r3] \n\t" + "madd %[r4], %[r5] \n\t" + "madd %[r6], %[r7] \n\t" +#endif + "addiu %[in], %[in], 8 \n\t" + "bnez %[loop_size], 1b \n\t" + " addiu %[in1], %[in1], 8 \n\t" + "2: \n\t" + "beqz %[count], 4f \n\t" +#if defined(MIPS_DSP_R1_LE) + " extrv.w %[r0], $ac0, %[scaling] \n\t" +#else + " mfhi %[r0] \n\t" +#endif + "3: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 0(%[in1]) \n\t" + "addiu %[count], %[count], -1 \n\t" + "addiu %[in], %[in], 2 \n\t" + "madd %[r0], %[r1] \n\t" + "bnez %[count], 3b \n\t" + " addiu %[in1], %[in1], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "extrv.w %[r0], $ac0, %[scaling] \n\t" +#else + "mfhi %[r0] \n\t" +#endif + "4: \n\t" +#if !defined(MIPS_DSP_R1_LE) + "mflo %[r1] \n\t" + "sllv %[r0], %[r0], %[tmp_shift] \n\t" + "srlv %[r1], %[r1], %[scaling] \n\t" + "addu %[r0], %[r0], %[r1] \n\t" +#endif + ".set pop \n\t" + : [loop_size] "+r" (loop_size), [in] "+r" (in), [in1] "+r" (in1), +#if !defined(MIPS_DSP_R2_LE) + [r4] "=&r" (r4), [r5] "=&r" (r5), [r6] "=&r" (r6), [r7] "=&r" (r7), +#endif + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), + [count] "+r" (count) + : [scaling] "r" (scaling) +#if !defined(MIPS_DSP_R1_LE) + , [tmp_shift] "r" (tmp_shift) +#endif + : "memory", "hi", "lo" + ); + r[i] = r0; + } + } + *scale = scaling; + + return (order + 1); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_neon.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_neon.c new file mode 100644 index 0000000000..1734a969cb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_neon.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "rtc_base/checks.h" +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" + +// Autocorrelation function in fixed point. +// NOTE! Different from SPLIB-version in how it scales the signal. +int WebRtcIsacfix_AutocorrNeon(int32_t* __restrict r, + const int16_t* x, + int16_t n, + int16_t order, + int16_t* __restrict scale) { + int i = 0; + int16_t scaling = 0; + uint32_t temp = 0; + int64_t prod = 0; + int64_t prod_tail = 0; + + RTC_DCHECK_EQ(0, n % 4); + RTC_DCHECK_GE(n, 8); + + // Calculate r[0]. + int16x4_t x0_v; + int32x4_t tmpa0_v; + int64x2_t tmpb_v; + + tmpb_v = vdupq_n_s64(0); + const int16_t* x_start = x; + const int16_t* x_end0 = x_start + n; + while (x_start < x_end0) { + x0_v = vld1_s16(x_start); + tmpa0_v = vmull_s16(x0_v, x0_v); + tmpb_v = vpadalq_s32(tmpb_v, tmpa0_v); + x_start += 4; + } + +#ifdef WEBRTC_ARCH_ARM64 + prod = vaddvq_s64(tmpb_v); +#else + prod = vget_lane_s64(vadd_s64(vget_low_s64(tmpb_v), vget_high_s64(tmpb_v)), + 0); +#endif + // Calculate scaling (the value of shifting). + temp = (uint32_t)(prod >> 31); + + scaling = temp ? 32 - WebRtcSpl_NormU32(temp) : 0; + r[0] = (int32_t)(prod >> scaling); + + int16x8_t x1_v; + int16x8_t y_v; + int32x4_t tmpa1_v; + // Perform the actual correlation calculation. + for (i = 1; i < order + 1; i++) { + tmpb_v = vdupq_n_s64(0); + int rest = (n - i) % 8; + x_start = x; + x_end0 = x_start + n - i - rest; + const int16_t* y_start = x_start + i; + while (x_start < x_end0) { + x1_v = vld1q_s16(x_start); + y_v = vld1q_s16(y_start); + tmpa0_v = vmull_s16(vget_low_s16(x1_v), vget_low_s16(y_v)); +#ifdef WEBRTC_ARCH_ARM64 + tmpa1_v = vmull_high_s16(x1_v, y_v); +#else + tmpa1_v = vmull_s16(vget_high_s16(x1_v), vget_high_s16(y_v)); +#endif + tmpb_v = vpadalq_s32(tmpb_v, tmpa0_v); + tmpb_v = vpadalq_s32(tmpb_v, tmpa1_v); + x_start += 8; + y_start += 8; + } + // The remaining calculation. + const int16_t* x_end1 = x + n - i; + if (rest >= 4) { + int16x4_t x2_v = vld1_s16(x_start); + int16x4_t y2_v = vld1_s16(y_start); + tmpa0_v = vmull_s16(x2_v, y2_v); + tmpb_v = vpadalq_s32(tmpb_v, tmpa0_v); + x_start += 4; + y_start += 4; + } +#ifdef WEBRTC_ARCH_ARM64 + prod = vaddvq_s64(tmpb_v); +#else + prod = vget_lane_s64(vadd_s64(vget_low_s64(tmpb_v), vget_high_s64(tmpb_v)), + 0); +#endif + + prod_tail = 0; + while (x_start < x_end1) { + prod_tail += *x_start * *y_start; + ++x_start; + ++y_start; + } + + r[i] = (int32_t)((prod + prod_tail) >> scaling); + } + + *scale = scaling; + + return order + 1; +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_unittest.cc new file mode 100644 index 0000000000..fa529863c0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_unittest.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +class FiltersTest : public testing::Test { + protected: + // Pass a function pointer to the Tester function. + void FiltersTester(AutocorrFix WebRtcIsacfix_AutocorrFixFunction) { + const int kOrder = 12; + const int kBuffer = 40; + int16_t scale = 0; + int32_t r_buffer[kOrder + 2] = {0}; + + // Test an overflow case. + const int16_t x_buffer_0[kBuffer] = {0, 0, 3010, 22351, 21106, 16969, -2095, + -664, 3513, -30980, 32767, -23839, 13335, 20289, -6831, 339, -17207, + 32767, 4959, 6177, 32767, 16599, -4747, 20504, 3513, -30980, 32767, + -23839, 13335, 20289, 0, -16969, -2095, -664, 3513, 31981, 32767, + -13839, 23336, 30281}; + const int32_t r_expected_0[kOrder + 2] = {1872498461, -224288754, 203789985, + 483400487, -208272635, 2436500, 137785322, 266600814, -208486262, + 329510080, 137949184, -161738972, -26894267, 237630192}; + + WebRtcIsacfix_AutocorrFixFunction(r_buffer, x_buffer_0, + kBuffer, kOrder + 1, &scale); + for (int i = 0; i < kOrder + 2; i++) { + EXPECT_EQ(r_expected_0[i], r_buffer[i]); + } + EXPECT_EQ(3, scale); + + // Test a no-overflow case. + const int16_t x_buffer_1[kBuffer] = {0, 0, 300, 21, 206, 169, -295, + -664, 3513, -300, 327, -29, 15, 289, -6831, 339, -107, + 37, 59, 6177, 327, 169, -4747, 204, 313, -980, 767, + -9, 135, 289, 0, -6969, -2095, -664, 0, 1, 7, + -39, 236, 281}; + const int32_t r_expected_1[kOrder + 2] = {176253864, 8126617, 1983287, + -26196788, -3487363, -42839676, -24644043, 3469813, 30559879, 31905045, + 5101567, 29328896, -55787438, -13163978}; + + WebRtcIsacfix_AutocorrFixFunction(r_buffer, x_buffer_1, + kBuffer, kOrder + 1, &scale); + for (int i = 0; i < kOrder + 2; i++) { + EXPECT_EQ(r_expected_1[i], r_buffer[i]); + } + EXPECT_EQ(0, scale); + } +}; + +TEST_F(FiltersTest, AutocorrFixTest) { + FiltersTester(WebRtcIsacfix_AutocorrC); +#if defined(WEBRTC_HAS_NEON) + FiltersTester(WebRtcIsacfix_AutocorrNeon); +#endif +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/initialize.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/initialize.c new file mode 100644 index 0000000000..4481f98122 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/initialize.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * initialize.c + * + * Internal initfunctions + * + */ + +#include "codec.h" +#include "structs.h" +#include "pitch_estimator.h" + + +void WebRtcIsacfix_InitMaskingEnc(MaskFiltstr_enc *maskdata) { + + int k; + + for (k = 0; k < WINLEN; k++) { + maskdata->DataBufferLoQ0[k] = (int16_t) 0; + maskdata->DataBufferHiQ0[k] = (int16_t) 0; + } + for (k = 0; k < ORDERLO+1; k++) { + maskdata->CorrBufLoQQ[k] = (int32_t) 0; + maskdata->CorrBufLoQdom[k] = 0; + + maskdata->PreStateLoGQ15[k] = 0; + + } + for (k = 0; k < ORDERHI+1; k++) { + maskdata->CorrBufHiQQ[k] = (int32_t) 0; + maskdata->CorrBufHiQdom[k] = 0; + maskdata->PreStateHiGQ15[k] = 0; + } + + maskdata->OldEnergy = 10; + + return; +} + +void WebRtcIsacfix_InitMaskingDec(MaskFiltstr_dec *maskdata) { + + int k; + + for (k = 0; k < ORDERLO+1; k++) + { + maskdata->PostStateLoGQ0[k] = 0; + } + for (k = 0; k < ORDERHI+1; k++) + { + maskdata->PostStateHiGQ0[k] = 0; + } + + maskdata->OldEnergy = 10; + + return; +} + + + + + + + +void WebRtcIsacfix_InitPreFilterbank(PreFiltBankstr *prefiltdata) +{ + int k; + + for (k = 0; k < QLOOKAHEAD; k++) { + prefiltdata->INLABUF1_fix[k] = 0; + prefiltdata->INLABUF2_fix[k] = 0; + } + for (k = 0; k < 2 * (QORDER - 1); k++) { + prefiltdata->INSTAT1_fix[k] = 0; + prefiltdata->INSTAT2_fix[k] = 0; + } + + /* High pass filter states */ + prefiltdata->HPstates_fix[0] = 0; + prefiltdata->HPstates_fix[1] = 0; + + return; +} + +void WebRtcIsacfix_InitPostFilterbank(PostFiltBankstr *postfiltdata) +{ + int k; + + for (k = 0; k < 2 * POSTQORDER; k++) { + postfiltdata->STATE_0_LOWER_fix[k] = 0; + postfiltdata->STATE_0_UPPER_fix[k] = 0; + } + + /* High pass filter states */ + + postfiltdata->HPstates1_fix[0] = 0; + postfiltdata->HPstates1_fix[1] = 0; + + postfiltdata->HPstates2_fix[0] = 0; + postfiltdata->HPstates2_fix[1] = 0; + + return; +} + + +void WebRtcIsacfix_InitPitchFilter(PitchFiltstr *pitchfiltdata) +{ + int k; + + for (k = 0; k < PITCH_BUFFSIZE; k++) + pitchfiltdata->ubufQQ[k] = 0; + for (k = 0; k < (PITCH_DAMPORDER); k++) + pitchfiltdata->ystateQQ[k] = 0; + + pitchfiltdata->oldlagQ7 = 6400; /* 50.0 in Q7 */ + pitchfiltdata->oldgainQ12 = 0; +} + +void WebRtcIsacfix_InitPitchAnalysis(PitchAnalysisStruct *State) +{ + int k; + + for (k = 0; k < PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2; k++) { + State->dec_buffer16[k] = 0; + } + for (k = 0; k < 2 * ALLPASSSECTIONS + 1; k++) { + State->decimator_state32[k] = 0; + } + + for (k = 0; k < QLOOKAHEAD; k++) + State->inbuf[k] = 0; + + WebRtcIsacfix_InitPitchFilter(&(State->PFstr_wght)); + + WebRtcIsacfix_InitPitchFilter(&(State->PFstr)); +} + + +void WebRtcIsacfix_InitPlc( PLCstr *State ) +{ + State->decayCoeffPriodic = WEBRTC_SPL_WORD16_MAX; + State->decayCoeffNoise = WEBRTC_SPL_WORD16_MAX; + + State->used = PLC_WAS_USED; + + WebRtcSpl_ZerosArrayW16(State->overlapLP, RECOVERY_OVERLAP); + WebRtcSpl_ZerosArrayW16(State->lofilt_coefQ15, ORDERLO); + WebRtcSpl_ZerosArrayW16(State->hifilt_coefQ15, ORDERHI ); + + State->AvgPitchGain_Q12 = 0; + State->lastPitchGain_Q12 = 0; + State->lastPitchLag_Q7 = 0; + State->gain_lo_hiQ17[0]=State->gain_lo_hiQ17[1] = 0; + WebRtcSpl_ZerosArrayW16(State->prevPitchInvIn, FRAMESAMPLES/2); + WebRtcSpl_ZerosArrayW16(State->prevPitchInvOut, PITCH_MAX_LAG + 10 ); + WebRtcSpl_ZerosArrayW32(State->prevHP, PITCH_MAX_LAG + 10 ); + State->pitchCycles = 0; + State->A = 0; + State->B = 0; + State->pitchIndex = 0; + State->stretchLag = 240; + State->seed = 4447; + + +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isac_fix_type.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isac_fix_type.h new file mode 100644 index 0000000000..5f0f822a1c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isac_fix_type.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ISAC_FIX_TYPE_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ISAC_FIX_TYPE_H_ + +#include "modules/audio_coding/codecs/isac/fix/include/isacfix.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +class IsacFix { + public: + using instance_type = ISACFIX_MainStruct; + static const bool has_swb = false; + static inline int16_t Control(instance_type* inst, + int32_t rate, + int framesize) { + return WebRtcIsacfix_Control(inst, rate, framesize); + } + static inline int16_t ControlBwe(instance_type* inst, + int32_t rate_bps, + int frame_size_ms, + int16_t enforce_frame_size) { + return WebRtcIsacfix_ControlBwe(inst, rate_bps, frame_size_ms, + enforce_frame_size); + } + static inline int16_t Create(instance_type** inst) { + return WebRtcIsacfix_Create(inst); + } + static inline int DecodeInternal(instance_type* inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speech_type) { + return WebRtcIsacfix_Decode(inst, encoded, len, decoded, speech_type); + } + static inline size_t DecodePlc(instance_type* inst, + int16_t* decoded, + size_t num_lost_frames) { + return WebRtcIsacfix_DecodePlc(inst, decoded, num_lost_frames); + } + static inline void DecoderInit(instance_type* inst) { + WebRtcIsacfix_DecoderInit(inst); + } + static inline int Encode(instance_type* inst, + const int16_t* speech_in, + uint8_t* encoded) { + return WebRtcIsacfix_Encode(inst, speech_in, encoded); + } + static inline int16_t EncoderInit(instance_type* inst, int16_t coding_mode) { + return WebRtcIsacfix_EncoderInit(inst, coding_mode); + } + static inline uint16_t EncSampRate(instance_type* inst) { + return kFixSampleRate; + } + + static inline int16_t Free(instance_type* inst) { + return WebRtcIsacfix_Free(inst); + } + static inline void GetBandwidthInfo(instance_type* inst, + IsacBandwidthInfo* bwinfo) { + WebRtcIsacfix_GetBandwidthInfo(inst, bwinfo); + } + static inline int16_t GetErrorCode(instance_type* inst) { + return WebRtcIsacfix_GetErrorCode(inst); + } + + static inline int16_t GetNewFrameLen(instance_type* inst) { + return WebRtcIsacfix_GetNewFrameLen(inst); + } + static inline void SetBandwidthInfo(instance_type* inst, + const IsacBandwidthInfo* bwinfo) { + WebRtcIsacfix_SetBandwidthInfo(inst, bwinfo); + } + static inline int16_t SetDecSampRate(instance_type* inst, + uint16_t sample_rate_hz) { + RTC_DCHECK_EQ(sample_rate_hz, kFixSampleRate); + return 0; + } + static inline int16_t SetEncSampRate(instance_type* inst, + uint16_t sample_rate_hz) { + RTC_DCHECK_EQ(sample_rate_hz, kFixSampleRate); + return 0; + } + static inline void SetEncSampRateInDecoder(instance_type* inst, + uint16_t sample_rate_hz) { + RTC_DCHECK_EQ(sample_rate_hz, kFixSampleRate); + } + static inline void SetInitialBweBottleneck(instance_type* inst, + int bottleneck_bits_per_second) { + WebRtcIsacfix_SetInitialBweBottleneck(inst, bottleneck_bits_per_second); + } + static inline int16_t UpdateBwEstimate(instance_type* inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts) { + return WebRtcIsacfix_UpdateBwEstimate(inst, encoded, packet_size, + rtp_seq_number, send_ts, arr_ts); + } + static inline int16_t SetMaxPayloadSize(instance_type* inst, + int16_t max_payload_size_bytes) { + return WebRtcIsacfix_SetMaxPayloadSize(inst, max_payload_size_bytes); + } + static inline int16_t SetMaxRate(instance_type* inst, int32_t max_bit_rate) { + return WebRtcIsacfix_SetMaxRate(inst, max_bit_rate); + } + + private: + enum { kFixSampleRate = 16000 }; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_ISAC_FIX_TYPE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c new file mode 100644 index 0000000000..bbe9098b01 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c @@ -0,0 +1,1551 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * isacfix.c + * + * This C file contains the functions for the ISAC API + * + */ + +#include "modules/audio_coding/codecs/isac/fix/include/isacfix.h" + +#include + +#include "rtc_base/checks.h" +#include "modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h" +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" +#include "modules/audio_coding/codecs/isac/fix/source/entropy_coding.h" +#include "modules/audio_coding/codecs/isac/fix/source/filterbank_internal.h" +#include "modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h" +#include "modules/audio_coding/codecs/isac/fix/source/structs.h" +#include "system_wrappers/include/cpu_features_wrapper.h" + +// Declare function pointers. +FilterMaLoopFix WebRtcIsacfix_FilterMaLoopFix; +Spec2Time WebRtcIsacfix_Spec2Time; +Time2Spec WebRtcIsacfix_Time2Spec; +MatrixProduct1 WebRtcIsacfix_MatrixProduct1; +MatrixProduct2 WebRtcIsacfix_MatrixProduct2; + +/* This method assumes that |stream_size_bytes| is in valid range, + * i.e. >= 0 && <= STREAM_MAXW16_60MS + */ +static void InitializeDecoderBitstream(size_t stream_size_bytes, + Bitstr_dec* bitstream) { + bitstream->W_upper = 0xFFFFFFFF; + bitstream->streamval = 0; + bitstream->stream_index = 0; + bitstream->full = 1; + bitstream->stream_size = (stream_size_bytes + 1) >> 1; + memset(bitstream->stream, 0, sizeof(bitstream->stream)); +} + +/************************************************************************** + * WebRtcIsacfix_AssignSize(...) + * + * Functions used when malloc is not allowed + * Returns number of bytes needed to allocate for iSAC struct. + * + */ + +int16_t WebRtcIsacfix_AssignSize(int *sizeinbytes) { + *sizeinbytes=sizeof(ISACFIX_SubStruct)*2/sizeof(int16_t); + return(0); +} + +/*************************************************************************** + * WebRtcIsacfix_Assign(...) + * + * Functions used when malloc is not allowed + * Place struct at given address + * + * If successful, Return 0, else Return -1 + */ + +int16_t WebRtcIsacfix_Assign(ISACFIX_MainStruct **inst, void *ISACFIX_inst_Addr) { + if (ISACFIX_inst_Addr!=NULL) { + ISACFIX_SubStruct* self = ISACFIX_inst_Addr; + *inst = (ISACFIX_MainStruct*)self; + self->errorcode = 0; + self->initflag = 0; + self->ISACenc_obj.SaveEnc_ptr = NULL; + WebRtcIsacfix_InitBandwidthEstimator(&self->bwestimator_obj); + return(0); + } else { + return(-1); + } +} + + +#ifndef ISACFIX_NO_DYNAMIC_MEM + +/**************************************************************************** + * WebRtcIsacfix_Create(...) + * + * This function creates a ISAC instance, which will contain the state + * information for one coding/decoding channel. + * + * Input: + * - *ISAC_main_inst : a pointer to the coder instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_Create(ISACFIX_MainStruct **ISAC_main_inst) +{ + ISACFIX_SubStruct *tempo; + tempo = malloc(1 * sizeof(ISACFIX_SubStruct)); + *ISAC_main_inst = (ISACFIX_MainStruct *)tempo; + if (*ISAC_main_inst!=NULL) { + (*(ISACFIX_SubStruct**)ISAC_main_inst)->errorcode = 0; + (*(ISACFIX_SubStruct**)ISAC_main_inst)->initflag = 0; + (*(ISACFIX_SubStruct**)ISAC_main_inst)->ISACenc_obj.SaveEnc_ptr = NULL; + WebRtcSpl_Init(); + WebRtcIsacfix_InitBandwidthEstimator(&tempo->bwestimator_obj); + return(0); + } else { + return(-1); + } +} + + +/**************************************************************************** + * WebRtcIsacfix_CreateInternal(...) + * + * This function creates the memory that is used to store data in the encoder + * + * Input: + * - *ISAC_main_inst : a pointer to the coder instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_CreateInternal(ISACFIX_MainStruct *ISAC_main_inst) +{ + ISACFIX_SubStruct *ISAC_inst; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* Allocate memory for storing encoder data */ + ISAC_inst->ISACenc_obj.SaveEnc_ptr = malloc(1 * sizeof(IsacSaveEncoderData)); + + if (ISAC_inst->ISACenc_obj.SaveEnc_ptr!=NULL) { + return(0); + } else { + return(-1); + } +} + + +#endif + + + +/**************************************************************************** + * WebRtcIsacfix_Free(...) + * + * This function frees the ISAC instance created at the beginning. + * + * Input: + * - ISAC_main_inst : a ISAC instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_Free(ISACFIX_MainStruct *ISAC_main_inst) +{ + free(ISAC_main_inst); + return(0); +} + +/**************************************************************************** + * WebRtcIsacfix_FreeInternal(...) + * + * This function frees the internal memory for storing encoder data. + * + * Input: + * - ISAC_main_inst : a ISAC instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_FreeInternal(ISACFIX_MainStruct *ISAC_main_inst) +{ + ISACFIX_SubStruct *ISAC_inst; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* Release memory */ + free(ISAC_inst->ISACenc_obj.SaveEnc_ptr); + + return(0); +} + +/**************************************************************************** + * WebRtcIsacfix_InitNeon(...) + * + * This function initializes function pointers for ARM Neon platform. + */ + +#if defined(WEBRTC_HAS_NEON) +static void WebRtcIsacfix_InitNeon(void) { + WebRtcIsacfix_AutocorrFix = WebRtcIsacfix_AutocorrNeon; + WebRtcIsacfix_FilterMaLoopFix = WebRtcIsacfix_FilterMaLoopNeon; + WebRtcIsacfix_Spec2Time = WebRtcIsacfix_Spec2TimeNeon; + WebRtcIsacfix_Time2Spec = WebRtcIsacfix_Time2SpecNeon; + WebRtcIsacfix_AllpassFilter2FixDec16 = + WebRtcIsacfix_AllpassFilter2FixDec16Neon; + WebRtcIsacfix_MatrixProduct1 = WebRtcIsacfix_MatrixProduct1Neon; + WebRtcIsacfix_MatrixProduct2 = WebRtcIsacfix_MatrixProduct2Neon; +} +#endif + +/**************************************************************************** + * WebRtcIsacfix_InitMIPS(...) + * + * This function initializes function pointers for MIPS platform. + */ + +#if defined(MIPS32_LE) +static void WebRtcIsacfix_InitMIPS(void) { + WebRtcIsacfix_AutocorrFix = WebRtcIsacfix_AutocorrMIPS; + WebRtcIsacfix_FilterMaLoopFix = WebRtcIsacfix_FilterMaLoopMIPS; + WebRtcIsacfix_Spec2Time = WebRtcIsacfix_Spec2TimeMIPS; + WebRtcIsacfix_Time2Spec = WebRtcIsacfix_Time2SpecMIPS; + WebRtcIsacfix_MatrixProduct1 = WebRtcIsacfix_MatrixProduct1MIPS; + WebRtcIsacfix_MatrixProduct2 = WebRtcIsacfix_MatrixProduct2MIPS; +#if defined(MIPS_DSP_R1_LE) + WebRtcIsacfix_AllpassFilter2FixDec16 = + WebRtcIsacfix_AllpassFilter2FixDec16MIPS; + WebRtcIsacfix_HighpassFilterFixDec32 = + WebRtcIsacfix_HighpassFilterFixDec32MIPS; +#endif +#if defined(MIPS_DSP_R2_LE) + WebRtcIsacfix_CalculateResidualEnergy = + WebRtcIsacfix_CalculateResidualEnergyMIPS; +#endif +} +#endif + +static void InitFunctionPointers(void) { + WebRtcIsacfix_AutocorrFix = WebRtcIsacfix_AutocorrC; + WebRtcIsacfix_FilterMaLoopFix = WebRtcIsacfix_FilterMaLoopC; + WebRtcIsacfix_CalculateResidualEnergy = + WebRtcIsacfix_CalculateResidualEnergyC; + WebRtcIsacfix_AllpassFilter2FixDec16 = WebRtcIsacfix_AllpassFilter2FixDec16C; + WebRtcIsacfix_HighpassFilterFixDec32 = WebRtcIsacfix_HighpassFilterFixDec32C; + WebRtcIsacfix_Time2Spec = WebRtcIsacfix_Time2SpecC; + WebRtcIsacfix_Spec2Time = WebRtcIsacfix_Spec2TimeC; + WebRtcIsacfix_MatrixProduct1 = WebRtcIsacfix_MatrixProduct1C; + WebRtcIsacfix_MatrixProduct2 = WebRtcIsacfix_MatrixProduct2C; + +#if defined(WEBRTC_HAS_NEON) + WebRtcIsacfix_InitNeon(); +#endif + +#if defined(MIPS32_LE) + WebRtcIsacfix_InitMIPS(); +#endif +} + +/**************************************************************************** + * WebRtcIsacfix_EncoderInit(...) + * + * This function initializes a ISAC instance prior to the encoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - CodingMode : 0 -> Bit rate and frame length are automatically + * adjusted to available bandwidth on + * transmission channel. + * 1 -> User sets a frame length and a target bit + * rate which is taken as the maximum short-term + * average bit rate. + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_EncoderInit(ISACFIX_MainStruct *ISAC_main_inst, + int16_t CodingMode) +{ + int k; + int16_t statusInit; + ISACFIX_SubStruct *ISAC_inst; + + statusInit = 0; + /* typecast pointer to rela structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* flag encoder init */ + ISAC_inst->initflag |= 2; + + if (CodingMode == 0) + /* Adaptive mode */ + ISAC_inst->ISACenc_obj.new_framelength = INITIAL_FRAMESAMPLES; + else if (CodingMode == 1) + /* Instantaneous mode */ + ISAC_inst->ISACenc_obj.new_framelength = 480; /* default for I-mode */ + else { + ISAC_inst->errorcode = ISAC_DISALLOWED_CODING_MODE; + statusInit = -1; + } + + ISAC_inst->CodingMode = CodingMode; + + WebRtcIsacfix_InitMaskingEnc(&ISAC_inst->ISACenc_obj.maskfiltstr_obj); + WebRtcIsacfix_InitPreFilterbank(&ISAC_inst->ISACenc_obj.prefiltbankstr_obj); + WebRtcIsacfix_InitPitchFilter(&ISAC_inst->ISACenc_obj.pitchfiltstr_obj); + WebRtcIsacfix_InitPitchAnalysis(&ISAC_inst->ISACenc_obj.pitchanalysisstr_obj); + + WebRtcIsacfix_InitRateModel(&ISAC_inst->ISACenc_obj.rate_data_obj); + + + ISAC_inst->ISACenc_obj.buffer_index = 0; + ISAC_inst->ISACenc_obj.frame_nb = 0; + ISAC_inst->ISACenc_obj.BottleNeck = 32000; /* default for I-mode */ + ISAC_inst->ISACenc_obj.MaxDelay = 10; /* default for I-mode */ + ISAC_inst->ISACenc_obj.current_framesamples = 0; + ISAC_inst->ISACenc_obj.s2nr = 0; + ISAC_inst->ISACenc_obj.MaxBits = 0; + ISAC_inst->ISACenc_obj.bitstr_seed = 4447; + ISAC_inst->ISACenc_obj.payloadLimitBytes30 = STREAM_MAXW16_30MS << 1; + ISAC_inst->ISACenc_obj.payloadLimitBytes60 = STREAM_MAXW16_60MS << 1; + ISAC_inst->ISACenc_obj.maxPayloadBytes = STREAM_MAXW16_60MS << 1; + ISAC_inst->ISACenc_obj.maxRateInBytes = STREAM_MAXW16_30MS << 1; + ISAC_inst->ISACenc_obj.enforceFrameSize = 0; + + /* Init the bistream data area to zero */ + for (k=0; kISACenc_obj.bitstr_obj.stream[k] = 0; + } + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + WebRtcIsacfix_InitPostFilterbank(&ISAC_inst->ISACenc_obj.interpolatorstr_obj); +#endif + + InitFunctionPointers(); + + return statusInit; +} + +/* Read the given number of bytes of big-endian 16-bit integers from |src| and + write them to |dest| in host endian. If |nbytes| is odd, the number of + output elements is rounded up, and the least significant byte of the last + element is set to 0. */ +static void read_be16(const uint8_t* src, size_t nbytes, uint16_t* dest) { + size_t i; + for (i = 0; i < nbytes / 2; ++i) + dest[i] = src[2 * i] << 8 | src[2 * i + 1]; + if (nbytes % 2 == 1) + dest[nbytes / 2] = src[nbytes - 1] << 8; +} + +/* Read the given number of bytes of host-endian 16-bit integers from |src| and + write them to |dest| in big endian. If |nbytes| is odd, the number of source + elements is rounded up (but only the most significant byte of the last + element is used), and the number of output bytes written will be + nbytes + 1. */ +static void write_be16(const uint16_t* src, size_t nbytes, uint8_t* dest) { + size_t i; + for (i = 0; i < nbytes / 2; ++i) { + dest[2 * i] = src[i] >> 8; + dest[2 * i + 1] = src[i]; + } + if (nbytes % 2 == 1) { + dest[nbytes - 1] = src[nbytes / 2] >> 8; + dest[nbytes] = 0; + } +} + +/**************************************************************************** + * WebRtcIsacfix_Encode(...) + * + * This function encodes 10ms frame(s) and inserts it into a package. + * Input speech length has to be 160 samples (10ms). The encoder buffers those + * 10ms frames until it reaches the chosen Framesize (480 or 960 samples + * corresponding to 30 or 60 ms frames), and then proceeds to the encoding. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - speechIn : input speech vector. + * + * Output: + * - encoded : the encoded data vector + * + * Return value: + * : >0 - Length (in bytes) of coded data + * : 0 - The buffer didn't reach the chosen framesize + * so it keeps buffering speech samples. + * : -1 - Error + */ + +int WebRtcIsacfix_Encode(ISACFIX_MainStruct *ISAC_main_inst, + const int16_t *speechIn, + uint8_t* encoded) +{ + ISACFIX_SubStruct *ISAC_inst; + int stream_len; + + /* typecast pointer to rela structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + + /* check if encoder initiated */ + if ((ISAC_inst->initflag & 2) != 2) { + ISAC_inst->errorcode = ISAC_ENCODER_NOT_INITIATED; + return (-1); + } + + stream_len = WebRtcIsacfix_EncodeImpl((int16_t*)speechIn, + &ISAC_inst->ISACenc_obj, + &ISAC_inst->bwestimator_obj, + ISAC_inst->CodingMode); + if (stream_len<0) { + ISAC_inst->errorcode = -(int16_t)stream_len; + return -1; + } + + write_be16(ISAC_inst->ISACenc_obj.bitstr_obj.stream, (size_t)stream_len, + encoded); + return stream_len; + +} + + + + +/**************************************************************************** + * WebRtcIsacfix_EncodeNb(...) + * + * This function encodes 10ms narrow band (8 kHz sampling) frame(s) and inserts + * it into a package. Input speech length has to be 80 samples (10ms). The encoder + * interpolates into wide-band (16 kHz sampling) buffers those + * 10ms frames until it reaches the chosen Framesize (480 or 960 wide-band samples + * corresponding to 30 or 60 ms frames), and then proceeds to the encoding. + * + * The function is enabled if WEBRTC_ISAC_FIX_NB_CALLS_ENABLED is defined + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - speechIn : input speech vector. + * + * Output: + * - encoded : the encoded data vector + * + * Return value: + * : >0 - Length (in bytes) of coded data + * : 0 - The buffer didn't reach the chosen framesize + * so it keeps buffering speech samples. + * : -1 - Error + */ +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED +int16_t WebRtcIsacfix_EncodeNb(ISACFIX_MainStruct *ISAC_main_inst, + const int16_t *speechIn, + int16_t *encoded) +{ + ISACFIX_SubStruct *ISAC_inst; + int16_t stream_len; + int16_t speechInWB[FRAMESAMPLES_10ms]; + int16_t Vector_Word16_1[FRAMESAMPLES_10ms/2]; + int16_t Vector_Word16_2[FRAMESAMPLES_10ms/2]; + + int k; + + + /* typecast pointer to rela structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + + /* check if encoder initiated */ + if ((ISAC_inst->initflag & 2) != 2) { + ISAC_inst->errorcode = ISAC_ENCODER_NOT_INITIATED; + return (-1); + } + + + /* Oversample to WB */ + + /* Form polyphase signals, and compensate for DC offset */ + for (k=0;kISACenc_obj.interpolatorstr_obj, FRAMESAMPLES_10ms); + + + /* Encode WB signal */ + stream_len = WebRtcIsacfix_EncodeImpl((int16_t*)speechInWB, + &ISAC_inst->ISACenc_obj, + &ISAC_inst->bwestimator_obj, + ISAC_inst->CodingMode); + if (stream_len<0) { + ISAC_inst->errorcode = - stream_len; + return -1; + } + + write_be16(ISAC_inst->ISACenc_obj.bitstr_obj.stream, + stream_len, + (uint8_t*)encoded); + return stream_len; +} +#endif /* WEBRTC_ISAC_FIX_NB_CALLS_ENABLED */ + + +/**************************************************************************** + * WebRtcIsacfix_GetNewBitStream(...) + * + * This function returns encoded data, with the recieved bwe-index in the + * stream. It should always return a complete packet, i.e. only called once + * even for 60 msec frames + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - bweIndex : index of bandwidth estimate to put in new bitstream + * + * Output: + * - encoded : the encoded data vector + * + * Return value: + * : >0 - Length (in bytes) of coded data + * : -1 - Error + */ + +int16_t WebRtcIsacfix_GetNewBitStream(ISACFIX_MainStruct *ISAC_main_inst, + int16_t bweIndex, + float scale, + uint8_t* encoded) +{ + ISACFIX_SubStruct *ISAC_inst; + int16_t stream_len; + + /* typecast pointer to rela structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + + /* check if encoder initiated */ + if ((ISAC_inst->initflag & 2) != 2) { + ISAC_inst->errorcode = ISAC_ENCODER_NOT_INITIATED; + return (-1); + } + + stream_len = WebRtcIsacfix_EncodeStoredData(&ISAC_inst->ISACenc_obj, + bweIndex, + scale); + if (stream_len<0) { + ISAC_inst->errorcode = - stream_len; + return -1; + } + + write_be16(ISAC_inst->ISACenc_obj.bitstr_obj.stream, stream_len, encoded); + return stream_len; +} + + + +/**************************************************************************** + * WebRtcIsacfix_DecoderInit(...) + * + * This function initializes a ISAC instance prior to the decoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + */ + +void WebRtcIsacfix_DecoderInit(ISACFIX_MainStruct *ISAC_main_inst) +{ + ISACFIX_SubStruct *ISAC_inst; + + InitFunctionPointers(); + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* flag decoder init */ + ISAC_inst->initflag |= 1; + + WebRtcIsacfix_InitMaskingDec(&ISAC_inst->ISACdec_obj.maskfiltstr_obj); + WebRtcIsacfix_InitPostFilterbank(&ISAC_inst->ISACdec_obj.postfiltbankstr_obj); + WebRtcIsacfix_InitPitchFilter(&ISAC_inst->ISACdec_obj.pitchfiltstr_obj); + + /* TS */ + WebRtcIsacfix_InitPlc( &ISAC_inst->ISACdec_obj.plcstr_obj ); + + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + WebRtcIsacfix_InitPreFilterbank(&ISAC_inst->ISACdec_obj.decimatorstr_obj); +#endif +} + + +/**************************************************************************** + * WebRtcIsacfix_UpdateBwEstimate1(...) + * + * This function updates the estimate of the bandwidth. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s). + * - packet_size : size of the packet. + * - rtp_seq_number : the RTP number of the packet. + * - arr_ts : the arrival time of the packet (from NetEq) + * in samples. + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_UpdateBwEstimate1(ISACFIX_MainStruct *ISAC_main_inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t arr_ts) +{ + ISACFIX_SubStruct *ISAC_inst; + Bitstr_dec streamdata; + int16_t err; + const size_t kRequiredEncodedLenBytes = 10; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* Sanity check of packet length */ + if (packet_size == 0) { + /* return error code if the packet length is null or less */ + ISAC_inst->errorcode = ISAC_EMPTY_PACKET; + return -1; + } else if (packet_size > (STREAM_MAXW16<<1)) { + /* return error code if length of stream is too long */ + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + return -1; + } + + /* check if decoder initiated */ + if ((ISAC_inst->initflag & 1) != 1) { + ISAC_inst->errorcode = ISAC_DECODER_NOT_INITIATED; + return (-1); + } + + InitializeDecoderBitstream(packet_size, &streamdata); + + read_be16(encoded, kRequiredEncodedLenBytes, streamdata.stream); + + err = WebRtcIsacfix_EstimateBandwidth(&ISAC_inst->bwestimator_obj, + &streamdata, + packet_size, + rtp_seq_number, + 0, + arr_ts); + + + if (err < 0) + { + /* return error code if something went wrong */ + ISAC_inst->errorcode = -err; + return -1; + } + + + return 0; +} + +/**************************************************************************** + * WebRtcIsacfix_UpdateBwEstimate(...) + * + * This function updates the estimate of the bandwidth. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s). + * - packet_size : size of the packet. + * - rtp_seq_number : the RTP number of the packet. + * - send_ts : Send Time Stamp from RTP header + * - arr_ts : the arrival time of the packet (from NetEq) + * in samples. + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_UpdateBwEstimate(ISACFIX_MainStruct *ISAC_main_inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts) +{ + ISACFIX_SubStruct *ISAC_inst; + Bitstr_dec streamdata; + int16_t err; + const size_t kRequiredEncodedLenBytes = 10; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* Sanity check of packet length */ + if (packet_size == 0) { + /* return error code if the packet length is null or less */ + ISAC_inst->errorcode = ISAC_EMPTY_PACKET; + return -1; + } else if (packet_size < kRequiredEncodedLenBytes) { + ISAC_inst->errorcode = ISAC_PACKET_TOO_SHORT; + return -1; + } else if (packet_size > (STREAM_MAXW16<<1)) { + /* return error code if length of stream is too long */ + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + return -1; + } + + /* check if decoder initiated */ + if ((ISAC_inst->initflag & 1) != 1) { + ISAC_inst->errorcode = ISAC_DECODER_NOT_INITIATED; + return (-1); + } + + InitializeDecoderBitstream(packet_size, &streamdata); + + read_be16(encoded, kRequiredEncodedLenBytes, streamdata.stream); + + err = WebRtcIsacfix_EstimateBandwidth(&ISAC_inst->bwestimator_obj, + &streamdata, + packet_size, + rtp_seq_number, + send_ts, + arr_ts); + + if (err < 0) + { + /* return error code if something went wrong */ + ISAC_inst->errorcode = -err; + return -1; + } + + + return 0; +} + +/**************************************************************************** + * WebRtcIsacfix_Decode(...) + * + * This function decodes a ISAC frame. Output speech length + * will be a multiple of 480 samples: 480 or 960 samples, + * depending on the framesize (30 or 60 ms). + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s) + * - len : bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded vector + * -1 - Error + */ + + +int WebRtcIsacfix_Decode(ISACFIX_MainStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + ISACFIX_SubStruct *ISAC_inst; + /* number of samples (480 or 960), output from decoder */ + /* that were actually used in the encoder/decoder (determined on the fly) */ + size_t number_of_samples; + int declen_int = 0; + size_t declen; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* check if decoder initiated */ + if ((ISAC_inst->initflag & 1) != 1) { + ISAC_inst->errorcode = ISAC_DECODER_NOT_INITIATED; + return (-1); + } + + /* Sanity check of packet length */ + if (len == 0) { + /* return error code if the packet length is null or less */ + ISAC_inst->errorcode = ISAC_EMPTY_PACKET; + return -1; + } else if (len > (STREAM_MAXW16<<1)) { + /* return error code if length of stream is too long */ + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + return -1; + } + + InitializeDecoderBitstream(len, &ISAC_inst->ISACdec_obj.bitstr_obj); + + read_be16(encoded, len, ISAC_inst->ISACdec_obj.bitstr_obj.stream); + + /* added for NetEq purposes (VAD/DTX related) */ + *speechType=1; + + declen_int = WebRtcIsacfix_DecodeImpl(decoded, &ISAC_inst->ISACdec_obj, + &number_of_samples); + if (declen_int < 0) { + /* Some error inside the decoder */ + ISAC_inst->errorcode = -(int16_t)declen_int; + memset(decoded, 0, sizeof(int16_t) * MAX_FRAMESAMPLES); + return -1; + } + declen = (size_t)declen_int; + + /* error check */ + + if (declen & 1) { + if (len != declen && + len != declen + + ((ISAC_inst->ISACdec_obj.bitstr_obj.stream[declen >> 1]) & 0xFF)) { + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + memset(decoded, 0, sizeof(int16_t) * number_of_samples); + return -1; + } + } else { + if (len != declen && + len != declen + + ((ISAC_inst->ISACdec_obj.bitstr_obj.stream[declen >> 1]) >> 8)) { + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + memset(decoded, 0, sizeof(int16_t) * number_of_samples); + return -1; + } + } + + return (int)number_of_samples; +} + + + + + +/**************************************************************************** + * WebRtcIsacfix_DecodeNb(...) + * + * This function decodes a ISAC frame in narrow-band (8 kHz sampling). + * Output speech length will be a multiple of 240 samples: 240 or 480 samples, + * depending on the framesize (30 or 60 ms). + * + * The function is enabled if WEBRTC_ISAC_FIX_NB_CALLS_ENABLED is defined + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s) + * - len : bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded vector + * -1 - Error + */ + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED +int WebRtcIsacfix_DecodeNb(ISACFIX_MainStruct* ISAC_main_inst, + const uint16_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + ISACFIX_SubStruct *ISAC_inst; + /* twice the number of samples (480 or 960), output from decoder */ + /* that were actually used in the encoder/decoder (determined on the fly) */ + size_t number_of_samples; + int declen_int = 0; + size_t declen; + int16_t dummy[FRAMESAMPLES/2]; + + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* check if decoder initiated */ + if ((ISAC_inst->initflag & 1) != 1) { + ISAC_inst->errorcode = ISAC_DECODER_NOT_INITIATED; + return (-1); + } + + if (len == 0) { + /* return error code if the packet length is null or less */ + ISAC_inst->errorcode = ISAC_EMPTY_PACKET; + return -1; + } else if (len > (STREAM_MAXW16<<1)) { + /* return error code if length of stream is too long */ + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + return -1; + } + + InitializeDecoderBitstream(len, &ISAC_inst->ISACdec_obj.bitstr_obj); + + read_be16(encoded, len, ISAC_inst->ISACdec_obj.bitstr_obj.stream); + + /* added for NetEq purposes (VAD/DTX related) */ + *speechType=1; + + declen_int = WebRtcIsacfix_DecodeImpl(decoded, &ISAC_inst->ISACdec_obj, + &number_of_samples); + if (declen_int < 0) { + /* Some error inside the decoder */ + ISAC_inst->errorcode = -(int16_t)declen_int; + memset(decoded, 0, sizeof(int16_t) * FRAMESAMPLES); + return -1; + } + declen = (size_t)declen_int; + + /* error check */ + + if (declen & 1) { + if (len != declen && + len != declen + + ((ISAC_inst->ISACdec_obj.bitstr_obj.stream[declen >> 1]) & 0xFF)) { + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + memset(decoded, 0, sizeof(int16_t) * number_of_samples); + return -1; + } + } else { + if (len != declen && + len != declen + + ((ISAC_inst->ISACdec_obj.bitstr_obj.stream[declen >>1]) >> 8)) { + ISAC_inst->errorcode = ISAC_LENGTH_MISMATCH; + memset(decoded, 0, sizeof(int16_t) * number_of_samples); + return -1; + } + } + + WebRtcIsacfix_SplitAndFilter2(decoded, decoded, dummy, &ISAC_inst->ISACdec_obj.decimatorstr_obj); + + if (number_of_samples>FRAMESAMPLES) { + WebRtcIsacfix_SplitAndFilter2(decoded + FRAMESAMPLES, decoded + FRAMESAMPLES/2, + dummy, &ISAC_inst->ISACdec_obj.decimatorstr_obj); + } + + return (int)(number_of_samples / 2); +} +#endif /* WEBRTC_ISAC_FIX_NB_CALLS_ENABLED */ + + +/**************************************************************************** + * WebRtcIsacfix_DecodePlcNb(...) + * + * This function conducts PLC for ISAC frame(s) in narrow-band (8kHz sampling). + * Output speech length will be "240*noOfLostFrames" samples + * that is equevalent of "30*noOfLostFrames" millisecond. + * + * The function is enabled if WEBRTC_ISAC_FIX_NB_CALLS_ENABLED is defined + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - noOfLostFrames : Number of PLC frames (240 sample=30ms) to produce + * + * Output: + * - decoded : The decoded vector + * + * Return value : Number of samples in decoded PLC vector + */ + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED +size_t WebRtcIsacfix_DecodePlcNb(ISACFIX_MainStruct* ISAC_main_inst, + int16_t* decoded, + size_t noOfLostFrames ) +{ + size_t no_of_samples, declen, k; + int16_t outframeNB[FRAMESAMPLES]; + int16_t outframeWB[FRAMESAMPLES]; + int16_t dummy[FRAMESAMPLES/2]; + + + ISACFIX_SubStruct *ISAC_inst; + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* Limit number of frames to two = 60 msec. Otherwise we exceed data vectors */ + if (noOfLostFrames > 2){ + noOfLostFrames = 2; + } + + k = 0; + declen = 0; + while( noOfLostFrames > 0 ) + { + WebRtcIsacfix_DecodePlcImpl(outframeWB, &ISAC_inst->ISACdec_obj, + &no_of_samples); + + WebRtcIsacfix_SplitAndFilter2(outframeWB, &(outframeNB[k*240]), dummy, &ISAC_inst->ISACdec_obj.decimatorstr_obj); + + declen += no_of_samples; + noOfLostFrames--; + k++; + } + + declen>>=1; + + for (k=0;k 2) { + noOfLostFrames = 2; + } + k = 0; + declen = 0; + while( noOfLostFrames > 0 ) + { + WebRtcIsacfix_DecodePlcImpl(&(outframe16[k*480]), &ISAC_inst->ISACdec_obj, + &no_of_samples); + declen += no_of_samples; + noOfLostFrames--; + k++; + } + + for (k=0;kCodingMode == 0) + { + /* in adaptive mode */ + ISAC_inst->errorcode = ISAC_MODE_MISMATCH; + return -1; + } + + + if (rate >= 10000 && rate <= 32000) + ISAC_inst->ISACenc_obj.BottleNeck = rate; + else { + ISAC_inst->errorcode = ISAC_DISALLOWED_BOTTLENECK; + return -1; + } + + + + if (framesize == 30 || framesize == 60) + ISAC_inst->ISACenc_obj.new_framelength = (int16_t)((FS/1000) * framesize); + else { + ISAC_inst->errorcode = ISAC_DISALLOWED_FRAME_LENGTH; + return -1; + } + + return 0; +} + +void WebRtcIsacfix_SetInitialBweBottleneck(ISACFIX_MainStruct* ISAC_main_inst, + int bottleneck_bits_per_second) { + ISACFIX_SubStruct* inst = (ISACFIX_SubStruct*)ISAC_main_inst; + RTC_DCHECK_GE(bottleneck_bits_per_second, 10000); + RTC_DCHECK_LE(bottleneck_bits_per_second, 32000); + inst->bwestimator_obj.sendBwAvg = ((uint32_t)bottleneck_bits_per_second) << 7; +} + +/**************************************************************************** + * WebRtcIsacfix_ControlBwe(...) + * + * This function sets the initial values of bottleneck and frame-size if + * iSAC is used in channel-adaptive mode. Through this API, users can + * enforce a frame-size for all values of bottleneck. Then iSAC will not + * automatically change the frame-size. + * + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - rateBPS : initial value of bottleneck in bits/second + * 10000 <= rateBPS <= 32000 is accepted + * For default bottleneck set rateBPS = 0 + * - frameSizeMs : number of milliseconds per frame (30 or 60) + * - enforceFrameSize : 1 to enforce the given frame-size through out + * the adaptation process, 0 to let iSAC change + * the frame-size if required. + * + * Return value : 0 - ok + * -1 - Error + */ + +int16_t WebRtcIsacfix_ControlBwe(ISACFIX_MainStruct *ISAC_main_inst, + int16_t rateBPS, + int frameSizeMs, + int16_t enforceFrameSize) +{ + ISACFIX_SubStruct *ISAC_inst; + /* Typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* check if encoder initiated */ + if ((ISAC_inst->initflag & 2) != 2) { + ISAC_inst->errorcode = ISAC_ENCODER_NOT_INITIATED; + return (-1); + } + + /* Check that we are in channel-adaptive mode, otherwise, return -1 */ + if (ISAC_inst->CodingMode != 0) { + ISAC_inst->errorcode = ISAC_MODE_MISMATCH; + return (-1); + } + + /* Set struct variable if enforceFrameSize is set. ISAC will then keep the */ + /* chosen frame size. */ + ISAC_inst->ISACenc_obj.enforceFrameSize = (enforceFrameSize != 0)? 1:0; + + /* Set initial rate, if value between 10000 and 32000, */ + /* if rateBPS is 0, keep the default initial bottleneck value (15000) */ + if ((rateBPS >= 10000) && (rateBPS <= 32000)) { + ISAC_inst->bwestimator_obj.sendBwAvg = (((uint32_t)rateBPS) << 7); + } else if (rateBPS != 0) { + ISAC_inst->errorcode = ISAC_DISALLOWED_BOTTLENECK; + return -1; + } + + /* Set initial framesize. If enforceFrameSize is set the frame size will not change */ + if ((frameSizeMs == 30) || (frameSizeMs == 60)) { + ISAC_inst->ISACenc_obj.new_framelength = (int16_t)((FS/1000) * frameSizeMs); + } else { + ISAC_inst->errorcode = ISAC_DISALLOWED_FRAME_LENGTH; + return -1; + } + + return 0; +} + + + + + +/**************************************************************************** + * WebRtcIsacfix_GetDownLinkBwIndex(...) + * + * This function returns index representing the Bandwidth estimate from + * other side to this side. + * + * Input: + * - ISAC_main_inst: iSAC struct + * + * Output: + * - rateIndex : Bandwidth estimate to transmit to other side. + * + */ + +int16_t WebRtcIsacfix_GetDownLinkBwIndex(ISACFIX_MainStruct* ISAC_main_inst, + int16_t* rateIndex) +{ + ISACFIX_SubStruct *ISAC_inst; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* Call function to get Bandwidth Estimate */ + *rateIndex = WebRtcIsacfix_GetDownlinkBwIndexImpl(&ISAC_inst->bwestimator_obj); + + return 0; +} + + +/**************************************************************************** + * WebRtcIsacfix_UpdateUplinkBw(...) + * + * This function takes an index representing the Bandwidth estimate from + * this side to other side and updates BWE. + * + * Input: + * - ISAC_main_inst: iSAC struct + * - rateIndex : Bandwidth estimate from other side. + * + */ + +int16_t WebRtcIsacfix_UpdateUplinkBw(ISACFIX_MainStruct* ISAC_main_inst, + int16_t rateIndex) +{ + int16_t err = 0; + ISACFIX_SubStruct *ISAC_inst; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + /* Call function to update BWE with received Bandwidth Estimate */ + err = WebRtcIsacfix_UpdateUplinkBwRec(&ISAC_inst->bwestimator_obj, rateIndex); + if (err < 0) { + ISAC_inst->errorcode = -err; + return (-1); + } + + return 0; +} + +/**************************************************************************** + * WebRtcIsacfix_ReadFrameLen(...) + * + * This function returns the length of the frame represented in the packet. + * + * Input: + * - encoded : Encoded bitstream + * + * Output: + * - frameLength : Length of frame in packet (in samples) + * + */ + +int16_t WebRtcIsacfix_ReadFrameLen(const uint8_t* encoded, + size_t encoded_len_bytes, + size_t* frameLength) +{ + Bitstr_dec streamdata; + int16_t err; + const size_t kRequiredEncodedLenBytes = 10; + + if (encoded_len_bytes < kRequiredEncodedLenBytes) { + return -1; + } + + InitializeDecoderBitstream(encoded_len_bytes, &streamdata); + + read_be16(encoded, kRequiredEncodedLenBytes, streamdata.stream); + + /* decode frame length */ + err = WebRtcIsacfix_DecodeFrameLen(&streamdata, frameLength); + if (err<0) // error check + return err; + + return 0; +} + + +/**************************************************************************** + * WebRtcIsacfix_ReadBwIndex(...) + * + * This function returns the index of the Bandwidth estimate from the bitstream. + * + * Input: + * - encoded : Encoded bitstream + * + * Output: + * - frameLength : Length of frame in packet (in samples) + * - rateIndex : Bandwidth estimate in bitstream + * + */ + +int16_t WebRtcIsacfix_ReadBwIndex(const uint8_t* encoded, + size_t encoded_len_bytes, + int16_t* rateIndex) +{ + Bitstr_dec streamdata; + int16_t err; + const size_t kRequiredEncodedLenBytes = 10; + + if (encoded_len_bytes < kRequiredEncodedLenBytes) { + return -1; + } + + InitializeDecoderBitstream(encoded_len_bytes, &streamdata); + + read_be16(encoded, kRequiredEncodedLenBytes, streamdata.stream); + + /* decode frame length, needed to get to the rateIndex in the bitstream */ + size_t frameLength; + err = WebRtcIsacfix_DecodeFrameLen(&streamdata, &frameLength); + if (err<0) // error check + return err; + + /* decode BW estimation */ + err = WebRtcIsacfix_DecodeSendBandwidth(&streamdata, rateIndex); + if (err<0) // error check + return err; + + return 0; +} + + + + +/**************************************************************************** + * WebRtcIsacfix_GetErrorCode(...) + * + * This function can be used to check the error code of an iSAC instance. When + * a function returns -1 a error code will be set for that instance. The + * function below extract the code of the last error that occured in the + * specified instance. + * + * Input: + * - ISAC_main_inst : ISAC instance + * + * Return value : Error code + */ + +int16_t WebRtcIsacfix_GetErrorCode(ISACFIX_MainStruct *ISAC_main_inst) +{ + ISACFIX_SubStruct *ISAC_inst; + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + return ISAC_inst->errorcode; +} + + + +/**************************************************************************** + * WebRtcIsacfix_GetUplinkBw(...) + * + * This function returns the inst quantized iSAC send bitrate + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Return value : bitrate + */ + +int32_t WebRtcIsacfix_GetUplinkBw(ISACFIX_MainStruct *ISAC_main_inst) +{ + ISACFIX_SubStruct *ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + BwEstimatorstr * bw = (BwEstimatorstr*)&(ISAC_inst->bwestimator_obj); + + return (int32_t) WebRtcIsacfix_GetUplinkBandwidth(bw); +} + +/**************************************************************************** + * WebRtcIsacfix_GetNewFrameLen(...) + * + * This function return the next frame length (in samples) of iSAC. + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Return value : frame lenght in samples + */ + +int16_t WebRtcIsacfix_GetNewFrameLen(ISACFIX_MainStruct *ISAC_main_inst) +{ + ISACFIX_SubStruct *ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + return ISAC_inst->ISACenc_obj.new_framelength; +} + + +/**************************************************************************** + * WebRtcIsacfix_SetMaxPayloadSize(...) + * + * This function sets a limit for the maximum payload size of iSAC. The same + * value is used both for 30 and 60 msec packets. + * The absolute max will be valid until next time the function is called. + * NOTE! This function may override the function WebRtcIsacfix_SetMaxRate() + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxPayloadBytes : maximum size of the payload in bytes + * valid values are between 100 and 400 bytes + * + * + * Return value : 0 if sucessful + * -1 if error happens + */ + +int16_t WebRtcIsacfix_SetMaxPayloadSize(ISACFIX_MainStruct *ISAC_main_inst, + int16_t maxPayloadBytes) +{ + ISACFIX_SubStruct *ISAC_inst; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + if((maxPayloadBytes < 100) || (maxPayloadBytes > 400)) + { + /* maxPayloadBytes is out of valid range */ + return -1; + } + else + { + /* Set new absolute max, which will not change unless this function + is called again with a new value */ + ISAC_inst->ISACenc_obj.maxPayloadBytes = maxPayloadBytes; + + /* Set new maximum values for 30 and 60 msec packets */ + if (maxPayloadBytes < ISAC_inst->ISACenc_obj.maxRateInBytes) { + ISAC_inst->ISACenc_obj.payloadLimitBytes30 = maxPayloadBytes; + } else { + ISAC_inst->ISACenc_obj.payloadLimitBytes30 = ISAC_inst->ISACenc_obj.maxRateInBytes; + } + + if ( maxPayloadBytes < (ISAC_inst->ISACenc_obj.maxRateInBytes << 1)) { + ISAC_inst->ISACenc_obj.payloadLimitBytes60 = maxPayloadBytes; + } else { + ISAC_inst->ISACenc_obj.payloadLimitBytes60 = (ISAC_inst->ISACenc_obj.maxRateInBytes << 1); + } + } + return 0; +} + + +/**************************************************************************** + * WebRtcIsacfix_SetMaxRate(...) + * + * This function sets the maximum rate which the codec may not exceed for a + * singel packet. The maximum rate is set in bits per second. + * The codec has an absolute maximum rate of 53400 bits per second (200 bytes + * per 30 msec). + * It is possible to set a maximum rate between 32000 and 53400 bits per second. + * + * The rate limit is valid until next time the function is called. + * + * NOTE! Packet size will never go above the value set if calling + * WebRtcIsacfix_SetMaxPayloadSize() (default max packet size is 400 bytes). + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxRateInBytes : maximum rate in bits per second, + * valid values are 32000 to 53400 bits + * + * Return value : 0 if sucessful + * -1 if error happens + */ + +int16_t WebRtcIsacfix_SetMaxRate(ISACFIX_MainStruct *ISAC_main_inst, + int32_t maxRate) +{ + ISACFIX_SubStruct *ISAC_inst; + int16_t maxRateInBytes; + + /* typecast pointer to real structure */ + ISAC_inst = (ISACFIX_SubStruct *)ISAC_main_inst; + + if((maxRate < 32000) || (maxRate > 53400)) + { + /* maxRate is out of valid range */ + return -1; + } + else + { + /* Calculate maximum number of bytes per 30 msec packets for the given + maximum rate. Multiply with 30/1000 to get number of bits per 30 msec, + divide by 8 to get number of bytes per 30 msec: + maxRateInBytes = floor((maxRate * 30/1000) / 8); */ + maxRateInBytes = (int16_t)( WebRtcSpl_DivW32W16ResW16(WEBRTC_SPL_MUL(maxRate, 3), 800) ); + + /* Store the value for usage in the WebRtcIsacfix_SetMaxPayloadSize-function */ + ISAC_inst->ISACenc_obj.maxRateInBytes = maxRateInBytes; + + /* For 30 msec packets: if the new limit is below the maximum + payload size, set a new limit */ + if (maxRateInBytes < ISAC_inst->ISACenc_obj.maxPayloadBytes) { + ISAC_inst->ISACenc_obj.payloadLimitBytes30 = maxRateInBytes; + } else { + ISAC_inst->ISACenc_obj.payloadLimitBytes30 = ISAC_inst->ISACenc_obj.maxPayloadBytes; + } + + /* For 60 msec packets: if the new limit (times 2) is below the + maximum payload size, set a new limit */ + if ( (maxRateInBytes << 1) < ISAC_inst->ISACenc_obj.maxPayloadBytes) { + ISAC_inst->ISACenc_obj.payloadLimitBytes60 = (maxRateInBytes << 1); + } else { + ISAC_inst->ISACenc_obj.payloadLimitBytes60 = ISAC_inst->ISACenc_obj.maxPayloadBytes; + } + } + + return 0; +} + + + +/**************************************************************************** + * WebRtcIsacfix_version(...) + * + * This function returns the version number. + * + * Output: + * - version : Pointer to character string + * + */ + +void WebRtcIsacfix_version(char *version) +{ + strcpy(version, "3.6.0"); +} + +void WebRtcIsacfix_GetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + IsacBandwidthInfo* bwinfo) { + ISACFIX_SubStruct* inst = (ISACFIX_SubStruct*)ISAC_main_inst; + RTC_DCHECK_NE(0, inst->initflag & 1); // Decoder initialized. + WebRtcIsacfixBw_GetBandwidthInfo(&inst->bwestimator_obj, bwinfo); +} + +void WebRtcIsacfix_SetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + const IsacBandwidthInfo* bwinfo) { + ISACFIX_SubStruct* inst = (ISACFIX_SubStruct*)ISAC_main_inst; + RTC_DCHECK_NE(0, inst->initflag & 2); // Encoder initialized. + WebRtcIsacfixBw_SetBandwidthInfo(&inst->bwestimator_obj, bwinfo); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice.c new file mode 100644 index 0000000000..81369991d5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice.c @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lattice.c + * + * Contains the normalized lattice filter routines (MA and AR) for iSAC codec + * + */ + +#include "codec.h" +#include "settings.h" +#include "rtc_base/sanitizer.h" + +#define LATTICE_MUL_32_32_RSFT16(a32a, a32b, b32) \ + ((int32_t)(WEBRTC_SPL_MUL(a32a, b32) + (WEBRTC_SPL_MUL_16_32_RSFT16(a32b, b32)))) +/* This macro is FORBIDDEN to use elsewhere than in a function in this file and + its corresponding neon version. It might give unpredictable results, since a + general int32_t*int32_t multiplication results in a 64 bit value. + The result is then shifted just 16 steps to the right, giving need for 48 + bits, i.e. in the generel case, it will NOT fit in a int32_t. In the + cases used in here, the int32_t will be enough, since (for a good + reason) the involved multiplicands aren't big enough to overflow a + int32_t after shifting right 16 bits. I have compared the result of a + multiplication between t32 and tmp32, done in two ways: + 1) Using (int32_t) (((float)(tmp32))*((float)(tmp32b))/65536.0); + 2) Using LATTICE_MUL_32_32_RSFT16(t16a, t16b, tmp32b); + By running 25 files, I haven't found any bigger diff than 64 - this was in the + case when method 1) gave 650235648 and 2) gave 650235712. +*/ + +/* Function prototype: filtering ar_g_Q0[] and ar_f_Q0[] through an AR filter + with coefficients cth_Q15[] and sth_Q15[]. + Implemented for both generic and ARMv7 platforms. + */ +void WebRtcIsacfix_FilterArLoop(int16_t* ar_g_Q0, + int16_t* ar_f_Q0, + int16_t* cth_Q15, + int16_t* sth_Q15, + size_t order_coef); + +/* Inner loop used for function WebRtcIsacfix_NormLatticeFilterMa(). It does: + for 0 <= n < HALF_SUBFRAMELEN - 1: + *ptr2 = input2 * (*ptr2) + input0 * (*ptr0)); + *ptr1 = input1 * (*ptr0) + input0 * (*ptr2); + Note, function WebRtcIsacfix_FilterMaLoopNeon and WebRtcIsacfix_FilterMaLoopC + are not bit-exact. The accuracy by the ARM Neon function is same or better. +*/ +void WebRtcIsacfix_FilterMaLoopC(int16_t input0, // Filter coefficient + int16_t input1, // Filter coefficient + int32_t input2, // Inverse coeff. (1/input1) + int32_t* ptr0, // Sample buffer + int32_t* ptr1, // Sample buffer + int32_t* ptr2) { // Sample buffer + int n = 0; + + // Separate the 32-bit variable input2 into two 16-bit integers (high 16 and + // low 16 bits), for using LATTICE_MUL_32_32_RSFT16 in the loop. + int16_t t16a = (int16_t)(input2 >> 16); + int16_t t16b = (int16_t)input2; + if (t16b < 0) t16a++; + + // The loop filtering the samples *ptr0, *ptr1, *ptr2 with filter coefficients + // input0, input1, and input2. + for(n = 0; n < HALF_SUBFRAMELEN - 1; n++, ptr0++, ptr1++, ptr2++) { + int32_t tmp32a = 0; + int32_t tmp32b = 0; + + // Calculate *ptr2 = input2 * (*ptr2 + input0 * (*ptr0)); + tmp32a = WEBRTC_SPL_MUL_16_32_RSFT15(input0, *ptr0); // Q15 * Q15 >> 15 = Q15 + tmp32b = *ptr2 + tmp32a; // Q15 + Q15 = Q15 + *ptr2 = LATTICE_MUL_32_32_RSFT16(t16a, t16b, tmp32b); + + // Calculate *ptr1 = input1 * (*ptr0) + input0 * (*ptr2); + tmp32a = WEBRTC_SPL_MUL_16_32_RSFT15(input1, *ptr0); // Q15*Q15>>15 = Q15 + tmp32b = WEBRTC_SPL_MUL_16_32_RSFT15(input0, *ptr2); // Q15*Q15>>15 = Q15 + *ptr1 = tmp32a + tmp32b; // Q15 + Q15 = Q15 + } +} + +/* filter the signal using normalized lattice filter */ +/* MA filter */ +void WebRtcIsacfix_NormLatticeFilterMa(size_t orderCoef, + int32_t *stateGQ15, + int16_t *lat_inQ0, + int16_t *filt_coefQ15, + int32_t *gain_lo_hiQ17, + int16_t lo_hi, + int16_t *lat_outQ9) +{ + int16_t sthQ15[MAX_AR_MODEL_ORDER]; + int16_t cthQ15[MAX_AR_MODEL_ORDER]; + + int u, n; + size_t i, k; + int16_t temp2,temp3; + size_t ord_1 = orderCoef+1; + int32_t inv_cthQ16[MAX_AR_MODEL_ORDER]; + + int32_t gain32, fQtmp; + int16_t gain16; + int16_t gain_sh; + + int32_t tmp32, tmp32b; + int32_t fQ15vec[HALF_SUBFRAMELEN]; + int32_t gQ15[MAX_AR_MODEL_ORDER+1][HALF_SUBFRAMELEN]; + int16_t sh; + int16_t t16a; + int16_t t16b; + + for (u=0;u>15 = Q(17+gain_sh) + inv_cthQ16[k] = WebRtcSpl_DivW32W16((int32_t)2147483647, cthQ15[k]); // 1/cth[k] in Q31/Q15 = Q16 + } + gain16 = (int16_t)(gain32 >> 16); // Q(1+gain_sh). + + /* normalized lattice filter */ + /*****************************/ + + /* initial conditions */ + for (i=0;i>15 = Q15 + tmp32b= fQtmp + tmp32; //Q15+Q15=Q15 + tmp32 = inv_cthQ16[i-1]; //Q16 + t16a = (int16_t)(tmp32 >> 16); + t16b = (int16_t)(tmp32 - (t16a << 16)); + if (t16b<0) t16a++; + tmp32 = LATTICE_MUL_32_32_RSFT16(t16a, t16b, tmp32b); + fQtmp = tmp32; // Q15 + + // Calculate g[i][0] = cth[i-1]*stateG[i-1] + sth[i-1]* f[i][0]; + tmp32 = WEBRTC_SPL_MUL_16_32_RSFT15(cthQ15[i-1], stateGQ15[i-1]); //Q15*Q15>>15 = Q15 + tmp32b = WEBRTC_SPL_MUL_16_32_RSFT15(sthQ15[i-1], fQtmp); //Q15*Q15>>15 = Q15 + tmp32 = tmp32 + tmp32b;//Q15+Q15 = Q15 + gQ15[i][0] = tmp32; // Q15 + } + + /* filtering */ + /* save the states */ + for(k=0;k>= gain_sh; // Q(17+gain_sh) -> Q17 + tmp32 = WEBRTC_SPL_MUL_16_32_RSFT16(gain16, fQ15vec[n]); //Q(1+gain_sh)*Q15>>16 = Q(gain_sh) + sh = 9-gain_sh; //number of needed shifts to reach Q9 + t16a = (int16_t) WEBRTC_SPL_SHIFT_W32(tmp32, sh); + lat_outQ9[n + temp1] = t16a; + } + + /* save the states */ + for (i=0;i>15 = Q27 + } + + sh = WebRtcSpl_NormW32(tmp32); // tmp32 is the gain + den16 = (int16_t) WEBRTC_SPL_SHIFT_W32(tmp32, sh-16); //Q(27+sh-16) = Q(sh+11) (all 16 bits are value bits) + inv_gain32 = WebRtcSpl_DivW32W16((int32_t)2147483647, den16); // 1/gain in Q31/Q(sh+11) = Q(20-sh) + + //initial conditions + inv_gain16 = (int16_t)(inv_gain32 >> 2); // 1/gain in Q(20-sh-2) = Q(18-sh) + + for (i=0;iQ26 + tmp32 = WEBRTC_SPL_MUL_16_32_RSFT16(inv_gain16, tmp32); //lat_in[]*inv_gain in (Q(18-sh)*Q26)>>16 = Q(28-sh) + tmp32 = WEBRTC_SPL_SHIFT_W32(tmp32, -(28-sh)); // lat_in[]*inv_gain in Q0 + + ARfQ0vec[i] = (int16_t)WebRtcSpl_SatW32ToW16(tmp32); // Q0 + } + + // Get the state of f & g for the first input, for all orders. + for (i = orderCoef; i > 0; i--) + { + tmp32 = (cthQ15[i - 1] * ARfQ0vec[0] - sthQ15[i - 1] * stateGQ0[i - 1] + + 16384) >> 15; + tmpAR = (int16_t)WebRtcSpl_SatW32ToW16(tmp32); // Q0 + + tmp32 = (sthQ15[i - 1] * ARfQ0vec[0] + cthQ15[i - 1] * stateGQ0[i - 1] + + 16384) >> 15; + ARgQ0vec[i] = (int16_t)WebRtcSpl_SatW32ToW16(tmp32); // Q0 + ARfQ0vec[0] = tmpAR; + } + ARgQ0vec[0] = ARfQ0vec[0]; + + // Filter ARgQ0vec[] and ARfQ0vec[] through coefficients cthQ15[] and sthQ15[]. + WebRtcIsacfix_FilterArLoop(ARgQ0vec, ARfQ0vec, cthQ15, sthQ15, orderCoef); + + for(n=0;n 0; k--) + + ldrh r7, [r3, #-2]! @ sth_Q15[k - 1] + ldrh r6, [r2, #-2]! @ cth_Q15[k - 1] + + ldrh r8, [r0, #-2] @ ar_g_Q0[k - 1] + smlabb r11, r7, r5, r12 @ sth_Q15[k - 1] * tmpAR + 16384 + smlabb r10, r6, r5, r12 @ cth_Q15[k - 1] * tmpAR + 16384 + smulbb r7, r7, r8 @ sth_Q15[k - 1] * ar_g_Q0[k - 1] + smlabb r11, r6, r8, r11 @ cth_Q15[k - 1] * ar_g_Q0[k - 1] + + @ (sth_Q15[k - 1] * tmpAR + 16384) + + sub r10, r10, r7 @ cth_Q15[k - 1] * tmpAR + 16384 - + @ (sth_Q15[k - 1] * ar_g_Q0[k - 1]) + ssat r11, #16, r11, asr #15 + ssat r5, #16, r10, asr #15 + strh r11, [r0], #-2 @ Output: ar_g_Q0[k] + + subs r9, #1 + bgt ORDER_COEF_LOOP + + strh r5, [r0] @ Output: ar_g_Q0[0] = tmpAR; + strh r5, [r1], #2 @ Output: ar_f_Q0[n+1] = tmpAR; + + subs r4, #1 + bne HALF_SUBFRAME_LOOP + + pop {r4-r11} + bx lr diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c new file mode 100644 index 0000000000..d2356af514 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * Contains the core loop function for the lattice filter AR routine + * for iSAC codec. + * + */ + +#include "settings.h" +#include "signal_processing_library.h" +#include "typedefs.h" // NOLINT(build/include) + +/* Filter ar_g_Q0[] and ar_f_Q0[] through an AR filter with coefficients + * cth_Q15[] and sth_Q15[]. + */ +void WebRtcIsacfix_FilterArLoop(int16_t* ar_g_Q0, // Input samples + int16_t* ar_f_Q0, // Input samples + int16_t* cth_Q15, // Filter coefficients + int16_t* sth_Q15, // Filter coefficients + size_t order_coef) { // order of the filter + int n = 0; + + for (n = 0; n < HALF_SUBFRAMELEN - 1; n++) { + size_t k = 0; + int16_t tmpAR = 0; + int32_t tmp32 = 0; + int32_t tmp32_2 = 0; + + tmpAR = ar_f_Q0[n + 1]; + for (k = order_coef; k > 0; k--) { + tmp32 = (cth_Q15[k - 1] * tmpAR - sth_Q15[k - 1] * ar_g_Q0[k - 1] + + 16384) >> 15; + tmp32_2 = (sth_Q15[k - 1] * tmpAR + cth_Q15[k - 1] * ar_g_Q0[k - 1] + + 16384) >> 15; + tmpAR = (int16_t)WebRtcSpl_SatW32ToW16(tmp32); + ar_g_Q0[k] = (int16_t)WebRtcSpl_SatW32ToW16(tmp32_2); + } + ar_f_Q0[n + 1] = tmpAR; + ar_g_Q0[0] = tmpAR; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_mips.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_mips.c new file mode 100644 index 0000000000..613c07c9bd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_mips.c @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" +#include "typedefs.h" // NOLINT(build/include) + +// Filter ar_g_Q0[] and ar_f_Q0[] through an AR filter with coefficients +// cth_Q15[] and sth_Q15[]. +void WebRtcIsacfix_FilterArLoop(int16_t* ar_g_Q0, // Input samples + int16_t* ar_f_Q0, // Input samples + int16_t* cth_Q15, // Filter coefficients + int16_t* sth_Q15, // Filter coefficients + size_t order_coef) { // order of the filter + int n = 0; + + for (n = 0; n < HALF_SUBFRAMELEN - 1; n++) { + int count = (int)(order_coef - 1); + int offset; +#if !defined(MIPS_DSP_R1_LE) + int16_t* tmp_cth; + int16_t* tmp_sth; + int16_t* tmp_arg; + int32_t max_q16 = 0x7fff; + int32_t min_q16 = 0xffff8000; +#endif + // Declare variables used as temporary registers. + int32_t r0, r1, r2, t0, t1, t2, t_ar; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "bltz %[count], 2f \n\t" + " lh %[t_ar], 0(%[tmp]) \n\t" + // Inner loop + "1: \n\t" + "sll %[offset], %[count], 1 \n\t" +#if defined(MIPS_DSP_R1_LE) + "lhx %[r0], %[offset](%[cth_Q15]) \n\t" + "lhx %[r1], %[offset](%[sth_Q15]) \n\t" + "lhx %[r2], %[offset](%[ar_g_Q0]) \n\t" +#else + "addu %[tmp_cth], %[cth_Q15], %[offset] \n\t" + "addu %[tmp_sth], %[sth_Q15], %[offset] \n\t" + "addu %[tmp_arg], %[ar_g_Q0], %[offset] \n\t" + "lh %[r0], 0(%[tmp_cth]) \n\t" + "lh %[r1], 0(%[tmp_sth]) \n\t" + "lh %[r2], 0(%[tmp_arg]) \n\t" +#endif + "mul %[t0], %[r0], %[t_ar] \n\t" + "mul %[t1], %[r1], %[t_ar] \n\t" + "mul %[t2], %[r1], %[r2] \n\t" + "mul %[r0], %[r0], %[r2] \n\t" + "subu %[t0], %[t0], %[t2] \n\t" + "addu %[t1], %[t1], %[r0] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[t1], %[t1], 15 \n\t" + "shra_r.w %[t0], %[t0], 15 \n\t" +#else + "addiu %[t1], %[t1], 0x4000 \n\t" + "sra %[t1], %[t1], 15 \n\t" + "addiu %[t0], %[t0], 0x4000 \n\t" + "sra %[t0], %[t0], 15 \n\t" +#endif + "addiu %[offset], %[offset], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shll_s.w %[t1], %[t1], 16 \n\t" + "shll_s.w %[t_ar], %[t0], 16 \n\t" +#else + "slt %[r0], %[t1], %[max_q16] \n\t" + "slt %[r1], %[t0], %[max_q16] \n\t" + "movz %[t1], %[max_q16], %[r0] \n\t" + "movz %[t0], %[max_q16], %[r1] \n\t" +#endif + "addu %[offset], %[offset], %[ar_g_Q0] \n\t" +#if defined(MIPS_DSP_R1_LE) + "sra %[t1], %[t1], 16 \n\t" + "sra %[t_ar], %[t_ar], 16 \n\t" +#else + "slt %[r0], %[t1], %[min_q16] \n\t" + "slt %[r1], %[t0], %[min_q16] \n\t" + "movn %[t1], %[min_q16], %[r0] \n\t" + "movn %[t0], %[min_q16], %[r1] \n\t" + "addu %[t_ar], $zero, %[t0] \n\t" +#endif + "sh %[t1], 0(%[offset]) \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[count], %[count], -1 \n\t" + "2: \n\t" + "sh %[t_ar], 0(%[tmp]) \n\t" + "sh %[t_ar], 0(%[ar_g_Q0]) \n\t" + ".set pop \n\t" + : [t_ar] "=&r" (t_ar), [count] "+r" (count), [offset] "=&r" (offset), + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), [t0] "=&r" (t0), +#if !defined(MIPS_DSP_R1_LE) + [tmp_cth] "=&r" (tmp_cth), [tmp_sth] "=&r" (tmp_sth), + [tmp_arg] "=&r" (tmp_arg), +#endif + [t1] "=&r" (t1), [t2] "=&r" (t2) + : [tmp] "r" (&ar_f_Q0[n+1]), [cth_Q15] "r" (cth_Q15), +#if !defined(MIPS_DSP_R1_LE) + [max_q16] "r" (max_q16), [min_q16] "r" (min_q16), +#endif + [sth_Q15] "r" (sth_Q15), [ar_g_Q0] "r" (ar_g_Q0) + : "memory", "hi", "lo" + ); + } +} + +// MIPS optimization of the inner loop used for function +// WebRtcIsacfix_NormLatticeFilterMa(). It does: +// +// for 0 <= n < HALF_SUBFRAMELEN - 1: +// *ptr2 = input2 * (*ptr2) + input0 * (*ptr0)); +// *ptr1 = input1 * (*ptr0) + input0 * (*ptr2); +// +// Note, function WebRtcIsacfix_FilterMaLoopMIPS and WebRtcIsacfix_FilterMaLoopC +// are not bit-exact. The accuracy of the MIPS function is same or better. +void WebRtcIsacfix_FilterMaLoopMIPS(int16_t input0, // Filter coefficient + int16_t input1, // Filter coefficient + int32_t input2, // Inverse coeff (1/input1) + int32_t* ptr0, // Sample buffer + int32_t* ptr1, // Sample buffer + int32_t* ptr2) { // Sample buffer +#if defined(MIPS_DSP_R2_LE) + // MIPS DSPR2 version. 4 available accumulators allows loop unrolling 4 times. + // This variant is not bit-exact with WebRtcIsacfix_FilterMaLoopC, since we + // are exploiting 64-bit accumulators. The accuracy of the MIPS DSPR2 function + // is same or better. + int n = (HALF_SUBFRAMELEN - 1) >> 2; + int m = (HALF_SUBFRAMELEN - 1) & 3; + + int r0, r1, r2, r3; + int t0, t1, t2, t3; + int s0, s1, s2, s3; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "lw %[r0], 0(%[ptr0]) \n\t" + "lw %[r1], 4(%[ptr0]) \n\t" + "lw %[r2], 8(%[ptr0]) \n\t" + "lw %[r3], 12(%[ptr0]) \n\t" + "mult $ac0, %[r0], %[input0] \n\t" + "mult $ac1, %[r1], %[input0] \n\t" + "mult $ac2, %[r2], %[input0] \n\t" + "mult $ac3, %[r3], %[input0] \n\t" + "lw %[t0], 0(%[ptr2]) \n\t" + "extr_rs.w %[s0], $ac0, 15 \n\t" + "extr_rs.w %[s1], $ac1, 15 \n\t" + "extr_rs.w %[s2], $ac2, 15 \n\t" + "extr_rs.w %[s3], $ac3, 15 \n\t" + "lw %[t1], 4(%[ptr2]) \n\t" + "lw %[t2], 8(%[ptr2]) \n\t" + "lw %[t3], 12(%[ptr2]) \n\t" + "addu %[t0], %[t0], %[s0] \n\t" + "addu %[t1], %[t1], %[s1] \n\t" + "addu %[t2], %[t2], %[s2] \n\t" + "addu %[t3], %[t3], %[s3] \n\t" + "mult $ac0, %[t0], %[input2] \n\t" + "mult $ac1, %[t1], %[input2] \n\t" + "mult $ac2, %[t2], %[input2] \n\t" + "mult $ac3, %[t3], %[input2] \n\t" + "addiu %[ptr0], %[ptr0], 16 \n\t" + "extr_rs.w %[t0], $ac0, 16 \n\t" + "extr_rs.w %[t1], $ac1, 16 \n\t" + "extr_rs.w %[t2], $ac2, 16 \n\t" + "extr_rs.w %[t3], $ac3, 16 \n\t" + "addiu %[n], %[n], -1 \n\t" + "mult $ac0, %[r0], %[input1] \n\t" + "mult $ac1, %[r1], %[input1] \n\t" + "mult $ac2, %[r2], %[input1] \n\t" + "mult $ac3, %[r3], %[input1] \n\t" + "sw %[t0], 0(%[ptr2]) \n\t" + "extr_rs.w %[s0], $ac0, 15 \n\t" + "extr_rs.w %[s1], $ac1, 15 \n\t" + "extr_rs.w %[s2], $ac2, 15 \n\t" + "extr_rs.w %[s3], $ac3, 15 \n\t" + "sw %[t1], 4(%[ptr2]) \n\t" + "sw %[t2], 8(%[ptr2]) \n\t" + "sw %[t3], 12(%[ptr2]) \n\t" + "mult $ac0, %[t0], %[input0] \n\t" + "mult $ac1, %[t1], %[input0] \n\t" + "mult $ac2, %[t2], %[input0] \n\t" + "mult $ac3, %[t3], %[input0] \n\t" + "addiu %[ptr2], %[ptr2], 16 \n\t" + "extr_rs.w %[t0], $ac0, 15 \n\t" + "extr_rs.w %[t1], $ac1, 15 \n\t" + "extr_rs.w %[t2], $ac2, 15 \n\t" + "extr_rs.w %[t3], $ac3, 15 \n\t" + "addu %[t0], %[t0], %[s0] \n\t" + "addu %[t1], %[t1], %[s1] \n\t" + "addu %[t2], %[t2], %[s2] \n\t" + "addu %[t3], %[t3], %[s3] \n\t" + "sw %[t0], 0(%[ptr1]) \n\t" + "sw %[t1], 4(%[ptr1]) \n\t" + "sw %[t2], 8(%[ptr1]) \n\t" + "sw %[t3], 12(%[ptr1]) \n\t" + "bgtz %[n], 1b \n\t" + " addiu %[ptr1], %[ptr1], 16 \n\t" + "beq %[m], %0, 3f \n\t" + " nop \n\t" + "2: \n\t" + "lw %[r0], 0(%[ptr0]) \n\t" + "lw %[t0], 0(%[ptr2]) \n\t" + "addiu %[ptr0], %[ptr0], 4 \n\t" + "mult $ac0, %[r0], %[input0] \n\t" + "mult $ac1, %[r0], %[input1] \n\t" + "extr_rs.w %[r1], $ac0, 15 \n\t" + "extr_rs.w %[t1], $ac1, 15 \n\t" + "addu %[t0], %[t0], %[r1] \n\t" + "mult $ac0, %[t0], %[input2] \n\t" + "extr_rs.w %[t0], $ac0, 16 \n\t" + "sw %[t0], 0(%[ptr2]) \n\t" + "mult $ac0, %[t0], %[input0] \n\t" + "addiu %[ptr2], %[ptr2], 4 \n\t" + "addiu %[m], %[m], -1 \n\t" + "extr_rs.w %[t0], $ac0, 15 \n\t" + "addu %[t0], %[t0], %[t1] \n\t" + "sw %[t0], 0(%[ptr1]) \n\t" + "bgtz %[m], 2b \n\t" + " addiu %[ptr1], %[ptr1], 4 \n\t" + "3: \n\t" + ".set pop \n\t" + : [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), + [r3] "=&r" (r3), [t0] "=&r" (t0), [t1] "=&r" (t1), + [t2] "=&r" (t2), [t3] "=&r" (t3), [s0] "=&r" (s0), + [s1] "=&r" (s1), [s2] "=&r" (s2), [s3] "=&r" (s3), + [ptr0] "+r" (ptr0), [ptr1] "+r" (ptr1), [m] "+r" (m), + [ptr2] "+r" (ptr2), [n] "+r" (n) + : [input0] "r" (input0), [input1] "r" (input1), + [input2] "r" (input2) + : "memory", "hi", "lo", "$ac1hi", "$ac1lo", "$ac2hi", + "$ac2lo", "$ac3hi", "$ac3lo" + ); +#else + // Non-DSPR2 version of the function. Avoiding the accumulator usage due to + // large latencies. This variant is bit-exact with C code. + int n = HALF_SUBFRAMELEN - 1; + int32_t t16a, t16b; + int32_t r0, r1, r2, r3, r4; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "sra %[t16a], %[input2], 16 \n\t" + "andi %[t16b], %[input2], 0xFFFF \n\t" +#if defined(MIPS32R2_LE) + "seh %[t16b], %[t16b] \n\t" + "seh %[input0], %[input0] \n\t" + "seh %[input1], %[input1] \n\t" +#else + "sll %[t16b], %[t16b], 16 \n\t" + "sra %[t16b], %[t16b], 16 \n\t" + "sll %[input0], %[input0], 16 \n\t" + "sra %[input0], %[input0], 16 \n\t" + "sll %[input1], %[input1], 16 \n\t" + "sra %[input1], %[input1], 16 \n\t" +#endif + "addiu %[r0], %[t16a], 1 \n\t" + "slt %[r1], %[t16b], $zero \n\t" + "movn %[t16a], %[r0], %[r1] \n\t" + "1: \n\t" + "lw %[r0], 0(%[ptr0]) \n\t" + "lw %[r1], 0(%[ptr2]) \n\t" + "addiu %[ptr0], %[ptr0], 4 \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "mul %[r3], %[r2], %[input0] \n\t" + "mul %[r4], %[r0], %[input0] \n\t" + "mul %[r2], %[r2], %[input1] \n\t" + "mul %[r0], %[r0], %[input1] \n\t" + "addiu %[ptr2], %[ptr2], 4 \n\t" + "sll %[r3], %[r3], 1 \n\t" + "sra %[r4], %[r4], 1 \n\t" + "addiu %[r4], %[r4], 0x2000 \n\t" + "sra %[r4], %[r4], 14 \n\t" + "addu %[r3], %[r3], %[r4] \n\t" + "addu %[r1], %[r1], %[r3] \n\t" + "sra %[r3], %[r1], 16 \n\t" + "andi %[r4], %[r1], 0xFFFF \n\t" + "sra %[r4], %[r4], 1 \n\t" + "mul %[r1], %[r1], %[t16a] \n\t" + "mul %[r3], %[r3], %[t16b] \n\t" + "mul %[r4], %[r4], %[t16b] \n\t" + "sll %[r2], %[r2], 1 \n\t" + "sra %[r0], %[r0], 1 \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "sra %[r0], %[r0], 14 \n\t" + "addu %[r0], %[r0], %[r2] \n\t" + "addiu %[n], %[n], -1 \n\t" + "addu %[r1], %[r1], %[r3] \n\t" + "addiu %[r4], %[r4], 0x4000 \n\t" + "sra %[r4], %[r4], 15 \n\t" + "addu %[r1], %[r1], %[r4] \n\t" + "sra %[r2], %[r1], 16 \n\t" + "andi %[r3], %[r1], 0xFFFF \n\t" + "mul %[r3], %[r3], %[input0] \n\t" + "mul %[r2], %[r2], %[input0] \n\t" + "sw %[r1], -4(%[ptr2]) \n\t" + "sra %[r3], %[r3], 1 \n\t" + "addiu %[r3], %[r3], 0x2000 \n\t" + "sra %[r3], %[r3], 14 \n\t" + "addu %[r0], %[r0], %[r3] \n\t" + "sll %[r2], %[r2], 1 \n\t" + "addu %[r0], %[r0], %[r2] \n\t" + "sw %[r0], 0(%[ptr1]) \n\t" + "bgtz %[n], 1b \n\t" + " addiu %[ptr1], %[ptr1], 4 \n\t" + ".set pop \n\t" + : [t16a] "=&r" (t16a), [t16b] "=&r" (t16b), [r0] "=&r" (r0), + [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), + [r4] "=&r" (r4), [ptr0] "+r" (ptr0), [ptr1] "+r" (ptr1), + [ptr2] "+r" (ptr2), [n] "+r" (n) + : [input0] "r" (input0), [input1] "r" (input1), + [input2] "r" (input2) + : "hi", "lo", "memory" + ); +#endif +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_neon.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_neon.c new file mode 100644 index 0000000000..8ea9b63578 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_neon.c @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" + +// Contains a function for the core loop in the normalized lattice MA +// filter routine for iSAC codec, optimized for ARM Neon platform. +// It does: +// for 0 <= n < HALF_SUBFRAMELEN - 1: +// *ptr2 = input2 * ((*ptr2) + input0 * (*ptr0)); +// *ptr1 = input1 * (*ptr0) + input0 * (*ptr2); +// Output is not bit-exact with the reference C code, due to the replacement +// of WEBRTC_SPL_MUL_16_32_RSFT15 and LATTICE_MUL_32_32_RSFT16 with Neon +// instructions. The difference should not be bigger than 1. +void WebRtcIsacfix_FilterMaLoopNeon(int16_t input0, // Filter coefficient + int16_t input1, // Filter coefficient + int32_t input2, // Inverse coefficient + int32_t* ptr0, // Sample buffer + int32_t* ptr1, // Sample buffer + int32_t* ptr2) // Sample buffer +{ + int n = 0; + int loop = (HALF_SUBFRAMELEN - 1) >> 3; + int loop_tail = (HALF_SUBFRAMELEN - 1) & 0x7; + + int32x4_t input0_v = vdupq_n_s32((int32_t)input0 << 16); + int32x4_t input1_v = vdupq_n_s32((int32_t)input1 << 16); + int32x4_t input2_v = vdupq_n_s32(input2); + int32x4_t tmp0a, tmp1a, tmp2a, tmp3a; + int32x4_t tmp0b, tmp1b, tmp2b, tmp3b; + int32x4_t ptr0va, ptr1va, ptr2va; + int32x4_t ptr0vb, ptr1vb, ptr2vb; + + int64x2_t tmp2al_low, tmp2al_high, tmp2bl_low, tmp2bl_high; + // Unroll to process 8 samples at once. + for (n = 0; n < loop; n++) { + ptr0va = vld1q_s32(ptr0); + ptr0vb = vld1q_s32(ptr0 + 4); + ptr0 += 8; + + ptr2va = vld1q_s32(ptr2); + ptr2vb = vld1q_s32(ptr2 + 4); + + // Calculate tmp0 = (*ptr0) * input0. + tmp0a = vqrdmulhq_s32(ptr0va, input0_v); + tmp0b = vqrdmulhq_s32(ptr0vb, input0_v); + + // Calculate tmp1 = (*ptr0) * input1. + tmp1a = vqrdmulhq_s32(ptr0va, input1_v); + tmp1b = vqrdmulhq_s32(ptr0vb, input1_v); + + // Calculate tmp2 = tmp0 + *(ptr2). + tmp2a = vaddq_s32(tmp0a, ptr2va); + tmp2b = vaddq_s32(tmp0b, ptr2vb); + + // Calculate *ptr2 = input2 * tmp2. + tmp2al_low = vmull_s32(vget_low_s32(tmp2a), vget_low_s32(input2_v)); +#if defined(WEBRTC_ARCH_ARM64) + tmp2al_high = vmull_high_s32(tmp2a, input2_v); +#else + tmp2al_high = vmull_s32(vget_high_s32(tmp2a), vget_high_s32(input2_v)); +#endif + ptr2va = vcombine_s32(vrshrn_n_s64(tmp2al_low, 16), + vrshrn_n_s64(tmp2al_high, 16)); + + tmp2bl_low = vmull_s32(vget_low_s32(tmp2b), vget_low_s32(input2_v)); +#if defined(WEBRTC_ARCH_ARM64) + tmp2bl_high = vmull_high_s32(tmp2b, input2_v); +#else + tmp2bl_high = vmull_s32(vget_high_s32(tmp2b), vget_high_s32(input2_v)); +#endif + ptr2vb = vcombine_s32(vrshrn_n_s64(tmp2bl_low, 16), + vrshrn_n_s64(tmp2bl_high, 16)); + + vst1q_s32(ptr2, ptr2va); + vst1q_s32(ptr2 + 4, ptr2vb); + ptr2 += 8; + + // Calculate tmp3 = ptr2v * input0. + tmp3a = vqrdmulhq_s32(ptr2va, input0_v); + tmp3b = vqrdmulhq_s32(ptr2vb, input0_v); + + // Calculate *ptr1 = tmp1 + tmp3. + ptr1va = vaddq_s32(tmp1a, tmp3a); + ptr1vb = vaddq_s32(tmp1b, tmp3b); + + vst1q_s32(ptr1, ptr1va); + vst1q_s32(ptr1 + 4, ptr1vb); + ptr1 += 8; + } + + // Process four more samples. + if (loop_tail & 0x4) { + ptr0va = vld1q_s32(ptr0); + ptr2va = vld1q_s32(ptr2); + ptr0 += 4; + + // Calculate tmp0 = (*ptr0) * input0. + tmp0a = vqrdmulhq_s32(ptr0va, input0_v); + + // Calculate tmp1 = (*ptr0) * input1. + tmp1a = vqrdmulhq_s32(ptr0va, input1_v); + + // Calculate tmp2 = tmp0 + *(ptr2). + tmp2a = vaddq_s32(tmp0a, ptr2va); + + // Calculate *ptr2 = input2 * tmp2. + tmp2al_low = vmull_s32(vget_low_s32(tmp2a), vget_low_s32(input2_v)); + +#if defined(WEBRTC_ARCH_ARM64) + tmp2al_high = vmull_high_s32(tmp2a, input2_v); +#else + tmp2al_high = vmull_s32(vget_high_s32(tmp2a), vget_high_s32(input2_v)); +#endif + ptr2va = vcombine_s32(vrshrn_n_s64(tmp2al_low, 16), + vrshrn_n_s64(tmp2al_high, 16)); + + vst1q_s32(ptr2, ptr2va); + ptr2 += 4; + + // Calculate tmp3 = *(ptr2) * input0. + tmp3a = vqrdmulhq_s32(ptr2va, input0_v); + + // Calculate *ptr1 = tmp1 + tmp3. + ptr1va = vaddq_s32(tmp1a, tmp3a); + + vst1q_s32(ptr1, ptr1va); + ptr1 += 4; + } + + // Process two more samples. + if (loop_tail & 0x2) { + int32x2_t ptr0v_tail, ptr2v_tail, ptr1v_tail; + int32x2_t tmp0_tail, tmp1_tail, tmp2_tail, tmp3_tail; + int64x2_t tmp2l_tail; + ptr0v_tail = vld1_s32(ptr0); + ptr2v_tail = vld1_s32(ptr2); + ptr0 += 2; + + // Calculate tmp0 = (*ptr0) * input0. + tmp0_tail = vqrdmulh_s32(ptr0v_tail, vget_low_s32(input0_v)); + + // Calculate tmp1 = (*ptr0) * input1. + tmp1_tail = vqrdmulh_s32(ptr0v_tail, vget_low_s32(input1_v)); + + // Calculate tmp2 = tmp0 + *(ptr2). + tmp2_tail = vadd_s32(tmp0_tail, ptr2v_tail); + + // Calculate *ptr2 = input2 * tmp2. + tmp2l_tail = vmull_s32(tmp2_tail, vget_low_s32(input2_v)); + ptr2v_tail = vrshrn_n_s64(tmp2l_tail, 16); + + vst1_s32(ptr2, ptr2v_tail); + ptr2 += 2; + + // Calculate tmp3 = *(ptr2) * input0. + tmp3_tail = vqrdmulh_s32(ptr2v_tail, vget_low_s32(input0_v)); + + // Calculate *ptr1 = tmp1 + tmp3. + ptr1v_tail = vadd_s32(tmp1_tail, tmp3_tail); + + vst1_s32(ptr1, ptr1v_tail); + ptr1 += 2; + } + + // Process one more sample. + if (loop_tail & 0x1) { + int16_t t16a = (int16_t)(input2 >> 16); + int16_t t16b = (int16_t)input2; + if (t16b < 0) t16a++; + int32_t tmp32a; + int32_t tmp32b; + + // Calculate *ptr2 = input2 * (*ptr2 + input0 * (*ptr0)). + tmp32a = WEBRTC_SPL_MUL_16_32_RSFT15(input0, *ptr0); + tmp32b = *ptr2 + tmp32a; + *ptr2 = (int32_t)(WEBRTC_SPL_MUL(t16a, tmp32b) + + (WEBRTC_SPL_MUL_16_32_RSFT16(t16b, tmp32b))); + + // Calculate *ptr1 = input1 * (*ptr0) + input0 * (*ptr2). + tmp32a = WEBRTC_SPL_MUL_16_32_RSFT15(input1, *ptr0); + tmp32b = WEBRTC_SPL_MUL_16_32_RSFT15(input0, *ptr2); + *ptr1 = tmp32a + tmp32b; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.c new file mode 100644 index 0000000000..97e4ce03a4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.c @@ -0,0 +1,948 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_masking_model.c + * + * LPC analysis and filtering functions + * + */ + +#include "lpc_masking_model.h" + +#include /* For LLONG_MAX and LLONG_MIN. */ +#include "codec.h" +#include "entropy_coding.h" +#include "settings.h" + +/* The conversion is implemented by the step-down algorithm */ +void WebRtcSpl_AToK_JSK( + int16_t *a16, /* Q11 */ + int16_t useOrder, + int16_t *k16 /* Q15 */ + ) +{ + int m, k; + int32_t tmp32[MAX_AR_MODEL_ORDER]; + int32_t tmp32b; + int32_t tmp_inv_denum32; + int16_t tmp_inv_denum16; + + k16[useOrder-1] = a16[useOrder] << 4; // Q11<<4 => Q15 + + for (m=useOrder-1; m>0; m--) { + // (1 - k^2) in Q30 + tmp_inv_denum32 = 1073741823 - k16[m] * k16[m]; + tmp_inv_denum16 = (int16_t)(tmp_inv_denum32 >> 15); // (1 - k^2) in Q15. + + for (k=1; k<=m; k++) { + tmp32b = (a16[k] << 16) - ((k16[m] * a16[m - k + 1]) << 1); + + tmp32[k] = WebRtcSpl_DivW32W16(tmp32b, tmp_inv_denum16); //Q27/Q15 = Q12 + } + + for (k=1; k> 1); // Q12>>1 => Q11 + } + + tmp32[m] = WEBRTC_SPL_SAT(4092, tmp32[m], -4092); + k16[m - 1] = (int16_t)(tmp32[m] << 3); // Q12<<3 => Q15 + } + + return; +} + + + + + +int16_t WebRtcSpl_LevinsonW32_JSK( + int32_t *R, /* (i) Autocorrelation of length >= order+1 */ + int16_t *A, /* (o) A[0..order] LPC coefficients (Q11) */ + int16_t *K, /* (o) K[0...order-1] Reflection coefficients (Q15) */ + int16_t order /* (i) filter order */ + ) { + int16_t i, j; + int16_t R_hi[LEVINSON_MAX_ORDER+1], R_low[LEVINSON_MAX_ORDER+1]; + /* Aurocorr coefficients in high precision */ + int16_t A_hi[LEVINSON_MAX_ORDER+1], A_low[LEVINSON_MAX_ORDER+1]; + /* LPC coefficients in high precicion */ + int16_t A_upd_hi[LEVINSON_MAX_ORDER+1], A_upd_low[LEVINSON_MAX_ORDER+1]; + /* LPC coefficients for next iteration */ + int16_t K_hi, K_low; /* reflection coefficient in high precision */ + int16_t Alpha_hi, Alpha_low, Alpha_exp; /* Prediction gain Alpha in high precision + and with scale factor */ + int16_t tmp_hi, tmp_low; + int32_t temp1W32, temp2W32, temp3W32; + int16_t norm; + + /* Normalize the autocorrelation R[0]...R[order+1] */ + + norm = WebRtcSpl_NormW32(R[0]); + + for (i=order;i>=0;i--) { + temp1W32 = R[i] << norm; + /* Put R in hi and low format */ + R_hi[i] = (int16_t)(temp1W32 >> 16); + R_low[i] = (int16_t)((temp1W32 - ((int32_t)R_hi[i] << 16)) >> 1); + } + + /* K = A[1] = -R[1] / R[0] */ + + temp2W32 = (R_hi[1] << 16) + (R_low[1] << 1); /* R[1] in Q31 */ + temp3W32 = WEBRTC_SPL_ABS_W32(temp2W32); /* abs R[1] */ + temp1W32 = WebRtcSpl_DivW32HiLow(temp3W32, R_hi[0], R_low[0]); /* abs(R[1])/R[0] in Q31 */ + /* Put back the sign on R[1] */ + if (temp2W32 > 0) { + temp1W32 = -temp1W32; + } + + /* Put K in hi and low format */ + K_hi = (int16_t)(temp1W32 >> 16); + K_low = (int16_t)((temp1W32 - ((int32_t)K_hi << 16)) >> 1); + + /* Store first reflection coefficient */ + K[0] = K_hi; + + temp1W32 >>= 4; /* A[1] in Q27. */ + + /* Put A[1] in hi and low format */ + A_hi[1] = (int16_t)(temp1W32 >> 16); + A_low[1] = (int16_t)((temp1W32 - ((int32_t)A_hi[1] << 16)) >> 1); + + /* Alpha = R[0] * (1-K^2) */ + + temp1W32 = (((K_hi * K_low) >> 14) + K_hi * K_hi) << 1; /* = k^2 in Q31 */ + + temp1W32 = WEBRTC_SPL_ABS_W32(temp1W32); /* Guard against <0 */ + temp1W32 = (int32_t)0x7fffffffL - temp1W32; /* temp1W32 = (1 - K[0]*K[0]) in Q31 */ + + /* Store temp1W32 = 1 - K[0]*K[0] on hi and low format */ + tmp_hi = (int16_t)(temp1W32 >> 16); + tmp_low = (int16_t)((temp1W32 - ((int32_t)tmp_hi << 16)) >> 1); + + /* Calculate Alpha in Q31 */ + temp1W32 = (R_hi[0] * tmp_hi + ((R_hi[0] * tmp_low) >> 15) + + ((R_low[0] * tmp_hi) >> 15)) << 1; + + /* Normalize Alpha and put it in hi and low format */ + + Alpha_exp = WebRtcSpl_NormW32(temp1W32); + temp1W32 <<= Alpha_exp; + Alpha_hi = (int16_t)(temp1W32 >> 16); + Alpha_low = (int16_t)((temp1W32 - ((int32_t)Alpha_hi<< 16)) >> 1); + + /* Perform the iterative calculations in the + Levinson Durbin algorithm */ + + for (i=2; i<=order; i++) + { + + /* ---- + \ + temp1W32 = R[i] + > R[j]*A[i-j] + / + ---- + j=1..i-1 + */ + + temp1W32 = 0; + + for(j=1; j> 15) + + ((R_low[j] * A_hi[i - j]) >> 15)) << 1); + } + + temp1W32 <<= 4; + temp1W32 += (R_hi[i] << 16) + (R_low[i] << 1); + + /* K = -temp1W32 / Alpha */ + temp2W32 = WEBRTC_SPL_ABS_W32(temp1W32); /* abs(temp1W32) */ + temp3W32 = WebRtcSpl_DivW32HiLow(temp2W32, Alpha_hi, Alpha_low); /* abs(temp1W32)/Alpha */ + + /* Put the sign of temp1W32 back again */ + if (temp1W32 > 0) { + temp3W32 = -temp3W32; + } + + /* Use the Alpha shifts from earlier to denormalize */ + norm = WebRtcSpl_NormW32(temp3W32); + if ((Alpha_exp <= norm)||(temp3W32==0)) { + temp3W32 <<= Alpha_exp; + } else { + if (temp3W32 > 0) + { + temp3W32 = (int32_t)0x7fffffffL; + } else + { + temp3W32 = (int32_t)0x80000000L; + } + } + + /* Put K on hi and low format */ + K_hi = (int16_t)(temp3W32 >> 16); + K_low = (int16_t)((temp3W32 - ((int32_t)K_hi << 16)) >> 1); + + /* Store Reflection coefficient in Q15 */ + K[i-1] = K_hi; + + /* Test for unstable filter. If unstable return 0 and let the + user decide what to do in that case + */ + + if ((int32_t)WEBRTC_SPL_ABS_W16(K_hi) > (int32_t)32740) { + return(-i); /* Unstable filter */ + } + + /* + Compute updated LPC coefficient: Anew[i] + Anew[j]= A[j] + K*A[i-j] for j=1..i-1 + Anew[i]= K + */ + + for(j=1; j> 15) + + ((K_low * A_hi[i - j]) >> 15)) << 1; // temp1W32 += K*A[i-j] in Q27. + + /* Put Anew in hi and low format */ + A_upd_hi[j] = (int16_t)(temp1W32 >> 16); + A_upd_low[j] = (int16_t)((temp1W32 - ((int32_t)A_upd_hi[j] << 16)) >> 1); + } + + temp3W32 >>= 4; /* temp3W32 = K in Q27 (Convert from Q31 to Q27) */ + + /* Store Anew in hi and low format */ + A_upd_hi[i] = (int16_t)(temp3W32 >> 16); + A_upd_low[i] = (int16_t)((temp3W32 - ((int32_t)A_upd_hi[i] << 16)) >> 1); + + /* Alpha = Alpha * (1-K^2) */ + + temp1W32 = (((K_hi * K_low) >> 14) + K_hi * K_hi) << 1; /* K*K in Q31 */ + + temp1W32 = WEBRTC_SPL_ABS_W32(temp1W32); /* Guard against <0 */ + temp1W32 = (int32_t)0x7fffffffL - temp1W32; /* 1 - K*K in Q31 */ + + /* Convert 1- K^2 in hi and low format */ + tmp_hi = (int16_t)(temp1W32 >> 16); + tmp_low = (int16_t)((temp1W32 - ((int32_t)tmp_hi << 16)) >> 1); + + /* Calculate Alpha = Alpha * (1-K^2) in Q31 */ + temp1W32 = (Alpha_hi * tmp_hi + ((Alpha_hi * tmp_low) >> 15) + + ((Alpha_low * tmp_hi) >> 15)) << 1; + + /* Normalize Alpha and store it on hi and low format */ + + norm = WebRtcSpl_NormW32(temp1W32); + temp1W32 <<= norm; + + Alpha_hi = (int16_t)(temp1W32 >> 16); + Alpha_low = (int16_t)((temp1W32 - ((int32_t)Alpha_hi << 16)) >> 1); + + /* Update the total nomalization of Alpha */ + Alpha_exp = Alpha_exp + norm; + + /* Update A[] */ + + for(j=1; j<=i; j++) + { + A_hi[j] =A_upd_hi[j]; + A_low[j] =A_upd_low[j]; + } + } + + /* + Set A[0] to 1.0 and store the A[i] i=1...order in Q12 + (Convert from Q27 and use rounding) + */ + + A[0] = 2048; + + for(i=1; i<=order; i++) { + /* temp1W32 in Q27 */ + temp1W32 = (A_hi[i] << 16) + (A_low[i] << 1); + /* Round and store upper word */ + A[i] = (int16_t)((temp1W32 + 32768) >> 16); + } + return(1); /* Stable filters */ +} + + + + + +/* window */ +/* Matlab generation of floating point code: + * t = (1:256)/257; r = 1-(1-t).^.45; w = sin(r*pi).^3; w = w/sum(w); plot((1:256)/8, w); grid; + * for k=1:16, fprintf(1, '%.8f, ', w(k*16 + (-15:0))); fprintf(1, '\n'); end + * All values are multiplyed with 2^21 in fixed point code. + */ +static const int16_t kWindowAutocorr[WINLEN] = { + 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 5, 6, + 8, 10, 12, 14, 17, 20, 24, 28, 33, 38, 43, 49, + 56, 63, 71, 79, 88, 98, 108, 119, 131, 143, 157, 171, + 186, 202, 219, 237, 256, 275, 296, 318, 341, 365, 390, 416, + 444, 472, 502, 533, 566, 600, 635, 671, 709, 748, 789, 831, + 875, 920, 967, 1015, 1065, 1116, 1170, 1224, 1281, 1339, 1399, 1461, + 1525, 1590, 1657, 1726, 1797, 1870, 1945, 2021, 2100, 2181, 2263, 2348, + 2434, 2523, 2614, 2706, 2801, 2898, 2997, 3099, 3202, 3307, 3415, 3525, + 3637, 3751, 3867, 3986, 4106, 4229, 4354, 4481, 4611, 4742, 4876, 5012, + 5150, 5291, 5433, 5578, 5725, 5874, 6025, 6178, 6333, 6490, 6650, 6811, + 6974, 7140, 7307, 7476, 7647, 7820, 7995, 8171, 8349, 8529, 8711, 8894, + 9079, 9265, 9453, 9642, 9833, 10024, 10217, 10412, 10607, 10803, 11000, 11199, + 11398, 11597, 11797, 11998, 12200, 12401, 12603, 12805, 13008, 13210, 13412, 13614, + 13815, 14016, 14216, 14416, 14615, 14813, 15009, 15205, 15399, 15591, 15782, 15971, + 16157, 16342, 16524, 16704, 16881, 17056, 17227, 17395, 17559, 17720, 17877, 18030, + 18179, 18323, 18462, 18597, 18727, 18851, 18970, 19082, 19189, 19290, 19384, 19471, + 19551, 19623, 19689, 19746, 19795, 19835, 19867, 19890, 19904, 19908, 19902, 19886, + 19860, 19823, 19775, 19715, 19644, 19561, 19465, 19357, 19237, 19102, 18955, 18793, + 18618, 18428, 18223, 18004, 17769, 17518, 17252, 16970, 16672, 16357, 16025, 15677, + 15311, 14929, 14529, 14111, 13677, 13225, 12755, 12268, 11764, 11243, 10706, 10152, + 9583, 8998, 8399, 7787, 7162, 6527, 5883, 5231, 4576, 3919, 3265, 2620, + 1990, 1386, 825, 333 +}; + + +/* By using a hearing threshold level in dB of -28 dB (higher value gives more noise), + the H_T_H (in float) can be calculated as: + H_T_H = pow(10.0, 0.05 * (-28.0)) = 0.039810717055350 + In Q19, H_T_H becomes round(0.039810717055350*2^19) ~= 20872, i.e. + H_T_H = 20872/524288.0, and H_T_HQ19 = 20872; +*/ + + +/* The bandwidth expansion vectors are created from: + kPolyVecLo=[0.900000,0.810000,0.729000,0.656100,0.590490,0.531441,0.478297,0.430467,0.387420,0.348678,0.313811,0.282430]; + kPolyVecHi=[0.800000,0.640000,0.512000,0.409600,0.327680,0.262144]; + round(kPolyVecLo*32768) + round(kPolyVecHi*32768) +*/ +static const int16_t kPolyVecLo[12] = { + 29491, 26542, 23888, 21499, 19349, 17414, 15673, 14106, 12695, 11425, 10283, 9255 +}; +static const int16_t kPolyVecHi[6] = { + 26214, 20972, 16777, 13422, 10737, 8590 +}; + +static __inline int32_t log2_Q8_LPC( uint32_t x ) { + + int32_t zeros; + int16_t frac; + + zeros=WebRtcSpl_NormU32(x); + frac = (int16_t)(((x << zeros) & 0x7FFFFFFF) >> 23); + + /* log2(x) */ + return ((31 - zeros) << 8) + frac; +} + +static const int16_t kMulPitchGain = -25; /* 200/256 in Q5 */ +static const int16_t kChngFactor = 3523; /* log10(2)*10/4*0.4/1.4=log10(2)/1.4= 0.2150 in Q14 */ +static const int16_t kExp2 = 11819; /* 1/log(2) */ +const int kShiftLowerBand = 11; /* Shift value for lower band in Q domain. */ +const int kShiftHigherBand = 12; /* Shift value for higher band in Q domain. */ + +void WebRtcIsacfix_GetVars(const int16_t *input, const int16_t *pitchGains_Q12, + uint32_t *oldEnergy, int16_t *varscale) +{ + int k; + uint32_t nrgQ[4]; + int16_t nrgQlog[4]; + int16_t tmp16, chng1, chng2, chng3, chng4, tmp, chngQ, oldNrgQlog, pgQ, pg3; + int32_t expPg32; + int16_t expPg, divVal; + int16_t tmp16_1, tmp16_2; + + /* Calculate energies of first and second frame halfs */ + nrgQ[0]=0; + for (k = QLOOKAHEAD/2; k < (FRAMESAMPLES/4 + QLOOKAHEAD) / 2; k++) { + nrgQ[0] += (uint32_t)(input[k] * input[k]); + } + nrgQ[1]=0; + for ( ; k < (FRAMESAMPLES/2 + QLOOKAHEAD) / 2; k++) { + nrgQ[1] += (uint32_t)(input[k] * input[k]); + } + nrgQ[2]=0; + for ( ; k < (FRAMESAMPLES * 3 / 4 + QLOOKAHEAD) / 2; k++) { + nrgQ[2] += (uint32_t)(input[k] * input[k]); + } + nrgQ[3]=0; + for ( ; k < (FRAMESAMPLES + QLOOKAHEAD) / 2; k++) { + nrgQ[3] += (uint32_t)(input[k] * input[k]); + } + + for ( k=0; k<4; k++) { + nrgQlog[k] = (int16_t)log2_Q8_LPC(nrgQ[k]); /* log2(nrgQ) */ + } + oldNrgQlog = (int16_t)log2_Q8_LPC(*oldEnergy); + + /* Calculate average level change */ + chng1 = WEBRTC_SPL_ABS_W16(nrgQlog[3]-nrgQlog[2]); + chng2 = WEBRTC_SPL_ABS_W16(nrgQlog[2]-nrgQlog[1]); + chng3 = WEBRTC_SPL_ABS_W16(nrgQlog[1]-nrgQlog[0]); + chng4 = WEBRTC_SPL_ABS_W16(nrgQlog[0]-oldNrgQlog); + tmp = chng1+chng2+chng3+chng4; + chngQ = (int16_t)(tmp * kChngFactor >> 10); /* Q12 */ + chngQ += 2926; /* + 1.0/1.4 in Q12 */ + + /* Find average pitch gain */ + pgQ = 0; + for (k=0; k<4; k++) + { + pgQ += pitchGains_Q12[k]; + } + + pg3 = (int16_t)(pgQ * pgQ >> 11); // pgQ in Q(12+2)=Q14. Q14*Q14>>11 => Q17 + pg3 = (int16_t)(pgQ * pg3 >> 13); /* Q14*Q17>>13 =>Q18 */ + /* kMulPitchGain = -25 = -200 in Q-3. */ + pg3 = (int16_t)(pg3 * kMulPitchGain >> 5); // Q10 + tmp16=(int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(kExp2,pg3,13);/* Q13*Q10>>13 => Q10*/ + if (tmp16<0) { + tmp16_2 = (0x0400 | (tmp16 & 0x03FF)); + tmp16_1 = ((uint16_t)(tmp16 ^ 0xFFFF) >> 10) - 3; /* Gives result in Q14 */ + if (tmp16_1<0) + expPg = -(tmp16_2 << -tmp16_1); + else + expPg = -(tmp16_2 >> tmp16_1); + } else + expPg = (int16_t) -16384; /* 1 in Q14, since 2^0=1 */ + + expPg32 = (int32_t)expPg << 8; /* Q22 */ + divVal = WebRtcSpl_DivW32W16ResW16(expPg32, chngQ); /* Q22/Q12=Q10 */ + + tmp16=(int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(kExp2,divVal,13);/* Q13*Q10>>13 => Q10*/ + if (tmp16<0) { + tmp16_2 = (0x0400 | (tmp16 & 0x03FF)); + tmp16_1 = ((uint16_t)(tmp16 ^ 0xFFFF) >> 10) - 3; /* Gives result in Q14 */ + if (tmp16_1<0) + expPg = tmp16_2 << -tmp16_1; + else + expPg = tmp16_2 >> tmp16_1; + } else + expPg = (int16_t) 16384; /* 1 in Q14, since 2^0=1 */ + + *varscale = expPg-1; + *oldEnergy = nrgQ[3]; +} + + + +static __inline int16_t exp2_Q10_T(int16_t x) { // Both in and out in Q10 + + int16_t tmp16_1, tmp16_2; + + tmp16_2=(int16_t)(0x0400|(x&0x03FF)); + tmp16_1 = -(x >> 10); + if(tmp16_1>0) + return tmp16_2 >> tmp16_1; + else + return tmp16_2 << -tmp16_1; + +} + + +// Declare function pointers. +AutocorrFix WebRtcIsacfix_AutocorrFix; +CalculateResidualEnergy WebRtcIsacfix_CalculateResidualEnergy; + +/* This routine calculates the residual energy for LPC. + * Formula as shown in comments inside. + */ +int32_t WebRtcIsacfix_CalculateResidualEnergyC(int lpc_order, + int32_t q_val_corr, + int q_val_polynomial, + int16_t* a_polynomial, + int32_t* corr_coeffs, + int* q_val_residual_energy) { + int i = 0, j = 0; + int shift_internal = 0, shift_norm = 0; + int32_t tmp32 = 0, word32_high = 0, word32_low = 0, residual_energy = 0; + int64_t sum64 = 0, sum64_tmp = 0; + + for (i = 0; i <= lpc_order; i++) { + for (j = i; j <= lpc_order; j++) { + /* For the case of i == 0: residual_energy += + * a_polynomial[j] * corr_coeffs[i] * a_polynomial[j - i]; + * For the case of i != 0: residual_energy += + * a_polynomial[j] * corr_coeffs[i] * a_polynomial[j - i] * 2; + */ + + tmp32 = a_polynomial[j] * a_polynomial[j - i]; + /* tmp32 in Q(q_val_polynomial * 2). */ + if (i != 0) { + tmp32 <<= 1; + } + sum64_tmp = (int64_t)tmp32 * (int64_t)corr_coeffs[i]; + sum64_tmp >>= shift_internal; + + /* Test overflow and sum the result. */ + if(((sum64_tmp > 0 && sum64 > 0) && (LLONG_MAX - sum64 < sum64_tmp)) || + ((sum64_tmp < 0 && sum64 < 0) && (LLONG_MIN - sum64 > sum64_tmp))) { + /* Shift right for overflow. */ + shift_internal += 1; + sum64 >>= 1; + sum64 += sum64_tmp >> 1; + } else { + sum64 += sum64_tmp; + } + } + } + + word32_high = (int32_t)(sum64 >> 32); + word32_low = (int32_t)sum64; + + // Calculate the value of shifting (shift_norm) for the 64-bit sum. + if(word32_high != 0) { + shift_norm = 32 - WebRtcSpl_NormW32(word32_high); + residual_energy = (int32_t)(sum64 >> shift_norm); + } else { + if((word32_low & 0x80000000) != 0) { + shift_norm = 1; + residual_energy = (uint32_t)word32_low >> 1; + } else { + shift_norm = WebRtcSpl_NormW32(word32_low); + residual_energy = word32_low << shift_norm; + shift_norm = -shift_norm; + } + } + + /* Q(q_val_polynomial * 2) * Q(q_val_corr) >> shift_internal >> shift_norm + * = Q(q_val_corr - shift_internal - shift_norm + q_val_polynomial * 2) + */ + *q_val_residual_energy = q_val_corr - shift_internal - shift_norm + + q_val_polynomial * 2; + + return residual_energy; +} + +void WebRtcIsacfix_GetLpcCoef(int16_t *inLoQ0, + int16_t *inHiQ0, + MaskFiltstr_enc *maskdata, + int16_t snrQ10, + const int16_t *pitchGains_Q12, + int32_t *gain_lo_hiQ17, + int16_t *lo_coeffQ15, + int16_t *hi_coeffQ15) +{ + int k, n, ii; + int pos1, pos2; + int sh_lo, sh_hi, sh, ssh, shMem; + int16_t varscaleQ14; + + int16_t tmpQQlo, tmpQQhi; + int32_t tmp32; + int16_t tmp16,tmp16b; + + int16_t polyHI[ORDERHI+1]; + int16_t rcQ15_lo[ORDERLO], rcQ15_hi[ORDERHI]; + + + int16_t DataLoQ6[WINLEN], DataHiQ6[WINLEN]; + int32_t corrloQQ[ORDERLO+2]; + int32_t corrhiQQ[ORDERHI+1]; + int32_t corrlo2QQ[ORDERLO+1]; + int16_t scale; + int16_t QdomLO, QdomHI, newQdomHI, newQdomLO; + + int32_t res_nrgQQ; + int32_t sqrt_nrg; + + /* less-noise-at-low-frequencies factor */ + int16_t aaQ14; + + /* Multiplication with 1/sqrt(12) ~= 0.28901734104046 can be done by convertion to + Q15, i.e. round(0.28901734104046*32768) = 9471, and use 9471/32768.0 ~= 0.289032 + */ + int16_t snrq; + int shft; + + int16_t tmp16a; + int32_t tmp32a, tmp32b, tmp32c; + + int16_t a_LOQ11[ORDERLO+1]; + int16_t k_vecloQ15[ORDERLO]; + int16_t a_HIQ12[ORDERHI+1]; + int16_t k_vechiQ15[ORDERHI]; + + int16_t stab; + + snrq=snrQ10; + + /* SNR= C * 2 ^ (D * snrq) ; C=0.289, D=0.05*log2(10)=0.166 (~=172 in Q10)*/ + tmp16 = (int16_t)(snrq * 172 >> 10); // Q10 + tmp16b = exp2_Q10_T(tmp16); // Q10 + snrq = (int16_t)(tmp16b * 285 >> 10); // Q10 + + /* change quallevel depending on pitch gains and level fluctuations */ + WebRtcIsacfix_GetVars(inLoQ0, pitchGains_Q12, &(maskdata->OldEnergy), &varscaleQ14); + + /* less-noise-at-low-frequencies factor */ + /* Calculation of 0.35 * (0.5 + 0.5 * varscale) in fixpoint: + With 0.35 in Q16 (0.35 ~= 22938/65536.0 = 0.3500061) and varscaleQ14 in Q14, + we get Q16*Q14>>16 = Q14 + */ + aaQ14 = (int16_t)((22938 * (8192 + (varscaleQ14 >> 1)) + 32768) >> 16); + + /* Calculate tmp = (1.0 + aa*aa); in Q12 */ + tmp16 = (int16_t)(aaQ14 * aaQ14 >> 15); // Q14*Q14>>15 = Q13 + tmpQQlo = 4096 + (tmp16 >> 1); // Q12 + Q13>>1 = Q12. + + /* Calculate tmp = (1.0+aa) * (1.0+aa); */ + tmp16 = 8192 + (aaQ14 >> 1); // 1+a in Q13. + tmpQQhi = (int16_t)(tmp16 * tmp16 >> 14); // Q13*Q13>>14 = Q12 + + /* replace data in buffer by new look-ahead data */ + for (pos1 = 0; pos1 < QLOOKAHEAD; pos1++) { + maskdata->DataBufferLoQ0[pos1 + WINLEN - QLOOKAHEAD] = inLoQ0[pos1]; + } + + for (k = 0; k < SUBFRAMES; k++) { + + /* Update input buffer and multiply signal with window */ + for (pos1 = 0; pos1 < WINLEN - UPDATE/2; pos1++) { + maskdata->DataBufferLoQ0[pos1] = maskdata->DataBufferLoQ0[pos1 + UPDATE/2]; + maskdata->DataBufferHiQ0[pos1] = maskdata->DataBufferHiQ0[pos1 + UPDATE/2]; + DataLoQ6[pos1] = (int16_t)(maskdata->DataBufferLoQ0[pos1] * + kWindowAutocorr[pos1] >> 15); // Q0*Q21>>15 = Q6 + DataHiQ6[pos1] = (int16_t)(maskdata->DataBufferHiQ0[pos1] * + kWindowAutocorr[pos1] >> 15); // Q0*Q21>>15 = Q6 + } + pos2 = (int16_t)(k * UPDATE / 2); + for (n = 0; n < UPDATE/2; n++, pos1++) { + maskdata->DataBufferLoQ0[pos1] = inLoQ0[QLOOKAHEAD + pos2]; + maskdata->DataBufferHiQ0[pos1] = inHiQ0[pos2++]; + DataLoQ6[pos1] = (int16_t)(maskdata->DataBufferLoQ0[pos1] * + kWindowAutocorr[pos1] >> 15); // Q0*Q21>>15 = Q6 + DataHiQ6[pos1] = (int16_t)(maskdata->DataBufferHiQ0[pos1] * + kWindowAutocorr[pos1] >> 15); // Q0*Q21>>15 = Q6 + } + + /* Get correlation coefficients */ + /* The highest absolute value measured inside DataLo in the test set + For DataHi, corresponding value was 160. + + This means that it should be possible to represent the input values + to WebRtcSpl_AutoCorrelation() as Q6 values (since 307*2^6 = + 19648). Of course, Q0 will also work, but due to the low energy in + DataLo and DataHi, the outputted autocorrelation will be more accurate + and mimic the floating point code better, by being in an high as possible + Q-domain. + */ + + WebRtcIsacfix_AutocorrFix(corrloQQ,DataLoQ6,WINLEN, ORDERLO+1, &scale); + QdomLO = 12-scale; // QdomLO is the Q-domain of corrloQQ + sh_lo = WebRtcSpl_NormW32(corrloQQ[0]); + QdomLO += sh_lo; + for (ii=0; ii> 1) - + (WEBRTC_SPL_MUL_16_32_RSFT16(aaQ14, corrloQQ[1]) >> 2); + + /* Calculate corrlo2[n] = tmpQQlo * corrlo[n] - tmpQQlo * (corrlo[n-1] + corrlo[n+1]);*/ + for (n = 1; n <= ORDERLO; n++) { + + tmp32 = (corrloQQ[n - 1] >> 1) + (corrloQQ[n + 1] >> 1); // Q(QdomLO-1). + corrlo2QQ[n] = (WEBRTC_SPL_MUL_16_32_RSFT16(tmpQQlo, corrloQQ[n]) >> 1) - + (WEBRTC_SPL_MUL_16_32_RSFT16(aaQ14, tmp32) >> 2); + } + QdomLO -= 5; + + /* Calculate corrhi[n] = tmpQQhi * corrhi[n]; */ + for (n = 0; n <= ORDERHI; n++) { + corrhiQQ[n] = WEBRTC_SPL_MUL_16_32_RSFT16(tmpQQhi, corrhiQQ[n]); // Q(12+QdomHI-16) = Q(QdomHI-4) + } + QdomHI -= 4; + + /* add white noise floor */ + /* corrlo2QQ is in Q(QdomLO) and corrhiQQ is in Q(QdomHI) */ + /* Calculate corrlo2[0] += 9.5367431640625e-7; and + corrhi[0] += 9.5367431640625e-7, where the constant is 1/2^20 */ + + tmp32 = WEBRTC_SPL_SHIFT_W32((int32_t) 1, QdomLO-20); + corrlo2QQ[0] += tmp32; + tmp32 = WEBRTC_SPL_SHIFT_W32((int32_t) 1, QdomHI-20); + corrhiQQ[0] += tmp32; + + /* corrlo2QQ is in Q(QdomLO) and corrhiQQ is in Q(QdomHI) before the following + code segment, where we want to make sure we get a 1-bit margin */ + for (n = 0; n <= ORDERLO; n++) { + corrlo2QQ[n] >>= 1; // Make sure we have a 1-bit margin. + } + QdomLO -= 1; // Now, corrlo2QQ is in Q(QdomLO), with a 1-bit margin + + for (n = 0; n <= ORDERHI; n++) { + corrhiQQ[n] >>= 1; // Make sure we have a 1-bit margin. + } + QdomHI -= 1; // Now, corrhiQQ is in Q(QdomHI), with a 1-bit margin + + + newQdomLO = QdomLO; + + for (n = 0; n <= ORDERLO; n++) { + int32_t tmp, tmpB, tmpCorr; + int16_t alpha=328; //0.01 in Q15 + int16_t beta=324; //(1-0.01)*0.01=0.0099 in Q15 + int16_t gamma=32440; //(1-0.01)=0.99 in Q15 + + if (maskdata->CorrBufLoQQ[n] != 0) { + shMem=WebRtcSpl_NormW32(maskdata->CorrBufLoQQ[n]); + sh = QdomLO - maskdata->CorrBufLoQdom[n]; + if (sh<=shMem) { + tmp = WEBRTC_SPL_SHIFT_W32(maskdata->CorrBufLoQQ[n], sh); // Get CorrBufLoQQ to same domain as corrlo2 + tmp = WEBRTC_SPL_MUL_16_32_RSFT15(alpha, tmp); + } else if ((sh-shMem)<7){ + tmp = WEBRTC_SPL_SHIFT_W32(maskdata->CorrBufLoQQ[n], shMem); // Shift up CorrBufLoQQ as much as possible + // Shift |alpha| the number of times required to get |tmp| in QdomLO. + tmp = WEBRTC_SPL_MUL_16_32_RSFT15(alpha << (sh - shMem), tmp); + } else { + tmp = WEBRTC_SPL_SHIFT_W32(maskdata->CorrBufLoQQ[n], shMem); // Shift up CorrBufHiQQ as much as possible + // Shift |alpha| as much as possible without overflow the number of + // times required to get |tmp| in QdomLO. + tmp = WEBRTC_SPL_MUL_16_32_RSFT15(alpha << 6, tmp); + tmpCorr = corrloQQ[n] >> (sh - shMem - 6); + tmp = tmp + tmpCorr; + maskdata->CorrBufLoQQ[n] = tmp; + newQdomLO = QdomLO-(sh-shMem-6); + maskdata->CorrBufLoQdom[n] = newQdomLO; + } + } else + tmp = 0; + + tmp = tmp + corrlo2QQ[n]; + + maskdata->CorrBufLoQQ[n] = tmp; + maskdata->CorrBufLoQdom[n] = QdomLO; + + tmp=WEBRTC_SPL_MUL_16_32_RSFT15(beta, tmp); + tmpB=WEBRTC_SPL_MUL_16_32_RSFT15(gamma, corrlo2QQ[n]); + corrlo2QQ[n] = tmp + tmpB; + } + if( newQdomLO!=QdomLO) { + for (n = 0; n <= ORDERLO; n++) { + if (maskdata->CorrBufLoQdom[n] != newQdomLO) + corrloQQ[n] >>= maskdata->CorrBufLoQdom[n] - newQdomLO; + } + QdomLO = newQdomLO; + } + + + newQdomHI = QdomHI; + + for (n = 0; n <= ORDERHI; n++) { + int32_t tmp, tmpB, tmpCorr; + int16_t alpha=328; //0.01 in Q15 + int16_t beta=324; //(1-0.01)*0.01=0.0099 in Q15 + int16_t gamma=32440; //(1-0.01)=0.99 in Q1 + if (maskdata->CorrBufHiQQ[n] != 0) { + shMem=WebRtcSpl_NormW32(maskdata->CorrBufHiQQ[n]); + sh = QdomHI - maskdata->CorrBufHiQdom[n]; + if (sh<=shMem) { + tmp = WEBRTC_SPL_SHIFT_W32(maskdata->CorrBufHiQQ[n], sh); // Get CorrBufHiQQ to same domain as corrhi + tmp = WEBRTC_SPL_MUL_16_32_RSFT15(alpha, tmp); + tmpCorr = corrhiQQ[n]; + tmp = tmp + tmpCorr; + maskdata->CorrBufHiQQ[n] = tmp; + maskdata->CorrBufHiQdom[n] = QdomHI; + } else if ((sh-shMem)<7) { + tmp = WEBRTC_SPL_SHIFT_W32(maskdata->CorrBufHiQQ[n], shMem); // Shift up CorrBufHiQQ as much as possible + // Shift |alpha| the number of times required to get |tmp| in QdomHI. + tmp = WEBRTC_SPL_MUL_16_32_RSFT15(alpha << (sh - shMem), tmp); + tmpCorr = corrhiQQ[n]; + tmp = tmp + tmpCorr; + maskdata->CorrBufHiQQ[n] = tmp; + maskdata->CorrBufHiQdom[n] = QdomHI; + } else { + tmp = WEBRTC_SPL_SHIFT_W32(maskdata->CorrBufHiQQ[n], shMem); // Shift up CorrBufHiQQ as much as possible + // Shift |alpha| as much as possible without overflow the number of + // times required to get |tmp| in QdomHI. + tmp = WEBRTC_SPL_MUL_16_32_RSFT15(alpha << 6, tmp); + tmpCorr = corrhiQQ[n] >> (sh - shMem - 6); + tmp = tmp + tmpCorr; + maskdata->CorrBufHiQQ[n] = tmp; + newQdomHI = QdomHI-(sh-shMem-6); + maskdata->CorrBufHiQdom[n] = newQdomHI; + } + } else { + tmp = corrhiQQ[n]; + tmpCorr = tmp; + maskdata->CorrBufHiQQ[n] = tmp; + maskdata->CorrBufHiQdom[n] = QdomHI; + } + + tmp=WEBRTC_SPL_MUL_16_32_RSFT15(beta, tmp); + tmpB=WEBRTC_SPL_MUL_16_32_RSFT15(gamma, tmpCorr); + corrhiQQ[n] = tmp + tmpB; + } + + if( newQdomHI!=QdomHI) { + for (n = 0; n <= ORDERHI; n++) { + if (maskdata->CorrBufHiQdom[n] != newQdomHI) + corrhiQQ[n] >>= maskdata->CorrBufHiQdom[n] - newQdomHI; + } + QdomHI = newQdomHI; + } + + stab=WebRtcSpl_LevinsonW32_JSK(corrlo2QQ, a_LOQ11, k_vecloQ15, ORDERLO); + + if (stab<0) { // If unstable use lower order + a_LOQ11[0]=2048; + for (n = 1; n <= ORDERLO; n++) { + a_LOQ11[n]=0; + } + + stab=WebRtcSpl_LevinsonW32_JSK(corrlo2QQ, a_LOQ11, k_vecloQ15, 8); + } + + + WebRtcSpl_LevinsonDurbin(corrhiQQ, a_HIQ12, k_vechiQ15, ORDERHI); + + /* bandwidth expansion */ + for (n = 1; n <= ORDERLO; n++) { + a_LOQ11[n] = (int16_t)((kPolyVecLo[n - 1] * a_LOQ11[n] + (1 << 14)) >> + 15); + } + + + polyHI[0] = a_HIQ12[0]; + for (n = 1; n <= ORDERHI; n++) { + a_HIQ12[n] = (int16_t)(((int32_t)(kPolyVecHi[n - 1] * a_HIQ12[n]) + + (1 << 14)) >> 15); + polyHI[n] = a_HIQ12[n]; + } + + /* Normalize the corrlo2 vector */ + sh = WebRtcSpl_NormW32(corrlo2QQ[0]); + for (n = 0; n <= ORDERLO; n++) { + corrlo2QQ[n] <<= sh; + } + QdomLO += sh; /* Now, corrlo2QQ is still in Q(QdomLO) */ + + + /* residual energy */ + + sh_lo = 31; + res_nrgQQ = WebRtcIsacfix_CalculateResidualEnergy(ORDERLO, QdomLO, + kShiftLowerBand, a_LOQ11, corrlo2QQ, &sh_lo); + + /* Convert to reflection coefficients */ + WebRtcSpl_AToK_JSK(a_LOQ11, ORDERLO, rcQ15_lo); + + if (sh_lo & 0x0001) { + res_nrgQQ >>= 1; + sh_lo-=1; + } + + + if( res_nrgQQ > 0 ) + { + sqrt_nrg=WebRtcSpl_Sqrt(res_nrgQQ); + + /* add hearing threshold and compute the gain */ + /* lo_coeff = varscale * S_N_R / (sqrt_nrg + varscale * H_T_H); */ + + tmp32a = varscaleQ14 >> 1; // H_T_HQ19=65536 (16-17=-1) + ssh = sh_lo >> 1; // sqrt_nrg is in Qssh. + sh = ssh - 14; + tmp32b = WEBRTC_SPL_SHIFT_W32(tmp32a, sh); // Q14->Qssh + tmp32c = sqrt_nrg + tmp32b; // Qssh (denominator) + tmp32a = varscaleQ14 * snrq; // Q24 (numerator) + + sh = WebRtcSpl_NormW32(tmp32c); + shft = 16 - sh; + tmp16a = (int16_t) WEBRTC_SPL_SHIFT_W32(tmp32c, -shft); // Q(ssh-shft) (denominator) + + tmp32b = WebRtcSpl_DivW32W16(tmp32a, tmp16a); // Q(24-ssh+shft) + sh = ssh-shft-7; + *gain_lo_hiQ17 = WEBRTC_SPL_SHIFT_W32(tmp32b, sh); // Gains in Q17 + } + else + { + *gain_lo_hiQ17 = 100; // Gains in Q17 + } + gain_lo_hiQ17++; + + /* copy coefficients to output array */ + for (n = 0; n < ORDERLO; n++) { + *lo_coeffQ15 = (int16_t) (rcQ15_lo[n]); + lo_coeffQ15++; + } + /* residual energy */ + sh_hi = 31; + res_nrgQQ = WebRtcIsacfix_CalculateResidualEnergy(ORDERHI, QdomHI, + kShiftHigherBand, a_HIQ12, corrhiQQ, &sh_hi); + + /* Convert to reflection coefficients */ + WebRtcSpl_LpcToReflCoef(polyHI, ORDERHI, rcQ15_hi); + + if (sh_hi & 0x0001) { + res_nrgQQ >>= 1; + sh_hi-=1; + } + + + if( res_nrgQQ > 0 ) + { + sqrt_nrg=WebRtcSpl_Sqrt(res_nrgQQ); + + + /* add hearing threshold and compute the gain */ + /* hi_coeff = varscale * S_N_R / (sqrt_nrg + varscale * H_T_H); */ + + tmp32a = varscaleQ14 >> 1; // H_T_HQ19=65536 (16-17=-1) + + ssh = sh_hi >> 1; // |sqrt_nrg| is in Qssh. + sh = ssh - 14; + tmp32b = WEBRTC_SPL_SHIFT_W32(tmp32a, sh); // Q14->Qssh + tmp32c = sqrt_nrg + tmp32b; // Qssh (denominator) + tmp32a = varscaleQ14 * snrq; // Q24 (numerator) + + sh = WebRtcSpl_NormW32(tmp32c); + shft = 16 - sh; + tmp16a = (int16_t) WEBRTC_SPL_SHIFT_W32(tmp32c, -shft); // Q(ssh-shft) (denominator) + + tmp32b = WebRtcSpl_DivW32W16(tmp32a, tmp16a); // Q(24-ssh+shft) + sh = ssh-shft-7; + *gain_lo_hiQ17 = WEBRTC_SPL_SHIFT_W32(tmp32b, sh); // Gains in Q17 + } + else + { + *gain_lo_hiQ17 = 100; // Gains in Q17 + } + gain_lo_hiQ17++; + + + /* copy coefficients to output array */ + for (n = 0; n < ORDERHI; n++) { + *hi_coeffQ15 = rcQ15_hi[n]; + hi_coeffQ15++; + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h new file mode 100644 index 0000000000..97d291aca2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_masking_model.h + * + * LPC functions + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_LPC_MASKING_MODEL_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_LPC_MASKING_MODEL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "structs.h" + +void WebRtcIsacfix_GetVars(const int16_t *input, + const int16_t *pitchGains_Q12, + uint32_t *oldEnergy, + int16_t *varscale); + +void WebRtcIsacfix_GetLpcCoef(int16_t *inLoQ0, + int16_t *inHiQ0, + MaskFiltstr_enc *maskdata, + int16_t snrQ10, + const int16_t *pitchGains_Q12, + int32_t *gain_lo_hiQ17, + int16_t *lo_coeffQ15, + int16_t *hi_coeffQ15); + +typedef int32_t (*CalculateResidualEnergy)(int lpc_order, + int32_t q_val_corr, + int q_val_polynomial, + int16_t* a_polynomial, + int32_t* corr_coeffs, + int* q_val_residual_energy); +extern CalculateResidualEnergy WebRtcIsacfix_CalculateResidualEnergy; + +int32_t WebRtcIsacfix_CalculateResidualEnergyC(int lpc_order, + int32_t q_val_corr, + int q_val_polynomial, + int16_t* a_polynomial, + int32_t* corr_coeffs, + int* q_val_residual_energy); + +#if defined(MIPS_DSP_R2_LE) +int32_t WebRtcIsacfix_CalculateResidualEnergyMIPS(int lpc_order, + int32_t q_val_corr, + int q_val_polynomial, + int16_t* a_polynomial, + int32_t* corr_coeffs, + int* q_val_residual_energy); +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_LPC_MASKING_MODEL_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_mips.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_mips.c new file mode 100644 index 0000000000..727008da32 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_mips.c @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h" + +// MIPS DSPR2 optimization for function WebRtcIsacfix_CalculateResidualEnergy +// Bit-exact with WebRtcIsacfix_CalculateResidualEnergyC from file +// lpc_masking_model.c +int32_t WebRtcIsacfix_CalculateResidualEnergyMIPS(int lpc_order, + int32_t q_val_corr, + int q_val_polynomial, + int16_t* a_polynomial, + int32_t* corr_coeffs, + int* q_val_residual_energy) { + + int i = 0, j = 0; + int shift_internal = 0, shift_norm = 0; + int32_t tmp32 = 0, word32_high = 0, word32_low = 0, residual_energy = 0; + int32_t tmp_corr_c = corr_coeffs[0]; + int16_t* tmp_a_poly = &a_polynomial[0]; + int32_t sum64_hi = 0; + int32_t sum64_lo = 0; + + for (j = 0; j <= lpc_order; j++) { + // For the case of i == 0: + // residual_energy += + // a_polynomial[j] * corr_coeffs[i] * a_polynomial[j - i]; + + int32_t tmp2, tmp3; + int16_t sign_1; + int16_t sign_2; + int16_t sign_3; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lh %[tmp2], 0(%[tmp_a_poly]) \n\t" + "mul %[tmp32], %[tmp2], %[tmp2] \n\t" + "addiu %[tmp_a_poly], %[tmp_a_poly], 2 \n\t" + "sra %[sign_2], %[sum64_hi], 31 \n\t" + "mult $ac0, %[tmp32], %[tmp_corr_c] \n\t" + "shilov $ac0, %[shift_internal] \n\t" + "mfhi %[tmp2], $ac0 \n\t" + "mflo %[tmp3], $ac0 \n\t" + "sra %[sign_1], %[tmp2], 31 \n\t" + "xor %[sign_3], %[sign_1], %[sign_2] \n\t" + ".set pop \n\t" + : [tmp2] "=&r" (tmp2), [tmp3] "=&r" (tmp3), [tmp32] "=&r" (tmp32), + [tmp_a_poly] "+r" (tmp_a_poly), [sign_1] "=&r" (sign_1), + [sign_3] "=&r" (sign_3), [sign_2] "=&r" (sign_2), + [sum64_hi] "+r" (sum64_hi), [sum64_lo] "+r" (sum64_lo) + : [tmp_corr_c] "r" (tmp_corr_c), [shift_internal] "r" (shift_internal) + : "hi", "lo", "memory" + ); + + if (sign_3 != 0) { + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addsc %[sum64_lo], %[sum64_lo], %[tmp3] \n\t" + "addwc %[sum64_hi], %[sum64_hi], %[tmp2] \n\t" + ".set pop \n\t" + : [sum64_hi] "+r" (sum64_hi), [sum64_lo] "+r" (sum64_lo) + : [tmp2] "r" (tmp2), [tmp3] "r" (tmp3) + : "hi", "lo", "memory" + ); + } else { + if (((!(sign_1 || sign_2)) && (0x7FFFFFFF - sum64_hi < tmp2)) || + ((sign_1 && sign_2) && (sum64_hi + tmp2 > 0))) { + // Shift right for overflow. + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[shift_internal], %[shift_internal], 1 \n\t" + "prepend %[sum64_lo], %[sum64_hi], 1 \n\t" + "sra %[sum64_hi], %[sum64_hi], 1 \n\t" + "prepend %[tmp3], %[tmp2], 1 \n\t" + "sra %[tmp2], %[tmp2], 1 \n\t" + "addsc %[sum64_lo], %[sum64_lo], %[tmp3] \n\t" + "addwc %[sum64_hi], %[sum64_hi], %[tmp2] \n\t" + ".set pop \n\t" + : [tmp2] "+r" (tmp2), [tmp3] "+r" (tmp3), + [shift_internal] "+r" (shift_internal), + [sum64_hi] "+r" (sum64_hi), [sum64_lo] "+r" (sum64_lo) + : + : "hi", "lo", "memory" + ); + } else { + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addsc %[sum64_lo], %[sum64_lo], %[tmp3] \n\t" + "addwc %[sum64_hi], %[sum64_hi], %[tmp2] \n\t" + ".set pop \n\t" + : [sum64_hi] "+r" (sum64_hi), [sum64_lo] "+r" (sum64_lo) + : [tmp2] "r" (tmp2), [tmp3] "r" (tmp3) + : "hi", "lo", "memory" + ); + } + } + } + + for (i = 1; i <= lpc_order; i++) { + tmp_corr_c = corr_coeffs[i]; + int16_t* tmp_a_poly_j = &a_polynomial[i]; + int16_t* tmp_a_poly_j_i = &a_polynomial[0]; + for (j = i; j <= lpc_order; j++) { + // For the case of i = 1 .. lpc_order: + // residual_energy += + // a_polynomial[j] * corr_coeffs[i] * a_polynomial[j - i] * 2; + + int32_t tmp2, tmp3; + int16_t sign_1; + int16_t sign_2; + int16_t sign_3; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lh %[tmp3], 0(%[tmp_a_poly_j]) \n\t" + "lh %[tmp2], 0(%[tmp_a_poly_j_i]) \n\t" + "addiu %[tmp_a_poly_j], %[tmp_a_poly_j], 2 \n\t" + "addiu %[tmp_a_poly_j_i], %[tmp_a_poly_j_i], 2 \n\t" + "mul %[tmp32], %[tmp3], %[tmp2] \n\t" + "sll %[tmp32], %[tmp32], 1 \n\t" + "mult $ac0, %[tmp32], %[tmp_corr_c] \n\t" + "shilov $ac0, %[shift_internal] \n\t" + "mfhi %[tmp2], $ac0 \n\t" + "mflo %[tmp3], $ac0 \n\t" + "sra %[sign_1], %[tmp2], 31 \n\t" + "sra %[sign_2], %[sum64_hi], 31 \n\t" + "xor %[sign_3], %[sign_1], %[sign_2] \n\t" + ".set pop \n\t" + : [tmp2] "=&r" (tmp2), [tmp3] "=&r" (tmp3), [tmp32] "=&r" (tmp32), + [tmp_a_poly_j] "+r" (tmp_a_poly_j), [sign_1] "=&r" (sign_1), + [tmp_a_poly_j_i] "+r" (tmp_a_poly_j_i), [sign_2] "=&r" (sign_2), + [sign_3] "=&r" (sign_3), [sum64_hi] "+r" (sum64_hi), + [sum64_lo] "+r" (sum64_lo) + : [tmp_corr_c] "r" (tmp_corr_c), [shift_internal] "r" (shift_internal) + : "hi", "lo", "memory" + ); + if (sign_3 != 0) { + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addsc %[sum64_lo], %[sum64_lo], %[tmp3] \n\t" + "addwc %[sum64_hi], %[sum64_hi], %[tmp2] \n\t" + ".set pop \n\t" + : [tmp2] "+r" (tmp2), [tmp3] "+r" (tmp3), [sum64_hi] "+r" (sum64_hi), + [sum64_lo] "+r" (sum64_lo) + : + :"memory" + ); + } else { + // Test overflow and sum the result. + if (((!(sign_1 || sign_2)) && (0x7FFFFFFF - sum64_hi < tmp2)) || + ((sign_1 && sign_2) && (sum64_hi + tmp2 > 0))) { + // Shift right for overflow. + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[shift_internal], %[shift_internal], 1 \n\t" + "prepend %[sum64_lo], %[sum64_hi], 1 \n\t" + "sra %[sum64_hi], %[sum64_hi], 1 \n\t" + "prepend %[tmp3], %[tmp2], 1 \n\t" + "sra %[tmp2], %[tmp2], 1 \n\t" + "addsc %[sum64_lo], %[sum64_lo], %[tmp3] \n\t" + "addwc %[sum64_hi], %[sum64_hi], %[tmp2] \n\t" + ".set pop \n\t" + : [tmp2] "+r" (tmp2), [tmp3] "+r" (tmp3), + [shift_internal] "+r" (shift_internal), + [sum64_hi] "+r" (sum64_hi), [sum64_lo] "+r" (sum64_lo) + : + : "hi", "lo", "memory" + ); + } else { + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addsc %[sum64_lo], %[sum64_lo], %[tmp3] \n\t" + "addwc %[sum64_hi], %[sum64_hi], %[tmp2] \n\t" + ".set pop \n\t" + : [tmp2] "+r" (tmp2), [tmp3] "+r" (tmp3), + [sum64_hi] "+r" (sum64_hi), [sum64_lo] "+r" (sum64_lo) + : + : "hi", "lo", "memory" + ); + } + } + } + } + word32_high = sum64_hi; + word32_low = sum64_lo; + + // Calculate the value of shifting (shift_norm) for the 64-bit sum. + if (word32_high != 0) { + shift_norm = 32 - WebRtcSpl_NormW32(word32_high); + int tmp1; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "srl %[residual_energy], %[sum64_lo], %[shift_norm] \n\t" + "li %[tmp1], 32 \n\t" + "subu %[tmp1], %[tmp1], %[shift_norm] \n\t" + "sll %[tmp1], %[sum64_hi], %[tmp1] \n\t" + "or %[residual_energy], %[residual_energy], %[tmp1] \n\t" + ".set pop \n\t" + : [residual_energy] "=&r" (residual_energy), [tmp1]"=&r"(tmp1), + [sum64_hi] "+r" (sum64_hi), [sum64_lo] "+r" (sum64_lo) + : [shift_norm] "r" (shift_norm) + : "memory" + ); + } else { + if ((word32_low & 0x80000000) != 0) { + shift_norm = 1; + residual_energy = (uint32_t)word32_low >> 1; + } else { + shift_norm = WebRtcSpl_NormW32(word32_low); + residual_energy = word32_low << shift_norm; + shift_norm = -shift_norm; + } + } + + // Q(q_val_polynomial * 2) * Q(q_val_corr) >> shift_internal >> shift_norm + // = Q(q_val_corr - shift_internal - shift_norm + q_val_polynomial * 2) + *q_val_residual_energy = + q_val_corr - shift_internal - shift_norm + q_val_polynomial * 2; + + return residual_energy; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_unittest.cc new file mode 100644 index 0000000000..1604cc4144 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model_unittest.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +class LpcMaskingModelTest : public testing::Test { + protected: + // Pass a function pointer to the Tester function. + void CalculateResidualEnergyTester(CalculateResidualEnergy + CalculateResidualEnergyFunction) { + const int kIntOrder = 10; + const int32_t kInt32QDomain = 5; + const int kIntShift = 11; + int16_t a[kIntOrder + 1] = {32760, 122, 7, 0, -32760, -3958, + -48, 18745, 498, 9, 23456}; + int32_t corr[kIntOrder + 1] = {11443647, -27495, 0, + 98745, -11443600, 1, 1, 498, 9, 888, 23456}; + int q_shift_residual = 0; + int32_t residual_energy = 0; + + // Test the code path where (residual_energy >= 0x10000). + residual_energy = CalculateResidualEnergyFunction(kIntOrder, + kInt32QDomain, kIntShift, a, corr, &q_shift_residual); + EXPECT_EQ(1789023310, residual_energy); + EXPECT_EQ(2, q_shift_residual); + + // Test the code path where (residual_energy < 0x10000) + // and ((energy & 0x8000) != 0). + for (int i = 0; i < kIntOrder + 1; i++) { + a[i] = 24575 >> i; + corr[i] = i; + } + residual_energy = CalculateResidualEnergyFunction(kIntOrder, + kInt32QDomain, kIntShift, a, corr, &q_shift_residual); + EXPECT_EQ(1595279092, residual_energy); + EXPECT_EQ(26, q_shift_residual); + + // Test the code path where (residual_energy <= 0x7fff). + for (int i = 0; i < kIntOrder + 1; i++) { + a[i] = 2457 >> i; + } + residual_energy = CalculateResidualEnergyFunction(kIntOrder, + kInt32QDomain, kIntShift, a, corr, &q_shift_residual); + EXPECT_EQ(2029266944, residual_energy); + EXPECT_EQ(33, q_shift_residual); + } +}; + +TEST_F(LpcMaskingModelTest, CalculateResidualEnergyTest) { + CalculateResidualEnergyTester(WebRtcIsacfix_CalculateResidualEnergyC); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.c new file mode 100644 index 0000000000..bc0f129711 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.c @@ -0,0 +1,1280 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_tables.c + * + * Coding tables for the KLT coefficients + * + */ + + +#include "settings.h" +#include "lpc_tables.h" + +/* indices of KLT coefficients used */ +const uint16_t WebRtcIsacfix_kSelIndGain[12] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11}; + +const uint16_t WebRtcIsacfix_kSelIndShape[108] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 106, 107 +}; + +/* cdf array for model indicator */ +const uint16_t WebRtcIsacfix_kModelCdf[4] = { + 0, 15434, 37548, 65535 +}; + +/* pointer to cdf array for model indicator */ +const uint16_t *WebRtcIsacfix_kModelCdfPtr[1] = { + WebRtcIsacfix_kModelCdf +}; + +/* initial cdf index for decoder of model indicator */ +const uint16_t WebRtcIsacfix_kModelInitIndex[1] = { + 1 +}; + +/* offset to go from rounded value to quantization index */ +const int16_t WebRtcIsacfix_kQuantMinGain[12] ={ + 3, 6, 4, 6, 6, 9, 5, 16, 11, 34, 32, 47 +}; + +const int16_t WebRtcIsacfix_kQuantMinShape[108] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 2, 2, 2, 3, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, + 1, 1, 1, 2, 2, 3, 0, 0, 0, 0, + 1, 0, 1, 1, 1, 1, 1, 1, 1, 2, + 2, 4, 3, 5, 0, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 2, 3, 4, + 4, 7, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 2, 3, 2, 3, 4, 4, 5, 7, 13, + 0, 1, 1, 2, 3, 2, 2, 2, 4, 4, + 5, 6, 7, 11, 9, 13, 12, 26 +}; + +/* maximum quantization index */ +const uint16_t WebRtcIsacfix_kMaxIndGain[12] = { + 6, 12, 8, 14, 10, 19, 12, 31, 22, 56, 52, 138 +}; + +const uint16_t WebRtcIsacfix_kMaxIndShape[108] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 2, 2, 2, 2, 4, 4, 5, 6, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 1, 2, 2, + 2, 2, 3, 4, 5, 7, 0, 0, 0, 0, + 2, 0, 2, 2, 2, 2, 3, 2, 2, 4, + 4, 6, 6, 9, 0, 0, 0, 0, 2, 2, + 2, 2, 2, 2, 3, 2, 4, 4, 7, 7, + 9, 13, 0, 0, 2, 2, 2, 2, 2, 2, + 3, 4, 5, 4, 6, 8, 8, 10, 16, 25, + 0, 2, 2, 4, 5, 4, 4, 4, 7, 8, + 9, 10, 13, 19, 17, 23, 25, 49 +}; + +/* index offset */ +const uint16_t WebRtcIsacfix_kOffsetGain[3][12] = { + { 0, 7, 20, 29, 44, 55, 75, 88, 120, 143, 200, 253}, + { 0, 7, 19, 27, 42, 53, 73, 86, 117, 140, 197, 249}, + { 0, 7, 20, 28, 44, 55, 75, 89, 121, 145, 202, 257} +}; + +const uint16_t WebRtcIsacfix_kOffsetShape[3][108] = { + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 11, 14, 17, 20, 23, 28, 33, 39, 46, 47, + 48, 49, 50, 52, 53, 54, 55, 56, 58, 61, + 64, 67, 70, 74, 79, 85, 93, 94, 95, 96, + 97, 100, 101, 104, 107, 110, 113, 117, 120, 123, + 128, 133, 140, 147, 157, 158, 159, 160, 161, 164, + 167, 170, 173, 176, 179, 183, 186, 191, 196, 204, + 212, 222, 236, 237, 238, 241, 244, 247, 250, 253, + 256, 260, 265, 271, 276, 283, 292, 301, 312, 329, + 355, 356, 359, 362, 367, 373, 378, 383, 388, 396, + 405, 415, 426, 440, 460, 478, 502, 528 + }, + { + 0, 1, 2, 3, 4, 6, 7, 8, 9, 11, + 13, 16, 19, 22, 26, 29, 34, 39, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 55, 57, 60, + 63, 66, 70, 73, 78, 84, 91, 92, 93, 94, + 95, 96, 97, 99, 102, 105, 108, 111, 114, 118, + 123, 128, 134, 141, 151, 152, 153, 154, 156, 159, + 162, 165, 168, 171, 174, 177, 181, 186, 194, 200, + 208, 218, 233, 234, 235, 236, 239, 242, 245, 248, + 251, 254, 258, 263, 270, 277, 288, 297, 308, 324, + 349, 351, 354, 357, 361, 366, 372, 378, 383, 390, + 398, 407, 420, 431, 450, 472, 496, 524 + }, + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 11, + 14, 17, 20, 23, 26, 29, 34, 40, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 58, 61, 64, + 67, 70, 73, 77, 82, 88, 96, 97, 98, 99, + 101, 102, 104, 107, 110, 113, 116, 119, 122, 125, + 129, 134, 141, 150, 160, 161, 162, 163, 166, 168, + 171, 174, 177, 180, 183, 186, 190, 195, 201, 208, + 216, 226, 243, 244, 245, 248, 251, 254, 257, 260, + 263, 268, 273, 278, 284, 291, 299, 310, 323, 340, + 366, 368, 371, 374, 379, 383, 389, 394, 399, 406, + 414, 422, 433, 445, 461, 480, 505, 533 + } +}; + +/* initial cdf index for KLT coefficients */ +const uint16_t WebRtcIsacfix_kInitIndexGain[3][12] = { + { 3, 6, 4, 7, 5, 10, 6, 16, 11, 28, 26, 69}, + { 3, 6, 4, 7, 5, 10, 6, 15, 11, 28, 26, 69}, + { 3, 6, 4, 8, 5, 10, 7, 16, 12, 28, 27, 70} +}; + +const uint16_t WebRtcIsacfix_kInitIndexShape[3][108] = { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 2, 2, 3, 3, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, + 1, 1, 2, 2, 3, 4, 0, 0, 0, 0, + 1, 0, 1, 1, 1, 1, 2, 1, 1, 2, + 2, 3, 3, 5, 0, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 2, 4, 4, + 5, 7, 0, 0, 1, 1, 1, 1, 1, 1, + 2, 2, 3, 2, 3, 4, 4, 5, 8, 13, + 0, 1, 1, 2, 3, 2, 2, 2, 4, 4, + 5, 5, 7, 10, 9, 12, 13, 25 + }, + { + 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 1, 1, 1, 2, 1, 2, 2, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 2, 1, 2, 3, 3, 0, 0, 0, 0, + 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, + 2, 3, 3, 5, 0, 0, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 2, 2, 4, 3, 4, + 5, 7, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 2, 2, 3, 3, 5, 4, 5, 8, 12, + 1, 1, 1, 2, 2, 3, 3, 2, 3, 4, + 4, 6, 5, 9, 11, 12, 14, 25 + }, + { + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 2, 3, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 1, 2, 2, 3, 4, 0, 0, 0, 1, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, + 2, 3, 4, 5, 0, 0, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, + 5, 8, 0, 0, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 3, 3, 4, 5, 6, 8, 13, + 1, 1, 1, 2, 2, 3, 2, 2, 3, 4, + 4, 5, 6, 8, 9, 12, 14, 25 + } +}; + +/* offsets for quantizer representation levels*/ +const uint16_t WebRtcIsacfix_kOfLevelsGain[3] = { + 0, 392, 779 +}; + +const uint16_t WebRtcIsacfix_kOfLevelsShape[3] = { + 0, 578, 1152 +}; + +/* quantizer representation levels */ + + + +const int32_t WebRtcIsacfix_kLevelsGainQ17[1176] = { + -364547,-231664,-102123,-573,104807,238257,368823,-758583,-640135,-510291 + ,-377662,-252785,-113177,2627,112906,248601,389461,522691,644517,763974 + ,-538963,-368179,-245823,-106095,-890,104299,241111,350730,493190,-800763 + ,-646230,-510239,-382115,-248623,-111829,-2983,113852,251105,388114,519757 + ,644048,774712,896334,1057931,-770009,-635310,-503690,-375087,-248106,-108525 + ,-105,108259,243472,377948,519271,-1160885,-1032044,-914636,-777593,-647891 + ,-518408,-388028,-254321,-115293,-598,117849,251296,385367,515510,652727 + ,777432,920363,1038625,1153637,1316836,-632287,-505430,-379430,-248458,-118013 + ,-888,118762,250266,381650,513327,652169,766471,932113,-2107480,-1971030 + ,-1827020,-1698263,-1558670,-1436314,-1305377,-1172252,-1047355,-914202,-779651,-651001 + ,-520999,-390394,-255761,-123490,-1893,126839,256703,385493,518607,651760 + ,782750,908693,1044131,1163865,1311066,1424177,1582628,1709823,1831740,1955391 + ,-1423044,-1288917,-1181281,-1043222,-911770,-780354,-646799,-522664,-386721,-258266 + ,-128060,-1101,128233,259996,390336,519590,649290,778701,908010,1040796 + ,1161235,1306889,1441882,-4446002,-4301031,-4194304,-4080591,-3947740,-3808975,-3686530 + ,-3567839,-3383251,-3287089,-3136577,-3017405,-2869860,-2751321,-2619984,-2482932,-2354790 + ,-2223147,-2090669,-1964135,-1831208,-1706697,-1570817,-1446008,-1305386,-1175773,-1046066 + ,-915356,-785120,-653614,-524331,-393767,-260442,-130187,-799,128841,261466 + ,393616,520542,652117,784613,914159,1045399,1181072,1308971,1442502,1570346 + ,1693912,1843986,1966014,2090474,2224869,2364593,2475934,2628403,2752512,2856640 + ,-4192441,-4063232,-3917821,-3799195,-3666233,-3519199,-3411021,-3269192,-3135684,-3008826 + ,-2880875,-2747342,-2620981,-2494872,-2354979,-2229718,-2098939,-1964971,-1835399,-1703452 + ,-1572806,-1440482,-1311794,-1179338,-1046521,-919823,-785914,-655335,-523416,-395507 + ,-264833,-132184,-2546,131698,256217,391372,522688,651248,789964,909618 + ,1035305,1179145,1313824,1436934,1552353,1693722,1815508,1972826,2096328,2228224 + ,2359296,2490368,2598848,-6160384,-6029312,-5881382,-5767168,-5636096,-5505024,-5373952 + ,-5228418,-5110384,-4954923,-4880576,-4710990,-4587364,-4471340,-4333905,-4211513,-4051293 + ,-3907927,-3800105,-3675961,-3538640,-3413663,-3271148,-3152105,-3019103,-2869647,-2744015 + ,-2620639,-2479385,-2364211,-2227611,-2095427,-1974497,-1834168,-1703561,-1568961,-1439826 + ,-1309192,-1174050,-1050191,-917836,-786015,-656943,-518934,-394831,-257708,-128041 + ,1610,128991,264442,393977,521383,653849,788164,918641,1049122,1181971 + ,1308934,1439505,1571808,1706305,1836318,1966235,2097269,2228990,2357005,2490292 + ,2617400,2749680,2881234,3014880,3145637,3276467,3409099,3536637,3671493,3802918 + ,3929740,4065036,4194143,4325999,4456126,4586857,4717194,4843923,4978676,5110913 + ,5245281,5371394,5499780,5633779,5762611,5897682,6028688,6167546,6296465,6421682 + ,6548882,6682074,6809432,6941956,7078143,7204509,7334296,7475137,7609896,7732044 + ,7861604,8002039,8131670,8259222,8390299,8522399,8650037,8782348,8908402,9037815 + ,9164594,9300338,9434679,9574500,9699702,9833934,9948152,10083972,10244937,10332822 + ,10485760,10600122,10760754,10892964,11010048,11111004,11272192,11403264,11525091,11624984 + ,11796480,11915146,-393216,-262144,-101702,-740,100568,262144,393216,-786432 + ,-655360,-524288,-383907,-243301,-94956,-156,95547,269629,416691,524288 + ,655360,-393216,-262144,-88448,-37,87318,262144,393216,524288,-917504 + ,-786432,-655360,-495894,-373308,-267503,-93211,4119,91308,250895,393216 + ,526138,655360,786432,917504,-786432,-655360,-524288,-393216,-262144,-83497 + ,222,86893,240922,393216,524288,-1048576,-917504,-790472,-655360,-508639 + ,-383609,-262016,-95550,-3775,96692,256797,364847,534906,655360,786432 + ,889679,1048576,1179648,1310720,1441792,-655360,-524288,-377684,-248408,-93690 + ,1261,95441,227519,393216,524288,655360,786432,917504,-2097152,-1966080 + ,-1809470,-1703936,-1572864,-1441792,-1314289,-1195149,-1056205,-917504,-809951,-657769 + ,-521072,-383788,-248747,-106350,-2944,105550,243408,388548,521064,628732 + ,786432,885456,1064548,1179648,1310720,1441792,1572864,1703936,1835008,-1441792 + ,-1310720,-1179648,-1037570,-888492,-767774,-646634,-519935,-373458,-248029,-111915 + ,760,111232,247735,379432,507672,672699,786432,917504,1048576,1179648 + ,1310720,1441792,-4456448,-4325376,-4194304,-4063232,-3932160,-3801088,-3670016,-3538944 + ,-3407872,-3276800,-3145728,-3014656,-2883584,-2752512,-2647002,-2490368,-2359296,-2228224 + ,-2097152,-1951753,-1835008,-1703936,-1594177,-1462001,-1289150,-1160774,-1025917,-924928 + ,-782509,-641294,-516191,-386630,-251910,-118886,5210,121226,253949,386008 + ,517973,649374,780064,917783,1052462,1183856,1290593,1419389,1556641,1699884 + ,1835008,1988314,2090470,2228224,2359296,2490368,2621440,2752512,2883584,-3801088 + ,-3643514,-3539937,-3409931,-3263294,-3145658,-3012952,-2879230,-2752359,-2622556,-2483471 + ,-2357556,-2226500,-2093112,-1965892,-1833664,-1701035,-1567767,-1440320,-1310556,-1178339 + ,-1049625,-916812,-786477,-655277,-525050,-393773,-264828,-130696,-480,132126 + ,260116,394197,527846,652294,785563,917183,1049511,1175958,1308161,1438759 + ,1572253,1698835,1828535,1967072,2089391,2212798,2348901,2461547,2621440,2752512 + ,2883584,-7309870,-7203780,-7062699,-6939106,-6790819,-6672036,-6553600,-6422317,-6288422 + ,-6164694,-6026456,-5901410,-5754168,-5621459,-5502710,-5369686,-5240454,-5120712,-4976140 + ,-4847970,-4723070,-4589083,-4450923,-4324680,-4189892,-4065551,-3931803,-3800209,-3668539 + ,-3539395,-3404801,-3277470,-3141389,-3016710,-2885724,-2752612,-2618541,-2486762,-2354153 + ,-2225059,-2094984,-1968194,-1830895,-1699508,-1575743,-1444516,-1308683,-1179714,-1053088 + ,-917981,-783707,-653900,-524980,-395409,-260309,-131948,-3452,132113,263241 + ,392185,522597,654134,788288,919810,1045795,1179210,1314201,1444235,1574447 + ,1705193,1834009,1967332,2098102,2229019,2359147,2489859,2619878,2754966,2879671 + ,3014438,3146143,3276733,3405958,3542196,3667493,3798815,3932961,4062458,4187125 + ,4322346,4454875,4587752,4716809,4848274,4975027,5111957,5242215,5373085,5501158 + ,5640140,5762918,5895358,6024008,6157906,6290628,6422713,6546339,6675888,6815606 + ,6955288,7077501,7211630,7337893,7473635,7607175,7728310,7866475,7999658,8127888 + ,8241758,8386483,8522550,8641582,8771915,8922139,9038632,9179385,9313426,9437184 + ,9568256,9699328,9830400,9952933,10120004,10223616,10354688,10474645,10616832,-393216 + ,-262144,-85425,-121,82533,262144,393216,-786432,-655360,-524288,-379928 + ,-222821,-95200,287,95541,227093,393216,493567,655360,786432,-393216 + ,-262144,-86805,510,86722,262144,393216,524288,-1048576,-917504,-786432 + ,-624456,-529951,-395071,-241627,-101168,81,99975,241605,393216,524288 + ,655360,786432,917504,-786432,-655360,-524288,-393216,-230359,-95619,-137 + ,94425,226222,393216,524288,-1179648,-1048576,-917504,-773841,-655360,-492258 + ,-379715,-244707,-103621,-434,104523,242680,381575,523659,650565,786432 + ,917504,1048576,1179648,1310720,-786432,-629344,-524288,-376757,-242858,-101932 + ,-2715,107155,239212,366480,514943,655360,786432,917504,-2228224,-2097152 + ,-1966080,-1835008,-1703936,-1572864,-1441792,-1284584,-1179648,-1048819,-934658,-777181 + ,-626371,-515660,-377493,-248975,-113036,436,113584,248354,379718,512475 + ,653932,796494,917504,1048576,1179648,1310720,1441792,1572864,1703936,1835008 + ,-1572864,-1441792,-1297608,-1161159,-1032316,-917092,-779770,-647384,-515529,-384269 + ,-250003,-119252,1053,118111,249512,380545,512039,648101,770656,907003 + ,1021725,1178082,1310720,1441792,-4587520,-4456448,-4325376,-4194304,-4063232,-3932160 + ,-3801088,-3670016,-3538944,-3407872,-3276800,-3145728,-2999335,-2883584,-2752512,-2621440 + ,-2490368,-2359296,-2228224,-2112691,-1966080,-1848781,-1709830,-1566109,-1438427,-1303530 + ,-1176124,-1040936,-913876,-784585,-652025,-518361,-385267,-256342,-127297,-2733 + ,125422,257792,389363,519911,651106,783805,909407,1044143,1174156,1309267 + ,1436173,1553771,1708958,1814083,1967036,2095386,2255169,2359296,2478303,2621440 + ,2752512,-4456448,-4325376,-4194304,-4063232,-3932160,-3797524,-3670016,-3560250,-3413217 + ,-3257719,-3166416,-2986626,-2878000,-2781144,-2625383,-2495465,-2346792,-2230930,-2077063 + ,-1949225,-1819274,-1697261,-1568664,-1443074,-1304302,-1175289,-1043794,-913423,-785561 + ,-652104,-522835,-392667,-260517,-130088,-2,129509,260990,391931,522470 + ,655770,784902,917093,1046445,1176951,1303121,1441362,1565401,1702022,1822856 + ,1952852,2090384,2214607,2338436,2457483,2621440,-8781824,-8650752,-8519680,-8388608 + ,-8260828,-8126464,-8003337,-7859030,-7750057,-7602176,-7471104,-7340032,-7193045,-7090588 + ,-6946816,-6843344,-6676635,-6557575,-6447804,-6277614,-6159736,-6035729,-5884723,-5739567 + ,-5634818,-5489867,-5372864,-5243300,-5098939,-4988639,-4856258,-4728494,-4591717,-4447428 + ,-4322409,-4192918,-4062638,-3934141,-3797545,-3673373,-3531587,-3407391,-3277404,-3147797 + ,-3013578,-2886548,-2749811,-2616428,-2490949,-2361301,-2228482,-2096883,-1964343,-1831754 + ,-1702201,-1572495,-1442012,-1309242,-1182451,-1048996,-916905,-786510,-657079,-524730 + ,-393672,-261313,-128743,166,130678,261334,393287,524155,655570,786839 + ,917353,1052167,1179013,1309360,1442634,1571153,1703961,1832027,1965014,2097912 + ,2224861,2355341,2490455,2623051,2753484,2877015,3015783,3144157,3273705,3405255 + ,3542006,3669580,3802417,3935413,4065088,4190896,4333521,4456355,4579781,4713832 + ,4845707,4978625,5113278,5243817,5382318,5500592,5638135,5761179,5900822,6029270 + ,6186398,6297816,6436435,6559163,6666389,6806548,6950461,7086078,7195777,7350973 + ,7480132,7614852,7743514,7847288,8014762,8126464,8257536,8388608,8519680,8650752 + ,8781824,8912896,9043968,9175040,9306112,9437184 +}; + + + +const int16_t WebRtcIsacfix_kLevelsShapeQ10[1735] = { + 0, 0, -1, 0, 0, 1, 0, 1, 0, -821 + , 1, -763, -1, 656, -620, 0, 633, -636, 4, 615 + , -630, 1, 649, -1773, -670, 5, 678, 1810, -1876, -676 + , 0, 691, 1843, -1806, -743, -1, 749, 1795, 2920, -2872 + , -1761, -772, -3, 790, 1763, 2942, 0, 0, 0, 0 + , -792, 2, 0, 0, 1, 0, -854, 0, -702, -1 + , 662, -624, -5, 638, -611, -6, 638, -647, 0, 651 + , -685, -4, 679, 2123, -1814, -693, 0, 664, 1791, -1735 + , -737, 0, 771, 1854, 2873, -2867, -1842, -793, -1, 821 + , 1826, 2805, 3922, 0, 0, 0, -1, -779, 1, 786 + , 1, -708, 0, 789, -799, 1, 797, -663, 2, 646 + , -600, 3, 609, -600, 1, 658, 1807, -627, -3, 612 + , -625, 3, 632, -1732, -674, 1, 672, 2048, -1768, -715 + , 0, 724, 1784, -3881, -3072, -1774, -719, -1, 730, 1811 + , -2963, -1829, -806, -1, 816, 1795, 3050, -5389, -3784, -2942 + , -1893, -865, -12, 867, 1885, 2945, 3928, -2, 1, 4 + , 0, -694, 2, 665, -598, 5, 587, -599, -1, 661 + , -656, -7, 611, -607, 5, 603, -618, -4, 620, -1794 + , -645, -2, 654, -655, -1, 658, -1801, -700, 5, 707 + , 1927, -1752, -745, -8, 752, 1843, -2838, -1781, -801, 11 + , 796, 1811, 2942, 3866, -3849, -3026, -1848, -819, 2, 827 + , 1825, 2963, -3873, -2904, -1869, -910, -6, 903, 1902, 2885 + , 3978, 5286, -7168, -6081, -4989, -3968, -2963, -1970, -943, -2 + , 953, 1951, 2968, 3974, 5009, 6032, -2, 3, -1024, 2 + , 1024, -637, 1, 669, -613, -7, 630, -603, 4, 612 + , -612, 0, 590, -645, -11, 627, -657, -2, 671, 1849 + , -1853, -694, 2, 702, 1838, -3304, -1780, -736, -8, 732 + , 1772, -1709, -755, -6, 760, 1780, -2994, -1780, -800, 8 + , 819, 1830, 2816, -4096, -2822, -1881, -851, -4, 855, 1872 + , 2840, 3899, -3908, -2904, -1878, -887, 6, 897, 1872, 2942 + , 4008, -4992, -3881, -2933, -1915, -928, 1, 937, 1919, 2900 + , 4009, 4881, -6848, -6157, -5065, -3981, -2983, -1972, -978, -1 + , 968, 1979, 2988, 4008, 5007, 6108, 7003, 8051, 9027,-13272 + ,-12012,-11228,-10213, -9261, -8084, -7133, -6075, -5052, -4050, -3036 + , -2014, -996, -4, 1007, 2031, 3038, 4049, 5074, 6134, 7069 + , 8094, 9069, 10212, 11049, 12104, 51, -1024, -13, 1024, -609 + , -107, 613, -2048, -687, -95, 667, 2048, -3072, -1724, -785 + , -34, 732, 1819, -2048, -703, -26, 681, 2048, -2048, -686 + , -9, 665, 2048, -2048, -702, 37, 748, 1723, -4096, -2786 + , -1844, -837, 37, 811, 1742, 3072, -4096, -2783, -1848, -881 + , 39, 898, 1843, 2792, 3764, -5120, -4096, -2923, -1833, -852 + , -14, 862, 1824, 2834, 4096, -6144, -5120, -3914, -2842, -1870 + , -886, -27, 888, 1929, 2931, 4051, -7168, -6144, -5120, -3866 + , -2933, -1915, -927, 64, 933, 1902, 2929, 3912, 5063, 6144 + ,-11264,-10240, -9216, -8192, -7086, -6144, -5039, -3972, -2943, -1929 + , -941, 3, 938, 1942, 2959, 3933, 4905, 6088, 6983, 8192 + , -9216, -8192, -7202, -6088, -4983, -4019, -2955, -1975, -966, 17 + , 997, 1981, 2967, 3990, 4948, 6022, 6967, 8192,-13312,-12288 + ,-11264,-10240, -9216, -8049, -6997, -6040, -5026, -4043, -3029, -2034 + , -1015, -23, 984, 1997, 3010, 4038, 5002, 6015, 6946, 8061 + , 9216, 10240,-12381,-11264,-10240, -9060, -8058, -7153, -6085, -5075 + , -4051, -3042, -2037, -1017, -5, 1007, 2028, 3035, 4050, 5088 + , 6111, 7160, 8156, 9215, 10095, 11229, 12202, 13016,-26624,-25600 + ,-24582,-23671,-22674,-21400,-20355,-19508,-18315,-17269,-16361,-15299 + ,-14363,-13294,-12262,-11237,-10203, -9227, -8165, -7156, -6116, -5122 + , -4076, -3056, -2043, -1020, -8, 1027, 2047, 3065, 4110, 5130 + , 6125, 7168, 8195, 9206, 10230, 11227, 12256, 13304, 14281, 15316 + , 16374, 17382, 18428, 19388, 20361, 21468, 22448, 23781, 0, 0 + , -1, 0, -2, 1024, 0, 0, 0, -1, 1024, -1024 + , 1, -1024, 4, 1024, -1024, 2, 1024, -1024, 2, 1024 + , -2048, -1024, -4, 1024, -1024, 2, 1024, -2048, -1024, -3 + , 1024, 2048, -2048, -1024, 4, 1024, 2048, -3072, -2048, -1024 + , -1, 662, 2048, 0, 1, 0, 0, 1, -2, -2 + , 0, 2, 1024, -1, 1024, -1024, 4, 1024, -1024, 1 + , 1024, -1024, 1, 1024, -2048, -781, -4, 844, -807, -5 + , 866, -2048, -726, -13, 777, 2048, -2048, -643, -4, 617 + , 2048, 3072, -3072, -2048, -629, 1, 630, 2048, 3072, 0 + , -1, 1, -2, 2, 1, -1024, 5, -1024, 6, 1024 + , -1024, 4, 1024, -1024, 1, 1024, -1024, -9, 1024, -673 + , -7, 655, -2048, -665, -15, 716, -2048, -647, 4, 640 + , 2048, -2048, -615, -1, 635, 2048, -2048, -613, 10, 637 + , 2048, 3072, -3072, -2048, -647, -3, 641, 2048, 3072, -5120 + , -4096, -3072, -2048, -681, 6, 685, 2048, 3072, 4096, 1 + , 1, 0, -1, 1024, -1024, -3, 1024, -1024, 6, 1024 + , -1024, -1, 769, -733, 0, 1024, -876, -2, 653, -1024 + , -4, 786, -596, -13, 595, -634, -2, 638, 2048, -2048 + , -620, -5, 620, 2048, -4096, -3072, -2048, -639, 11, 655 + , 2048, 3072, -3072, -2048, -659, 5, 663, 2048, -3072, -1823 + , -687, 22, 695, 2048, 3072, 4096, -4096, -3072, -1848, -715 + , -3, 727, 1816, 3072, 4096, 5120, -8192, -7168, -6144, -5120 + , -4096, -2884, -1771, -756, -14, 775, 1844, 3072, 4096, 5120 + , 6144, -1, 1, 0, -1024, 2, 815, -768, 2, 708 + , -1024, -3, 693, -661, -7, 607, -643, -5, 609, -624 + , 3, 631, -682, -3, 691, 2048, -2048, -640, 5, 650 + , 2048, -3072, -2048, -701, 9, 704, 2048, 3072, -3072, -2048 + , -670, 10, 674, 2048, 3072, -5120, -4096, -3072, -1749, -738 + , 0, 733, 1811, 3072, 4096, 5120, -4096, -3072, -1873, -753 + , 0, 756, 1874, 3072, 4096, -5120, -4096, -2900, -1838, -793 + , -6, 793, 1868, 2837, 4096, 5120, -7168, -6144, -5120, -4096 + , -2832, -1891, -828, 1, 828, 1901, 2823, 3912, 5120, 6144 + , 7168, 8192,-13312,-12288,-11264,-10240, -9216, -8192, -7168, -6144 + , -5120, -3976, -3004, -1911, -869, 7, 869, 1932, 3024, 3992 + , 5009, 6144, 7168, 8192, 9216, 10240, 11264, -4, 1024, -629 + , -22, 609, -623, 9, 640, -2048, -768, 1, 682, -2048 + , -741, 49, 722, 2048, -3072, -1706, -808, -20, 768, 1750 + , -1684, -727, -29, 788, 1840, 3033, -1758, -784, 0, 801 + , 1702, -3072, -1813, -814, 38, 820, 1884, 2927, -4096, -3241 + , -1839, -922, 25, 882, 1886, 2812, -4096, -2982, -1923, -894 + , 84, 912, 1869, 2778, 4096, -4928, -3965, -2902, -1920, -883 + , 3, 917, 1953, 2921, 3957, 4922, 6144, 7168, -5120, -3916 + , -2897, -1949, -930, 31, 959, 1934, 2901, 3851, 5120, -9216 + , -8192, -7046, -6029, -5030, -4034, -2980, -1969, -1013, -76, 963 + , 1963, 2901, 3929, 4893, 6270, 7168, 8192, 9216,-12288,-11264 + ,-10240, -9216, -8192, -6846, -6123, -5108, -4008, -3000, -1963, -954 + , -6, 958, 1992, 3009, 4020, 5085, 6097, 7168, 8192, 9216 + ,-11264,-10139, -9194, -8127, -7156, -6102, -5053, -4049, -3036, -2025 + , -1009, -34, 974, 1984, 3034, 4028, 5138, 6000, 7057, 8166 + , 9070, 10033, 11360, 12288,-13312,-12288,-10932,-10190, -9120, -8123 + , -7128, -6103, -5074, -4081, -3053, -2029, -989, -4, 1010, 2028 + , 3051, 4073, 5071, 6099, 7132, 8147, 9295, 10159, 11023, 12263 + , 13312, 14336,-25600,-24576,-23552,-22529,-21504,-20480,-19456,-18637 + ,-17425,-16165,-15316,-14327,-13606,-12135,-11182,-10107, -9153, -8144 + , -7146, -6160, -5129, -4095, -3064, -2038, -1025, 1, 1031, 2072 + , 3074, 4088, 5123, 6149, 7157, 8173, 9198, 10244, 11250, 12268 + , 13263, 14289, 15351, 16370, 17402, 18413, 19474, 20337, 21386, 22521 + , 23367, 24350, 0, 0, 0, 0, 0, 0, 0, 0 + , -1024, 0, 1024, -1024, 0, 1024, -1024, 0, 1024, -1024 + , 0, 1024, -1024, 0, 1024, -773, 0, 1024, -674, 0 + , 645, -2048, -745, 0, 628, 2048, -2048, -712, 0, 681 + , 2048, 3072, -3072, -2048, -673, 0, 682, 1964, 3257, 0 + , 0, 0, 0, 0, 0, 0, 0, -1024, 0, 1024 + , -1024, 0, 1024, -1024, 0, 1024, -705, 0, 623, -771 + , 0, 1024, -786, 0, 688, -631, 0, 652, 2048, -2048 + , -627, -1, 666, 2048, -3072, -1756, -694, 0, 674, 2048 + , -3098, -1879, -720, 5, 694, 1886, 2958, 4096, 0, 0 + , 0, 0, 1024, 0, 0, 1024, -769, 0, 1024, -1024 + , 0, 1024, -1024, 0, 1024, -817, 0, 734, -786, 0 + , 651, -638, 0, 637, -623, 0, 671, -652, 0, 619 + , 2048, -2048, -670, -1, 663, 2048, -1908, -680, 1, 686 + , 2048, 3072, 4096, -4096, -3072, -1833, -711, 0, 727, 1747 + , 3072, 4096, -4096, -2971, -1826, -762, 2, 766, 1832, 2852 + , 3928, 5079, 0, 0, 0, -1024, 0, 1024, -1024, 0 + , -656, 0, 1024, -599, 0, 620, -1024, 0, 1024, -603 + , 0, 622, -643, 0, 660, -599, 0, 611, -641, -1 + , 651, 2048, -2048, -648, -2, 647, 1798, -3072, -2048, -672 + , 2, 670, 2048, -3072, -1780, -694, -1, 706, 1751, 3072 + , -3072, -1862, -757, 7, 739, 1798, 3072, 4096, -5120, -4096 + , -3253, -1811, -787, 3, 782, 1887, 3123, 4096, -7252, -6144 + , -5354, -4060, -2864, -1863, -820, -11, 847, 1903, 2970, 3851 + , 4921, 5957, 7168, 8192, 9306, 0, 0, -1024, 0, 1024 + , -726, 0, 706, -692, 0, 593, -598, 0, 616, -624 + , 0, 616, -605, 0, 613, -2048, -652, 1, 635, 2048 + , -2048, -647, -1, 660, 2048, -1811, -668, -2, 685, 2048 + , -1796, -731, -2, 730, 1702, 3072, -3072, -1766, -747, -4 + , 756, 1770, 3072, -4096, -3024, -1762, -783, 4, 771, 1781 + , 3072, -5120, -4057, -2807, -1832, -822, 0, 816, 1804, 2851 + , 3949, 5120, -6144, -4899, -3927, -2920, -1893, -874, -2, 868 + , 1881, 2905, 3960, 4912, 6144, -9216, -8192, -7168, -6225, -4963 + , -3943, -2956, -1890, -902, 0, 897, 1914, 2916, 3984, 4990 + , 6050, 7168,-11264,-10217, -9114, -8132, -7035, -5988, -4984, -4000 + , -2980, -1962, -927, 7, 931, 1956, 2981, 4031, 4972, 6213 + , 7227, 8192, 9216, 10240, 11170, 12288, 13312, 14336, 0, 1024 + , -557, 1, 571, -606, -4, 612, -1676, -707, 10, 673 + , 2048, -2048, -727, 5, 686, -3072, -1772, -755, 12, 716 + , 1877, -1856, -786, 2, 786, 1712, -1685, -818, -16, 863 + , 1729, -3072, -1762, -857, 3, 866, 1838, 2841, -3862, -2816 + , -1864, -925, -2, 923, 1897, 2779, -2782, -1838, -920, -28 + , 931, 1951, 2835, 3804, -4815, -4001, -2940, -1934, -959, -22 + , 975, 1957, 2904, 3971, 4835, -5148, -3892, -2944, -1953, -986 + , -11, 989, 1968, 2939, 3949, 4947, 5902, -9216, -8192, -6915 + , -6004, -4965, -4013, -3009, -1977, -987, -1, 982, 1972, 3000 + , 3960, 4939, 5814, -8976, -7888, -7084, -5955, -5043, -4009, -2991 + , -2002, -1000, -8, 993, 2011, 3023, 4026, 5028, 6023, 7052 + , 8014, 9216,-11240,-10036, -9125, -8118, -7105, -6062, -5048, -4047 + , -3044, -2025, -1009, -1, 1011, 2023, 3042, 4074, 5085, 6108 + , 7119, 8142, 9152, 10114, 11141, 12250, 13307,-15360,-14099,-13284 + ,-12291,-11223,-10221, -9152, -8147, -7128, -6104, -5077, -4072, -3062 + , -2033, -1020, 7, 1018, 2038, 3059, 4081, 5084, 6109, 7102 + , 8128, 9134, 10125, 11239, 12080,-23552,-22528,-21504,-20480,-19456 + ,-18159,-17240,-16291,-15364,-14285,-13305,-12271,-11233,-10217, -9198 + , -8175, -7157, -6134, -5122, -4089, -3071, -2047, -1018, 3, 1026 + , 2041, 3077, 4090, 5108, 6131, 7150, 8172, 9175, 10196, 11272 + , 12303, 13273, 14328, 15332, 16334, 17381, 18409, 19423, 20423, 21451 + , 22679, 23391, 24568, 25600, 26589 +}; + +/* cdf tables for quantizer indices */ +const uint16_t WebRtcIsacfix_kCdfGain[1212] = { + 0, 13, 301, 3730, 61784, 65167, 65489, 65535, 0, 17, + 142, 314, 929, 2466, 7678, 56450, 63463, 64740, 65204, 65426, + 65527, 65535, 0, 8, 100, 724, 6301, 60105, 65125, 65510, + 65531, 65535, 0, 13, 117, 368, 1068, 3010, 11928, 53603, + 61177, 63404, 64505, 65108, 65422, 65502, 65531, 65535, 0, 4, + 17, 96, 410, 1859, 12125, 54361, 64103, 65305, 65497, 65535, + 0, 4, 88, 230, 469, 950, 1746, 3228, 6092, 16592, + 44756, 56848, 61256, 63308, 64325, 64920, 65309, 65460, 65502, 65522, + 65535, 0, 88, 352, 1675, 6339, 20749, 46686, 59284, 63525, + 64949, 65359, 65502, 65527, 65535, 0, 13, 38, 63, 117, + 234, 381, 641, 929, 1407, 2043, 2809, 4032, 5753, 8792, + 14407, 24308, 38941, 48947, 55403, 59293, 61411, 62688, 63630, 64329, + 64840, 65188, 65376, 65472, 65506, 65527, 65531, 65535, 0, 8, + 29, 75, 222, 615, 1327, 2801, 5623, 9931, 16094, 24966, + 34419, 43458, 50676, 56186, 60055, 62500, 63936, 64765, 65225, 65435, + 65514, 65535, 0, 8, 13, 15, 17, 21, 33, 59, + 71, 92, 151, 243, 360, 456, 674, 934, 1223, 1583, + 1989, 2504, 3031, 3617, 4354, 5154, 6163, 7411, 8780, 10747, + 12874, 15591, 18974, 23027, 27436, 32020, 36948, 41830, 46205, 49797, + 53042, 56094, 58418, 60360, 61763, 62818, 63559, 64103, 64509, 64798, + 65045, 65162, 65288, 65363, 65447, 65506, 65522, 65531, 65533, 65535, + 0, 4, 6, 25, 38, 71, 138, 264, 519, 808, + 1227, 1825, 2516, 3408, 4279, 5560, 7092, 9197, 11420, 14108, + 16947, 20300, 23926, 27459, 31164, 34827, 38575, 42178, 45540, 48747, + 51444, 54090, 56426, 58460, 60080, 61595, 62734, 63668, 64275, 64673, + 64936, 65112, 65217, 65334, 65426, 65464, 65477, 65489, 65518, 65527, + 65529, 65531, 65533, 65535, 0, 2, 4, 8, 10, 12, + 14, 16, 21, 33, 50, 71, 84, 92, 105, 138, + 180, 255, 318, 377, 435, 473, 511, 590, 682, 758, + 913, 1097, 1256, 1449, 1671, 1884, 2169, 2445, 2772, 3157, + 3563, 3944, 4375, 4848, 5334, 5820, 6448, 7101, 7716, 8378, + 9102, 9956, 10752, 11648, 12707, 13670, 14758, 15910, 17187, 18472, + 19627, 20649, 21951, 23169, 24283, 25552, 26862, 28227, 29391, 30764, + 31882, 33213, 34432, 35600, 36910, 38116, 39464, 40729, 41872, 43144, + 44371, 45514, 46762, 47813, 48968, 50069, 51032, 51974, 52908, 53737, + 54603, 55445, 56282, 56990, 57572, 58191, 58840, 59410, 59887, 60264, + 60607, 60946, 61269, 61516, 61771, 61960, 62198, 62408, 62558, 62776, + 62985, 63207, 63408, 63546, 63739, 63906, 64070, 64237, 64371, 64551, + 64677, 64836, 64999, 65095, 65213, 65284, 65338, 65380, 65426, 65447, + 65472, 65485, 65487, 65489, 65502, 65510, 65512, 65514, 65516, 65518, + 65522, 65531, 65533, 65535, 0, 2, 4, 6, 65528, 65531, + 65533, 65535, 0, 2, 4, 6, 8, 10, 222, 65321, + 65513, 65528, 65531, 65533, 65535, 0, 2, 4, 50, 65476, + 65529, 65531, 65533, 65535, 0, 2, 4, 6, 8, 12, + 38, 544, 64936, 65509, 65523, 65525, 65529, 65531, 65533, 65535, + 0, 2, 4, 6, 8, 10, 1055, 64508, 65528, 65531, + 65533, 65535, 0, 2, 4, 6, 8, 10, 12, 123, + 3956, 62999, 65372, 65495, 65515, 65521, 65523, 65525, 65527, 65529, + 65531, 65533, 65535, 0, 2, 4, 12, 53, 4707, 59445, + 65467, 65525, 65527, 65529, 65531, 65533, 65535, 0, 2, 4, + 6, 8, 10, 12, 14, 16, 38, 40, 50, 67, + 96, 234, 929, 14345, 55750, 64866, 65389, 65462, 65514, 65517, + 65519, 65521, 65523, 65525, 65527, 65529, 65531, 65533, 65535, 0, + 2, 4, 6, 8, 10, 15, 35, 91, 377, 1946, + 13618, 52565, 63714, 65184, 65465, 65520, 65523, 65525, 65527, 65529, + 65531, 65533, 65535, 0, 2, 4, 6, 8, 10, 12, + 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, + 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, + 54, 82, 149, 362, 751, 1701, 4239, 12893, 38627, 55072, + 60875, 63071, 64158, 64702, 65096, 65283, 65412, 65473, 65494, 65505, + 65508, 65517, 65519, 65521, 65523, 65525, 65527, 65529, 65531, 65533, + 65535, 0, 2, 15, 23, 53, 143, 260, 418, 698, + 988, 1353, 1812, 2411, 3144, 4015, 5143, 6401, 7611, 8999, + 10653, 12512, 14636, 16865, 19404, 22154, 24798, 27521, 30326, 33102, + 35790, 38603, 41415, 43968, 46771, 49435, 52152, 54715, 57143, 59481, + 61178, 62507, 63603, 64489, 64997, 65257, 65427, 65473, 65503, 65520, + 65529, 65531, 65533, 65535, 0, 3, 6, 9, 26, 32, + 44, 46, 64, 94, 111, 164, 205, 254, 327, 409, + 506, 608, 733, 885, 1093, 1292, 1482, 1742, 1993, 2329, + 2615, 3029, 3374, 3798, 4257, 4870, 5405, 5992, 6618, 7225, + 7816, 8418, 9051, 9761, 10532, 11380, 12113, 13010, 13788, 14594, + 15455, 16361, 17182, 18088, 18997, 20046, 20951, 21968, 22947, 24124, + 25296, 26547, 27712, 28775, 29807, 30835, 31709, 32469, 33201, 34014, + 34876, 35773, 36696, 37620, 38558, 39547, 40406, 41277, 42367, 43290, + 44445, 45443, 46510, 47684, 48973, 50157, 51187, 52242, 53209, 54083, + 55006, 55871, 56618, 57293, 57965, 58556, 59222, 59722, 60180, 60554, + 60902, 61250, 61554, 61837, 62100, 62372, 62631, 62856, 63078, 63324, + 63557, 63768, 63961, 64089, 64235, 64352, 64501, 64633, 64770, 64887, + 65001, 65059, 65121, 65188, 65246, 65302, 65346, 65390, 65428, 65463, + 65477, 65506, 65515, 65517, 65519, 65521, 65523, 65525, 65527, 65529, + 65531, 65533, 65535, 0, 2, 4, 109, 65332, 65531, 65533, + 65535, 0, 2, 4, 6, 8, 25, 1817, 63874, 65511, + 65527, 65529, 65531, 65533, 65535, 0, 2, 4, 907, 65014, + 65529, 65531, 65533, 65535, 0, 2, 4, 6, 8, 10, + 12, 132, 2743, 62708, 65430, 65525, 65527, 65529, 65531, 65533, + 65535, 0, 2, 4, 6, 8, 35, 3743, 61666, 65485, + 65531, 65533, 65535, 0, 2, 4, 6, 8, 10, 23, + 109, 683, 6905, 58417, 64911, 65398, 65497, 65518, 65525, 65527, + 65529, 65531, 65533, 65535, 0, 2, 4, 6, 53, 510, + 10209, 55212, 64573, 65441, 65522, 65529, 65531, 65533, 65535, 0, + 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, + 22, 32, 90, 266, 1037, 3349, 14468, 50488, 62394, 64685, + 65341, 65480, 65514, 65519, 65521, 65523, 65525, 65527, 65529, 65531, + 65533, 65535, 0, 2, 4, 6, 9, 16, 37, 106, + 296, 748, 1868, 5733, 18897, 45553, 60165, 63949, 64926, 65314, + 65441, 65508, 65524, 65529, 65531, 65533, 65535, 0, 2, 4, + 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, + 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, + 46, 48, 50, 83, 175, 344, 667, 1293, 2337, 4357, + 8033, 14988, 28600, 43244, 52011, 57042, 59980, 61779, 63065, 63869, + 64390, 64753, 64988, 65164, 65326, 65422, 65462, 65492, 65506, 65522, + 65524, 65526, 65531, 65533, 65535, 0, 2, 4, 6, 8, + 10, 12, 14, 16, 25, 39, 48, 55, 62, 65, + 85, 106, 139, 169, 194, 252, 323, 485, 688, 1074, + 1600, 2544, 3863, 5733, 8303, 11397, 15529, 20273, 25734, 31455, + 36853, 41891, 46410, 50306, 53702, 56503, 58673, 60479, 61880, 62989, + 63748, 64404, 64852, 65124, 65309, 65424, 65480, 65524, 65528, 65533, + 65535, 0, 2, 4, 6, 8, 10, 12, 14, 21, + 23, 25, 27, 29, 31, 39, 41, 43, 48, 60, + 72, 79, 106, 136, 166, 187, 224, 252, 323, 381, + 427, 478, 568, 660, 783, 912, 1046, 1175, 1365, 1567, + 1768, 2024, 2347, 2659, 3049, 3529, 4033, 4623, 5281, 5925, + 6726, 7526, 8417, 9468, 10783, 12141, 13571, 15222, 16916, 18659, + 20350, 22020, 23725, 25497, 27201, 29026, 30867, 32632, 34323, 36062, + 37829, 39466, 41144, 42654, 43981, 45343, 46579, 47759, 49013, 50171, + 51249, 52283, 53245, 54148, 54938, 55669, 56421, 57109, 57791, 58464, + 59092, 59674, 60105, 60653, 61083, 61407, 61757, 62095, 62388, 62649, + 62873, 63157, 63358, 63540, 63725, 63884, 64046, 64155, 64278, 64426, + 64548, 64654, 64806, 64906, 64994, 65077, 65137, 65215, 65277, 65324, + 65354, 65409, 65437, 65455, 65462, 65490, 65495, 65499, 65508, 65511, + 65513, 65515, 65517, 65519, 65521, 65523, 65525, 65527, 65529, 65531, + 65533, 65535 +}; + +const uint16_t WebRtcIsacfix_kCdfShape[2059] = { + 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65535, + 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, 4, + 65535, 0, 8, 65514, 65535, 0, 29, 65481, 65535, 0, + 121, 65439, 65535, 0, 239, 65284, 65535, 0, 8, 779, + 64999, 65527, 65535, 0, 8, 888, 64693, 65522, 65535, 0, + 29, 2604, 62843, 65497, 65531, 65535, 0, 25, 176, 4576, + 61164, 65275, 65527, 65535, 0, 65535, 0, 65535, 0, 65535, + 0, 65535, 0, 4, 65535, 0, 65535, 0, 65535, 0, + 65535, 0, 65535, 0, 4, 65535, 0, 33, 65502, 65535, + 0, 54, 65481, 65535, 0, 251, 65309, 65535, 0, 611, + 65074, 65535, 0, 1273, 64292, 65527, 65535, 0, 4, 1809, + 63940, 65518, 65535, 0, 88, 4392, 60603, 65426, 65531, 65535, + 0, 25, 419, 7046, 57756, 64961, 65514, 65531, 65535, 0, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 4, 65531, + 65535, 0, 65535, 0, 8, 65531, 65535, 0, 4, 65527, + 65535, 0, 17, 65510, 65535, 0, 42, 65481, 65535, 0, + 197, 65342, 65531, 65535, 0, 385, 65154, 65535, 0, 1005, + 64522, 65535, 0, 8, 1985, 63469, 65533, 65535, 0, 38, + 3119, 61884, 65514, 65535, 0, 4, 6, 67, 4961, 60804, + 65472, 65535, 0, 17, 565, 9182, 56538, 65087, 65514, 65535, + 0, 8, 63, 327, 2118, 14490, 52774, 63839, 65376, 65522, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, + 17, 65522, 65535, 0, 59, 65489, 65535, 0, 50, 65522, + 65535, 0, 54, 65489, 65535, 0, 310, 65179, 65535, 0, + 615, 64836, 65535, 0, 4, 1503, 63965, 65535, 0, 2780, + 63383, 65535, 0, 21, 3919, 61051, 65527, 65535, 0, 84, + 6674, 59929, 65435, 65535, 0, 4, 255, 7976, 55784, 65150, + 65518, 65531, 65535, 0, 4, 8, 582, 10726, 53465, 64949, + 65518, 65535, 0, 29, 339, 3006, 17555, 49517, 62956, 65200, + 65497, 65531, 65535, 0, 2, 33, 138, 565, 2324, 7670, + 22089, 45966, 58949, 63479, 64966, 65380, 65518, 65535, 0, 65535, + 0, 65535, 0, 2, 65533, 65535, 0, 46, 65514, 65535, + 0, 414, 65091, 65535, 0, 540, 64911, 65535, 0, 419, + 65162, 65535, 0, 976, 64790, 65535, 0, 2977, 62495, 65531, + 65535, 0, 4, 3852, 61034, 65527, 65535, 0, 4, 29, + 6021, 60243, 65468, 65535, 0, 84, 6711, 58066, 65418, 65535, + 0, 13, 281, 9550, 54917, 65125, 65506, 65535, 0, 2, + 63, 984, 12108, 52644, 64342, 65435, 65527, 65535, 0, 29, + 251, 2014, 14871, 47553, 62881, 65229, 65518, 65535, 0, 13, + 142, 749, 4220, 18497, 45200, 60913, 64823, 65426, 65527, 65535, + 0, 13, 71, 264, 1176, 3789, 10500, 24480, 43488, 56324, + 62315, 64493, 65242, 65464, 65514, 65522, 65531, 65535, 0, 4, + 13, 38, 109, 205, 448, 850, 1708, 3429, 6276, 11371, + 19221, 29734, 40955, 49391, 55411, 59460, 62102, 63793, 64656, 65150, + 65401, 65485, 65522, 65531, 65535, 0, 65535, 0, 2, 65533, + 65535, 0, 1160, 65476, 65535, 0, 2, 6640, 64763, 65533, + 65535, 0, 2, 38, 9923, 61009, 65527, 65535, 0, 2, + 4949, 63092, 65533, 65535, 0, 2, 3090, 63398, 65533, 65535, + 0, 2, 2520, 58744, 65510, 65535, 0, 2, 13, 544, + 8784, 51403, 65148, 65533, 65535, 0, 2, 25, 1017, 10412, + 43550, 63651, 65489, 65527, 65535, 0, 2, 4, 29, 783, + 13377, 52462, 64524, 65495, 65533, 65535, 0, 2, 4, 6, + 100, 1817, 18451, 52590, 63559, 65376, 65531, 65535, 0, 2, + 4, 6, 46, 385, 2562, 11225, 37416, 60488, 65026, 65487, + 65529, 65533, 65535, 0, 2, 4, 6, 8, 10, 12, + 42, 222, 971, 5221, 19811, 45048, 60312, 64486, 65294, 65474, + 65525, 65529, 65533, 65535, 0, 2, 4, 8, 71, 167, + 666, 2533, 7875, 19622, 38082, 54359, 62108, 64633, 65290, 65495, + 65529, 65533, 65535, 0, 2, 4, 6, 8, 10, 13, + 109, 586, 1930, 4949, 11600, 22641, 36125, 48312, 56899, 61495, + 63927, 64932, 65389, 65489, 65518, 65531, 65533, 65535, 0, 4, + 6, 8, 67, 209, 712, 1838, 4195, 8432, 14432, 22834, + 31723, 40523, 48139, 53929, 57865, 60657, 62403, 63584, 64363, 64907, + 65167, 65372, 65472, 65514, 65535, 0, 2, 4, 13, 25, + 42, 46, 50, 75, 113, 147, 281, 448, 657, 909, + 1185, 1591, 1976, 2600, 3676, 5317, 7398, 9914, 12941, 16169, + 19477, 22885, 26464, 29851, 33360, 37228, 41139, 44802, 48654, 52058, + 55181, 57676, 59581, 61022, 62190, 63107, 63676, 64199, 64547, 64924, + 65158, 65313, 65430, 65481, 65518, 65535, 0, 65535, 0, 65535, + 0, 65535, 0, 65535, 0, 65533, 65535, 0, 65535, 0, + 65535, 0, 65535, 0, 65533, 65535, 0, 2, 65535, 0, + 2, 65533, 65535, 0, 2, 65533, 65535, 0, 2, 65533, + 65535, 0, 2, 4, 65533, 65535, 0, 2, 65533, 65535, + 0, 2, 4, 65531, 65533, 65535, 0, 2, 4, 65531, + 65533, 65535, 0, 2, 4, 6, 65524, 65533, 65535, 0, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, + 65535, 0, 65535, 0, 65535, 0, 65533, 65535, 0, 65533, + 65535, 0, 2, 65533, 65535, 0, 2, 65533, 65535, 0, + 2, 65533, 65535, 0, 2, 4, 65532, 65535, 0, 6, + 65523, 65535, 0, 2, 15, 65530, 65533, 65535, 0, 2, + 35, 65493, 65531, 65533, 65535, 0, 2, 4, 158, 65382, + 65531, 65533, 65535, 0, 65535, 0, 65535, 0, 65535, 0, + 65535, 0, 65535, 0, 65535, 0, 2, 65535, 0, 2, + 65533, 65535, 0, 2, 65533, 65535, 0, 2, 65533, 65535, + 0, 2, 65533, 65535, 0, 9, 65512, 65535, 0, 2, + 12, 65529, 65535, 0, 2, 73, 65434, 65533, 65535, 0, + 2, 240, 65343, 65533, 65535, 0, 2, 476, 65017, 65531, + 65533, 65535, 0, 2, 4, 1046, 64686, 65531, 65533, 65535, + 0, 2, 4, 6, 8, 1870, 63898, 65529, 65531, 65533, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65533, 65535, + 0, 2, 65533, 65535, 0, 2, 65533, 65535, 0, 2, + 65532, 65535, 0, 6, 65533, 65535, 0, 6, 65523, 65535, + 0, 2, 65532, 65535, 0, 137, 65439, 65535, 0, 576, + 64899, 65533, 65535, 0, 2, 289, 65299, 65533, 65535, 0, + 2, 4, 6, 880, 64134, 65531, 65533, 65535, 0, 2, + 4, 1853, 63347, 65533, 65535, 0, 2, 6, 2516, 61762, + 65529, 65531, 65533, 65535, 0, 2, 4, 9, 3980, 61380, + 65503, 65529, 65531, 65533, 65535, 0, 2, 4, 6, 8, + 10, 12, 61, 6393, 59859, 65466, 65527, 65529, 65531, 65533, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 2, 65532, + 65535, 0, 3, 65529, 65535, 0, 2, 65529, 65535, 0, + 61, 65453, 65535, 0, 234, 65313, 65535, 0, 503, 65138, + 65535, 0, 155, 65402, 65533, 65535, 0, 2, 1058, 64554, + 65533, 65535, 0, 2, 4, 3138, 62109, 65531, 65533, 65535, + 0, 2, 4, 2031, 63339, 65531, 65533, 65535, 0, 2, + 4, 6, 9, 4155, 60778, 65523, 65529, 65531, 65533, 65535, + 0, 2, 4, 41, 6189, 59269, 65490, 65531, 65533, 65535, + 0, 2, 4, 6, 210, 8789, 57043, 65400, 65528, 65531, + 65533, 65535, 0, 2, 4, 6, 8, 26, 453, 10086, + 55499, 64948, 65483, 65524, 65527, 65529, 65531, 65533, 65535, 0, + 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, + 114, 1014, 11202, 52670, 64226, 65356, 65503, 65514, 65523, 65525, + 65527, 65529, 65531, 65533, 65535, 0, 65533, 65535, 0, 15, + 65301, 65535, 0, 152, 64807, 65535, 0, 2, 3328, 63308, + 65535, 0, 2, 4050, 59730, 65533, 65535, 0, 2, 164, + 10564, 61894, 65529, 65535, 0, 15, 6712, 59831, 65076, 65532, + 65535, 0, 32, 7712, 57449, 65459, 65535, 0, 2, 210, + 7849, 53110, 65021, 65523, 65535, 0, 2, 12, 1081, 13883, + 48262, 62870, 65477, 65535, 0, 2, 88, 847, 6145, 37852, + 62012, 65454, 65533, 65535, 0, 9, 47, 207, 1823, 14522, + 45521, 61069, 64891, 65481, 65528, 65531, 65533, 65535, 0, 2, + 9, 488, 2881, 12758, 38703, 58412, 64420, 65410, 65533, 65535, + 0, 2, 4, 6, 61, 333, 1891, 6486, 19720, 43188, + 57547, 62472, 64796, 65421, 65497, 65523, 65529, 65531, 65533, 65535, + 0, 2, 4, 6, 8, 10, 12, 29, 117, 447, + 1528, 6138, 21242, 43133, 56495, 62432, 64746, 65362, 65500, 65529, + 65531, 65533, 65535, 0, 2, 18, 105, 301, 760, 1490, + 3472, 7568, 15002, 26424, 40330, 53029, 60048, 62964, 64274, 64890, + 65337, 65445, 65489, 65513, 65527, 65530, 65533, 65535, 0, 2, + 4, 6, 41, 102, 409, 853, 2031, 4316, 7302, 11328, + 16869, 24825, 34926, 43481, 50877, 56126, 59874, 62103, 63281, 63857, + 64166, 64675, 65382, 65522, 65531, 65533, 65535, 0, 2, 4, + 6, 8, 10, 12, 14, 16, 18, 29, 38, 53, + 58, 96, 181, 503, 1183, 2849, 5590, 8600, 11379, 13942, + 16478, 19453, 22638, 26039, 29411, 32921, 37596, 41433, 44998, 48560, + 51979, 55106, 57666, 59892, 61485, 62616, 63484, 64018, 64375, 64685, + 64924, 65076, 65278, 65395, 65471, 65509, 65529, 65535, 0, 65535, + 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65535, + 0, 65535, 0, 65535, 0, 2, 65533, 65535, 0, 2, + 65533, 65535, 0, 2, 65533, 65535, 0, 2, 65533, 65535, + 0, 2, 65533, 65535, 0, 2, 65533, 65535, 0, 7, + 65519, 65535, 0, 2, 14, 65491, 65533, 65535, 0, 2, + 81, 65427, 65531, 65533, 65535, 0, 2, 4, 312, 65293, + 65528, 65533, 65535, 0, 65535, 0, 65535, 0, 65535, 0, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, + 2, 65533, 65535, 0, 2, 65533, 65535, 0, 2, 65533, + 65535, 0, 5, 65523, 65535, 0, 2, 65533, 65535, 0, + 7, 65526, 65535, 0, 46, 65464, 65533, 65535, 0, 2, + 120, 65309, 65533, 65535, 0, 2, 5, 362, 65097, 65533, + 65535, 0, 2, 18, 1164, 64785, 65528, 65531, 65533, 65535, + 0, 65535, 0, 65535, 0, 65535, 0, 65533, 65535, 0, + 65535, 0, 65533, 65535, 0, 2, 65533, 65535, 0, 2, + 65533, 65535, 0, 2, 65533, 65535, 0, 2, 65530, 65535, + 0, 2, 65523, 65535, 0, 69, 65477, 65535, 0, 141, + 65459, 65535, 0, 194, 65325, 65533, 65535, 0, 2, 543, + 64912, 65533, 65535, 0, 5, 1270, 64301, 65529, 65531, 65533, + 65535, 0, 2, 4, 12, 2055, 63538, 65508, 65531, 65533, + 65535, 0, 2, 7, 102, 3775, 61970, 65429, 65526, 65528, + 65533, 65535, 0, 65535, 0, 65535, 0, 65535, 0, 2, + 65533, 65535, 0, 2, 65535, 0, 9, 65533, 65535, 0, + 25, 65512, 65535, 0, 2, 65533, 65535, 0, 44, 65480, + 65535, 0, 48, 65475, 65535, 0, 162, 65373, 65535, 0, + 637, 64806, 65533, 65535, 0, 2, 935, 64445, 65533, 65535, + 0, 2, 4, 1662, 64083, 65533, 65535, 0, 2, 12, + 3036, 62469, 65521, 65533, 65535, 0, 2, 120, 5405, 60468, + 65469, 65531, 65533, 65535, 0, 2, 4, 18, 254, 6663, + 58999, 65272, 65528, 65533, 65535, 0, 2, 4, 9, 12, + 67, 591, 8981, 56781, 64564, 65365, 65508, 65524, 65526, 65529, + 65531, 65533, 65535, 0, 65535, 0, 65535, 0, 2, 65533, + 65535, 0, 9, 65526, 65535, 0, 14, 65503, 65535, 0, + 127, 65390, 65535, 0, 517, 64990, 65535, 0, 178, 65330, + 65535, 0, 2, 1055, 64533, 65533, 65535, 0, 2, 1558, + 63942, 65533, 65535, 0, 2, 2205, 63173, 65533, 65535, 0, + 25, 4493, 60862, 65505, 65533, 65535, 0, 2, 48, 5890, + 59442, 65482, 65533, 65535, 0, 2, 4, 127, 7532, 58191, + 65394, 65533, 65535, 0, 2, 5, 32, 550, 10388, 54924, + 65046, 65510, 65531, 65533, 65535, 0, 2, 4, 30, 150, + 1685, 14340, 51375, 63619, 65288, 65503, 65528, 65533, 65535, 0, + 2, 4, 6, 8, 28, 97, 473, 2692, 15407, 50020, + 62880, 65064, 65445, 65508, 65531, 65533, 65535, 0, 2, 4, + 12, 32, 79, 150, 372, 907, 2184, 5868, 18207, 45431, + 59856, 64031, 65096, 65401, 65481, 65507, 65521, 65523, 65525, 65527, + 65529, 65531, 65533, 65535, 0, 65533, 65535, 0, 182, 65491, + 65535, 0, 877, 64286, 65535, 0, 9, 2708, 63612, 65533, + 65535, 0, 2, 6038, 59532, 65535, 0, 2, 92, 5500, + 60539, 65533, 65535, 0, 268, 8908, 56512, 65385, 65535, 0, + 129, 13110, 52742, 65036, 65535, 0, 2, 806, 14003, 51929, + 64732, 65523, 65535, 0, 7, 92, 2667, 18159, 47678, 62610, + 65355, 65535, 0, 32, 1836, 19676, 48237, 61677, 64960, 65526, + 65535, 0, 21, 159, 967, 5668, 22782, 44709, 58317, 64020, + 65406, 65528, 65535, 0, 7, 162, 1838, 8328, 23929, 43014, + 56394, 63374, 65216, 65484, 65521, 65535, 0, 2, 4, 6, + 28, 268, 1120, 3613, 10688, 24185, 40989, 54917, 61684, 64510, + 65403, 65530, 65535, 0, 2, 16, 44, 139, 492, 1739, + 5313, 13558, 26766, 41566, 52446, 58937, 62815, 64480, 65201, 65454, + 65524, 65533, 65535, 0, 7, 25, 76, 263, 612, 1466, + 3325, 6832, 12366, 20152, 29466, 39255, 47360, 53506, 57740, 60726, + 62845, 64131, 64882, 65260, 65459, 65521, 65528, 65530, 65535, 0, + 2, 4, 14, 48, 136, 312, 653, 1240, 2369, 4327, + 7028, 10759, 15449, 21235, 28027, 35386, 42938, 49562, 54990, 59119, + 62086, 63916, 64863, 65249, 65445, 65493, 65523, 65535, 0, 2, + 4, 6, 8, 10, 12, 21, 83, 208, 409, 723, + 1152, 1868, 2951, 4463, 6460, 8979, 11831, 15195, 18863, 22657, + 26762, 30881, 34963, 39098, 43054, 47069, 50620, 53871, 56821, 59386, + 61340, 62670, 63512, 64023, 64429, 64750, 64944, 65126, 65279, 65366, + 65413, 65445, 65473, 65505, 65510, 65521, 65528, 65530, 65535 +}; + +/* pointers to cdf tables for quantizer indices */ +const uint16_t *WebRtcIsacfix_kCdfGainPtr[3][12] = { + { WebRtcIsacfix_kCdfGain +0 +0, WebRtcIsacfix_kCdfGain +0 +8, WebRtcIsacfix_kCdfGain +0 +22, + WebRtcIsacfix_kCdfGain +0 +32, WebRtcIsacfix_kCdfGain +0 +48, WebRtcIsacfix_kCdfGain +0 +60, + WebRtcIsacfix_kCdfGain +0 +81, WebRtcIsacfix_kCdfGain +0 +95, WebRtcIsacfix_kCdfGain +0 +128, + WebRtcIsacfix_kCdfGain +0 +152, WebRtcIsacfix_kCdfGain +0 +210, WebRtcIsacfix_kCdfGain +0 +264 + }, + { WebRtcIsacfix_kCdfGain +404 +0, WebRtcIsacfix_kCdfGain +404 +8, WebRtcIsacfix_kCdfGain +404 +21, + WebRtcIsacfix_kCdfGain +404 +30, WebRtcIsacfix_kCdfGain +404 +46, WebRtcIsacfix_kCdfGain +404 +58, + WebRtcIsacfix_kCdfGain +404 +79, WebRtcIsacfix_kCdfGain +404 +93, WebRtcIsacfix_kCdfGain +404 +125, + WebRtcIsacfix_kCdfGain +404 +149, WebRtcIsacfix_kCdfGain +404 +207, WebRtcIsacfix_kCdfGain +404 +260 + }, + { WebRtcIsacfix_kCdfGain +803 +0, WebRtcIsacfix_kCdfGain +803 +8, WebRtcIsacfix_kCdfGain +803 +22, + WebRtcIsacfix_kCdfGain +803 +31, WebRtcIsacfix_kCdfGain +803 +48, WebRtcIsacfix_kCdfGain +803 +60, + WebRtcIsacfix_kCdfGain +803 +81, WebRtcIsacfix_kCdfGain +803 +96, WebRtcIsacfix_kCdfGain +803 +129, + WebRtcIsacfix_kCdfGain +803 +154, WebRtcIsacfix_kCdfGain +803 +212, WebRtcIsacfix_kCdfGain +803 +268 + } +}; + +const uint16_t *WebRtcIsacfix_kCdfShapePtr[3][108] = { + { WebRtcIsacfix_kCdfShape +0 +0, WebRtcIsacfix_kCdfShape +0 +2, WebRtcIsacfix_kCdfShape +0 +4, + WebRtcIsacfix_kCdfShape +0 +6, WebRtcIsacfix_kCdfShape +0 +8, WebRtcIsacfix_kCdfShape +0 +10, + WebRtcIsacfix_kCdfShape +0 +12, WebRtcIsacfix_kCdfShape +0 +14, WebRtcIsacfix_kCdfShape +0 +16, + WebRtcIsacfix_kCdfShape +0 +18, WebRtcIsacfix_kCdfShape +0 +21, WebRtcIsacfix_kCdfShape +0 +25, + WebRtcIsacfix_kCdfShape +0 +29, WebRtcIsacfix_kCdfShape +0 +33, WebRtcIsacfix_kCdfShape +0 +37, + WebRtcIsacfix_kCdfShape +0 +43, WebRtcIsacfix_kCdfShape +0 +49, WebRtcIsacfix_kCdfShape +0 +56, + WebRtcIsacfix_kCdfShape +0 +64, WebRtcIsacfix_kCdfShape +0 +66, WebRtcIsacfix_kCdfShape +0 +68, + WebRtcIsacfix_kCdfShape +0 +70, WebRtcIsacfix_kCdfShape +0 +72, WebRtcIsacfix_kCdfShape +0 +75, + WebRtcIsacfix_kCdfShape +0 +77, WebRtcIsacfix_kCdfShape +0 +79, WebRtcIsacfix_kCdfShape +0 +81, + WebRtcIsacfix_kCdfShape +0 +83, WebRtcIsacfix_kCdfShape +0 +86, WebRtcIsacfix_kCdfShape +0 +90, + WebRtcIsacfix_kCdfShape +0 +94, WebRtcIsacfix_kCdfShape +0 +98, WebRtcIsacfix_kCdfShape +0 +102, + WebRtcIsacfix_kCdfShape +0 +107, WebRtcIsacfix_kCdfShape +0 +113, WebRtcIsacfix_kCdfShape +0 +120, + WebRtcIsacfix_kCdfShape +0 +129, WebRtcIsacfix_kCdfShape +0 +131, WebRtcIsacfix_kCdfShape +0 +133, + WebRtcIsacfix_kCdfShape +0 +135, WebRtcIsacfix_kCdfShape +0 +137, WebRtcIsacfix_kCdfShape +0 +141, + WebRtcIsacfix_kCdfShape +0 +143, WebRtcIsacfix_kCdfShape +0 +147, WebRtcIsacfix_kCdfShape +0 +151, + WebRtcIsacfix_kCdfShape +0 +155, WebRtcIsacfix_kCdfShape +0 +159, WebRtcIsacfix_kCdfShape +0 +164, + WebRtcIsacfix_kCdfShape +0 +168, WebRtcIsacfix_kCdfShape +0 +172, WebRtcIsacfix_kCdfShape +0 +178, + WebRtcIsacfix_kCdfShape +0 +184, WebRtcIsacfix_kCdfShape +0 +192, WebRtcIsacfix_kCdfShape +0 +200, + WebRtcIsacfix_kCdfShape +0 +211, WebRtcIsacfix_kCdfShape +0 +213, WebRtcIsacfix_kCdfShape +0 +215, + WebRtcIsacfix_kCdfShape +0 +217, WebRtcIsacfix_kCdfShape +0 +219, WebRtcIsacfix_kCdfShape +0 +223, + WebRtcIsacfix_kCdfShape +0 +227, WebRtcIsacfix_kCdfShape +0 +231, WebRtcIsacfix_kCdfShape +0 +235, + WebRtcIsacfix_kCdfShape +0 +239, WebRtcIsacfix_kCdfShape +0 +243, WebRtcIsacfix_kCdfShape +0 +248, + WebRtcIsacfix_kCdfShape +0 +252, WebRtcIsacfix_kCdfShape +0 +258, WebRtcIsacfix_kCdfShape +0 +264, + WebRtcIsacfix_kCdfShape +0 +273, WebRtcIsacfix_kCdfShape +0 +282, WebRtcIsacfix_kCdfShape +0 +293, + WebRtcIsacfix_kCdfShape +0 +308, WebRtcIsacfix_kCdfShape +0 +310, WebRtcIsacfix_kCdfShape +0 +312, + WebRtcIsacfix_kCdfShape +0 +316, WebRtcIsacfix_kCdfShape +0 +320, WebRtcIsacfix_kCdfShape +0 +324, + WebRtcIsacfix_kCdfShape +0 +328, WebRtcIsacfix_kCdfShape +0 +332, WebRtcIsacfix_kCdfShape +0 +336, + WebRtcIsacfix_kCdfShape +0 +341, WebRtcIsacfix_kCdfShape +0 +347, WebRtcIsacfix_kCdfShape +0 +354, + WebRtcIsacfix_kCdfShape +0 +360, WebRtcIsacfix_kCdfShape +0 +368, WebRtcIsacfix_kCdfShape +0 +378, + WebRtcIsacfix_kCdfShape +0 +388, WebRtcIsacfix_kCdfShape +0 +400, WebRtcIsacfix_kCdfShape +0 +418, + WebRtcIsacfix_kCdfShape +0 +445, WebRtcIsacfix_kCdfShape +0 +447, WebRtcIsacfix_kCdfShape +0 +451, + WebRtcIsacfix_kCdfShape +0 +455, WebRtcIsacfix_kCdfShape +0 +461, WebRtcIsacfix_kCdfShape +0 +468, + WebRtcIsacfix_kCdfShape +0 +474, WebRtcIsacfix_kCdfShape +0 +480, WebRtcIsacfix_kCdfShape +0 +486, + WebRtcIsacfix_kCdfShape +0 +495, WebRtcIsacfix_kCdfShape +0 +505, WebRtcIsacfix_kCdfShape +0 +516, + WebRtcIsacfix_kCdfShape +0 +528, WebRtcIsacfix_kCdfShape +0 +543, WebRtcIsacfix_kCdfShape +0 +564, + WebRtcIsacfix_kCdfShape +0 +583, WebRtcIsacfix_kCdfShape +0 +608, WebRtcIsacfix_kCdfShape +0 +635 + }, + { WebRtcIsacfix_kCdfShape +686 +0, WebRtcIsacfix_kCdfShape +686 +2, WebRtcIsacfix_kCdfShape +686 +4, + WebRtcIsacfix_kCdfShape +686 +6, WebRtcIsacfix_kCdfShape +686 +8, WebRtcIsacfix_kCdfShape +686 +11, + WebRtcIsacfix_kCdfShape +686 +13, WebRtcIsacfix_kCdfShape +686 +15, WebRtcIsacfix_kCdfShape +686 +17, + WebRtcIsacfix_kCdfShape +686 +20, WebRtcIsacfix_kCdfShape +686 +23, WebRtcIsacfix_kCdfShape +686 +27, + WebRtcIsacfix_kCdfShape +686 +31, WebRtcIsacfix_kCdfShape +686 +35, WebRtcIsacfix_kCdfShape +686 +40, + WebRtcIsacfix_kCdfShape +686 +44, WebRtcIsacfix_kCdfShape +686 +50, WebRtcIsacfix_kCdfShape +686 +56, + WebRtcIsacfix_kCdfShape +686 +63, WebRtcIsacfix_kCdfShape +686 +65, WebRtcIsacfix_kCdfShape +686 +67, + WebRtcIsacfix_kCdfShape +686 +69, WebRtcIsacfix_kCdfShape +686 +71, WebRtcIsacfix_kCdfShape +686 +73, + WebRtcIsacfix_kCdfShape +686 +75, WebRtcIsacfix_kCdfShape +686 +77, WebRtcIsacfix_kCdfShape +686 +79, + WebRtcIsacfix_kCdfShape +686 +82, WebRtcIsacfix_kCdfShape +686 +85, WebRtcIsacfix_kCdfShape +686 +89, + WebRtcIsacfix_kCdfShape +686 +93, WebRtcIsacfix_kCdfShape +686 +97, WebRtcIsacfix_kCdfShape +686 +102, + WebRtcIsacfix_kCdfShape +686 +106, WebRtcIsacfix_kCdfShape +686 +112, WebRtcIsacfix_kCdfShape +686 +119, + WebRtcIsacfix_kCdfShape +686 +127, WebRtcIsacfix_kCdfShape +686 +129, WebRtcIsacfix_kCdfShape +686 +131, + WebRtcIsacfix_kCdfShape +686 +133, WebRtcIsacfix_kCdfShape +686 +135, WebRtcIsacfix_kCdfShape +686 +137, + WebRtcIsacfix_kCdfShape +686 +139, WebRtcIsacfix_kCdfShape +686 +142, WebRtcIsacfix_kCdfShape +686 +146, + WebRtcIsacfix_kCdfShape +686 +150, WebRtcIsacfix_kCdfShape +686 +154, WebRtcIsacfix_kCdfShape +686 +158, + WebRtcIsacfix_kCdfShape +686 +162, WebRtcIsacfix_kCdfShape +686 +167, WebRtcIsacfix_kCdfShape +686 +173, + WebRtcIsacfix_kCdfShape +686 +179, WebRtcIsacfix_kCdfShape +686 +186, WebRtcIsacfix_kCdfShape +686 +194, + WebRtcIsacfix_kCdfShape +686 +205, WebRtcIsacfix_kCdfShape +686 +207, WebRtcIsacfix_kCdfShape +686 +209, + WebRtcIsacfix_kCdfShape +686 +211, WebRtcIsacfix_kCdfShape +686 +214, WebRtcIsacfix_kCdfShape +686 +218, + WebRtcIsacfix_kCdfShape +686 +222, WebRtcIsacfix_kCdfShape +686 +226, WebRtcIsacfix_kCdfShape +686 +230, + WebRtcIsacfix_kCdfShape +686 +234, WebRtcIsacfix_kCdfShape +686 +238, WebRtcIsacfix_kCdfShape +686 +242, + WebRtcIsacfix_kCdfShape +686 +247, WebRtcIsacfix_kCdfShape +686 +253, WebRtcIsacfix_kCdfShape +686 +262, + WebRtcIsacfix_kCdfShape +686 +269, WebRtcIsacfix_kCdfShape +686 +278, WebRtcIsacfix_kCdfShape +686 +289, + WebRtcIsacfix_kCdfShape +686 +305, WebRtcIsacfix_kCdfShape +686 +307, WebRtcIsacfix_kCdfShape +686 +309, + WebRtcIsacfix_kCdfShape +686 +311, WebRtcIsacfix_kCdfShape +686 +315, WebRtcIsacfix_kCdfShape +686 +319, + WebRtcIsacfix_kCdfShape +686 +323, WebRtcIsacfix_kCdfShape +686 +327, WebRtcIsacfix_kCdfShape +686 +331, + WebRtcIsacfix_kCdfShape +686 +335, WebRtcIsacfix_kCdfShape +686 +340, WebRtcIsacfix_kCdfShape +686 +346, + WebRtcIsacfix_kCdfShape +686 +354, WebRtcIsacfix_kCdfShape +686 +362, WebRtcIsacfix_kCdfShape +686 +374, + WebRtcIsacfix_kCdfShape +686 +384, WebRtcIsacfix_kCdfShape +686 +396, WebRtcIsacfix_kCdfShape +686 +413, + WebRtcIsacfix_kCdfShape +686 +439, WebRtcIsacfix_kCdfShape +686 +442, WebRtcIsacfix_kCdfShape +686 +446, + WebRtcIsacfix_kCdfShape +686 +450, WebRtcIsacfix_kCdfShape +686 +455, WebRtcIsacfix_kCdfShape +686 +461, + WebRtcIsacfix_kCdfShape +686 +468, WebRtcIsacfix_kCdfShape +686 +475, WebRtcIsacfix_kCdfShape +686 +481, + WebRtcIsacfix_kCdfShape +686 +489, WebRtcIsacfix_kCdfShape +686 +498, WebRtcIsacfix_kCdfShape +686 +508, + WebRtcIsacfix_kCdfShape +686 +522, WebRtcIsacfix_kCdfShape +686 +534, WebRtcIsacfix_kCdfShape +686 +554, + WebRtcIsacfix_kCdfShape +686 +577, WebRtcIsacfix_kCdfShape +686 +602, WebRtcIsacfix_kCdfShape +686 +631 + }, + { WebRtcIsacfix_kCdfShape +1368 +0, WebRtcIsacfix_kCdfShape +1368 +2, WebRtcIsacfix_kCdfShape +1368 +4, + WebRtcIsacfix_kCdfShape +1368 +6, WebRtcIsacfix_kCdfShape +1368 +8, WebRtcIsacfix_kCdfShape +1368 +10, + WebRtcIsacfix_kCdfShape +1368 +12, WebRtcIsacfix_kCdfShape +1368 +14, WebRtcIsacfix_kCdfShape +1368 +16, + WebRtcIsacfix_kCdfShape +1368 +20, WebRtcIsacfix_kCdfShape +1368 +24, WebRtcIsacfix_kCdfShape +1368 +28, + WebRtcIsacfix_kCdfShape +1368 +32, WebRtcIsacfix_kCdfShape +1368 +36, WebRtcIsacfix_kCdfShape +1368 +40, + WebRtcIsacfix_kCdfShape +1368 +44, WebRtcIsacfix_kCdfShape +1368 +50, WebRtcIsacfix_kCdfShape +1368 +57, + WebRtcIsacfix_kCdfShape +1368 +65, WebRtcIsacfix_kCdfShape +1368 +67, WebRtcIsacfix_kCdfShape +1368 +69, + WebRtcIsacfix_kCdfShape +1368 +71, WebRtcIsacfix_kCdfShape +1368 +73, WebRtcIsacfix_kCdfShape +1368 +75, + WebRtcIsacfix_kCdfShape +1368 +77, WebRtcIsacfix_kCdfShape +1368 +79, WebRtcIsacfix_kCdfShape +1368 +81, + WebRtcIsacfix_kCdfShape +1368 +85, WebRtcIsacfix_kCdfShape +1368 +89, WebRtcIsacfix_kCdfShape +1368 +93, + WebRtcIsacfix_kCdfShape +1368 +97, WebRtcIsacfix_kCdfShape +1368 +101, WebRtcIsacfix_kCdfShape +1368 +105, + WebRtcIsacfix_kCdfShape +1368 +110, WebRtcIsacfix_kCdfShape +1368 +116, WebRtcIsacfix_kCdfShape +1368 +123, + WebRtcIsacfix_kCdfShape +1368 +132, WebRtcIsacfix_kCdfShape +1368 +134, WebRtcIsacfix_kCdfShape +1368 +136, + WebRtcIsacfix_kCdfShape +1368 +138, WebRtcIsacfix_kCdfShape +1368 +141, WebRtcIsacfix_kCdfShape +1368 +143, + WebRtcIsacfix_kCdfShape +1368 +146, WebRtcIsacfix_kCdfShape +1368 +150, WebRtcIsacfix_kCdfShape +1368 +154, + WebRtcIsacfix_kCdfShape +1368 +158, WebRtcIsacfix_kCdfShape +1368 +162, WebRtcIsacfix_kCdfShape +1368 +166, + WebRtcIsacfix_kCdfShape +1368 +170, WebRtcIsacfix_kCdfShape +1368 +174, WebRtcIsacfix_kCdfShape +1368 +179, + WebRtcIsacfix_kCdfShape +1368 +185, WebRtcIsacfix_kCdfShape +1368 +193, WebRtcIsacfix_kCdfShape +1368 +203, + WebRtcIsacfix_kCdfShape +1368 +214, WebRtcIsacfix_kCdfShape +1368 +216, WebRtcIsacfix_kCdfShape +1368 +218, + WebRtcIsacfix_kCdfShape +1368 +220, WebRtcIsacfix_kCdfShape +1368 +224, WebRtcIsacfix_kCdfShape +1368 +227, + WebRtcIsacfix_kCdfShape +1368 +231, WebRtcIsacfix_kCdfShape +1368 +235, WebRtcIsacfix_kCdfShape +1368 +239, + WebRtcIsacfix_kCdfShape +1368 +243, WebRtcIsacfix_kCdfShape +1368 +247, WebRtcIsacfix_kCdfShape +1368 +251, + WebRtcIsacfix_kCdfShape +1368 +256, WebRtcIsacfix_kCdfShape +1368 +262, WebRtcIsacfix_kCdfShape +1368 +269, + WebRtcIsacfix_kCdfShape +1368 +277, WebRtcIsacfix_kCdfShape +1368 +286, WebRtcIsacfix_kCdfShape +1368 +297, + WebRtcIsacfix_kCdfShape +1368 +315, WebRtcIsacfix_kCdfShape +1368 +317, WebRtcIsacfix_kCdfShape +1368 +319, + WebRtcIsacfix_kCdfShape +1368 +323, WebRtcIsacfix_kCdfShape +1368 +327, WebRtcIsacfix_kCdfShape +1368 +331, + WebRtcIsacfix_kCdfShape +1368 +335, WebRtcIsacfix_kCdfShape +1368 +339, WebRtcIsacfix_kCdfShape +1368 +343, + WebRtcIsacfix_kCdfShape +1368 +349, WebRtcIsacfix_kCdfShape +1368 +355, WebRtcIsacfix_kCdfShape +1368 +361, + WebRtcIsacfix_kCdfShape +1368 +368, WebRtcIsacfix_kCdfShape +1368 +376, WebRtcIsacfix_kCdfShape +1368 +385, + WebRtcIsacfix_kCdfShape +1368 +397, WebRtcIsacfix_kCdfShape +1368 +411, WebRtcIsacfix_kCdfShape +1368 +429, + WebRtcIsacfix_kCdfShape +1368 +456, WebRtcIsacfix_kCdfShape +1368 +459, WebRtcIsacfix_kCdfShape +1368 +463, + WebRtcIsacfix_kCdfShape +1368 +467, WebRtcIsacfix_kCdfShape +1368 +473, WebRtcIsacfix_kCdfShape +1368 +478, + WebRtcIsacfix_kCdfShape +1368 +485, WebRtcIsacfix_kCdfShape +1368 +491, WebRtcIsacfix_kCdfShape +1368 +497, + WebRtcIsacfix_kCdfShape +1368 +505, WebRtcIsacfix_kCdfShape +1368 +514, WebRtcIsacfix_kCdfShape +1368 +523, + WebRtcIsacfix_kCdfShape +1368 +535, WebRtcIsacfix_kCdfShape +1368 +548, WebRtcIsacfix_kCdfShape +1368 +565, + WebRtcIsacfix_kCdfShape +1368 +585, WebRtcIsacfix_kCdfShape +1368 +611, WebRtcIsacfix_kCdfShape +1368 +640 + } +}; + +/* code length for all coefficients using different models */ + +const int16_t WebRtcIsacfix_kCodeLenGainQ11[392] = { + 25189, 16036, 8717, 358, 8757, 15706, 21456, 24397, 18502, 17559 + , 13794, 11088, 7480, 873, 6603, 11636, 14627, 16805, 19132, 26624 + , 26624, 19408, 13751, 7280, 583, 7591, 15178, 23773, 28672, 25189 + , 19045, 16442, 13412, 10397, 5893, 1338, 6376, 9992, 12074, 13853 + , 15781, 19821, 22819, 28672, 28672, 25189, 19858, 15781, 11262, 5477 + , 1298, 5632, 11814, 17234, 22020, 28672, 19677, 18125, 16587, 14521 + , 13032, 11196, 9249, 5411, 2495, 4994, 7975, 10234, 12308, 13892 + , 15148, 17944, 21725, 23917, 25189, 19539, 16293, 11531, 7808, 4475 + , 2739, 4872, 8089, 11314, 14992, 18105, 23257, 26624, 25189, 23257 + , 23257, 20982, 18697, 18023, 16338, 16036, 14539, 13695, 13146, 11763 + , 10754, 9074, 7260, 5584, 4430, 5553, 6848, 8344, 10141, 11636 + , 12535, 13416, 14342, 15477, 17296, 19282, 22349, 23773, 28672, 28672 + , 26624, 23773, 21456, 18023, 15118, 13362, 11212, 9293, 8043, 6985 + , 5908, 5721, 5853, 6518, 7316, 8360, 9716, 11289, 12912, 14652 + , 16969, 19858, 23773, 26624, 28013, 30720, 30720, 28672, 25426, 23141 + , 25426, 23773, 20720, 19408, 18697, 19282, 16859, 16338, 16026, 15377 + , 15021, 14319, 14251, 13937, 13260, 13017, 12332, 11703, 11430, 10359 + , 10128, 9405, 8757, 8223, 7974, 7859, 7646, 7673, 7997, 8580 + , 8880, 9061, 9866, 10397, 11358, 12200, 13244, 14157, 15021, 16026 + , 16490, 18697, 18479, 20011, 19677, 20720, 24576, 26276, 30720, 30720 + , 28672, 30720, 24068, 25189, 22437, 20345, 18479, 16396, 16026, 14928 + , 13877, 13450, 12696, 12766, 11626, 11098, 10159, 9998, 9437, 9275 + , 8783, 8552, 8629, 8488, 8522, 8454, 8571, 8775, 8915, 9427 + , 9483, 9851, 10260, 10933, 11131, 11974, 12560, 13833, 15080, 16304 + , 17491, 19017, 18697, 19408, 22020, 25189, 25426, 22819, 26276, 30720 + , 30720, 30720, 30720, 30720, 30720, 28672, 30720, 30720, 30720, 30720 + , 28013, 25426, 24397, 23773, 25189, 26624, 25189, 22437, 21725, 20011 + , 20527, 20720, 20771, 22020, 22020, 19858, 19408, 19972, 17866, 17360 + , 17791, 17219, 16805, 16927, 16067, 16162, 15661, 15178, 15021, 15209 + , 14845, 14570, 14490, 14490, 13733, 13617, 13794, 13577, 13312, 12824 + , 13032, 12683, 12189, 12469, 12109, 11940, 11636, 11617, 11932, 12294 + , 11578, 11775, 12039, 11654, 11560, 11439, 11909, 11421, 12029, 11513 + , 11773, 11899, 11560, 11805, 11476, 11664, 11963, 11647, 11754, 11963 + , 11703, 12211, 11932, 12074, 12469, 12535, 12560, 12912, 12783, 12866 + , 12884, 13378, 13957, 13775, 13635, 14019, 14545, 15240, 15520, 15554 + , 15697, 16490, 16396, 17281, 16599, 16969, 17963, 16859, 16983, 16805 + , 17099, 18210, 17219, 17646, 17700, 17646, 18297, 17425, 18479, 17791 + , 17718, 19282, 18672, 20173, 20982, 21725, 21456, 23773, 23257, 25189 + , 30720, 30720, 25189, 26624, 30720, 30720, 30720, 30720, 28672, 26276 + , 30720, 30720 +}; + +const int16_t WebRtcIsacfix_kCodeLenShapeQ11[577] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 28672 + , 0, 26624, 1, 23773, 22819, 4, 20982, 18598, 10, 19282 + , 16587, 22, 16442, 26624, 13126, 60, 14245, 26624, 26624, 12736 + , 79, 12912, 25189, 22819, 9563, 249, 9474, 22349, 28672, 23257 + , 17944, 7980, 434, 8181, 16431, 26624, 0, 0, 0, 0 + , 28672, 0, 0, 0, 0, 0, 28672, 0, 22437, 3 + , 22437, 20982, 5, 20982, 16442, 22, 16752, 13814, 49, 14646 + , 11645, 116, 11734, 26624, 28672, 10613, 158, 11010, 24397, 19539 + , 8046, 453, 7709, 19017, 28672, 23257, 15110, 6770, 758, 6523 + , 14108, 24397, 28672, 0, 0, 0, 0, 28672, 0, 28672 + , 0, 26624, 1, 28672, 28672, 1, 26624, 24397, 2, 23257 + , 21725, 4, 20982, 17158, 18, 17281, 28672, 15178, 35, 15209 + , 12343, 92, 12320, 26624, 10344, 189, 10217, 30720, 22020, 9033 + , 322, 8549, 23773, 28672, 30720, 20622, 7666, 473, 7806, 20527 + , 24397, 14135, 5995, 960, 6018, 14872, 23773, 26624, 20928, 16293 + , 10636, 4926, 1588, 5256, 11088, 18043, 25189, 0, 0, 0 + , 0, 24397, 1, 25189, 20720, 5, 21456, 21209, 3, 25189 + , 20982, 5, 21456, 15818, 30, 15410, 13794, 60, 13416, 28672 + , 11162, 142, 11025, 9337, 231, 10094, 23773, 8338, 405, 7930 + , 26624, 19677, 6787, 613, 7318, 19161, 28672, 16442, 6319, 932 + , 5748, 15312, 25189, 28672, 28672, 28672, 13998, 5513, 1263, 5146 + , 14024, 24397, 22819, 15818, 9460, 4447, 2122, 4681, 9970, 15945 + , 22349, 28672, 30720, 22622, 19017, 14872, 10689, 7405, 4473, 2983 + , 4783, 7894, 11186, 14964, 18210, 24397, 0, 0, 30720, 0 + , 30720, 21456, 3, 23773, 14964, 39, 14757, 14179, 53, 13751 + , 14928, 36, 15272, 12430, 79, 13228, 9135, 285, 9077, 28672 + , 28672, 8377, 403, 7919, 26624, 28672, 23257, 7068, 560, 7473 + , 20345, 19677, 6770, 720, 6464, 18697, 25189, 16249, 5779, 1087 + , 5494, 15209, 22819, 30720, 20622, 12601, 5240, 1419, 5091, 12095 + , 19408, 26624, 22819, 16805, 10683, 4812, 2056, 4293, 9836, 16026 + , 24397, 25189, 18409, 13833, 8681, 4503, 2653, 4220, 8329, 13853 + , 19132, 26624, 25189, 20771, 17219, 12630, 9520, 6733, 4565, 3657 + , 4817, 7069, 10058, 13212, 16805, 21209, 26624, 26276, 28672, 28672 + , 26276, 23257, 20173, 19282, 16538, 15051, 12811, 10754, 9267, 7547 + , 6270, 5407, 5214, 6057, 7054, 8226, 9488, 10806, 12793, 14442 + , 16442, 19677, 22099, 26276, 28672, 0, 30720, 0, 30720, 11920 + , 56, 20720, 30720, 6766, 355, 13130, 30720, 30720, 22180, 5589 + , 736, 7902, 26624, 30720, 7634, 354, 9721, 30720, 30720, 9027 + , 246, 10117, 30720, 30720, 9630, 453, 6709, 23257, 30720, 25683 + , 14228, 6127, 1271, 4615, 15178, 30720, 30720, 23504, 12382, 5739 + , 2015, 3492, 10560, 22020, 26624, 30720, 30720, 23257, 13192, 4873 + , 1527, 5001, 12445, 22020, 30720, 30720, 30720, 30720, 19344, 10761 + , 4051, 1927, 5281, 10594, 17866, 28672, 30720, 30720, 30720, 21869 + , 15554, 10060, 5979, 2710, 3085, 7889, 14646, 21725, 28672, 30720 + , 30720, 30720, 30720, 30720, 30720, 30720, 22719, 17425, 13212, 8083 + , 4439, 2820, 4305, 8136, 12988, 17425, 21151, 28672, 28672, 30720 + , 30720, 30720, 28672, 20527, 19282, 14412, 10513, 7407, 5079, 3744 + , 4115, 6308, 9621, 13599, 17040, 22349, 28672, 30720, 30720, 30720 + , 30720, 30720, 30720, 29522, 19282, 14545, 11485, 9093, 6760, 5262 + , 4672, 4970, 6005, 7852, 9732, 12343, 14672, 19161, 22819, 25189 + , 30720, 30720, 28672, 30720, 30720, 20720, 18125, 14388, 12007, 9825 + , 8092, 7064, 6069, 5903, 5932, 6359, 7169, 8310, 9324, 10711 + , 11867, 13096, 14157, 16338, 17040, 19161, 21725, 23773, 30720, 30720 + , 26276, 25426, 24397, 28672, 28672, 23257, 22020, 22349, 18297, 17646 + , 16983, 16431, 16162, 15021, 15178, 13751, 12142, 10895, 10193, 9632 + , 9086, 8896, 8823, 8735, 8591, 8754, 8649, 8361, 8329, 8522 + , 8373, 8739, 8993, 9657, 10454, 11279, 11899, 12614, 14024, 14273 + , 15477, 15240, 16649, 17866, 18697, 21151, 22099 +}; + +/* left KLT transforms */ +const int16_t WebRtcIsacfix_kT1GainQ15[3][4] = { + { -26130, 19773, 19773, 26130 }, + { -26664, 19046, 19046, 26664 }, + { -23538, 22797, 22797, 23538 } +}; + + + +const int16_t WebRtcIsacfix_kT1ShapeQ15[3][324] = { + { 52,16,168,7,439,-138,-89,306,671,882, + 157,1301,291,1598,-3571,-1943,-1119,32404,96,-12, + 379,-64,-307,345,-836,539,1045,2541,-2865,-992, + 1683,-4717,5808,7427,30599,2319,183,-73,451,481, + 933,-198,781,-397,1244,-777,3690,-2414,149,-1356, + -2593,-31140,8289,-1737,-202,-14,-214,360,501,450, + -245,-7,797,3638,-2804,3042,-337,22137,-22103,2264, + 6838,-3381,305,172,263,-195,-355,351,179,513, + 2234,3343,5509,7531,19075,-17740,-16836,2244,-629,-1505, + -153,108,124,-324,2694,-124,1492,-850,5347,4285, + 7439,-10229,-22822,-12467,-12891,3645,822,-232,131,13, + 374,565,536,4681,1294,-1935,1926,-5734,-10643,26462, + -12480,-5589,-1038,-2468,964,-704,-247,-106,186,-558, + -4050,3760,2972,2141,-7393,6294,26740,11991,-3251,5461, + 5341,1574,2208,-51,-552,-297,-753,-154,2068,-5371, + 3578,4106,28043,-10533,8041,2353,2389,4609,3410,1906, + 351,-249,18,-15,1117,539,2870,9084,17585,-24528, + -366,-6490,2009,-3170,2942,1116,-232,1672,1065,606, + -399,-388,-518,38,3728,28948,-11936,4543,4104,-4441, + 1545,-4044,1485,622,-68,186,-473,135,-280,125, + -546,-1813,6989,6606,23711,19376,-2636,2870,-4553,-1687, + 878,-375,205,-208,-409,-108,-200,-45,-1670,-337, + 8213,-5524,-2334,5240,-12939,-26205,5937,-1582,-592,-959, + -5374,2449,3400,559,349,-492,668,12379,-27684,3419, + 5117,4415,-297,-8270,-1252,-3490,-1272,-1199,-3159,191, + 630,488,-797,-3071,12912,-27783,-10249,1047,647,619, + 111,-3722,-915,-1055,-502,5,-1384,-306,221,68, + 5219,13173,-26474,-11663,-5626,927,806,-1127,236,-589, + -522,-230,-312,-315,-428,-573,426,192,-11830,-26883, + -14121,-2785,-1429,-109,410,-832,-302,539,-459,104, + 1,-530,-202,-289,153,116,30082,-12944,-671,20, + 649,98,103,215,234,0,280,-51,-169,298, + 31,230,-73,-51 + }, + { -154,-7,-192,61,-739,-389,-947,-162,-60,94, + 511,-716,1520,-1428,4168,-2214,1816,32270,-123,-77, + -199,-99,-42,-588,203,-240,-930,-35,1580,234, + 3206,-5507,-1495,-10946,30000,-2667,-136,-176,-240,-175, + -204,-661,-1796,-1039,-1271,498,3143,734,2663,2699, + -8127,29333,10495,2356,-72,113,-91,118,-2840,-723, + -1733,-1158,-389,-2116,-3054,-3,-5179,8071,29546,6308, + 5657,-3178,-186,-294,-473,-635,1213,-983,-1437,-1715, + -1094,1280,-92,-9573,948,29576,-7060,-5921,2954,1349, + -337,-108,-1099,962,418,-413,-1149,-334,1241,3975, + -6825,26725,-14377,7051,-4772,-1707,2335,2008,-150,570, + 1371,42,-1649,-619,2039,3369,-1225,1583,-2755,-15207, + -27504,-4855,-4304,1495,2733,1324,15,-448,403,353, + 3016,-1242,2338,2673,2064,-7496,-30447,-3686,5833,-1301, + -2455,2122,1519,608,43,-653,773,-3072,912,-1537, + 4505,10284,30237,1549,3200,-691,205,1702,658,1014, + 1499,148,79,-322,-1162,-4639,-813,7536,3204,29109, + -10747,-26,1611,2286,2114,2561,1022,372,348,207, + 1062,-1088,-443,-9849,2381,5671,29097,-7612,-2927,3853, + 194,1155,275,1438,1438,1312,581,888,-784,906, + 112,-11103,25104,14438,-9311,-3068,1210,368,370,-940, + -2434,-1148,1925,392,657,258,-526,1475,-2281,-4265, + -1880,1534,2185,-1472,959,-30934,6306,3114,-4109,1768, + -2612,-703,45,644,2185,2033,5670,7211,19114,-22427, + 6432,5150,-4090,-2694,3860,1245,-596,293,1829,369, + -319,229,-3256,2170,-6374,-26216,-4570,-16053,-5766,-262, + -2006,2873,-1477,147,378,-1544,-344,-544,-985,-481, + 4210,4542,30757,-7291,-4863,1529,-2079,-628,-603,-783, + -408,1646,697,808,-620,-292,181,158,-13313,-29173, + 5984,-1262,859,-1776,-558,-24,-883,-1421,739,210, + -531,-285,131,-160,-246,-56,29345,-13706,-2859,-2966, + -300,-970,-2382,-268,-103,-636,-12,-62,-691,-253, + -147,-127,27,66 + }, + { 55,-212,-198,489,-274,81,682,399,328,-934, + -389,-37,1357,-3632,5276,6581,-9493,-29921,29,-45, + 2,190,172,-15,311,-130,-1085,-25,324,-684, + 3223,-6580,4485,-5280,-29521,9933,82,-320,-530,229, + -705,-533,-414,848,-1842,-4473,1390,-857,6717,-6692, + 4648,29397,576,8339,-68,-85,238,-330,264,-1012, + -381,-203,-3384,-3329,3906,6810,3790,-6250,28312,-8078, + 8089,1565,160,-569,-612,-613,-1063,-1928,-1125,3421, + -7481,-7484,4942,-6984,4330,-25591,-10574,-6982,5682,-1781, + -308,89,178,-1715,-420,-3530,-5776,1219,-8617,-7137, + 7015,4981,24875,12657,-5408,-3356,-785,-1972,326,-858, + -506,-3382,-986,-6258,-2259,4015,-8374,-10482,3127,23826, + -14126,-514,-5417,2178,-2912,-17,-587,80,67,-5881, + -1702,-5351,-4481,398,-10156,-225,20727,-15460,-11603,7752, + 3660,1714,-2001,-359,499,-527,-1225,-7820,-1297,-6326, + -8526,7900,-18328,13311,-17488,-2926,-196,-17,2281,873, + 480,-160,-624,471,780,-8729,1707,-14262,-20647,1721, + 18590,-2206,-1214,-1066,312,-2602,783,-412,-113,49, + -119,1305,-2371,-15132,-1833,-18252,20295,-8316,2227,341, + -2074,-702,3082,-262,-465,-198,430,30,-70,-788, + 2342,-25132,-4863,19783,-484,2137,2811,-1906,799,1586, + 962,-734,-191,-30,-129,-93,-1126,1729,5860,-2030, + 8953,603,-3338,-10869,-1144,22070,12130,10513,3191,-6881, + -3514,2090,711,-666,1843,-5997,-5681,2921,-17641,-2801, + 4969,18590,7169,12214,8587,4405,3008,-1074,-371,-77, + 253,331,-5611,5014,13152,-1985,18483,-1696,8043,20463, + 2381,-393,1688,-1205,618,1220,457,248,-83,176, + 7920,-13676,-22139,-3038,17402,2036,844,3258,994,719, + 2087,-44,426,494,12,-91,46,5,-14204,22912, + -18156,-361,442,2298,-829,2229,386,1433,1335,1323, + 55,-592,-139,49,-12,-57,27783,17134,350,-282, + 552,158,142,2488,465,329,1087,118,143,10, + 56,65,-15,-31 + } +}; + +/* right KLT transforms */ +const int16_t WebRtcIsacfix_kT2GainQ15[3][36] = { + { 4775, -14892, 20313, -17104, 10533, -3613, -6782, 16044, -8889, + -11019, 21330, -10720, 13193, -15678, -11101, 14461, 12250, -13096, + -16951, 2167, 16066, 15569, -702, -16754, -19195, -12823, -4321, + 5128, 13348, 17825, 13232, 13404, 13494, 13490, 13383, 13261 + }, + { -3725, 11408, -18493, 20031, -13097, 3865, 9344, -19294, 10740, + 8856, -18432, 8982, 13975, -14444, -11930, 11774, 14285, -13594, + -16323, -4, 16340, 15609, 359, -17220, -18401, -13471, -4643, + 5225, 13375, 18053, 13124, 13463, 13621, 13583, 13393, 13072 + }, + { -3513, 11402, -17883, 19504, -14399, 4885, 8702, -19513, 12046, + 8533, -18110, 8447, 12778, -14838, -12444, 13177, 14107, -12759, + -17268, 914, 15822, 15661, 838, -16686, -18907, -12936, -4820, + 4175, 12398, 18830, 12913, 13215, 13433, 13572, 13601, 13518 + } +}; + +const int16_t WebRtcIsacfix_kT2ShapeQ15[3][36] = { + { 4400, -11512, 17205, -19470, 14770, -5345, 9784, -19222, 11228, + 6842, -18371, 9909, 14191, -13496, -11563, 14015, 11827, -14839, + -15439, 948, 17802, 14827, -2053, -17132, 18723, 14516, 4135, + -6822, -13869, -16016, 12975, 13341, 13563, 13603, 13478, 13296 + }, + { 5420, -14215, 19060, -18073, 11709, -3911, 9645, -18335, 7717, + 10842, -19283, 9777, 14898, -12555, -13661, 11668, 13520, -13733, + -15936, -1358, 15671, 16728, 328, -17100, 17527, 13973, 5587, + -5194, -14165, -17677, 12970, 13446, 13693, 13660, 13462, 13015 + }, + { 4386, -12426, 18019, -18895, 13894, -5034, 9713, -19270, 10283, + 8692, -18439, 9317, 13992, -13454, -13241, 12850, 13366, -13336, + -16334, -498, 15976, 16213, -114, -16987, 18191, 13659, 4958, + -5116, -13444, -18021, 12911, 13424, 13718, 13674, 13464, 13054 + } +}; + +/* means of log gains and LAR coefficients*/ +const int16_t WebRtcIsacfix_kMeansGainQ8[3][12] = { + { -1758, -1370, -1758, -1373, -1757, -1375, + -1758, -1374, -1758, -1373, -1755, -1370 + }, + { -1569, -1224, -1569, -1225, -1569, -1227, + -1569, -1226, -1567, -1225, -1565, -1224 + }, + { -1452, -957, -1447, -951, -1438, -944, + -1431, -938, -1419, -931, -1406, -926 + } +}; + + +const int32_t WebRtcIsacfix_kMeansShapeQ17[3][108] = { + { -119581, 34418, -44193, 11112, -4428, 18906, 9222, 8068, 1953, 5425, + 1871, 1689, 109933, 33751, 10471, -2566, 1090, 2320, -119219, 33728, + -43759, 11450, -4870, 19117, 9174, 8037, 1972, 5331, 1872, 1843, + 109899, 34301, 10629, -2316, 1272, 2562, -118608, 32318, -44012, 11591, + -4914, 18932, 9456, 8088, 1900, 5419, 1723, 1853, 109963, 35059, + 10745, -2335, 1161, 2520, -119174, 32107, -44462, 11635, -4694, 18611, + 9757, 8108, 1969, 5486, 1673, 1777, 109636, 34907, 10643, -2406, + 1034, 2420, -118597, 32320, -44590, 10854, -4569, 18821, 9701, 7866, + 2003, 5577, 1732, 1626, 109913, 34448, 10714, -2752, 990, 2228, + -118138, 32996, -44352, 10334, -3772, 18488, 9464, 7865, 2208, 5540, + 1745, 1664, 109880, 33381, 10640, -2779, 980, 2054 + }, + { -146328, 46370, 1047, 26431, 10035, 13933, 6415, 14359, -2368, 6661, + 2269, 1764, 96623, 7802, 4163, 10742, 1643, 2954, -146871, 46561, 1127, + 26225, 10113, 14096, 6771, 14323, -2037, 6788, 2297, 1761, 96324, 8382, + 4309, 10450, 1695, 3016, -146502, 46475, 1580, 26118, 10487, 14179, 6622, + 14439, -2034, 6757, 2342, 1761, 95869, 8966, 4347, 10358, 1999, 2855, + -146958, 47717, 826, 25952, 10263, 14061, 5266, 13681, -2417, 6582, 2047, + 1608, 96257, 9107, 4452, 10301, 1792, 2676, -146992, 47123, 446, 25822, + 10405, 14292, 5140, 13804, -2403, 6496, 1834, 1735, 97489, 9253, 4414, + 10684, 1549, 2721, -145811, 46182, 901, 26482, 10241, 14524, 6075, 14514, + -2147, 6691, 2196, 1899, 97011, 8178, 4102, 10758, 1638, 2869 + }, + { -166617, 46969, -43908, 17726, 6330, 25615, 6913, 5450, -2301, 1984, + 507, 2883, 149998, 28709, 19333, 16703, 11093, 8965, -168254, 46604, + -44315, 17862, 6474, 25746, 7018, 5373, -2343, 1930, 513, 2819, 150391, + 28627, 19194, 16678, 10998, 8929, -169093, 46084, -44767, 17427, 6401, + 25674, 7147, 5472, -2336, 1820, 491, 2802, 149860, 28430, 19064, 16524, + 10898, 8875, -170205, 46189, -44877, 17403, 6190, 25209, 7035, 5673, -2173, + 1894, 574, 2756, 148830, 28230, 18819, 16418, 10789, 8811, -171263, 45045, + -44834, 16858, 6103, 24726, 7014, 5713, -2103, 1877, 518, 2729, 147073, + 27744, 18629, 16277, 10690, 8703, -171720, 44153, -45062, 15951, 5872, + 24429, 7044, 5585, -2082, 1807, 519, 2769, 144791, 27402, 18490, 16126, + 10548, 8635 + } +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.h new file mode 100644 index 0000000000..05c53dd94c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_tables.h + * + * header file for coding tables for the LPC coefficients + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_LPC_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_LPC_TABLES_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/* indices of KLT coefficients used */ +extern const uint16_t WebRtcIsacfix_kSelIndGain[12]; + +extern const uint16_t WebRtcIsacfix_kSelIndShape[108]; + +/* cdf array for model indicator */ +extern const uint16_t WebRtcIsacfix_kModelCdf[KLT_NUM_MODELS+1]; + +/* pointer to cdf array for model indicator */ +extern const uint16_t *WebRtcIsacfix_kModelCdfPtr[1]; + +/* initial cdf index for decoder of model indicator */ +extern const uint16_t WebRtcIsacfix_kModelInitIndex[1]; + +/* offset to go from rounded value to quantization index */ +extern const int16_t WebRtcIsacfix_kQuantMinGain[12]; + +extern const int16_t WebRtcIsacfix_kQuantMinShape[108]; + +/* maximum quantization index */ +extern const uint16_t WebRtcIsacfix_kMaxIndGain[12]; + +extern const uint16_t WebRtcIsacfix_kMaxIndShape[108]; + +/* index offset */ +extern const uint16_t WebRtcIsacfix_kOffsetGain[KLT_NUM_MODELS][12]; + +extern const uint16_t WebRtcIsacfix_kOffsetShape[KLT_NUM_MODELS][108]; + +/* initial cdf index for KLT coefficients */ +extern const uint16_t WebRtcIsacfix_kInitIndexGain[KLT_NUM_MODELS][12]; + +extern const uint16_t WebRtcIsacfix_kInitIndexShape[KLT_NUM_MODELS][108]; + +/* offsets for quantizer representation levels */ +extern const uint16_t WebRtcIsacfix_kOfLevelsGain[3]; + +extern const uint16_t WebRtcIsacfix_kOfLevelsShape[3]; + +/* quantizer representation levels */ +extern const int32_t WebRtcIsacfix_kLevelsGainQ17[1176]; + +extern const int16_t WebRtcIsacfix_kLevelsShapeQ10[1735]; + +/* cdf tables for quantizer indices */ +extern const uint16_t WebRtcIsacfix_kCdfGain[1212]; + +extern const uint16_t WebRtcIsacfix_kCdfShape[2059]; + +/* pointers to cdf tables for quantizer indices */ +extern const uint16_t *WebRtcIsacfix_kCdfGainPtr[KLT_NUM_MODELS][12]; + +extern const uint16_t *WebRtcIsacfix_kCdfShapePtr[KLT_NUM_MODELS][108]; + +/* code length for all coefficients using different models */ +extern const int16_t WebRtcIsacfix_kCodeLenGainQ11[392]; + +extern const int16_t WebRtcIsacfix_kCodeLenShapeQ11[577]; + +/* left KLT transforms */ +extern const int16_t WebRtcIsacfix_kT1GainQ15[KLT_NUM_MODELS][4]; + +extern const int16_t WebRtcIsacfix_kT1ShapeQ15[KLT_NUM_MODELS][324]; + +/* right KLT transforms */ +extern const int16_t WebRtcIsacfix_kT2GainQ15[KLT_NUM_MODELS][36]; + +extern const int16_t WebRtcIsacfix_kT2ShapeQ15[KLT_NUM_MODELS][36]; + +/* means of log gains and LAR coefficients */ +extern const int16_t WebRtcIsacfix_kMeansGainQ8[KLT_NUM_MODELS][12]; + +extern const int32_t WebRtcIsacfix_kMeansShapeQ17[3][108]; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_LPC_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.c new file mode 100644 index 0000000000..78cb93f7ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.c @@ -0,0 +1,435 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "rtc_base/compile_assert_c.h" + +/* log2[0.2, 0.5, 0.98] in Q8 */ +static const int16_t kLogLagWinQ8[3] = { + -594, -256, -7 +}; + +/* [1 -0.75 0.25] in Q12 */ +static const int16_t kACoefQ12[3] = { + 4096, -3072, 1024 +}; + +int32_t WebRtcIsacfix_Log2Q8(uint32_t x) { + int32_t zeros; + int16_t frac; + + zeros=WebRtcSpl_NormU32(x); + frac = (int16_t)(((x << zeros) & 0x7FFFFFFF) >> 23); + /* log2(magn(i)) */ + + return ((31 - zeros) << 8) + frac; +} + +static __inline int16_t Exp2Q10(int16_t x) { // Both in and out in Q10 + + int16_t tmp16_1, tmp16_2; + + tmp16_2=(int16_t)(0x0400|(x&0x03FF)); + tmp16_1 = -(x >> 10); + if(tmp16_1>0) + return tmp16_2 >> tmp16_1; + else + return tmp16_2 << -tmp16_1; + +} + + + +/* 1D parabolic interpolation . All input and output values are in Q8 */ +static __inline void Intrp1DQ8(int32_t *x, int32_t *fx, int32_t *y, int32_t *fy) { + + int16_t sign1=1, sign2=1; + int32_t r32, q32, t32, nom32, den32; + int16_t t16, tmp16, tmp16_1; + + if ((fx[0]>0) && (fx[2]>0)) { + r32=fx[1]-fx[2]; + q32=fx[0]-fx[1]; + nom32=q32+r32; + den32 = (q32 - r32) * 2; + if (nom32<0) + sign1=-1; + if (den32<0) + sign2=-1; + + /* t = (q32+r32)/(2*(q32-r32)) = (fx[0]-fx[1] + fx[1]-fx[2])/(2 * fx[0]-fx[1] - (fx[1]-fx[2]))*/ + /* (Signs are removed because WebRtcSpl_DivResultInQ31 can't handle negative numbers) */ + /* t in Q31, without signs */ + t32 = WebRtcSpl_DivResultInQ31(nom32 * sign1, den32 * sign2); + + t16 = (int16_t)(t32 >> 23); /* Q8 */ + t16=t16*sign1*sign2; /* t in Q8 with signs */ + + *y = x[0]+t16; /* Q8 */ + // *y = x[1]+t16; /* Q8 */ + + /* The following code calculates fy in three steps */ + /* fy = 0.5 * t * (t-1) * fx[0] + (1-t*t) * fx[1] + 0.5 * t * (t+1) * fx[2]; */ + + /* Part I: 0.5 * t * (t-1) * fx[0] */ + tmp16_1 = (int16_t)(t16 * t16); /* Q8*Q8=Q16 */ + tmp16_1 >>= 2; /* Q16>>2 = Q14 */ + t16 <<= 6; /* Q8<<6 = Q14 */ + tmp16 = tmp16_1-t16; + *fy = WEBRTC_SPL_MUL_16_32_RSFT15(tmp16, fx[0]); /* (Q14 * Q8 >>15)/2 = Q8 */ + + /* Part II: (1-t*t) * fx[1] */ + tmp16 = 16384-tmp16_1; /* 1 in Q14 - Q14 */ + *fy += WEBRTC_SPL_MUL_16_32_RSFT14(tmp16, fx[1]);/* Q14 * Q8 >> 14 = Q8 */ + + /* Part III: 0.5 * t * (t+1) * fx[2] */ + tmp16 = tmp16_1+t16; + *fy += WEBRTC_SPL_MUL_16_32_RSFT15(tmp16, fx[2]);/* (Q14 * Q8 >>15)/2 = Q8 */ + } else { + *y = x[0]; + *fy= fx[1]; + } +} + + +static void FindFour32(int32_t *in, int16_t length, int16_t *bestind) +{ + int32_t best[4]= {-100, -100, -100, -100}; + int16_t k; + + for (k=0; k best[3]) { + if (in[k] > best[2]) { + if (in[k] > best[1]) { + if (in[k] > best[0]) { // The Best + best[3] = best[2]; + bestind[3] = bestind[2]; + best[2] = best[1]; + bestind[2] = bestind[1]; + best[1] = best[0]; + bestind[1] = bestind[0]; + best[0] = in[k]; + bestind[0] = k; + } else { // 2nd best + best[3] = best[2]; + bestind[3] = bestind[2]; + best[2] = best[1]; + bestind[2] = bestind[1]; + best[1] = in[k]; + bestind[1] = k; + } + } else { // 3rd best + best[3] = best[2]; + bestind[3] = bestind[2]; + best[2] = in[k]; + bestind[2] = k; + } + } else { // 4th best + best[3] = in[k]; + bestind[3] = k; + } + } + } +} + + + + + +extern void WebRtcIsacfix_PCorr2Q32(const int16_t *in, int32_t *logcorQ8); + + + +void WebRtcIsacfix_InitialPitch(const int16_t *in, /* Q0 */ + PitchAnalysisStruct *State, + int16_t *lagsQ7 /* Q7 */ + ) +{ + int16_t buf_dec16[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2+2]; + int32_t *crrvecQ8_1,*crrvecQ8_2; + int32_t cv1q[PITCH_LAG_SPAN2+2],cv2q[PITCH_LAG_SPAN2+2], peakvq[PITCH_LAG_SPAN2+2]; + int k; + int16_t peaks_indq; + int16_t peakiq[PITCH_LAG_SPAN2]; + int32_t corr; + int32_t corr32, corr_max32, corr_max_o32; + int16_t npkq; + int16_t best4q[4]={0,0,0,0}; + int32_t xq[3],yq[1],fyq[1]; + int32_t *fxq; + int32_t best_lag1q, best_lag2q; + int32_t tmp32a,tmp32b,lag32,ratq; + int16_t start; + int16_t oldgQ12, tmp16a, tmp16b, gain_bias16,tmp16c, tmp16d, bias16; + int32_t tmp32c,tmp32d, tmp32e; + int16_t old_lagQ; + int32_t old_lagQ8; + int32_t lagsQ8[4]; + + old_lagQ = State->PFstr_wght.oldlagQ7; // Q7 + old_lagQ8 = old_lagQ << 1; // Q8 + + oldgQ12= State->PFstr_wght.oldgainQ12; + + crrvecQ8_1=&cv1q[1]; + crrvecQ8_2=&cv2q[1]; + + + /* copy old values from state buffer */ + memcpy(buf_dec16, State->dec_buffer16, sizeof(State->dec_buffer16)); + + /* decimation; put result after the old values */ + WebRtcIsacfix_DecimateAllpass32(in, State->decimator_state32, PITCH_FRAME_LEN, + &buf_dec16[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2]); + + /* low-pass filtering */ + start= PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2; + WebRtcSpl_FilterARFastQ12(&buf_dec16[start],&buf_dec16[start],(int16_t*)kACoefQ12,3, PITCH_FRAME_LEN/2); + + /* copy end part back into state buffer */ + for (k = 0; k < (PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2); k++) + State->dec_buffer16[k] = buf_dec16[k+PITCH_FRAME_LEN/2]; + + + /* compute correlation for first and second half of the frame */ + WebRtcIsacfix_PCorr2Q32(buf_dec16, crrvecQ8_1); + WebRtcIsacfix_PCorr2Q32(buf_dec16 + PITCH_CORR_STEP2, crrvecQ8_2); + + + /* bias towards pitch lag of previous frame */ + tmp32a = WebRtcIsacfix_Log2Q8((uint32_t) old_lagQ8) - 2304; + // log2(0.5*oldlag) in Q8 + tmp32b = oldgQ12 * oldgQ12 >> 10; // Q12 & * 4.0; + gain_bias16 = (int16_t) tmp32b; //Q12 + if (gain_bias16 > 3276) gain_bias16 = 3276; // 0.8 in Q12 + + + for (k = 0; k < PITCH_LAG_SPAN2; k++) + { + if (crrvecQ8_1[k]>0) { + tmp32b = WebRtcIsacfix_Log2Q8((uint32_t) (k + (PITCH_MIN_LAG/2-2))); + tmp16a = (int16_t) (tmp32b - tmp32a); // Q8 & fabs(ratio)<4 + tmp32c = tmp16a * tmp16a >> 6; // Q10 + tmp16b = (int16_t) tmp32c; // Q10 & <8 + tmp32d = tmp16b * 177 >> 8; // mult with ln2 in Q8 + tmp16c = (int16_t) tmp32d; // Q10 & <4 + tmp16d = Exp2Q10((int16_t) -tmp16c); //Q10 + tmp32c = gain_bias16 * tmp16d >> 13; // Q10 & * 0.5 + bias16 = (int16_t) (1024 + tmp32c); // Q10 + tmp32b = WebRtcIsacfix_Log2Q8((uint32_t)bias16) - 2560; + // Q10 in -> Q8 out with 10*2^8 offset + crrvecQ8_1[k] += tmp32b ; // -10*2^8 offset + } + } + + /* taper correlation functions */ + for (k = 0; k < 3; k++) { + crrvecQ8_1[k] += kLogLagWinQ8[k]; + crrvecQ8_2[k] += kLogLagWinQ8[k]; + + crrvecQ8_1[PITCH_LAG_SPAN2-1-k] += kLogLagWinQ8[k]; + crrvecQ8_2[PITCH_LAG_SPAN2-1-k] += kLogLagWinQ8[k]; + } + + + /* Make zeropadded corr vectors */ + cv1q[0]=0; + cv2q[0]=0; + cv1q[PITCH_LAG_SPAN2+1]=0; + cv2q[PITCH_LAG_SPAN2+1]=0; + corr_max32 = 0; + + for (k = 1; k <= PITCH_LAG_SPAN2; k++) + { + + + corr32=crrvecQ8_1[k-1]; + if (corr32 > corr_max32) + corr_max32 = corr32; + + corr32=crrvecQ8_2[k-1]; + corr32 += -4; // Compensate for later (log2(0.99)) + + if (corr32 > corr_max32) + corr_max32 = corr32; + + } + + /* threshold value to qualify as a peak */ + // corr_max32 += -726; // log(0.14)/log(2.0) in Q8 + corr_max32 += -1000; // log(0.14)/log(2.0) in Q8 + corr_max_o32 = corr_max32; + + + /* find peaks in corr1 */ + peaks_indq = 0; + for (k = 1; k <= PITCH_LAG_SPAN2; k++) + { + corr32=cv1q[k]; + if (corr32>corr_max32) { // Disregard small peaks + if ((corr32>=cv1q[k-1]) && (corr32>cv1q[k+1])) { // Peak? + peakvq[peaks_indq] = corr32; + peakiq[peaks_indq++] = k; + } + } + } + + + /* find highest interpolated peak */ + corr_max32=0; + best_lag1q =0; + if (peaks_indq > 0) { + FindFour32(peakvq, (int16_t) peaks_indq, best4q); + npkq = WEBRTC_SPL_MIN(peaks_indq, 4); + + for (k=0;k> 8; + tmp32c= tmp32b + 256; + *fyq += tmp32c; + if (*fyq > corr_max32) { + corr_max32 = *fyq; + best_lag1q = *yq; + } + } + tmp32b = (best_lag1q - OFFSET_Q8) * 2; + lagsQ8[0] = tmp32b + PITCH_MIN_LAG_Q8; + lagsQ8[1] = lagsQ8[0]; + } else { + lagsQ8[0] = old_lagQ8; + lagsQ8[1] = lagsQ8[0]; + } + + /* Bias towards constant pitch */ + tmp32a = lagsQ8[0] - PITCH_MIN_LAG_Q8; + ratq = (tmp32a >> 1) + OFFSET_Q8; + + for (k = 1; k <= PITCH_LAG_SPAN2; k++) + { + tmp32a = k << 7; // 0.5*k Q8 + tmp32b = tmp32a * 2 - ratq; // Q8 + tmp32c = (int16_t)tmp32b * (int16_t)tmp32b >> 8; // Q8 + + tmp32b = tmp32c + (ratq >> 1); + // (k-r)^2 + 0.5 * r Q8 + tmp32c = WebRtcIsacfix_Log2Q8((uint32_t)tmp32a) - 2048; + // offset 8*2^8 , log2(0.5*k) Q8 + tmp32d = WebRtcIsacfix_Log2Q8((uint32_t)tmp32b) - 2048; + // offset 8*2^8 , log2(0.5*k) Q8 + tmp32e = tmp32c - tmp32d; + + cv2q[k] += tmp32e >> 1; + + } + + /* find peaks in corr2 */ + corr_max32 = corr_max_o32; + peaks_indq = 0; + + for (k = 1; k <= PITCH_LAG_SPAN2; k++) + { + corr=cv2q[k]; + if (corr>corr_max32) { // Disregard small peaks + if ((corr>=cv2q[k-1]) && (corr>cv2q[k+1])) { // Peak? + peakvq[peaks_indq] = corr; + peakiq[peaks_indq++] = k; + } + } + } + + + + /* find highest interpolated peak */ + corr_max32 = 0; + best_lag2q =0; + if (peaks_indq > 0) { + + FindFour32(peakvq, (int16_t) peaks_indq, best4q); + npkq = WEBRTC_SPL_MIN(peaks_indq, 4); + for (k=0;k> 8; + tmp32c= tmp32b + 256; + *fyq += tmp32c; + if (*fyq > corr_max32) { + corr_max32 = *fyq; + best_lag2q = *yq; + } + } + + tmp32b = (best_lag2q - OFFSET_Q8) * 2; + lagsQ8[2] = tmp32b + PITCH_MIN_LAG_Q8; + lagsQ8[3] = lagsQ8[2]; + } else { + lagsQ8[2] = lagsQ8[0]; + lagsQ8[3] = lagsQ8[0]; + } + + lagsQ7[0] = (int16_t)(lagsQ8[0] >> 1); + lagsQ7[1] = (int16_t)(lagsQ8[1] >> 1); + lagsQ7[2] = (int16_t)(lagsQ8[2] >> 1); + lagsQ7[3] = (int16_t)(lagsQ8[3] >> 1); +} + + + +void WebRtcIsacfix_PitchAnalysis(const int16_t *inn, /* PITCH_FRAME_LEN samples */ + int16_t *outQ0, /* PITCH_FRAME_LEN+QLOOKAHEAD samples */ + PitchAnalysisStruct *State, + int16_t *PitchLags_Q7, + int16_t *PitchGains_Q12) +{ + int16_t inbufQ0[PITCH_FRAME_LEN + QLOOKAHEAD]; + int16_t k; + + /* inital pitch estimate */ + WebRtcIsacfix_InitialPitch(inn, State, PitchLags_Q7); + + + /* Calculate gain */ + WebRtcIsacfix_PitchFilterGains(inn, &(State->PFstr_wght), PitchLags_Q7, PitchGains_Q12); + + /* concatenate previous input's end and current input */ + for (k = 0; k < QLOOKAHEAD; k++) { + inbufQ0[k] = State->inbuf[k]; + } + for (k = 0; k < PITCH_FRAME_LEN; k++) { + inbufQ0[k+QLOOKAHEAD] = (int16_t) inn[k]; + } + + /* lookahead pitch filtering for masking analysis */ + WebRtcIsacfix_PitchFilter(inbufQ0, outQ0, &(State->PFstr), PitchLags_Q7,PitchGains_Q12, 2); + + + /* store last part of input */ + for (k = 0; k < QLOOKAHEAD; k++) { + State->inbuf[k] = inbufQ0[k + PITCH_FRAME_LEN]; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h new file mode 100644 index 0000000000..0aa727dd41 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_estimator.h + * + * Pitch functions + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_ESTIMATOR_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_ESTIMATOR_H_ + +#include "structs.h" + +void WebRtcIsacfix_PitchAnalysis(const int16_t *in, /* PITCH_FRAME_LEN samples */ + int16_t *outQ0, /* PITCH_FRAME_LEN+QLOOKAHEAD samples */ + PitchAnalysisStruct *State, + int16_t *lagsQ7, + int16_t *PitchGains_Q12); + +void WebRtcIsacfix_InitialPitch(const int16_t *in, + PitchAnalysisStruct *State, + int16_t *qlags); + +void WebRtcIsacfix_PitchFilter(int16_t *indatFix, + int16_t *outdatQQ, + PitchFiltstr *pfp, + int16_t *lagsQ7, + int16_t *gainsQ12, + int16_t type); + +void WebRtcIsacfix_PitchFilterCore(int loopNumber, + int16_t gain, + size_t index, + int16_t sign, + int16_t* inputState, + int16_t* outputBuff2, + const int16_t* coefficient, + int16_t* inputBuf, + int16_t* outputBuf, + int* index2); + +void WebRtcIsacfix_PitchFilterGains(const int16_t *indatQ0, + PitchFiltstr *pfp, + int16_t *lagsQ7, + int16_t *gainsQ12); + +void WebRtcIsacfix_DecimateAllpass32(const int16_t *in, + int32_t *state_in, /* array of size: 2*ALLPASSSECTIONS+1 */ + int16_t N, /* number of input samples */ + int16_t *out); /* array of size N/2 */ + +int32_t WebRtcIsacfix_Log2Q8( uint32_t x ); + +void WebRtcIsacfix_PCorr2Q32(const int16_t* in, int32_t* logcorQ8); + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_ESTIMATOR_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_c.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_c.c new file mode 100644 index 0000000000..1214e23bef --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_c.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h" + +#ifdef WEBRTC_HAS_NEON +#include +#endif + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "rtc_base/compile_assert_c.h" + +extern int32_t WebRtcIsacfix_Log2Q8(uint32_t x); + +void WebRtcIsacfix_PCorr2Q32(const int16_t* in, int32_t* logcorQ8) { + int16_t scaling,n,k; + int32_t ysum32,csum32, lys, lcs; + const int32_t oneQ8 = 1 << 8; // 1.00 in Q8 + const int16_t* x; + const int16_t* inptr; + + x = in + PITCH_MAX_LAG / 2 + 2; + scaling = WebRtcSpl_GetScalingSquare((int16_t*)in, + PITCH_CORR_LEN2, + PITCH_CORR_LEN2); + ysum32 = 1; + csum32 = 0; + x = in + PITCH_MAX_LAG / 2 + 2; + for (n = 0; n < PITCH_CORR_LEN2; n++) { + ysum32 += in[n] * in[n] >> scaling; // Q0 + csum32 += x[n] * in[n] >> scaling; // Q0 + } + logcorQ8 += PITCH_LAG_SPAN2 - 1; + lys = WebRtcIsacfix_Log2Q8((uint32_t)ysum32) >> 1; // Q8, sqrt(ysum) + if (csum32 > 0) { + lcs = WebRtcIsacfix_Log2Q8((uint32_t)csum32); // 2log(csum) in Q8 + if (lcs > (lys + oneQ8)) { // csum/sqrt(ysum) > 2 in Q8 + *logcorQ8 = lcs - lys; // log2(csum/sqrt(ysum)) + } else { + *logcorQ8 = oneQ8; // 1.00 + } + } else { + *logcorQ8 = 0; + } + + + for (k = 1; k < PITCH_LAG_SPAN2; k++) { + inptr = &in[k]; + ysum32 -= in[k - 1] * in[k - 1] >> scaling; + ysum32 += in[PITCH_CORR_LEN2 + k - 1] * in[PITCH_CORR_LEN2 + k - 1] >> + scaling; + +#ifdef WEBRTC_HAS_NEON + { + int32_t vbuff[4]; + int32x4_t int_32x4_sum = vmovq_n_s32(0); + // Can't shift a Neon register to right with a non-constant shift value. + int32x4_t int_32x4_scale = vdupq_n_s32(-scaling); + // Assert a codition used in loop unrolling at compile-time. + RTC_COMPILE_ASSERT(PITCH_CORR_LEN2 %4 == 0); + + for (n = 0; n < PITCH_CORR_LEN2; n += 4) { + int16x4_t int_16x4_x = vld1_s16(&x[n]); + int16x4_t int_16x4_in = vld1_s16(&inptr[n]); + int32x4_t int_32x4 = vmull_s16(int_16x4_x, int_16x4_in); + int_32x4 = vshlq_s32(int_32x4, int_32x4_scale); + int_32x4_sum = vaddq_s32(int_32x4_sum, int_32x4); + } + + // Use vector store to avoid long stall from data trasferring + // from vector to general register. + vst1q_s32(vbuff, int_32x4_sum); + csum32 = vbuff[0] + vbuff[1]; + csum32 += vbuff[2]; + csum32 += vbuff[3]; + } +#else + csum32 = 0; + if(scaling == 0) { + for (n = 0; n < PITCH_CORR_LEN2; n++) { + csum32 += x[n] * inptr[n]; + } + } else { + for (n = 0; n < PITCH_CORR_LEN2; n++) { + csum32 += (x[n] * inptr[n]) >> scaling; + } + } +#endif + + logcorQ8--; + + lys = WebRtcIsacfix_Log2Q8((uint32_t)ysum32) >> 1; // Q8, sqrt(ysum) + + if (csum32 > 0) { + lcs = WebRtcIsacfix_Log2Q8((uint32_t)csum32); // 2log(csum) in Q8 + if (lcs > (lys + oneQ8)) { // csum/sqrt(ysum) > 2 + *logcorQ8 = lcs - lys; // log2(csum/sqrt(ysum)) + } else { + *logcorQ8 = oneQ8; // 1.00 + } + } else { + *logcorQ8 = 0; + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_mips.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_mips.c new file mode 100644 index 0000000000..4ead84c492 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_mips.c @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "rtc_base/compile_assert_c.h" + +extern int32_t WebRtcIsacfix_Log2Q8(uint32_t x); + +void WebRtcIsacfix_PCorr2Q32(const int16_t* in, int32_t* logcorQ8) { + int16_t scaling,n,k; + int32_t ysum32,csum32, lys, lcs; + const int32_t oneQ8 = 1 << 8; // 1.00 in Q8 + const int16_t* x; + const int16_t* inptr; + + x = in + PITCH_MAX_LAG / 2 + 2; + scaling = WebRtcSpl_GetScalingSquare((int16_t*)in, + PITCH_CORR_LEN2, + PITCH_CORR_LEN2); + ysum32 = 1; + csum32 = 0; + x = in + PITCH_MAX_LAG / 2 + 2; + { + const int16_t* tmp_x = x; + const int16_t* tmp_in = in; + int32_t tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + n = PITCH_CORR_LEN2; + RTC_COMPILE_ASSERT(PITCH_CORR_LEN2 % 4 == 0); + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "lh %[tmp1], 0(%[tmp_in]) \n\t" + "lh %[tmp2], 2(%[tmp_in]) \n\t" + "lh %[tmp3], 4(%[tmp_in]) \n\t" + "lh %[tmp4], 6(%[tmp_in]) \n\t" + "lh %[tmp5], 0(%[tmp_x]) \n\t" + "lh %[tmp6], 2(%[tmp_x]) \n\t" + "lh %[tmp7], 4(%[tmp_x]) \n\t" + "lh %[tmp8], 6(%[tmp_x]) \n\t" + "mul %[tmp5], %[tmp1], %[tmp5] \n\t" + "mul %[tmp1], %[tmp1], %[tmp1] \n\t" + "mul %[tmp6], %[tmp2], %[tmp6] \n\t" + "mul %[tmp2], %[tmp2], %[tmp2] \n\t" + "mul %[tmp7], %[tmp3], %[tmp7] \n\t" + "mul %[tmp3], %[tmp3], %[tmp3] \n\t" + "mul %[tmp8], %[tmp4], %[tmp8] \n\t" + "mul %[tmp4], %[tmp4], %[tmp4] \n\t" + "addiu %[n], %[n], -4 \n\t" + "srav %[tmp5], %[tmp5], %[scaling] \n\t" + "srav %[tmp1], %[tmp1], %[scaling] \n\t" + "srav %[tmp6], %[tmp6], %[scaling] \n\t" + "srav %[tmp2], %[tmp2], %[scaling] \n\t" + "srav %[tmp7], %[tmp7], %[scaling] \n\t" + "srav %[tmp3], %[tmp3], %[scaling] \n\t" + "srav %[tmp8], %[tmp8], %[scaling] \n\t" + "srav %[tmp4], %[tmp4], %[scaling] \n\t" + "addu %[ysum32], %[ysum32], %[tmp1] \n\t" + "addu %[csum32], %[csum32], %[tmp5] \n\t" + "addu %[ysum32], %[ysum32], %[tmp2] \n\t" + "addu %[csum32], %[csum32], %[tmp6] \n\t" + "addu %[ysum32], %[ysum32], %[tmp3] \n\t" + "addu %[csum32], %[csum32], %[tmp7] \n\t" + "addu %[ysum32], %[ysum32], %[tmp4] \n\t" + "addu %[csum32], %[csum32], %[tmp8] \n\t" + "addiu %[tmp_in], %[tmp_in], 8 \n\t" + "bgtz %[n], 1b \n\t" + " addiu %[tmp_x], %[tmp_x], 8 \n\t" + ".set pop \n\t" + : [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [tmp3] "=&r" (tmp3), + [tmp4] "=&r" (tmp4), [tmp5] "=&r" (tmp5), [tmp6] "=&r" (tmp6), + [tmp7] "=&r" (tmp7), [tmp8] "=&r" (tmp8), [tmp_in] "+r" (tmp_in), + [ysum32] "+r" (ysum32), [tmp_x] "+r" (tmp_x), [csum32] "+r" (csum32), + [n] "+r" (n) + : [scaling] "r" (scaling) + : "memory", "hi", "lo" + ); + } + logcorQ8 += PITCH_LAG_SPAN2 - 1; + lys = WebRtcIsacfix_Log2Q8((uint32_t)ysum32) >> 1; // Q8, sqrt(ysum) + if (csum32 > 0) { + lcs = WebRtcIsacfix_Log2Q8((uint32_t)csum32); // 2log(csum) in Q8 + if (lcs > (lys + oneQ8)) { // csum/sqrt(ysum) > 2 in Q8 + *logcorQ8 = lcs - lys; // log2(csum/sqrt(ysum)) + } else { + *logcorQ8 = oneQ8; // 1.00 + } + } else { + *logcorQ8 = 0; + } + + for (k = 1; k < PITCH_LAG_SPAN2; k++) { + inptr = &in[k]; + const int16_t* tmp_in1 = &in[k - 1]; + const int16_t* tmp_in2 = &in[PITCH_CORR_LEN2 + k - 1]; + const int16_t* tmp_x = x; + int32_t tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + n = PITCH_CORR_LEN2; + csum32 = 0; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lh %[tmp1], 0(%[tmp_in1]) \n\t" + "lh %[tmp2], 0(%[tmp_in2]) \n\t" + "mul %[tmp1], %[tmp1], %[tmp1] \n\t" + "mul %[tmp2], %[tmp2], %[tmp2] \n\t" + "srav %[tmp1], %[tmp1], %[scaling] \n\t" + "srav %[tmp2], %[tmp2], %[scaling] \n\t" + "subu %[ysum32], %[ysum32], %[tmp1] \n\t" + "bnez %[scaling], 2f \n\t" + " addu %[ysum32], %[ysum32], %[tmp2] \n\t" + "1: \n\t" + "lh %[tmp1], 0(%[inptr]) \n\t" + "lh %[tmp2], 0(%[tmp_x]) \n\t" + "lh %[tmp3], 2(%[inptr]) \n\t" + "lh %[tmp4], 2(%[tmp_x]) \n\t" + "lh %[tmp5], 4(%[inptr]) \n\t" + "lh %[tmp6], 4(%[tmp_x]) \n\t" + "lh %[tmp7], 6(%[inptr]) \n\t" + "lh %[tmp8], 6(%[tmp_x]) \n\t" + "mul %[tmp1], %[tmp1], %[tmp2] \n\t" + "mul %[tmp2], %[tmp3], %[tmp4] \n\t" + "mul %[tmp3], %[tmp5], %[tmp6] \n\t" + "mul %[tmp4], %[tmp7], %[tmp8] \n\t" + "addiu %[n], %[n], -4 \n\t" + "addiu %[inptr], %[inptr], 8 \n\t" + "addiu %[tmp_x], %[tmp_x], 8 \n\t" + "addu %[csum32], %[csum32], %[tmp1] \n\t" + "addu %[csum32], %[csum32], %[tmp2] \n\t" + "addu %[csum32], %[csum32], %[tmp3] \n\t" + "bgtz %[n], 1b \n\t" + " addu %[csum32], %[csum32], %[tmp4] \n\t" + "b 3f \n\t" + " nop \n\t" + "2: \n\t" + "lh %[tmp1], 0(%[inptr]) \n\t" + "lh %[tmp2], 0(%[tmp_x]) \n\t" + "lh %[tmp3], 2(%[inptr]) \n\t" + "lh %[tmp4], 2(%[tmp_x]) \n\t" + "lh %[tmp5], 4(%[inptr]) \n\t" + "lh %[tmp6], 4(%[tmp_x]) \n\t" + "lh %[tmp7], 6(%[inptr]) \n\t" + "lh %[tmp8], 6(%[tmp_x]) \n\t" + "mul %[tmp1], %[tmp1], %[tmp2] \n\t" + "mul %[tmp2], %[tmp3], %[tmp4] \n\t" + "mul %[tmp3], %[tmp5], %[tmp6] \n\t" + "mul %[tmp4], %[tmp7], %[tmp8] \n\t" + "addiu %[n], %[n], -4 \n\t" + "addiu %[inptr], %[inptr], 8 \n\t" + "addiu %[tmp_x], %[tmp_x], 8 \n\t" + "srav %[tmp1], %[tmp1], %[scaling] \n\t" + "srav %[tmp2], %[tmp2], %[scaling] \n\t" + "srav %[tmp3], %[tmp3], %[scaling] \n\t" + "srav %[tmp4], %[tmp4], %[scaling] \n\t" + "addu %[csum32], %[csum32], %[tmp1] \n\t" + "addu %[csum32], %[csum32], %[tmp2] \n\t" + "addu %[csum32], %[csum32], %[tmp3] \n\t" + "bgtz %[n], 2b \n\t" + " addu %[csum32], %[csum32], %[tmp4] \n\t" + "3: \n\t" + ".set pop \n\t" + : [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [tmp3] "=&r" (tmp3), + [tmp4] "=&r" (tmp4), [tmp5] "=&r" (tmp5), [tmp6] "=&r" (tmp6), + [tmp7] "=&r" (tmp7), [tmp8] "=&r" (tmp8), [inptr] "+r" (inptr), + [csum32] "+r" (csum32), [tmp_x] "+r" (tmp_x), [ysum32] "+r" (ysum32), + [n] "+r" (n) + : [tmp_in1] "r" (tmp_in1), [tmp_in2] "r" (tmp_in2), + [scaling] "r" (scaling) + : "memory", "hi", "lo" + ); + + logcorQ8--; + lys = WebRtcIsacfix_Log2Q8((uint32_t)ysum32) >> 1; // Q8, sqrt(ysum) + if (csum32 > 0) { + lcs = WebRtcIsacfix_Log2Q8((uint32_t)csum32); // 2log(csum) in Q8 + if (lcs > (lys + oneQ8)) { // csum/sqrt(ysum) > 2 + *logcorQ8 = lcs - lys; // log2(csum/sqrt(ysum)) + } else { + *logcorQ8 = oneQ8; // 1.00 + } + } else { + *logcorQ8 = 0; + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c new file mode 100644 index 0000000000..e565e8564b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h" + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" +#include "modules/audio_coding/codecs/isac/fix/source/structs.h" +#include "rtc_base/compile_assert_c.h" + +// Number of segments in a pitch subframe. +static const int kSegments = 5; + +// A division factor of 1/5 in Q15. +static const int16_t kDivFactor = 6553; + +// Interpolation coefficients; generated by design_pitch_filter.m. +// Coefficients are stored in Q14. +static const int16_t kIntrpCoef[PITCH_FRACS][PITCH_FRACORDER] = { + {-367, 1090, -2706, 9945, 10596, -3318, 1626, -781, 287}, + {-325, 953, -2292, 7301, 12963, -3320, 1570, -743, 271}, + {-240, 693, -1622, 4634, 14809, -2782, 1262, -587, 212}, + {-125, 358, -817, 2144, 15982, -1668, 721, -329, 118}, + { 0, 0, -1, 1, 16380, 1, -1, 0, 0}, + { 118, -329, 721, -1668, 15982, 2144, -817, 358, -125}, + { 212, -587, 1262, -2782, 14809, 4634, -1622, 693, -240}, + { 271, -743, 1570, -3320, 12963, 7301, -2292, 953, -325} +}; + +static __inline size_t CalcLrIntQ(int16_t fixVal, + int16_t qDomain) { + int32_t roundVal = 1 << (qDomain - 1); + + return (fixVal + roundVal) >> qDomain; +} + +void WebRtcIsacfix_PitchFilter(int16_t* indatQQ, // Q10 if type is 1 or 4, + // Q0 if type is 2. + int16_t* outdatQQ, + PitchFiltstr* pfp, + int16_t* lagsQ7, + int16_t* gainsQ12, + int16_t type) { + int k, ind, cnt; + int16_t sign = 1; + int16_t inystateQQ[PITCH_DAMPORDER]; + int16_t ubufQQ[PITCH_INTBUFFSIZE + QLOOKAHEAD]; + const int16_t Gain = 21299; // 1.3 in Q14 + int16_t oldLagQ7; + int16_t oldGainQ12, lagdeltaQ7, curLagQ7, gaindeltaQ12, curGainQ12; + size_t indW32 = 0, frcQQ = 0; + const int16_t* fracoeffQQ = NULL; + + // Assumptions in ARM assembly for WebRtcIsacfix_PitchFilterCoreARM(). + RTC_COMPILE_ASSERT(PITCH_FRACORDER == 9); + RTC_COMPILE_ASSERT(PITCH_DAMPORDER == 5); + + // Set up buffer and states. + memcpy(ubufQQ, pfp->ubufQQ, sizeof(pfp->ubufQQ)); + memcpy(inystateQQ, pfp->ystateQQ, sizeof(inystateQQ)); + + // Get old lag and gain value from memory. + oldLagQ7 = pfp->oldlagQ7; + oldGainQ12 = pfp->oldgainQ12; + + if (type == 4) { + sign = -1; + + // Make output more periodic. + for (k = 0; k < PITCH_SUBFRAMES; k++) { + gainsQ12[k] = (int16_t)(gainsQ12[k] * Gain >> 14); + } + } + + // No interpolation if pitch lag step is big. + if (((lagsQ7[0] * 3 >> 1) < oldLagQ7) || (lagsQ7[0] > (oldLagQ7 * 3 >> 1))) { + oldLagQ7 = lagsQ7[0]; + oldGainQ12 = gainsQ12[0]; + } + + ind = 0; + + for (k = 0; k < PITCH_SUBFRAMES; k++) { + // Calculate interpolation steps. + lagdeltaQ7 = lagsQ7[k] - oldLagQ7; + lagdeltaQ7 = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + lagdeltaQ7, kDivFactor, 15); + curLagQ7 = oldLagQ7; + gaindeltaQ12 = gainsQ12[k] - oldGainQ12; + gaindeltaQ12 = (int16_t)(gaindeltaQ12 * kDivFactor >> 15); + + curGainQ12 = oldGainQ12; + oldLagQ7 = lagsQ7[k]; + oldGainQ12 = gainsQ12[k]; + + // Each frame has 4 60-sample pitch subframes, and each subframe has 5 + // 12-sample segments. Each segment need to be processed with + // newly-updated parameters, so we break the pitch filtering into + // two for-loops (5 x 12) below. It's also why kDivFactor = 0.2 (in Q15). + for (cnt = 0; cnt < kSegments; cnt++) { + // Update parameters for each segment. + curGainQ12 += gaindeltaQ12; + curLagQ7 += lagdeltaQ7; + indW32 = CalcLrIntQ(curLagQ7, 7); + if (indW32 < PITCH_FRACORDER - 2) { + // WebRtcIsacfix_PitchFilterCore requires indW32 >= PITCH_FRACORDER - + // 2; otherwise, it will read from entries of ubufQQ that haven't been + // written yet. (This problem has only been seen in fuzzer tests, not + // in real life.) See Chromium bug 581901. + indW32 = PITCH_FRACORDER - 2; + } + frcQQ = ((indW32 << 7) + 64 - curLagQ7) >> 4; + + if (frcQQ >= PITCH_FRACS) { + frcQQ = 0; + } + fracoeffQQ = kIntrpCoef[frcQQ]; + + // Pitch filtering. + WebRtcIsacfix_PitchFilterCore(PITCH_SUBFRAME_LEN / kSegments, curGainQ12, + indW32, sign, inystateQQ, ubufQQ, fracoeffQQ, indatQQ, outdatQQ, &ind); + } + } + + // Export buffer and states. + memcpy(pfp->ubufQQ, ubufQQ + PITCH_FRAME_LEN, sizeof(pfp->ubufQQ)); + memcpy(pfp->ystateQQ, inystateQQ, sizeof(pfp->ystateQQ)); + + pfp->oldlagQ7 = oldLagQ7; + pfp->oldgainQ12 = oldGainQ12; + + if (type == 2) { + // Filter look-ahead segment. + WebRtcIsacfix_PitchFilterCore(QLOOKAHEAD, curGainQ12, indW32, 1, inystateQQ, + ubufQQ, fracoeffQQ, indatQQ, outdatQQ, &ind); + } +} + + +void WebRtcIsacfix_PitchFilterGains(const int16_t* indatQ0, + PitchFiltstr* pfp, + int16_t* lagsQ7, + int16_t* gainsQ12) { + int k, n, m; + size_t ind, pos, pos3QQ; + + int16_t ubufQQ[PITCH_INTBUFFSIZE]; + int16_t oldLagQ7, lagdeltaQ7, curLagQ7; + const int16_t* fracoeffQQ = NULL; + int16_t scale; + int16_t cnt = 0, tmpW16; + size_t frcQQ, indW16 = 0; + int32_t tmpW32, tmp2W32, csum1QQ, esumxQQ; + + // Set up buffer and states. + memcpy(ubufQQ, pfp->ubufQQ, sizeof(pfp->ubufQQ)); + oldLagQ7 = pfp->oldlagQ7; + + // No interpolation if pitch lag step is big. + if (((lagsQ7[0] * 3 >> 1) < oldLagQ7) || (lagsQ7[0] > (oldLagQ7 * 3 >> 1))) { + oldLagQ7 = lagsQ7[0]; + } + + ind = 0; + pos = ind + PITCH_BUFFSIZE; + scale = 0; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + + // Calculate interpolation steps. + lagdeltaQ7 = lagsQ7[k] - oldLagQ7; + lagdeltaQ7 = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + lagdeltaQ7, kDivFactor, 15); + curLagQ7 = oldLagQ7; + oldLagQ7 = lagsQ7[k]; + + csum1QQ = 1; + esumxQQ = 1; + + // Same as function WebRtcIsacfix_PitchFilter(), we break the pitch + // filtering into two for-loops (5 x 12) below. + for (cnt = 0; cnt < kSegments; cnt++) { + // Update parameters for each segment. + curLagQ7 += lagdeltaQ7; + indW16 = CalcLrIntQ(curLagQ7, 7); + frcQQ = ((indW16 << 7) + 64 - curLagQ7) >> 4; + + if (frcQQ >= PITCH_FRACS) { + frcQQ = 0; + } + fracoeffQQ = kIntrpCoef[frcQQ]; + + pos3QQ = pos - (indW16 + 4); + + for (n = 0; n < PITCH_SUBFRAME_LEN / kSegments; n++) { + // Filter to get fractional pitch. + + tmpW32 = 0; + for (m = 0; m < PITCH_FRACORDER; m++) { + tmpW32 += ubufQQ[pos3QQ + m] * fracoeffQQ[m]; + } + + // Subtract from input and update buffer. + ubufQQ[pos] = indatQ0[ind]; + + tmp2W32 = WEBRTC_SPL_MUL_16_32_RSFT14(indatQ0[ind], tmpW32); + tmpW32 += 8192; + tmpW16 = tmpW32 >> 14; + tmpW32 = tmpW16 * tmpW16; + + if ((tmp2W32 > 1073700000) || (csum1QQ > 1073700000) || + (tmpW32 > 1073700000) || (esumxQQ > 1073700000)) { // 2^30 + scale++; + csum1QQ >>= 1; + esumxQQ >>= 1; + } + csum1QQ += tmp2W32 >> scale; + esumxQQ += tmpW32 >> scale; + + ind++; + pos++; + pos3QQ++; + } + } + + if (csum1QQ < esumxQQ) { + tmp2W32 = WebRtcSpl_DivResultInQ31(csum1QQ, esumxQQ); + + // Gain should be half the correlation. + tmpW32 = tmp2W32 >> 20; + } else { + tmpW32 = 4096; + } + gainsQ12[k] = (int16_t)WEBRTC_SPL_SAT(PITCH_MAX_GAIN_Q12, tmpW32, 0); + } + + // Export buffer and states. + memcpy(pfp->ubufQQ, ubufQQ + PITCH_FRAME_LEN, sizeof(pfp->ubufQQ)); + pfp->oldlagQ7 = lagsQ7[PITCH_SUBFRAMES - 1]; + pfp->oldgainQ12 = gainsQ12[PITCH_SUBFRAMES - 1]; + +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S new file mode 100644 index 0000000000..44fcac86f8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S @@ -0,0 +1,143 @@ +@ +@ Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +@ +@ Use of this source code is governed by a BSD-style license +@ that can be found in the LICENSE file in the root of the source +@ tree. An additional intellectual property rights grant can be found +@ in the file PATENTS. All contributing project authors may +@ be found in the AUTHORS file in the root of the source tree. +@ + +@ Contains the core loop routine for the pitch filter function in iSAC, +@ optimized for ARMv7 platforms. +@ +@ Output is bit-exact with the reference C code in pitch_filter.c. + +#include "system_wrappers/include/asm_defines.h" +#include "settings.h" + +GLOBAL_FUNCTION WebRtcIsacfix_PitchFilterCore +.align 2 + +@ void WebRtcIsacfix_PitchFilterCore(int loopNumber, +@ int16_t gain, +@ size_t index, +@ int16_t sign, +@ int16_t* inputState, +@ int16_t* outputBuf2, +@ const int16_t* coefficient, +@ int16_t* inputBuf, +@ int16_t* outputBuf, +@ int* index2) { +DEFINE_FUNCTION WebRtcIsacfix_PitchFilterCore + push {r4-r11} + sub sp, #8 + + str r0, [sp] @ loopNumber + str r3, [sp, #4] @ sign + ldr r3, [sp, #44] @ outputBuf2 + ldr r6, [sp, #60] @ index2 + ldr r7, [r6] @ *index2 + ldr r8, [sp, #52] @ inputBuf + ldr r12, [sp, #56] @ outputBuf + + add r4, r7, r0 + str r4, [r6] @ Store return value to index2. + + mov r10, r7, asl #1 + add r12, r10 @ &outputBuf[*index2] + add r8, r10 @ &inputBuf[*index2] + + add r4, r7, #PITCH_BUFFSIZE @ *index2 + PITCH_BUFFSIZE + add r6, r3, r4, lsl #1 @ &outputBuf2[*index2 + PITCH_BUFFSIZE] + sub r4, r2 @ r2: index + sub r4, #2 @ *index2 + PITCH_BUFFSIZE - index - 2 + add r3, r4, lsl #1 @ &ubufQQpos2[*index2] + ldr r9, [sp, #48] @ coefficient + +LOOP: +@ Usage of registers in the loop: +@ r0: loop counter +@ r1: gain +@ r2: tmpW32 +@ r3: &ubufQQpos2[] +@ r6: &outputBuf2[] +@ r8: &inputBuf[] +@ r9: &coefficient[] +@ r12: &outputBuf[] +@ r4, r5, r7, r10, r11: scratch + + @ Filter to get fractional pitch. + @ The pitch filter loop here is unrolled with 9 multipications. + pld [r3] + ldr r10, [r3], #4 @ ubufQQpos2[*index2 + 0, *index2 + 1] + ldr r4, [r9], #4 @ coefficient[0, 1] + ldr r11, [r3], #4 + ldr r5, [r9], #4 + smuad r2, r10, r4 + smlad r2, r11, r5, r2 + + ldr r10, [r3], #4 + ldr r4, [r9], #4 + ldr r11, [r3], #4 + ldr r5, [r9], #4 + smlad r2, r10, r4, r2 + ldrh r10, [r3], #-14 @ r3 back to &ubufQQpos2[*index2]. + ldrh r4, [r9], #-16 @ r9 back to &coefficient[0]. + smlad r2, r11, r5, r2 + smlabb r2, r10, r4, r2 + + @ Saturate to avoid overflow in tmpW16. + asr r2, #1 + add r4, r2, #0x1000 + ssat r7, #16, r4, asr #13 + + @ Shift low pass filter state, and excute the low pass filter. + @ The memmove() and the low pass filter loop are unrolled and mixed. + smulbb r5, r1, r7 + add r7, r5, #0x800 + asr r7, #12 @ Get the value for inputState[0]. + ldr r11, [sp, #40] @ inputState + pld [r11] + adr r10, kDampFilter + ldrsh r4, [r10], #2 @ kDampFilter[0] + mul r2, r7, r4 + ldr r4, [r11] @ inputState[0, 1], before shift. + strh r7, [r11] @ inputState[0], after shift. + ldr r5, [r11, #4] @ inputState[2, 3], before shift. + ldr r7, [r10], #4 @ kDampFilter[1, 2] + ldr r10, [r10] @ kDampFilter[3, 4] + str r4, [r11, #2] @ inputState[1, 2], after shift. + str r5, [r11, #6] @ inputState[3, 4], after shift. + smlad r2, r4, r7, r2 + smlad r2, r5, r10, r2 + + @ Saturate to avoid overflow. + @ First shift the sample to the range of [0xC0000000, 0x3FFFFFFF], + @ to avoid overflow in the next saturation step. + asr r2, #1 + add r10, r2, #0x2000 + ssat r10, #16, r10, asr #14 + + @ Subtract from input and update buffer. + ldr r11, [sp, #4] @ sign + ldrsh r4, [r8] + ldrsh r7, [r8], #2 @ inputBuf[*index2] + smulbb r5, r11, r10 + subs r0, #1 + sub r4, r5 + ssat r2, #16, r4 + strh r2, [r12], #2 @ outputBuf[*index2] + + add r2, r7 + ssat r2, #16, r2 + strh r2, [r6], #2 @ outputBuff2[*index2 + PITCH_BUFFSIZE] + bgt LOOP + + add sp, #8 + pop {r4-r11} + bx lr + +.align 2 +kDampFilter: + .short -2294, 8192, 20972, 8192, -2294 diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c new file mode 100644 index 0000000000..f23d19de9c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h" + +/* Filter coefficicients in Q15. */ +static const int16_t kDampFilter[PITCH_DAMPORDER] = { + -2294, 8192, 20972, 8192, -2294 +}; + +void WebRtcIsacfix_PitchFilterCore(int loopNumber, + int16_t gain, + size_t index, + int16_t sign, + int16_t* inputState, + int16_t* outputBuf2, + const int16_t* coefficient, + int16_t* inputBuf, + int16_t* outputBuf, + int* index2) { + int i = 0, j = 0; /* Loop counters. */ + int16_t* ubufQQpos2 = &outputBuf2[PITCH_BUFFSIZE - (index + 2)]; + int16_t tmpW16 = 0; + + for (i = 0; i < loopNumber; i++) { + int32_t tmpW32 = 0; + + /* Filter to get fractional pitch. */ + for (j = 0; j < PITCH_FRACORDER; j++) { + tmpW32 += ubufQQpos2[*index2 + j] * coefficient[j]; + } + + /* Saturate to avoid overflow in tmpW16. */ + tmpW32 = WEBRTC_SPL_SAT(536862719, tmpW32, -536879104); + tmpW32 += 8192; + tmpW16 = (int16_t)(tmpW32 >> 14); + + /* Shift low pass filter state. */ + memmove(&inputState[1], &inputState[0], + (PITCH_DAMPORDER - 1) * sizeof(int16_t)); + inputState[0] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + gain, tmpW16, 12); + + /* Low pass filter. */ + tmpW32 = 0; + /* TODO(kma): Define a static inline function WebRtcSpl_DotProduct() + in spl_inl.h to replace this and other similar loops. */ + for (j = 0; j < PITCH_DAMPORDER; j++) { + tmpW32 += inputState[j] * kDampFilter[j]; + } + + /* Saturate to avoid overflow in tmpW16. */ + tmpW32 = WEBRTC_SPL_SAT(1073725439, tmpW32, -1073758208); + tmpW32 += 16384; + tmpW16 = (int16_t)(tmpW32 >> 15); + + /* Subtract from input and update buffer. */ + tmpW32 = inputBuf[*index2] - sign * tmpW16; + outputBuf[*index2] = WebRtcSpl_SatW32ToW16(tmpW32); + tmpW32 = inputBuf[*index2] + outputBuf[*index2]; + outputBuf2[*index2 + PITCH_BUFFSIZE] = WebRtcSpl_SatW32ToW16(tmpW32); + + (*index2)++; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_mips.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_mips.c new file mode 100644 index 0000000000..785fd9464f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_mips.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/pitch_estimator.h" + +void WebRtcIsacfix_PitchFilterCore(int loopNumber, + int16_t gain, + size_t index, + int16_t sign, + int16_t* inputState, + int16_t* outputBuf2, + const int16_t* coefficient, + int16_t* inputBuf, + int16_t* outputBuf, + int* index2) { + int ind2t = *index2; + int i = 0; + int16_t* out2_pos2 = &outputBuf2[PITCH_BUFFSIZE - (index + 2)] + ind2t; + int32_t w1, w2, w3, w4, w5, gain32, sign32; + int32_t coef1, coef2, coef3, coef4, coef5 = 0; + // Define damp factors as int32_t (pair of int16_t) + int32_t kDampF0 = 0x0000F70A; + int32_t kDampF1 = 0x51EC2000; + int32_t kDampF2 = 0xF70A2000; + int16_t* input1 = inputBuf + ind2t; + int16_t* output1 = outputBuf + ind2t; + int16_t* output2 = outputBuf2 + ind2t + PITCH_BUFFSIZE; + + // Load coefficients outside the loop and sign-extend gain and sign + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lwl %[coef1], 3(%[coefficient]) \n\t" + "lwl %[coef2], 7(%[coefficient]) \n\t" + "lwl %[coef3], 11(%[coefficient]) \n\t" + "lwl %[coef4], 15(%[coefficient]) \n\t" + "lwr %[coef1], 0(%[coefficient]) \n\t" + "lwr %[coef2], 4(%[coefficient]) \n\t" + "lwr %[coef3], 8(%[coefficient]) \n\t" + "lwr %[coef4], 12(%[coefficient]) \n\t" + "lhu %[coef5], 16(%[coefficient]) \n\t" + "seh %[gain32], %[gain] \n\t" + "seh %[sign32], %[sign] \n\t" + ".set pop \n\t" + : [coef1] "=&r" (coef1), [coef2] "=&r" (coef2), [coef3] "=&r" (coef3), + [coef4] "=&r" (coef4), [coef5] "=&r" (coef5), [gain32] "=&r" (gain32), + [sign32] "=&r" (sign32) + : [coefficient] "r" (coefficient), [gain] "r" (gain), + [sign] "r" (sign) + : "memory" + ); + + for (i = 0; i < loopNumber; i++) { + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + // Filter to get fractional pitch + "li %[w1], 8192 \n\t" + "mtlo %[w1] \n\t" + "mthi $0 \n\t" + "lwl %[w1], 3(%[out2_pos2]) \n\t" + "lwl %[w2], 7(%[out2_pos2]) \n\t" + "lwl %[w3], 11(%[out2_pos2]) \n\t" + "lwl %[w4], 15(%[out2_pos2]) \n\t" + "lwr %[w1], 0(%[out2_pos2]) \n\t" + "lwr %[w2], 4(%[out2_pos2]) \n\t" + "lwr %[w3], 8(%[out2_pos2]) \n\t" + "lwr %[w4], 12(%[out2_pos2]) \n\t" + "lhu %[w5], 16(%[out2_pos2]) \n\t" + "dpa.w.ph $ac0, %[w1], %[coef1] \n\t" + "dpa.w.ph $ac0, %[w2], %[coef2] \n\t" + "dpa.w.ph $ac0, %[w3], %[coef3] \n\t" + "dpa.w.ph $ac0, %[w4], %[coef4] \n\t" + "dpa.w.ph $ac0, %[w5], %[coef5] \n\t" + "addiu %[out2_pos2], %[out2_pos2], 2 \n\t" + "mthi $0, $ac1 \n\t" + "lwl %[w2], 3(%[inputState]) \n\t" + "lwl %[w3], 7(%[inputState]) \n\t" + // Fractional pitch shift & saturation + "extr_s.h %[w1], $ac0, 14 \n\t" + "li %[w4], 16384 \n\t" + "lwr %[w2], 0(%[inputState]) \n\t" + "lwr %[w3], 4(%[inputState]) \n\t" + "mtlo %[w4], $ac1 \n\t" + // Shift low pass filter state + "swl %[w2], 5(%[inputState]) \n\t" + "swl %[w3], 9(%[inputState]) \n\t" + "mul %[w1], %[gain32], %[w1] \n\t" + "swr %[w2], 2(%[inputState]) \n\t" + "swr %[w3], 6(%[inputState]) \n\t" + // Low pass filter accumulation + "dpa.w.ph $ac1, %[kDampF1], %[w2] \n\t" + "dpa.w.ph $ac1, %[kDampF2], %[w3] \n\t" + "lh %[w4], 0(%[input1]) \n\t" + "addiu %[input1], %[input1], 2 \n\t" + "shra_r.w %[w1], %[w1], 12 \n\t" + "sh %[w1], 0(%[inputState]) \n\t" + "dpa.w.ph $ac1, %[kDampF0], %[w1] \n\t" + // Low pass filter shift & saturation + "extr_s.h %[w2], $ac1, 15 \n\t" + "mul %[w2], %[w2], %[sign32] \n\t" + // Buffer update + "subu %[w2], %[w4], %[w2] \n\t" + "shll_s.w %[w2], %[w2], 16 \n\t" + "sra %[w2], %[w2], 16 \n\t" + "sh %[w2], 0(%[output1]) \n\t" + "addu %[w2], %[w2], %[w4] \n\t" + "shll_s.w %[w2], %[w2], 16 \n\t" + "addiu %[output1], %[output1], 2 \n\t" + "sra %[w2], %[w2], 16 \n\t" + "sh %[w2], 0(%[output2]) \n\t" + "addiu %[output2], %[output2], 2 \n\t" + ".set pop \n\t" + : [w1] "=&r" (w1), [w2] "=&r" (w2), [w3] "=&r" (w3), [w4] "=&r" (w4), + [w5] "=&r" (w5), [input1] "+r" (input1), [out2_pos2] "+r" (out2_pos2), + [output1] "+r" (output1), [output2] "+r" (output2) + : [coefficient] "r" (coefficient), [inputState] "r" (inputState), + [gain32] "r" (gain32), [sign32] "r" (sign32), [kDampF0] "r" (kDampF0), + [kDampF1] "r" (kDampF1), [kDampF2] "r" (kDampF2), + [coef1] "r" (coef1), [coef2] "r" (coef2), [coef3] "r" (coef3), + [coef4] "r" (coef4), [coef5] "r" (coef5) + : "hi", "lo", "$ac1hi", "$ac1lo", "memory" + ); + } + (*index2) += loopNumber; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.c new file mode 100644 index 0000000000..bfd83b7258 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_gain_tables.c + * + * This file contains tables for the pitch filter side-info in the entropy coder. + * + */ + +#include "pitch_gain_tables.h" + + +/********************* Pitch Filter Gain Coefficient Tables ************************/ + +/* cdf for quantized pitch filter gains */ +const uint16_t WebRtcIsacfix_kPitchGainCdf[255] = { + 0, 2, 4, 6, 64, 901, 903, 905, 16954, 16956, + 16961, 17360, 17362, 17364, 17366, 17368, 17370, 17372, 17374, 17411, + 17514, 17516, 17583, 18790, 18796, 18802, 20760, 20777, 20782, 21722, + 21724, 21728, 21738, 21740, 21742, 21744, 21746, 21748, 22224, 22227, + 22230, 23214, 23229, 23239, 25086, 25108, 25120, 26088, 26094, 26098, + 26175, 26177, 26179, 26181, 26183, 26185, 26484, 26507, 26522, 27705, + 27731, 27750, 29767, 29799, 29817, 30866, 30883, 30885, 31025, 31029, + 31031, 31033, 31035, 31037, 31114, 31126, 31134, 32687, 32722, 32767, + 35718, 35742, 35757, 36943, 36952, 36954, 37115, 37128, 37130, 37132, + 37134, 37136, 37143, 37145, 37152, 38843, 38863, 38897, 47458, 47467, + 47474, 49040, 49061, 49063, 49145, 49157, 49159, 49161, 49163, 49165, + 49167, 49169, 49171, 49757, 49770, 49782, 61333, 61344, 61346, 62860, + 62883, 62885, 62887, 62889, 62891, 62893, 62895, 62897, 62899, 62901, + 62903, 62905, 62907, 62909, 65496, 65498, 65500, 65521, 65523, 65525, + 65527, 65529, 65531, 65533, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535 +}; + +/* index limits and ranges */ +const int16_t WebRtcIsacfix_kLowerlimiGain[3] = { + -7, -2, -1 +}; + +const int16_t WebRtcIsacfix_kUpperlimitGain[3] = { + 0, 3, 1 +}; + +const uint16_t WebRtcIsacfix_kMultsGain[2] = { + 18, 3 +}; + +/* size of cdf table */ +const uint16_t WebRtcIsacfix_kCdfTableSizeGain[1] = { + 256 +}; + +/* mean values of pitch filter gains in FIXED point Q12 */ +const int16_t WebRtcIsacfix_kPitchGain1[144] = { + 843, 1092, 1336, 1222, 1405, 1656, 1500, 1815, 1843, 1838, 1839, + 1843, 1843, 1843, 1843, 1843, 1843, 1843, 814, 846, 1092, 1013, + 1174, 1383, 1391, 1511, 1584, 1734, 1753, 1843, 1843, 1843, 1843, + 1843, 1843, 1843, 524, 689, 777, 845, 947, 1069, 1090, 1263, + 1380, 1447, 1559, 1676, 1645, 1749, 1843, 1843, 1843, 1843, 81, + 477, 563, 611, 706, 806, 849, 1012, 1192, 1128, 1330, 1489, + 1425, 1576, 1826, 1741, 1843, 1843, 0, 290, 305, 356, 488, + 575, 602, 741, 890, 835, 1079, 1196, 1182, 1376, 1519, 1506, + 1680, 1843, 0, 47, 97, 69, 289, 381, 385, 474, 617, + 664, 803, 1079, 935, 1160, 1269, 1265, 1506, 1741, 0, 0, + 0, 0, 112, 120, 190, 283, 442, 343, 526, 809, 684, + 935, 1134, 1020, 1265, 1506, 0, 0, 0, 0, 0, 0, + 0, 111, 256, 87, 373, 597, 430, 684, 935, 770, 1020, + 1265 +}; + +const int16_t WebRtcIsacfix_kPitchGain2[144] = { + 1760, 1525, 1285, 1747, 1671, 1393, 1843, 1826, 1555, 1843, 1784, + 1606, 1843, 1843, 1711, 1843, 1843, 1814, 1389, 1275, 1040, 1564, + 1414, 1252, 1610, 1495, 1343, 1753, 1592, 1405, 1804, 1720, 1475, + 1843, 1814, 1581, 1208, 1061, 856, 1349, 1148, 994, 1390, 1253, + 1111, 1495, 1343, 1178, 1770, 1465, 1234, 1814, 1581, 1342, 1040, + 793, 713, 1053, 895, 737, 1128, 1003, 861, 1277, 1094, 981, + 1475, 1192, 1019, 1581, 1342, 1098, 855, 570, 483, 833, 648, + 540, 948, 744, 572, 1009, 844, 636, 1234, 934, 685, 1342, + 1217, 984, 537, 318, 124, 603, 423, 350, 687, 479, 322, + 791, 581, 430, 987, 671, 488, 1098, 849, 597, 283, 27, + 0, 397, 222, 38, 513, 271, 124, 624, 325, 157, 737, + 484, 233, 849, 597, 343, 27, 0, 0, 141, 0, 0, + 256, 69, 0, 370, 87, 0, 484, 229, 0, 597, 343, + 87 +}; + +const int16_t WebRtcIsacfix_kPitchGain3[144] = { + 1843, 1843, 1711, 1843, 1818, 1606, 1843, 1827, 1511, 1814, 1639, + 1393, 1760, 1525, 1285, 1656, 1419, 1176, 1835, 1718, 1475, 1841, + 1650, 1387, 1648, 1498, 1287, 1600, 1411, 1176, 1522, 1299, 1040, + 1419, 1176, 928, 1773, 1461, 1128, 1532, 1355, 1202, 1429, 1260, + 1115, 1398, 1151, 1025, 1172, 1080, 790, 1176, 928, 677, 1475, + 1147, 1019, 1276, 1096, 922, 1214, 1010, 901, 1057, 893, 800, + 1040, 796, 734, 928, 677, 424, 1137, 897, 753, 1120, 830, + 710, 875, 751, 601, 795, 642, 583, 790, 544, 475, 677, + 474, 140, 987, 750, 482, 697, 573, 450, 691, 487, 303, + 661, 394, 332, 537, 303, 220, 424, 168, 0, 737, 484, + 229, 624, 348, 153, 441, 261, 136, 397, 166, 51, 283, + 27, 0, 168, 0, 0, 484, 229, 0, 370, 57, 0, + 256, 43, 0, 141, 0, 0, 27, 0, 0, 0, 0, + 0 +}; + + +const int16_t WebRtcIsacfix_kPitchGain4[144] = { + 1843, 1843, 1843, 1843, 1841, 1843, 1500, 1821, 1843, 1222, 1434, + 1656, 843, 1092, 1336, 504, 757, 1007, 1843, 1843, 1843, 1838, + 1791, 1843, 1265, 1505, 1599, 965, 1219, 1425, 730, 821, 1092, + 249, 504, 757, 1783, 1819, 1843, 1351, 1567, 1727, 1096, 1268, + 1409, 805, 961, 1131, 444, 670, 843, 0, 249, 504, 1425, + 1655, 1743, 1096, 1324, 1448, 822, 1019, 1199, 490, 704, 867, + 81, 450, 555, 0, 0, 249, 1247, 1428, 1530, 881, 1073, + 1283, 610, 759, 939, 278, 464, 645, 0, 200, 270, 0, + 0, 0, 935, 1163, 1410, 528, 790, 1068, 377, 499, 717, + 173, 240, 274, 0, 43, 62, 0, 0, 0, 684, 935, + 1182, 343, 551, 735, 161, 262, 423, 0, 55, 27, 0, + 0, 0, 0, 0, 0, 430, 684, 935, 87, 377, 597, + 0, 46, 256, 0, 0, 0, 0, 0, 0, 0, 0, + 0 +}; + + + +/* transform matrix in Q12*/ +const int16_t WebRtcIsacfix_kTransform[4][4] = { + { -2048, -2048, -2048, -2048 }, + { 2748, 916, -916, -2748 }, + { 2048, -2048, -2048, 2048 }, + { 916, -2748, 2748, -916 } +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.h new file mode 100644 index 0000000000..fe4d2883d4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_gain_tables.h + * + * This file contains tables for the pitch filter side-info in the entropy coder. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_GAIN_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_GAIN_TABLES_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/********************* Pitch Filter Gain Coefficient Tables ************************/ +/* cdf for quantized pitch filter gains */ +extern const uint16_t WebRtcIsacfix_kPitchGainCdf[255]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsacfix_kLowerlimiGain[3]; +extern const int16_t WebRtcIsacfix_kUpperlimitGain[3]; +extern const uint16_t WebRtcIsacfix_kMultsGain[2]; + +/* mean values of pitch filter gains in Q12*/ +extern const int16_t WebRtcIsacfix_kPitchGain1[144]; +extern const int16_t WebRtcIsacfix_kPitchGain2[144]; +extern const int16_t WebRtcIsacfix_kPitchGain3[144]; +extern const int16_t WebRtcIsacfix_kPitchGain4[144]; + +/* size of cdf table */ +extern const uint16_t WebRtcIsacfix_kCdfTableSizeGain[1]; + +/* transform matrix */ +extern const int16_t WebRtcIsacfix_kTransform[4][4]; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_GAIN_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.c new file mode 100644 index 0000000000..4566b6eb78 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_lag_tables.c + * + * This file contains tables for the pitch filter side-info in the entropy coder. + * + */ + +#include "settings.h" +#include "pitch_lag_tables.h" + + +/********************* Pitch Filter Gain Coefficient Tables ************************/ + +/* tables for use with small pitch gain */ + +/* cdf for quantized pitch filter lags */ +const uint16_t WebRtcIsacfix_kPitchLagCdf1Lo[127] = { + 0, 134, 336, 549, 778, 998, 1264, 1512, 1777, 2070, + 2423, 2794, 3051, 3361, 3708, 3979, 4315, 4610, 4933, 5269, + 5575, 5896, 6155, 6480, 6816, 7129, 7477, 7764, 8061, 8358, + 8718, 9020, 9390, 9783, 10177, 10543, 10885, 11342, 11795, 12213, + 12680, 13096, 13524, 13919, 14436, 14903, 15349, 15795, 16267, 16734, + 17266, 17697, 18130, 18632, 19080, 19447, 19884, 20315, 20735, 21288, + 21764, 22264, 22723, 23193, 23680, 24111, 24557, 25022, 25537, 26082, + 26543, 27090, 27620, 28139, 28652, 29149, 29634, 30175, 30692, 31273, + 31866, 32506, 33059, 33650, 34296, 34955, 35629, 36295, 36967, 37726, + 38559, 39458, 40364, 41293, 42256, 43215, 44231, 45253, 46274, 47359, + 48482, 49678, 50810, 51853, 53016, 54148, 55235, 56263, 57282, 58363, + 59288, 60179, 61076, 61806, 62474, 63129, 63656, 64160, 64533, 64856, + 65152, 65535, 65535, 65535, 65535, 65535, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf2Lo[20] = { + 0, 429, 3558, 5861, 8558, 11639, 15210, 19502, 24773, 31983, + 42602, 48567, 52601, 55676, 58160, 60172, 61889, 63235, 65383, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf3Lo[2] = { + 0, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf4Lo[10] = { + 0, 2966, 6368, 11182, 19431, 37793, 48532, 55353, 60626, 65535 +}; + +const uint16_t *WebRtcIsacfix_kPitchLagPtrLo[4] = { + WebRtcIsacfix_kPitchLagCdf1Lo, + WebRtcIsacfix_kPitchLagCdf2Lo, + WebRtcIsacfix_kPitchLagCdf3Lo, + WebRtcIsacfix_kPitchLagCdf4Lo +}; + +/* size of first cdf table */ +const uint16_t WebRtcIsacfix_kPitchLagSizeLo[1] = { + 128 +}; + +/* index limits and ranges */ +const int16_t WebRtcIsacfix_kLowerLimitLo[4] = { + -140, -9, 0, -4 +}; + +const int16_t WebRtcIsacfix_kUpperLimitLo[4] = { + -20, 9, 0, 4 +}; + +/* initial index for arithmetic decoder */ +const uint16_t WebRtcIsacfix_kInitIndLo[3] = { + 10, 1, 5 +}; + +/* mean values of pitch filter lags in Q10 */ + +const int16_t WebRtcIsacfix_kMeanLag2Lo[19] = { + -17627, -16207, -14409, -12319, -10253, -8200, -6054, -3986, -1948, -19, + 1937, 3974, 6064, 8155, 10229, 12270, 14296, 16127, 17520 +}; + +const int16_t WebRtcIsacfix_kMeanLag4Lo[9] = { + -7949, -6063, -4036, -1941, 38, 1977, 4060, 6059 +}; + + + +/* tables for use with medium pitch gain */ + +/* cdf for quantized pitch filter lags */ +const uint16_t WebRtcIsacfix_kPitchLagCdf1Mid[255] = { + 0, 28, 61, 88, 121, 149, 233, 331, 475, 559, + 624, 661, 689, 712, 745, 791, 815, 843, 866, 922, + 959, 1024, 1061, 1117, 1178, 1238, 1280, 1350, 1453, 1513, + 1564, 1625, 1671, 1741, 1788, 1904, 2072, 2421, 2626, 2770, + 2840, 2900, 2942, 3012, 3068, 3115, 3147, 3194, 3254, 3319, + 3366, 3520, 3678, 3780, 3850, 3911, 3957, 4032, 4106, 4185, + 4292, 4474, 4683, 4842, 5019, 5191, 5321, 5428, 5540, 5675, + 5763, 5847, 5959, 6127, 6304, 6564, 6839, 7090, 7263, 7421, + 7556, 7728, 7872, 7984, 8142, 8361, 8580, 8743, 8938, 9227, + 9409, 9539, 9674, 9795, 9930, 10060, 10177, 10382, 10614, 10861, + 11038, 11271, 11415, 11629, 11792, 12044, 12193, 12416, 12574, 12821, + 13007, 13235, 13445, 13654, 13901, 14134, 14488, 15000, 15703, 16285, + 16504, 16797, 17086, 17328, 17579, 17807, 17998, 18268, 18538, 18836, + 19087, 19274, 19474, 19716, 19935, 20270, 20833, 21303, 21532, 21741, + 21978, 22207, 22523, 22770, 23054, 23613, 23943, 24204, 24399, 24651, + 24832, 25074, 25270, 25549, 25759, 26015, 26150, 26424, 26713, 27048, + 27342, 27504, 27681, 27854, 28021, 28207, 28412, 28664, 28859, 29064, + 29278, 29548, 29748, 30107, 30377, 30656, 30856, 31164, 31452, 31755, + 32011, 32328, 32626, 32919, 33319, 33789, 34329, 34925, 35396, 35973, + 36443, 36964, 37551, 38156, 38724, 39357, 40023, 40908, 41587, 42602, + 43924, 45037, 45810, 46597, 47421, 48291, 49092, 50051, 51448, 52719, + 53440, 54241, 54944, 55977, 56676, 57299, 57872, 58389, 59059, 59688, + 60237, 60782, 61094, 61573, 61890, 62290, 62658, 63030, 63217, 63454, + 63622, 63882, 64003, 64273, 64427, 64529, 64581, 64697, 64758, 64902, + 65414, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf2Mid[36] = { + 0, 71, 335, 581, 836, 1039, 1323, 1795, 2258, 2608, + 3005, 3591, 4243, 5344, 7163, 10583, 16848, 28078, 49448, 57007, + 60357, 61850, 62837, 63437, 63872, 64188, 64377, 64614, 64774, 64949, + 65039, 65115, 65223, 65360, 65474, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf3Mid[2] = { + 0, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf4Mid[20] = { + 0, 28, 246, 459, 667, 1045, 1523, 2337, 4337, 11347, + 44231, 56709, 60781, 62243, 63161, 63969, 64608, 65062, 65502, 65535 +}; + +const uint16_t *WebRtcIsacfix_kPitchLagPtrMid[4] = { + WebRtcIsacfix_kPitchLagCdf1Mid, + WebRtcIsacfix_kPitchLagCdf2Mid, + WebRtcIsacfix_kPitchLagCdf3Mid, + WebRtcIsacfix_kPitchLagCdf4Mid +}; + +/* size of first cdf table */ +const uint16_t WebRtcIsacfix_kPitchLagSizeMid[1] = { + 256 +}; + +/* index limits and ranges */ +const int16_t WebRtcIsacfix_kLowerLimitMid[4] = { + -280, -17, 0, -9 +}; + +const int16_t WebRtcIsacfix_kUpperLimitMid[4] = { + -40, 17, 0, 9 +}; + +/* initial index for arithmetic decoder */ +const uint16_t WebRtcIsacfix_kInitIndMid[3] = { + 18, 1, 10 +}; + +/* mean values of pitch filter lags in Q10 */ + +const int16_t WebRtcIsacfix_kMeanLag2Mid[35] = { + -17297, -16250, -15416, -14343, -13341, -12363, -11270, + -10355, -9122, -8217, -7172, -6083, -5102, -4004, -3060, + -1982, -952, -18, 935, 1976, 3040, 4032, + 5082, 6065, 7257, 8202, 9264, 10225, 11242, + 12234, 13337, 14336, 15374, 16187, 17347 +}; + + +const int16_t WebRtcIsacfix_kMeanLag4Mid[19] = { + -8811, -8081, -7203, -6003, -5057, -4025, -2983, -1964, + -891, 29, 921, 1920, 2988, 4064, 5187, 6079, 7173, 8074, 8849 +}; + + +/* tables for use with large pitch gain */ + +/* cdf for quantized pitch filter lags */ +const uint16_t WebRtcIsacfix_kPitchLagCdf1Hi[511] = { + 0, 7, 18, 33, 69, 105, 156, 228, 315, 612, + 680, 691, 709, 724, 735, 738, 742, 746, 749, 753, + 756, 760, 764, 774, 782, 785, 789, 796, 800, 803, + 807, 814, 818, 822, 829, 832, 847, 854, 858, 869, + 876, 883, 898, 908, 934, 977, 1010, 1050, 1060, 1064, + 1075, 1078, 1086, 1089, 1093, 1104, 1111, 1122, 1133, 1136, + 1151, 1162, 1183, 1209, 1252, 1281, 1339, 1364, 1386, 1401, + 1411, 1415, 1426, 1430, 1433, 1440, 1448, 1455, 1462, 1477, + 1487, 1495, 1502, 1506, 1509, 1516, 1524, 1531, 1535, 1542, + 1553, 1556, 1578, 1589, 1611, 1625, 1639, 1643, 1654, 1665, + 1672, 1687, 1694, 1705, 1708, 1719, 1730, 1744, 1752, 1759, + 1791, 1795, 1820, 1867, 1886, 1915, 1936, 1943, 1965, 1987, + 2041, 2099, 2161, 2175, 2200, 2211, 2226, 2233, 2244, 2251, + 2266, 2280, 2287, 2298, 2309, 2316, 2331, 2342, 2356, 2378, + 2403, 2418, 2447, 2497, 2544, 2602, 2863, 2895, 2903, 2935, + 2950, 2971, 3004, 3011, 3018, 3029, 3040, 3062, 3087, 3127, + 3152, 3170, 3199, 3243, 3293, 3322, 3340, 3377, 3402, 3427, + 3474, 3518, 3543, 3579, 3601, 3637, 3659, 3706, 3731, 3760, + 3818, 3847, 3869, 3901, 3920, 3952, 4068, 4169, 4220, 4271, + 4524, 4571, 4604, 4632, 4672, 4730, 4777, 4806, 4857, 4904, + 4951, 5002, 5031, 5060, 5107, 5150, 5212, 5266, 5331, 5382, + 5432, 5490, 5544, 5610, 5700, 5762, 5812, 5874, 5972, 6022, + 6091, 6163, 6232, 6305, 6402, 6540, 6685, 6880, 7090, 7271, + 7379, 7452, 7542, 7625, 7687, 7770, 7843, 7911, 7966, 8024, + 8096, 8190, 8252, 8320, 8411, 8501, 8585, 8639, 8751, 8842, + 8918, 8986, 9066, 9127, 9203, 9269, 9345, 9406, 9464, 9536, + 9612, 9667, 9735, 9844, 9931, 10036, 10119, 10199, 10260, 10358, + 10441, 10514, 10666, 10734, 10872, 10951, 11053, 11125, 11223, 11324, + 11516, 11664, 11737, 11816, 11892, 12008, 12120, 12200, 12280, 12392, + 12490, 12576, 12685, 12812, 12917, 13003, 13108, 13210, 13300, 13384, + 13470, 13579, 13673, 13771, 13879, 13999, 14136, 14201, 14368, 14614, + 14759, 14867, 14958, 15030, 15121, 15189, 15280, 15385, 15461, 15555, + 15653, 15768, 15884, 15971, 16069, 16145, 16210, 16279, 16380, 16463, + 16539, 16615, 16688, 16818, 16919, 17017, 18041, 18338, 18523, 18649, + 18790, 18917, 19047, 19167, 19315, 19460, 19601, 19731, 19858, 20068, + 20173, 20318, 20466, 20625, 20741, 20911, 21045, 21201, 21396, 21588, + 21816, 22022, 22305, 22547, 22786, 23072, 23322, 23600, 23879, 24168, + 24433, 24769, 25120, 25511, 25895, 26289, 26792, 27219, 27683, 28077, + 28566, 29094, 29546, 29977, 30491, 30991, 31573, 32105, 32594, 33173, + 33788, 34497, 35181, 35833, 36488, 37255, 37921, 38645, 39275, 39894, + 40505, 41167, 41790, 42431, 43096, 43723, 44385, 45134, 45858, 46607, + 47349, 48091, 48768, 49405, 49955, 50555, 51167, 51985, 52611, 53078, + 53494, 53965, 54435, 54996, 55601, 56125, 56563, 56838, 57244, 57566, + 57967, 58297, 58771, 59093, 59419, 59647, 59886, 60143, 60461, 60693, + 60917, 61170, 61416, 61634, 61891, 62122, 62310, 62455, 62632, 62839, + 63103, 63436, 63639, 63805, 63906, 64015, 64192, 64355, 64475, 64558, + 64663, 64742, 64811, 64865, 64916, 64956, 64981, 65025, 65068, 65115, + 65195, 65314, 65419, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf2Hi[68] = { + 0, 7, 11, 22, 37, 52, 56, 59, 81, 85, + 89, 96, 115, 130, 137, 152, 170, 181, 193, 200, + 207, 233, 237, 259, 289, 318, 363, 433, 592, 992, + 1607, 3062, 6149, 12206, 25522, 48368, 58223, 61918, 63640, 64584, + 64943, 65098, 65206, 65268, 65294, 65335, 65350, 65372, 65387, 65402, + 65413, 65420, 65428, 65435, 65439, 65450, 65454, 65468, 65472, 65476, + 65483, 65491, 65498, 65505, 65516, 65520, 65528, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf3Hi[2] = { + 0, 65535 +}; + +const uint16_t WebRtcIsacfix_kPitchLagCdf4Hi[35] = { + 0, 7, 19, 30, 41, 48, 63, 74, 82, 96, + 122, 152, 215, 330, 701, 2611, 10931, 48106, 61177, 64341, + 65112, 65238, 65309, 65338, 65364, 65379, 65401, 65427, 65453, + 65465, 65476, 65490, 65509, 65528, 65535 +}; + +const uint16_t *WebRtcIsacfix_kPitchLagPtrHi[4] = { + WebRtcIsacfix_kPitchLagCdf1Hi, + WebRtcIsacfix_kPitchLagCdf2Hi, + WebRtcIsacfix_kPitchLagCdf3Hi, + WebRtcIsacfix_kPitchLagCdf4Hi +}; + +/* size of first cdf table */ +const uint16_t WebRtcIsacfix_kPitchLagSizeHi[1] = { + 512 +}; + +/* index limits and ranges */ +const int16_t WebRtcIsacfix_kLowerLimitHi[4] = { + -552, -34, 0, -16 +}; + +const int16_t WebRtcIsacfix_kUpperLimitHi[4] = { + -80, 32, 0, 17 +}; + +/* initial index for arithmetic decoder */ +const uint16_t WebRtcIsacfix_kInitIndHi[3] = { + 34, 1, 18 +}; + +/* mean values of pitch filter lags */ + +const int16_t WebRtcIsacfix_kMeanLag2Hi[67] = { + -17482, -16896, -16220, -15929, -15329, -14848, -14336, -13807, -13312, -12800, -12218, -11720, + -11307, -10649, -10396, -9742, -9148, -8668, -8297, -7718, -7155, -6656, -6231, -5600, -5129, + -4610, -4110, -3521, -3040, -2525, -2016, -1506, -995, -477, -5, 469, 991, 1510, 2025, 2526, 3079, + 3555, 4124, 4601, 5131, 5613, 6194, 6671, 7140, 7645, 8207, 8601, 9132, 9728, 10359, 10752, 11302, + 11776, 12288, 12687, 13204, 13759, 14295, 14810, 15360, 15764, 16350 +}; + + +const int16_t WebRtcIsacfix_kMeanLag4Hi[34] = { + -8175, -7659, -7205, -6684, -6215, -5651, -5180, -4566, -4087, -3536, -3096, + -2532, -1990, -1482, -959, -440, 11, 451, 954, 1492, 2020, 2562, 3059, + 3577, 4113, 4618, 5134, 5724, 6060, 6758, 7015, 7716, 8066, 8741 +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.h new file mode 100644 index 0000000000..a8c0c3ae7e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_lag_tables.h + * + * This file contains tables for the pitch filter side-info in the entropy coder. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_LAG_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_LAG_TABLES_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/********************* Pitch Filter Lag Coefficient Tables ************************/ + +/* tables for use with small pitch gain */ + +/* cdfs for quantized pitch lags */ +extern const uint16_t WebRtcIsacfix_kPitchLagCdf1Lo[127]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf2Lo[20]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf3Lo[2]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf4Lo[10]; + +extern const uint16_t *WebRtcIsacfix_kPitchLagPtrLo[4]; + +/* size of first cdf table */ +extern const uint16_t WebRtcIsacfix_kPitchLagSizeLo[1]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsacfix_kLowerLimitLo[4]; +extern const int16_t WebRtcIsacfix_kUpperLimitLo[4]; + +/* initial index for arithmetic decoder */ +extern const uint16_t WebRtcIsacfix_kInitIndLo[3]; + +/* mean values of pitch filter lags */ +extern const int16_t WebRtcIsacfix_kMeanLag2Lo[19]; +extern const int16_t WebRtcIsacfix_kMeanLag4Lo[9]; + + + +/* tables for use with medium pitch gain */ + +/* cdfs for quantized pitch lags */ +extern const uint16_t WebRtcIsacfix_kPitchLagCdf1Mid[255]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf2Mid[36]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf3Mid[2]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf4Mid[20]; + +extern const uint16_t *WebRtcIsacfix_kPitchLagPtrMid[4]; + +/* size of first cdf table */ +extern const uint16_t WebRtcIsacfix_kPitchLagSizeMid[1]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsacfix_kLowerLimitMid[4]; +extern const int16_t WebRtcIsacfix_kUpperLimitMid[4]; + +/* initial index for arithmetic decoder */ +extern const uint16_t WebRtcIsacfix_kInitIndMid[3]; + +/* mean values of pitch filter lags */ +extern const int16_t WebRtcIsacfix_kMeanLag2Mid[35]; +extern const int16_t WebRtcIsacfix_kMeanLag4Mid[19]; + + +/* tables for use with large pitch gain */ + +/* cdfs for quantized pitch lags */ +extern const uint16_t WebRtcIsacfix_kPitchLagCdf1Hi[511]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf2Hi[68]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf3Hi[2]; +extern const uint16_t WebRtcIsacfix_kPitchLagCdf4Hi[35]; + +extern const uint16_t *WebRtcIsacfix_kPitchLagPtrHi[4]; + +/* size of first cdf table */ +extern const uint16_t WebRtcIsacfix_kPitchLagSizeHi[1]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsacfix_kLowerLimitHi[4]; +extern const int16_t WebRtcIsacfix_kUpperLimitHi[4]; + +/* initial index for arithmetic decoder */ +extern const uint16_t WebRtcIsacfix_kInitIndHi[3]; + +/* mean values of pitch filter lags */ +extern const int16_t WebRtcIsacfix_kMeanLag2Hi[67]; +extern const int16_t WebRtcIsacfix_kMeanLag4Hi[34]; + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_PITCH_LAG_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/settings.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/settings.h new file mode 100644 index 0000000000..34c0efefd0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/settings.h @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * settings.h + * + * Declaration of #defines used in the iSAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_SETTINGS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_SETTINGS_H_ + + +/* sampling frequency (Hz) */ +#define FS 16000 +/* 1.5 times Sampling frequency */ +#define FS_1_HALF (uint32_t) 24000 +/* Three times Sampling frequency */ +#define FS3 (uint32_t) 48000 +/* Eight times Sampling frequency */ +#define FS8 (uint32_t) 128000 + +/* number of samples per frame (either 480 (30ms) or 960 (60ms)) */ +#define INITIAL_FRAMESAMPLES 960 + +/* miliseconds */ +#define FRAMESIZE 30 +/* number of samples per frame processed in the encoder (30ms) */ +#define FRAMESAMPLES 480 /* ((FRAMESIZE*FS)/1000) */ +#define FRAMESAMPLES_HALF 240 +/* max number of samples per frame (= 60 ms frame) */ +#define MAX_FRAMESAMPLES 960 +/* number of samples per 10ms frame */ +#define FRAMESAMPLES_10ms 160 /* ((10*FS)/1000) */ +/* Number of samples per 1 ms */ +#define SAMPLES_PER_MSEC 16 +/* number of subframes */ +#define SUBFRAMES 6 +/* length of a subframe */ +#define UPDATE 80 +/* length of half a subframe (low/high band) */ +#define HALF_SUBFRAMELEN 40 /* (UPDATE/2) */ +/* samples of look ahead (in a half-band, so actually half the samples of look ahead @ FS) */ +#define QLOOKAHEAD 24 /* 3 ms */ + +/* order of AR model in spectral entropy coder */ +#define AR_ORDER 6 +#define MAX_ORDER 13 +#define LEVINSON_MAX_ORDER 12 + +/* window length (masking analysis) */ +#define WINLEN 256 +/* order of low-band pole filter used to approximate masking curve */ +#define ORDERLO 12 +/* order of hi-band pole filter used to approximate masking curve */ +#define ORDERHI 6 + +#define KLT_NUM_AVG_GAIN 0 +#define KLT_NUM_AVG_SHAPE 0 +#define KLT_NUM_MODELS 3 +#define LPC_SHAPE_ORDER 18 /* (ORDERLO + ORDERHI) */ + +#define KLT_ORDER_GAIN 12 /* (2 * SUBFRAMES) */ +#define KLT_ORDER_SHAPE 108 /* (LPC_SHAPE_ORDER * SUBFRAMES) */ + + + +/* order for post_filter_bank */ +#define POSTQORDER 3 +/* order for pre-filterbank */ +#define QORDER 3 +/* for decimator */ +#define ALLPASSSECTIONS 2 +/* The number of composite all-pass filter factors */ +#define NUMBEROFCOMPOSITEAPSECTIONS 4 + +/* The number of all-pass filter factors in an upper or lower channel*/ +#define NUMBEROFCHANNELAPSECTIONS 2 + + + +#define DPMIN_Q10 -10240 /* -10.00 in Q10 */ +#define DPMAX_Q10 10240 /* 10.00 in Q10 */ +#define MINBITS_Q10 10240 /* 10.0 in Q10 */ + + +/* array size for byte stream in number of Word16. */ +#define STREAM_MAXW16 300 /* The old maximum size still needed for the decoding */ +#define STREAM_MAXW16_30MS 100 /* 100 Word16 = 200 bytes = 53.4 kbit/s @ 30 ms.framelength */ +#define STREAM_MAXW16_60MS 200 /* 200 Word16 = 400 bytes = 53.4 kbit/s @ 60 ms.framelength */ +/* This is used only at the decoder bit-stream struct. + * - The encoder and decoder bitstream containers are of different size because + * old iSAC limited the encoded bitstream to 600 bytes. But newer versions + * restrict to shorter bitstream. + * - We add 10 bytes of guards to the internal bitstream container. The reason + * is that entropy decoder might read few bytes (3 according to our + * observations) more than the actual size of the bitstream. To avoid reading + * outside memory, in rare occasion of full-size bitstream we add 10 bytes + * of guard. */ +#define INTERNAL_STREAM_SIZE_W16 (STREAM_MAXW16 + 5) + +/* storage size for bit counts */ +//#define BIT_COUNTER_SIZE 30 +/* maximum order of any AR model or filter */ +#define MAX_AR_MODEL_ORDER 12 + +/* Maximum number of iterations allowed to limit payload size */ +#define MAX_PAYLOAD_LIMIT_ITERATION 1 + +/* Bandwidth estimator */ + +#define MIN_ISAC_BW 10000 /* Minimum bandwidth in bits per sec */ +#define MAX_ISAC_BW 32000 /* Maxmum bandwidth in bits per sec */ +#define MIN_ISAC_MD 5 /* Minimum Max Delay in ?? */ +#define MAX_ISAC_MD 25 /* Maxmum Max Delay in ?? */ +#define DELAY_CORRECTION_MAX 717 +#define DELAY_CORRECTION_MED 819 +#define Thld_30_60 18000 +#define Thld_60_30 27000 + +/* assumed header size; we don't know the exact number (header compression may be used) */ +#define HEADER_SIZE 35 /* bytes */ +#define INIT_FRAME_LEN 60 +#define INIT_BN_EST 20000 +#define INIT_BN_EST_Q7 2560000 /* 20 kbps in Q7 */ +#define INIT_REC_BN_EST_Q5 789312 /* INIT_BN_EST + INIT_HDR_RATE in Q5 */ + +/* 8738 in Q18 is ~ 1/30 */ +/* #define INIT_HDR_RATE (((HEADER_SIZE * 8 * 1000) * 8738) >> NUM_BITS_TO_SHIFT (INIT_FRAME_LEN)) */ +#define INIT_HDR_RATE 4666 +/* number of packets in a row for a high rate burst */ +#define BURST_LEN 3 +/* ms, max time between two full bursts */ +#define BURST_INTERVAL 800 +/* number of packets in a row for initial high rate burst */ +#define INIT_BURST_LEN 5 +/* bits/s, rate for the first BURST_LEN packets */ +#define INIT_RATE 10240000 /* INIT_BN_EST in Q9 */ + + +/* For pitch analysis */ +#define PITCH_FRAME_LEN 240 /* (FRAMESAMPLES/2) 30 ms */ +#define PITCH_MAX_LAG 140 /* 57 Hz */ +#define PITCH_MIN_LAG 20 /* 400 Hz */ +#define PITCH_MIN_LAG_Q8 5120 /* 256 * PITCH_MIN_LAG */ +#define OFFSET_Q8 768 /* 256 * 3 */ + +#define PITCH_MAX_GAIN_Q12 1843 /* 0.45 */ +#define PITCH_LAG_SPAN2 65 /* (PITCH_MAX_LAG/2-PITCH_MIN_LAG/2+5) */ +#define PITCH_CORR_LEN2 60 /* 15 ms */ +#define PITCH_CORR_STEP2 60 /* (PITCH_FRAME_LEN/4) */ +#define PITCH_SUBFRAMES 4 +#define PITCH_SUBFRAME_LEN 60 /* (PITCH_FRAME_LEN/PITCH_SUBFRAMES) */ + +/* For pitch filter */ +#define PITCH_BUFFSIZE 190 /* (PITCH_MAX_LAG + 50) Extra 50 for fraction and LP filters */ +#define PITCH_INTBUFFSIZE 430 /* (PITCH_FRAME_LEN+PITCH_BUFFSIZE) */ +#define PITCH_FRACS 8 +#define PITCH_FRACORDER 9 +#define PITCH_DAMPORDER 5 + + +/* Order of high pass filter */ +#define HPORDER 2 + + +/* PLC */ +#define DECAY_RATE 10 /* Q15, 20% of decay every lost frame apllied linearly sample by sample*/ +#define PLC_WAS_USED 1 +#define PLC_NOT_USED 3 +#define RECOVERY_OVERLAP 80 +#define RESAMP_RES 256 +#define RESAMP_RES_BIT 8 + + + +/* Define Error codes */ +/* 6000 General */ +#define ISAC_MEMORY_ALLOCATION_FAILED 6010 +#define ISAC_MODE_MISMATCH 6020 +#define ISAC_DISALLOWED_BOTTLENECK 6030 +#define ISAC_DISALLOWED_FRAME_LENGTH 6040 +/* 6200 Bandwidth estimator */ +#define ISAC_RANGE_ERROR_BW_ESTIMATOR 6240 +/* 6400 Encoder */ +#define ISAC_ENCODER_NOT_INITIATED 6410 +#define ISAC_DISALLOWED_CODING_MODE 6420 +#define ISAC_DISALLOWED_FRAME_MODE_ENCODER 6430 +#define ISAC_DISALLOWED_BITSTREAM_LENGTH 6440 +#define ISAC_PAYLOAD_LARGER_THAN_LIMIT 6450 +/* 6600 Decoder */ +#define ISAC_DECODER_NOT_INITIATED 6610 +#define ISAC_EMPTY_PACKET 6620 +#define ISAC_PACKET_TOO_SHORT 6625 +#define ISAC_DISALLOWED_FRAME_MODE_DECODER 6630 +#define ISAC_RANGE_ERROR_DECODE_FRAME_LENGTH 6640 +#define ISAC_RANGE_ERROR_DECODE_BANDWIDTH 6650 +#define ISAC_RANGE_ERROR_DECODE_PITCH_GAIN 6660 +#define ISAC_RANGE_ERROR_DECODE_PITCH_LAG 6670 +#define ISAC_RANGE_ERROR_DECODE_LPC 6680 +#define ISAC_RANGE_ERROR_DECODE_SPECTRUM 6690 +#define ISAC_LENGTH_MISMATCH 6730 +/* 6800 Call setup formats */ +#define ISAC_INCOMPATIBLE_FORMATS 6810 + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_SETTINGS_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.c new file mode 100644 index 0000000000..cf2dea7507 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.c @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * spectrum_ar_model_tables.c + * + * This file contains tables with AR coefficients, Gain coefficients + * and cosine tables. + * + */ + +#include "spectrum_ar_model_tables.h" +#include "settings.h" + +/********************* AR Coefficient Tables ************************/ + +/* cdf for quantized reflection coefficient 1 */ +const uint16_t WebRtcIsacfix_kRc1Cdf[12] = { + 0, 2, 4, 129, 7707, 57485, 65495, 65527, 65529, 65531, + 65533, 65535 +}; + +/* cdf for quantized reflection coefficient 2 */ +const uint16_t WebRtcIsacfix_kRc2Cdf[12] = { + 0, 2, 4, 7, 531, 25298, 64525, 65526, 65529, 65531, + 65533, 65535 +}; + +/* cdf for quantized reflection coefficient 3 */ +const uint16_t WebRtcIsacfix_kRc3Cdf[12] = { + 0, 2, 4, 6, 620, 22898, 64843, 65527, 65529, 65531, + 65533, 65535 +}; + +/* cdf for quantized reflection coefficient 4 */ +const uint16_t WebRtcIsacfix_kRc4Cdf[12] = { + 0, 2, 4, 6, 35, 10034, 60733, 65506, 65529, 65531, + 65533, 65535 +}; + +/* cdf for quantized reflection coefficient 5 */ +const uint16_t WebRtcIsacfix_kRc5Cdf[12] = { + 0, 2, 4, 6, 36, 7567, 56727, 65385, 65529, 65531, + 65533, 65535 +}; + +/* cdf for quantized reflection coefficient 6 */ +const uint16_t WebRtcIsacfix_kRc6Cdf[12] = { + 0, 2, 4, 6, 14, 6579, 57360, 65409, 65529, 65531, + 65533, 65535 +}; + +/* representation levels for quantized reflection coefficient 1 */ +const int16_t WebRtcIsacfix_kRc1Levels[11] = { + -32104, -29007, -23202, -15496, -9279, -2577, 5934, 17535, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 2 */ +const int16_t WebRtcIsacfix_kRc2Levels[11] = { + -32104, -29503, -23494, -15261, -7309, -1399, 6158, 16381, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 3 */ +const int16_t WebRtcIsacfix_kRc3Levels[11] = { + -32104, -29503, -23157, -15186, -7347, -1359, 5829, 17535, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 4 */ +const int16_t WebRtcIsacfix_kRc4Levels[11] = { + -32104, -29503, -24512, -15362, -6665, -342, 6596, 14585, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 5 */ +const int16_t WebRtcIsacfix_kRc5Levels[11] = { + -32104, -29503, -24512, -15005, -6564, -106, 7123, 14920, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 6 */ +const int16_t WebRtcIsacfix_kRc6Levels[11] = { + -32104, -29503, -24512, -15096, -6656, -37, 7036, 14847, 24512, 29503, 32104 +}; + +/* quantization boundary levels for reflection coefficients */ +const int16_t WebRtcIsacfix_kRcBound[12] = { + -32768, -31441, -27566, -21458, -13612, -4663, + 4663, 13612, 21458, 27566, 31441, 32767 +}; + +/* initial index for AR reflection coefficient quantizer and cdf table search */ +const uint16_t WebRtcIsacfix_kRcInitInd[6] = { + 5, 5, 5, 5, 5, 5 +}; + +/* pointers to AR cdf tables */ +const uint16_t *WebRtcIsacfix_kRcCdfPtr[AR_ORDER] = { + WebRtcIsacfix_kRc1Cdf, + WebRtcIsacfix_kRc2Cdf, + WebRtcIsacfix_kRc3Cdf, + WebRtcIsacfix_kRc4Cdf, + WebRtcIsacfix_kRc5Cdf, + WebRtcIsacfix_kRc6Cdf +}; + +/* pointers to AR representation levels tables */ +const int16_t *WebRtcIsacfix_kRcLevPtr[AR_ORDER] = { + WebRtcIsacfix_kRc1Levels, + WebRtcIsacfix_kRc2Levels, + WebRtcIsacfix_kRc3Levels, + WebRtcIsacfix_kRc4Levels, + WebRtcIsacfix_kRc5Levels, + WebRtcIsacfix_kRc6Levels +}; + + +/******************** GAIN Coefficient Tables ***********************/ + +/* cdf for Gain coefficient */ +const uint16_t WebRtcIsacfix_kGainCdf[19] = { + 0, 2, 4, 6, 8, 10, 12, 14, 16, 1172, + 11119, 29411, 51699, 64445, 65527, 65529, 65531, 65533, 65535 +}; + +/* representation levels for quantized squared Gain coefficient */ +const int32_t WebRtcIsacfix_kGain2Lev[18] = { + 128, 128, 128, 128, 128, 215, 364, 709, 1268, + 1960, 3405, 6078, 11286, 17827, 51918, 134498, 487432, 2048000 +}; + +/* quantization boundary levels for squared Gain coefficient */ +const int32_t WebRtcIsacfix_kGain2Bound[19] = { + 0, 21, 35, 59, 99, 166, 280, 475, 815, 1414, + 2495, 4505, 8397, 16405, 34431, 81359, 240497, 921600, 0x7FFFFFFF +}; + +/* pointers to Gain cdf table */ +const uint16_t *WebRtcIsacfix_kGainPtr[1] = { + WebRtcIsacfix_kGainCdf +}; + +/* gain initial index for gain quantizer and cdf table search */ +const uint16_t WebRtcIsacfix_kGainInitInd[1] = { + 11 +}; + + +/************************* Cosine Tables ****************************/ + +/* cosine table */ +const int16_t WebRtcIsacfix_kCos[6][60] = { + { 512, 512, 511, 510, 508, 507, 505, 502, 499, 496, + 493, 489, 485, 480, 476, 470, 465, 459, 453, 447, + 440, 433, 426, 418, 410, 402, 394, 385, 376, 367, + 357, 348, 338, 327, 317, 306, 295, 284, 273, 262, + 250, 238, 226, 214, 202, 190, 177, 165, 152, 139, + 126, 113, 100, 87, 73, 60, 47, 33, 20, 7 }, + { 512, 510, 508, 503, 498, 491, 483, 473, 462, 450, + 437, 422, 406, 389, 371, 352, 333, 312, 290, 268, + 244, 220, 196, 171, 145, 120, 93, 67, 40, 13, + -13, -40, -67, -93, -120, -145, -171, -196, -220, -244, + -268, -290, -312, -333, -352, -371, -389, -406, -422, -437, + -450, -462, -473, -483, -491, -498, -503, -508, -510, -512 }, + { 512, 508, 502, 493, 480, 465, 447, 426, 402, 376, + 348, 317, 284, 250, 214, 177, 139, 100, 60, 20, + -20, -60, -100, -139, -177, -214, -250, -284, -317, -348, + -376, -402, -426, -447, -465, -480, -493, -502, -508, -512, + -512, -508, -502, -493, -480, -465, -447, -426, -402, -376, + -348, -317, -284, -250, -214, -177, -139, -100, -60, -20 }, + { 511, 506, 495, 478, 456, 429, 398, 362, 322, 279, + 232, 183, 133, 80, 27, -27, -80, -133, -183, -232, + -279, -322, -362, -398, -429, -456, -478, -495, -506, -511, + -511, -506, -495, -478, -456, -429, -398, -362, -322, -279, + -232, -183, -133, -80, -27, 27, 80, 133, 183, 232, + 279, 322, 362, 398, 429, 456, 478, 495, 506, 511 }, + { 511, 502, 485, 459, 426, 385, 338, 284, 226, 165, + 100, 33, -33, -100, -165, -226, -284, -338, -385, -426, + -459, -485, -502, -511, -511, -502, -485, -459, -426, -385, + -338, -284, -226, -165, -100, -33, 33, 100, 165, 226, + 284, 338, 385, 426, 459, 485, 502, 511, 511, 502, + 485, 459, 426, 385, 338, 284, 226, 165, 100, 33 }, + { 510, 498, 473, 437, 389, 333, 268, 196, 120, 40, + -40, -120, -196, -268, -333, -389, -437, -473, -498, -510, + -510, -498, -473, -437, -389, -333, -268, -196, -120, -40, + 40, 120, 196, 268, 333, 389, 437, 473, 498, 510, + 510, 498, 473, 437, 389, 333, 268, 196, 120, 40, + -40, -120, -196, -268, -333, -389, -437, -473, -498, -510 } +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.h new file mode 100644 index 0000000000..9a7e946396 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * spectrum_ar_model_tables.h + * + * This file contains definitions of tables with AR coefficients, + * Gain coefficients and cosine tables. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_SPECTRUM_AR_MODEL_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_SPECTRUM_AR_MODEL_TABLES_H_ + +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +/********************* AR Coefficient Tables ************************/ +/* cdf for quantized reflection coefficient 1 */ +extern const uint16_t WebRtcIsacfix_kRc1Cdf[12]; + +/* cdf for quantized reflection coefficient 2 */ +extern const uint16_t WebRtcIsacfix_kRc2Cdf[12]; + +/* cdf for quantized reflection coefficient 3 */ +extern const uint16_t WebRtcIsacfix_kRc3Cdf[12]; + +/* cdf for quantized reflection coefficient 4 */ +extern const uint16_t WebRtcIsacfix_kRc4Cdf[12]; + +/* cdf for quantized reflection coefficient 5 */ +extern const uint16_t WebRtcIsacfix_kRc5Cdf[12]; + +/* cdf for quantized reflection coefficient 6 */ +extern const uint16_t WebRtcIsacfix_kRc6Cdf[12]; + +/* representation levels for quantized reflection coefficient 1 */ +extern const int16_t WebRtcIsacfix_kRc1Levels[11]; + +/* representation levels for quantized reflection coefficient 2 */ +extern const int16_t WebRtcIsacfix_kRc2Levels[11]; + +/* representation levels for quantized reflection coefficient 3 */ +extern const int16_t WebRtcIsacfix_kRc3Levels[11]; + +/* representation levels for quantized reflection coefficient 4 */ +extern const int16_t WebRtcIsacfix_kRc4Levels[11]; + +/* representation levels for quantized reflection coefficient 5 */ +extern const int16_t WebRtcIsacfix_kRc5Levels[11]; + +/* representation levels for quantized reflection coefficient 6 */ +extern const int16_t WebRtcIsacfix_kRc6Levels[11]; + +/* quantization boundary levels for reflection coefficients */ +extern const int16_t WebRtcIsacfix_kRcBound[12]; + +/* initial indices for AR reflection coefficient quantizer and cdf table search */ +extern const uint16_t WebRtcIsacfix_kRcInitInd[AR_ORDER]; + +/* pointers to AR cdf tables */ +extern const uint16_t *WebRtcIsacfix_kRcCdfPtr[AR_ORDER]; + +/* pointers to AR representation levels tables */ +extern const int16_t *WebRtcIsacfix_kRcLevPtr[AR_ORDER]; + + +/******************** GAIN Coefficient Tables ***********************/ +/* cdf for Gain coefficient */ +extern const uint16_t WebRtcIsacfix_kGainCdf[19]; + +/* representation levels for quantized Gain coefficient */ +extern const int32_t WebRtcIsacfix_kGain2Lev[18]; + +/* squared quantization boundary levels for Gain coefficient */ +extern const int32_t WebRtcIsacfix_kGain2Bound[19]; + +/* pointer to Gain cdf table */ +extern const uint16_t *WebRtcIsacfix_kGainPtr[1]; + +/* Gain initial index for gain quantizer and cdf table search */ +extern const uint16_t WebRtcIsacfix_kGainInitInd[1]; + +/************************* Cosine Tables ****************************/ +/* Cosine table */ +extern const int16_t WebRtcIsacfix_kCos[6][60]; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_SPECTRUM_AR_MODEL_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h new file mode 100644 index 0000000000..7a14e5c639 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * structs.h + * + * This header file contains all the structs used in the ISAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_STRUCTS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_STRUCTS_H_ + + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/codecs/isac/bandwidth_info.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" +#include "typedefs.h" // NOLINT(build/include) + +/* Bitstream struct for decoder */ +typedef struct Bitstreamstruct_dec { + + uint16_t stream[INTERNAL_STREAM_SIZE_W16]; /* Array bytestream to decode */ + uint32_t W_upper; /* Upper boundary of interval W */ + uint32_t streamval; + uint16_t stream_index; /* Index to the current position in bytestream */ + int16_t full; /* 0 - first byte in memory filled, second empty*/ + /* 1 - both bytes are empty (we just filled the previous memory */ + + size_t stream_size; /* The size of stream in bytes. */ +} Bitstr_dec; + +/* Bitstream struct for encoder */ +typedef struct Bitstreamstruct_enc { + + uint16_t stream[STREAM_MAXW16_60MS]; /* Vector for adding encoded bytestream */ + uint32_t W_upper; /* Upper boundary of interval W */ + uint32_t streamval; + uint16_t stream_index; /* Index to the current position in bytestream */ + int16_t full; /* 0 - first byte in memory filled, second empty*/ + /* 1 - both bytes are empty (we just filled the previous memory */ + +} Bitstr_enc; + + +typedef struct { + + int16_t DataBufferLoQ0[WINLEN]; + int16_t DataBufferHiQ0[WINLEN]; + + int32_t CorrBufLoQQ[ORDERLO+1]; + int32_t CorrBufHiQQ[ORDERHI+1]; + + int16_t CorrBufLoQdom[ORDERLO+1]; + int16_t CorrBufHiQdom[ORDERHI+1]; + + int32_t PreStateLoGQ15[ORDERLO+1]; + int32_t PreStateHiGQ15[ORDERHI+1]; + + uint32_t OldEnergy; + +} MaskFiltstr_enc; + + + +typedef struct { + + int16_t PostStateLoGQ0[ORDERLO+1]; + int16_t PostStateHiGQ0[ORDERHI+1]; + + uint32_t OldEnergy; + +} MaskFiltstr_dec; + + + + + + + + +typedef struct { + + //state vectors for each of the two analysis filters + + int32_t INSTAT1_fix[2*(QORDER-1)]; + int32_t INSTAT2_fix[2*(QORDER-1)]; + int16_t INLABUF1_fix[QLOOKAHEAD]; + int16_t INLABUF2_fix[QLOOKAHEAD]; + + /* High pass filter */ + int32_t HPstates_fix[HPORDER]; + +} PreFiltBankstr; + + +typedef struct { + + //state vectors for each of the two analysis filters + int32_t STATE_0_LOWER_fix[2*POSTQORDER]; + int32_t STATE_0_UPPER_fix[2*POSTQORDER]; + + /* High pass filter */ + + int32_t HPstates1_fix[HPORDER]; + int32_t HPstates2_fix[HPORDER]; + +} PostFiltBankstr; + +typedef struct { + + + /* data buffer for pitch filter */ + int16_t ubufQQ[PITCH_BUFFSIZE]; + + /* low pass state vector */ + int16_t ystateQQ[PITCH_DAMPORDER]; + + /* old lag and gain */ + int16_t oldlagQ7; + int16_t oldgainQ12; + +} PitchFiltstr; + + + +typedef struct { + + //for inital estimator + int16_t dec_buffer16[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2]; + int32_t decimator_state32[2*ALLPASSSECTIONS+1]; + int16_t inbuf[QLOOKAHEAD]; + + PitchFiltstr PFstr_wght; + PitchFiltstr PFstr; + + +} PitchAnalysisStruct; + + +typedef struct { + /* Parameters used in PLC to avoid re-computation */ + + /* --- residual signals --- */ + int16_t prevPitchInvIn[FRAMESAMPLES/2]; + int16_t prevPitchInvOut[PITCH_MAX_LAG + 10]; // [FRAMESAMPLES/2]; save 90 + int32_t prevHP[PITCH_MAX_LAG + 10]; // [FRAMESAMPLES/2]; save 90 + + + int16_t decayCoeffPriodic; /* how much to supress a sample */ + int16_t decayCoeffNoise; + int16_t used; /* if PLC is used */ + + + int16_t *lastPitchLP; // [FRAMESAMPLES/2]; saved 240; + + + /* --- LPC side info --- */ + int16_t lofilt_coefQ15[ ORDERLO ]; + int16_t hifilt_coefQ15[ ORDERHI ]; + int32_t gain_lo_hiQ17[2]; + + /* --- LTP side info --- */ + int16_t AvgPitchGain_Q12; + int16_t lastPitchGain_Q12; + int16_t lastPitchLag_Q7; + + /* --- Add-overlap in recovery packet --- */ + int16_t overlapLP[ RECOVERY_OVERLAP ]; // [FRAMESAMPLES/2]; saved 160 + + int16_t pitchCycles; + int16_t A; + int16_t B; + size_t pitchIndex; + size_t stretchLag; + int16_t *prevPitchLP; // [ FRAMESAMPLES/2 ]; saved 240 + int16_t seed; + + int16_t std; +} PLCstr; + + + +/* Have instance of struct together with other iSAC structs */ +typedef struct { + + int16_t prevFrameSizeMs; /* Previous frame size (in ms) */ + uint16_t prevRtpNumber; /* Previous RTP timestamp from received packet */ + /* (in samples relative beginning) */ + uint32_t prevSendTime; /* Send time for previous packet, from RTP header */ + uint32_t prevArrivalTime; /* Arrival time for previous packet (in ms using timeGetTime()) */ + uint16_t prevRtpRate; /* rate of previous packet, derived from RTP timestamps (in bits/s) */ + uint32_t lastUpdate; /* Time since the last update of the Bottle Neck estimate (in samples) */ + uint32_t lastReduction; /* Time sinse the last reduction (in samples) */ + int32_t countUpdates; /* How many times the estimate was update in the beginning */ + + /* The estimated bottle neck rate from there to here (in bits/s) */ + uint32_t recBw; + uint32_t recBwInv; + uint32_t recBwAvg; + uint32_t recBwAvgQ; + + uint32_t minBwInv; + uint32_t maxBwInv; + + /* The estimated mean absolute jitter value, as seen on this side (in ms) */ + int32_t recJitter; + int32_t recJitterShortTerm; + int32_t recJitterShortTermAbs; + int32_t recMaxDelay; + int32_t recMaxDelayAvgQ; + + + int16_t recHeaderRate; /* (assumed) bitrate for headers (bps) */ + + uint32_t sendBwAvg; /* The estimated bottle neck rate from here to there (in bits/s) */ + int32_t sendMaxDelayAvg; /* The estimated mean absolute jitter value, as seen on the other siee (in ms) */ + + + int16_t countRecPkts; /* number of packets received since last update */ + int16_t highSpeedRec; /* flag for marking that a high speed network has been detected downstream */ + + /* number of consecutive pkts sent during which the bwe estimate has + remained at a value greater than the downstream threshold for determining highspeed network */ + int16_t countHighSpeedRec; + + /* flag indicating bwe should not adjust down immediately for very late pckts */ + int16_t inWaitPeriod; + + /* variable holding the time of the start of a window of time when + bwe should not adjust down immediately for very late pckts */ + uint32_t startWaitPeriod; + + /* number of consecutive pkts sent during which the bwe estimate has + remained at a value greater than the upstream threshold for determining highspeed network */ + int16_t countHighSpeedSent; + + /* flag indicated the desired number of packets over threshold rate have been sent and + bwe will assume the connection is over broadband network */ + int16_t highSpeedSend; + + IsacBandwidthInfo external_bw_info; +} BwEstimatorstr; + + +typedef struct { + + /* boolean, flags if previous packet exceeded B.N. */ + int16_t PrevExceed; + /* ms */ + int16_t ExceedAgo; + /* packets left to send in current burst */ + int16_t BurstCounter; + /* packets */ + int16_t InitCounter; + /* ms remaining in buffer when next packet will be sent */ + int16_t StillBuffered; + +} RateModel; + +/* The following strutc is used to store data from encoding, to make it + fast and easy to construct a new bitstream with a different Bandwidth + estimate. All values (except framelength and minBytes) is double size to + handle 60 ms of data. +*/ +typedef struct { + + /* Used to keep track of if it is first or second part of 60 msec packet */ + int startIdx; + + /* Frame length in samples */ + int16_t framelength; + + /* Pitch Gain */ + int16_t pitchGain_index[2]; + + /* Pitch Lag */ + int32_t meanGain[2]; + int16_t pitchIndex[PITCH_SUBFRAMES*2]; + + /* LPC */ + int32_t LPCcoeffs_g[12*2]; /* KLT_ORDER_GAIN = 12 */ + int16_t LPCindex_s[108*2]; /* KLT_ORDER_SHAPE = 108 */ + int16_t LPCindex_g[12*2]; /* KLT_ORDER_GAIN = 12 */ + + /* Encode Spec */ + int16_t fre[FRAMESAMPLES]; + int16_t fim[FRAMESAMPLES]; + int16_t AvgPitchGain[2]; + + /* Used in adaptive mode only */ + int minBytes; + +} IsacSaveEncoderData; + +typedef struct { + + Bitstr_enc bitstr_obj; + MaskFiltstr_enc maskfiltstr_obj; + PreFiltBankstr prefiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + PitchAnalysisStruct pitchanalysisstr_obj; + RateModel rate_data_obj; + + int16_t buffer_index; + int16_t current_framesamples; + + int16_t data_buffer_fix[FRAMESAMPLES]; // the size was MAX_FRAMESAMPLES + + int16_t frame_nb; + int16_t BottleNeck; + int16_t MaxDelay; + int16_t new_framelength; + int16_t s2nr; + uint16_t MaxBits; + + int16_t bitstr_seed; +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + PostFiltBankstr interpolatorstr_obj; +#endif + + IsacSaveEncoderData *SaveEnc_ptr; + int16_t payloadLimitBytes30; /* Maximum allowed number of bits for a 30 msec packet */ + int16_t payloadLimitBytes60; /* Maximum allowed number of bits for a 30 msec packet */ + int16_t maxPayloadBytes; /* Maximum allowed number of bits for both 30 and 60 msec packet */ + int16_t maxRateInBytes; /* Maximum allowed rate in bytes per 30 msec packet */ + int16_t enforceFrameSize; /* If set iSAC will never change packet size */ + +} IsacFixEncoderInstance; + + +typedef struct { + + Bitstr_dec bitstr_obj; + MaskFiltstr_dec maskfiltstr_obj; + PostFiltBankstr postfiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + PLCstr plcstr_obj; /* TS; for packet loss concealment */ + +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + PreFiltBankstr decimatorstr_obj; +#endif + +} IsacFixDecoderInstance; + + + +typedef struct { + + IsacFixEncoderInstance ISACenc_obj; + IsacFixDecoderInstance ISACdec_obj; + BwEstimatorstr bwestimator_obj; + int16_t CodingMode; /* 0 = adaptive; 1 = instantaneous */ + int16_t errorcode; + int16_t initflag; /* 0 = nothing initiated; 1 = encoder or decoder */ + /* not initiated; 2 = all initiated */ +} ISACFIX_SubStruct; + + +typedef struct { + int32_t lpcGains[12]; /* 6 lower-band & 6 upper-band we may need to double it for 60*/ + /* */ + uint32_t W_upper; /* Upper boundary of interval W */ + uint32_t streamval; + uint16_t stream_index; /* Index to the current position in bytestream */ + int16_t full; /* 0 - first byte in memory filled, second empty*/ + /* 1 - both bytes are empty (we just filled the previous memory */ + uint16_t beforeLastWord; + uint16_t lastWord; +} transcode_obj; + + +//Bitstr_enc myBitStr; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_FIX_SOURCE_STRUCTS_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform.c new file mode 100644 index 0000000000..2f1275da5c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform.c @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * WebRtcIsacfix_kTransform.c + * + * Transform functions + * + */ + +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" +#include "modules/audio_coding/codecs/isac/fix/source/fft.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" + +/* Tables are defined in transform_tables.c file or ARM assembly files. */ +/* Cosine table 1 in Q14 */ +extern const int16_t WebRtcIsacfix_kCosTab1[FRAMESAMPLES/2]; +/* Sine table 1 in Q14 */ +extern const int16_t WebRtcIsacfix_kSinTab1[FRAMESAMPLES/2]; +/* Sine table 2 in Q14 */ +extern const int16_t WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4]; + +void WebRtcIsacfix_Time2SpecC(int16_t *inre1Q9, + int16_t *inre2Q9, + int16_t *outreQ7, + int16_t *outimQ7) +{ + + int k; + int32_t tmpreQ16[FRAMESAMPLES/2], tmpimQ16[FRAMESAMPLES/2]; + int16_t tmp1rQ14, tmp1iQ14; + int32_t xrQ16, xiQ16, yrQ16, yiQ16; + int32_t v1Q16, v2Q16; + int16_t factQ19, sh; + + /* Multiply with complex exponentials and combine into one complex vector */ + factQ19 = 16921; // 0.5/sqrt(240) in Q19 is round(.5/sqrt(240)*(2^19)) = 16921 + for (k = 0; k < FRAMESAMPLES/2; k++) { + tmp1rQ14 = WebRtcIsacfix_kCosTab1[k]; + tmp1iQ14 = WebRtcIsacfix_kSinTab1[k]; + xrQ16 = (tmp1rQ14 * inre1Q9[k] + tmp1iQ14 * inre2Q9[k]) >> 7; + xiQ16 = (tmp1rQ14 * inre2Q9[k] - tmp1iQ14 * inre1Q9[k]) >> 7; + // Q-domains below: (Q16*Q19>>16)>>3 = Q16 + tmpreQ16[k] = (WEBRTC_SPL_MUL_16_32_RSFT16(factQ19, xrQ16) + 4) >> 3; + tmpimQ16[k] = (WEBRTC_SPL_MUL_16_32_RSFT16(factQ19, xiQ16) + 4) >> 3; + } + + + xrQ16 = WebRtcSpl_MaxAbsValueW32(tmpreQ16, FRAMESAMPLES/2); + yrQ16 = WebRtcSpl_MaxAbsValueW32(tmpimQ16, FRAMESAMPLES/2); + if (yrQ16>xrQ16) { + xrQ16 = yrQ16; + } + + sh = WebRtcSpl_NormW32(xrQ16); + sh = sh-24; //if sh becomes >=0, then we should shift sh steps to the left, and the domain will become Q(16+sh) + //if sh becomes <0, then we should shift -sh steps to the right, and the domain will become Q(16+sh) + + //"Fastest" vectors + if (sh>=0) { + for (k=0; k> -sh); // Q(16+sh) + inre2Q9[k] = (int16_t)((tmpimQ16[k] + round) >> -sh); // Q(16+sh) + } + } + + /* Get DFT */ + WebRtcIsacfix_FftRadix16Fastest(inre1Q9, inre2Q9, -1); // real call + + //"Fastest" vectors + if (sh>=0) { + for (k=0; k> sh; // Q(16+sh) -> Q16 + tmpimQ16[k] = inre2Q9[k] >> sh; // Q(16+sh) -> Q16 + } + } else { + for (k=0; k Q16 + tmpimQ16[k] = inre2Q9[k] << -sh; // Q(16+sh) -> Q16 + } + } + + + /* Use symmetry to separate into two complex vectors and center frames in time around zero */ + for (k = 0; k < FRAMESAMPLES/4; k++) { + xrQ16 = tmpreQ16[k] + tmpreQ16[FRAMESAMPLES/2 - 1 - k]; + yiQ16 = -tmpreQ16[k] + tmpreQ16[FRAMESAMPLES/2 - 1 - k]; + xiQ16 = tmpimQ16[k] - tmpimQ16[FRAMESAMPLES/2 - 1 - k]; + yrQ16 = tmpimQ16[k] + tmpimQ16[FRAMESAMPLES/2 - 1 - k]; + tmp1rQ14 = -WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4 - 1 - k]; + tmp1iQ14 = WebRtcIsacfix_kSinTab2[k]; + v1Q16 = WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, xrQ16) - WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, xiQ16); + v2Q16 = WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, xrQ16) + WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, xiQ16); + outreQ7[k] = (int16_t)(v1Q16 >> 9); + outimQ7[k] = (int16_t)(v2Q16 >> 9); + v1Q16 = -WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, yrQ16) - WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, yiQ16); + v2Q16 = -WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, yrQ16) + WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, yiQ16); + // CalcLrIntQ(v1Q16, 9); + outreQ7[FRAMESAMPLES / 2 - 1 - k] = (int16_t)(v1Q16 >> 9); + // CalcLrIntQ(v2Q16, 9); + outimQ7[FRAMESAMPLES / 2 - 1 - k] = (int16_t)(v2Q16 >> 9); + + } +} + + +void WebRtcIsacfix_Spec2TimeC(int16_t *inreQ7, int16_t *inimQ7, int32_t *outre1Q16, int32_t *outre2Q16) +{ + + int k; + int16_t tmp1rQ14, tmp1iQ14; + int32_t xrQ16, xiQ16, yrQ16, yiQ16; + int32_t tmpInRe, tmpInIm, tmpInRe2, tmpInIm2; + int16_t factQ11; + int16_t sh; + + for (k = 0; k < FRAMESAMPLES/4; k++) { + /* Move zero in time to beginning of frames */ + tmp1rQ14 = -WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4 - 1 - k]; + tmp1iQ14 = WebRtcIsacfix_kSinTab2[k]; + + tmpInRe = inreQ7[k] * (1 << 9); // Q7 -> Q16 + tmpInIm = inimQ7[k] * (1 << 9); // Q7 -> Q16 + tmpInRe2 = inreQ7[FRAMESAMPLES / 2 - 1 - k] * (1 << 9); // Q7 -> Q16 + tmpInIm2 = inimQ7[FRAMESAMPLES / 2 - 1 - k] * (1 << 9); // Q7 -> Q16 + + xrQ16 = WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, tmpInRe) + WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, tmpInIm); + xiQ16 = WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, tmpInIm) - WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, tmpInRe); + yrQ16 = -WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, tmpInIm2) - WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, tmpInRe2); + yiQ16 = -WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, tmpInRe2) + WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, tmpInIm2); + + /* Combine into one vector, z = x + j * y */ + outre1Q16[k] = xrQ16 - yiQ16; + outre1Q16[FRAMESAMPLES/2 - 1 - k] = xrQ16 + yiQ16; + outre2Q16[k] = xiQ16 + yrQ16; + outre2Q16[FRAMESAMPLES/2 - 1 - k] = -xiQ16 + yrQ16; + } + + /* Get IDFT */ + tmpInRe = WebRtcSpl_MaxAbsValueW32(outre1Q16, 240); + tmpInIm = WebRtcSpl_MaxAbsValueW32(outre2Q16, 240); + if (tmpInIm>tmpInRe) { + tmpInRe = tmpInIm; + } + + sh = WebRtcSpl_NormW32(tmpInRe); + sh = sh-24; //if sh becomes >=0, then we should shift sh steps to the left, and the domain will become Q(16+sh) + //if sh becomes <0, then we should shift -sh steps to the right, and the domain will become Q(16+sh) + + //"Fastest" vectors + if (sh>=0) { + for (k=0; k<240; k++) { + inreQ7[k] = (int16_t)(outre1Q16[k] << sh); // Q(16+sh) + inimQ7[k] = (int16_t)(outre2Q16[k] << sh); // Q(16+sh) + } + } else { + int32_t round = 1 << (-sh - 1); + for (k=0; k<240; k++) { + inreQ7[k] = (int16_t)((outre1Q16[k] + round) >> -sh); // Q(16+sh) + inimQ7[k] = (int16_t)((outre2Q16[k] + round) >> -sh); // Q(16+sh) + } + } + + WebRtcIsacfix_FftRadix16Fastest(inreQ7, inimQ7, 1); // real call + + //"Fastest" vectors + if (sh>=0) { + for (k=0; k<240; k++) { + outre1Q16[k] = inreQ7[k] >> sh; // Q(16+sh) -> Q16 + outre2Q16[k] = inimQ7[k] >> sh; // Q(16+sh) -> Q16 + } + } else { + for (k=0; k<240; k++) { + outre1Q16[k] = inreQ7[k] * (1 << -sh); // Q(16+sh) -> Q16 + outre2Q16[k] = inimQ7[k] * (1 << -sh); // Q(16+sh) -> Q16 + } + } + + /* Divide through by the normalizing constant: */ + /* scale all values with 1/240, i.e. with 273 in Q16 */ + /* 273/65536 ~= 0.0041656 */ + /* 1/240 ~= 0.0041666 */ + for (k=0; k<240; k++) { + outre1Q16[k] = WEBRTC_SPL_MUL_16_32_RSFT16(273, outre1Q16[k]); + outre2Q16[k] = WEBRTC_SPL_MUL_16_32_RSFT16(273, outre2Q16[k]); + } + + /* Demodulate and separate */ + factQ11 = 31727; // sqrt(240) in Q11 is round(15.49193338482967*2048) = 31727 + for (k = 0; k < FRAMESAMPLES/2; k++) { + tmp1rQ14 = WebRtcIsacfix_kCosTab1[k]; + tmp1iQ14 = WebRtcIsacfix_kSinTab1[k]; + xrQ16 = WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, outre1Q16[k]) - WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, outre2Q16[k]); + xiQ16 = WEBRTC_SPL_MUL_16_32_RSFT14(tmp1rQ14, outre2Q16[k]) + WEBRTC_SPL_MUL_16_32_RSFT14(tmp1iQ14, outre1Q16[k]); + xrQ16 = WEBRTC_SPL_MUL_16_32_RSFT11(factQ11, xrQ16); + xiQ16 = WEBRTC_SPL_MUL_16_32_RSFT11(factQ11, xiQ16); + outre2Q16[k] = xiQ16; + outre1Q16[k] = xrQ16; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_mips.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_mips.c new file mode 100644 index 0000000000..a87b3b54f2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_mips.c @@ -0,0 +1,1294 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" +#include "modules/audio_coding/codecs/isac/fix/source/fft.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" + +// The tables are defined in transform_tables.c file. +extern const int16_t WebRtcIsacfix_kCosTab1[FRAMESAMPLES/2]; +extern const int16_t WebRtcIsacfix_kSinTab1[FRAMESAMPLES/2]; +extern const int16_t WebRtcIsacfix_kCosTab2[FRAMESAMPLES/4]; +extern const int16_t WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4]; + +// MIPS DSPr2 version of the WebRtcIsacfix_Time2Spec function +// is not bit-exact with the C version. +// The accuracy of the MIPS DSPr2 version is same or better. +void WebRtcIsacfix_Time2SpecMIPS(int16_t* inre1Q9, + int16_t* inre2Q9, + int16_t* outreQ7, + int16_t* outimQ7) { + int k = FRAMESAMPLES / 2; + int32_t tmpreQ16[FRAMESAMPLES / 2], tmpimQ16[FRAMESAMPLES / 2]; + int32_t r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + int32_t inre1, inre2, tmpre, tmpim, factor, max, max1; + int16_t* cosptr; + int16_t* sinptr; + + cosptr = (int16_t*)WebRtcIsacfix_kCosTab1; + sinptr = (int16_t*)WebRtcIsacfix_kSinTab1; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[inre1], %[inre1Q9], 0 \n\t" + "addiu %[inre2], %[inre2Q9], 0 \n\t" + "addiu %[tmpre], %[tmpreQ16], 0 \n\t" + "addiu %[tmpim], %[tmpimQ16], 0 \n\t" + "addiu %[factor], $zero, 16921 \n\t" + "mul %[max], $zero, $zero \n\t" + // Multiply with complex exponentials and combine into one complex vector. + // Also, calculate the maximal absolute value in the same loop. + "1: \n\t" +#if defined(MIPS_DSP_R2_LE) + "lwl %[r0], 0(%[inre1]) \n\t" + "lwl %[r2], 0(%[cosptr]) \n\t" + "lwl %[r3], 0(%[sinptr]) \n\t" + "lwl %[r1], 0(%[inre2]) \n\t" + "lwr %[r0], 0(%[inre1]) \n\t" + "lwr %[r2], 0(%[cosptr]) \n\t" + "lwr %[r3], 0(%[sinptr]) \n\t" + "lwr %[r1], 0(%[inre2]) \n\t" + "muleq_s.w.phr %[r4], %[r2], %[r0] \n\t" + "muleq_s.w.phr %[r5], %[r3], %[r0] \n\t" + "muleq_s.w.phr %[r6], %[r3], %[r1] \n\t" + "muleq_s.w.phr %[r7], %[r2], %[r1] \n\t" + "muleq_s.w.phl %[r8], %[r2], %[r0] \n\t" + "muleq_s.w.phl %[r0], %[r3], %[r0] \n\t" + "muleq_s.w.phl %[r3], %[r3], %[r1] \n\t" + "muleq_s.w.phl %[r1], %[r2], %[r1] \n\t" + "addiu %[k], %[k], -2 \n\t" + "addu %[r4], %[r4], %[r6] \n\t" + "subu %[r5], %[r7], %[r5] \n\t" + "sra %[r4], %[r4], 8 \n\t" + "sra %[r5], %[r5], 8 \n\t" + "mult $ac0, %[factor], %[r4] \n\t" + "mult $ac1, %[factor], %[r5] \n\t" + "addu %[r3], %[r8], %[r3] \n\t" + "subu %[r0], %[r1], %[r0] \n\t" + "sra %[r3], %[r3], 8 \n\t" + "sra %[r0], %[r0], 8 \n\t" + "mult $ac2, %[factor], %[r3] \n\t" + "mult $ac3, %[factor], %[r0] \n\t" + "extr_r.w %[r4], $ac0, 16 \n\t" + "extr_r.w %[r5], $ac1, 16 \n\t" + "addiu %[inre1], %[inre1], 4 \n\t" + "addiu %[inre2], %[inre2], 4 \n\t" + "extr_r.w %[r6], $ac2, 16 \n\t" + "extr_r.w %[r7], $ac3, 16 \n\t" + "addiu %[cosptr], %[cosptr], 4 \n\t" + "addiu %[sinptr], %[sinptr], 4 \n\t" + "shra_r.w %[r4], %[r4], 3 \n\t" + "shra_r.w %[r5], %[r5], 3 \n\t" + "sw %[r4], 0(%[tmpre]) \n\t" + "absq_s.w %[r4], %[r4] \n\t" + "sw %[r5], 0(%[tmpim]) \n\t" + "absq_s.w %[r5], %[r5] \n\t" + "shra_r.w %[r6], %[r6], 3 \n\t" + "shra_r.w %[r7], %[r7], 3 \n\t" + "sw %[r6], 4(%[tmpre]) \n\t" + "absq_s.w %[r6], %[r6] \n\t" + "sw %[r7], 4(%[tmpim]) \n\t" + "absq_s.w %[r7], %[r7] \n\t" + "slt %[r0], %[r4], %[r5] \n\t" + "movn %[r4], %[r5], %[r0] \n\t" + "slt %[r1], %[r6], %[r7] \n\t" + "movn %[r6], %[r7], %[r1] \n\t" + "slt %[r0], %[max], %[r4] \n\t" + "movn %[max], %[r4], %[r0] \n\t" + "slt %[r1], %[max], %[r6] \n\t" + "movn %[max], %[r6], %[r1] \n\t" + "addiu %[tmpre], %[tmpre], 8 \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[tmpim], %[tmpim], 8 \n\t" +#else // #if defined(MIPS_DSP_R2_LE) + "lh %[r0], 0(%[inre1]) \n\t" + "lh %[r1], 0(%[inre2]) \n\t" + "lh %[r2], 0(%[cosptr]) \n\t" + "lh %[r3], 0(%[sinptr]) \n\t" + "addiu %[k], %[k], -1 \n\t" + "mul %[r4], %[r0], %[r2] \n\t" + "mul %[r5], %[r1], %[r3] \n\t" + "mul %[r0], %[r0], %[r3] \n\t" + "mul %[r2], %[r1], %[r2] \n\t" + "addiu %[inre1], %[inre1], 2 \n\t" + "addiu %[inre2], %[inre2], 2 \n\t" + "addiu %[cosptr], %[cosptr], 2 \n\t" + "addiu %[sinptr], %[sinptr], 2 \n\t" + "addu %[r1], %[r4], %[r5] \n\t" + "sra %[r1], %[r1], 7 \n\t" + "sra %[r3], %[r1], 16 \n\t" + "andi %[r1], %[r1], 0xFFFF \n\t" + "sra %[r1], %[r1], 1 \n\t" + "mul %[r1], %[factor], %[r1] \n\t" + "mul %[r3], %[factor], %[r3] \n\t" + "subu %[r0], %[r2], %[r0] \n\t" + "sra %[r0], %[r0], 7 \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "sra %[r0], %[r0], 1 \n\t" + "mul %[r0], %[factor], %[r0] \n\t" + "mul %[r2], %[factor], %[r2] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r1], %[r1], 15 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r1], %[r1], 0x4000 \n\t" + "sra %[r1], %[r1], 15 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r1], %[r3], %[r1] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r1], %[r1], 3 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r1], %[r1], 4 \n\t" + "sra %[r1], %[r1], 3 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sw %[r1], 0(%[tmpre]) \n\t" + "addiu %[tmpre], %[tmpre], 4 \n\t" +#if defined(MIPS_DSP_R1_LE) + "absq_s.w %[r1], %[r1] \n\t" + "shra_r.w %[r0], %[r0], 15 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "negu %[r4], %[r1] \n\t" + "slt %[r3], %[r1], $zero \n\t" + "movn %[r1], %[r4], %[r3] \n\t" + "addiu %[r0], %[r0], 0x4000 \n\t" + "sra %[r0], %[r0], 15 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r0], %[r2] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r0], %[r0], 3 \n\t" + "sw %[r0], 0(%[tmpim]) \n\t" + "absq_s.w %[r0], %[r0] \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r0], %[r0], 4 \n\t" + "sra %[r0], %[r0], 3 \n\t" + "sw %[r0], 0(%[tmpim]) \n\t" + "negu %[r2], %[r0] \n\t" + "slt %[r3], %[r0], $zero \n\t" + "movn %[r0], %[r2], %[r3] \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "slt %[r2], %[max], %[r1] \n\t" + "movn %[max], %[r1], %[r2] \n\t" + "slt %[r2], %[max], %[r0] \n\t" + "movn %[max], %[r0], %[r2] \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[tmpim], %[tmpim], 4 \n\t" +#endif // #if defined(MIPS_DSP_R2_LE) + // Calculate WebRtcSpl_NormW32(max). + // If max gets value >=0, we should shift max steps to the left, and the + // domain will be Q(16+shift). If max gets value <0, we should shift -max + // steps to the right, and the domain will be Q(16+max) + "clz %[max], %[max] \n\t" + "addiu %[max], %[max], -25 \n\t" + ".set pop \n\t" + : [k] "+r" (k), [inre1] "=&r" (inre1), [inre2] "=&r" (inre2), + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), + [r3] "=&r" (r3), [r4] "=&r" (r4), [tmpre] "=&r" (tmpre), + [tmpim] "=&r" (tmpim), [max] "=&r" (max), [factor] "=&r" (factor), +#if defined(MIPS_DSP_R2_LE) + [r6] "=&r" (r6), [r7] "=&r" (r7), [r8] "=&r" (r8), +#endif // #if defined(MIPS_DSP_R2_LE) + [r5] "=&r" (r5) + : [inre1Q9] "r" (inre1Q9), [inre2Q9] "r" (inre2Q9), + [tmpreQ16] "r" (tmpreQ16), [tmpimQ16] "r" (tmpimQ16), + [cosptr] "r" (cosptr), [sinptr] "r" (sinptr) + : "hi", "lo", "memory" +#if defined(MIPS_DSP_R2_LE) + , "$ac1hi", "$ac1lo", "$ac2hi", "$ac2lo", "$ac3hi", "$ac3lo" +#endif // #if defined(MIPS_DSP_R2_LE) + ); + + // "Fastest" vectors + k = FRAMESAMPLES / 4; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[tmpre], %[tmpreQ16], 0 \n\t" + "addiu %[tmpim], %[tmpimQ16], 0 \n\t" + "addiu %[inre1], %[inre1Q9], 0 \n\t" + "addiu %[inre2], %[inre2Q9], 0 \n\t" + "blez %[max], 2f \n\t" + " subu %[max1], $zero, %[max] \n\t" + "1: \n\t" + "lw %[r0], 0(%[tmpre]) \n\t" + "lw %[r1], 0(%[tmpim]) \n\t" + "lw %[r2], 4(%[tmpre]) \n\t" + "lw %[r3], 4(%[tmpim]) \n\t" + "addiu %[k], %[k], -1 \n\t" + "sllv %[r0], %[r0], %[max] \n\t" + "sllv %[r1], %[r1], %[max] \n\t" + "sllv %[r2], %[r2], %[max] \n\t" + "sllv %[r3], %[r3], %[max] \n\t" + "addiu %[tmpre], %[tmpre], 8 \n\t" + "addiu %[tmpim], %[tmpim], 8 \n\t" + "sh %[r0], 0(%[inre1]) \n\t" + "sh %[r1], 0(%[inre2]) \n\t" + "sh %[r2], 2(%[inre1]) \n\t" + "sh %[r3], 2(%[inre2]) \n\t" + "addiu %[inre1], %[inre1], 4 \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[inre2], %[inre2], 4 \n\t" + "b 4f \n\t" + " nop \n\t" + "2: \n\t" +#if !defined(MIPS_DSP_R1_LE) + "addiu %[r4], %[max1], -1 \n\t" + "addiu %[r5], $zero, 1 \n\t" + "sllv %[r4], %[r5], %[r4] \n\t" +#endif // #if !defined(MIPS_DSP_R1_LE) + "3: \n\t" + "lw %[r0], 0(%[tmpre]) \n\t" + "lw %[r1], 0(%[tmpim]) \n\t" + "lw %[r2], 4(%[tmpre]) \n\t" + "lw %[r3], 4(%[tmpim]) \n\t" + "addiu %[k], %[k], -1 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shrav_r.w %[r0], %[r0], %[max1] \n\t" + "shrav_r.w %[r1], %[r1], %[max1] \n\t" + "shrav_r.w %[r2], %[r2], %[max1] \n\t" + "shrav_r.w %[r3], %[r3], %[max1] \n\t" +#else // #if !defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r0], %[r4] \n\t" + "addu %[r1], %[r1], %[r4] \n\t" + "addu %[r2], %[r2], %[r4] \n\t" + "addu %[r3], %[r3], %[r4] \n\t" + "srav %[r0], %[r0], %[max1] \n\t" + "srav %[r1], %[r1], %[max1] \n\t" + "srav %[r2], %[r2], %[max1] \n\t" + "srav %[r3], %[r3], %[max1] \n\t" +#endif // #if !defined(MIPS_DSP_R1_LE) + "addiu %[tmpre], %[tmpre], 8 \n\t" + "addiu %[tmpim], %[tmpim], 8 \n\t" + "sh %[r0], 0(%[inre1]) \n\t" + "sh %[r1], 0(%[inre2]) \n\t" + "sh %[r2], 2(%[inre1]) \n\t" + "sh %[r3], 2(%[inre2]) \n\t" + "addiu %[inre1], %[inre1], 4 \n\t" + "bgtz %[k], 3b \n\t" + " addiu %[inre2], %[inre2], 4 \n\t" + "4: \n\t" + ".set pop \n\t" + : [tmpre] "=&r" (tmpre), [tmpim] "=&r" (tmpim), [inre1] "=&r" (inre1), + [inre2] "=&r" (inre2), [k] "+r" (k), [max1] "=&r" (max1), +#if !defined(MIPS_DSP_R1_LE) + [r4] "=&r" (r4), [r5] "=&r" (r5), +#endif // #if !defined(MIPS_DSP_R1_LE) + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3) + : [tmpreQ16] "r" (tmpreQ16), [tmpimQ16] "r" (tmpimQ16), + [inre1Q9] "r" (inre1Q9), [inre2Q9] "r" (inre2Q9), [max] "r" (max) + : "memory" + ); + + // Get DFT + WebRtcIsacfix_FftRadix16Fastest(inre1Q9, inre2Q9, -1); // real call + + // "Fastest" vectors and + // Use symmetry to separate into two complex vectors + // and center frames in time around zero + // merged into one loop + cosptr = (int16_t*)WebRtcIsacfix_kCosTab2; + sinptr = (int16_t*)WebRtcIsacfix_kSinTab2; + k = FRAMESAMPLES / 4; + factor = FRAMESAMPLES - 2; // offset for FRAMESAMPLES / 2 - 1 array member + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[inre1], %[inre1Q9], 0 \n\t" + "addiu %[inre2], %[inre2Q9], 0 \n\t" + "addiu %[tmpre], %[outreQ7], 0 \n\t" + "addiu %[tmpim], %[outimQ7], 0 \n\t" + "bltz %[max], 2f \n\t" + " subu %[max1], $zero, %[max] \n\t" + "1: \n\t" +#if !defined(MIPS_DSP_R1_LE) + "addu %[r4], %[inre1], %[offset] \n\t" + "addu %[r5], %[inre2], %[offset] \n\t" +#endif // #if !defined(MIPS_DSP_R1_LE) + "lh %[r0], 0(%[inre1]) \n\t" + "lh %[r1], 0(%[inre2]) \n\t" +#if defined(MIPS_DSP_R1_LE) + "lhx %[r2], %[offset](%[inre1]) \n\t" + "lhx %[r3], %[offset](%[inre2]) \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "lh %[r2], 0(%[r4]) \n\t" + "lh %[r3], 0(%[r5]) \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "srav %[r0], %[r0], %[max] \n\t" + "srav %[r1], %[r1], %[max] \n\t" + "srav %[r2], %[r2], %[max] \n\t" + "srav %[r3], %[r3], %[max] \n\t" + "addu %[r4], %[r0], %[r2] \n\t" + "subu %[r0], %[r2], %[r0] \n\t" + "subu %[r2], %[r1], %[r3] \n\t" + "addu %[r1], %[r1], %[r3] \n\t" + "lh %[r3], 0(%[cosptr]) \n\t" + "lh %[r5], 0(%[sinptr]) \n\t" + "andi %[r6], %[r4], 0xFFFF \n\t" + "sra %[r4], %[r4], 16 \n\t" + "mul %[r7], %[r3], %[r6] \n\t" + "mul %[r8], %[r3], %[r4] \n\t" + "mul %[r6], %[r5], %[r6] \n\t" + "mul %[r4], %[r5], %[r4] \n\t" + "addiu %[k], %[k], -1 \n\t" + "addiu %[inre1], %[inre1], 2 \n\t" + "addiu %[inre2], %[inre2], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r7], %[r7], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r7], %[r7], 0x2000 \n\t" + "sra %[r7], %[r7], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r8], %[r8], 2 \n\t" + "addu %[r8], %[r8], %[r7] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r6], %[r6], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r6], %[r6], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r4], %[r4], 2 \n\t" + "addu %[r4], %[r4], %[r6] \n\t" + "andi %[r6], %[r2], 0xFFFF \n\t" + "sra %[r2], %[r2], 16 \n\t" + "mul %[r7], %[r5], %[r6] \n\t" + "mul %[r9], %[r5], %[r2] \n\t" + "mul %[r6], %[r3], %[r6] \n\t" + "mul %[r2], %[r3], %[r2] \n\t" + "addiu %[cosptr], %[cosptr], 2 \n\t" + "addiu %[sinptr], %[sinptr], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r7], %[r7], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r7], %[r7], 0x2000 \n\t" + "sra %[r7], %[r7], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r9], %[r9], 2 \n\t" + "addu %[r9], %[r7], %[r9] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r6], %[r6], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r6], %[r6], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r2], %[r2], 2 \n\t" + "addu %[r2], %[r6], %[r2] \n\t" + "subu %[r8], %[r8], %[r9] \n\t" + "sra %[r8], %[r8], 9 \n\t" + "addu %[r2], %[r4], %[r2] \n\t" + "sra %[r2], %[r2], 9 \n\t" + "sh %[r8], 0(%[tmpre]) \n\t" + "sh %[r2], 0(%[tmpim]) \n\t" + + "andi %[r4], %[r1], 0xFFFF \n\t" + "sra %[r1], %[r1], 16 \n\t" + "andi %[r6], %[r0], 0xFFFF \n\t" + "sra %[r0], %[r0], 16 \n\t" + "mul %[r7], %[r5], %[r4] \n\t" + "mul %[r9], %[r5], %[r1] \n\t" + "mul %[r4], %[r3], %[r4] \n\t" + "mul %[r1], %[r3], %[r1] \n\t" + "mul %[r8], %[r3], %[r0] \n\t" + "mul %[r3], %[r3], %[r6] \n\t" + "mul %[r6], %[r5], %[r6] \n\t" + "mul %[r0], %[r5], %[r0] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r7], %[r7], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r7], %[r7], 0x2000 \n\t" + "sra %[r7], %[r7], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r9], %[r9], 2 \n\t" + "addu %[r9], %[r9], %[r7] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r4], %[r4], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r4], %[r4], 0x2000 \n\t" + "sra %[r4], %[r4], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r1], %[r1], 2 \n\t" + "addu %[r1], %[r1], %[r4] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r3], %[r3], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r3], %[r3], 0x2000 \n\t" + "sra %[r3], %[r3], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r8], %[r8], 2 \n\t" + "addu %[r8], %[r8], %[r3] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r6], %[r6], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r6], %[r6], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r0], %[r0], 2 \n\t" + "addu %[r0], %[r0], %[r6] \n\t" + "addu %[r3], %[tmpre], %[offset] \n\t" + "addu %[r2], %[tmpim], %[offset] \n\t" + "addu %[r9], %[r9], %[r8] \n\t" + "negu %[r9], %[r9] \n\t" + "sra %[r9], %[r9], 9 \n\t" + "subu %[r0], %[r0], %[r1] \n\t" + "addiu %[offset], %[offset], -4 \n\t" + "sh %[r9], 0(%[r3]) \n\t" + "sh %[r0], 0(%[r2]) \n\t" + "addiu %[tmpre], %[tmpre], 2 \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[tmpim], %[tmpim], 2 \n\t" + "b 3f \n\t" + " nop \n\t" + "2: \n\t" +#if !defined(MIPS_DSP_R1_LE) + "addu %[r4], %[inre1], %[offset] \n\t" + "addu %[r5], %[inre2], %[offset] \n\t" +#endif // #if !defined(MIPS_DSP_R1_LE) + "lh %[r0], 0(%[inre1]) \n\t" + "lh %[r1], 0(%[inre2]) \n\t" +#if defined(MIPS_DSP_R1_LE) + "lhx %[r2], %[offset](%[inre1]) \n\t" + "lhx %[r3], %[offset](%[inre2]) \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "lh %[r2], 0(%[r4]) \n\t" + "lh %[r3], 0(%[r5]) \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sllv %[r0], %[r0], %[max1] \n\t" + "sllv %[r1], %[r1], %[max1] \n\t" + "sllv %[r2], %[r2], %[max1] \n\t" + "sllv %[r3], %[r3], %[max1] \n\t" + "addu %[r4], %[r0], %[r2] \n\t" + "subu %[r0], %[r2], %[r0] \n\t" + "subu %[r2], %[r1], %[r3] \n\t" + "addu %[r1], %[r1], %[r3] \n\t" + "lh %[r3], 0(%[cosptr]) \n\t" + "lh %[r5], 0(%[sinptr]) \n\t" + "andi %[r6], %[r4], 0xFFFF \n\t" + "sra %[r4], %[r4], 16 \n\t" + "mul %[r7], %[r3], %[r6] \n\t" + "mul %[r8], %[r3], %[r4] \n\t" + "mul %[r6], %[r5], %[r6] \n\t" + "mul %[r4], %[r5], %[r4] \n\t" + "addiu %[k], %[k], -1 \n\t" + "addiu %[inre1], %[inre1], 2 \n\t" + "addiu %[inre2], %[inre2], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r7], %[r7], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r7], %[r7], 0x2000 \n\t" + "sra %[r7], %[r7], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r8], %[r8], 2 \n\t" + "addu %[r8], %[r8], %[r7] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r6], %[r6], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r6], %[r6], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r4], %[r4], 2 \n\t" + "addu %[r4], %[r4], %[r6] \n\t" + "andi %[r6], %[r2], 0xFFFF \n\t" + "sra %[r2], %[r2], 16 \n\t" + "mul %[r7], %[r5], %[r6] \n\t" + "mul %[r9], %[r5], %[r2] \n\t" + "mul %[r6], %[r3], %[r6] \n\t" + "mul %[r2], %[r3], %[r2] \n\t" + "addiu %[cosptr], %[cosptr], 2 \n\t" + "addiu %[sinptr], %[sinptr], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r7], %[r7], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r7], %[r7], 0x2000 \n\t" + "sra %[r7], %[r7], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r9], %[r9], 2 \n\t" + "addu %[r9], %[r7], %[r9] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r6], %[r6], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r6], %[r6], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r2], %[r2], 2 \n\t" + "addu %[r2], %[r6], %[r2] \n\t" + "subu %[r8], %[r8], %[r9] \n\t" + "sra %[r8], %[r8], 9 \n\t" + "addu %[r2], %[r4], %[r2] \n\t" + "sra %[r2], %[r2], 9 \n\t" + "sh %[r8], 0(%[tmpre]) \n\t" + "sh %[r2], 0(%[tmpim]) \n\t" + "andi %[r4], %[r1], 0xFFFF \n\t" + "sra %[r1], %[r1], 16 \n\t" + "andi %[r6], %[r0], 0xFFFF \n\t" + "sra %[r0], %[r0], 16 \n\t" + "mul %[r7], %[r5], %[r4] \n\t" + "mul %[r9], %[r5], %[r1] \n\t" + "mul %[r4], %[r3], %[r4] \n\t" + "mul %[r1], %[r3], %[r1] \n\t" + "mul %[r8], %[r3], %[r0] \n\t" + "mul %[r3], %[r3], %[r6] \n\t" + "mul %[r6], %[r5], %[r6] \n\t" + "mul %[r0], %[r5], %[r0] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r7], %[r7], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r7], %[r7], 0x2000 \n\t" + "sra %[r7], %[r7], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r9], %[r9], 2 \n\t" + "addu %[r9], %[r9], %[r7] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r4], %[r4], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r4], %[r4], 0x2000 \n\t" + "sra %[r4], %[r4], 14 \n\t" +#endif + "sll %[r1], %[r1], 2 \n\t" + "addu %[r1], %[r1], %[r4] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r3], %[r3], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r3], %[r3], 0x2000 \n\t" + "sra %[r3], %[r3], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r8], %[r8], 2 \n\t" + "addu %[r8], %[r8], %[r3] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r6], %[r6], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r6], %[r6], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sll %[r0], %[r0], 2 \n\t" + "addu %[r0], %[r0], %[r6] \n\t" + "addu %[r3], %[tmpre], %[offset] \n\t" + "addu %[r2], %[tmpim], %[offset] \n\t" + "addu %[r9], %[r9], %[r8] \n\t" + "negu %[r9], %[r9] \n\t" + "sra %[r9], %[r9], 9 \n\t" + "subu %[r0], %[r0], %[r1] \n\t" + "sra %[r0], %[r0], 9 \n\t" + "addiu %[offset], %[offset], -4 \n\t" + "sh %[r9], 0(%[r3]) \n\t" + "sh %[r0], 0(%[r2]) \n\t" + "addiu %[tmpre], %[tmpre], 2 \n\t" + "bgtz %[k], 2b \n\t" + " addiu %[tmpim], %[tmpim], 2 \n\t" + "3: \n\t" + ".set pop \n\t" + : [inre1] "=&r" (inre1), [inre2] "=&r" (inre2), [tmpre] "=&r" (tmpre), + [tmpim] "=&r" (tmpim), [offset] "+r" (factor), [k] "+r" (k), + [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), + [r4] "=&r" (r4), [r5] "=&r" (r5), [r6] "=&r" (r6), [r7] "=&r" (r7), + [r8] "=&r" (r8), [r9] "=&r" (r9), [max1] "=&r" (max1) + : [inre1Q9] "r" (inre1Q9), [inre2Q9] "r" (inre2Q9), + [outreQ7] "r" (outreQ7), [outimQ7] "r" (outimQ7), + [max] "r" (max), [cosptr] "r" (cosptr), [sinptr] "r" (sinptr) + : "hi", "lo", "memory" + ); +} + +void WebRtcIsacfix_Spec2TimeMIPS(int16_t *inreQ7, + int16_t *inimQ7, + int32_t *outre1Q16, + int32_t *outre2Q16) { + int k = FRAMESAMPLES / 4; + int16_t* inre; + int16_t* inim; + int32_t* outre1; + int32_t* outre2; + int16_t* cosptr = (int16_t*)WebRtcIsacfix_kCosTab2; + int16_t* sinptr = (int16_t*)WebRtcIsacfix_kSinTab2; + int32_t r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, max, max1; +#if defined(MIPS_DSP_R1_LE) + int32_t offset = FRAMESAMPLES - 4; +#else // #if defined(MIPS_DSP_R1_LE) + int32_t offset = FRAMESAMPLES - 2; +#endif // #if defined(MIPS_DSP_R1_LE) + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[inre], %[inreQ7], 0 \n\t" + "addiu %[inim] , %[inimQ7], 0 \n\t" + "addiu %[outre1], %[outre1Q16], 0 \n\t" + "addiu %[outre2], %[outre2Q16], 0 \n\t" + "mul %[max], $zero, $zero \n\t" + "1: \n\t" +#if defined(MIPS_DSP_R1_LE) + // Process two samples in one iteration avoiding left shift before + // multiplication. MaxAbsValueW32 function inlined into the loop. + "addu %[r8], %[inre], %[offset] \n\t" + "addu %[r9], %[inim], %[offset] \n\t" + "lwl %[r4], 0(%[r8]) \n\t" + "lwl %[r5], 0(%[r9]) \n\t" + "lwl %[r0], 0(%[inre]) \n\t" + "lwl %[r1], 0(%[inim]) \n\t" + "lwl %[r2], 0(%[cosptr]) \n\t" + "lwl %[r3], 0(%[sinptr]) \n\t" + "lwr %[r4], 0(%[r8]) \n\t" + "lwr %[r5], 0(%[r9]) \n\t" + "lwr %[r0], 0(%[inre]) \n\t" + "lwr %[r1], 0(%[inim]) \n\t" + "lwr %[r2], 0(%[cosptr]) \n\t" + "lwr %[r3], 0(%[sinptr]) \n\t" + "packrl.ph %[r4], %[r4], %[r4] \n\t" + "packrl.ph %[r5], %[r5], %[r5] \n\t" + "muleq_s.w.phr %[r6], %[r0], %[r2] \n\t" + "muleq_s.w.phr %[r7], %[r1], %[r3] \n\t" + "muleq_s.w.phr %[r8], %[r4], %[r2] \n\t" + "muleq_s.w.phr %[r9], %[r5], %[r3] \n\t" + "addiu %[k], %[k], -2 \n\t" + "addiu %[cosptr], %[cosptr], 4 \n\t" + "addiu %[sinptr], %[sinptr], 4 \n\t" + "addiu %[inre], %[inre], 4 \n\t" + "addiu %[inim], %[inim], 4 \n\t" + "shra_r.w %[r6], %[r6], 6 \n\t" + "shra_r.w %[r7], %[r7], 6 \n\t" + "shra_r.w %[r8], %[r8], 6 \n\t" + "shra_r.w %[r9], %[r9], 6 \n\t" + "addu %[r6], %[r6], %[r7] \n\t" + "subu %[r9], %[r9], %[r8] \n\t" + "subu %[r7], %[r6], %[r9] \n\t" + "addu %[r6], %[r6], %[r9] \n\t" + "sw %[r7], 0(%[outre1]) \n\t" + "absq_s.w %[r7], %[r7] \n\t" + "slt %[r8], %[max], %[r7] \n\t" + "movn %[max], %[r7], %[r8] \n\t" + "sll %[r7], %[offset], 1 \n\t" + "addu %[r7], %[outre1], %[r7] \n\t" + "sw %[r6], 4(%[r7]) \n\t" + "absq_s.w %[r6], %[r6] \n\t" + "slt %[r8], %[max], %[r6] \n\t" + "movn %[max], %[r6], %[r8] \n\t" + "muleq_s.w.phl %[r6], %[r0], %[r2] \n\t" + "muleq_s.w.phl %[r7], %[r1], %[r3] \n\t" + "muleq_s.w.phl %[r8], %[r4], %[r2] \n\t" + "muleq_s.w.phl %[r9], %[r5], %[r3] \n\t" + "shra_r.w %[r6], %[r6], 6 \n\t" + "shra_r.w %[r7], %[r7], 6 \n\t" + "shra_r.w %[r8], %[r8], 6 \n\t" + "shra_r.w %[r9], %[r9], 6 \n\t" + "addu %[r6], %[r6], %[r7] \n\t" + "subu %[r9], %[r9], %[r8] \n\t" + "subu %[r7], %[r6], %[r9] \n\t" + "addu %[r6], %[r6], %[r9] \n\t" + "sw %[r7], 4(%[outre1]) \n\t" + "absq_s.w %[r7], %[r7] \n\t" + "slt %[r8], %[max], %[r7] \n\t" + "movn %[max], %[r7], %[r8] \n\t" + "sll %[r7], %[offset], 1 \n\t" + "addu %[r7], %[outre1], %[r7] \n\t" + "sw %[r6], 0(%[r7]) \n\t" + "absq_s.w %[r6], %[r6] \n\t" + "slt %[r8], %[max], %[r6] \n\t" + "movn %[max], %[r6], %[r8] \n\t" + "muleq_s.w.phr %[r6], %[r1], %[r2] \n\t" + "muleq_s.w.phr %[r7], %[r0], %[r3] \n\t" + "muleq_s.w.phr %[r8], %[r5], %[r2] \n\t" + "muleq_s.w.phr %[r9], %[r4], %[r3] \n\t" + "addiu %[outre1], %[outre1], 8 \n\t" + "shra_r.w %[r6], %[r6], 6 \n\t" + "shra_r.w %[r7], %[r7], 6 \n\t" + "shra_r.w %[r8], %[r8], 6 \n\t" + "shra_r.w %[r9], %[r9], 6 \n\t" + "subu %[r6], %[r6], %[r7] \n\t" + "addu %[r9], %[r9], %[r8] \n\t" + "subu %[r7], %[r6], %[r9] \n\t" + "addu %[r6], %[r9], %[r6] \n\t" + "negu %[r6], %[r6] \n\t" + "sw %[r7], 0(%[outre2]) \n\t" + "absq_s.w %[r7], %[r7] \n\t" + "slt %[r8], %[max], %[r7] \n\t" + "movn %[max], %[r7], %[r8] \n\t" + "sll %[r7], %[offset], 1 \n\t" + "addu %[r7], %[outre2], %[r7] \n\t" + "sw %[r6], 4(%[r7]) \n\t" + "absq_s.w %[r6], %[r6] \n\t" + "slt %[r8], %[max], %[r6] \n\t" + "movn %[max], %[r6], %[r8] \n\t" + "muleq_s.w.phl %[r6], %[r1], %[r2] \n\t" + "muleq_s.w.phl %[r7], %[r0], %[r3] \n\t" + "muleq_s.w.phl %[r8], %[r5], %[r2] \n\t" + "muleq_s.w.phl %[r9], %[r4], %[r3] \n\t" + "addiu %[offset], %[offset], -8 \n\t" + "shra_r.w %[r6], %[r6], 6 \n\t" + "shra_r.w %[r7], %[r7], 6 \n\t" + "shra_r.w %[r8], %[r8], 6 \n\t" + "shra_r.w %[r9], %[r9], 6 \n\t" + "subu %[r6], %[r6], %[r7] \n\t" + "addu %[r9], %[r9], %[r8] \n\t" + "subu %[r7], %[r6], %[r9] \n\t" + "addu %[r6], %[r9], %[r6] \n\t" + "negu %[r6], %[r6] \n\t" + "sw %[r7], 4(%[outre2]) \n\t" + "absq_s.w %[r7], %[r7] \n\t" + "slt %[r8], %[max], %[r7] \n\t" + "movn %[max], %[r7], %[r8] \n\t" + "sll %[r7], %[offset], 1 \n\t" + "addu %[r7], %[outre2], %[r7] \n\t" + "sw %[r6], 0(%[r7]) \n\t" + "absq_s.w %[r6], %[r6] \n\t" + "slt %[r8], %[max], %[r6] \n\t" + "movn %[max], %[r6], %[r8] \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[outre2], %[outre2], 8 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "lh %[r0], 0(%[inre]) \n\t" + "lh %[r1], 0(%[inim]) \n\t" + "lh %[r4], 0(%[cosptr]) \n\t" + "lh %[r5], 0(%[sinptr]) \n\t" + "addiu %[k], %[k], -1 \n\t" + "mul %[r2], %[r0], %[r4] \n\t" + "mul %[r0], %[r0], %[r5] \n\t" + "mul %[r3], %[r1], %[r5] \n\t" + "mul %[r1], %[r1], %[r4] \n\t" + "addiu %[cosptr], %[cosptr], 2 \n\t" + "addiu %[sinptr], %[sinptr], 2 \n\t" + "addu %[r8], %[inre], %[offset] \n\t" + "addu %[r9], %[inim], %[offset] \n\t" + "addiu %[r2], %[r2], 16 \n\t" + "sra %[r2], %[r2], 5 \n\t" + "addiu %[r0], %[r0], 16 \n\t" + "sra %[r0], %[r0], 5 \n\t" + "addiu %[r3], %[r3], 16 \n\t" + "sra %[r3], %[r3], 5 \n\t" + "lh %[r6], 0(%[r8]) \n\t" + "lh %[r7], 0(%[r9]) \n\t" + "addiu %[r1], %[r1], 16 \n\t" + "sra %[r1], %[r1], 5 \n\t" + "mul %[r8], %[r7], %[r4] \n\t" + "mul %[r7], %[r7], %[r5] \n\t" + "mul %[r9], %[r6], %[r4] \n\t" + "mul %[r6], %[r6], %[r5] \n\t" + "addu %[r2], %[r2], %[r3] \n\t" + "subu %[r1], %[r1], %[r0] \n\t" + "sll %[r0], %[offset], 1 \n\t" + "addu %[r4], %[outre1], %[r0] \n\t" + "addu %[r5], %[outre2], %[r0] \n\t" + "addiu %[r8], %[r8], 16 \n\t" + "sra %[r8], %[r8], 5 \n\t" + "addiu %[r7], %[r7], 16 \n\t" + "sra %[r7], %[r7], 5 \n\t" + "addiu %[r6], %[r6], 16 \n\t" + "sra %[r6], %[r6], 5 \n\t" + "addiu %[r9], %[r9], 16 \n\t" + "sra %[r9], %[r9], 5 \n\t" + "addu %[r8], %[r8], %[r6] \n\t" + "negu %[r8], %[r8] \n\t" + "subu %[r7], %[r7], %[r9] \n\t" + "subu %[r6], %[r2], %[r7] \n\t" + "addu %[r0], %[r2], %[r7] \n\t" + "addu %[r3], %[r1], %[r8] \n\t" + "subu %[r1], %[r8], %[r1] \n\t" + "sw %[r6], 0(%[outre1]) \n\t" + "sw %[r0], 0(%[r4]) \n\t" + "sw %[r3], 0(%[outre2]) \n\t" + "sw %[r1], 0(%[r5]) \n\t" + "addiu %[outre1], %[outre1], 4 \n\t" + "addiu %[offset], %[offset], -4 \n\t" + "addiu %[inre], %[inre], 2 \n\t" + "addiu %[inim], %[inim], 2 \n\t" + // Inlined WebRtcSpl_MaxAbsValueW32 + "negu %[r5], %[r6] \n\t" + "slt %[r2], %[r6], $zero \n\t" + "movn %[r6], %[r5], %[r2] \n\t" + "negu %[r5], %[r0] \n\t" + "slt %[r2], %[r0], $zero \n\t" + "movn %[r0], %[r5], %[r2] \n\t" + "negu %[r5], %[r3] \n\t" + "slt %[r2], %[r3], $zero \n\t" + "movn %[r3], %[r5], %[r2] \n\t" + "negu %[r5], %[r1] \n\t" + "slt %[r2], %[r1], $zero \n\t" + "movn %[r1], %[r5], %[r2] \n\t" + "slt %[r2], %[r6], %[r0] \n\t" + "slt %[r5], %[r3], %[r1] \n\t" + "movn %[r6], %[r0], %[r2] \n\t" + "movn %[r3], %[r1], %[r5] \n\t" + "slt %[r2], %[r6], %[r3] \n\t" + "movn %[r6], %[r3], %[r2] \n\t" + "slt %[r2], %[max], %[r6] \n\t" + "movn %[max], %[r6], %[r2] \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[outre2], %[outre2], 4 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "clz %[max], %[max] \n\t" + "addiu %[max], %[max], -25 \n\t" + ".set pop \n\t" + : [inre] "=&r" (inre), [inim] "=&r" (inim), + [outre1] "=&r" (outre1), [outre2] "=&r" (outre2), + [offset] "+r" (offset), [k] "+r" (k), [r0] "=&r" (r0), + [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3), + [r4] "=&r" (r4), [r5] "=&r" (r5), [r6] "=&r" (r6), + [r7] "=&r" (r7), [r8] "=&r" (r8), [r9] "=&r" (r9), + [max] "=&r" (max) + : [inreQ7] "r" (inreQ7), [inimQ7] "r" (inimQ7), + [cosptr] "r" (cosptr), [sinptr] "r" (sinptr), + [outre1Q16] "r" (outre1Q16), [outre2Q16] "r" (outre2Q16) + : "hi", "lo", "memory" + ); + + // "Fastest" vectors + k = FRAMESAMPLES / 4; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[inre], %[inreQ7], 0 \n\t" + "addiu %[inim], %[inimQ7], 0 \n\t" + "addiu %[outre1], %[outre1Q16], 0 \n\t" + "addiu %[outre2], %[outre2Q16], 0 \n\t" + "bltz %[max], 2f \n\t" + " subu %[max1], $zero, %[max] \n\t" + "1: \n\t" + "lw %[r0], 0(%[outre1]) \n\t" + "lw %[r1], 0(%[outre2]) \n\t" + "lw %[r2], 4(%[outre1]) \n\t" + "lw %[r3], 4(%[outre2]) \n\t" + "sllv %[r0], %[r0], %[max] \n\t" + "sllv %[r1], %[r1], %[max] \n\t" + "sllv %[r2], %[r2], %[max] \n\t" + "sllv %[r3], %[r3], %[max] \n\t" + "addiu %[k], %[k], -1 \n\t" + "addiu %[outre1], %[outre1], 8 \n\t" + "addiu %[outre2], %[outre2], 8 \n\t" + "sh %[r0], 0(%[inre]) \n\t" + "sh %[r1], 0(%[inim]) \n\t" + "sh %[r2], 2(%[inre]) \n\t" + "sh %[r3], 2(%[inim]) \n\t" + "addiu %[inre], %[inre], 4 \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[inim], %[inim], 4 \n\t" + "b 4f \n\t" + " nop \n\t" + "2: \n\t" +#if !defined(MIPS_DSP_R1_LE) + "addiu %[r4], $zero, 1 \n\t" + "addiu %[r5], %[max1], -1 \n\t" + "sllv %[r4], %[r4], %[r5] \n\t" +#endif // #if !defined(MIPS_DSP_R1_LE) + "3: \n\t" + "lw %[r0], 0(%[outre1]) \n\t" + "lw %[r1], 0(%[outre2]) \n\t" + "lw %[r2], 4(%[outre1]) \n\t" + "lw %[r3], 4(%[outre2]) \n\t" +#if defined(MIPS_DSP_R1_LE) + "shrav_r.w %[r0], %[r0], %[max1] \n\t" + "shrav_r.w %[r1], %[r1], %[max1] \n\t" + "shrav_r.w %[r2], %[r2], %[max1] \n\t" + "shrav_r.w %[r3], %[r3], %[max1] \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r0], %[r4] \n\t" + "addu %[r1], %[r1], %[r4] \n\t" + "addu %[r2], %[r2], %[r4] \n\t" + "addu %[r3], %[r3], %[r4] \n\t" + "srav %[r0], %[r0], %[max1] \n\t" + "srav %[r1], %[r1], %[max1] \n\t" + "srav %[r2], %[r2], %[max1] \n\t" + "srav %[r3], %[r3], %[max1] \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addiu %[outre1], %[outre1], 8 \n\t" + "addiu %[outre2], %[outre2], 8 \n\t" + "sh %[r0], 0(%[inre]) \n\t" + "sh %[r1], 0(%[inim]) \n\t" + "sh %[r2], 2(%[inre]) \n\t" + "sh %[r3], 2(%[inim]) \n\t" + "addiu %[k], %[k], -1 \n\t" + "addiu %[inre], %[inre], 4 \n\t" + "bgtz %[k], 3b \n\t" + " addiu %[inim], %[inim], 4 \n\t" + "4: \n\t" + ".set pop \n\t" + : [k] "+r" (k), [max1] "=&r" (max1), [r0] "=&r" (r0), + [inre] "=&r" (inre), [inim] "=&r" (inim), + [outre1] "=&r" (outre1), [outre2] "=&r" (outre2), +#if !defined(MIPS_DSP_R1_LE) + [r4] "=&r" (r4), [r5] "=&r" (r5), +#endif // #if !defined(MIPS_DSP_R1_LE) + [r1] "=&r" (r1), [r2] "=&r" (r2), [r3] "=&r" (r3) + : [max] "r" (max), [inreQ7] "r" (inreQ7), + [inimQ7] "r" (inimQ7), [outre1Q16] "r" (outre1Q16), + [outre2Q16] "r" (outre2Q16) + : "memory" + ); + + WebRtcIsacfix_FftRadix16Fastest(inreQ7, inimQ7, 1); // real call + + // All the remaining processing is done inside a single loop to avoid + // unnecessary memory accesses. MIPS DSPr2 version processes two samples + // at a time. + cosptr = (int16_t*)WebRtcIsacfix_kCosTab1; + sinptr = (int16_t*)WebRtcIsacfix_kSinTab1; + k = FRAMESAMPLES / 2; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[inre], %[inreQ7], 0 \n\t" + "addiu %[inim], %[inimQ7], 0 \n\t" + "addiu %[outre1], %[outre1Q16], 0 \n\t" + "addiu %[outre2], %[outre2Q16], 0 \n\t" + "addiu %[r4], $zero, 273 \n\t" + "addiu %[r5], $zero, 31727 \n\t" +#if defined(MIPS_DSP_R2_LE) + "addiu %[max], %[max], 16 \n\t" + "replv.ph %[r4], %[r4] \n\t" +#endif // #if defined(MIPS_DSP_R2_LE) + "bltz %[max], 2f \n\t" + " subu %[max1], $zero, %[max] \n\t" +#if defined(MIPS_DSP_R2_LE) + "addiu %[max], %[max], 1 \n\t" +#endif // #if defined(MIPS_DSP_R2_LE) + "1: \n\t" +#if defined(MIPS_DSP_R2_LE) + "lwl %[r0], 0(%[inre]) \n\t" + "lwl %[r1], 0(%[inim]) \n\t" + "lh %[r2], 0(%[cosptr]) \n\t" + "lwr %[r0], 0(%[inre]) \n\t" + "lwr %[r1], 0(%[inim]) \n\t" + "lh %[r3], 0(%[sinptr]) \n\t" + "muleq_s.w.phr %[r6], %[r0], %[r4] \n\t" + "muleq_s.w.phr %[r7], %[r1], %[r4] \n\t" + "muleq_s.w.phl %[r0], %[r0], %[r4] \n\t" + "muleq_s.w.phl %[r1], %[r1], %[r4] \n\t" + "addiu %[k], %[k], -2 \n\t" + "addiu %[inre], %[inre], 4 \n\t" + "addiu %[inim], %[inim], 4 \n\t" + "shrav_r.w %[r6], %[r6], %[max] \n\t" + "shrav_r.w %[r7], %[r7], %[max] \n\t" + "mult $ac0, %[r2], %[r6] \n\t" + "mult $ac1, %[r3], %[r7] \n\t" + "mult $ac2, %[r2], %[r7] \n\t" + "mult $ac3, %[r3], %[r6] \n\t" + "lh %[r2], 2(%[cosptr]) \n\t" + "lh %[r3], 2(%[sinptr]) \n\t" + "extr_r.w %[r6], $ac0, 14 \n\t" + "extr_r.w %[r7], $ac1, 14 \n\t" + "extr_r.w %[r8], $ac2, 14 \n\t" + "extr_r.w %[r9], $ac3, 14 \n\t" + "shrav_r.w %[r0], %[r0], %[max] \n\t" + "shrav_r.w %[r1], %[r1], %[max] \n\t" + "mult $ac0, %[r2], %[r0] \n\t" + "mult $ac1, %[r3], %[r1] \n\t" + "mult $ac2, %[r2], %[r1] \n\t" + "mult $ac3, %[r3], %[r0] \n\t" + "addiu %[cosptr], %[cosptr], 4 \n\t" + "extr_r.w %[r0], $ac0, 14 \n\t" + "extr_r.w %[r1], $ac1, 14 \n\t" + "extr_r.w %[r2], $ac2, 14 \n\t" + "extr_r.w %[r3], $ac3, 14 \n\t" + "subu %[r6], %[r6], %[r7] \n\t" + "addu %[r8], %[r8], %[r9] \n\t" + "mult $ac0, %[r5], %[r6] \n\t" + "mult $ac1, %[r5], %[r8] \n\t" + "addiu %[sinptr], %[sinptr], 4 \n\t" + "subu %[r0], %[r0], %[r1] \n\t" + "addu %[r2], %[r2], %[r3] \n\t" + "extr_r.w %[r1], $ac0, 11 \n\t" + "extr_r.w %[r3], $ac1, 11 \n\t" + "mult $ac2, %[r5], %[r0] \n\t" + "mult $ac3, %[r5], %[r2] \n\t" + "sw %[r1], 0(%[outre1]) \n\t" + "sw %[r3], 0(%[outre2]) \n\t" + "addiu %[outre1], %[outre1], 8 \n\t" + "extr_r.w %[r0], $ac2, 11 \n\t" + "extr_r.w %[r2], $ac3, 11 \n\t" + "sw %[r0], -4(%[outre1]) \n\t" + "sw %[r2], 4(%[outre2]) \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[outre2], %[outre2], 8 \n\t" + "b 3f \n\t" +#else // #if defined(MIPS_DSP_R2_LE) + "lh %[r0], 0(%[inre]) \n\t" + "lh %[r1], 0(%[inim]) \n\t" + "addiu %[k], %[k], -1 \n\t" + "srav %[r0], %[r0], %[max] \n\t" + "srav %[r1], %[r1], %[max] \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "sra %[r0], %[r0], 1 \n\t" + "sra %[r3], %[r1], 16 \n\t" + "andi %[r1], %[r1], 0xFFFF \n\t" + "sra %[r1], %[r1], 1 \n\t" + "mul %[r2], %[r2], %[r4] \n\t" + "mul %[r0], %[r0], %[r4] \n\t" + "mul %[r3], %[r3], %[r4] \n\t" + "mul %[r1], %[r1], %[r4] \n\t" + "addiu %[inre], %[inre], 2 \n\t" + "addiu %[inim], %[inim], 2 \n\t" + "lh %[r6], 0(%[cosptr]) \n\t" + "lh %[r7], 0(%[sinptr]) \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r0], %[r0], 15 \n\t" + "shra_r.w %[r1], %[r1], 15 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r0], %[r0], 0x4000 \n\t" + "addiu %[r1], %[r1], 0x4000 \n\t" + "sra %[r0], %[r0], 15 \n\t" + "sra %[r1], %[r1], 15 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r2], %[r0] \n\t" + "addu %[r1], %[r3], %[r1] \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "mul %[r9], %[r2], %[r6] \n\t" + "mul %[r2], %[r2], %[r7] \n\t" + "mul %[r8], %[r0], %[r6] \n\t" + "mul %[r0], %[r0], %[r7] \n\t" + "sra %[r3], %[r3], 16 \n\t" + "andi %[r1], %[r1], 0xFFFF \n\t" + "sll %[r9], %[r9], 2 \n\t" + "sll %[r2], %[r2], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r8], %[r8], 14 \n\t" + "shra_r.w %[r0], %[r0], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r8], %[r8], 0x2000 \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "sra %[r8], %[r8], 14 \n\t" + "sra %[r0], %[r0], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r9], %[r9], %[r8] \n\t" + "addu %[r2], %[r2], %[r0] \n\t" + "mul %[r0], %[r3], %[r6] \n\t" + "mul %[r3], %[r3], %[r7] \n\t" + "mul %[r8], %[r1], %[r6] \n\t" + "mul %[r1], %[r1], %[r8] \n\t" + "addiu %[cosptr], %[cosptr], 2 \n\t" + "addiu %[sinptr], %[sinptr], 2 \n\t" + "sll %[r0], %[r0], 2 \n\t" + "sll %[r3], %[r3], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r8], %[r8], 14 \n\t" + "shra_r.w %[r1], %[r1], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r8], %[r8], 0x2000 \n\t" + "addiu %[r1], %[r1], 0x2000 \n\t" + "sra %[r8], %[r8], 14 \n\t" + "sra %[r1], %[r1], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r0], %[r8] \n\t" + "addu %[r3], %[r3], %[r1] \n\t" + "subu %[r9], %[r9], %[r3] \n\t" + "addu %[r0], %[r0], %[r2] \n\t" + "sra %[r1], %[r9], 16 \n\t" + "andi %[r9], %[r9], 0xFFFF \n\t" + "mul %[r1], %[r1], %[r5] \n\t" + "mul %[r9], %[r9], %[r5] \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "mul %[r2], %[r2], %[r5] \n\t" + "mul %[r0], %[r0], %[r5] \n\t" + "sll %[r1], %[r1], 5 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r9], %[r9], 11 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r9], %[r9], 0x400 \n\t" + "sra %[r9], %[r9], 11 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r1], %[r1], %[r9] \n\t" + "sll %[r2], %[r2], 5 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r0], %[r0], 11 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r0], %[r0], 0x400 \n\t" + "sra %[r0], %[r0], 11 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r0], %[r2] \n\t" + "sw %[r1], 0(%[outre1]) \n\t" + "addiu %[outre1], %[outre1], 4 \n\t" + "sw %[r0], 0(%[outre2]) \n\t" + "bgtz %[k], 1b \n\t" + " addiu %[outre2], %[outre2], 4 \n\t" + "b 3f \n\t" + " nop \n\t" +#endif // #if defined(MIPS_DSP_R2_LE) + "2: \n\t" +#if defined(MIPS_DSP_R2_LE) + "addiu %[max1], %[max1], -1 \n\t" + "21: \n\t" + "lwl %[r0], 0(%[inre]) \n\t" + "lwl %[r1], 0(%[inim]) \n\t" + "lh %[r2], 0(%[cosptr]) \n\t" + "lwr %[r0], 0(%[inre]) \n\t" + "lwr %[r1], 0(%[inim]) \n\t" + "lh %[r3], 0(%[sinptr]) \n\t" + "muleq_s.w.phr %[r6], %[r0], %[r4] \n\t" + "muleq_s.w.phr %[r7], %[r1], %[r4] \n\t" + "muleq_s.w.phl %[r0], %[r0], %[r4] \n\t" + "muleq_s.w.phl %[r1], %[r1], %[r4] \n\t" + "addiu %[k], %[k], -2 \n\t" + "addiu %[inre], %[inre], 4 \n\t" + "addiu %[inim], %[inim], 4 \n\t" + "sllv %[r6], %[r6], %[max1] \n\t" + "sllv %[r7], %[r7], %[max1] \n\t" + "mult $ac0, %[r2], %[r6] \n\t" + "mult $ac1, %[r3], %[r7] \n\t" + "mult $ac2, %[r2], %[r7] \n\t" + "mult $ac3, %[r3], %[r6] \n\t" + "lh %[r2], 2(%[cosptr]) \n\t" + "lh %[r3], 2(%[sinptr]) \n\t" + "extr_r.w %[r6], $ac0, 14 \n\t" + "extr_r.w %[r7], $ac1, 14 \n\t" + "extr_r.w %[r8], $ac2, 14 \n\t" + "extr_r.w %[r9], $ac3, 14 \n\t" + "sllv %[r0], %[r0], %[max1] \n\t" + "sllv %[r1], %[r1], %[max1] \n\t" + "mult $ac0, %[r2], %[r0] \n\t" + "mult $ac1, %[r3], %[r1] \n\t" + "mult $ac2, %[r2], %[r1] \n\t" + "mult $ac3, %[r3], %[r0] \n\t" + "addiu %[cosptr], %[cosptr], 4 \n\t" + "extr_r.w %[r0], $ac0, 14 \n\t" + "extr_r.w %[r1], $ac1, 14 \n\t" + "extr_r.w %[r2], $ac2, 14 \n\t" + "extr_r.w %[r3], $ac3, 14 \n\t" + "subu %[r6], %[r6], %[r7] \n\t" + "addu %[r8], %[r8], %[r9] \n\t" + "mult $ac0, %[r5], %[r6] \n\t" + "mult $ac1, %[r5], %[r8] \n\t" + "addiu %[sinptr], %[sinptr], 4 \n\t" + "subu %[r0], %[r0], %[r1] \n\t" + "addu %[r2], %[r2], %[r3] \n\t" + "extr_r.w %[r1], $ac0, 11 \n\t" + "extr_r.w %[r3], $ac1, 11 \n\t" + "mult $ac2, %[r5], %[r0] \n\t" + "mult $ac3, %[r5], %[r2] \n\t" + "sw %[r1], 0(%[outre1]) \n\t" + "sw %[r3], 0(%[outre2]) \n\t" + "addiu %[outre1], %[outre1], 8 \n\t" + "extr_r.w %[r0], $ac2, 11 \n\t" + "extr_r.w %[r2], $ac3, 11 \n\t" + "sw %[r0], -4(%[outre1]) \n\t" + "sw %[r2], 4(%[outre2]) \n\t" + "bgtz %[k], 21b \n\t" + " addiu %[outre2], %[outre2], 8 \n\t" + "b 3f \n\t" + " nop \n\t" +#else // #if defined(MIPS_DSP_R2_LE) + "lh %[r0], 0(%[inre]) \n\t" + "lh %[r1], 0(%[inim]) \n\t" + "addiu %[k], %[k], -1 \n\t" + "sllv %[r0], %[r0], %[max1] \n\t" + "sllv %[r1], %[r1], %[max1] \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "sra %[r0], %[r0], 1 \n\t" + "sra %[r3], %[r1], 16 \n\t" + "andi %[r1], %[r1], 0xFFFF \n\t" + "sra %[r1], %[r1], 1 \n\t" + "mul %[r2], %[r2], %[r4] \n\t" + "mul %[r0], %[r0], %[r4] \n\t" + "mul %[r3], %[r3], %[r4] \n\t" + "mul %[r1], %[r1], %[r4] \n\t" + "addiu %[inre], %[inre], 2 \n\t" + "addiu %[inim], %[inim], 2 \n\t" + "lh %[r6], 0(%[cosptr]) \n\t" + "lh %[r7], 0(%[sinptr]) \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r0], %[r0], 15 \n\t" + "shra_r.w %[r1], %[r1], 15 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r0], %[r0], 0x4000 \n\t" + "addiu %[r1], %[r1], 0x4000 \n\t" + "sra %[r0], %[r0], 15 \n\t" + "sra %[r1], %[r1], 15 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r2], %[r0] \n\t" + "addu %[r1], %[r3], %[r1] \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "mul %[r9], %[r2], %[r6] \n\t" + "mul %[r2], %[r2], %[r7] \n\t" + "mul %[r8], %[r0], %[r6] \n\t" + "mul %[r0], %[r0], %[r7] \n\t" + "sra %[r3], %[r1], 16 \n\t" + "andi %[r1], %[r1], 0xFFFF \n\t" + "sll %[r9], %[r9], 2 \n\t" + "sll %[r2], %[r2], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r8], %[r8], 14 \n\t" + "shra_r.w %[r0], %[r0], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r8], %[r8], 0x2000 \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "sra %[r8], %[r8], 14 \n\t" + "sra %[r0], %[r0], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r9], %[r9], %[r8] \n\t" + "addu %[r2], %[r2], %[r0] \n\t" + "mul %[r0], %[r3], %[r6] \n\t" + "mul %[r3], %[r3], %[r7] \n\t" + "mul %[r8], %[r1], %[r6] \n\t" + "mul %[r1], %[r1], %[r7] \n\t" + "addiu %[cosptr], %[cosptr], 2 \n\t" + "addiu %[sinptr], %[sinptr], 2 \n\t" + "sll %[r0], %[r0], 2 \n\t" + "sll %[r3], %[r3], 2 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r8], %[r8], 14 \n\t" + "shra_r.w %[r1], %[r1], 14 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r8], %[r8], 0x2000 \n\t" + "addiu %[r1], %[r1], 0x2000 \n\t" + "sra %[r8], %[r8], 14 \n\t" + "sra %[r1], %[r1], 14 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r0], %[r8] \n\t" + "addu %[r3], %[r3], %[r1] \n\t" + "subu %[r9], %[r9], %[r3] \n\t" + "addu %[r0], %[r0], %[r2] \n\t" + "sra %[r1], %[r9], 16 \n\t" + "andi %[r9], %[r9], 0xFFFF \n\t" + "mul %[r1], %[r1], %[r5] \n\t" + "mul %[r9], %[r9], %[r5] \n\t" + "sra %[r2], %[r0], 16 \n\t" + "andi %[r0], %[r0], 0xFFFF \n\t" + "mul %[r2], %[r2], %[r5] \n\t" + "mul %[r0], %[r0], %[r5] \n\t" + "sll %[r1], %[r1], 5 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r9], %[r9], 11 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r9], %[r9], 0x400 \n\t" + "sra %[r9], %[r9], 11 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r1], %[r1], %[r9] \n\t" + "sll %[r2], %[r2], 5 \n\t" +#if defined(MIPS_DSP_R1_LE) + "shra_r.w %[r0], %[r0], 11 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "addiu %[r0], %[r0], 0x400 \n\t" + "sra %[r0], %[r0], 11 \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "addu %[r0], %[r0], %[r2] \n\t" + "sw %[r1], 0(%[outre1]) \n\t" + "addiu %[outre1], %[outre1], 4 \n\t" + "sw %[r0], 0(%[outre2]) \n\t" + "bgtz %[k], 2b \n\t" + " addiu %[outre2], %[outre2], 4 \n\t" +#endif // #if defined(MIPS_DSP_R2_LE) + "3: \n\t" + ".set pop \n\t" + : [k] "+r" (k), [r0] "=&r" (r0), [r1] "=&r" (r1), + [r2] "=&r" (r2), [r3] "=&r" (r3), [r4] "=&r" (r4), + [r5] "=&r" (r5), [r6] "=&r" (r6), [r7] "=&r" (r7), + [r8] "=&r" (r8), [r9] "=&r" (r9), [max1] "=&r" (max1), + [inre] "=&r" (inre), [inim] "=&r" (inim), + [outre1] "=&r" (outre1), [outre2] "=&r" (outre2) + : [max] "r" (max), [inreQ7] "r" (inreQ7), + [inimQ7] "r" (inimQ7), [cosptr] "r" (cosptr), + [sinptr] "r" (sinptr), [outre1Q16] "r" (outre1Q16), + [outre2Q16] "r" (outre2Q16) + : "hi", "lo", "memory" +#if defined(MIPS_DSP_R2_LE) + , "$ac1hi", "$ac1lo", "$ac2hi", "$ac2lo", "$ac3hi", "$ac3lo" +#endif // #if defined(MIPS_DSP_R2_LE) + ); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_neon.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_neon.c new file mode 100644 index 0000000000..79dadc4600 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_neon.c @@ -0,0 +1,479 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" +#include "modules/audio_coding/codecs/isac/fix/source/fft.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" + +// Tables are defined in transform_tables.c file. +// Cosine table 1 in Q14. +extern const int16_t WebRtcIsacfix_kCosTab1[FRAMESAMPLES/2]; +// Sine table 1 in Q14. +extern const int16_t WebRtcIsacfix_kSinTab1[FRAMESAMPLES/2]; +// Sine table 2 in Q14. +extern const int16_t WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4]; + +static inline int32_t ComplexMulAndFindMaxNeon(int16_t* inre1Q9, + int16_t* inre2Q9, + int32_t* outreQ16, + int32_t* outimQ16) { + int k; + const int16_t* kCosTab = &WebRtcIsacfix_kCosTab1[0]; + const int16_t* kSinTab = &WebRtcIsacfix_kSinTab1[0]; + // 0.5 / sqrt(240) in Q19 is round((.5 / sqrt(240)) * (2^19)) = 16921. + // Use "16921 << 5" and vqdmulh, instead of ">> 26" as in the C code. + int32_t fact = 16921 << 5; + int32x4_t factq = vdupq_n_s32(fact); + uint32x4_t max_r = vdupq_n_u32(0); + uint32x4_t max_i = vdupq_n_u32(0); + + for (k = 0; k < FRAMESAMPLES/2; k += 8) { + int16x8_t tmpr = vld1q_s16(kCosTab); + int16x8_t tmpi = vld1q_s16(kSinTab); + int16x8_t inre1 = vld1q_s16(inre1Q9); + int16x8_t inre2 = vld1q_s16(inre2Q9); + kCosTab += 8; + kSinTab += 8; + inre1Q9 += 8; + inre2Q9 += 8; + + // Use ">> 26", instead of ">> 7", ">> 16" and then ">> 3" as in the C code. + int32x4_t tmp0 = vmull_s16(vget_low_s16(tmpr), vget_low_s16(inre1)); + int32x4_t tmp1 = vmull_s16(vget_low_s16(tmpr), vget_low_s16(inre2)); + tmp0 = vmlal_s16(tmp0, vget_low_s16(tmpi), vget_low_s16(inre2)); + tmp1 = vmlsl_s16(tmp1, vget_low_s16(tmpi), vget_low_s16(inre1)); +#if defined(WEBRTC_ARCH_ARM64) + int32x4_t tmp2 = vmull_high_s16(tmpr, inre1); + int32x4_t tmp3 = vmull_high_s16(tmpr, inre2); + tmp2 = vmlal_high_s16(tmp2, tmpi, inre2); + tmp3 = vmlsl_high_s16(tmp3, tmpi, inre1); +#else + int32x4_t tmp2 = vmull_s16(vget_high_s16(tmpr), vget_high_s16(inre1)); + int32x4_t tmp3 = vmull_s16(vget_high_s16(tmpr), vget_high_s16(inre2)); + tmp2 = vmlal_s16(tmp2, vget_high_s16(tmpi), vget_high_s16(inre2)); + tmp3 = vmlsl_s16(tmp3, vget_high_s16(tmpi), vget_high_s16(inre1)); +#endif + + int32x4_t outr_0 = vqdmulhq_s32(tmp0, factq); + int32x4_t outr_1 = vqdmulhq_s32(tmp2, factq); + int32x4_t outi_0 = vqdmulhq_s32(tmp1, factq); + int32x4_t outi_1 = vqdmulhq_s32(tmp3, factq); + vst1q_s32(outreQ16, outr_0); + outreQ16 += 4; + vst1q_s32(outreQ16, outr_1); + outreQ16 += 4; + vst1q_s32(outimQ16, outi_0); + outimQ16 += 4; + vst1q_s32(outimQ16, outi_1); + outimQ16 += 4; + + // Find the absolute maximum in the vectors. + tmp0 = vabsq_s32(outr_0); + tmp1 = vabsq_s32(outr_1); + tmp2 = vabsq_s32(outi_0); + tmp3 = vabsq_s32(outi_1); + // vabs doesn't change the value of 0x80000000. + // Use u32 so we don't lose the value 0x80000000. + max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp0)); + max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp2)); + max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp1)); + max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp3)); + } + + max_r = vmaxq_u32(max_r, max_i); +#if defined(WEBRTC_ARCH_ARM64) + uint32_t maximum = vmaxvq_u32(max_r); +#else + uint32x2_t max32x2_r = vmax_u32(vget_low_u32(max_r), vget_high_u32(max_r)); + max32x2_r = vpmax_u32(max32x2_r, max32x2_r); + uint32_t maximum = vget_lane_u32(max32x2_r, 0); +#endif + + return (int32_t)maximum; +} + +static inline void PreShiftW32toW16Neon(int32_t* inre, + int32_t* inim, + int16_t* outre, + int16_t* outim, + int32_t sh) { + int k; + int32x4_t sh32x4 = vdupq_n_s32(sh); + for (k = 0; k < FRAMESAMPLES/2; k += 16) { + int32x4x4_t inre32x4x4 = vld4q_s32(inre); + int32x4x4_t inim32x4x4 = vld4q_s32(inim); + inre += 16; + inim += 16; + inre32x4x4.val[0] = vrshlq_s32(inre32x4x4.val[0], sh32x4); + inre32x4x4.val[1] = vrshlq_s32(inre32x4x4.val[1], sh32x4); + inre32x4x4.val[2] = vrshlq_s32(inre32x4x4.val[2], sh32x4); + inre32x4x4.val[3] = vrshlq_s32(inre32x4x4.val[3], sh32x4); + inim32x4x4.val[0] = vrshlq_s32(inim32x4x4.val[0], sh32x4); + inim32x4x4.val[1] = vrshlq_s32(inim32x4x4.val[1], sh32x4); + inim32x4x4.val[2] = vrshlq_s32(inim32x4x4.val[2], sh32x4); + inim32x4x4.val[3] = vrshlq_s32(inim32x4x4.val[3], sh32x4); + int16x4x4_t outre16x4x4; + int16x4x4_t outim16x4x4; + outre16x4x4.val[0] = vmovn_s32(inre32x4x4.val[0]); + outre16x4x4.val[1] = vmovn_s32(inre32x4x4.val[1]); + outre16x4x4.val[2] = vmovn_s32(inre32x4x4.val[2]); + outre16x4x4.val[3] = vmovn_s32(inre32x4x4.val[3]); + outim16x4x4.val[0] = vmovn_s32(inim32x4x4.val[0]); + outim16x4x4.val[1] = vmovn_s32(inim32x4x4.val[1]); + outim16x4x4.val[2] = vmovn_s32(inim32x4x4.val[2]); + outim16x4x4.val[3] = vmovn_s32(inim32x4x4.val[3]); + vst4_s16(outre, outre16x4x4); + vst4_s16(outim, outim16x4x4); + outre += 16; + outim += 16; + } +} + +static inline void PostShiftAndSeparateNeon(int16_t* inre, + int16_t* inim, + int16_t* outre, + int16_t* outim, + int32_t sh) { + int k; + int16_t* inre1 = inre; + int16_t* inre2 = &inre[FRAMESAMPLES/2 - 4]; + int16_t* inim1 = inim; + int16_t* inim2 = &inim[FRAMESAMPLES/2 - 4]; + int16_t* outre1 = outre; + int16_t* outre2 = &outre[FRAMESAMPLES/2 - 4]; + int16_t* outim1 = outim; + int16_t* outim2 = &outim[FRAMESAMPLES/2 - 4]; + const int16_t* kSinTab1 = &WebRtcIsacfix_kSinTab2[0]; + const int16_t* kSinTab2 = &WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4 -4]; + // By vshl, we effectively did "<< (-sh - 23)", instead of "<< (-sh)", + // ">> 14" and then ">> 9" as in the C code. + int32x4_t shift = vdupq_n_s32(-sh - 23); + + for (k = 0; k < FRAMESAMPLES/4; k += 4) { + int16x4_t tmpi = vld1_s16(kSinTab1); + kSinTab1 += 4; + int16x4_t tmpr = vld1_s16(kSinTab2); + kSinTab2 -= 4; + int16x4_t inre_0 = vld1_s16(inre1); + inre1 += 4; + int16x4_t inre_1 = vld1_s16(inre2); + inre2 -= 4; + int16x4_t inim_0 = vld1_s16(inim1); + inim1 += 4; + int16x4_t inim_1 = vld1_s16(inim2); + inim2 -= 4; + tmpr = vneg_s16(tmpr); + inre_1 = vrev64_s16(inre_1); + inim_1 = vrev64_s16(inim_1); + tmpr = vrev64_s16(tmpr); + + int16x4_t xr = vqadd_s16(inre_0, inre_1); + int16x4_t xi = vqsub_s16(inim_0, inim_1); + int16x4_t yr = vqadd_s16(inim_0, inim_1); + int16x4_t yi = vqsub_s16(inre_1, inre_0); + + int32x4_t outr0 = vmull_s16(tmpr, xr); + int32x4_t outi0 = vmull_s16(tmpi, xr); + int32x4_t outr1 = vmull_s16(tmpi, yr); + int32x4_t outi1 = vmull_s16(tmpi, yi); + outr0 = vmlsl_s16(outr0, tmpi, xi); + outi0 = vmlal_s16(outi0, tmpr, xi); + outr1 = vmlal_s16(outr1, tmpr, yi); + outi1 = vmlsl_s16(outi1, tmpr, yr); + + outr0 = vshlq_s32(outr0, shift); + outi0 = vshlq_s32(outi0, shift); + outr1 = vshlq_s32(outr1, shift); + outi1 = vshlq_s32(outi1, shift); + outr1 = vnegq_s32(outr1); + + int16x4_t outre_0 = vmovn_s32(outr0); + int16x4_t outim_0 = vmovn_s32(outi0); + int16x4_t outre_1 = vmovn_s32(outr1); + int16x4_t outim_1 = vmovn_s32(outi1); + outre_1 = vrev64_s16(outre_1); + outim_1 = vrev64_s16(outim_1); + + vst1_s16(outre1, outre_0); + outre1 += 4; + vst1_s16(outim1, outim_0); + outim1 += 4; + vst1_s16(outre2, outre_1); + outre2 -= 4; + vst1_s16(outim2, outim_1); + outim2 -= 4; + } +} + +void WebRtcIsacfix_Time2SpecNeon(int16_t* inre1Q9, + int16_t* inre2Q9, + int16_t* outreQ7, + int16_t* outimQ7) { + int32_t tmpreQ16[FRAMESAMPLES/2], tmpimQ16[FRAMESAMPLES/2]; + int32_t max; + int32_t sh; + + // Multiply with complex exponentials and combine into one complex vector. + // And find the maximum. + max = ComplexMulAndFindMaxNeon(inre1Q9, inre2Q9, tmpreQ16, tmpimQ16); + + sh = (int32_t)WebRtcSpl_NormW32(max); + sh = sh - 24; + + // If sh becomes >= 0, then we should shift sh steps to the left, + // and the domain will become Q(16 + sh). + // If sh becomes < 0, then we should shift -sh steps to the right, + // and the domain will become Q(16 + sh). + PreShiftW32toW16Neon(tmpreQ16, tmpimQ16, inre1Q9, inre2Q9, sh); + + // Get DFT. + WebRtcIsacfix_FftRadix16Fastest(inre1Q9, inre2Q9, -1); + + // If sh >= 0, shift sh steps to the right, + // If sh < 0, shift -sh steps to the left. + // Use symmetry to separate into two complex vectors + // and center frames in time around zero. + PostShiftAndSeparateNeon(inre1Q9, inre2Q9, outreQ7, outimQ7, sh); +} + +static inline int32_t TransformAndFindMaxNeon(int16_t* inre, + int16_t* inim, + int32_t* outre, + int32_t* outim) { + int k; + int16_t* inre1 = inre; + int16_t* inre2 = &inre[FRAMESAMPLES/2 - 4]; + int16_t* inim1 = inim; + int16_t* inim2 = &inim[FRAMESAMPLES/2 - 4]; + int32_t* outre1 = outre; + int32_t* outre2 = &outre[FRAMESAMPLES/2 - 4]; + int32_t* outim1 = outim; + int32_t* outim2 = &outim[FRAMESAMPLES/2 - 4]; + const int16_t* kSinTab1 = &WebRtcIsacfix_kSinTab2[0]; + const int16_t* kSinTab2 = &WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4 - 4]; + uint32x4_t max_r = vdupq_n_u32(0); + uint32x4_t max_i = vdupq_n_u32(0); + + // Use ">> 5", instead of "<< 9" and then ">> 14" as in the C code. + for (k = 0; k < FRAMESAMPLES/4; k += 4) { + int16x4_t tmpi = vld1_s16(kSinTab1); + kSinTab1 += 4; + int16x4_t tmpr = vld1_s16(kSinTab2); + kSinTab2 -= 4; + int16x4_t inre_0 = vld1_s16(inre1); + inre1 += 4; + int16x4_t inre_1 = vld1_s16(inre2); + inre2 -= 4; + int16x4_t inim_0 = vld1_s16(inim1); + inim1 += 4; + int16x4_t inim_1 = vld1_s16(inim2); + inim2 -= 4; + tmpr = vneg_s16(tmpr); + inre_1 = vrev64_s16(inre_1); + inim_1 = vrev64_s16(inim_1); + tmpr = vrev64_s16(tmpr); + + int32x4_t xr = vmull_s16(tmpr, inre_0); + int32x4_t xi = vmull_s16(tmpr, inim_0); + int32x4_t yr = vmull_s16(tmpr, inim_1); + int32x4_t yi = vmull_s16(tmpi, inim_1); + xr = vmlal_s16(xr, tmpi, inim_0); + xi = vmlsl_s16(xi, tmpi, inre_0); + yr = vmlal_s16(yr, tmpi, inre_1); + yi = vmlsl_s16(yi, tmpr, inre_1); + yr = vnegq_s32(yr); + + xr = vshrq_n_s32(xr, 5); + xi = vshrq_n_s32(xi, 5); + yr = vshrq_n_s32(yr, 5); + yi = vshrq_n_s32(yi, 5); + + int32x4_t outr0 = vsubq_s32(xr, yi); + int32x4_t outr1 = vaddq_s32(xr, yi); + int32x4_t outi0 = vaddq_s32(xi, yr); + int32x4_t outi1 = vsubq_s32(yr, xi); + + // Find the absolute maximum in the vectors. + int32x4_t tmp0 = vabsq_s32(outr0); + int32x4_t tmp1 = vabsq_s32(outr1); + int32x4_t tmp2 = vabsq_s32(outi0); + int32x4_t tmp3 = vabsq_s32(outi1); + // vabs doesn't change the value of 0x80000000. + // Use u32 so we don't lose the value 0x80000000. + max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp0)); + max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp2)); + max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp1)); + max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp3)); + + // Store the vectors. + outr1 = vrev64q_s32(outr1); + outi1 = vrev64q_s32(outi1); + int32x4_t outr_1 = vcombine_s32(vget_high_s32(outr1), vget_low_s32(outr1)); + int32x4_t outi_1 = vcombine_s32(vget_high_s32(outi1), vget_low_s32(outi1)); + + vst1q_s32(outre1, outr0); + outre1 += 4; + vst1q_s32(outim1, outi0); + outim1 += 4; + vst1q_s32(outre2, outr_1); + outre2 -= 4; + vst1q_s32(outim2, outi_1); + outim2 -= 4; + } + + max_r = vmaxq_u32(max_r, max_i); +#if defined(WEBRTC_ARCH_ARM64) + uint32_t maximum = vmaxvq_u32(max_r); +#else + uint32x2_t max32x2_r = vmax_u32(vget_low_u32(max_r), vget_high_u32(max_r)); + max32x2_r = vpmax_u32(max32x2_r, max32x2_r); + uint32_t maximum = vget_lane_u32(max32x2_r, 0); +#endif + + return (int32_t)maximum; +} + +static inline void PostShiftAndDivideAndDemodulateNeon(int16_t* inre, + int16_t* inim, + int32_t* outre1, + int32_t* outre2, + int32_t sh) { + int k; + int16_t* p_inre = inre; + int16_t* p_inim = inim; + int32_t* p_outre1 = outre1; + int32_t* p_outre2 = outre2; + const int16_t* kCosTab = &WebRtcIsacfix_kCosTab1[0]; + const int16_t* kSinTab = &WebRtcIsacfix_kSinTab1[0]; + int32x4_t shift = vdupq_n_s32(-sh - 16); + // Divide through by the normalizing constant: + // scale all values with 1/240, i.e. with 273 in Q16. + // 273/65536 ~= 0.0041656 + // 1/240 ~= 0.0041666 + int16x8_t scale = vdupq_n_s16(273); + // Sqrt(240) in Q11 is round(15.49193338482967 * 2048) = 31727. + int factQ19 = 31727 << 16; + int32x4_t fact = vdupq_n_s32(factQ19); + + for (k = 0; k < FRAMESAMPLES/2; k += 8) { + int16x8_t inre16x8 = vld1q_s16(p_inre); + int16x8_t inim16x8 = vld1q_s16(p_inim); + p_inre += 8; + p_inim += 8; + int16x8_t tmpr = vld1q_s16(kCosTab); + int16x8_t tmpi = vld1q_s16(kSinTab); + kCosTab += 8; + kSinTab += 8; + // By vshl and vmull, we effectively did "<< (-sh - 16)", + // instead of "<< (-sh)" and ">> 16" as in the C code. + int32x4_t outre1_0 = vmull_s16(vget_low_s16(inre16x8), vget_low_s16(scale)); + int32x4_t outre2_0 = vmull_s16(vget_low_s16(inim16x8), vget_low_s16(scale)); +#if defined(WEBRTC_ARCH_ARM64) + int32x4_t outre1_1 = vmull_high_s16(inre16x8, scale); + int32x4_t outre2_1 = vmull_high_s16(inim16x8, scale); +#else + int32x4_t outre1_1 = vmull_s16(vget_high_s16(inre16x8), + vget_high_s16(scale)); + int32x4_t outre2_1 = vmull_s16(vget_high_s16(inim16x8), + vget_high_s16(scale)); +#endif + + outre1_0 = vshlq_s32(outre1_0, shift); + outre1_1 = vshlq_s32(outre1_1, shift); + outre2_0 = vshlq_s32(outre2_0, shift); + outre2_1 = vshlq_s32(outre2_1, shift); + + // Demodulate and separate. + int32x4_t tmpr_0 = vmovl_s16(vget_low_s16(tmpr)); + int32x4_t tmpi_0 = vmovl_s16(vget_low_s16(tmpi)); +#if defined(WEBRTC_ARCH_ARM64) + int32x4_t tmpr_1 = vmovl_high_s16(tmpr); + int32x4_t tmpi_1 = vmovl_high_s16(tmpi); +#else + int32x4_t tmpr_1 = vmovl_s16(vget_high_s16(tmpr)); + int32x4_t tmpi_1 = vmovl_s16(vget_high_s16(tmpi)); +#endif + + int64x2_t xr0 = vmull_s32(vget_low_s32(tmpr_0), vget_low_s32(outre1_0)); + int64x2_t xi0 = vmull_s32(vget_low_s32(tmpr_0), vget_low_s32(outre2_0)); + int64x2_t xr2 = vmull_s32(vget_low_s32(tmpr_1), vget_low_s32(outre1_1)); + int64x2_t xi2 = vmull_s32(vget_low_s32(tmpr_1), vget_low_s32(outre2_1)); + xr0 = vmlsl_s32(xr0, vget_low_s32(tmpi_0), vget_low_s32(outre2_0)); + xi0 = vmlal_s32(xi0, vget_low_s32(tmpi_0), vget_low_s32(outre1_0)); + xr2 = vmlsl_s32(xr2, vget_low_s32(tmpi_1), vget_low_s32(outre2_1)); + xi2 = vmlal_s32(xi2, vget_low_s32(tmpi_1), vget_low_s32(outre1_1)); + +#if defined(WEBRTC_ARCH_ARM64) + int64x2_t xr1 = vmull_high_s32(tmpr_0, outre1_0); + int64x2_t xi1 = vmull_high_s32(tmpr_0, outre2_0); + int64x2_t xr3 = vmull_high_s32(tmpr_1, outre1_1); + int64x2_t xi3 = vmull_high_s32(tmpr_1, outre2_1); + xr1 = vmlsl_high_s32(xr1, tmpi_0, outre2_0); + xi1 = vmlal_high_s32(xi1, tmpi_0, outre1_0); + xr3 = vmlsl_high_s32(xr3, tmpi_1, outre2_1); + xi3 = vmlal_high_s32(xi3, tmpi_1, outre1_1); +#else + int64x2_t xr1 = vmull_s32(vget_high_s32(tmpr_0), vget_high_s32(outre1_0)); + int64x2_t xi1 = vmull_s32(vget_high_s32(tmpr_0), vget_high_s32(outre2_0)); + int64x2_t xr3 = vmull_s32(vget_high_s32(tmpr_1), vget_high_s32(outre1_1)); + int64x2_t xi3 = vmull_s32(vget_high_s32(tmpr_1), vget_high_s32(outre2_1)); + xr1 = vmlsl_s32(xr1, vget_high_s32(tmpi_0), vget_high_s32(outre2_0)); + xi1 = vmlal_s32(xi1, vget_high_s32(tmpi_0), vget_high_s32(outre1_0)); + xr3 = vmlsl_s32(xr3, vget_high_s32(tmpi_1), vget_high_s32(outre2_1)); + xi3 = vmlal_s32(xi3, vget_high_s32(tmpi_1), vget_high_s32(outre1_1)); +#endif + + outre1_0 = vcombine_s32(vrshrn_n_s64(xr0, 10), vrshrn_n_s64(xr1, 10)); + outre2_0 = vcombine_s32(vrshrn_n_s64(xi0, 10), vrshrn_n_s64(xi1, 10)); + outre1_1 = vcombine_s32(vrshrn_n_s64(xr2, 10), vrshrn_n_s64(xr3, 10)); + outre2_1 = vcombine_s32(vrshrn_n_s64(xi2, 10), vrshrn_n_s64(xi3, 10)); + outre1_0 = vqdmulhq_s32(outre1_0, fact); + outre2_0 = vqdmulhq_s32(outre2_0, fact); + outre1_1 = vqdmulhq_s32(outre1_1, fact); + outre2_1 = vqdmulhq_s32(outre2_1, fact); + + vst1q_s32(p_outre1, outre1_0); + p_outre1 += 4; + vst1q_s32(p_outre1, outre1_1); + p_outre1 += 4; + vst1q_s32(p_outre2, outre2_0); + p_outre2 += 4; + vst1q_s32(p_outre2, outre2_1); + p_outre2 += 4; + } +} + +void WebRtcIsacfix_Spec2TimeNeon(int16_t* inreQ7, + int16_t* inimQ7, + int32_t* outre1Q16, + int32_t* outre2Q16) { + int32_t max; + int32_t sh; + + max = TransformAndFindMaxNeon(inreQ7, inimQ7, outre1Q16, outre2Q16); + + + sh = (int32_t)WebRtcSpl_NormW32(max); + sh = sh - 24; + // If sh becomes >= 0, then we should shift sh steps to the left, + // and the domain will become Q(16 + sh). + // If sh becomes < 0, then we should shift -sh steps to the right, + // and the domain will become Q(16 + sh). + + // "Fastest" vectors. + PreShiftW32toW16Neon(outre1Q16, outre2Q16, inreQ7, inimQ7, sh); + + // Get IDFT. + WebRtcIsacfix_FftRadix16Fastest(inreQ7, inimQ7, 1); + + PostShiftAndDivideAndDemodulateNeon(inreQ7, inimQ7, outre1Q16, outre2Q16, sh); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_tables.c new file mode 100644 index 0000000000..eed88e4a3b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_tables.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This file contains trigonometric functions look-up tables used in + * transform functions WebRtcIsacfix_Time2Spec and WebRtcIsacfix_Spec2Time. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" +#include "typedefs.h" // NOLINT(build/include) + +/* Cosine table 1 in Q14. */ +const int16_t WebRtcIsacfix_kCosTab1[FRAMESAMPLES/2] = { + 16384, 16383, 16378, 16371, 16362, 16349, 16333, 16315, 16294, 16270, + 16244, 16214, 16182, 16147, 16110, 16069, 16026, 15980, 15931, 15880, + 15826, 15769, 15709, 15647, 15582, 15515, 15444, 15371, 15296, 15218, + 15137, 15053, 14968, 14879, 14788, 14694, 14598, 14500, 14399, 14295, + 14189, 14081, 13970, 13856, 13741, 13623, 13502, 13380, 13255, 13128, + 12998, 12867, 12733, 12597, 12458, 12318, 12176, 12031, 11885, 11736, + 11585, 11433, 11278, 11121, 10963, 10803, 10641, 10477, 10311, 10143, + 9974, 9803, 9630, 9456, 9280, 9102, 8923, 8743, 8561, 8377, + 8192, 8006, 7818, 7629, 7438, 7246, 7053, 6859, 6664, 6467, + 6270, 6071, 5872, 5671, 5469, 5266, 5063, 4859, 4653, 4447, + 4240, 4033, 3825, 3616, 3406, 3196, 2986, 2775, 2563, 2351, + 2139, 1926, 1713, 1499, 1285, 1072, 857, 643, 429, 214, + 0, -214, -429, -643, -857, -1072, -1285, -1499, -1713, -1926, + -2139, -2351, -2563, -2775, -2986, -3196, -3406, -3616, -3825, -4033, + -4240, -4447, -4653, -4859, -5063, -5266, -5469, -5671, -5872, -6071, + -6270, -6467, -6664, -6859, -7053, -7246, -7438, -7629, -7818, -8006, + -8192, -8377, -8561, -8743, -8923, -9102, -9280, -9456, -9630, -9803, + -9974, -10143, -10311, -10477, -10641, -10803, -10963, -11121, -11278, -11433, + -11585, -11736, -11885, -12031, -12176, -12318, -12458, -12597, -12733, + -12867, -12998, -13128, -13255, -13380, -13502, -13623, -13741, -13856, + -13970, -14081, -14189, -14295, -14399, -14500, -14598, -14694, -14788, + -14879, -14968, -15053, -15137, -15218, -15296, -15371, -15444, -15515, + -15582, -15647, -15709, -15769, -15826, -15880, -15931, -15980, -16026, + -16069, -16110, -16147, -16182, -16214, -16244, -16270, -16294, -16315, + -16333, -16349, -16362, -16371, -16378, -16383 +}; + +/* Sine table 1 in Q14. */ +const int16_t WebRtcIsacfix_kSinTab1[FRAMESAMPLES/2] = { + 0, 214, 429, 643, 857, 1072, 1285, 1499, 1713, 1926, + 2139, 2351, 2563, 2775, 2986, 3196, 3406, 3616, 3825, 4033, + 4240, 4447, 4653, 4859, 5063, 5266, 5469, 5671, 5872, 6071, + 6270, 6467, 6664, 6859, 7053, 7246, 7438, 7629, 7818, 8006, + 8192, 8377, 8561, 8743, 8923, 9102, 9280, 9456, 9630, 9803, + 9974, 10143, 10311, 10477, 10641, 10803, 10963, 11121, 11278, 11433, + 11585, 11736, 11885, 12031, 12176, 12318, 12458, 12597, 12733, 12867, + 12998, 13128, 13255, 13380, 13502, 13623, 13741, 13856, 13970, 14081, + 14189, 14295, 14399, 14500, 14598, 14694, 14788, 14879, 14968, 15053, + 15137, 15218, 15296, 15371, 15444, 15515, 15582, 15647, 15709, 15769, + 15826, 15880, 15931, 15980, 16026, 16069, 16110, 16147, 16182, 16214, + 16244, 16270, 16294, 16315, 16333, 16349, 16362, 16371, 16378, 16383, + 16384, 16383, 16378, 16371, 16362, 16349, 16333, 16315, 16294, 16270, + 16244, 16214, 16182, 16147, 16110, 16069, 16026, 15980, 15931, 15880, + 15826, 15769, 15709, 15647, 15582, 15515, 15444, 15371, 15296, 15218, + 15137, 15053, 14968, 14879, 14788, 14694, 14598, 14500, 14399, 14295, + 14189, 14081, 13970, 13856, 13741, 13623, 13502, 13380, 13255, 13128, + 12998, 12867, 12733, 12597, 12458, 12318, 12176, 12031, 11885, 11736, + 11585, 11433, 11278, 11121, 10963, 10803, 10641, 10477, 10311, 10143, + 9974, 9803, 9630, 9456, 9280, 9102, 8923, 8743, 8561, 8377, + 8192, 8006, 7818, 7629, 7438, 7246, 7053, 6859, 6664, 6467, + 6270, 6071, 5872, 5671, 5469, 5266, 5063, 4859, 4653, 4447, + 4240, 4033, 3825, 3616, 3406, 3196, 2986, 2775, 2563, 2351, + 2139, 1926, 1713, 1499, 1285, 1072, 857, 643, 429, 214 +}; + + +/* Sine table 2 in Q14. */ +const int16_t WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4] = { + 16384, -16381, 16375, -16367, 16356, -16342, 16325, -16305, 16283, -16257, + 16229, -16199, 16165, -16129, 16090, -16048, 16003, -15956, 15906, -15853, + 15798, -15739, 15679, -15615, 15549, -15480, 15408, -15334, 15257, -15178, + 15095, -15011, 14924, -14834, 14741, -14647, 14549, -14449, 14347, -14242, + 14135, -14025, 13913, -13799, 13682, -13563, 13441, -13318, 13192, -13063, + 12933, -12800, 12665, -12528, 12389, -12247, 12104, -11958, 11810, -11661, + 11509, -11356, 11200, -11042, 10883, -10722, 10559, -10394, 10227, -10059, + 9889, -9717, 9543, -9368, 9191, -9013, 8833, -8652, 8469, -8285, + 8099, -7912, 7723, -7534, 7342, -7150, 6957, -6762, 6566, -6369, + 6171, -5971, 5771, -5570, 5368, -5165, 4961, -4756, 4550, -4344, + 4137, -3929, 3720, -3511, 3301, -3091, 2880, -2669, 2457, -2245, + 2032, -1819, 1606, -1392, 1179, -965, 750, -536, 322, -107 +}; + +#if defined(MIPS32_LE) +/* Cosine table 2 in Q14. Used only on MIPS platforms. */ +const int16_t WebRtcIsacfix_kCosTab2[FRAMESAMPLES/4] = { + 107, -322, 536, -750, 965, -1179, 1392, -1606, 1819, -2032, + 2245, -2457, 2669, -2880, 3091, -3301, 3511, -3720, 3929, -4137, + 4344, -4550, 4756, -4961, 5165, -5368, 5570, -5771, 5971, -6171, + 6369, -6566, 6762, -6957, 7150, -7342, 7534, -7723, 7912, -8099, + 8285, -8469, 8652, -8833, 9013, -9191, 9368, -9543, 9717, -9889, + 10059, -10227, 10394, -10559, 10722, -10883, 11042, -11200, 11356, -11509, + 11661, -11810, 11958, -12104, 12247, -12389, 12528, -12665, 12800, -12933, + 13063, -13192, 13318, -13441, 13563, -13682, 13799, -13913, 14025, -14135, + 14242, -14347, 14449, -14549, 14647, -14741, 14834, -14924, 15011, -15095, + 15178, -15257, 15334, -15408, 15480, -15549, 15615, -15679, 15739, -15798, + 15853, -15906, 15956, -16003, 16048, -16090, 16129, -16165, 16199, -16229, + 16257, -16283, 16305, -16325, 16342, -16356, 16367, -16375, 16381, -16384 +}; +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_unittest.cc new file mode 100644 index 0000000000..347b0492e9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_unittest.cc @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/source/codec.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" + +static const int kSamples = FRAMESAMPLES/2; +static const int32_t spec2time_out_expected_1[kSamples] = { + -3366470, -2285227, + -3415765, -2310215, -3118030, -2222470, -3030254, -2192091, -3423170, + -2216041, -3305541, -2171936, -3195767, -2095779, -3153304, -2157560, + -3071167, -2032108, -3101190, -1972016, -3103824, -2089118, -3139811, + -1898337, -3102801, -2055082, -3029665, -1854140, -2962586, -1966454, + -3071167, -1894588, -2851743, -1917315, -2848087, -1594932, -2799242, + -1462184, -2845887, -1437599, -2691776, -1329637, -2770659, -1268491, + -2625161, -1578991, -2460299, -1186385, -2365613, -1039354, -2322608, + -958518, -2271749, -789860, -2254538, -850308, -2384436, -850959, -2133734, + -587678, -2093316, -495115, -1973364, -475177, -1801282, -173507, + -1848516, -158015, -1792018, -62648, -1643313, 214746, -1500758, 267077, + -1450193, 560521, -1521579, 675283, -1345408, 857559, -1300822, 1116332, + -1294533, 1241117, -1070027, 1263503, -983816, 1529821, -1019586, + 1910421, -955420, 2073688, -836459, 2401105, -653905, 2690474, -731425, + 2930131, -935234, 3299500, -875978, 3523432, -878906, 3924822, -1081630, + 4561267, -1203023, 5105274, -1510983, 6052762, -2294646, 7021597, + -3108053, 8826736, -4935222, 11678789, -8442713, 18725700, -21526692, + 25420577, 19589811, -28108666, 12634054, -14483066, 6263217, -9979706, + 3665661, -7909736, 2531530, -6434896, 1700772, -5525393, 1479473, + -4894262, 1231760, -4353044, 1032940, -3786590, 941152, -3331614, + 665090, -2851619, 830696, -2762201, 958007, -2483118, 788233, -2184965, + 804825, -1967306, 1007255, -1862474, 920889, -1457506, 755406, -1405841, + 890230, -1302124, 1161599, -701867, 1154163, -1083366, 1204743, -513581, + 1547264, -650636, 1493384, -285543, 1771863, -277906, 1841343, -9078, + 1751863, 230222, 1819578, 207170, 1978972, 398137, 2106468, 552155, + 1997624, 685213, 2129520, 601078, 2238736, 944591, 2441879, 1194178, + 2355280, 986124, 2393328, 1049005, 2417944, 1208368, 2489516, 1352023, + 2572118, 1445283, 2856081, 1532997, 2742279, 1615877, 2915274, 1808036, + 2856871, 1806936, 3241747, 1622461, 2978558, 1841297, 3010378, 1923666, + 3271367, 2126700, 3070935, 1956958, 3107588, 2128405, 3288872, 2114911, + 3315952, 2406651, 3344038, 2370199, 3368980, 2144361, 3305030, 2183803, + 3401450, 2523102, 3405463, 2452475, 3463355, 2421678, 3551968, 2431949, + 3477251, 2148125, 3244489, 2174090}; +static const int32_t spec2time_out_expected_2[kSamples] = { + 1691694, -2499988, -2035547, + 1060469, 988634, -2044502, -306271, 2041000, 201454, -2289456, 93694, + 2129427, -369152, -1887834, 860796, 2089102, -929424, -1673956, 1395291, + 1785651, -1619673, -1380109, 1963449, 1093311, -2111007, -840456, + 2372786, 578119, -2242702, 89774, 2463304, -132717, -2121480, 643634, + 2277636, -1125999, -1995858, 1543748, 2227861, -1483779, -1495491, + 2102642, 1833876, -1920568, -958378, 2485101, 772261, -2454257, -24942, + 2918714, 136838, -2500453, 816118, 3039735, -746560, -2365815, 1586396, + 2714951, -1511696, -1942334, 2571792, 2182827, -2325335, -1311543, + 3055970, 1367220, -2737182, -110626, 3889222, 631008, -3280879, 853066, + 4122279, -706638, -3334449, 2148311, 3993512, -1846301, -3004894, + 3426779, 3329522, -3165264, -2242423, 4756866, 2557711, -4131280, + -805259, 5702711, 1120592, -4852821, 743664, 6476444, -621186, -5465828, + 2815787, 6768835, -3017442, -5338409, 5658126, 6838454, -5492288, + -4682382, 8874947, 6153814, -8832561, -2649251, 12817398, 4237692, + -13000247, 1190661, 18986363, -115738, -19693978, 9908367, 30660381, + -10632635, -37962068, 47022884, 89744622, -42087632, 40279224, + -88869341, -47542383, 38572364, 10441576, -30339718, -9926740, 19896578, + 28009, -18886612, -1124047, 13232498, -4150304, -12770551, 2637074, + 9051831, -6162211, -8713972, 4557937, 5489716, -6862312, -5532349, + 5415449, 2791310, -6999367, -2790102, 5375806, 546222, -6486452, + -821261, 4994973, -1278840, -5645501, 1060484, 3996285, -2503954, + -4653629, 2220549, 3036977, -3282133, -3318585, 2780636, 1789880, + -4004589, -2041031, 3105373, 574819, -3992722, -971004, 3001703, + -676739, -3841508, 417284, 2897970, -1427018, -3058480, 1189948, + 2210960, -2268992, -2603272, 1949785, 1576172, -2720404, -1891738, + 2309456, 769178, -2975646, -707150, 2424652, -88039, -2966660, -65452, + 2320780, -957557, -2798978, 744640, 1879794, -1672081, -2365319, + 1253309, 1366383, -2204082, -1544367, 1801452, 613828, -2531994, + -983847, 2064842, 118326, -2613790, -203220, 2219635, -730341, -2641861, + 563557, 1765434, -1329916, -2272927, 1037138, 1266725, -1939220, + -1588643, 1754528, 816552, -2376303, -1099167, 1864999, 122477, + -2422762, -400027, 1889228, -579916, -2490353, 287139, 2011318, + -1176657, -2502978, 812896, 1116502, -1940211}; +static const int16_t time2spec_out_expected_1[kSamples] = { + 20342, 23889, -10063, -9419, + 3242, 7280, -2012, -5029, 332, 4478, -97, -3244, -891, 3117, 773, -2204, + -1335, 2009, 1236, -1469, -1562, 1277, 1366, -815, -1619, 599, 1449, -177, + -1507, 116, 1294, 263, -1338, -244, 1059, 553, -1045, -549, 829, 826, + -731, -755, 516, 909, -427, -853, 189, 1004, -184, -828, -108, 888, 72, + -700, -280, 717, 342, -611, -534, 601, 534, -374, -646, 399, 567, -171, + -720, 234, 645, -11, -712, -26, 593, 215, -643, -172, 536, 361, -527, + -403, 388, 550, -361, -480, 208, 623, -206, -585, 41, 578, 12, -504, + -182, 583, 218, -437, -339, 499, 263, -354, -450, 347, 456, -193, -524, + 212, 475, -74, -566, 94, 511, 112, -577, -201, 408, 217, -546, -295, 338, + 387, -13, 4, -46, 2, -76, 103, -83, 108, -55, 100, -150, 131, -156, 141, + -171, 179, -190, 128, -227, 172, -214, 215, -189, 265, -244, 322, -335, + 337, -352, 358, -368, 362, -355, 366, -381, 403, -395, 411, -392, 446, + -458, 504, -449, 507, -464, 452, -491, 481, -534, 486, -516, 560, -535, + 525, -537, 559, -554, 570, -616, 591, -585, 627, -509, 588, -584, 547, + -610, 580, -614, 635, -620, 655, -554, 546, -591, 642, -590, 660, -656, + 629, -604, 620, -580, 617, -645, 648, -573, 612, -604, 584, -571, 597, + -562, 627, -550, 560, -606, 529, -584, 568, -503, 532, -463, 512, -440, + 399, -457, 437, -349, 278, -317, 257, -220, 163, -8, -61, 18, -161, 367, + -1306}; +static const int16_t time2spec_out_expected_2[kSamples] = { + 14283, -11552, -15335, 6626, + 7554, -2150, -6309, 1307, 4523, -4, -3908, -314, 3001, 914, -2715, -1042, + 2094, 1272, -1715, -1399, 1263, 1508, -1021, -1534, 735, 1595, -439, -1447, + 155, 1433, 22, -1325, -268, 1205, 424, -1030, -608, 950, 643, -733, -787, + 661, 861, -502, -888, 331, 852, -144, -849, 19, 833, 99, -826, -154, + 771, 368, -735, -459, 645, 513, -491, -604, 431, 630, -314, -598, 183, + 622, -78, -612, -48, 641, 154, -645, -257, 610, 281, -529, -444, 450, + 441, -327, -506, 274, 476, -232, -570, 117, 554, -86, -531, -21, 572, + 151, -606, -221, 496, 322, -407, -388, 407, 394, -268, -428, 280, 505, + -115, -588, 19, 513, -29, -539, -109, 468, 173, -501, -242, 442, 278, + -478, -680, 656, -659, 656, -669, 602, -688, 612, -667, 612, -642, 627, + -648, 653, -676, 596, -680, 655, -649, 678, -672, 587, -608, 637, -645, + 637, -620, 556, -580, 553, -635, 518, -599, 583, -501, 536, -544, 473, + -552, 583, -511, 541, -532, 563, -486, 461, -453, 486, -388, 424, -416, + 432, -374, 399, -462, 364, -346, 293, -329, 331, -313, 281, -247, 309, + -337, 241, -190, 207, -194, 179, -163, 155, -156, 117, -135, 107, -126, + 29, -22, 81, -8, 17, -61, -10, 8, -37, 80, -44, 72, -88, 65, -89, 130, + -114, 181, -215, 189, -245, 260, -288, 294, -339, 344, -396, 407, -429, + 438, -439, 485, -556, 629, -612, 637, -645, 661, -737, 829, -830, 831, + -1041}; + +class TransformTest : public testing::Test { + protected: + TransformTest() { + WebRtcSpl_Init(); + } + + // Pass a function pointer to the Tester function. + void Time2SpecTester(Time2Spec Time2SpecFunction) { + // WebRtcIsacfix_Time2Spec functions hard coded the buffer lengths. It's a + // large buffer but we have to test it here. + int16_t data_in_1[kSamples] = {0}; + int16_t data_in_2[kSamples] = {0}; + int16_t data_out_1[kSamples] = {0}; + int16_t data_out_2[kSamples] = {0}; + + for(int i = 0; i < kSamples; i++) { + data_in_1[i] = i * i + 1777; + data_in_2[i] = WEBRTC_SPL_WORD16_MAX / (i + 1) + 17; + } + + Time2SpecFunction(data_in_1, data_in_2, data_out_1, data_out_2); + + for (int i = 0; i < kSamples; i++) { + // We don't require bit-exact for ARM assembly code. + EXPECT_LE(abs(time2spec_out_expected_1[i] - data_out_1[i]), 1); + EXPECT_LE(abs(time2spec_out_expected_2[i] - data_out_2[i]), 1); + } + } + + // Pass a function pointer to the Tester function. + void Spec2TimeTester(Spec2Time Spec2TimeFunction) { + // WebRtcIsacfix_Spec2Time functions hard coded the buffer lengths. It's a + // large buffer but we have to test it here. + int16_t data_in_1[kSamples] = {0}; + int16_t data_in_2[kSamples] = {0}; + int32_t data_out_1[kSamples] = {0}; + int32_t data_out_2[kSamples] = {0}; + for(int i = 0; i < kSamples; i++) { + data_in_1[i] = i * i + 1777; + data_in_2[i] = WEBRTC_SPL_WORD16_MAX / (i + 1) + 17; + } + + Spec2TimeFunction(data_in_1, data_in_2, data_out_1, data_out_2); + + for (int i = 0; i < kSamples; i++) { + // We don't require bit-exact for ARM assembly code. + EXPECT_LE(abs(spec2time_out_expected_1[i] - data_out_1[i]), 16); + EXPECT_LE(abs(spec2time_out_expected_2[i] - data_out_2[i]), 16); + } + } + +}; + +TEST_F(TransformTest, Time2SpecTest) { + Time2SpecTester(WebRtcIsacfix_Time2SpecC); +#if defined(WEBRTC_HAS_NEON) + Time2SpecTester(WebRtcIsacfix_Time2SpecNeon); +#endif +} + +TEST_F(TransformTest, Spec2TimeTest) { + Spec2TimeTester(WebRtcIsacfix_Spec2TimeC); +#if defined(WEBRTC_HAS_NEON) + Spec2TimeTester(WebRtcIsacfix_Spec2TimeNeon); +#endif +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ChannelFiles.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ChannelFiles.txt new file mode 100644 index 0000000000..05f7410141 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ChannelFiles.txt @@ -0,0 +1,3 @@ +bottlenecks.txt +lowrates.txt +tworates.txt diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFiles.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFiles.txt new file mode 100644 index 0000000000..f26b7afb6c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFiles.txt @@ -0,0 +1,31 @@ +DTMF_16kHz_long.pcm +DTMF_16kHz_short.pcm +F00.INP +F01.INP +F02.INP +F03.INP +F04.INP +F05.INP +F06.INP +longtest.pcm +ltest_speech_clean.pcm +ltest_music.pcm +ltest_speech_noisy.pcm +misc2.pcm +purenb.pcm +sawsweep_380_60.pcm +sinesweep.pcm +sinesweep_half.pcm +speechmusic.pcm +speechmusic_nb.pcm +speechoffice0dB.pcm +speech_and_misc_NB.pcm +speech_and_misc_WB.pcm +testM4.pcm +testM4D_rev.pcm +testM4D.pcm +testfile.pcm +tone_cisco.pcm +tone_cisco_long.pcm +wb_contspeech.pcm +wb_speech_office25db.pcm \ No newline at end of file diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFilesFew.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFilesFew.txt new file mode 100644 index 0000000000..08bbde30d7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/InputFilesFew.txt @@ -0,0 +1,6 @@ +DTMF_16kHz_short.pcm +ltest_speech_noisy.pcm +misc2.pcm +sinesweep.pcm +speechmusic.pcm +tone_cisco.pcm diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ListOfTestCases.xls b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ListOfTestCases.xls new file mode 100644 index 0000000000..f0889ef4ed Binary files /dev/null and b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/ListOfTestCases.xls differ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSAC.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSAC.txt new file mode 100644 index 0000000000..96b87c066b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSAC.txt @@ -0,0 +1,481 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +diff ../dataqa350/i30_1DTMF_16kHz_long.pcm ../dataqa351/i30_1DTMF_16kHz_long.pcm +diff ../dataqa350/i60_1DTMF_16kHz_long.pcm ../dataqa351/i60_1DTMF_16kHz_long.pcm +diff ../dataqa350/i30_2DTMF_16kHz_long.pcm ../dataqa351/i30_2DTMF_16kHz_long.pcm +diff ../dataqa350/i60_2DTMF_16kHz_long.pcm ../dataqa351/i60_2DTMF_16kHz_long.pcm +diff ../dataqa350/i30_3DTMF_16kHz_long.pcm ../dataqa351/i30_3DTMF_16kHz_long.pcm +diff ../dataqa350/i60_3DTMF_16kHz_long.pcm ../dataqa351/i60_3DTMF_16kHz_long.pcm +diff ../dataqa350/i30_4DTMF_16kHz_long.pcm ../dataqa351/i30_4DTMF_16kHz_long.pcm +diff ../dataqa350/i60_4DTMF_16kHz_long.pcm ../dataqa351/i60_4DTMF_16kHz_long.pcm +diff ../dataqa350/i30_5DTMF_16kHz_long.pcm ../dataqa351/i30_5DTMF_16kHz_long.pcm +diff ../dataqa350/i60_5DTMF_16kHz_long.pcm ../dataqa351/i60_5DTMF_16kHz_long.pcm +diff ../dataqa350/i30_6DTMF_16kHz_long.pcm ../dataqa351/i30_6DTMF_16kHz_long.pcm +diff ../dataqa350/i60_6DTMF_16kHz_long.pcm ../dataqa351/i60_6DTMF_16kHz_long.pcm +diff ../dataqa350/a1DTMF_16kHz_long.pcm ../dataqa351/a1DTMF_16kHz_long.pcm +diff ../dataqa350/a2DTMF_16kHz_long.pcm ../dataqa351/a2DTMF_16kHz_long.pcm +diff ../dataqa350/a3DTMF_16kHz_long.pcm ../dataqa351/a3DTMF_16kHz_long.pcm +diff ../dataqa350/i30_7DTMF_16kHz_short.pcm ../dataqa351/i30_7DTMF_16kHz_short.pcm +diff ../dataqa350/i60_7DTMF_16kHz_short.pcm ../dataqa351/i60_7DTMF_16kHz_short.pcm +diff ../dataqa350/i30_8DTMF_16kHz_short.pcm ../dataqa351/i30_8DTMF_16kHz_short.pcm +diff ../dataqa350/i60_8DTMF_16kHz_short.pcm ../dataqa351/i60_8DTMF_16kHz_short.pcm +diff ../dataqa350/i30_9DTMF_16kHz_short.pcm ../dataqa351/i30_9DTMF_16kHz_short.pcm +diff ../dataqa350/i60_9DTMF_16kHz_short.pcm ../dataqa351/i60_9DTMF_16kHz_short.pcm +diff ../dataqa350/i30_10DTMF_16kHz_short.pcm ../dataqa351/i30_10DTMF_16kHz_short.pcm +diff ../dataqa350/i60_10DTMF_16kHz_short.pcm ../dataqa351/i60_10DTMF_16kHz_short.pcm +diff ../dataqa350/i30_11DTMF_16kHz_short.pcm ../dataqa351/i30_11DTMF_16kHz_short.pcm +diff ../dataqa350/i60_11DTMF_16kHz_short.pcm ../dataqa351/i60_11DTMF_16kHz_short.pcm +diff ../dataqa350/i30_12DTMF_16kHz_short.pcm ../dataqa351/i30_12DTMF_16kHz_short.pcm +diff ../dataqa350/i60_12DTMF_16kHz_short.pcm ../dataqa351/i60_12DTMF_16kHz_short.pcm +diff ../dataqa350/a4DTMF_16kHz_short.pcm ../dataqa351/a4DTMF_16kHz_short.pcm +diff ../dataqa350/a5DTMF_16kHz_short.pcm ../dataqa351/a5DTMF_16kHz_short.pcm +diff ../dataqa350/a6DTMF_16kHz_short.pcm ../dataqa351/a6DTMF_16kHz_short.pcm +diff ../dataqa350/i30_13F00.INP ../dataqa350/i30_13F00.INP +diff ../dataqa350/i60_13F00.INP ../dataqa350/i60_13F00.INP +diff ../dataqa350/i30_14F00.INP ../dataqa350/i30_14F00.INP +diff ../dataqa350/i60_14F00.INP ../dataqa350/i60_14F00.INP +diff ../dataqa350/i30_15F00.INP ../dataqa350/i30_15F00.INP +diff ../dataqa350/i60_15F00.INP ../dataqa350/i60_15F00.INP +diff ../dataqa350/i30_16F00.INP ../dataqa350/i30_16F00.INP +diff ../dataqa350/i60_16F00.INP ../dataqa350/i60_16F00.INP +diff ../dataqa350/i30_17F00.INP ../dataqa350/i30_17F00.INP +diff ../dataqa350/i60_17F00.INP ../dataqa350/i60_17F00.INP +diff ../dataqa350/i30_18F00.INP ../dataqa350/i30_18F00.INP +diff ../dataqa350/i60_18F00.INP ../dataqa350/i60_18F00.INP +diff ../dataqa350/a7F00.INP ../dataqa350/a7F00.INP +diff ../dataqa350/a8F00.INP ../dataqa350/a8F00.INP +diff ../dataqa350/a9F00.INP ../dataqa350/a9F00.INP +diff ../dataqa350/i30_19F01.INP ../dataqa350/i30_19F01.INP +diff ../dataqa350/i60_19F01.INP ../dataqa350/i60_19F01.INP +diff ../dataqa350/i30_20F01.INP ../dataqa350/i30_20F01.INP +diff ../dataqa350/i60_20F01.INP ../dataqa350/i60_20F01.INP +diff ../dataqa350/i30_21F01.INP ../dataqa350/i30_21F01.INP +diff ../dataqa350/i60_21F01.INP ../dataqa350/i60_21F01.INP +diff ../dataqa350/i30_22F01.INP ../dataqa350/i30_22F01.INP +diff ../dataqa350/i60_22F01.INP ../dataqa350/i60_22F01.INP +diff ../dataqa350/i30_23F01.INP ../dataqa350/i30_23F01.INP +diff ../dataqa350/i60_23F01.INP ../dataqa350/i60_23F01.INP +diff ../dataqa350/i30_24F01.INP ../dataqa350/i30_24F01.INP +diff ../dataqa350/i60_24F01.INP ../dataqa350/i60_24F01.INP +diff ../dataqa350/a10F01.INP ../dataqa350/a10F01.INP +diff ../dataqa350/a11F01.INP ../dataqa350/a11F01.INP +diff ../dataqa350/a12F01.INP ../dataqa350/a12F01.INP +diff ../dataqa350/i30_25F02.INP ../dataqa350/i30_25F02.INP +diff ../dataqa350/i60_25F02.INP ../dataqa350/i60_25F02.INP +diff ../dataqa350/i30_26F02.INP ../dataqa350/i30_26F02.INP +diff ../dataqa350/i60_26F02.INP ../dataqa350/i60_26F02.INP +diff ../dataqa350/i30_27F02.INP ../dataqa350/i30_27F02.INP +diff ../dataqa350/i60_27F02.INP ../dataqa350/i60_27F02.INP +diff ../dataqa350/i30_28F02.INP ../dataqa350/i30_28F02.INP +diff ../dataqa350/i60_28F02.INP ../dataqa350/i60_28F02.INP +diff ../dataqa350/i30_29F02.INP ../dataqa350/i30_29F02.INP +diff ../dataqa350/i60_29F02.INP ../dataqa350/i60_29F02.INP +diff ../dataqa350/i30_30F02.INP ../dataqa350/i30_30F02.INP +diff ../dataqa350/i60_30F02.INP ../dataqa350/i60_30F02.INP +diff ../dataqa350/a13F02.INP ../dataqa350/a13F02.INP +diff ../dataqa350/a14F02.INP ../dataqa350/a14F02.INP +diff ../dataqa350/a15F02.INP ../dataqa350/a15F02.INP +diff ../dataqa350/i30_31F03.INP ../dataqa350/i30_31F03.INP +diff ../dataqa350/i60_31F03.INP ../dataqa350/i60_31F03.INP +diff ../dataqa350/i30_32F03.INP ../dataqa350/i30_32F03.INP +diff ../dataqa350/i60_32F03.INP ../dataqa350/i60_32F03.INP +diff ../dataqa350/i30_33F03.INP ../dataqa350/i30_33F03.INP +diff ../dataqa350/i60_33F03.INP ../dataqa350/i60_33F03.INP +diff ../dataqa350/i30_34F03.INP ../dataqa350/i30_34F03.INP +diff ../dataqa350/i60_34F03.INP ../dataqa350/i60_34F03.INP +diff ../dataqa350/i30_35F03.INP ../dataqa350/i30_35F03.INP +diff ../dataqa350/i60_35F03.INP ../dataqa350/i60_35F03.INP +diff ../dataqa350/i30_36F03.INP ../dataqa350/i30_36F03.INP +diff ../dataqa350/i60_36F03.INP ../dataqa350/i60_36F03.INP +diff ../dataqa350/a16F03.INP ../dataqa350/a16F03.INP +diff ../dataqa350/a17F03.INP ../dataqa350/a17F03.INP +diff ../dataqa350/a18F03.INP ../dataqa350/a18F03.INP +diff ../dataqa350/i30_37F04.INP ../dataqa350/i30_37F04.INP +diff ../dataqa350/i60_37F04.INP ../dataqa350/i60_37F04.INP +diff ../dataqa350/i30_38F04.INP ../dataqa350/i30_38F04.INP +diff ../dataqa350/i60_38F04.INP ../dataqa350/i60_38F04.INP +diff ../dataqa350/i30_39F04.INP ../dataqa350/i30_39F04.INP +diff ../dataqa350/i60_39F04.INP ../dataqa350/i60_39F04.INP +diff ../dataqa350/i30_40F04.INP ../dataqa350/i30_40F04.INP +diff ../dataqa350/i60_40F04.INP ../dataqa350/i60_40F04.INP +diff ../dataqa350/i30_41F04.INP ../dataqa350/i30_41F04.INP +diff ../dataqa350/i60_41F04.INP ../dataqa350/i60_41F04.INP +diff ../dataqa350/i30_42F04.INP ../dataqa350/i30_42F04.INP +diff ../dataqa350/i60_42F04.INP ../dataqa350/i60_42F04.INP +diff ../dataqa350/a19F04.INP ../dataqa350/a19F04.INP +diff ../dataqa350/a20F04.INP ../dataqa350/a20F04.INP +diff ../dataqa350/a21F04.INP ../dataqa350/a21F04.INP +diff ../dataqa350/i30_43F05.INP ../dataqa350/i30_43F05.INP +diff ../dataqa350/i60_43F05.INP ../dataqa350/i60_43F05.INP +diff ../dataqa350/i30_44F05.INP ../dataqa350/i30_44F05.INP +diff ../dataqa350/i60_44F05.INP ../dataqa350/i60_44F05.INP +diff ../dataqa350/i30_45F05.INP ../dataqa350/i30_45F05.INP +diff ../dataqa350/i60_45F05.INP ../dataqa350/i60_45F05.INP +diff ../dataqa350/i30_46F05.INP ../dataqa350/i30_46F05.INP +diff ../dataqa350/i60_46F05.INP ../dataqa350/i60_46F05.INP +diff ../dataqa350/i30_47F05.INP ../dataqa350/i30_47F05.INP +diff ../dataqa350/i60_47F05.INP ../dataqa350/i60_47F05.INP +diff ../dataqa350/i30_48F05.INP ../dataqa350/i30_48F05.INP +diff ../dataqa350/i60_48F05.INP ../dataqa350/i60_48F05.INP +diff ../dataqa350/a22F05.INP ../dataqa350/a22F05.INP +diff ../dataqa350/a23F05.INP ../dataqa350/a23F05.INP +diff ../dataqa350/a24F05.INP ../dataqa350/a24F05.INP +diff ../dataqa350/i30_49F06.INP ../dataqa350/i30_49F06.INP +diff ../dataqa350/i60_49F06.INP ../dataqa350/i60_49F06.INP +diff ../dataqa350/i30_50F06.INP ../dataqa350/i30_50F06.INP +diff ../dataqa350/i60_50F06.INP ../dataqa350/i60_50F06.INP +diff ../dataqa350/i30_51F06.INP ../dataqa350/i30_51F06.INP +diff ../dataqa350/i60_51F06.INP ../dataqa350/i60_51F06.INP +diff ../dataqa350/i30_52F06.INP ../dataqa350/i30_52F06.INP +diff ../dataqa350/i60_52F06.INP ../dataqa350/i60_52F06.INP +diff ../dataqa350/i30_53F06.INP ../dataqa350/i30_53F06.INP +diff ../dataqa350/i60_53F06.INP ../dataqa350/i60_53F06.INP +diff ../dataqa350/i30_54F06.INP ../dataqa350/i30_54F06.INP +diff ../dataqa350/i60_54F06.INP ../dataqa350/i60_54F06.INP +diff ../dataqa350/a25F06.INP ../dataqa350/a25F06.INP +diff ../dataqa350/a26F06.INP ../dataqa350/a26F06.INP +diff ../dataqa350/a27F06.INP ../dataqa350/a27F06.INP +diff ../dataqa350/i30_55longtest.pcm ../dataqa351/i30_55longtest.pcm +diff ../dataqa350/i60_55longtest.pcm ../dataqa351/i60_55longtest.pcm +diff ../dataqa350/i30_56longtest.pcm ../dataqa351/i30_56longtest.pcm +diff ../dataqa350/i60_56longtest.pcm ../dataqa351/i60_56longtest.pcm +diff ../dataqa350/i30_57longtest.pcm ../dataqa351/i30_57longtest.pcm +diff ../dataqa350/i60_57longtest.pcm ../dataqa351/i60_57longtest.pcm +diff ../dataqa350/i30_58longtest.pcm ../dataqa351/i30_58longtest.pcm +diff ../dataqa350/i60_58longtest.pcm ../dataqa351/i60_58longtest.pcm +diff ../dataqa350/i30_59longtest.pcm ../dataqa351/i30_59longtest.pcm +diff ../dataqa350/i60_59longtest.pcm ../dataqa351/i60_59longtest.pcm +diff ../dataqa350/i30_60longtest.pcm ../dataqa351/i30_60longtest.pcm +diff ../dataqa350/i60_60longtest.pcm ../dataqa351/i60_60longtest.pcm +diff ../dataqa350/a28longtest.pcm ../dataqa351/a28longtest.pcm +diff ../dataqa350/a29longtest.pcm ../dataqa351/a29longtest.pcm +diff ../dataqa350/a30longtest.pcm ../dataqa351/a30longtest.pcm +diff ../dataqa350/i30_61ltest_speech_clean.pcm ../dataqa351/i30_61ltest_speech_clean.pcm +diff ../dataqa350/i60_61ltest_speech_clean.pcm ../dataqa351/i60_61ltest_speech_clean.pcm +diff ../dataqa350/i30_62ltest_speech_clean.pcm ../dataqa351/i30_62ltest_speech_clean.pcm +diff ../dataqa350/i60_62ltest_speech_clean.pcm ../dataqa351/i60_62ltest_speech_clean.pcm +diff ../dataqa350/i30_63ltest_speech_clean.pcm ../dataqa351/i30_63ltest_speech_clean.pcm +diff ../dataqa350/i60_63ltest_speech_clean.pcm ../dataqa351/i60_63ltest_speech_clean.pcm +diff ../dataqa350/i30_64ltest_speech_clean.pcm ../dataqa351/i30_64ltest_speech_clean.pcm +diff ../dataqa350/i60_64ltest_speech_clean.pcm ../dataqa351/i60_64ltest_speech_clean.pcm +diff ../dataqa350/i30_65ltest_speech_clean.pcm ../dataqa351/i30_65ltest_speech_clean.pcm +diff ../dataqa350/i60_65ltest_speech_clean.pcm ../dataqa351/i60_65ltest_speech_clean.pcm +diff ../dataqa350/i30_66ltest_speech_clean.pcm ../dataqa351/i30_66ltest_speech_clean.pcm +diff ../dataqa350/i60_66ltest_speech_clean.pcm ../dataqa351/i60_66ltest_speech_clean.pcm +diff ../dataqa350/a31ltest_speech_clean.pcm ../dataqa351/a31ltest_speech_clean.pcm +diff ../dataqa350/a32ltest_speech_clean.pcm ../dataqa351/a32ltest_speech_clean.pcm +diff ../dataqa350/a33ltest_speech_clean.pcm ../dataqa351/a33ltest_speech_clean.pcm +diff ../dataqa350/i30_67ltest_music.pcm ../dataqa351/i30_67ltest_music.pcm +diff ../dataqa350/i60_67ltest_music.pcm ../dataqa351/i60_67ltest_music.pcm +diff ../dataqa350/i30_68ltest_music.pcm ../dataqa351/i30_68ltest_music.pcm +diff ../dataqa350/i60_68ltest_music.pcm ../dataqa351/i60_68ltest_music.pcm +diff ../dataqa350/i30_69ltest_music.pcm ../dataqa351/i30_69ltest_music.pcm +diff ../dataqa350/i60_69ltest_music.pcm ../dataqa351/i60_69ltest_music.pcm +diff ../dataqa350/i30_70ltest_music.pcm ../dataqa351/i30_70ltest_music.pcm +diff ../dataqa350/i60_70ltest_music.pcm ../dataqa351/i60_70ltest_music.pcm +diff ../dataqa350/i30_71ltest_music.pcm ../dataqa351/i30_71ltest_music.pcm +diff ../dataqa350/i60_71ltest_music.pcm ../dataqa351/i60_71ltest_music.pcm +diff ../dataqa350/i30_72ltest_music.pcm ../dataqa351/i30_72ltest_music.pcm +diff ../dataqa350/i60_72ltest_music.pcm ../dataqa351/i60_72ltest_music.pcm +diff ../dataqa350/a34ltest_music.pcm ../dataqa351/a34ltest_music.pcm +diff ../dataqa350/a35ltest_music.pcm ../dataqa351/a35ltest_music.pcm +diff ../dataqa350/a36ltest_music.pcm ../dataqa351/a36ltest_music.pcm +diff ../dataqa350/i30_73ltest_speech_noisy.pcm ../dataqa351/i30_73ltest_speech_noisy.pcm +diff ../dataqa350/i60_73ltest_speech_noisy.pcm ../dataqa351/i60_73ltest_speech_noisy.pcm +diff ../dataqa350/i30_74ltest_speech_noisy.pcm ../dataqa351/i30_74ltest_speech_noisy.pcm +diff ../dataqa350/i60_74ltest_speech_noisy.pcm ../dataqa351/i60_74ltest_speech_noisy.pcm +diff ../dataqa350/i30_75ltest_speech_noisy.pcm ../dataqa351/i30_75ltest_speech_noisy.pcm +diff ../dataqa350/i60_75ltest_speech_noisy.pcm ../dataqa351/i60_75ltest_speech_noisy.pcm +diff ../dataqa350/i30_76ltest_speech_noisy.pcm ../dataqa351/i30_76ltest_speech_noisy.pcm +diff ../dataqa350/i60_76ltest_speech_noisy.pcm ../dataqa351/i60_76ltest_speech_noisy.pcm +diff ../dataqa350/i30_77ltest_speech_noisy.pcm ../dataqa351/i30_77ltest_speech_noisy.pcm +diff ../dataqa350/i60_77ltest_speech_noisy.pcm ../dataqa351/i60_77ltest_speech_noisy.pcm +diff ../dataqa350/i30_78ltest_speech_noisy.pcm ../dataqa351/i30_78ltest_speech_noisy.pcm +diff ../dataqa350/i60_78ltest_speech_noisy.pcm ../dataqa351/i60_78ltest_speech_noisy.pcm +diff ../dataqa350/a37ltest_speech_noisy.pcm ../dataqa351/a37ltest_speech_noisy.pcm +diff ../dataqa350/a38ltest_speech_noisy.pcm ../dataqa351/a38ltest_speech_noisy.pcm +diff ../dataqa350/a39ltest_speech_noisy.pcm ../dataqa351/a39ltest_speech_noisy.pcm +diff ../dataqa350/i30_79misc2.pcm ../dataqa351/i30_79misc2.pcm +diff ../dataqa350/i60_79misc2.pcm ../dataqa351/i60_79misc2.pcm +diff ../dataqa350/i30_80misc2.pcm ../dataqa351/i30_80misc2.pcm +diff ../dataqa350/i60_80misc2.pcm ../dataqa351/i60_80misc2.pcm +diff ../dataqa350/i30_81misc2.pcm ../dataqa351/i30_81misc2.pcm +diff ../dataqa350/i60_81misc2.pcm ../dataqa351/i60_81misc2.pcm +diff ../dataqa350/i30_82misc2.pcm ../dataqa351/i30_82misc2.pcm +diff ../dataqa350/i60_82misc2.pcm ../dataqa351/i60_82misc2.pcm +diff ../dataqa350/i30_83misc2.pcm ../dataqa351/i30_83misc2.pcm +diff ../dataqa350/i60_83misc2.pcm ../dataqa351/i60_83misc2.pcm +diff ../dataqa350/i30_84misc2.pcm ../dataqa351/i30_84misc2.pcm +diff ../dataqa350/i60_84misc2.pcm ../dataqa351/i60_84misc2.pcm +diff ../dataqa350/a40misc2.pcm ../dataqa351/a40misc2.pcm +diff ../dataqa350/a41misc2.pcm ../dataqa351/a41misc2.pcm +diff ../dataqa350/a42misc2.pcm ../dataqa351/a42misc2.pcm +diff ../dataqa350/i30_85purenb.pcm ../dataqa351/i30_85purenb.pcm +diff ../dataqa350/i60_85purenb.pcm ../dataqa351/i60_85purenb.pcm +diff ../dataqa350/i30_86purenb.pcm ../dataqa351/i30_86purenb.pcm +diff ../dataqa350/i60_86purenb.pcm ../dataqa351/i60_86purenb.pcm +diff ../dataqa350/i30_87purenb.pcm ../dataqa351/i30_87purenb.pcm +diff ../dataqa350/i60_87purenb.pcm ../dataqa351/i60_87purenb.pcm +diff ../dataqa350/i30_88purenb.pcm ../dataqa351/i30_88purenb.pcm +diff ../dataqa350/i60_88purenb.pcm ../dataqa351/i60_88purenb.pcm +diff ../dataqa350/i30_89purenb.pcm ../dataqa351/i30_89purenb.pcm +diff ../dataqa350/i60_89purenb.pcm ../dataqa351/i60_89purenb.pcm +diff ../dataqa350/i30_90purenb.pcm ../dataqa351/i30_90purenb.pcm +diff ../dataqa350/i60_90purenb.pcm ../dataqa351/i60_90purenb.pcm +diff ../dataqa350/a43purenb.pcm ../dataqa351/a43purenb.pcm +diff ../dataqa350/a44purenb.pcm ../dataqa351/a44purenb.pcm +diff ../dataqa350/a45purenb.pcm ../dataqa351/a45purenb.pcm +diff ../dataqa350/i30_91sawsweep_380_60.pcm ../dataqa351/i30_91sawsweep_380_60.pcm +diff ../dataqa350/i60_91sawsweep_380_60.pcm ../dataqa351/i60_91sawsweep_380_60.pcm +diff ../dataqa350/i30_92sawsweep_380_60.pcm ../dataqa351/i30_92sawsweep_380_60.pcm +diff ../dataqa350/i60_92sawsweep_380_60.pcm ../dataqa351/i60_92sawsweep_380_60.pcm +diff ../dataqa350/i30_93sawsweep_380_60.pcm ../dataqa351/i30_93sawsweep_380_60.pcm +diff ../dataqa350/i60_93sawsweep_380_60.pcm ../dataqa351/i60_93sawsweep_380_60.pcm +diff ../dataqa350/i30_94sawsweep_380_60.pcm ../dataqa351/i30_94sawsweep_380_60.pcm +diff ../dataqa350/i60_94sawsweep_380_60.pcm ../dataqa351/i60_94sawsweep_380_60.pcm +diff ../dataqa350/i30_95sawsweep_380_60.pcm ../dataqa351/i30_95sawsweep_380_60.pcm +diff ../dataqa350/i60_95sawsweep_380_60.pcm ../dataqa351/i60_95sawsweep_380_60.pcm +diff ../dataqa350/i30_96sawsweep_380_60.pcm ../dataqa351/i30_96sawsweep_380_60.pcm +diff ../dataqa350/i60_96sawsweep_380_60.pcm ../dataqa351/i60_96sawsweep_380_60.pcm +diff ../dataqa350/a46sawsweep_380_60.pcm ../dataqa351/a46sawsweep_380_60.pcm +diff ../dataqa350/a47sawsweep_380_60.pcm ../dataqa351/a47sawsweep_380_60.pcm +diff ../dataqa350/a48sawsweep_380_60.pcm ../dataqa351/a48sawsweep_380_60.pcm +diff ../dataqa350/i30_97sinesweep.pcm ../dataqa351/i30_97sinesweep.pcm +diff ../dataqa350/i60_97sinesweep.pcm ../dataqa351/i60_97sinesweep.pcm +diff ../dataqa350/i30_98sinesweep.pcm ../dataqa351/i30_98sinesweep.pcm +diff ../dataqa350/i60_98sinesweep.pcm ../dataqa351/i60_98sinesweep.pcm +diff ../dataqa350/i30_99sinesweep.pcm ../dataqa351/i30_99sinesweep.pcm +diff ../dataqa350/i60_99sinesweep.pcm ../dataqa351/i60_99sinesweep.pcm +diff ../dataqa350/i30_100sinesweep.pcm ../dataqa351/i30_100sinesweep.pcm +diff ../dataqa350/i60_100sinesweep.pcm ../dataqa351/i60_100sinesweep.pcm +diff ../dataqa350/i30_101sinesweep.pcm ../dataqa351/i30_101sinesweep.pcm +diff ../dataqa350/i60_101sinesweep.pcm ../dataqa351/i60_101sinesweep.pcm +diff ../dataqa350/i30_102sinesweep.pcm ../dataqa351/i30_102sinesweep.pcm +diff ../dataqa350/i60_102sinesweep.pcm ../dataqa351/i60_102sinesweep.pcm +diff ../dataqa350/a49sinesweep.pcm ../dataqa351/a49sinesweep.pcm +diff ../dataqa350/a50sinesweep.pcm ../dataqa351/a50sinesweep.pcm +diff ../dataqa350/a51sinesweep.pcm ../dataqa351/a51sinesweep.pcm +diff ../dataqa350/i30_103sinesweep_half.pcm ../dataqa351/i30_103sinesweep_half.pcm +diff ../dataqa350/i60_103sinesweep_half.pcm ../dataqa351/i60_103sinesweep_half.pcm +diff ../dataqa350/i30_104sinesweep_half.pcm ../dataqa351/i30_104sinesweep_half.pcm +diff ../dataqa350/i60_104sinesweep_half.pcm ../dataqa351/i60_104sinesweep_half.pcm +diff ../dataqa350/i30_105sinesweep_half.pcm ../dataqa351/i30_105sinesweep_half.pcm +diff ../dataqa350/i60_105sinesweep_half.pcm ../dataqa351/i60_105sinesweep_half.pcm +diff ../dataqa350/i30_106sinesweep_half.pcm ../dataqa351/i30_106sinesweep_half.pcm +diff ../dataqa350/i60_106sinesweep_half.pcm ../dataqa351/i60_106sinesweep_half.pcm +diff ../dataqa350/i30_107sinesweep_half.pcm ../dataqa351/i30_107sinesweep_half.pcm +diff ../dataqa350/i60_107sinesweep_half.pcm ../dataqa351/i60_107sinesweep_half.pcm +diff ../dataqa350/i30_108sinesweep_half.pcm ../dataqa351/i30_108sinesweep_half.pcm +diff ../dataqa350/i60_108sinesweep_half.pcm ../dataqa351/i60_108sinesweep_half.pcm +diff ../dataqa350/a52sinesweep_half.pcm ../dataqa351/a52sinesweep_half.pcm +diff ../dataqa350/a53sinesweep_half.pcm ../dataqa351/a53sinesweep_half.pcm +diff ../dataqa350/a54sinesweep_half.pcm ../dataqa351/a54sinesweep_half.pcm +diff ../dataqa350/i30_109speechmusic.pcm ../dataqa351/i30_109speechmusic.pcm +diff ../dataqa350/i60_109speechmusic.pcm ../dataqa351/i60_109speechmusic.pcm +diff ../dataqa350/i30_110speechmusic.pcm ../dataqa351/i30_110speechmusic.pcm +diff ../dataqa350/i60_110speechmusic.pcm ../dataqa351/i60_110speechmusic.pcm +diff ../dataqa350/i30_111speechmusic.pcm ../dataqa351/i30_111speechmusic.pcm +diff ../dataqa350/i60_111speechmusic.pcm ../dataqa351/i60_111speechmusic.pcm +diff ../dataqa350/i30_112speechmusic.pcm ../dataqa351/i30_112speechmusic.pcm +diff ../dataqa350/i60_112speechmusic.pcm ../dataqa351/i60_112speechmusic.pcm +diff ../dataqa350/i30_113speechmusic.pcm ../dataqa351/i30_113speechmusic.pcm +diff ../dataqa350/i60_113speechmusic.pcm ../dataqa351/i60_113speechmusic.pcm +diff ../dataqa350/i30_114speechmusic.pcm ../dataqa351/i30_114speechmusic.pcm +diff ../dataqa350/i60_114speechmusic.pcm ../dataqa351/i60_114speechmusic.pcm +diff ../dataqa350/a55speechmusic.pcm ../dataqa351/a55speechmusic.pcm +diff ../dataqa350/a56speechmusic.pcm ../dataqa351/a56speechmusic.pcm +diff ../dataqa350/a57speechmusic.pcm ../dataqa351/a57speechmusic.pcm +diff ../dataqa350/i30_115speechmusic_nb.pcm ../dataqa351/i30_115speechmusic_nb.pcm +diff ../dataqa350/i60_115speechmusic_nb.pcm ../dataqa351/i60_115speechmusic_nb.pcm +diff ../dataqa350/i30_116speechmusic_nb.pcm ../dataqa351/i30_116speechmusic_nb.pcm +diff ../dataqa350/i60_116speechmusic_nb.pcm ../dataqa351/i60_116speechmusic_nb.pcm +diff ../dataqa350/i30_117speechmusic_nb.pcm ../dataqa351/i30_117speechmusic_nb.pcm +diff ../dataqa350/i60_117speechmusic_nb.pcm ../dataqa351/i60_117speechmusic_nb.pcm +diff ../dataqa350/i30_118speechmusic_nb.pcm ../dataqa351/i30_118speechmusic_nb.pcm +diff ../dataqa350/i60_118speechmusic_nb.pcm ../dataqa351/i60_118speechmusic_nb.pcm +diff ../dataqa350/i30_119speechmusic_nb.pcm ../dataqa351/i30_119speechmusic_nb.pcm +diff ../dataqa350/i60_119speechmusic_nb.pcm ../dataqa351/i60_119speechmusic_nb.pcm +diff ../dataqa350/i30_120speechmusic_nb.pcm ../dataqa351/i30_120speechmusic_nb.pcm +diff ../dataqa350/i60_120speechmusic_nb.pcm ../dataqa351/i60_120speechmusic_nb.pcm +diff ../dataqa350/a58speechmusic_nb.pcm ../dataqa351/a58speechmusic_nb.pcm +diff ../dataqa350/a59speechmusic_nb.pcm ../dataqa351/a59speechmusic_nb.pcm +diff ../dataqa350/a60speechmusic_nb.pcm ../dataqa351/a60speechmusic_nb.pcm +diff ../dataqa350/i30_121speechoffice0dB.pcm ../dataqa351/i30_121speechoffice0dB.pcm +diff ../dataqa350/i60_121speechoffice0dB.pcm ../dataqa351/i60_121speechoffice0dB.pcm +diff ../dataqa350/i30_122speechoffice0dB.pcm ../dataqa351/i30_122speechoffice0dB.pcm +diff ../dataqa350/i60_122speechoffice0dB.pcm ../dataqa351/i60_122speechoffice0dB.pcm +diff ../dataqa350/i30_123speechoffice0dB.pcm ../dataqa351/i30_123speechoffice0dB.pcm +diff ../dataqa350/i60_123speechoffice0dB.pcm ../dataqa351/i60_123speechoffice0dB.pcm +diff ../dataqa350/i30_124speechoffice0dB.pcm ../dataqa351/i30_124speechoffice0dB.pcm +diff ../dataqa350/i60_124speechoffice0dB.pcm ../dataqa351/i60_124speechoffice0dB.pcm +diff ../dataqa350/i30_125speechoffice0dB.pcm ../dataqa351/i30_125speechoffice0dB.pcm +diff ../dataqa350/i60_125speechoffice0dB.pcm ../dataqa351/i60_125speechoffice0dB.pcm +diff ../dataqa350/i30_126speechoffice0dB.pcm ../dataqa351/i30_126speechoffice0dB.pcm +diff ../dataqa350/i60_126speechoffice0dB.pcm ../dataqa351/i60_126speechoffice0dB.pcm +diff ../dataqa350/a61speechoffice0dB.pcm ../dataqa351/a61speechoffice0dB.pcm +diff ../dataqa350/a62speechoffice0dB.pcm ../dataqa351/a62speechoffice0dB.pcm +diff ../dataqa350/a63speechoffice0dB.pcm ../dataqa351/a63speechoffice0dB.pcm +diff ../dataqa350/i30_127speech_and_misc_NB.pcm ../dataqa351/i30_127speech_and_misc_NB.pcm +diff ../dataqa350/i60_127speech_and_misc_NB.pcm ../dataqa351/i60_127speech_and_misc_NB.pcm +diff ../dataqa350/i30_128speech_and_misc_NB.pcm ../dataqa351/i30_128speech_and_misc_NB.pcm +diff ../dataqa350/i60_128speech_and_misc_NB.pcm ../dataqa351/i60_128speech_and_misc_NB.pcm +diff ../dataqa350/i30_129speech_and_misc_NB.pcm ../dataqa351/i30_129speech_and_misc_NB.pcm +diff ../dataqa350/i60_129speech_and_misc_NB.pcm ../dataqa351/i60_129speech_and_misc_NB.pcm +diff ../dataqa350/i30_130speech_and_misc_NB.pcm ../dataqa351/i30_130speech_and_misc_NB.pcm +diff ../dataqa350/i60_130speech_and_misc_NB.pcm ../dataqa351/i60_130speech_and_misc_NB.pcm +diff ../dataqa350/i30_131speech_and_misc_NB.pcm ../dataqa351/i30_131speech_and_misc_NB.pcm +diff ../dataqa350/i60_131speech_and_misc_NB.pcm ../dataqa351/i60_131speech_and_misc_NB.pcm +diff ../dataqa350/i30_132speech_and_misc_NB.pcm ../dataqa351/i30_132speech_and_misc_NB.pcm +diff ../dataqa350/i60_132speech_and_misc_NB.pcm ../dataqa351/i60_132speech_and_misc_NB.pcm +diff ../dataqa350/a64speech_and_misc_NB.pcm ../dataqa351/a64speech_and_misc_NB.pcm +diff ../dataqa350/a65speech_and_misc_NB.pcm ../dataqa351/a65speech_and_misc_NB.pcm +diff ../dataqa350/a66speech_and_misc_NB.pcm ../dataqa351/a66speech_and_misc_NB.pcm +diff ../dataqa350/i30_133speech_and_misc_WB.pcm ../dataqa351/i30_133speech_and_misc_WB.pcm +diff ../dataqa350/i60_133speech_and_misc_WB.pcm ../dataqa351/i60_133speech_and_misc_WB.pcm +diff ../dataqa350/i30_134speech_and_misc_WB.pcm ../dataqa351/i30_134speech_and_misc_WB.pcm +diff ../dataqa350/i60_134speech_and_misc_WB.pcm ../dataqa351/i60_134speech_and_misc_WB.pcm +diff ../dataqa350/i30_135speech_and_misc_WB.pcm ../dataqa351/i30_135speech_and_misc_WB.pcm +diff ../dataqa350/i60_135speech_and_misc_WB.pcm ../dataqa351/i60_135speech_and_misc_WB.pcm +diff ../dataqa350/i30_136speech_and_misc_WB.pcm ../dataqa351/i30_136speech_and_misc_WB.pcm +diff ../dataqa350/i60_136speech_and_misc_WB.pcm ../dataqa351/i60_136speech_and_misc_WB.pcm +diff ../dataqa350/i30_137speech_and_misc_WB.pcm ../dataqa351/i30_137speech_and_misc_WB.pcm +diff ../dataqa350/i60_137speech_and_misc_WB.pcm ../dataqa351/i60_137speech_and_misc_WB.pcm +diff ../dataqa350/i30_138speech_and_misc_WB.pcm ../dataqa351/i30_138speech_and_misc_WB.pcm +diff ../dataqa350/i60_138speech_and_misc_WB.pcm ../dataqa351/i60_138speech_and_misc_WB.pcm +diff ../dataqa350/a67speech_and_misc_WB.pcm ../dataqa351/a67speech_and_misc_WB.pcm +diff ../dataqa350/a68speech_and_misc_WB.pcm ../dataqa351/a68speech_and_misc_WB.pcm +diff ../dataqa350/a69speech_and_misc_WB.pcm ../dataqa351/a69speech_and_misc_WB.pcm +diff ../dataqa350/i30_139testM4.pcm ../dataqa351/i30_139testM4.pcm +diff ../dataqa350/i60_139testM4.pcm ../dataqa351/i60_139testM4.pcm +diff ../dataqa350/i30_140testM4.pcm ../dataqa351/i30_140testM4.pcm +diff ../dataqa350/i60_140testM4.pcm ../dataqa351/i60_140testM4.pcm +diff ../dataqa350/i30_141testM4.pcm ../dataqa351/i30_141testM4.pcm +diff ../dataqa350/i60_141testM4.pcm ../dataqa351/i60_141testM4.pcm +diff ../dataqa350/i30_142testM4.pcm ../dataqa351/i30_142testM4.pcm +diff ../dataqa350/i60_142testM4.pcm ../dataqa351/i60_142testM4.pcm +diff ../dataqa350/i30_143testM4.pcm ../dataqa351/i30_143testM4.pcm +diff ../dataqa350/i60_143testM4.pcm ../dataqa351/i60_143testM4.pcm +diff ../dataqa350/i30_144testM4.pcm ../dataqa351/i30_144testM4.pcm +diff ../dataqa350/i60_144testM4.pcm ../dataqa351/i60_144testM4.pcm +diff ../dataqa350/a70testM4.pcm ../dataqa351/a70testM4.pcm +diff ../dataqa350/a71testM4.pcm ../dataqa351/a71testM4.pcm +diff ../dataqa350/a72testM4.pcm ../dataqa351/a72testM4.pcm +diff ../dataqa350/i30_145testM4D_rev.pcm ../dataqa351/i30_145testM4D_rev.pcm +diff ../dataqa350/i60_145testM4D_rev.pcm ../dataqa351/i60_145testM4D_rev.pcm +diff ../dataqa350/i30_146testM4D_rev.pcm ../dataqa351/i30_146testM4D_rev.pcm +diff ../dataqa350/i60_146testM4D_rev.pcm ../dataqa351/i60_146testM4D_rev.pcm +diff ../dataqa350/i30_147testM4D_rev.pcm ../dataqa351/i30_147testM4D_rev.pcm +diff ../dataqa350/i60_147testM4D_rev.pcm ../dataqa351/i60_147testM4D_rev.pcm +diff ../dataqa350/i30_148testM4D_rev.pcm ../dataqa351/i30_148testM4D_rev.pcm +diff ../dataqa350/i60_148testM4D_rev.pcm ../dataqa351/i60_148testM4D_rev.pcm +diff ../dataqa350/i30_149testM4D_rev.pcm ../dataqa351/i30_149testM4D_rev.pcm +diff ../dataqa350/i60_149testM4D_rev.pcm ../dataqa351/i60_149testM4D_rev.pcm +diff ../dataqa350/i30_150testM4D_rev.pcm ../dataqa351/i30_150testM4D_rev.pcm +diff ../dataqa350/i60_150testM4D_rev.pcm ../dataqa351/i60_150testM4D_rev.pcm +diff ../dataqa350/a73testM4D_rev.pcm ../dataqa351/a73testM4D_rev.pcm +diff ../dataqa350/a74testM4D_rev.pcm ../dataqa351/a74testM4D_rev.pcm +diff ../dataqa350/a75testM4D_rev.pcm ../dataqa351/a75testM4D_rev.pcm +diff ../dataqa350/i30_151testM4D.pcm ../dataqa351/i30_151testM4D.pcm +diff ../dataqa350/i60_151testM4D.pcm ../dataqa351/i60_151testM4D.pcm +diff ../dataqa350/i30_152testM4D.pcm ../dataqa351/i30_152testM4D.pcm +diff ../dataqa350/i60_152testM4D.pcm ../dataqa351/i60_152testM4D.pcm +diff ../dataqa350/i30_153testM4D.pcm ../dataqa351/i30_153testM4D.pcm +diff ../dataqa350/i60_153testM4D.pcm ../dataqa351/i60_153testM4D.pcm +diff ../dataqa350/i30_154testM4D.pcm ../dataqa351/i30_154testM4D.pcm +diff ../dataqa350/i60_154testM4D.pcm ../dataqa351/i60_154testM4D.pcm +diff ../dataqa350/i30_155testM4D.pcm ../dataqa351/i30_155testM4D.pcm +diff ../dataqa350/i60_155testM4D.pcm ../dataqa351/i60_155testM4D.pcm +diff ../dataqa350/i30_156testM4D.pcm ../dataqa351/i30_156testM4D.pcm +diff ../dataqa350/i60_156testM4D.pcm ../dataqa351/i60_156testM4D.pcm +diff ../dataqa350/a76testM4D.pcm ../dataqa351/a76testM4D.pcm +diff ../dataqa350/a77testM4D.pcm ../dataqa351/a77testM4D.pcm +diff ../dataqa350/a78testM4D.pcm ../dataqa351/a78testM4D.pcm +diff ../dataqa350/i30_157testfile.pcm ../dataqa351/i30_157testfile.pcm +diff ../dataqa350/i60_157testfile.pcm ../dataqa351/i60_157testfile.pcm +diff ../dataqa350/i30_158testfile.pcm ../dataqa351/i30_158testfile.pcm +diff ../dataqa350/i60_158testfile.pcm ../dataqa351/i60_158testfile.pcm +diff ../dataqa350/i30_159testfile.pcm ../dataqa351/i30_159testfile.pcm +diff ../dataqa350/i60_159testfile.pcm ../dataqa351/i60_159testfile.pcm +diff ../dataqa350/i30_160testfile.pcm ../dataqa351/i30_160testfile.pcm +diff ../dataqa350/i60_160testfile.pcm ../dataqa351/i60_160testfile.pcm +diff ../dataqa350/i30_161testfile.pcm ../dataqa351/i30_161testfile.pcm +diff ../dataqa350/i60_161testfile.pcm ../dataqa351/i60_161testfile.pcm +diff ../dataqa350/i30_162testfile.pcm ../dataqa351/i30_162testfile.pcm +diff ../dataqa350/i60_162testfile.pcm ../dataqa351/i60_162testfile.pcm +diff ../dataqa350/a79testfile.pcm ../dataqa351/a79testfile.pcm +diff ../dataqa350/a80testfile.pcm ../dataqa351/a80testfile.pcm +diff ../dataqa350/a81testfile.pcm ../dataqa351/a81testfile.pcm +diff ../dataqa350/i30_163tone_cisco.pcm ../dataqa351/i30_163tone_cisco.pcm +diff ../dataqa350/i60_163tone_cisco.pcm ../dataqa351/i60_163tone_cisco.pcm +diff ../dataqa350/i30_164tone_cisco.pcm ../dataqa351/i30_164tone_cisco.pcm +diff ../dataqa350/i60_164tone_cisco.pcm ../dataqa351/i60_164tone_cisco.pcm +diff ../dataqa350/i30_165tone_cisco.pcm ../dataqa351/i30_165tone_cisco.pcm +diff ../dataqa350/i60_165tone_cisco.pcm ../dataqa351/i60_165tone_cisco.pcm +diff ../dataqa350/i30_166tone_cisco.pcm ../dataqa351/i30_166tone_cisco.pcm +diff ../dataqa350/i60_166tone_cisco.pcm ../dataqa351/i60_166tone_cisco.pcm +diff ../dataqa350/i30_167tone_cisco.pcm ../dataqa351/i30_167tone_cisco.pcm +diff ../dataqa350/i60_167tone_cisco.pcm ../dataqa351/i60_167tone_cisco.pcm +diff ../dataqa350/i30_168tone_cisco.pcm ../dataqa351/i30_168tone_cisco.pcm +diff ../dataqa350/i60_168tone_cisco.pcm ../dataqa351/i60_168tone_cisco.pcm +diff ../dataqa350/a82tone_cisco.pcm ../dataqa351/a82tone_cisco.pcm +diff ../dataqa350/a83tone_cisco.pcm ../dataqa351/a83tone_cisco.pcm +diff ../dataqa350/a84tone_cisco.pcm ../dataqa351/a84tone_cisco.pcm +diff ../dataqa350/i30_169tone_cisco_long.pcm ../dataqa351/i30_169tone_cisco_long.pcm +diff ../dataqa350/i60_169tone_cisco_long.pcm ../dataqa351/i60_169tone_cisco_long.pcm +diff ../dataqa350/i30_170tone_cisco_long.pcm ../dataqa351/i30_170tone_cisco_long.pcm +diff ../dataqa350/i60_170tone_cisco_long.pcm ../dataqa351/i60_170tone_cisco_long.pcm +diff ../dataqa350/i30_171tone_cisco_long.pcm ../dataqa351/i30_171tone_cisco_long.pcm +diff ../dataqa350/i60_171tone_cisco_long.pcm ../dataqa351/i60_171tone_cisco_long.pcm +diff ../dataqa350/i30_172tone_cisco_long.pcm ../dataqa351/i30_172tone_cisco_long.pcm +diff ../dataqa350/i60_172tone_cisco_long.pcm ../dataqa351/i60_172tone_cisco_long.pcm +diff ../dataqa350/i30_173tone_cisco_long.pcm ../dataqa351/i30_173tone_cisco_long.pcm +diff ../dataqa350/i60_173tone_cisco_long.pcm ../dataqa351/i60_173tone_cisco_long.pcm +diff ../dataqa350/i30_174tone_cisco_long.pcm ../dataqa351/i30_174tone_cisco_long.pcm +diff ../dataqa350/i60_174tone_cisco_long.pcm ../dataqa351/i60_174tone_cisco_long.pcm +diff ../dataqa350/a85tone_cisco_long.pcm ../dataqa351/a85tone_cisco_long.pcm +diff ../dataqa350/a86tone_cisco_long.pcm ../dataqa351/a86tone_cisco_long.pcm +diff ../dataqa350/a87tone_cisco_long.pcm ../dataqa351/a87tone_cisco_long.pcm +diff ../dataqa350/i30_175wb_contspeech.pcm ../dataqa351/i30_175wb_contspeech.pcm +diff ../dataqa350/i60_175wb_contspeech.pcm ../dataqa351/i60_175wb_contspeech.pcm +diff ../dataqa350/i30_176wb_contspeech.pcm ../dataqa351/i30_176wb_contspeech.pcm +diff ../dataqa350/i60_176wb_contspeech.pcm ../dataqa351/i60_176wb_contspeech.pcm +diff ../dataqa350/i30_177wb_contspeech.pcm ../dataqa351/i30_177wb_contspeech.pcm +diff ../dataqa350/i60_177wb_contspeech.pcm ../dataqa351/i60_177wb_contspeech.pcm +diff ../dataqa350/i30_178wb_contspeech.pcm ../dataqa351/i30_178wb_contspeech.pcm +diff ../dataqa350/i60_178wb_contspeech.pcm ../dataqa351/i60_178wb_contspeech.pcm +diff ../dataqa350/i30_179wb_contspeech.pcm ../dataqa351/i30_179wb_contspeech.pcm +diff ../dataqa350/i60_179wb_contspeech.pcm ../dataqa351/i60_179wb_contspeech.pcm +diff ../dataqa350/i30_180wb_contspeech.pcm ../dataqa351/i30_180wb_contspeech.pcm +diff ../dataqa350/i60_180wb_contspeech.pcm ../dataqa351/i60_180wb_contspeech.pcm +diff ../dataqa350/a88wb_contspeech.pcm ../dataqa351/a88wb_contspeech.pcm +diff ../dataqa350/a89wb_contspeech.pcm ../dataqa351/a89wb_contspeech.pcm +diff ../dataqa350/a90wb_contspeech.pcm ../dataqa351/a90wb_contspeech.pcm +diff ../dataqa350/i30_181wb_speech_office25db.pcm ../dataqa351/i30_181wb_speech_office25db.pcm +diff ../dataqa350/i60_181wb_speech_office25db.pcm ../dataqa351/i60_181wb_speech_office25db.pcm +diff ../dataqa350/i30_182wb_speech_office25db.pcm ../dataqa351/i30_182wb_speech_office25db.pcm +diff ../dataqa350/i60_182wb_speech_office25db.pcm ../dataqa351/i60_182wb_speech_office25db.pcm +diff ../dataqa350/i30_183wb_speech_office25db.pcm ../dataqa351/i30_183wb_speech_office25db.pcm +diff ../dataqa350/i60_183wb_speech_office25db.pcm ../dataqa351/i60_183wb_speech_office25db.pcm +diff ../dataqa350/i30_184wb_speech_office25db.pcm ../dataqa351/i30_184wb_speech_office25db.pcm +diff ../dataqa350/i60_184wb_speech_office25db.pcm ../dataqa351/i60_184wb_speech_office25db.pcm +diff ../dataqa350/i30_185wb_speech_office25db.pcm ../dataqa351/i30_185wb_speech_office25db.pcm +diff ../dataqa350/i60_185wb_speech_office25db.pcm ../dataqa351/i60_185wb_speech_office25db.pcm +diff ../dataqa350/i30_186wb_speech_office25db.pcm ../dataqa351/i30_186wb_speech_office25db.pcm +diff ../dataqa350/i60_186wb_speech_office25db.pcm ../dataqa351/i60_186wb_speech_office25db.pcm +diff ../dataqa350/a91wb_speech_office25db.pcm ../dataqa351/a91wb_speech_office25db.pcm +diff ../dataqa350/a92wb_speech_office25db.pcm ../dataqa351/a92wb_speech_office25db.pcm +diff ../dataqa350/a93wb_speech_office25db.pcm ../dataqa351/a93wb_speech_office25db.pcm +diff ../dataqa350/a30_1DTMF_16kHz_short.pcm ../dataqa351/a30_1DTMF_16kHz_short.pcm +diff ../dataqa350/a60_1DTMF_16kHz_short.pcm ../dataqa351/a60_1DTMF_16kHz_short.pcm +diff ../dataqa350/a30_2ltest_speech_noisy.pcm ../dataqa351/a30_2ltest_speech_noisy.pcm +diff ../dataqa350/a60_2ltest_speech_noisy.pcm ../dataqa351/a60_2ltest_speech_noisy.pcm +diff ../dataqa350/a30_3misc2.pcm ../dataqa351/a30_3misc2.pcm +diff ../dataqa350/a60_3misc2.pcm ../dataqa351/a60_3misc2.pcm +diff ../dataqa350/a30_4sinesweep.pcm ../dataqa351/a30_4sinesweep.pcm +diff ../dataqa350/a60_4sinesweep.pcm ../dataqa351/a60_4sinesweep.pcm +diff ../dataqa350/a30_5speechmusic.pcm ../dataqa351/a30_5speechmusic.pcm +diff ../dataqa350/a60_5speechmusic.pcm ../dataqa351/a60_5speechmusic.pcm +diff ../dataqa350/a30_6tone_cisco.pcm ../dataqa351/a30_6tone_cisco.pcm +diff ../dataqa350/a60_6tone_cisco.pcm ../dataqa351/a60_6tone_cisco.pcm +diff ../dataqa350/a60_7tone_cisco.pcm ../dataqa351/a60_7tone_cisco.pcm diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSACPLC.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSACPLC.txt new file mode 100644 index 0000000000..9e3629b2ca --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/diffiSACPLC.txt @@ -0,0 +1,20 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGFILE=logplc.txt +echo "START PLC TEST" > $LOGFILE + +OUTDIR1=../dataqaplc_0 +OUTDIR2=../dataqaplc_1 + +diff $OUTDIR1/outplc1.pcm $OUTDIR2/outplc1.pcm +diff $OUTDIR1/outplc2.pcm $OUTDIR2/outplc2.pcm +diff $OUTDIR1/outplc3.pcm $OUTDIR2/outplc3.pcm +diff $OUTDIR1/outplc4.pcm $OUTDIR2/outplc4.pcm +diff $OUTDIR1/outplc5.pcm $OUTDIR2/outplc5.pcm +diff $OUTDIR1/outplc6.pcm $OUTDIR2/outplc6.pcm + +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACLongtest.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACLongtest.txt new file mode 100644 index 0000000000..eeffc0c955 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACLongtest.txt @@ -0,0 +1,61 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGFILE=logNormal.txt +echo "START ISAC TEST" > $LOGFILE +echo >> $LOGFILE + +ISAC=../Release/kenny.exe +ISACFIXFLOAT=../Release/testFixFloat.exe + +INFILES=$(cat InputFiles.txt) +SUBSET=$(cat InputFilesFew.txt) +CHANNELFILES=$(cat ChannelFiles.txt) +CHANNELLIST=($(cat ChannelFiles.txt)) +INDIR=../data/orig +OUTDIR=../dataqa +mkdir -p $OUTDIR + +TARGETRATE=(10000 15000 20000 25000 30000 32000) +#echo ${CHANNELFILES[1]} + +index1=0 +index2=0 + +for file in $INFILES # loop over all input files + do + + for rate in ${TARGETRATE[*]} + do + let "index1=index1+1" + $ISAC -I $rate -FL 30 $INDIR/"$file" $OUTDIR/i30_$index1"$file" >> $LOGFILE + $ISAC -I $rate -FL 60 $INDIR/"$file" $OUTDIR/i60_$index1"$file" >> $LOGFILE + done + for channel in $CHANNELFILES + do + let "index2=index2+1" + $ISAC $INDIR/$channel $INDIR/"$file" $OUTDIR/a$index2"$file" >> $LOGFILE + done + +done + +index1=0 + +for file in $SUBSET # loop over the subset of input files + do + let "index1=index1+1" + $ISAC $INDIR/${CHANNELLIST[0]} -FL 30 -FIXED_FL $INDIR/"$file" $OUTDIR/a30_$index1"$file" >> $LOGFILE + $ISAC $INDIR/${CHANNELLIST[0]} -FL 60 -FIXED_FL $INDIR/"$file" $OUTDIR/a60_$index1"$file" >> $LOGFILE +done + +let "index1=index1+1" + $ISAC $INDIR/${CHANNELLIST[0]} -INITRATE 25000 -FL 30 $INDIR/"$file" $OUTDIR/a60_$index1"$file" >> $LOGFILE + +# Run fault test + +#./runiSACfault.txt + +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACNB.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACNB.txt new file mode 100644 index 0000000000..605595cc04 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACNB.txt @@ -0,0 +1,45 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGFILE=logNB.txt +echo "START NARROWBAND TEST" > $LOGFILE +echo >> $LOGFILE + +ISAC=../Release/kenny.exe +ISACFIXFLOAT=../Release/testFixFloat.exe + +INFILES=$(cat InputFiles.txt) +SUBSET=$(cat InputFilesFew.txt) +CHANNELFILES=$(cat ChannelFiles.txt) +CHANNELLIST=($(cat ChannelFiles.txt)) +INDIR=../data/orig +OUTDIR=../dataqaNB +mkdir -p $OUTDIR + +TARGETRATE=(10000 15000 20000 25000 30000 32000) +#echo ${CHANNELFILES[1]} + +index1=0 +index2=0 + +# Narrowband Interfaces + +for file in $SUBSET # loop over all input files + do + for rate in ${TARGETRATE[*]} + do + let "index1=index1+1" + $ISAC $rate -FL 30 -NB 1 $INDIR/"$file" $OUTDIR/nb130_$index1"$file" >> $LOGFILE + $ISAC $rate -FL 60 -NB 1 $INDIR/"$file" $OUTDIR/nb160_$index1"$file" >> $LOGFILE + $ISAC $rate -FL 30 -NB 2 $INDIR/"$file" $OUTDIR/nb230_$index1"$file" >> $LOGFILE + $ISAC $rate -FL 60 -NB 2 $INDIR/"$file" $OUTDIR/nb260_$index1"$file" >> $LOGFILE + $ISAC $rate -FL 30 -NB 2 -PL 10 $INDIR/"$file" $OUTDIR/nb2plc30_$index1"$file" >> $LOGFILE + $ISAC $rate -FL 60 -NB 2 -PL 10 $INDIR/"$file" $OUTDIR/nb2plc60_$index1"$file" >> $LOGFILE + done + +done + +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACPLC.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACPLC.txt new file mode 100644 index 0000000000..6bee6f7c3f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACPLC.txt @@ -0,0 +1,23 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGFILE=logplc.txt +echo "START PLC TEST" > $LOGFILE + +ISAC=../Release/kenny.exe + +INDIR=../data/orig +OUTDIR=../dataqaplc_0 +mkdir -p $OUTDIR + +$ISAC 12000 -PL 15 $INDIR/speechmusic.pcm $OUTDIR/outplc1.pcm +$ISAC 20000 -PL 15 $INDIR/speechmusic.pcm $OUTDIR/outplc2.pcm +$ISAC 32000 -PL 15 $INDIR/speechmusic.pcm $OUTDIR/outplc3.pcm +$ISAC 12000 -PL 15 $INDIR/tone_cisco.pcm $OUTDIR/outplc4.pcm +$ISAC 20000 -PL 15 $INDIR/tone_cisco.pcm $OUTDIR/outplc5.pcm +$ISAC 32000 -PL 15 $INDIR/tone_cisco.pcm $OUTDIR/outplc6.pcm + +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACRate.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACRate.txt new file mode 100644 index 0000000000..d8403e099d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACRate.txt @@ -0,0 +1,23 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGG=loggRate.txt +OUTDIR=../dataqaRate +mkdir -p $OUTDIR + +../Release/kenny.exe 13000 -FIXED_FL -FL 30 -MAX 100 ../data/orig/sawsweep_380_60.pcm $OUTDIR/out_napi_1.pcm > $LOGG +../Release/kenny.exe ../data/orig/bottlenecks.txt -FIXED_FL -FL 30 -MAXRATE 32000 ../data/orig/sawsweep_380_60.pcm $OUTDIR/out_napi_2.pcm >> $LOGG +../Release/kenny.exe 13000 -FIXED_FL -FL 30 -MAX 100 ../data/orig/sawsweep_380_60.pcm $OUTDIR/out_napi_3.pcm >> $LOGG +../Release/kenny.exe ../data/orig/bottlenecks.txt -FIXED_FL -FL 30 -MAXRATE 32000 ../data/orig/sawsweep_380_60.pcm $OUTDIR/out_napi_4.pcm >> $LOGG +../Release/kenny.exe 13000 -FIXED_FL -FL 60 -MAX 100 ../data/orig/sawsweep_380_60.pcm $OUTDIR/out_napi_5.pcm >> $LOGG +../Release/kenny.exe ../data/orig/bottlenecks.txt -FIXED_FL -FL 60 -MAXRATE 32000 ../data/orig/sawsweep_380_60.pcm $OUTDIR/out_napi_6.pcm >> $LOGG +../Release/kenny.exe 13000 -INIT_RATE 32000 -FIXED_FL -FL 60 -MAX 100 ../data/orig/sawsweep_380_60.pcm $OUTDIR/out_napi_7.pcm >> $LOGG + +../Release/kenny.exe 13000 -FIXED_FL -FL 30 -MAX 100 ../data/orig/longspeech.pcm $OUTDIR/out_napi_11.pcm >> $LOGG +../Release/kenny.exe ../data/orig/bottlenecks.txt -FIXED_FL -FL 30 -MAXRATE 32000 ../data/orig/longspeech.pcm $OUTDIR/out_napi_12.pcm >> $LOGG +../Release/kenny.exe 13000 -FIXED_FL -FL 30 -MAX 100 ../data/orig/longspeech.pcm $OUTDIR/out_napi_13.pcm >> $LOGG +../Release/kenny.exe ../data/orig/bottlenecks.txt -FIXED_FL -FL 30 -MAXRATE 32000 ../data/orig/longspeech.pcm $OUTDIR/out_napi_14.pcm >> $LOGG +../Release/kenny.exe 13000 -FIXED_FL -FL 60 -MAX 100 ../data/orig/longspeech.pcm $OUTDIR/out_napi_15.pcm >> $LOGG +../Release/kenny.exe ../data/orig/bottlenecks.txt -FIXED_FL -FL 60 -MAXRATE 32000 ../data/orig/longspeech.pcm $OUTDIR/out_napi_16.pcm >> $LOGG +../Release/kenny.exe 13000 -INIT_RATE 32000 -FIXED_FL -FL 60 -MAX 100 ../data/orig/longspeech.pcm $OUTDIR/out_napi_17.pcm >> $LOGG + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfault.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfault.txt new file mode 100644 index 0000000000..f4d9478fd4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfault.txt @@ -0,0 +1,40 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGFILE=logfault.txt +echo "START FAULT TEST" > $LOGFILE + +ISAC=../Release/kenny.exe +ISACFIXFLOAT=../Release/testFixFloat.exe + +INFILES=$(cat InputFiles.txt) +SUBSET=$(cat InputFilesFew.txt) +CHANNELFILES=$(cat ChannelFiles.txt) +CHANNELLIST=($(cat ChannelFiles.txt)) +INDIR=../data/orig +OUTDIR=../dataqaft +mkdir -p $OUTDIR + +TARGETRATE=(10000 15000 20000 25000 30000 32000) +FAULTTEST=(1 2 3 4 5 6 7 9) + +index1=0 + +file=wb_contspeech.pcm + +# Fault test +for testnr in ${FAULTTEST[*]} + do + $ISAC 32000 -F $testnr $INDIR/"$file" $OUTDIR/ft$testnr"$file" >> $LOGFILE +done + +# Fault test number 10, error in bitstream + $ISAC 32000 -F 10 $INDIR/"$file" $OUTDIR/ft10_"$file" >> $LOGFILE + $ISAC 32000 -F 10 -PL 10 $INDIR/"$file" $OUTDIR/ft10plc_"$file" >> $LOGFILE + $ISAC 32000 -F 10 -NB 1 $INDIR/"$file" $OUTDIR/ft10nb1_"$file" >> $LOGFILE + $ISAC 32000 -F 10 -NB 2 -PL 10 $INDIR/"$file" $OUTDIR/ft10nb2_"$file" >> $LOGFILE + +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfixfloat.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfixfloat.txt new file mode 100644 index 0000000000..c9e02df2e9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/QA/runiSACfixfloat.txt @@ -0,0 +1,47 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGFILE=logfxfl.txt +echo "START FIX-FLOAT TEST" > $LOGFILE + + +ISACFIXFLOAT=../testFixFloat.exe + +INFILES=$(cat InputFiles.txt) +SUBSET=$(cat InputFilesFew.txt) +CHANNELFILES=$(cat ChannelFiles.txt) +CHANNELLIST=($(cat ChannelFiles.txt)) +INDIR=../data/orig +OUTDIR=../dataqafxfl +mkdir -p $OUTDIR + +index1=0 + +for file in $INFILES # loop over all input files + do + + for channel in $CHANNELFILES + do + let "index1=index1+1" + + $ISACFIXFLOAT $INDIR/$channel -m 1 -PLC $INDIR/"$file" $OUTDIR/flfx$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 2 -PLC $INDIR/"$file" $OUTDIR/fxfl$index1"$file" >> $LOGFILE + done + +done + +index1=0 + +for file in $SUBSET # loop over the subset of input files + do + let "index1=index1+1" + $ISACFIXFLOAT $INDIR/$channel -m 1 -NB 1 $INDIR/"$file" $OUTDIR/flfxnb1_$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 2 -NB 1 $INDIR/"$file" $OUTDIR/fxflnb1_$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 1 -NB 2 -PLC $INDIR/"$file" $OUTDIR/flfxnb2_$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 2 -NB 2 -PLC $INDIR/"$file" $OUTDIR/fxflnb2_$index1"$file" >> $LOGFILE +done + +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/isac_speed_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/isac_speed_test.cc new file mode 100644 index 0000000000..fc779d8185 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/isac_speed_test.cc @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/include/isacfix.h" +#include "modules/audio_coding/codecs/isac/fix/source/settings.h" +#include "modules/audio_coding/codecs/tools/audio_codec_speed_test.h" + +using std::string; + +namespace webrtc { + +static const int kIsacBlockDurationMs = 30; +static const int kIsacInputSamplingKhz = 16; +static const int kIsacOutputSamplingKhz = 16; + +class IsacSpeedTest : public AudioCodecSpeedTest { + protected: + IsacSpeedTest(); + void SetUp() override; + void TearDown() override; + float EncodeABlock(int16_t* in_data, uint8_t* bit_stream, + size_t max_bytes, size_t* encoded_bytes) override; + float DecodeABlock(const uint8_t* bit_stream, size_t encoded_bytes, + int16_t* out_data) override; + ISACFIX_MainStruct *ISACFIX_main_inst_; +}; + +IsacSpeedTest::IsacSpeedTest() + : AudioCodecSpeedTest(kIsacBlockDurationMs, + kIsacInputSamplingKhz, + kIsacOutputSamplingKhz), + ISACFIX_main_inst_(NULL) { +} + +void IsacSpeedTest::SetUp() { + AudioCodecSpeedTest::SetUp(); + + // Check whether the allocated buffer for the bit stream is large enough. + EXPECT_GE(max_bytes_, static_cast(STREAM_MAXW16_60MS)); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcIsacfix_Create(&ISACFIX_main_inst_)); + EXPECT_EQ(0, WebRtcIsacfix_EncoderInit(ISACFIX_main_inst_, 1)); + WebRtcIsacfix_DecoderInit(ISACFIX_main_inst_); + // Set bitrate and block length. + EXPECT_EQ(0, WebRtcIsacfix_Control(ISACFIX_main_inst_, bit_rate_, + block_duration_ms_)); +} + +void IsacSpeedTest::TearDown() { + AudioCodecSpeedTest::TearDown(); + // Free memory. + EXPECT_EQ(0, WebRtcIsacfix_Free(ISACFIX_main_inst_)); +} + +float IsacSpeedTest::EncodeABlock(int16_t* in_data, uint8_t* bit_stream, + size_t max_bytes, size_t* encoded_bytes) { + // ISAC takes 10 ms everycall + const int subblocks = block_duration_ms_ / 10; + const int subblock_length = 10 * input_sampling_khz_; + int value = 0; + + clock_t clocks = clock(); + size_t pointer = 0; + for (int idx = 0; idx < subblocks; idx++, pointer += subblock_length) { + value = WebRtcIsacfix_Encode(ISACFIX_main_inst_, &in_data[pointer], + bit_stream); + if (idx == subblocks - 1) + EXPECT_GT(value, 0); + else + EXPECT_EQ(0, value); + } + clocks = clock() - clocks; + *encoded_bytes = static_cast(value); + assert(*encoded_bytes <= max_bytes); + return 1000.0 * clocks / CLOCKS_PER_SEC; +} + +float IsacSpeedTest::DecodeABlock(const uint8_t* bit_stream, + size_t encoded_bytes, + int16_t* out_data) { + int value; + int16_t audio_type; + clock_t clocks = clock(); + value = WebRtcIsacfix_Decode(ISACFIX_main_inst_, bit_stream, encoded_bytes, + out_data, &audio_type); + clocks = clock() - clocks; + EXPECT_EQ(output_length_sample_, static_cast(value)); + return 1000.0 * clocks / CLOCKS_PER_SEC; +} + +TEST_P(IsacSpeedTest, IsacEncodeDecodeTest) { + size_t kDurationSec = 400; // Test audio length in second. + EncodeDecode(kDurationSec); +} + +const coding_param param_set[] = { + std::make_tuple(1, + 32000, + string("audio_coding/speech_mono_16kHz"), + string("pcm"), + true)}; + +INSTANTIATE_TEST_CASE_P(AllTest, IsacSpeedTest, + ::testing::ValuesIn(param_set)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/kenny.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/kenny.cc new file mode 100644 index 0000000000..b11c6702ce --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/test/kenny.cc @@ -0,0 +1,845 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include +#include + +#include "modules/audio_coding/codecs/isac/fix/include/isacfix.h" +#include "test/gtest.h" +#include "test/testsupport/perf_test.h" + +// TODO(kma): Clean up the code and change benchmarking the whole codec to +// separate encoder and decoder. + +/* Defines */ +#define SEED_FILE "randseed.txt" /* Used when running decoder on garbage data */ +#define MAX_FRAMESAMPLES 960 /* max number of samples per frame (= 60 ms frame) */ +#define FRAMESAMPLES_10ms 160 /* number of samples per 10ms frame */ +#define FS 16000 /* sampling frequency (Hz) */ + +/* Function for reading audio data from PCM file */ +int readframe(int16_t *data, FILE *inp, int length) { + + short k, rlen, status = 0; + + rlen = fread(data, sizeof(int16_t), length, inp); + if (rlen < length) { + for (k = rlen; k < length; k++) + data[k] = 0; + status = 1; + } + + return status; +} + +// Globals needed because gtest does not provide access to argv. +// This should be reworked to use flags. +static int global_argc; +static char **global_argv; + +/* Struct for bottleneck model */ +typedef struct { + uint32_t send_time; /* samples */ + uint32_t arrival_time; /* samples */ + uint32_t sample_count; /* samples */ + uint16_t rtp_number; +} BottleNeckModel; + +void get_arrival_time(int current_framesamples, /* samples */ + size_t packet_size, /* bytes */ + int bottleneck, /* excluding headers; bits/s */ + BottleNeckModel *BN_data) +{ + const int HeaderSize = 35; + int HeaderRate; + + HeaderRate = HeaderSize * 8 * FS / current_framesamples; /* bits/s */ + + /* everything in samples */ + BN_data->sample_count = BN_data->sample_count + current_framesamples; + + BN_data->arrival_time += static_cast( + ((packet_size + HeaderSize) * 8 * FS) / (bottleneck + HeaderRate)); + BN_data->send_time += current_framesamples; + + if (BN_data->arrival_time < BN_data->sample_count) + BN_data->arrival_time = BN_data->sample_count; + + BN_data->rtp_number++; +} + +void get_arrival_time2(int current_framesamples, + int current_delay, + BottleNeckModel *BN_data) +{ + if (current_delay == -1) + //dropped packet + { + BN_data->arrival_time += current_framesamples; + } + else if (current_delay != -2) + { + // + BN_data->arrival_time += (current_framesamples + ((FS/1000) * current_delay)); + } + //else + //current packet has same timestamp as previous packet + + BN_data->rtp_number++; +} + +TEST(IsacFixTest, Kenny) { + int argc = global_argc; + char **argv = global_argv; + + char inname[100], outname[100], outbitsname[100], bottleneck_file[100]; + FILE *inp, *outp, *f_bn, *outbits; + int endfile; + + size_t i; + int errtype, h = 0, k, packetLossPercent = 0; + int16_t CodingMode; + int16_t bottleneck; + int framesize = 30; /* ms */ + int cur_framesmpls, err = 0, lostPackets = 0; + + /* Runtime statistics */ + double starttime, runtime, length_file; + + int stream_len_int = 0; + size_t stream_len = 0; + int16_t framecnt; + int declen = 0; + int16_t shortdata[FRAMESAMPLES_10ms]; + int16_t decoded[MAX_FRAMESAMPLES]; + uint16_t streamdata[500]; + int16_t speechType[1]; + size_t prevFrameSize = 1; + int16_t rateBPS = 0; + int16_t fixedFL = 0; + int16_t payloadSize = 0; + int32_t payloadRate = 0; + int setControlBWE = 0; + int readLoss; + FILE *plFile = NULL; + + char version_number[20]; + char tmpBit[5] = ".bit"; + + int totalbits =0; + int totalsmpls =0; + int16_t testNum, testCE; + + FILE *fp_gns = NULL; + int gns = 0; + int cur_delay = 0; + char gns_file[100]; + + int nbTest = 0; + int16_t lostFrame; + float scale = (float)0.7; + /* only one structure used for ISAC encoder */ + ISACFIX_MainStruct *ISAC_main_inst = NULL; + + /* For fault test 10, garbage data */ + FILE *seedfile; + unsigned int random_seed = (unsigned int) time(NULL);//1196764538 + + BottleNeckModel BN_data; + f_bn = NULL; + + readLoss = 0; + packetLossPercent = 0; + + /* Handling wrong input arguments in the command line */ + if ((argc<3) || (argc>21)) { + printf("\n\nWrong number of arguments or flag values.\n\n"); + + printf("\n"); + WebRtcIsacfix_version(version_number); + printf("iSAC version %s \n\n", version_number); + + printf("Usage:\n\n"); + printf("%s [-F num][-I] bottleneck_value infile outfile \n\n", argv[0]); + printf("with:\n"); + printf("[-I] :if -I option is specified, the coder will use\n"); + printf(" an instantaneous Bottleneck value. If not, it\n"); + printf(" will be an adaptive Bottleneck value.\n\n"); + printf("bottleneck_value :the value of the bottleneck provided either\n"); + printf(" as a fixed value (e.g. 25000) or\n"); + printf(" read from a file (e.g. bottleneck.txt)\n\n"); + printf("[-INITRATE num] :Set a new value for initial rate. Note! Only used" + " in adaptive mode.\n\n"); + printf("[-FL num] :Set (initial) frame length in msec. Valid length" + " are 30 and 60 msec.\n\n"); + printf("[-FIXED_FL] :Frame length to be fixed to initial value.\n\n"); + printf("[-MAX num] :Set the limit for the payload size of iSAC" + " in bytes. \n"); + printf(" Minimum 100, maximum 400.\n\n"); + printf("[-MAXRATE num] :Set the maxrate for iSAC in bits per second. \n"); + printf(" Minimum 32000, maximum 53400.\n\n"); + printf("[-F num] :if -F option is specified, the test function\n"); + printf(" will run the iSAC API fault scenario specified" + " by the\n"); + printf(" supplied number.\n"); + printf(" F 1 - Call encoder prior to init encoder call\n"); + printf(" F 2 - Call decoder prior to init decoder call\n"); + printf(" F 3 - Call decoder prior to encoder call\n"); + printf(" F 4 - Call decoder with a too short coded" + " sequence\n"); + printf(" F 5 - Call decoder with a too long coded" + " sequence\n"); + printf(" F 6 - Call decoder with random bit stream\n"); + printf(" F 7 - Call init encoder/decoder at random" + " during a call\n"); + printf(" F 8 - Call encoder/decoder without having" + " allocated memory for \n"); + printf(" encoder/decoder instance\n"); + printf(" F 9 - Call decodeB without calling decodeA\n"); + printf(" F 10 - Call decodeB with garbage data\n"); + printf("[-PL num] : if -PL option is specified 0 encode with narrowband encoder" + " (infile is narrowband)\n"); + printf(" num=2 => decode with narrowband decoder" + " (outfile is narrowband)\n\n"); + printf("[-CE num] : Test of APIs used by Conference Engine.\n"); + printf(" CE 1 - createInternal, freeInternal," + " getNewBitstream \n"); + printf(" CE 2 - transcode, getBWE \n"); + printf(" CE 3 - getSendBWE, setSendBWE. \n\n"); + printf("[-RTP_INIT num] : if -RTP_INIT option is specified num will be" + " the initial\n"); + printf(" value of the rtp sequence number.\n\n"); + printf("infile : Normal speech input file\n\n"); + printf("outfile : Speech output file\n\n"); + printf("Example usage : \n\n"); + printf("%s -I bottleneck.txt speechIn.pcm speechOut.pcm\n\n", argv[0]); + exit(1); + + } + + /* Print version number */ + WebRtcIsacfix_version(version_number); + printf("iSAC version %s \n\n", version_number); + + /* Loop over all command line arguments */ + CodingMode = 0; + testNum = 0; + testCE = 0; + for (i = 1; i + 2 < static_cast(argc); i++) { + /* Instantaneous mode */ + if (!strcmp ("-I", argv[i])) { + printf("\nInstantaneous BottleNeck\n"); + CodingMode = 1; + i++; + } + + /* Set (initial) bottleneck value */ + if (!strcmp ("-INITRATE", argv[i])) { + rateBPS = atoi(argv[i + 1]); + setControlBWE = 1; + if ((rateBPS < 10000) || (rateBPS > 32000)) { + printf("\n%d is not a initial rate. " + "Valid values are in the range 10000 to 32000.\n", rateBPS); + exit(1); + } + printf("\nNew initial rate: %d\n", rateBPS); + i++; + } + + /* Set (initial) framelength */ + if (!strcmp ("-FL", argv[i])) { + framesize = atoi(argv[i + 1]); + if ((framesize != 30) && (framesize != 60)) { + printf("\n%d is not a valid frame length. " + "Valid length are 30 and 60 msec.\n", framesize); + exit(1); + } + printf("\nFrame Length: %d\n", framesize); + i++; + } + + /* Fixed frame length */ + if (!strcmp ("-FIXED_FL", argv[i])) { + fixedFL = 1; + setControlBWE = 1; + } + + /* Set maximum allowed payload size in bytes */ + if (!strcmp ("-MAX", argv[i])) { + payloadSize = atoi(argv[i + 1]); + printf("Maximum Payload Size: %d\n", payloadSize); + i++; + } + + /* Set maximum rate in bytes */ + if (!strcmp ("-MAXRATE", argv[i])) { + payloadRate = atoi(argv[i + 1]); + printf("Maximum Rate in kbps: %d\n", payloadRate); + i++; + } + + /* Test of fault scenarious */ + if (!strcmp ("-F", argv[i])) { + testNum = atoi(argv[i + 1]); + printf("\nFault test: %d\n", testNum); + if (testNum < 1 || testNum > 10) { + printf("\n%d is not a valid Fault Scenario number." + " Valid Fault Scenarios are numbered 1-10.\n", testNum); + exit(1); + } + i++; + } + + /* Packet loss test */ + if (!strcmp ("-PL", argv[i])) { + if( isdigit( *argv[i+1] ) ) { + packetLossPercent = atoi( argv[i+1] ); + if( (packetLossPercent < 0) | (packetLossPercent > 100) ) { + printf( "\nInvalid packet loss perentage \n" ); + exit( 1 ); + } + if( packetLossPercent > 0 ) { + printf( "\nSimulating %d %% of independent packet loss\n", + packetLossPercent ); + } else { + printf( "\nNo Packet Loss Is Simulated \n" ); + } + readLoss = 0; + } else { + readLoss = 1; + plFile = fopen( argv[i+1], "rb" ); + if( plFile == NULL ) { + FAIL() << "Couldn't open the frameloss file: " << argv[i+1]; + } + printf( "\nSimulating packet loss through the given " + "channel file: %s\n", argv[i+1] ); + } + i++; + } + + /* Random packetlosses */ + if (!strcmp ("-rnd", argv[i])) { + srand(time(NULL) ); + printf( "\n Random pattern in lossed packets \n" ); + } + + /* Use gns file */ + if (!strcmp ("-G", argv[i])) { + sscanf(argv[i + 1], "%s", gns_file); + fp_gns = fopen(gns_file, "rb"); + if (fp_gns == NULL) { + FAIL() << "Cannot read file " << gns_file << "."; + } + gns = 1; + i++; + } + + /* Run Narrowband interfaces (either encoder or decoder) */ + if (!strcmp ("-NB", argv[i])) { + nbTest = atoi(argv[i + 1]); + i++; + } + + /* Run Conference Engine APIs */ + if (!strcmp ("-CE", argv[i])) { + testCE = atoi(argv[i + 1]); + if (testCE==1 || testCE==2) { + i++; + scale = (float)atof( argv[i+1] ); + } else if (testCE < 1 || testCE > 3) { + printf("\n%d is not a valid CE-test number, valid Fault " + "Scenarios are numbered 1-3\n", testCE); + exit(1); + } + i++; + } + + /* Set initial RTP number */ + if (!strcmp ("-RTP_INIT", argv[i])) { + i++; + } + } + + /* Get Bottleneck value */ + /* Gns files and bottleneck should not and can not be used simultaneously */ + bottleneck = atoi(argv[CodingMode+1]); + if (bottleneck == 0 && gns == 0) { + sscanf(argv[CodingMode+1], "%s", bottleneck_file); + f_bn = fopen(bottleneck_file, "rb"); + if (f_bn == NULL) { + printf("No value provided for BottleNeck\n"); + FAIL() << "Cannot read file " << bottleneck_file; + } else { + int aux_var; + printf("reading bottleneck rates from file %s\n\n",bottleneck_file); + if (fscanf(f_bn, "%d", &aux_var) == EOF) { + /* Set pointer to beginning of file */ + fseek(f_bn, 0L, SEEK_SET); + if (fscanf(f_bn, "%d", &aux_var) == EOF) { + FAIL(); + } + } + bottleneck = (int16_t)aux_var; + /* Bottleneck is a cosine function + * Matlab code for writing the bottleneck file: + * BottleNeck_10ms = 20e3 + 10e3 * cos((0:5999)/5999*2*pi); + * fid = fopen('bottleneck.txt', 'wb'); + * fprintf(fid, '%d\n', BottleNeck_10ms); fclose(fid); + */ + } + } else { + f_bn = NULL; + printf("\nfixed bottleneck rate of %d bits/s\n\n", bottleneck); + } + + if (CodingMode == 0) { + printf("\nAdaptive BottleNeck\n"); + } + + /* Get Input and Output files */ + sscanf(argv[argc-2], "%s", inname); + sscanf(argv[argc-1], "%s", outname); + + /* Add '.bit' to output bitstream file */ + while ((int)outname[h] != 0) { + outbitsname[h] = outname[h]; + h++; + } + for (k=0; k<5; k++) { + outbitsname[h] = tmpBit[k]; + h++; + } + if ((inp = fopen(inname,"rb")) == NULL) { + FAIL() << " iSAC: Cannot read file " << inname; + } + if ((outp = fopen(outname,"wb")) == NULL) { + FAIL() << " iSAC: Cannot write file " << outname; + } + + if ((outbits = fopen(outbitsname,"wb")) == NULL) { + FAIL() << " iSAC: Cannot write file " << outbitsname; + } + printf("\nInput:%s\nOutput:%s\n\n", inname, outname); + + /* Error test number 10, garbage data */ + if (testNum == 10) { + /* Test to run decoder with garbage data */ + srand(random_seed); + + if ( (seedfile = fopen(SEED_FILE, "a+t") ) == NULL ) { + printf("Error: Could not open file %s\n", SEED_FILE); + } + else { + fprintf(seedfile, "%u\n", random_seed); + fclose(seedfile); + } + } + + /* Runtime statistics */ + starttime = clock()/(double)CLOCKS_PER_SEC; + + /* Initialize the ISAC and BN structs */ + if (testNum != 8) + { + if(1){ + err =WebRtcIsacfix_Create(&ISAC_main_inst); + }else{ + /* Test the Assign functions */ + int sss; + void *ppp; + err =WebRtcIsacfix_AssignSize(&sss); + ppp=malloc(sss); + err =WebRtcIsacfix_Assign(&ISAC_main_inst,ppp); + } + /* Error check */ + if (err < 0) { + printf("\n\n Error in create.\n\n"); + } + if (testCE == 1) { + err = WebRtcIsacfix_CreateInternal(ISAC_main_inst); + /* Error check */ + if (err < 0) { + printf("\n\n Error in createInternal.\n\n"); + } + } + } + + /* Init of bandwidth data */ + BN_data.send_time = 0; + BN_data.arrival_time = 0; + BN_data.sample_count = 0; + BN_data.rtp_number = 0; + + /* Initialize encoder and decoder */ + framecnt= 0; + endfile = 0; + if (testNum != 1) { + WebRtcIsacfix_EncoderInit(ISAC_main_inst, CodingMode); + } + if (testNum != 2) { + WebRtcIsacfix_DecoderInit(ISAC_main_inst); + } + + if (CodingMode == 1) { + err = WebRtcIsacfix_Control(ISAC_main_inst, bottleneck, framesize); + if (err < 0) { + /* exit if returned with error */ + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf("\n\n Error in control: %d.\n\n", errtype); + } + } else if(setControlBWE == 1) { + err = WebRtcIsacfix_ControlBwe(ISAC_main_inst, rateBPS, framesize, fixedFL); + } + + if (payloadSize != 0) { + err = WebRtcIsacfix_SetMaxPayloadSize(ISAC_main_inst, payloadSize); + if (err < 0) { + /* exit if returned with error */ + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + FAIL() << "Error in SetMaxPayloadSize: " << errtype; + } + } + if (payloadRate != 0) { + err = WebRtcIsacfix_SetMaxRate(ISAC_main_inst, payloadRate); + if (err < 0) { + /* exit if returned with error */ + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + FAIL() << "Error in SetMaxRateInBytes: " << errtype; + } + } + + *speechType = 1; + + + while (endfile == 0) { + + if(testNum == 7 && (rand()%2 == 0)) { + err = WebRtcIsacfix_EncoderInit(ISAC_main_inst, CodingMode); + /* Error check */ + if (err < 0) { + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf("\n\n Error in encoderinit: %d.\n\n", errtype); + } + + WebRtcIsacfix_DecoderInit(ISAC_main_inst); + } + + + cur_framesmpls = 0; + while (1) { + /* Read 10 ms speech block */ + if (nbTest != 1) { + endfile = readframe(shortdata, inp, FRAMESAMPLES_10ms); + } else { + endfile = readframe(shortdata, inp, (FRAMESAMPLES_10ms/2)); + } + + if (testNum == 7) { + srand(time(NULL)); + } + + /* iSAC encoding */ + if (!(testNum == 3 && framecnt == 0)) { + if (nbTest != 1) { + short bwe; + + /* Encode */ + stream_len_int = WebRtcIsacfix_Encode(ISAC_main_inst, + shortdata, + (uint8_t*)streamdata); + + /* If packet is ready, and CE testing, call the different API + functions from the internal API. */ + if (stream_len_int>0) { + if (testCE == 1) { + err = WebRtcIsacfix_ReadBwIndex( + reinterpret_cast(streamdata), + static_cast(stream_len_int), + &bwe); + stream_len_int = WebRtcIsacfix_GetNewBitStream( + ISAC_main_inst, + bwe, + scale, + reinterpret_cast(streamdata)); + } else if (testCE == 2) { + /* transcode function not supported */ + } else if (testCE == 3) { + /* Only for Function testing. The functions should normally + not be used in this way */ + + err = WebRtcIsacfix_GetDownLinkBwIndex(ISAC_main_inst, &bwe); + /* Error Check */ + if (err < 0) { + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf("\nError in getSendBWE: %d.\n", errtype); + } + + err = WebRtcIsacfix_UpdateUplinkBw(ISAC_main_inst, bwe); + /* Error Check */ + if (err < 0) { + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf("\nError in setBWE: %d.\n", errtype); + } + + } + } + } else { +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + stream_len_int = WebRtcIsacfix_EncodeNb(ISAC_main_inst, + shortdata, + streamdata); +#else + stream_len_int = -1; +#endif + } + } + else + { + break; + } + + if (stream_len_int < 0 || err < 0) { + /* exit if returned with error */ + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf("\nError in encoder: %d.\n", errtype); + } else { + stream_len = static_cast(stream_len_int); + if (fwrite(streamdata, sizeof(char), stream_len, outbits) != + stream_len) { + FAIL(); + } + } + + cur_framesmpls += FRAMESAMPLES_10ms; + + /* read next bottleneck rate */ + if (f_bn != NULL) { + int aux_var; + if (fscanf(f_bn, "%d", &aux_var) == EOF) { + /* Set pointer to beginning of file */ + fseek(f_bn, 0L, SEEK_SET); + if (fscanf(f_bn, "%d", &aux_var) == EOF) { + FAIL(); + } + } + bottleneck = (int16_t)aux_var; + if (CodingMode == 1) { + WebRtcIsacfix_Control(ISAC_main_inst, bottleneck, framesize); + } + } + + /* exit encoder loop if the encoder returned a bitstream */ + if (stream_len != 0) break; + } + + /* make coded sequence to short be inreasing */ + /* the length the decoder expects */ + if (testNum == 4) { + stream_len += 10; + } + + /* make coded sequence to long be decreasing */ + /* the length the decoder expects */ + if (testNum == 5) { + stream_len -= 10; + } + + if (testNum == 6) { + srand(time(NULL)); + for (i = 0; i < stream_len; i++ ) { + streamdata[i] = rand(); + } + } + + /* set pointer to beginning of file */ + if (fp_gns != NULL) { + if (fscanf(fp_gns, "%d", &cur_delay) == EOF) { + fseek(fp_gns, 0L, SEEK_SET); + if (fscanf(fp_gns, "%d", &cur_delay) == EOF) { + FAIL(); + } + } + } + + /* simulate packet handling through NetEq and the modem */ + if (!(testNum == 3 && framecnt == 0)) { + if (gns == 0) { + get_arrival_time(cur_framesmpls, stream_len, bottleneck, + &BN_data); + } else { + get_arrival_time2(cur_framesmpls, cur_delay, &BN_data); + } + } + + /* packet not dropped */ + if (cur_delay != -1) { + + /* Error test number 10, garbage data */ + if (testNum == 10) { + for ( i = 0; i < stream_len; i++) { + streamdata[i] = (short) (streamdata[i] + (short) rand()); + } + } + + if (testNum != 9) { + err = WebRtcIsacfix_UpdateBwEstimate( + ISAC_main_inst, + reinterpret_cast(streamdata), + stream_len, + BN_data.rtp_number, + BN_data.send_time, + BN_data.arrival_time); + + if (err < 0) { + /* exit if returned with error */ + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf("\nError in decoder: %d.\n", errtype); + } + } + + if( readLoss == 1 ) { + if( fread( &lostFrame, sizeof(int16_t), 1, plFile ) != 1 ) { + rewind( plFile ); + } + lostFrame = !lostFrame; + } else { + lostFrame = (rand()%100 < packetLossPercent); + } + + + + /* iSAC decoding */ + if( lostFrame && framecnt > 0) { + if (nbTest !=2) { + declen = static_cast( + WebRtcIsacfix_DecodePlc(ISAC_main_inst, decoded, prevFrameSize)); + } else { +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + declen = static_cast(WebRtcIsacfix_DecodePlcNb( + ISAC_main_inst, decoded, prevFrameSize)); +#else + declen = -1; +#endif + } + lostPackets++; + } else { + if (nbTest !=2 ) { + size_t FL; + /* Call getFramelen, only used here for function test */ + err = WebRtcIsacfix_ReadFrameLen( + reinterpret_cast(streamdata), stream_len, &FL); + declen = WebRtcIsacfix_Decode( + ISAC_main_inst, + reinterpret_cast(streamdata), + stream_len, + decoded, + speechType); + /* Error check */ + if (err < 0 || declen < 0 || FL != static_cast(declen)) { + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf( + "\nError %d in ReadFrameLen (%s), Decode (%s), with FL %zu and " + "declen %d.\n", + errtype, err < 0 ? "yes" : "no", declen < 0 ? "yes" : "no", FL, + declen); + } + prevFrameSize = static_cast(declen/480); + + } else { +#ifdef WEBRTC_ISAC_FIX_NB_CALLS_ENABLED + declen = WebRtcIsacfix_DecodeNb( ISAC_main_inst, streamdata, + stream_len, decoded, speechType ); +#else + declen = -1; +#endif + prevFrameSize = static_cast(declen / 240); + } + } + + if (declen <= 0) { + /* exit if returned with error */ + errtype=WebRtcIsacfix_GetErrorCode(ISAC_main_inst); + printf("\nError in decoder: %d.\n", errtype); + } + + /* Write decoded speech frame to file */ + if (fwrite(decoded, sizeof(int16_t), + declen, outp) != (size_t)declen) { + FAIL(); + } + // fprintf( ratefile, "%f \n", stream_len / ( ((double)declen)/ + // ((double)FS) ) * 8 ); + } else { + lostPackets++; + } + framecnt++; + + totalsmpls += declen; + totalbits += static_cast(8 * stream_len); + + /* Error test number 10, garbage data */ + if (testNum == 10) { + if ( (seedfile = fopen(SEED_FILE, "a+t") ) == NULL ) { + printf( "Error: Could not open file %s\n", SEED_FILE); + } + else { + fprintf(seedfile, "ok\n\n"); + fclose(seedfile); + } + } + } + printf("\nLost Frames %d ~ %4.1f%%\n", lostPackets, + (double)lostPackets/(double)framecnt*100.0 ); + printf("\n\ntotal bits = %d bits", totalbits); + printf("\nmeasured average bitrate = %0.3f kbits/s", + (double)totalbits *(FS/1000) / totalsmpls); + printf("\n"); + + /* Runtime statistics */ + + + runtime = (double)(((double)clock()/(double)CLOCKS_PER_SEC)-starttime); + length_file = ((double)framecnt*(double)declen/FS); + printf("\n\nLength of speech file: %.1f s\n", length_file); + printf("Time to run iSAC: %.2f s (%.2f %% of realtime)\n\n", + runtime, (100*runtime/length_file)); + printf("\n\n_______________________________________________\n"); + + // Record the results with Perf test tools. + webrtc::test::PrintResult("isac", "", "time_per_10ms_frame", + (runtime * 10000) / length_file, "us", false); + + fclose(inp); + fclose(outp); + fclose(outbits); + + if ( testCE == 1) { + WebRtcIsacfix_FreeInternal(ISAC_main_inst); + } + WebRtcIsacfix_Free(ISAC_main_inst); +} + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + global_argc = argc; + global_argv = argv; + + return RUN_ALL_TESTS(); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.cc new file mode 100644 index 0000000000..80d10abf7f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.cc @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/locked_bandwidth_info.h" + +namespace webrtc { + +LockedIsacBandwidthInfo::LockedIsacBandwidthInfo() : ref_count_(0) { + bwinfo_.in_use = 0; +} + +LockedIsacBandwidthInfo::~LockedIsacBandwidthInfo() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.h new file mode 100644 index 0000000000..37074f8f5b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_LOCKED_BANDWIDTH_INFO_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_LOCKED_BANDWIDTH_INFO_H_ + +#include "modules/audio_coding/codecs/isac/bandwidth_info.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +// An IsacBandwidthInfo that's safe to access from multiple threads because +// it's protected by a mutex. +class LockedIsacBandwidthInfo final { + public: + LockedIsacBandwidthInfo(); + ~LockedIsacBandwidthInfo(); + + IsacBandwidthInfo Get() const { + rtc::CritScope lock(&lock_); + return bwinfo_; + } + + void Set(const IsacBandwidthInfo& bwinfo) { + rtc::CritScope lock(&lock_); + bwinfo_ = bwinfo; + } + + int AddRef() const { return rtc::AtomicOps::Increment(&ref_count_); } + + int Release() const { + const int count = rtc::AtomicOps::Decrement(&ref_count_); + if (count == 0) { + delete this; + } + return count; + } + + private: + mutable volatile int ref_count_; + rtc::CriticalSection lock_; + IsacBandwidthInfo bwinfo_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_LOCKED_BANDWIDTH_INFO_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h new file mode 100644 index 0000000000..fae2f3d4a7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_AUDIO_DECODER_ISAC_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_AUDIO_DECODER_ISAC_H_ + +#include "modules/audio_coding/codecs/isac/audio_decoder_isac_t.h" +#include "modules/audio_coding/codecs/isac/main/source/isac_float_type.h" + +namespace webrtc { + +using AudioDecoderIsacFloatImpl = AudioDecoderIsacT; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_AUDIO_ENCODER_ISAC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h new file mode 100644 index 0000000000..dc32bcdde6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_AUDIO_ENCODER_ISAC_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_AUDIO_ENCODER_ISAC_H_ + +#include "modules/audio_coding/codecs/isac/audio_encoder_isac_t.h" +#include "modules/audio_coding/codecs/isac/main/source/isac_float_type.h" + +namespace webrtc { + +using AudioEncoderIsacFloatImpl = AudioEncoderIsacT; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_AUDIO_ENCODER_ISAC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/isac.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/isac.h new file mode 100644 index 0000000000..e1ee818059 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/isac.h @@ -0,0 +1,724 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_ISAC_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_ISAC_H_ + +#include + +#include "modules/audio_coding/codecs/isac/bandwidth_info.h" +#include "typedefs.h" // NOLINT(build/include) + +typedef struct WebRtcISACStruct ISACStruct; + +#if defined(__cplusplus) +extern "C" { +#endif + + /****************************************************************************** + * WebRtcIsac_AssignSize(...) + * + * This function returns the size of the ISAC instance, so that the instance + * can be created outside iSAC. + * + * Input: + * - samplingRate : sampling rate of the input/output audio. + * + * Output: + * - sizeinbytes : number of bytes needed to allocate for the + * instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsac_AssignSize( + int* sizeinbytes); + + + /****************************************************************************** + * WebRtcIsac_Assign(...) + * + * This function assignes the memory already created to the ISAC instance. + * + * Input: + * - *ISAC_main_inst : a pointer to the coder instance. + * - samplingRate : sampling rate of the input/output audio. + * - ISAC_inst_Addr : the already allocated memory, where we put the + * iSAC structure. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsac_Assign( + ISACStruct** ISAC_main_inst, + void* ISAC_inst_Addr); + + + /****************************************************************************** + * WebRtcIsac_Create(...) + * + * This function creates an ISAC instance, which will contain the state + * information for one coding/decoding channel. + * + * Input: + * - *ISAC_main_inst : a pointer to the coder instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsac_Create( + ISACStruct** ISAC_main_inst); + + + /****************************************************************************** + * WebRtcIsac_Free(...) + * + * This function frees the ISAC instance created at the beginning. + * + * Input: + * - ISAC_main_inst : an ISAC instance. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsac_Free( + ISACStruct* ISAC_main_inst); + + + /****************************************************************************** + * WebRtcIsac_EncoderInit(...) + * + * This function initializes an ISAC instance prior to the encoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - CodingMode : 0 -> Bit rate and frame length are + * automatically adjusted to available bandwidth + * on transmission channel, just valid if codec + * is created to work in wideband mode. + * 1 -> User sets a frame length and a target bit + * rate which is taken as the maximum + * short-term average bit rate. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsac_EncoderInit( + ISACStruct* ISAC_main_inst, + int16_t CodingMode); + + + /****************************************************************************** + * WebRtcIsac_Encode(...) + * + * This function encodes 10ms audio blocks and inserts it into a package. + * Input speech length has 160 samples if operating at 16 kHz sampling + * rate, or 320 if operating at 32 kHz sampling rate. The encoder buffers the + * input audio until the whole frame is buffered then proceeds with encoding. + * + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - speechIn : input speech vector. + * + * Output: + * - encoded : the encoded data vector + * + * Return value: + * : >0 - Length (in bytes) of coded data + * : 0 - The buffer didn't reach the chosen + * frame-size so it keeps buffering speech + * samples. + * : -1 - Error + */ + + int WebRtcIsac_Encode( + ISACStruct* ISAC_main_inst, + const int16_t* speechIn, + uint8_t* encoded); + + + /****************************************************************************** + * WebRtcIsac_DecoderInit(...) + * + * This function initializes an ISAC instance prior to the decoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + */ + + void WebRtcIsac_DecoderInit(ISACStruct* ISAC_main_inst); + + /****************************************************************************** + * WebRtcIsac_UpdateBwEstimate(...) + * + * This function updates the estimate of the bandwidth. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s). + * - packet_size : size of the packet. + * - rtp_seq_number : the RTP number of the packet. + * - send_ts : the RTP send timestamp, given in samples + * - arr_ts : the arrival time of the packet (from NetEq) + * in samples. + * + * Return value : 0 - Ok + * -1 - Error + */ + + int16_t WebRtcIsac_UpdateBwEstimate( + ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts); + + + /****************************************************************************** + * WebRtcIsac_Decode(...) + * + * This function decodes an ISAC frame. At 16 kHz sampling rate, the length + * of the output audio could be either 480 or 960 samples, equivalent to + * 30 or 60 ms respectively. At 32 kHz sampling rate, the length of the + * output audio is 960 samples, which is 30 ms. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s). + * - len : bytes in encoded vector. + * + * Output: + * - decoded : The decoded vector. + * + * Return value : >0 - number of samples in decoded vector. + * -1 - Error. + */ + + int WebRtcIsac_Decode( + ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + + + /****************************************************************************** + * WebRtcIsac_DecodePlc(...) + * + * This function conducts PLC for ISAC frame(s). Output speech length + * will be a multiple of frames, i.e. multiples of 30 ms audio. Therefore, + * the output is multiple of 480 samples if operating at 16 kHz and multiple + * of 960 if operating at 32 kHz. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - noOfLostFrames : Number of PLC frames to produce. + * + * Output: + * - decoded : The decoded vector. + * + * Return value : Number of samples in decoded PLC vector + */ + + size_t WebRtcIsac_DecodePlc( + ISACStruct* ISAC_main_inst, + int16_t* decoded, + size_t noOfLostFrames); + + + /****************************************************************************** + * WebRtcIsac_Control(...) + * + * This function sets the limit on the short-term average bit-rate and the + * frame length. Should be used only in Instantaneous mode. At 16 kHz sampling + * rate, an average bit-rate between 10000 to 32000 bps is valid and a + * frame-size of 30 or 60 ms is acceptable. At 32 kHz, an average bit-rate + * between 10000 to 56000 is acceptable, and the valid frame-size is 30 ms. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - rate : limit on the short-term average bit rate, + * in bits/second. + * - framesize : frame-size in millisecond. + * + * Return value : 0 - ok + * -1 - Error + */ + + int16_t WebRtcIsac_Control( + ISACStruct* ISAC_main_inst, + int32_t rate, + int framesize); + + void WebRtcIsac_SetInitialBweBottleneck(ISACStruct* ISAC_main_inst, + int bottleneck_bits_per_second); + + /****************************************************************************** + * WebRtcIsac_ControlBwe(...) + * + * This function sets the initial values of bottleneck and frame-size if + * iSAC is used in channel-adaptive mode. Therefore, this API is not + * applicable if the codec is created to operate in super-wideband mode. + * + * Through this API, users can enforce a frame-size for all values of + * bottleneck. Then iSAC will not automatically change the frame-size. + * + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - rateBPS : initial value of bottleneck in bits/second + * 10000 <= rateBPS <= 56000 is accepted + * For default bottleneck set rateBPS = 0 + * - frameSizeMs : number of milliseconds per frame (30 or 60) + * - enforceFrameSize : 1 to enforce the given frame-size through + * out the adaptation process, 0 to let iSAC + * change the frame-size if required. + * + * Return value : 0 - ok + * -1 - Error + */ + + int16_t WebRtcIsac_ControlBwe( + ISACStruct* ISAC_main_inst, + int32_t rateBPS, + int frameSizeMs, + int16_t enforceFrameSize); + + + /****************************************************************************** + * WebRtcIsac_ReadFrameLen(...) + * + * This function returns the length of the frame represented in the packet. + * + * Input: + * - encoded : Encoded bit-stream + * + * Output: + * - frameLength : Length of frame in packet (in samples) + * + */ + + int16_t WebRtcIsac_ReadFrameLen( + ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + int16_t* frameLength); + + + /****************************************************************************** + * WebRtcIsac_version(...) + * + * This function returns the version number. + * + * Output: + * - version : Pointer to character string + * + */ + + void WebRtcIsac_version( + char *version); + + + /****************************************************************************** + * WebRtcIsac_GetErrorCode(...) + * + * This function can be used to check the error code of an iSAC instance. When + * a function returns -1 a error code will be set for that instance. The + * function below extract the code of the last error that occurred in the + * specified instance. + * + * Input: + * - ISAC_main_inst : ISAC instance + * + * Return value : Error code + */ + + int16_t WebRtcIsac_GetErrorCode( + ISACStruct* ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsac_GetUplinkBw(...) + * + * This function outputs the target bottleneck of the codec. In + * channel-adaptive mode, the target bottleneck is specified through in-band + * signalling retreived by bandwidth estimator. + * In channel-independent, also called instantaneous mode, the target + * bottleneck is provided to the encoder by calling xxx_control(...). If + * xxx_control is never called the default values is returned. The default + * value for bottleneck at 16 kHz encoder sampling rate is 32000 bits/sec, + * and it is 56000 bits/sec for 32 kHz sampling rate. + * Note that the output is the iSAC internal operating bottleneck which might + * differ slightly from the one provided through xxx_control(). + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Output: + * - *bottleneck : bottleneck in bits/sec + * + * Return value : -1 if error happens + * 0 bit-rates computed correctly. + */ + + int16_t WebRtcIsac_GetUplinkBw( + ISACStruct* ISAC_main_inst, + int32_t* bottleneck); + + + /****************************************************************************** + * WebRtcIsac_SetMaxPayloadSize(...) + * + * This function sets a limit for the maximum payload size of iSAC. The same + * value is used both for 30 and 60 ms packets. If the encoder sampling rate + * is 16 kHz the maximum payload size is between 120 and 400 bytes. If the + * encoder sampling rate is 32 kHz the maximum payload size is between 120 + * and 600 bytes. + * + * If an out of range limit is used, the function returns -1, but the closest + * valid value will be applied. + * + * --------------- + * IMPORTANT NOTES + * --------------- + * The size of a packet is limited to the minimum of 'max-payload-size' and + * 'max-rate.' For instance, let's assume the max-payload-size is set to + * 170 bytes, and max-rate is set to 40 kbps. Note that a limit of 40 kbps + * translates to 150 bytes for 30ms frame-size & 300 bytes for 60ms + * frame-size. Then a packet with a frame-size of 30 ms is limited to 150, + * i.e. min(170, 150), and a packet with 60 ms frame-size is limited to + * 170 bytes, i.e. min(170, 300). + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxPayloadBytes : maximum size of the payload in bytes + * valid values are between 120 and 400 bytes + * if encoder sampling rate is 16 kHz. For + * 32 kHz encoder sampling rate valid values + * are between 120 and 600 bytes. + * + * Return value : 0 if successful + * -1 if error happens + */ + + int16_t WebRtcIsac_SetMaxPayloadSize( + ISACStruct* ISAC_main_inst, + int16_t maxPayloadBytes); + + + /****************************************************************************** + * WebRtcIsac_SetMaxRate(...) + * + * This function sets the maximum rate which the codec may not exceed for + * any signal packet. The maximum rate is defined and payload-size per + * frame-size in bits per second. + * + * The codec has a maximum rate of 53400 bits per second (200 bytes per 30 + * ms) if the encoder sampling rate is 16kHz, and 160 kbps (600 bytes/30 ms) + * if the encoder sampling rate is 32 kHz. + * + * It is possible to set a maximum rate between 32000 and 53400 bits/sec + * in wideband mode, and 32000 to 160000 bits/sec in super-wideband mode. + * + * If an out of range limit is used, the function returns -1, but the closest + * valid value will be applied. + * + * --------------- + * IMPORTANT NOTES + * --------------- + * The size of a packet is limited to the minimum of 'max-payload-size' and + * 'max-rate.' For instance, let's assume the max-payload-size is set to + * 170 bytes, and max-rate is set to 40 kbps. Note that a limit of 40 kbps + * translates to 150 bytes for 30ms frame-size & 300 bytes for 60ms + * frame-size. Then a packet with a frame-size of 30 ms is limited to 150, + * i.e. min(170, 150), and a packet with 60 ms frame-size is limited to + * 170 bytes, min(170, 300). + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxRate : maximum rate in bits per second, + * valid values are 32000 to 53400 bits/sec in + * wideband mode, and 32000 to 160000 bits/sec in + * super-wideband mode. + * + * Return value : 0 if successful + * -1 if error happens + */ + + int16_t WebRtcIsac_SetMaxRate( + ISACStruct* ISAC_main_inst, + int32_t maxRate); + + + /****************************************************************************** + * WebRtcIsac_DecSampRate() + * Return the sampling rate of the decoded audio. + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Return value : sampling frequency in Hertz. + * + */ + + uint16_t WebRtcIsac_DecSampRate(ISACStruct* ISAC_main_inst); + + + /****************************************************************************** + * WebRtcIsac_EncSampRate() + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Return value : sampling rate in Hertz. + * + */ + + uint16_t WebRtcIsac_EncSampRate(ISACStruct* ISAC_main_inst); + + + /****************************************************************************** + * WebRtcIsac_SetDecSampRate() + * Set the sampling rate of the decoder. Initialization of the decoder WILL + * NOT overwrite the sampling rate of the encoder. The default value is 16 kHz + * which is set when the instance is created. + * + * Input: + * - ISAC_main_inst : iSAC instance + * - sampRate : sampling rate in Hertz. + * + * Return value : 0 if successful + * -1 if failed. + */ + + int16_t WebRtcIsac_SetDecSampRate(ISACStruct* ISAC_main_inst, + uint16_t samp_rate_hz); + + + /****************************************************************************** + * WebRtcIsac_SetEncSampRate() + * Set the sampling rate of the encoder. Initialization of the encoder WILL + * NOT overwrite the sampling rate of the encoder. The default value is 16 kHz + * which is set when the instance is created. The encoding-mode and the + * bottleneck remain unchanged by this call, however, the maximum rate and + * maximum payload-size will reset to their default value. + * + * Input: + * - ISAC_main_inst : iSAC instance + * - sampRate : sampling rate in Hertz. + * + * Return value : 0 if successful + * -1 if failed. + */ + + int16_t WebRtcIsac_SetEncSampRate(ISACStruct* ISAC_main_inst, + uint16_t sample_rate_hz); + + + + /****************************************************************************** + * WebRtcIsac_GetNewBitStream(...) + * + * This function returns encoded data, with the recieved bwe-index in the + * stream. If the rate is set to a value less than bottleneck of codec + * the new bistream will be re-encoded with the given target rate. + * It should always return a complete packet, i.e. only called once + * even for 60 msec frames. + * + * NOTE 1! This function does not write in the ISACStruct, it is not allowed. + * NOTE 2! Currently not implemented for SWB mode. + * NOTE 3! Rates larger than the bottleneck of the codec will be limited + * to the current bottleneck. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - bweIndex : Index of bandwidth estimate to put in new + * bitstream + * - rate : target rate of the transcoder is bits/sec. + * Valid values are the accepted rate in iSAC, + * i.e. 10000 to 56000. + * - isRCU : if the new bit-stream is an RCU stream. + * Note that the rate parameter always indicates + * the target rate of the main payload, regardless + * of 'isRCU' value. + * + * Output: + * - encoded : The encoded data vector + * + * Return value : >0 - Length (in bytes) of coded data + * -1 - Error or called in SWB mode + * NOTE! No error code is written to + * the struct since it is only allowed to read + * the struct. + */ + int16_t WebRtcIsac_GetNewBitStream( + ISACStruct* ISAC_main_inst, + int16_t bweIndex, + int16_t jitterInfo, + int32_t rate, + uint8_t* encoded, + int16_t isRCU); + + + + /**************************************************************************** + * WebRtcIsac_GetDownLinkBwIndex(...) + * + * This function returns index representing the Bandwidth estimate from + * other side to this side. + * + * Input: + * - ISAC_main_inst : iSAC struct + * + * Output: + * - bweIndex : Bandwidth estimate to transmit to other side. + * + */ + + int16_t WebRtcIsac_GetDownLinkBwIndex( + ISACStruct* ISAC_main_inst, + int16_t* bweIndex, + int16_t* jitterInfo); + + + /**************************************************************************** + * WebRtcIsac_UpdateUplinkBw(...) + * + * This function takes an index representing the Bandwidth estimate from + * this side to other side and updates BWE. + * + * Input: + * - ISAC_main_inst : iSAC struct + * - bweIndex : Bandwidth estimate from other side. + * + */ + + int16_t WebRtcIsac_UpdateUplinkBw( + ISACStruct* ISAC_main_inst, + int16_t bweIndex); + + + /**************************************************************************** + * WebRtcIsac_ReadBwIndex(...) + * + * This function returns the index of the Bandwidth estimate from the bitstream. + * + * Input: + * - encoded : Encoded bitstream + * + * Output: + * - frameLength : Length of frame in packet (in samples) + * - bweIndex : Bandwidth estimate in bitstream + * + */ + + int16_t WebRtcIsac_ReadBwIndex( + const uint8_t* encoded, + int16_t* bweIndex); + + + + /******************************************************************************* + * WebRtcIsac_GetNewFrameLen(...) + * + * returns the frame lenght (in samples) of the next packet. In the case of channel-adaptive + * mode, iSAC decides on its frame lenght based on the estimated bottleneck + * this allows a user to prepare for the next packet (at the encoder) + * + * The primary usage is in CE to make the iSAC works in channel-adaptive mode + * + * Input: + * - ISAC_main_inst : iSAC struct + * + * Return Value : frame lenght in samples + * + */ + + int16_t WebRtcIsac_GetNewFrameLen( + ISACStruct* ISAC_main_inst); + + + /**************************************************************************** + * WebRtcIsac_GetRedPayload(...) + * + * Populates "encoded" with the redundant payload of the recently encoded + * frame. This function has to be called once that WebRtcIsac_Encode(...) + * returns a positive value. Regardless of the frame-size this function will + * be called only once after encoding is completed. + * + * Input: + * - ISAC_main_inst : iSAC struct + * + * Output: + * - encoded : the encoded data vector + * + * + * Return value: + * : >0 - Length (in bytes) of coded data + * : -1 - Error + * + * + */ + int16_t WebRtcIsac_GetRedPayload( + ISACStruct* ISAC_main_inst, + uint8_t* encoded); + + + /**************************************************************************** + * WebRtcIsac_DecodeRcu(...) + * + * This function decodes a redundant (RCU) iSAC frame. Function is called in + * NetEq with a stored RCU payload i case of packet loss. Output speech length + * will be a multiple of 480 samples: 480 or 960 samples, + * depending on the framesize (30 or 60 ms). + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC RCU frame(s) + * - len : bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded vector + * -1 - Error + */ + int WebRtcIsac_DecodeRcu( + ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + + /* Fills in an IsacBandwidthInfo struct. |inst| should be a decoder. */ + void WebRtcIsac_GetBandwidthInfo(ISACStruct* inst, IsacBandwidthInfo* bwinfo); + + /* Uses the values from an IsacBandwidthInfo struct. |inst| should be an + encoder. */ + void WebRtcIsac_SetBandwidthInfo(ISACStruct* inst, + const IsacBandwidthInfo* bwinfo); + + /* If |inst| is a decoder but not an encoder: tell it what sample rate the + encoder is using, for bandwidth estimation purposes. */ + void WebRtcIsac_SetEncSampRateInDecoder(ISACStruct* inst, int sample_rate_hz); + +#if defined(__cplusplus) +} +#endif + + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INCLUDE_ISAC_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.c new file mode 100644 index 0000000000..5c901bba00 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "arith_routines.h" +#include "settings.h" + + +/* + * terminate and return byte stream; + * returns the number of bytes in the stream + */ +int WebRtcIsac_EncTerminate(Bitstr *streamdata) /* in-/output struct containing bitstream */ +{ + uint8_t *stream_ptr; + + + /* point to the right place in the stream buffer */ + stream_ptr = streamdata->stream + streamdata->stream_index; + + /* find minimum length (determined by current interval width) */ + if ( streamdata->W_upper > 0x01FFFFFF ) + { + streamdata->streamval += 0x01000000; + /* add carry to buffer */ + if (streamdata->streamval < 0x01000000) + { + /* propagate carry */ + while ( !(++(*--stream_ptr)) ); + /* put pointer back to the old value */ + stream_ptr = streamdata->stream + streamdata->stream_index; + } + /* write remaining data to bitstream */ + *stream_ptr++ = (uint8_t) (streamdata->streamval >> 24); + } + else + { + streamdata->streamval += 0x00010000; + /* add carry to buffer */ + if (streamdata->streamval < 0x00010000) + { + /* propagate carry */ + while ( !(++(*--stream_ptr)) ); + /* put pointer back to the old value */ + stream_ptr = streamdata->stream + streamdata->stream_index; + } + /* write remaining data to bitstream */ + *stream_ptr++ = (uint8_t) (streamdata->streamval >> 24); + *stream_ptr++ = (uint8_t) ((streamdata->streamval >> 16) & 0x00FF); + } + + /* calculate stream length */ + return (int)(stream_ptr - streamdata->stream); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.h new file mode 100644 index 0000000000..73d0c2306e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * arith_routines.h + * + * Functions for arithmetic coding. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ARITH_ROUTINES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ARITH_ROUTINES_H_ + +#include "structs.h" + + +int WebRtcIsac_EncLogisticMulti2( + Bitstr *streamdata, /* in-/output struct containing bitstream */ + int16_t *dataQ7, /* input: data vector */ + const uint16_t *env, /* input: side info vector defining the width of the pdf */ + const int N, /* input: data vector length */ + const int16_t isSWB12kHz); /* if the codec is working in 12kHz bandwidth */ + +/* returns the number of bytes in the stream */ +int WebRtcIsac_EncTerminate(Bitstr *streamdata); /* in-/output struct containing bitstream */ + +/* returns the number of bytes in the stream so far */ +int WebRtcIsac_DecLogisticMulti2( + int16_t *data, /* output: data vector */ + Bitstr *streamdata, /* in-/output struct containing bitstream */ + const uint16_t *env, /* input: side info vector defining the width of the pdf */ + const int16_t *dither, /* input: dither vector */ + const int N, /* input: data vector length */ + const int16_t isSWB12kHz); /* if the codec is working in 12kHz bandwidth */ + +void WebRtcIsac_EncHistMulti( + Bitstr *streamdata, /* in-/output struct containing bitstream */ + const int *data, /* input: data vector */ + const uint16_t *const *cdf, /* input: array of cdf arrays */ + const int N); /* input: data vector length */ + +int WebRtcIsac_DecHistBisectMulti( + int *data, /* output: data vector */ + Bitstr *streamdata, /* in-/output struct containing bitstream */ + const uint16_t *const *cdf, /* input: array of cdf arrays */ + const uint16_t *cdf_size, /* input: array of cdf table sizes+1 (power of two: 2^k) */ + const int N); /* input: data vector length */ + +int WebRtcIsac_DecHistOneStepMulti( + int *data, /* output: data vector */ + Bitstr *streamdata, /* in-/output struct containing bitstream */ + const uint16_t *const *cdf, /* input: array of cdf arrays */ + const uint16_t *init_index,/* input: vector of initial cdf table search entries */ + const int N); /* input: data vector length */ + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ARITH_ROUTINES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_hist.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_hist.c new file mode 100644 index 0000000000..b229c679fc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_hist.c @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "settings.h" +#include "arith_routines.h" + + +/* + * code symbols into arithmetic bytestream + */ +void WebRtcIsac_EncHistMulti(Bitstr *streamdata, /* in-/output struct containing bitstream */ + const int *data, /* input: data vector */ + const uint16_t *const *cdf, /* input: array of cdf arrays */ + const int N) /* input: data vector length */ +{ + uint32_t W_lower, W_upper; + uint32_t W_upper_LSB, W_upper_MSB; + uint8_t *stream_ptr; + uint8_t *stream_ptr_carry; + uint32_t cdf_lo, cdf_hi; + int k; + + + /* point to beginning of stream buffer */ + stream_ptr = streamdata->stream + streamdata->stream_index; + W_upper = streamdata->W_upper; + + for (k=N; k>0; k--) + { + /* fetch cdf_lower and cdf_upper from cdf tables */ + cdf_lo = (uint32_t) *(*cdf + *data); + cdf_hi = (uint32_t) *(*cdf++ + *data++ + 1); + + /* update interval */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = W_upper >> 16; + W_lower = W_upper_MSB * cdf_lo; + W_lower += (W_upper_LSB * cdf_lo) >> 16; + W_upper = W_upper_MSB * cdf_hi; + W_upper += (W_upper_LSB * cdf_hi) >> 16; + + /* shift interval such that it begins at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamdata->streamval += W_lower; + + /* handle carry */ + if (streamdata->streamval < W_lower) + { + /* propagate carry */ + stream_ptr_carry = stream_ptr; + while (!(++(*--stream_ptr_carry))); + } + + /* renormalize interval, store most significant byte of streamval and update streamval */ + while ( !(W_upper & 0xFF000000) ) /* W_upper < 2^24 */ + { + W_upper <<= 8; + *stream_ptr++ = (uint8_t) (streamdata->streamval >> 24); + streamdata->streamval <<= 8; + } + } + + /* calculate new stream_index */ + streamdata->stream_index = (int)(stream_ptr - streamdata->stream); + streamdata->W_upper = W_upper; + + return; +} + + + +/* + * function to decode more symbols from the arithmetic bytestream, using method of bisection + * cdf tables should be of size 2^k-1 (which corresponds to an alphabet size of 2^k-2) + */ +int WebRtcIsac_DecHistBisectMulti(int *data, /* output: data vector */ + Bitstr *streamdata, /* in-/output struct containing bitstream */ + const uint16_t *const *cdf, /* input: array of cdf arrays */ + const uint16_t *cdf_size, /* input: array of cdf table sizes+1 (power of two: 2^k) */ + const int N) /* input: data vector length */ +{ + uint32_t W_lower, W_upper; + uint32_t W_tmp; + uint32_t W_upper_LSB, W_upper_MSB; + uint32_t streamval; + const uint8_t *stream_ptr; + const uint16_t *cdf_ptr; + int size_tmp; + int k; + + W_lower = 0; //to remove warning -DH + stream_ptr = streamdata->stream + streamdata->stream_index; + W_upper = streamdata->W_upper; + if (W_upper == 0) + /* Should not be possible in normal operation */ + return -2; + + if (streamdata->stream_index == 0) /* first time decoder is called for this stream */ + { + /* read first word from bytestream */ + streamval = *stream_ptr << 24; + streamval |= *++stream_ptr << 16; + streamval |= *++stream_ptr << 8; + streamval |= *++stream_ptr; + } else { + streamval = streamdata->streamval; + } + + for (k=N; k>0; k--) + { + /* find the integer *data for which streamval lies in [W_lower+1, W_upper] */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = W_upper >> 16; + + /* start halfway the cdf range */ + size_tmp = *cdf_size++ >> 1; + cdf_ptr = *cdf + (size_tmp - 1); + + /* method of bisection */ + for ( ;; ) + { + W_tmp = W_upper_MSB * *cdf_ptr; + W_tmp += (W_upper_LSB * *cdf_ptr) >> 16; + size_tmp >>= 1; + if (size_tmp == 0) break; + if (streamval > W_tmp) + { + W_lower = W_tmp; + cdf_ptr += size_tmp; + } else { + W_upper = W_tmp; + cdf_ptr -= size_tmp; + } + } + if (streamval > W_tmp) + { + W_lower = W_tmp; + *data++ = (int)(cdf_ptr - *cdf++); + } else { + W_upper = W_tmp; + *data++ = (int)(cdf_ptr - *cdf++ - 1); + } + + /* shift interval to start at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamval -= W_lower; + + /* renormalize interval and update streamval */ + while ( !(W_upper & 0xFF000000) ) /* W_upper < 2^24 */ + { + /* read next byte from stream */ + streamval = (streamval << 8) | *++stream_ptr; + W_upper <<= 8; + } + + if (W_upper == 0) + /* Should not be possible in normal operation */ + return -2; + + + } + + streamdata->stream_index = (int)(stream_ptr - streamdata->stream); + streamdata->W_upper = W_upper; + streamdata->streamval = streamval; + + + /* find number of bytes in original stream (determined by current interval width) */ + if ( W_upper > 0x01FFFFFF ) + return streamdata->stream_index - 2; + else + return streamdata->stream_index - 1; +} + + + +/* + * function to decode more symbols from the arithmetic bytestream, taking single step up or + * down at a time + * cdf tables can be of arbitrary size, but large tables may take a lot of iterations + */ +int WebRtcIsac_DecHistOneStepMulti(int *data, /* output: data vector */ + Bitstr *streamdata, /* in-/output struct containing bitstream */ + const uint16_t *const *cdf, /* input: array of cdf arrays */ + const uint16_t *init_index, /* input: vector of initial cdf table search entries */ + const int N) /* input: data vector length */ +{ + uint32_t W_lower, W_upper; + uint32_t W_tmp; + uint32_t W_upper_LSB, W_upper_MSB; + uint32_t streamval; + const uint8_t *stream_ptr; + const uint16_t *cdf_ptr; + int k; + + + stream_ptr = streamdata->stream + streamdata->stream_index; + W_upper = streamdata->W_upper; + if (W_upper == 0) + /* Should not be possible in normal operation */ + return -2; + + if (streamdata->stream_index == 0) /* first time decoder is called for this stream */ + { + /* read first word from bytestream */ + streamval = (uint32_t)(*stream_ptr) << 24; + streamval |= (uint32_t)(*++stream_ptr) << 16; + streamval |= (uint32_t)(*++stream_ptr) << 8; + streamval |= (uint32_t)(*++stream_ptr); + } else { + streamval = streamdata->streamval; + } + + + for (k=N; k>0; k--) + { + /* find the integer *data for which streamval lies in [W_lower+1, W_upper] */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = W_upper >> 16; + + /* start at the specified table entry */ + cdf_ptr = *cdf + (*init_index++); + W_tmp = W_upper_MSB * *cdf_ptr; + W_tmp += (W_upper_LSB * *cdf_ptr) >> 16; + if (streamval > W_tmp) + { + for ( ;; ) + { + W_lower = W_tmp; + if (cdf_ptr[0]==65535) + /* range check */ + return -3; + W_tmp = W_upper_MSB * *++cdf_ptr; + W_tmp += (W_upper_LSB * *cdf_ptr) >> 16; + if (streamval <= W_tmp) break; + } + W_upper = W_tmp; + *data++ = (int)(cdf_ptr - *cdf++ - 1); + } else { + for ( ;; ) + { + W_upper = W_tmp; + --cdf_ptr; + if (cdf_ptr<*cdf) { + /* range check */ + return -3; + } + W_tmp = W_upper_MSB * *cdf_ptr; + W_tmp += (W_upper_LSB * *cdf_ptr) >> 16; + if (streamval > W_tmp) break; + } + W_lower = W_tmp; + *data++ = (int)(cdf_ptr - *cdf++); + } + + /* shift interval to start at zero */ + W_upper -= ++W_lower; + /* add integer to bitstream */ + streamval -= W_lower; + + /* renormalize interval and update streamval */ + while ( !(W_upper & 0xFF000000) ) /* W_upper < 2^24 */ + { + /* read next byte from stream */ + streamval = (streamval << 8) | *++stream_ptr; + W_upper <<= 8; + } + } + + streamdata->stream_index = (int)(stream_ptr - streamdata->stream); + streamdata->W_upper = W_upper; + streamdata->streamval = streamval; + + + /* find number of bytes in original stream (determined by current interval width) */ + if ( W_upper > 0x01FFFFFF ) + return streamdata->stream_index - 2; + else + return streamdata->stream_index - 1; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_logist.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_logist.c new file mode 100644 index 0000000000..a463d4853a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_logist.c @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * arith_routines.h + * + * This file contains functions for arithmatically encoding and + * decoding DFT coefficients. + * + */ + + +#include "arith_routines.h" + + + +static const int32_t kHistEdgesQ15[51] = { + -327680, -314573, -301466, -288359, -275252, -262144, -249037, -235930, -222823, -209716, + -196608, -183501, -170394, -157287, -144180, -131072, -117965, -104858, -91751, -78644, + -65536, -52429, -39322, -26215, -13108, 0, 13107, 26214, 39321, 52428, + 65536, 78643, 91750, 104857, 117964, 131072, 144179, 157286, 170393, 183500, + 196608, 209715, 222822, 235929, 249036, 262144, 275251, 288358, 301465, 314572, + 327680}; + + +static const int kCdfSlopeQ0[51] = { /* Q0 */ + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 13, 23, 47, 87, 154, 315, 700, 1088, + 2471, 6064, 14221, 21463, 36634, 36924, 19750, 13270, 5806, 2312, + 1095, 660, 316, 145, 86, 41, 32, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 2, 0}; + + +static const int kCdfQ16[51] = { /* Q16 */ + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, + 20, 22, 24, 29, 38, 57, 92, 153, 279, 559, + 994, 1983, 4408, 10097, 18682, 33336, 48105, 56005, 61313, 63636, + 64560, 64998, 65262, 65389, 65447, 65481, 65497, 65510, 65512, 65514, + 65516, 65518, 65520, 65522, 65524, 65526, 65528, 65530, 65532, 65534, + 65535}; + + + +/* function to be converted to fixed point */ +static __inline uint32_t piecewise(int32_t xinQ15) { + + int32_t ind, qtmp1, qtmp2, qtmp3; + uint32_t tmpUW32; + + + qtmp2 = xinQ15; + + if (qtmp2 < kHistEdgesQ15[0]) { + qtmp2 = kHistEdgesQ15[0]; + } + if (qtmp2 > kHistEdgesQ15[50]) { + qtmp2 = kHistEdgesQ15[50]; + } + + qtmp1 = qtmp2 - kHistEdgesQ15[0]; /* Q15 - Q15 = Q15 */ + ind = (qtmp1 * 5) >> 16; /* 2^16 / 5 = 0.4 in Q15 */ + /* Q15 -> Q0 */ + qtmp1 = qtmp2 - kHistEdgesQ15[ind]; /* Q15 - Q15 = Q15 */ + qtmp2 = kCdfSlopeQ0[ind] * qtmp1; /* Q0 * Q15 = Q15 */ + qtmp3 = qtmp2>>15; /* Q15 -> Q0 */ + + tmpUW32 = kCdfQ16[ind] + qtmp3; /* Q0 + Q0 = Q0 */ + return tmpUW32; +} + + + +int WebRtcIsac_EncLogisticMulti2( + Bitstr *streamdata, /* in-/output struct containing bitstream */ + int16_t *dataQ7, /* input: data vector */ + const uint16_t *envQ8, /* input: side info vector defining the width of the pdf */ + const int N, /* input: data vector length / 2 */ + const int16_t isSWB12kHz) +{ + uint32_t W_lower, W_upper; + uint32_t W_upper_LSB, W_upper_MSB; + uint8_t *stream_ptr; + uint8_t *maxStreamPtr; + uint8_t *stream_ptr_carry; + uint32_t cdf_lo, cdf_hi; + int k; + + /* point to beginning of stream buffer */ + stream_ptr = streamdata->stream + streamdata->stream_index; + W_upper = streamdata->W_upper; + + maxStreamPtr = streamdata->stream + STREAM_SIZE_MAX_60 - 1; + for (k = 0; k < N; k++) + { + /* compute cdf_lower and cdf_upper by evaluating the piecewise linear cdf */ + cdf_lo = piecewise((*dataQ7 - 64) * *envQ8); + cdf_hi = piecewise((*dataQ7 + 64) * *envQ8); + + /* test and clip if probability gets too small */ + while (cdf_lo+1 >= cdf_hi) { + /* clip */ + if (*dataQ7 > 0) { + *dataQ7 -= 128; + cdf_hi = cdf_lo; + cdf_lo = piecewise((*dataQ7 - 64) * *envQ8); + } else { + *dataQ7 += 128; + cdf_lo = cdf_hi; + cdf_hi = piecewise((*dataQ7 + 64) * *envQ8); + } + } + + dataQ7++; + // increment only once per 4 iterations for SWB-16kHz or WB + // increment only once per 2 iterations for SWB-12kHz + envQ8 += (isSWB12kHz)? (k & 1):((k & 1) & (k >> 1)); + + + /* update interval */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = W_upper >> 16; + W_lower = W_upper_MSB * cdf_lo; + W_lower += (W_upper_LSB * cdf_lo) >> 16; + W_upper = W_upper_MSB * cdf_hi; + W_upper += (W_upper_LSB * cdf_hi) >> 16; + + /* shift interval such that it begins at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamdata->streamval += W_lower; + + /* handle carry */ + if (streamdata->streamval < W_lower) + { + /* propagate carry */ + stream_ptr_carry = stream_ptr; + while (!(++(*--stream_ptr_carry))); + } + + /* renormalize interval, store most significant byte of streamval and update streamval */ + while ( !(W_upper & 0xFF000000) ) /* W_upper < 2^24 */ + { + W_upper <<= 8; + *stream_ptr++ = (uint8_t) (streamdata->streamval >> 24); + + if(stream_ptr > maxStreamPtr) + { + return -ISAC_DISALLOWED_BITSTREAM_LENGTH; + } + streamdata->streamval <<= 8; + } + } + + /* calculate new stream_index */ + streamdata->stream_index = (int)(stream_ptr - streamdata->stream); + streamdata->W_upper = W_upper; + + return 0; +} + + + +int WebRtcIsac_DecLogisticMulti2( + int16_t *dataQ7, /* output: data vector */ + Bitstr *streamdata, /* in-/output struct containing bitstream */ + const uint16_t *envQ8, /* input: side info vector defining the width of the pdf */ + const int16_t *ditherQ7,/* input: dither vector */ + const int N, /* input: data vector length */ + const int16_t isSWB12kHz) +{ + uint32_t W_lower, W_upper; + uint32_t W_tmp; + uint32_t W_upper_LSB, W_upper_MSB; + uint32_t streamval; + const uint8_t *stream_ptr; + uint32_t cdf_tmp; + int16_t candQ7; + int k; + + // Position just past the end of the stream. STREAM_SIZE_MAX_60 instead of + // STREAM_SIZE_MAX (which is the size of the allocated buffer) because that's + // the limit to how much data is filled in. + const uint8_t* const stream_end = streamdata->stream + STREAM_SIZE_MAX_60; + + stream_ptr = streamdata->stream + streamdata->stream_index; + W_upper = streamdata->W_upper; + if (streamdata->stream_index == 0) /* first time decoder is called for this stream */ + { + /* read first word from bytestream */ + if (stream_ptr + 3 >= stream_end) + return -1; // Would read out of bounds. Malformed input? + streamval = *stream_ptr << 24; + streamval |= *++stream_ptr << 16; + streamval |= *++stream_ptr << 8; + streamval |= *++stream_ptr; + } else { + streamval = streamdata->streamval; + } + + + for (k = 0; k < N; k++) + { + /* find the integer *data for which streamval lies in [W_lower+1, W_upper] */ + W_upper_LSB = W_upper & 0x0000FFFF; + W_upper_MSB = W_upper >> 16; + + /* find first candidate by inverting the logistic cdf */ + candQ7 = - *ditherQ7 + 64; + cdf_tmp = piecewise(candQ7 * *envQ8); + + W_tmp = W_upper_MSB * cdf_tmp; + W_tmp += (W_upper_LSB * cdf_tmp) >> 16; + if (streamval > W_tmp) + { + W_lower = W_tmp; + candQ7 += 128; + cdf_tmp = piecewise(candQ7 * *envQ8); + + W_tmp = W_upper_MSB * cdf_tmp; + W_tmp += (W_upper_LSB * cdf_tmp) >> 16; + while (streamval > W_tmp) + { + W_lower = W_tmp; + candQ7 += 128; + cdf_tmp = piecewise(candQ7 * *envQ8); + + W_tmp = W_upper_MSB * cdf_tmp; + W_tmp += (W_upper_LSB * cdf_tmp) >> 16; + + /* error check */ + if (W_lower == W_tmp) return -1; + } + W_upper = W_tmp; + + /* another sample decoded */ + *dataQ7 = candQ7 - 64; + } + else + { + W_upper = W_tmp; + candQ7 -= 128; + cdf_tmp = piecewise(candQ7 * *envQ8); + + W_tmp = W_upper_MSB * cdf_tmp; + W_tmp += (W_upper_LSB * cdf_tmp) >> 16; + while ( !(streamval > W_tmp) ) + { + W_upper = W_tmp; + candQ7 -= 128; + cdf_tmp = piecewise(candQ7 * *envQ8); + + W_tmp = W_upper_MSB * cdf_tmp; + W_tmp += (W_upper_LSB * cdf_tmp) >> 16; + + /* error check */ + if (W_upper == W_tmp) return -1; + } + W_lower = W_tmp; + + /* another sample decoded */ + *dataQ7 = candQ7 + 64; + } + ditherQ7++; + dataQ7++; + // increment only once per 4 iterations for SWB-16kHz or WB + // increment only once per 2 iterations for SWB-12kHz + envQ8 += (isSWB12kHz)? (k & 1):((k & 1) & (k >> 1)); + + /* shift interval to start at zero */ + W_upper -= ++W_lower; + + /* add integer to bitstream */ + streamval -= W_lower; + + /* renormalize interval and update streamval */ + while ( !(W_upper & 0xFF000000) ) /* W_upper < 2^24 */ + { + /* read next byte from stream */ + if (stream_ptr + 1 >= stream_end) + return -1; // Would read out of bounds. Malformed input? + streamval = (streamval << 8) | *++stream_ptr; + W_upper <<= 8; + } + } + + streamdata->stream_index = (int)(stream_ptr - streamdata->stream); + streamdata->W_upper = W_upper; + streamdata->streamval = streamval; + + /* find number of bytes in original stream (determined by current interval width) */ + if ( W_upper > 0x01FFFFFF ) + return streamdata->stream_index - 2; + else + return streamdata->stream_index - 1; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_decoder_isac.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_decoder_isac.cc new file mode 100644 index 0000000000..b671002e1e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_decoder_isac.cc @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h" + +#include "modules/audio_coding/codecs/isac/audio_decoder_isac_t_impl.h" + +namespace webrtc { + +// Explicit instantiation: +template class AudioDecoderIsacT; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac.cc new file mode 100644 index 0000000000..b7f2c0b1af --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac.cc @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h" + +#include "modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h" + +namespace webrtc { + +// Explicit instantiation: +template class AudioEncoderIsacT; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac_unittest.cc new file mode 100644 index 0000000000..333ab52c00 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac_unittest.cc @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +void TestBadConfig(const AudioEncoderIsacFloatImpl::Config& config) { + EXPECT_FALSE(config.IsOk()); +} + +void TestGoodConfig(const AudioEncoderIsacFloatImpl::Config& config) { + EXPECT_TRUE(config.IsOk()); + AudioEncoderIsacFloatImpl aei(config); +} + +// Wrap subroutine calls that test things in this, so that the error messages +// will be accompanied by stack traces that make it possible to tell which +// subroutine invocation caused the failure. +#define S(x) do { SCOPED_TRACE(#x); x; } while (0) + +} // namespace + +TEST(AudioEncoderIsacTest, TestConfigBitrate) { + AudioEncoderIsacFloatImpl::Config config; + + // The default value is some real, positive value. + EXPECT_GT(config.bit_rate, 1); + S(TestGoodConfig(config)); + + // 0 is another way to ask for the default value. + config.bit_rate = 0; + S(TestGoodConfig(config)); + + // Try some unreasonable values and watch them fail. + config.bit_rate = -1; + S(TestBadConfig(config)); + config.bit_rate = 1; + S(TestBadConfig(config)); + config.bit_rate = std::numeric_limits::max(); + S(TestBadConfig(config)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c new file mode 100644 index 0000000000..4a16932341 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c @@ -0,0 +1,1031 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * BwEstimator.c + * + * This file contains the code for the Bandwidth Estimator designed + * for iSAC. + * + */ + +#include "bandwidth_estimator.h" +#include "settings.h" +#include "isac.h" +#include "rtc_base/checks.h" + +#include +#include + +/* array of quantization levels for bottle neck info; Matlab code: */ +/* sprintf('%4.1ff, ', logspace(log10(5000), log10(40000), 12)) */ +static const float kQRateTableWb[12] = +{ + 10000.0f, 11115.3f, 12355.1f, 13733.1f, 15264.8f, 16967.3f, + 18859.8f, 20963.3f, 23301.4f, 25900.3f, 28789.0f, 32000.0f}; + + +static const float kQRateTableSwb[24] = +{ + 10000.0f, 11115.3f, 12355.1f, 13733.1f, 15264.8f, 16967.3f, + 18859.8f, 20963.3f, 23153.1f, 25342.9f, 27532.7f, 29722.5f, + 31912.3f, 34102.1f, 36291.9f, 38481.7f, 40671.4f, 42861.2f, + 45051.0f, 47240.8f, 49430.6f, 51620.4f, 53810.2f, 56000.0f, +}; + + + + +int32_t WebRtcIsac_InitBandwidthEstimator( + BwEstimatorstr* bwest_str, + enum IsacSamplingRate encoderSampRate, + enum IsacSamplingRate decoderSampRate) +{ + switch(encoderSampRate) + { + case kIsacWideband: + { + bwest_str->send_bw_avg = INIT_BN_EST_WB; + break; + } + case kIsacSuperWideband: + { + bwest_str->send_bw_avg = INIT_BN_EST_SWB; + break; + } + } + + switch(decoderSampRate) + { + case kIsacWideband: + { + bwest_str->prev_frame_length = INIT_FRAME_LEN_WB; + bwest_str->rec_bw_inv = 1.0f / + (INIT_BN_EST_WB + INIT_HDR_RATE_WB); + bwest_str->rec_bw = (int32_t)INIT_BN_EST_WB; + bwest_str->rec_bw_avg_Q = INIT_BN_EST_WB; + bwest_str->rec_bw_avg = INIT_BN_EST_WB + INIT_HDR_RATE_WB; + bwest_str->rec_header_rate = INIT_HDR_RATE_WB; + break; + } + case kIsacSuperWideband: + { + bwest_str->prev_frame_length = INIT_FRAME_LEN_SWB; + bwest_str->rec_bw_inv = 1.0f / + (INIT_BN_EST_SWB + INIT_HDR_RATE_SWB); + bwest_str->rec_bw = (int32_t)INIT_BN_EST_SWB; + bwest_str->rec_bw_avg_Q = INIT_BN_EST_SWB; + bwest_str->rec_bw_avg = INIT_BN_EST_SWB + INIT_HDR_RATE_SWB; + bwest_str->rec_header_rate = INIT_HDR_RATE_SWB; + break; + } + } + + bwest_str->prev_rec_rtp_number = 0; + bwest_str->prev_rec_arr_ts = 0; + bwest_str->prev_rec_send_ts = 0; + bwest_str->prev_rec_rtp_rate = 1.0f; + bwest_str->last_update_ts = 0; + bwest_str->last_reduction_ts = 0; + bwest_str->count_tot_updates_rec = -9; + bwest_str->rec_jitter = 10.0f; + bwest_str->rec_jitter_short_term = 0.0f; + bwest_str->rec_jitter_short_term_abs = 5.0f; + bwest_str->rec_max_delay = 10.0f; + bwest_str->rec_max_delay_avg_Q = 10.0f; + bwest_str->num_pkts_rec = 0; + + bwest_str->send_max_delay_avg = 10.0f; + + bwest_str->hsn_detect_rec = 0; + + bwest_str->num_consec_rec_pkts_over_30k = 0; + + bwest_str->hsn_detect_snd = 0; + + bwest_str->num_consec_snt_pkts_over_30k = 0; + + bwest_str->in_wait_period = 0; + + bwest_str->change_to_WB = 0; + + bwest_str->numConsecLatePkts = 0; + bwest_str->consecLatency = 0; + bwest_str->inWaitLatePkts = 0; + bwest_str->senderTimestamp = 0; + bwest_str->receiverTimestamp = 0; + + bwest_str->external_bw_info.in_use = 0; + + return 0; +} + +/* This function updates both bottle neck rates */ +/* Parameters: */ +/* rtp_number - value from RTP packet, from NetEq */ +/* frame length - length of signal frame in ms, from iSAC decoder */ +/* send_ts - value in RTP header giving send time in samples */ +/* arr_ts - value given by timeGetTime() time of arrival in samples of packet from NetEq */ +/* pksize - size of packet in bytes, from NetEq */ +/* Index - integer (range 0...23) indicating bottle neck & jitter as estimated by other side */ +/* returns 0 if everything went fine, -1 otherwise */ +int16_t WebRtcIsac_UpdateBandwidthEstimator( + BwEstimatorstr* bwest_str, + const uint16_t rtp_number, + const int32_t frame_length, + const uint32_t send_ts, + const uint32_t arr_ts, + const size_t pksize + /*, const uint16_t Index*/) +{ + float weight = 0.0f; + float curr_bw_inv = 0.0f; + float rec_rtp_rate; + float t_diff_proj; + float arr_ts_diff; + float send_ts_diff; + float arr_time_noise; + float arr_time_noise_abs; + + float delay_correction_factor = 1; + float late_diff = 0.0f; + int immediate_set = 0; + int num_pkts_expected; + + RTC_DCHECK(!bwest_str->external_bw_info.in_use); + + // We have to adjust the header-rate if the first packet has a + // frame-size different than the initialized value. + if ( frame_length != bwest_str->prev_frame_length ) + { + bwest_str->rec_header_rate = (float)HEADER_SIZE * 8.0f * + 1000.0f / (float)frame_length; /* bits/s */ + } + + /* UPDATE ESTIMATES ON THIS SIDE */ + /* compute far-side transmission rate */ + rec_rtp_rate = ((float)pksize * 8.0f * 1000.0f / (float)frame_length) + + bwest_str->rec_header_rate; + // rec_rtp_rate packet bits/s + header bits/s + + /* check for timer wrap-around */ + if (arr_ts < bwest_str->prev_rec_arr_ts) + { + bwest_str->prev_rec_arr_ts = arr_ts; + bwest_str->last_update_ts = arr_ts; + bwest_str->last_reduction_ts = arr_ts + 3*FS; + bwest_str->num_pkts_rec = 0; + + /* store frame length */ + bwest_str->prev_frame_length = frame_length; + + /* store far-side transmission rate */ + bwest_str->prev_rec_rtp_rate = rec_rtp_rate; + + /* store far-side RTP time stamp */ + bwest_str->prev_rec_rtp_number = rtp_number; + + return 0; + } + + bwest_str->num_pkts_rec++; + + /* check that it's not one of the first 9 packets */ + if ( bwest_str->count_tot_updates_rec > 0 ) + { + if(bwest_str->in_wait_period > 0 ) + { + bwest_str->in_wait_period--; + } + + bwest_str->inWaitLatePkts -= ((bwest_str->inWaitLatePkts > 0)? 1:0); + send_ts_diff = (float)(send_ts - bwest_str->prev_rec_send_ts); + + if (send_ts_diff <= (16 * frame_length)*2) + //doesn't allow for a dropped packet, not sure necessary to be + // that strict -DH + { + /* if not been updated for a long time, reduce the BN estimate */ + if((uint32_t)(arr_ts - bwest_str->last_update_ts) * + 1000.0f / FS > 3000) + { + //how many frames should have been received since the last + // update if too many have been dropped or there have been + // big delays won't allow this reduction may no longer need + // the send_ts_diff here + num_pkts_expected = (int)(((float)(arr_ts - + bwest_str->last_update_ts) * 1000.0f /(float) FS) / + (float)frame_length); + + if(((float)bwest_str->num_pkts_rec/(float)num_pkts_expected) > + 0.9) + { + float inv_bitrate = (float) pow( 0.99995, + (double)((uint32_t)(arr_ts - + bwest_str->last_reduction_ts)*1000.0f/FS) ); + + if ( inv_bitrate ) + { + bwest_str->rec_bw_inv /= inv_bitrate; + + //precautionary, likely never necessary + if (bwest_str->hsn_detect_snd && + bwest_str->hsn_detect_rec) + { + if (bwest_str->rec_bw_inv > 0.000066f) + { + bwest_str->rec_bw_inv = 0.000066f; + } + } + } + else + { + bwest_str->rec_bw_inv = 1.0f / + (INIT_BN_EST_WB + INIT_HDR_RATE_WB); + } + /* reset time-since-update counter */ + bwest_str->last_reduction_ts = arr_ts; + } + else + //reset here? + { + bwest_str->last_reduction_ts = arr_ts + 3*FS; + bwest_str->last_update_ts = arr_ts; + bwest_str->num_pkts_rec = 0; + } + } + } + else + { + bwest_str->last_reduction_ts = arr_ts + 3*FS; + bwest_str->last_update_ts = arr_ts; + bwest_str->num_pkts_rec = 0; + } + + + /* temporarily speed up adaptation if frame length has changed */ + if ( frame_length != bwest_str->prev_frame_length ) + { + bwest_str->count_tot_updates_rec = 10; + bwest_str->rec_header_rate = (float)HEADER_SIZE * 8.0f * + 1000.0f / (float)frame_length; /* bits/s */ + + bwest_str->rec_bw_inv = 1.0f /((float)bwest_str->rec_bw + + bwest_str->rec_header_rate); + } + + //////////////////////// + arr_ts_diff = (float)(arr_ts - bwest_str->prev_rec_arr_ts); + + if (send_ts_diff > 0 ) + { + late_diff = arr_ts_diff - send_ts_diff; + } + else + { + late_diff = arr_ts_diff - (float)(16 * frame_length); + } + + if((late_diff > 0) && !bwest_str->inWaitLatePkts) + { + bwest_str->numConsecLatePkts++; + bwest_str->consecLatency += late_diff; + } + else + { + bwest_str->numConsecLatePkts = 0; + bwest_str->consecLatency = 0; + } + if(bwest_str->numConsecLatePkts > 50) + { + float latencyMs = bwest_str->consecLatency/(FS/1000); + float averageLatencyMs = latencyMs / bwest_str->numConsecLatePkts; + delay_correction_factor = frame_length / (frame_length + averageLatencyMs); + immediate_set = 1; + bwest_str->inWaitLatePkts = (int16_t)((bwest_str->consecLatency/(FS/1000)) / 30);// + 150; + bwest_str->start_wait_period = arr_ts; + } + /////////////////////////////////////////////// + + + + /* update only if previous packet was not lost */ + if ( rtp_number == bwest_str->prev_rec_rtp_number + 1 ) + { + + + if (!(bwest_str->hsn_detect_snd && bwest_str->hsn_detect_rec)) + { + if ((arr_ts_diff > (float)(16 * frame_length))) + { + //1/2 second + if ((late_diff > 8000.0f) && !bwest_str->in_wait_period) + { + delay_correction_factor = 0.7f; + bwest_str->in_wait_period = 55; + bwest_str->start_wait_period = arr_ts; + immediate_set = 1; + } + //320 ms + else if (late_diff > 5120.0f && !bwest_str->in_wait_period) + { + delay_correction_factor = 0.8f; + immediate_set = 1; + bwest_str->in_wait_period = 44; + bwest_str->start_wait_period = arr_ts; + } + } + } + + + if ((bwest_str->prev_rec_rtp_rate > bwest_str->rec_bw_avg) && + (rec_rtp_rate > bwest_str->rec_bw_avg) && + !bwest_str->in_wait_period) + { + /* test if still in initiation period and increment counter */ + if (bwest_str->count_tot_updates_rec++ > 99) + { + /* constant weight after initiation part */ + weight = 0.01f; + } + else + { + /* weight decreases with number of updates */ + weight = 1.0f / (float) bwest_str->count_tot_updates_rec; + } + /* Bottle Neck Estimation */ + + /* limit outliers */ + /* if more than 25 ms too much */ + if (arr_ts_diff > frame_length * FS/1000 + 400.0f) + { + // in samples, why 25ms?? + arr_ts_diff = frame_length * FS/1000 + 400.0f; + } + if(arr_ts_diff < (frame_length * FS/1000) - 160.0f) + { + /* don't allow it to be less than frame rate - 10 ms */ + arr_ts_diff = (float)frame_length * FS/1000 - 160.0f; + } + + /* compute inverse receiving rate for last packet */ + curr_bw_inv = arr_ts_diff / ((float)(pksize + HEADER_SIZE) * + 8.0f * FS); // (180+35)*8*16000 = 27.5 Mbit.... + + + if(curr_bw_inv < + (1.0f / (MAX_ISAC_BW + bwest_str->rec_header_rate))) + { + // don't allow inv rate to be larger than MAX + curr_bw_inv = (1.0f / + (MAX_ISAC_BW + bwest_str->rec_header_rate)); + } + + /* update bottle neck rate estimate */ + bwest_str->rec_bw_inv = weight * curr_bw_inv + + (1.0f - weight) * bwest_str->rec_bw_inv; + + /* reset time-since-update counter */ + bwest_str->last_update_ts = arr_ts; + bwest_str->last_reduction_ts = arr_ts + 3 * FS; + bwest_str->num_pkts_rec = 0; + + /* Jitter Estimation */ + /* projected difference between arrival times */ + t_diff_proj = ((float)(pksize + HEADER_SIZE) * 8.0f * + 1000.0f) / bwest_str->rec_bw_avg; + + + // difference between projected and actual + // arrival time differences + arr_time_noise = (float)(arr_ts_diff*1000.0f/FS) - + t_diff_proj; + arr_time_noise_abs = (float) fabs( arr_time_noise ); + + /* long term averaged absolute jitter */ + bwest_str->rec_jitter = weight * arr_time_noise_abs + + (1.0f - weight) * bwest_str->rec_jitter; + if (bwest_str->rec_jitter > 10.0f) + { + bwest_str->rec_jitter = 10.0f; + } + /* short term averaged absolute jitter */ + bwest_str->rec_jitter_short_term_abs = 0.05f * + arr_time_noise_abs + 0.95f * + bwest_str->rec_jitter_short_term_abs; + + /* short term averaged jitter */ + bwest_str->rec_jitter_short_term = 0.05f * arr_time_noise + + 0.95f * bwest_str->rec_jitter_short_term; + } + } + } + else + { + // reset time-since-update counter when + // receiving the first 9 packets + bwest_str->last_update_ts = arr_ts; + bwest_str->last_reduction_ts = arr_ts + 3*FS; + bwest_str->num_pkts_rec = 0; + + bwest_str->count_tot_updates_rec++; + } + + /* limit minimum bottle neck rate */ + if (bwest_str->rec_bw_inv > 1.0f / ((float)MIN_ISAC_BW + + bwest_str->rec_header_rate)) + { + bwest_str->rec_bw_inv = 1.0f / ((float)MIN_ISAC_BW + + bwest_str->rec_header_rate); + } + + // limit maximum bitrate + if (bwest_str->rec_bw_inv < 1.0f / ((float)MAX_ISAC_BW + + bwest_str->rec_header_rate)) + { + bwest_str->rec_bw_inv = 1.0f / ((float)MAX_ISAC_BW + + bwest_str->rec_header_rate); + } + + /* store frame length */ + bwest_str->prev_frame_length = frame_length; + + /* store far-side transmission rate */ + bwest_str->prev_rec_rtp_rate = rec_rtp_rate; + + /* store far-side RTP time stamp */ + bwest_str->prev_rec_rtp_number = rtp_number; + + // Replace bwest_str->rec_max_delay by the new + // value (atomic operation) + bwest_str->rec_max_delay = 3.0f * bwest_str->rec_jitter; + + /* store send and arrival time stamp */ + bwest_str->prev_rec_arr_ts = arr_ts ; + bwest_str->prev_rec_send_ts = send_ts; + + /* Replace bwest_str->rec_bw by the new value (atomic operation) */ + bwest_str->rec_bw = (int32_t)(1.0f / bwest_str->rec_bw_inv - + bwest_str->rec_header_rate); + + if (immediate_set) + { + bwest_str->rec_bw = (int32_t) (delay_correction_factor * + (float) bwest_str->rec_bw); + + if (bwest_str->rec_bw < (int32_t) MIN_ISAC_BW) + { + bwest_str->rec_bw = (int32_t) MIN_ISAC_BW; + } + + bwest_str->rec_bw_avg = bwest_str->rec_bw + + bwest_str->rec_header_rate; + + bwest_str->rec_bw_avg_Q = (float) bwest_str->rec_bw; + + bwest_str->rec_jitter_short_term = 0.0f; + + bwest_str->rec_bw_inv = 1.0f / (bwest_str->rec_bw + + bwest_str->rec_header_rate); + + bwest_str->count_tot_updates_rec = 1; + + immediate_set = 0; + bwest_str->consecLatency = 0; + bwest_str->numConsecLatePkts = 0; + } + + return 0; +} + + +/* This function updates the send bottle neck rate */ +/* Index - integer (range 0...23) indicating bottle neck & jitter as estimated by other side */ +/* returns 0 if everything went fine, -1 otherwise */ +int16_t WebRtcIsac_UpdateUplinkBwImpl( + BwEstimatorstr* bwest_str, + int16_t index, + enum IsacSamplingRate encoderSamplingFreq) +{ + RTC_DCHECK(!bwest_str->external_bw_info.in_use); + + if((index < 0) || (index > 23)) + { + return -ISAC_RANGE_ERROR_BW_ESTIMATOR; + } + + /* UPDATE ESTIMATES FROM OTHER SIDE */ + if(encoderSamplingFreq == kIsacWideband) + { + if(index > 11) + { + index -= 12; + /* compute the jitter estimate as decoded on the other side */ + bwest_str->send_max_delay_avg = 0.9f * bwest_str->send_max_delay_avg + + 0.1f * (float)MAX_ISAC_MD; + } + else + { + /* compute the jitter estimate as decoded on the other side */ + bwest_str->send_max_delay_avg = 0.9f * bwest_str->send_max_delay_avg + + 0.1f * (float)MIN_ISAC_MD; + } + + /* compute the BN estimate as decoded on the other side */ + bwest_str->send_bw_avg = 0.9f * bwest_str->send_bw_avg + + 0.1f * kQRateTableWb[index]; + } + else + { + /* compute the BN estimate as decoded on the other side */ + bwest_str->send_bw_avg = 0.9f * bwest_str->send_bw_avg + + 0.1f * kQRateTableSwb[index]; + } + + if (bwest_str->send_bw_avg > (float) 28000 && !bwest_str->hsn_detect_snd) + { + bwest_str->num_consec_snt_pkts_over_30k++; + + if (bwest_str->num_consec_snt_pkts_over_30k >= 66) + { + //approx 2 seconds with 30ms frames + bwest_str->hsn_detect_snd = 1; + } + } + else if (!bwest_str->hsn_detect_snd) + { + bwest_str->num_consec_snt_pkts_over_30k = 0; + } + return 0; +} + +// called when there is upper-band bit-stream to update jitter +// statistics. +int16_t WebRtcIsac_UpdateUplinkJitter( + BwEstimatorstr* bwest_str, + int32_t index) +{ + RTC_DCHECK(!bwest_str->external_bw_info.in_use); + + if((index < 0) || (index > 23)) + { + return -ISAC_RANGE_ERROR_BW_ESTIMATOR; + } + + if(index > 0) + { + /* compute the jitter estimate as decoded on the other side */ + bwest_str->send_max_delay_avg = 0.9f * bwest_str->send_max_delay_avg + + 0.1f * (float)MAX_ISAC_MD; + } + else + { + /* compute the jitter estimate as decoded on the other side */ + bwest_str->send_max_delay_avg = 0.9f * bwest_str->send_max_delay_avg + + 0.1f * (float)MIN_ISAC_MD; + } + + return 0; +} + + + +// Returns the bandwidth/jitter estimation code (integer 0...23) +// to put in the sending iSAC payload +void +WebRtcIsac_GetDownlinkBwJitIndexImpl( + BwEstimatorstr* bwest_str, + int16_t* bottleneckIndex, + int16_t* jitterInfo, + enum IsacSamplingRate decoderSamplingFreq) +{ + float MaxDelay; + //uint16_t MaxDelayBit; + + float rate; + float r; + float e1, e2; + const float weight = 0.1f; + const float* ptrQuantizationTable; + int16_t addJitterInfo; + int16_t minInd; + int16_t maxInd; + int16_t midInd; + + if (bwest_str->external_bw_info.in_use) { + *bottleneckIndex = bwest_str->external_bw_info.bottleneck_idx; + *jitterInfo = bwest_str->external_bw_info.jitter_info; + return; + } + + /* Get Max Delay Bit */ + /* get unquantized max delay */ + MaxDelay = (float)WebRtcIsac_GetDownlinkMaxDelay(bwest_str); + + if ( ((1.f - weight) * bwest_str->rec_max_delay_avg_Q + weight * + MAX_ISAC_MD - MaxDelay) > (MaxDelay - (1.f-weight) * + bwest_str->rec_max_delay_avg_Q - weight * MIN_ISAC_MD) ) + { + jitterInfo[0] = 0; + /* update quantized average */ + bwest_str->rec_max_delay_avg_Q = + (1.f - weight) * bwest_str->rec_max_delay_avg_Q + weight * + (float)MIN_ISAC_MD; + } + else + { + jitterInfo[0] = 1; + /* update quantized average */ + bwest_str->rec_max_delay_avg_Q = + (1.f-weight) * bwest_str->rec_max_delay_avg_Q + weight * + (float)MAX_ISAC_MD; + } + + // Get unquantized rate. + rate = (float)WebRtcIsac_GetDownlinkBandwidth(bwest_str); + + /* Get Rate Index */ + if(decoderSamplingFreq == kIsacWideband) + { + ptrQuantizationTable = kQRateTableWb; + addJitterInfo = 1; + maxInd = 11; + } + else + { + ptrQuantizationTable = kQRateTableSwb; + addJitterInfo = 0; + maxInd = 23; + } + + minInd = 0; + while(maxInd > minInd + 1) + { + midInd = (maxInd + minInd) >> 1; + if(rate > ptrQuantizationTable[midInd]) + { + minInd = midInd; + } + else + { + maxInd = midInd; + } + } + // Chose the index which gives results an average which is closest + // to rate + r = (1 - weight) * bwest_str->rec_bw_avg_Q - rate; + e1 = weight * ptrQuantizationTable[minInd] + r; + e2 = weight * ptrQuantizationTable[maxInd] + r; + e1 = (e1 > 0)? e1:-e1; + e2 = (e2 > 0)? e2:-e2; + if(e1 < e2) + { + bottleneckIndex[0] = minInd; + } + else + { + bottleneckIndex[0] = maxInd; + } + + bwest_str->rec_bw_avg_Q = (1 - weight) * bwest_str->rec_bw_avg_Q + + weight * ptrQuantizationTable[bottleneckIndex[0]]; + bottleneckIndex[0] += jitterInfo[0] * 12 * addJitterInfo; + + bwest_str->rec_bw_avg = (1 - weight) * bwest_str->rec_bw_avg + weight * + (rate + bwest_str->rec_header_rate); +} + + + +/* get the bottle neck rate from far side to here, as estimated on this side */ +int32_t WebRtcIsac_GetDownlinkBandwidth( const BwEstimatorstr *bwest_str) +{ + int32_t rec_bw; + float jitter_sign; + float bw_adjust; + + RTC_DCHECK(!bwest_str->external_bw_info.in_use); + + /* create a value between -1.0 and 1.0 indicating "average sign" of jitter */ + jitter_sign = bwest_str->rec_jitter_short_term / + bwest_str->rec_jitter_short_term_abs; + + /* adjust bw proportionally to negative average jitter sign */ + bw_adjust = 1.0f - jitter_sign * (0.15f + 0.15f * jitter_sign * jitter_sign); + + /* adjust Rate if jitter sign is mostly constant */ + rec_bw = (int32_t)(bwest_str->rec_bw * bw_adjust); + + /* limit range of bottle neck rate */ + if (rec_bw < MIN_ISAC_BW) + { + rec_bw = MIN_ISAC_BW; + } + else if (rec_bw > MAX_ISAC_BW) + { + rec_bw = MAX_ISAC_BW; + } + return rec_bw; +} + +/* Returns the max delay (in ms) */ +int32_t +WebRtcIsac_GetDownlinkMaxDelay(const BwEstimatorstr *bwest_str) +{ + int32_t rec_max_delay; + + RTC_DCHECK(!bwest_str->external_bw_info.in_use); + + rec_max_delay = (int32_t)(bwest_str->rec_max_delay); + + /* limit range of jitter estimate */ + if (rec_max_delay < MIN_ISAC_MD) + { + rec_max_delay = MIN_ISAC_MD; + } + else if (rec_max_delay > MAX_ISAC_MD) + { + rec_max_delay = MAX_ISAC_MD; + } + return rec_max_delay; +} + +/* Clamp val to the closed interval [min,max]. */ +static int32_t clamp(int32_t val, int32_t min, int32_t max) { + RTC_DCHECK_LE(min, max); + return val < min ? min : (val > max ? max : val); +} + +int32_t WebRtcIsac_GetUplinkBandwidth(const BwEstimatorstr* bwest_str) { + return bwest_str->external_bw_info.in_use + ? bwest_str->external_bw_info.send_bw_avg + : clamp(bwest_str->send_bw_avg, MIN_ISAC_BW, MAX_ISAC_BW); +} + +int32_t WebRtcIsac_GetUplinkMaxDelay(const BwEstimatorstr* bwest_str) { + return bwest_str->external_bw_info.in_use + ? bwest_str->external_bw_info.send_max_delay_avg + : clamp(bwest_str->send_max_delay_avg, MIN_ISAC_MD, MAX_ISAC_MD); +} + +void WebRtcIsacBw_GetBandwidthInfo(BwEstimatorstr* bwest_str, + enum IsacSamplingRate decoder_sample_rate_hz, + IsacBandwidthInfo* bwinfo) { + RTC_DCHECK(!bwest_str->external_bw_info.in_use); + bwinfo->in_use = 1; + bwinfo->send_bw_avg = WebRtcIsac_GetUplinkBandwidth(bwest_str); + bwinfo->send_max_delay_avg = WebRtcIsac_GetUplinkMaxDelay(bwest_str); + WebRtcIsac_GetDownlinkBwJitIndexImpl(bwest_str, &bwinfo->bottleneck_idx, + &bwinfo->jitter_info, + decoder_sample_rate_hz); +} + +void WebRtcIsacBw_SetBandwidthInfo(BwEstimatorstr* bwest_str, + const IsacBandwidthInfo* bwinfo) { + memcpy(&bwest_str->external_bw_info, bwinfo, + sizeof bwest_str->external_bw_info); +} + +/* + * update long-term average bitrate and amount of data in buffer + * returns minimum payload size (bytes) + */ +int WebRtcIsac_GetMinBytes( + RateModel* State, + int StreamSize, /* bytes in bitstream */ + const int FrameSamples, /* samples per frame */ + const double BottleNeck, /* bottle neck rate; excl headers (bps) */ + const double DelayBuildUp, /* max delay from bottleneck buffering (ms) */ + enum ISACBandwidth bandwidth + /*,int16_t frequentLargePackets*/) +{ + double MinRate = 0.0; + int MinBytes; + double TransmissionTime; + int burstInterval = BURST_INTERVAL; + + // first 10 packets @ low rate, then INIT_BURST_LEN packets @ + // fixed rate of INIT_RATE bps + if (State->InitCounter > 0) + { + if (State->InitCounter-- <= INIT_BURST_LEN) + { + if(bandwidth == isac8kHz) + { + MinRate = INIT_RATE_WB; + } + else + { + MinRate = INIT_RATE_SWB; + } + } + else + { + MinRate = 0; + } + } + else + { + /* handle burst */ + if (State->BurstCounter) + { + if (State->StillBuffered < (1.0 - 1.0/BURST_LEN) * DelayBuildUp) + { + /* max bps derived from BottleNeck and DelayBuildUp values */ + MinRate = (1.0 + (FS/1000) * DelayBuildUp / + (double)(BURST_LEN * FrameSamples)) * BottleNeck; + } + else + { + // max bps derived from StillBuffered and DelayBuildUp + // values + MinRate = (1.0 + (FS/1000) * (DelayBuildUp - + State->StillBuffered) / (double)FrameSamples) * BottleNeck; + if (MinRate < 1.04 * BottleNeck) + { + MinRate = 1.04 * BottleNeck; + } + } + State->BurstCounter--; + } + } + + + /* convert rate from bits/second to bytes/packet */ + MinBytes = (int) (MinRate * FrameSamples / (8.0 * FS)); + + /* StreamSize will be adjusted if less than MinBytes */ + if (StreamSize < MinBytes) + { + StreamSize = MinBytes; + } + + /* keep track of when bottle neck was last exceeded by at least 1% */ + if (StreamSize * 8.0 * FS / FrameSamples > 1.01 * BottleNeck) { + if (State->PrevExceed) { + /* bottle_neck exceded twice in a row, decrease ExceedAgo */ + State->ExceedAgo -= /*BURST_INTERVAL*/ burstInterval / (BURST_LEN - 1); + if (State->ExceedAgo < 0) + State->ExceedAgo = 0; + } + else + { + State->ExceedAgo += (FrameSamples * 1000) / FS; /* ms */ + State->PrevExceed = 1; + } + } + else + { + State->PrevExceed = 0; + State->ExceedAgo += (FrameSamples * 1000) / FS; /* ms */ + } + + /* set burst flag if bottle neck not exceeded for long time */ + if ((State->ExceedAgo > burstInterval) && + (State->BurstCounter == 0)) + { + if (State->PrevExceed) + { + State->BurstCounter = BURST_LEN - 1; + } + else + { + State->BurstCounter = BURST_LEN; + } + } + + + /* Update buffer delay */ + TransmissionTime = StreamSize * 8.0 * 1000.0 / BottleNeck; /* ms */ + State->StillBuffered += TransmissionTime; + State->StillBuffered -= (FrameSamples * 1000) / FS; /* ms */ + if (State->StillBuffered < 0.0) + { + State->StillBuffered = 0.0; + } + + return MinBytes; +} + + +/* + * update long-term average bitrate and amount of data in buffer + */ +void WebRtcIsac_UpdateRateModel( + RateModel *State, + int StreamSize, /* bytes in bitstream */ + const int FrameSamples, /* samples per frame */ + const double BottleNeck) /* bottle neck rate; excl headers (bps) */ +{ + double TransmissionTime; + + /* avoid the initial "high-rate" burst */ + State->InitCounter = 0; + + /* Update buffer delay */ + TransmissionTime = StreamSize * 8.0 * 1000.0 / BottleNeck; /* ms */ + State->StillBuffered += TransmissionTime; + State->StillBuffered -= (FrameSamples * 1000) / FS; /* ms */ + if (State->StillBuffered < 0.0) + State->StillBuffered = 0.0; + +} + + +void WebRtcIsac_InitRateModel( + RateModel *State) +{ + State->PrevExceed = 0; /* boolean */ + State->ExceedAgo = 0; /* ms */ + State->BurstCounter = 0; /* packets */ + State->InitCounter = INIT_BURST_LEN + 10; /* packets */ + State->StillBuffered = 1.0; /* ms */ +} + +int WebRtcIsac_GetNewFrameLength( + double bottle_neck, + int current_framesamples) +{ + int new_framesamples; + + const int Thld_20_30 = 20000; + + //const int Thld_30_20 = 30000; + const int Thld_30_20 = 1000000; // disable 20 ms frames + + const int Thld_30_60 = 18000; + //const int Thld_30_60 = 0; // disable 60 ms frames + + const int Thld_60_30 = 27000; + + + new_framesamples = current_framesamples; + + /* find new framelength */ + switch(current_framesamples) { + case 320: + if (bottle_neck < Thld_20_30) + new_framesamples = 480; + break; + case 480: + if (bottle_neck < Thld_30_60) + new_framesamples = 960; + else if (bottle_neck > Thld_30_20) + new_framesamples = 320; + break; + case 960: + if (bottle_neck >= Thld_60_30) + new_framesamples = 480; + break; + } + + return new_framesamples; +} + +double WebRtcIsac_GetSnr( + double bottle_neck, + int framesamples) +{ + double s2nr; + + const double a_20 = -30.0; + const double b_20 = 0.8; + const double c_20 = 0.0; + + const double a_30 = -23.0; + const double b_30 = 0.48; + const double c_30 = 0.0; + + const double a_60 = -23.0; + const double b_60 = 0.53; + const double c_60 = 0.0; + + + /* find new SNR value */ + switch(framesamples) { + case 320: + s2nr = a_20 + b_20 * bottle_neck * 0.001 + c_20 * bottle_neck * + bottle_neck * 0.000001; + break; + case 480: + s2nr = a_30 + b_30 * bottle_neck * 0.001 + c_30 * bottle_neck * + bottle_neck * 0.000001; + break; + case 960: + s2nr = a_60 + b_60 * bottle_neck * 0.001 + c_60 * bottle_neck * + bottle_neck * 0.000001; + break; + default: + s2nr = 0; + } + + return s2nr; + +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h new file mode 100644 index 0000000000..ddebdcd342 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * bandwidth_estimator.h + * + * This header file contains the API for the Bandwidth Estimator + * designed for iSAC. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_BANDWIDTH_ESTIMATOR_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_BANDWIDTH_ESTIMATOR_H_ + +#include "structs.h" +#include "settings.h" + + +#define MIN_ISAC_BW 10000 +#define MIN_ISAC_BW_LB 10000 +#define MIN_ISAC_BW_UB 25000 + +#define MAX_ISAC_BW 56000 +#define MAX_ISAC_BW_UB 32000 +#define MAX_ISAC_BW_LB 32000 + +#define MIN_ISAC_MD 5 +#define MAX_ISAC_MD 25 + +// assumed header size, in bytes; we don't know the exact number +// (header compression may be used) +#define HEADER_SIZE 35 + +// Initial Frame-Size, in ms, for Wideband & Super-Wideband Mode +#define INIT_FRAME_LEN_WB 60 +#define INIT_FRAME_LEN_SWB 30 + +// Initial Bottleneck Estimate, in bits/sec, for +// Wideband & Super-wideband mode +#define INIT_BN_EST_WB 20e3f +#define INIT_BN_EST_SWB 56e3f + +// Initial Header rate (header rate depends on frame-size), +// in bits/sec, for Wideband & Super-Wideband mode. +#define INIT_HDR_RATE_WB \ + ((float)HEADER_SIZE * 8.0f * 1000.0f / (float)INIT_FRAME_LEN_WB) +#define INIT_HDR_RATE_SWB \ + ((float)HEADER_SIZE * 8.0f * 1000.0f / (float)INIT_FRAME_LEN_SWB) + +// number of packets in a row for a high rate burst +#define BURST_LEN 3 + +// ms, max time between two full bursts +#define BURST_INTERVAL 500 + +// number of packets in a row for initial high rate burst +#define INIT_BURST_LEN 5 + +// bits/s, rate for the first BURST_LEN packets +#define INIT_RATE_WB INIT_BN_EST_WB +#define INIT_RATE_SWB INIT_BN_EST_SWB + + +#if defined(__cplusplus) +extern "C" { +#endif + + /* This function initializes the struct */ + /* to be called before using the struct for anything else */ + /* returns 0 if everything went fine, -1 otherwise */ + int32_t WebRtcIsac_InitBandwidthEstimator( + BwEstimatorstr* bwest_str, + enum IsacSamplingRate encoderSampRate, + enum IsacSamplingRate decoderSampRate); + + /* This function updates the receiving estimate */ + /* Parameters: */ + /* rtp_number - value from RTP packet, from NetEq */ + /* frame length - length of signal frame in ms, from iSAC decoder */ + /* send_ts - value in RTP header giving send time in samples */ + /* arr_ts - value given by timeGetTime() time of arrival in samples of packet from NetEq */ + /* pksize - size of packet in bytes, from NetEq */ + /* Index - integer (range 0...23) indicating bottle neck & jitter as estimated by other side */ + /* returns 0 if everything went fine, -1 otherwise */ + int16_t WebRtcIsac_UpdateBandwidthEstimator( + BwEstimatorstr* bwest_str, + const uint16_t rtp_number, + const int32_t frame_length, + const uint32_t send_ts, + const uint32_t arr_ts, + const size_t pksize); + + /* Update receiving estimates. Used when we only receive BWE index, no iSAC data packet. */ + int16_t WebRtcIsac_UpdateUplinkBwImpl( + BwEstimatorstr* bwest_str, + int16_t Index, + enum IsacSamplingRate encoderSamplingFreq); + + /* Returns the bandwidth/jitter estimation code (integer 0...23) to put in the sending iSAC payload */ + void WebRtcIsac_GetDownlinkBwJitIndexImpl( + BwEstimatorstr* bwest_str, + int16_t* bottleneckIndex, + int16_t* jitterInfo, + enum IsacSamplingRate decoderSamplingFreq); + + /* Returns the bandwidth estimation (in bps) */ + int32_t WebRtcIsac_GetDownlinkBandwidth( + const BwEstimatorstr *bwest_str); + + /* Returns the max delay (in ms) */ + int32_t WebRtcIsac_GetDownlinkMaxDelay( + const BwEstimatorstr *bwest_str); + + /* Returns the bandwidth that iSAC should send with in bps */ + int32_t WebRtcIsac_GetUplinkBandwidth(const BwEstimatorstr* bwest_str); + + /* Returns the max delay value from the other side in ms */ + int32_t WebRtcIsac_GetUplinkMaxDelay( + const BwEstimatorstr *bwest_str); + + /* Fills in an IsacExternalBandwidthInfo struct. */ + void WebRtcIsacBw_GetBandwidthInfo( + BwEstimatorstr* bwest_str, + enum IsacSamplingRate decoder_sample_rate_hz, + IsacBandwidthInfo* bwinfo); + + /* Uses the values from an IsacExternalBandwidthInfo struct. */ + void WebRtcIsacBw_SetBandwidthInfo(BwEstimatorstr* bwest_str, + const IsacBandwidthInfo* bwinfo); + + /* + * update amount of data in bottle neck buffer and burst handling + * returns minimum payload size (bytes) + */ + int WebRtcIsac_GetMinBytes( + RateModel* State, + int StreamSize, /* bytes in bitstream */ + const int FrameLen, /* ms per frame */ + const double BottleNeck, /* bottle neck rate; excl headers (bps) */ + const double DelayBuildUp, /* max delay from bottleneck buffering (ms) */ + enum ISACBandwidth bandwidth + /*,int16_t frequentLargePackets*/); + + /* + * update long-term average bitrate and amount of data in buffer + */ + void WebRtcIsac_UpdateRateModel( + RateModel* State, + int StreamSize, /* bytes in bitstream */ + const int FrameSamples, /* samples per frame */ + const double BottleNeck); /* bottle neck rate; excl headers (bps) */ + + + void WebRtcIsac_InitRateModel( + RateModel *State); + + /* Returns the new framelength value (input argument: bottle_neck) */ + int WebRtcIsac_GetNewFrameLength( + double bottle_neck, + int current_framelength); + + /* Returns the new SNR value (input argument: bottle_neck) */ + double WebRtcIsac_GetSnr( + double bottle_neck, + int new_framelength); + + + int16_t WebRtcIsac_UpdateUplinkJitter( + BwEstimatorstr* bwest_str, + int32_t index); + +#if defined(__cplusplus) +} +#endif + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_BANDWIDTH_ESTIMATOR_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/codec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/codec.h new file mode 100644 index 0000000000..d938725b92 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/codec.h @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * codec.h + * + * This header file contains the calls to the internal encoder + * and decoder functions. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_CODEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_CODEC_H_ + +#include "structs.h" + + +void WebRtcIsac_ResetBitstream(Bitstr* bit_stream); + +int WebRtcIsac_EstimateBandwidth(BwEstimatorstr* bwest_str, Bitstr* streamdata, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, uint32_t arr_ts, + enum IsacSamplingRate encoderSampRate, + enum IsacSamplingRate decoderSampRate); + +int WebRtcIsac_DecodeLb(const TransformTables* transform_tables, + float* signal_out, + ISACLBDecStruct* ISACdec_obj, + int16_t* current_framesamples, + int16_t isRCUPayload); + +int WebRtcIsac_DecodeRcuLb(float* signal_out, ISACLBDecStruct* ISACdec_obj, + int16_t* current_framesamples); + +int WebRtcIsac_EncodeLb(const TransformTables* transform_tables, + float* in, + ISACLBEncStruct* ISACencLB_obj, + int16_t codingMode, + int16_t bottleneckIndex); + +int WebRtcIsac_EncodeStoredDataLb(const IsacSaveEncoderData* ISACSavedEnc_obj, + Bitstr* ISACBitStr_obj, int BWnumber, + float scale); + +int WebRtcIsac_EncodeStoredDataUb( + const ISACUBSaveEncDataStruct* ISACSavedEnc_obj, Bitstr* bitStream, + int32_t jitterInfo, float scale, enum ISACBandwidth bandwidth); + +int16_t WebRtcIsac_GetRedPayloadUb( + const ISACUBSaveEncDataStruct* ISACSavedEncObj, Bitstr* bitStreamObj, + enum ISACBandwidth bandwidth); + +/****************************************************************************** + * WebRtcIsac_RateAllocation() + * Internal function to perform a rate-allocation for upper and lower-band, + * given a total rate. + * + * Input: + * - inRateBitPerSec : a total bit-rate in bits/sec. + * + * Output: + * - rateLBBitPerSec : a bit-rate allocated to the lower-band + * in bits/sec. + * - rateUBBitPerSec : a bit-rate allocated to the upper-band + * in bits/sec. + * + * Return value : 0 if rate allocation has been successful. + * -1 if failed to allocate rates. + */ + +int16_t WebRtcIsac_RateAllocation(int32_t inRateBitPerSec, + double* rateLBBitPerSec, + double* rateUBBitPerSec, + enum ISACBandwidth* bandwidthKHz); + + +/****************************************************************************** + * WebRtcIsac_DecodeUb16() + * + * Decode the upper-band if the codec is in 0-16 kHz mode. + * + * Input/Output: + * -ISACdec_obj : pointer to the upper-band decoder object. The + * bit-stream is stored inside the decoder object. + * + * Output: + * -signal_out : decoded audio, 480 samples 30 ms. + * + * Return value : >0 number of decoded bytes. + * <0 if an error occurred. + */ +int WebRtcIsac_DecodeUb16(const TransformTables* transform_tables, + float* signal_out, + ISACUBDecStruct* ISACdec_obj, + int16_t isRCUPayload); + +/****************************************************************************** + * WebRtcIsac_DecodeUb12() + * + * Decode the upper-band if the codec is in 0-12 kHz mode. + * + * Input/Output: + * -ISACdec_obj : pointer to the upper-band decoder object. The + * bit-stream is stored inside the decoder object. + * + * Output: + * -signal_out : decoded audio, 480 samples 30 ms. + * + * Return value : >0 number of decoded bytes. + * <0 if an error occurred. + */ +int WebRtcIsac_DecodeUb12(const TransformTables* transform_tables, + float* signal_out, + ISACUBDecStruct* ISACdec_obj, + int16_t isRCUPayload); + +/****************************************************************************** + * WebRtcIsac_EncodeUb16() + * + * Encode the upper-band if the codec is in 0-16 kHz mode. + * + * Input: + * -in : upper-band audio, 160 samples (10 ms). + * + * Input/Output: + * -ISACdec_obj : pointer to the upper-band encoder object. The + * bit-stream is stored inside the encoder object. + * + * Return value : >0 number of encoded bytes. + * <0 if an error occurred. + */ +int WebRtcIsac_EncodeUb16(const TransformTables* transform_tables, + float* in, + ISACUBEncStruct* ISACenc_obj, + int32_t jitterInfo); + +/****************************************************************************** + * WebRtcIsac_EncodeUb12() + * + * Encode the upper-band if the codec is in 0-12 kHz mode. + * + * Input: + * -in : upper-band audio, 160 samples (10 ms). + * + * Input/Output: + * -ISACdec_obj : pointer to the upper-band encoder object. The + * bit-stream is stored inside the encoder object. + * + * Return value : >0 number of encoded bytes. + * <0 if an error occurred. + */ +int WebRtcIsac_EncodeUb12(const TransformTables* transform_tables, + float* in, + ISACUBEncStruct* ISACenc_obj, + int32_t jitterInfo); + +/************************** initialization functions *************************/ + +void WebRtcIsac_InitMasking(MaskFiltstr* maskdata); + +void WebRtcIsac_InitPreFilterbank(PreFiltBankstr* prefiltdata); + +void WebRtcIsac_InitPostFilterbank(PostFiltBankstr* postfiltdata); + +void WebRtcIsac_InitPitchFilter(PitchFiltstr* pitchfiltdata); + +void WebRtcIsac_InitPitchAnalysis(PitchAnalysisStruct* State); + + +/**************************** transform functions ****************************/ + +void WebRtcIsac_InitTransform(TransformTables* tables); + +void WebRtcIsac_Time2Spec(const TransformTables* tables, + double* inre1, + double* inre2, + int16_t* outre, + int16_t* outim, + FFTstr* fftstr_obj); + +void WebRtcIsac_Spec2time(const TransformTables* tables, + double* inre, + double* inim, + double* outre1, + double* outre2, + FFTstr* fftstr_obj); + +/******************************* filter functions ****************************/ + +void WebRtcIsac_AllPoleFilter(double* InOut, double* Coef, size_t lengthInOut, + int orderCoef); + +void WebRtcIsac_AllZeroFilter(double* In, double* Coef, size_t lengthInOut, + int orderCoef, double* Out); + +void WebRtcIsac_ZeroPoleFilter(double* In, double* ZeroCoef, double* PoleCoef, + size_t lengthInOut, int orderCoef, double* Out); + + +/***************************** filterbank functions **************************/ + +void WebRtcIsac_SplitAndFilterFloat(float* in, float* LP, float* HP, + double* LP_la, double* HP_la, + PreFiltBankstr* prefiltdata); + + +void WebRtcIsac_FilterAndCombineFloat(float* InLP, float* InHP, float* Out, + PostFiltBankstr* postfiltdata); + + +/************************* normalized lattice filters ************************/ + +void WebRtcIsac_NormLatticeFilterMa(int orderCoef, float* stateF, float* stateG, + float* lat_in, double* filtcoeflo, + double* lat_out); + +void WebRtcIsac_NormLatticeFilterAr(int orderCoef, float* stateF, float* stateG, + double* lat_in, double* lo_filt_coef, + float* lat_out); + +void WebRtcIsac_Dir2Lat(double* a, int orderCoef, float* sth, float* cth); + +void WebRtcIsac_AutoCorr(double* r, const double* x, size_t N, size_t order); + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_CODEC_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.c new file mode 100644 index 0000000000..2419e24033 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "crc.h" +#include +#include "signal_processing_library.h" + +#define POLYNOMIAL 0x04c11db7L + + +static const uint32_t kCrcTable[256] = { + 0, 0x4c11db7, 0x9823b6e, 0xd4326d9, 0x130476dc, 0x17c56b6b, + 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, + 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, + 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, + 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, + 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0, + 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x18aeb13, 0x54bf6a4, + 0x808d07d, 0xcc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, + 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, + 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, + 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, + 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1, + 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, + 0x3f9b762c, 0x3b5a6b9b, 0x315d626, 0x7d4cb91, 0xa97ed48, 0xe56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9, + 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, + 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, + 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, + 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e, + 0x18197087, 0x1cd86d30, 0x29f3d35, 0x65e2082, 0xb1d065b, 0xfdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, + 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, + 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, + 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 +}; + + + + +/**************************************************************************** + * WebRtcIsac_GetCrc(...) + * + * This function returns a 32 bit CRC checksum of a bit stream + * + * Input: + * - bitstream : payload bitstream + * - len_bitstream_in_bytes : number of 8-bit words in the bit stream + * + * Output: + * - crc : checksum + * + * Return value : 0 - Ok + * -1 - Error + */ + +int WebRtcIsac_GetCrc(const int16_t* bitstream, + int len_bitstream_in_bytes, + uint32_t* crc) +{ + uint8_t* bitstream_ptr_uw8; + uint32_t crc_state; + int byte_cntr; + int crc_tbl_indx; + + /* Sanity Check. */ + if (bitstream == NULL) { + return -1; + } + /* cast to UWord8 pointer */ + bitstream_ptr_uw8 = (uint8_t *)bitstream; + + /* initialize */ + crc_state = 0xFFFFFFFF; + + for (byte_cntr = 0; byte_cntr < len_bitstream_in_bytes; byte_cntr++) { + crc_tbl_indx = (WEBRTC_SPL_RSHIFT_U32(crc_state, 24) ^ + bitstream_ptr_uw8[byte_cntr]) & 0xFF; + crc_state = (crc_state << 8) ^ kCrcTable[crc_tbl_indx]; + } + + *crc = ~crc_state; + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.h new file mode 100644 index 0000000000..b3197a1a7d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * crc.h + * + * Checksum functions + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_CRC_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_CRC_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/**************************************************************************** + * WebRtcIsac_GetCrc(...) + * + * This function returns a 32 bit CRC checksum of a bit stream + * + * Input: + * - encoded : payload bit stream + * - no_of_word8s : number of 8-bit words in the bit stream + * + * Output: + * - crc : checksum + * + * Return value : 0 - Ok + * -1 - Error + */ + +int WebRtcIsac_GetCrc( + const int16_t* encoded, + int no_of_word8s, + uint32_t* crc); + + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_CRC_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode.c new file mode 100644 index 0000000000..e925efba6e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode.c @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * decode_B.c + * + * This file contains definition of funtions for decoding. + * Decoding of lower-band, including normal-decoding and RCU decoding. + * Decoding of upper-band, including 8-12 kHz, when the bandwidth is + * 0-12 kHz, and 8-16 kHz, when the bandwidth is 0-16 kHz. + * + */ + + +#include "codec.h" +#include "entropy_coding.h" +#include "pitch_estimator.h" +#include "bandwidth_estimator.h" +#include "structs.h" +#include "settings.h" + +#include +#include +#include + + +/* + * function to decode the bitstream + * returns the total number of bytes in the stream + */ +int WebRtcIsac_DecodeLb(const TransformTables* transform_tables, + float* signal_out, ISACLBDecStruct* ISACdecLB_obj, + int16_t* current_framesamples, + int16_t isRCUPayload) { + int k; + int len, err; + int16_t bandwidthInd; + + float LP_dec_float[FRAMESAMPLES_HALF]; + float HP_dec_float[FRAMESAMPLES_HALF]; + + double LPw[FRAMESAMPLES_HALF]; + double HPw[FRAMESAMPLES_HALF]; + double LPw_pf[FRAMESAMPLES_HALF]; + + double lo_filt_coef[(ORDERLO + 1)*SUBFRAMES]; + double hi_filt_coef[(ORDERHI + 1)*SUBFRAMES]; + + double real_f[FRAMESAMPLES_HALF]; + double imag_f[FRAMESAMPLES_HALF]; + + double PitchLags[4]; + double PitchGains[4]; + double AvgPitchGain; + int16_t PitchGains_Q12[4]; + int16_t AvgPitchGain_Q12; + + float gain; + + int frame_nb; /* counter */ + int frame_mode; /* 0 30ms, 1 for 60ms */ + /* Processed_samples: 480 (30, 60 ms). Cannot take other values. */ + + WebRtcIsac_ResetBitstream(&(ISACdecLB_obj->bitstr_obj)); + + len = 0; + + /* Decode framelength and BW estimation - not used, + only for stream pointer*/ + err = WebRtcIsac_DecodeFrameLen(&ISACdecLB_obj->bitstr_obj, + current_framesamples); + if (err < 0) { + return err; + } + + /* Frame_mode: + * 0: indicates 30 ms frame (480 samples) + * 1: indicates 60 ms frame (960 samples) */ + frame_mode = *current_framesamples / MAX_FRAMESAMPLES; + + err = WebRtcIsac_DecodeSendBW(&ISACdecLB_obj->bitstr_obj, &bandwidthInd); + if (err < 0) { + return err; + } + + /* One loop if it's one frame (20 or 30ms), 2 loops if 2 frames + bundled together (60ms). */ + for (frame_nb = 0; frame_nb <= frame_mode; frame_nb++) { + /* Decode & de-quantize pitch parameters */ + err = WebRtcIsac_DecodePitchGain(&ISACdecLB_obj->bitstr_obj, + PitchGains_Q12); + if (err < 0) { + return err; + } + + err = WebRtcIsac_DecodePitchLag(&ISACdecLB_obj->bitstr_obj, PitchGains_Q12, + PitchLags); + if (err < 0) { + return err; + } + + AvgPitchGain_Q12 = (PitchGains_Q12[0] + PitchGains_Q12[1] + + PitchGains_Q12[2] + PitchGains_Q12[3]) >> 2; + + /* Decode & de-quantize filter coefficients. */ + err = WebRtcIsac_DecodeLpc(&ISACdecLB_obj->bitstr_obj, lo_filt_coef, + hi_filt_coef); + if (err < 0) { + return err; + } + /* Decode & de-quantize spectrum. */ + len = WebRtcIsac_DecodeSpec(&ISACdecLB_obj->bitstr_obj, AvgPitchGain_Q12, + kIsacLowerBand, real_f, imag_f); + if (len < 0) { + return len; + } + + /* Inverse transform. */ + WebRtcIsac_Spec2time(transform_tables, real_f, imag_f, LPw, HPw, + &ISACdecLB_obj->fftstr_obj); + + /* Convert PitchGains back to float for pitchfilter_post */ + for (k = 0; k < 4; k++) { + PitchGains[k] = ((float)PitchGains_Q12[k]) / 4096; + } + if (isRCUPayload) { + for (k = 0; k < 240; k++) { + LPw[k] *= RCU_TRANSCODING_SCALE_INVERSE; + HPw[k] *= RCU_TRANSCODING_SCALE_INVERSE; + } + } + + /* Inverse pitch filter. */ + WebRtcIsac_PitchfilterPost(LPw, LPw_pf, &ISACdecLB_obj->pitchfiltstr_obj, + PitchLags, PitchGains); + /* Convert AvgPitchGain back to float for computation of gain. */ + AvgPitchGain = ((float)AvgPitchGain_Q12) / 4096; + gain = 1.0f - 0.45f * (float)AvgPitchGain; + + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + /* Reduce gain to compensate for pitch enhancer. */ + LPw_pf[k] *= gain; + } + + if (isRCUPayload) { + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + /* Compensation for transcoding gain changes. */ + LPw_pf[k] *= RCU_TRANSCODING_SCALE; + HPw[k] *= RCU_TRANSCODING_SCALE; + } + } + /* Perceptual post-filtering (using normalized lattice filter). */ + WebRtcIsac_NormLatticeFilterAr( + ORDERLO, ISACdecLB_obj->maskfiltstr_obj.PostStateLoF, + (ISACdecLB_obj->maskfiltstr_obj).PostStateLoG, LPw_pf, lo_filt_coef, + LP_dec_float); + WebRtcIsac_NormLatticeFilterAr( + ORDERHI, ISACdecLB_obj->maskfiltstr_obj.PostStateHiF, + (ISACdecLB_obj->maskfiltstr_obj).PostStateHiG, HPw, hi_filt_coef, + HP_dec_float); + + /* Recombine the 2 bands. */ + WebRtcIsac_FilterAndCombineFloat(LP_dec_float, HP_dec_float, + signal_out + frame_nb * FRAMESAMPLES, + &ISACdecLB_obj->postfiltbankstr_obj); + } + return len; +} + + +/* + * This decode function is called when the codec is operating in 16 kHz + * bandwidth to decode the upperband, i.e. 8-16 kHz. + * + * Contrary to lower-band, the upper-band (8-16 kHz) is not split in + * frequency, but split to 12 sub-frames, i.e. twice as lower-band. + */ +int WebRtcIsac_DecodeUb16(const TransformTables* transform_tables, + float* signal_out, ISACUBDecStruct* ISACdecUB_obj, + int16_t isRCUPayload) { + int len, err; + + double halfFrameFirst[FRAMESAMPLES_HALF]; + double halfFrameSecond[FRAMESAMPLES_HALF]; + + double percepFilterParam[(UB_LPC_ORDER + 1) * (SUBFRAMES << 1) + + (UB_LPC_ORDER + 1)]; + + double real_f[FRAMESAMPLES_HALF]; + double imag_f[FRAMESAMPLES_HALF]; + const int16_t kAveragePitchGain = 0; /* No pitch-gain for upper-band. */ + len = 0; + + /* Decode & de-quantize filter coefficients. */ + memset(percepFilterParam, 0, sizeof(percepFilterParam)); + err = WebRtcIsac_DecodeInterpolLpcUb(&ISACdecUB_obj->bitstr_obj, + percepFilterParam, isac16kHz); + if (err < 0) { + return err; + } + + /* Decode & de-quantize spectrum. */ + len = WebRtcIsac_DecodeSpec(&ISACdecUB_obj->bitstr_obj, kAveragePitchGain, + kIsacUpperBand16, real_f, imag_f); + if (len < 0) { + return len; + } + if (isRCUPayload) { + int n; + for (n = 0; n < 240; n++) { + real_f[n] *= RCU_TRANSCODING_SCALE_UB_INVERSE; + imag_f[n] *= RCU_TRANSCODING_SCALE_UB_INVERSE; + } + } + /* Inverse transform. */ + WebRtcIsac_Spec2time(transform_tables, + real_f, imag_f, halfFrameFirst, halfFrameSecond, + &ISACdecUB_obj->fftstr_obj); + + /* Perceptual post-filtering (using normalized lattice filter). */ + WebRtcIsac_NormLatticeFilterAr( + UB_LPC_ORDER, ISACdecUB_obj->maskfiltstr_obj.PostStateLoF, + (ISACdecUB_obj->maskfiltstr_obj).PostStateLoG, halfFrameFirst, + &percepFilterParam[(UB_LPC_ORDER + 1)], signal_out); + + WebRtcIsac_NormLatticeFilterAr( + UB_LPC_ORDER, ISACdecUB_obj->maskfiltstr_obj.PostStateLoF, + (ISACdecUB_obj->maskfiltstr_obj).PostStateLoG, halfFrameSecond, + &percepFilterParam[(UB_LPC_ORDER + 1) * SUBFRAMES + (UB_LPC_ORDER + 1)], + &signal_out[FRAMESAMPLES_HALF]); + + return len; +} + +/* + * This decode function is called when the codec operates at 0-12 kHz + * bandwidth to decode the upperband, i.e. 8-12 kHz. + * + * At the encoder the upper-band is split into two band, 8-12 kHz & 12-16 + * kHz, and only 8-12 kHz is encoded. At the decoder, 8-12 kHz band is + * reconstructed and 12-16 kHz replaced with zeros. Then two bands + * are combined, to reconstruct the upperband 8-16 kHz. + */ +int WebRtcIsac_DecodeUb12(const TransformTables* transform_tables, + float* signal_out, ISACUBDecStruct* ISACdecUB_obj, + int16_t isRCUPayload) { + int len, err; + + float LP_dec_float[FRAMESAMPLES_HALF]; + float HP_dec_float[FRAMESAMPLES_HALF]; + + double LPw[FRAMESAMPLES_HALF]; + double HPw[FRAMESAMPLES_HALF]; + + double percepFilterParam[(UB_LPC_ORDER + 1)*SUBFRAMES]; + + double real_f[FRAMESAMPLES_HALF]; + double imag_f[FRAMESAMPLES_HALF]; + const int16_t kAveragePitchGain = 0; /* No pitch-gain for upper-band. */ + len = 0; + + /* Decode & dequantize filter coefficients. */ + err = WebRtcIsac_DecodeInterpolLpcUb(&ISACdecUB_obj->bitstr_obj, + percepFilterParam, isac12kHz); + if (err < 0) { + return err; + } + + /* Decode & de-quantize spectrum. */ + len = WebRtcIsac_DecodeSpec(&ISACdecUB_obj->bitstr_obj, kAveragePitchGain, + kIsacUpperBand12, real_f, imag_f); + if (len < 0) { + return len; + } + + if (isRCUPayload) { + int n; + for (n = 0; n < 240; n++) { + real_f[n] *= RCU_TRANSCODING_SCALE_UB_INVERSE; + imag_f[n] *= RCU_TRANSCODING_SCALE_UB_INVERSE; + } + } + /* Inverse transform. */ + WebRtcIsac_Spec2time(transform_tables, + real_f, imag_f, LPw, HPw, &ISACdecUB_obj->fftstr_obj); + /* perceptual post-filtering (using normalized lattice filter) */ + WebRtcIsac_NormLatticeFilterAr(UB_LPC_ORDER, + ISACdecUB_obj->maskfiltstr_obj.PostStateLoF, + (ISACdecUB_obj->maskfiltstr_obj).PostStateLoG, + LPw, percepFilterParam, LP_dec_float); + /* Zero for 12-16 kHz. */ + memset(HP_dec_float, 0, sizeof(float) * (FRAMESAMPLES_HALF)); + /* Recombine the 2 bands. */ + WebRtcIsac_FilterAndCombineFloat(HP_dec_float, LP_dec_float, signal_out, + &ISACdecUB_obj->postfiltbankstr_obj); + return len; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode_bwe.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode_bwe.c new file mode 100644 index 0000000000..019cc89528 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode_bwe.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "structs.h" +#include "bandwidth_estimator.h" +#include "entropy_coding.h" +#include "codec.h" + + +int +WebRtcIsac_EstimateBandwidth( + BwEstimatorstr* bwest_str, + Bitstr* streamdata, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts, + enum IsacSamplingRate encoderSampRate, + enum IsacSamplingRate decoderSampRate) +{ + int16_t index; + int16_t frame_samples; + uint32_t sendTimestampIn16kHz; + uint32_t arrivalTimestampIn16kHz; + uint32_t diffSendTime; + uint32_t diffArrivalTime; + int err; + + /* decode framelength and BW estimation */ + err = WebRtcIsac_DecodeFrameLen(streamdata, &frame_samples); + if(err < 0) // error check + { + return err; + } + err = WebRtcIsac_DecodeSendBW(streamdata, &index); + if(err < 0) // error check + { + return err; + } + + /* UPDATE ESTIMATES FROM OTHER SIDE */ + err = WebRtcIsac_UpdateUplinkBwImpl(bwest_str, index, encoderSampRate); + if(err < 0) + { + return err; + } + + // We like BWE to work at 16 kHz sampling rate, + // therefore, we have to change the timestamps accordingly. + // translate the send timestamp if required + diffSendTime = (uint32_t)((uint32_t)send_ts - + (uint32_t)bwest_str->senderTimestamp); + bwest_str->senderTimestamp = send_ts; + + diffArrivalTime = (uint32_t)((uint32_t)arr_ts - + (uint32_t)bwest_str->receiverTimestamp); + bwest_str->receiverTimestamp = arr_ts; + + if(decoderSampRate == kIsacSuperWideband) + { + diffArrivalTime = (uint32_t)diffArrivalTime >> 1; + diffSendTime = (uint32_t)diffSendTime >> 1; + } + + // arrival timestamp in 16 kHz + arrivalTimestampIn16kHz = (uint32_t)((uint32_t) + bwest_str->prev_rec_arr_ts + (uint32_t)diffArrivalTime); + // send timestamp in 16 kHz + sendTimestampIn16kHz = (uint32_t)((uint32_t) + bwest_str->prev_rec_send_ts + (uint32_t)diffSendTime); + + err = WebRtcIsac_UpdateBandwidthEstimator(bwest_str, rtp_seq_number, + (frame_samples * 1000) / FS, sendTimestampIn16kHz, + arrivalTimestampIn16kHz, packet_size); + // error check + if(err < 0) + { + return err; + } + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode.c new file mode 100644 index 0000000000..3f1912b6d3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode.c @@ -0,0 +1,1258 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * encode.c + * + * This file contains definition of funtions for encoding. + * Decoding of upper-band, including 8-12 kHz, when the bandwidth is + * 0-12 kHz, and 8-16 kHz, when the bandwidth is 0-16 kHz. + * + */ + +#include +#include +#include + +#include "structs.h" +#include "codec.h" +#include "pitch_estimator.h" +#include "entropy_coding.h" +#include "arith_routines.h" +#include "pitch_gain_tables.h" +#include "pitch_lag_tables.h" +#include "spectrum_ar_model_tables.h" +#include "lpc_tables.h" +#include "lpc_analysis.h" +#include "bandwidth_estimator.h" +#include "lpc_shape_swb12_tables.h" +#include "lpc_shape_swb16_tables.h" +#include "lpc_gain_swb_tables.h" + + +#define UB_LOOKAHEAD 24 + + +/* + Rate allocation tables of lower and upper-band bottleneck for + 12kHz & 16kHz bandwidth. + + 12 kHz bandwidth + ----------------- + The overall bottleneck of the coder is between 38 kbps and 45 kbps. We have + considered 7 enteries, uniformly distributed in this interval, i.e. 38, + 39.17, 40.33, 41.5, 42.67, 43.83 and 45. For every entery, the lower-band + and the upper-band bottlenecks are specified in + 'kLowerBandBitRate12' and 'kUpperBandBitRate12' + tables, respectively. E.g. the overall rate of 41.5 kbps corresponts to a + bottleneck of 31 kbps for lower-band and 27 kbps for upper-band. Given an + overall bottleneck of the codec, we use linear interpolation to get + lower-band and upper-band bottlenecks. + + 16 kHz bandwidth + ----------------- + The overall bottleneck of the coder is between 50 kbps and 56 kbps. We have + considered 7 enteries, uniformly distributed in this interval, i.e. 50, 51.2, + 52.4, 53.6, 54.8 and 56. For every entery, the lower-band and the upper-band + bottlenecks are specified in 'kLowerBandBitRate16' and + 'kUpperBandBitRate16' tables, respectively. E.g. the overall rate + of 53.6 kbps corresponts to a bottleneck of 32 kbps for lower-band and 30 + kbps for upper-band. Given an overall bottleneck of the codec, we use linear + interpolation to get lower-band and upper-band bottlenecks. + + */ + +/* 38 39.17 40.33 41.5 42.67 43.83 45 */ +static const int16_t kLowerBandBitRate12[7] = { + 29000, 30000, 30000, 31000, 31000, 32000, 32000 }; +static const int16_t kUpperBandBitRate12[7] = { + 25000, 25000, 27000, 27000, 29000, 29000, 32000 }; + +/* 50 51.2 52.4 53.6 54.8 56 */ +static const int16_t kLowerBandBitRate16[6] = { + 31000, 31000, 32000, 32000, 32000, 32000 }; +static const int16_t kUpperBandBitRate16[6] = { + 28000, 29000, 29000, 30000, 31000, 32000 }; + +/****************************************************************************** + * WebRtcIsac_RateAllocation() + * Internal function to perform a rate-allocation for upper and lower-band, + * given a total rate. + * + * Input: + * - inRateBitPerSec : a total bottleneck in bits/sec. + * + * Output: + * - rateLBBitPerSec : a bottleneck allocated to the lower-band + * in bits/sec. + * - rateUBBitPerSec : a bottleneck allocated to the upper-band + * in bits/sec. + * + * Return value : 0 if rate allocation has been successful. + * -1 if failed to allocate rates. + */ + +int16_t WebRtcIsac_RateAllocation(int32_t inRateBitPerSec, + double* rateLBBitPerSec, + double* rateUBBitPerSec, + enum ISACBandwidth* bandwidthKHz) { + int16_t idx; + double idxD; + double idxErr; + if (inRateBitPerSec < 38000) { + /* If the given overall bottleneck is less than 38000 then + * then codec has to operate in wideband mode, i.e. 8 kHz + * bandwidth. */ + *rateLBBitPerSec = (int16_t)((inRateBitPerSec > 32000) ? + 32000 : inRateBitPerSec); + *rateUBBitPerSec = 0; + *bandwidthKHz = isac8kHz; + } else if ((inRateBitPerSec >= 38000) && (inRateBitPerSec < 50000)) { + /* At a bottleneck between 38 and 50 kbps the codec is operating + * at 12 kHz bandwidth. Using xxxBandBitRate12[] to calculates + * upper/lower bottleneck */ + + /* Find the bottlenecks by linear interpolation, + * step is (45000 - 38000)/6.0 we use the inverse of it. */ + const double stepSizeInv = 8.5714286e-4; + idxD = (inRateBitPerSec - 38000) * stepSizeInv; + idx = (idxD >= 6) ? 6 : ((int16_t)idxD); + idxErr = idxD - idx; + *rateLBBitPerSec = kLowerBandBitRate12[idx]; + *rateUBBitPerSec = kUpperBandBitRate12[idx]; + + if (idx < 6) { + *rateLBBitPerSec += (int16_t)( + idxErr * (kLowerBandBitRate12[idx + 1] - kLowerBandBitRate12[idx])); + *rateUBBitPerSec += (int16_t)( + idxErr * (kUpperBandBitRate12[idx + 1] - kUpperBandBitRate12[idx])); + } + *bandwidthKHz = isac12kHz; + } else if ((inRateBitPerSec >= 50000) && (inRateBitPerSec <= 56000)) { + /* A bottleneck between 50 and 56 kbps corresponds to bandwidth + * of 16 kHz. Using xxxBandBitRate16[] to calculates + * upper/lower bottleneck. */ + + /* Find the bottlenecks by linear interpolation + * step is (56000 - 50000)/5 we use the inverse of it. */ + const double stepSizeInv = 8.3333333e-4; + idxD = (inRateBitPerSec - 50000) * stepSizeInv; + idx = (idxD >= 5) ? 5 : ((int16_t)idxD); + idxErr = idxD - idx; + *rateLBBitPerSec = kLowerBandBitRate16[idx]; + *rateUBBitPerSec = kUpperBandBitRate16[idx]; + + if (idx < 5) { + *rateLBBitPerSec += (int16_t)(idxErr * + (kLowerBandBitRate16[idx + 1] - + kLowerBandBitRate16[idx])); + + *rateUBBitPerSec += (int16_t)(idxErr * + (kUpperBandBitRate16[idx + 1] - + kUpperBandBitRate16[idx])); + } + *bandwidthKHz = isac16kHz; + } else { + /* Out-of-range botlteneck value. */ + return -1; + } + + /* limit the values. */ + *rateLBBitPerSec = (*rateLBBitPerSec > 32000) ? 32000 : *rateLBBitPerSec; + *rateUBBitPerSec = (*rateUBBitPerSec > 32000) ? 32000 : *rateUBBitPerSec; + return 0; +} + + +void WebRtcIsac_ResetBitstream(Bitstr* bit_stream) { + bit_stream->W_upper = 0xFFFFFFFF; + bit_stream->stream_index = 0; + bit_stream->streamval = 0; +} + +int WebRtcIsac_EncodeLb(const TransformTables* transform_tables, + float* in, ISACLBEncStruct* ISACencLB_obj, + int16_t codingMode, + int16_t bottleneckIndex) { + int stream_length = 0; + int err; + int k; + int iterCntr; + + double lofilt_coef[(ORDERLO + 1)*SUBFRAMES]; + double hifilt_coef[(ORDERHI + 1)*SUBFRAMES]; + float LP[FRAMESAMPLES_HALF]; + float HP[FRAMESAMPLES_HALF]; + + double LP_lookahead[FRAMESAMPLES_HALF]; + double HP_lookahead[FRAMESAMPLES_HALF]; + double LP_lookahead_pf[FRAMESAMPLES_HALF + QLOOKAHEAD]; + double LPw[FRAMESAMPLES_HALF]; + + double HPw[FRAMESAMPLES_HALF]; + double LPw_pf[FRAMESAMPLES_HALF]; + int16_t fre[FRAMESAMPLES_HALF]; /* Q7 */ + int16_t fim[FRAMESAMPLES_HALF]; /* Q7 */ + + double PitchLags[4]; + double PitchGains[4]; + int16_t PitchGains_Q12[4]; + int16_t AvgPitchGain_Q12; + + int frame_mode; /* 0 for 30ms, 1 for 60ms */ + int status = 0; + int my_index; + transcode_obj transcodingParam; + double bytesLeftSpecCoding; + uint16_t payloadLimitBytes; + + /* Copy new frame-length and bottleneck rate only for the first 10 ms data */ + if (ISACencLB_obj->buffer_index == 0) { + /* Set the framelength for the next packet. */ + ISACencLB_obj->current_framesamples = ISACencLB_obj->new_framelength; + } + /* 'frame_mode' is 0 (30 ms) or 1 (60 ms). */ + frame_mode = ISACencLB_obj->current_framesamples / MAX_FRAMESAMPLES; + + /* buffer speech samples (by 10ms packet) until the frame-length */ + /* is reached (30 or 60 ms). */ + /*****************************************************************/ + + /* fill the buffer with 10ms input data */ + for (k = 0; k < FRAMESAMPLES_10ms; k++) { + ISACencLB_obj->data_buffer_float[k + ISACencLB_obj->buffer_index] = in[k]; + } + + /* If buffersize is not equal to current framesize then increase index + * and return. We do no encoding untill we have enough audio. */ + if (ISACencLB_obj->buffer_index + FRAMESAMPLES_10ms != FRAMESAMPLES) { + ISACencLB_obj->buffer_index += FRAMESAMPLES_10ms; + return 0; + } + /* If buffer reached the right size, reset index and continue with + * encoding the frame. */ + ISACencLB_obj->buffer_index = 0; + + /* End of buffer function. */ + /**************************/ + + /* Encoding */ + /************/ + + if (frame_mode == 0 || ISACencLB_obj->frame_nb == 0) { + /* This is to avoid Linux warnings until we change 'int' to 'Word32' + * at all places. */ + int intVar; + /* reset bitstream */ + WebRtcIsac_ResetBitstream(&(ISACencLB_obj->bitstr_obj)); + + if ((codingMode == 0) && (frame_mode == 0) && + (ISACencLB_obj->enforceFrameSize == 0)) { + ISACencLB_obj->new_framelength = WebRtcIsac_GetNewFrameLength( + ISACencLB_obj->bottleneck, ISACencLB_obj->current_framesamples); + } + + ISACencLB_obj->s2nr = WebRtcIsac_GetSnr( + ISACencLB_obj->bottleneck, ISACencLB_obj->current_framesamples); + + /* Encode frame length. */ + status = WebRtcIsac_EncodeFrameLen( + ISACencLB_obj->current_framesamples, &ISACencLB_obj->bitstr_obj); + if (status < 0) { + /* Wrong frame size. */ + return status; + } + /* Save framelength for multiple packets memory. */ + ISACencLB_obj->SaveEnc_obj.framelength = + ISACencLB_obj->current_framesamples; + + /* To be used for Redundant Coding. */ + ISACencLB_obj->lastBWIdx = bottleneckIndex; + intVar = (int)bottleneckIndex; + WebRtcIsac_EncodeReceiveBw(&intVar, &ISACencLB_obj->bitstr_obj); + } + + /* Split signal in two bands. */ + WebRtcIsac_SplitAndFilterFloat(ISACencLB_obj->data_buffer_float, LP, HP, + LP_lookahead, HP_lookahead, + &ISACencLB_obj->prefiltbankstr_obj); + + /* estimate pitch parameters and pitch-filter lookahead signal */ + WebRtcIsac_PitchAnalysis(LP_lookahead, LP_lookahead_pf, + &ISACencLB_obj->pitchanalysisstr_obj, PitchLags, + PitchGains); + + /* Encode in FIX Q12. */ + + /* Convert PitchGain to Fixed point. */ + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchGains_Q12[k] = (int16_t)(PitchGains[k] * 4096.0); + } + + /* Set where to store data in multiple packets memory. */ + if (frame_mode == 0 || ISACencLB_obj->frame_nb == 0) { + ISACencLB_obj->SaveEnc_obj.startIdx = 0; + } else { + ISACencLB_obj->SaveEnc_obj.startIdx = 1; + } + + /* Quantize & encode pitch parameters. */ + WebRtcIsac_EncodePitchGain(PitchGains_Q12, &ISACencLB_obj->bitstr_obj, + &ISACencLB_obj->SaveEnc_obj); + WebRtcIsac_EncodePitchLag(PitchLags, PitchGains_Q12, + &ISACencLB_obj->bitstr_obj, + &ISACencLB_obj->SaveEnc_obj); + + AvgPitchGain_Q12 = (PitchGains_Q12[0] + PitchGains_Q12[1] + + PitchGains_Q12[2] + PitchGains_Q12[3]) >> 2; + + /* Find coefficients for perceptual pre-filters. */ + WebRtcIsac_GetLpcCoefLb(LP_lookahead_pf, HP_lookahead, + &ISACencLB_obj->maskfiltstr_obj, ISACencLB_obj->s2nr, + PitchGains_Q12, lofilt_coef, hifilt_coef); + + /* Code LPC model and shape - gains not quantized yet. */ + WebRtcIsac_EncodeLpcLb(lofilt_coef, hifilt_coef, &ISACencLB_obj->bitstr_obj, + &ISACencLB_obj->SaveEnc_obj); + + /* Convert PitchGains back to FLOAT for pitchfilter_pre. */ + for (k = 0; k < 4; k++) { + PitchGains[k] = ((float)PitchGains_Q12[k]) / 4096; + } + + /* Store the state of arithmetic coder before coding LPC gains. */ + transcodingParam.W_upper = ISACencLB_obj->bitstr_obj.W_upper; + transcodingParam.stream_index = ISACencLB_obj->bitstr_obj.stream_index; + transcodingParam.streamval = ISACencLB_obj->bitstr_obj.streamval; + transcodingParam.stream[0] = + ISACencLB_obj->bitstr_obj.stream[ISACencLB_obj->bitstr_obj.stream_index - + 2]; + transcodingParam.stream[1] = + ISACencLB_obj->bitstr_obj.stream[ISACencLB_obj->bitstr_obj.stream_index - + 1]; + transcodingParam.stream[2] = + ISACencLB_obj->bitstr_obj.stream[ISACencLB_obj->bitstr_obj.stream_index]; + + /* Store LPC Gains before encoding them. */ + for (k = 0; k < SUBFRAMES; k++) { + transcodingParam.loFiltGain[k] = lofilt_coef[(LPC_LOBAND_ORDER + 1) * k]; + transcodingParam.hiFiltGain[k] = hifilt_coef[(LPC_HIBAND_ORDER + 1) * k]; + } + + /* Code gains */ + WebRtcIsac_EncodeLpcGainLb(lofilt_coef, hifilt_coef, + &ISACencLB_obj->bitstr_obj, + &ISACencLB_obj->SaveEnc_obj); + + /* Get the correct value for the payload limit and calculate the + * number of bytes left for coding the spectrum. */ + if ((frame_mode == 1) && (ISACencLB_obj->frame_nb == 0)) { + /* It is a 60ms and we are in the first 30ms then the limit at + * this point should be half of the assigned value. */ + payloadLimitBytes = ISACencLB_obj->payloadLimitBytes60 >> 1; + } else if (frame_mode == 0) { + /* It is a 30ms frame */ + /* Subract 3 because termination process may add 3 bytes. */ + payloadLimitBytes = ISACencLB_obj->payloadLimitBytes30 - 3; + } else { + /* This is the second half of a 60ms frame. */ + /* Subract 3 because termination process may add 3 bytes. */ + payloadLimitBytes = ISACencLB_obj->payloadLimitBytes60 - 3; + } + bytesLeftSpecCoding = payloadLimitBytes - transcodingParam.stream_index; + + /* Perceptual pre-filtering (using normalized lattice filter). */ + /* Low-band filtering. */ + WebRtcIsac_NormLatticeFilterMa(ORDERLO, + ISACencLB_obj->maskfiltstr_obj.PreStateLoF, + ISACencLB_obj->maskfiltstr_obj.PreStateLoG, + LP, lofilt_coef, LPw); + /* High-band filtering. */ + WebRtcIsac_NormLatticeFilterMa(ORDERHI, + ISACencLB_obj->maskfiltstr_obj.PreStateHiF, + ISACencLB_obj->maskfiltstr_obj.PreStateHiG, + HP, hifilt_coef, HPw); + /* Pitch filter. */ + WebRtcIsac_PitchfilterPre(LPw, LPw_pf, &ISACencLB_obj->pitchfiltstr_obj, + PitchLags, PitchGains); + /* Transform */ + WebRtcIsac_Time2Spec(transform_tables, + LPw_pf, HPw, fre, fim, &ISACencLB_obj->fftstr_obj); + + /* Save data for multiple packets memory. */ + my_index = ISACencLB_obj->SaveEnc_obj.startIdx * FRAMESAMPLES_HALF; + memcpy(&ISACencLB_obj->SaveEnc_obj.fre[my_index], fre, sizeof(fre)); + memcpy(&ISACencLB_obj->SaveEnc_obj.fim[my_index], fim, sizeof(fim)); + + ISACencLB_obj->SaveEnc_obj.AvgPitchGain[ISACencLB_obj->SaveEnc_obj.startIdx] = + AvgPitchGain_Q12; + + /* Quantization and loss-less coding. */ + err = WebRtcIsac_EncodeSpec(fre, fim, AvgPitchGain_Q12, kIsacLowerBand, + &ISACencLB_obj->bitstr_obj); + if ((err < 0) && (err != -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + /* There has been an error but it was not too large payload + (we can cure too large payload). */ + if (frame_mode == 1 && ISACencLB_obj->frame_nb == 1) { + /* If this is the second 30ms of a 60ms frame reset + this such that in the next call encoder starts fresh. */ + ISACencLB_obj->frame_nb = 0; + } + return err; + } + iterCntr = 0; + while ((ISACencLB_obj->bitstr_obj.stream_index > payloadLimitBytes) || + (err == -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + double bytesSpecCoderUsed; + double transcodeScale; + + if (iterCntr >= MAX_PAYLOAD_LIMIT_ITERATION) { + /* We were not able to limit the payload size */ + if ((frame_mode == 1) && (ISACencLB_obj->frame_nb == 0)) { + /* This was the first 30ms of a 60ms frame. Although + the payload is larger than it should be but we let + the second 30ms be encoded. Maybe together we + won't exceed the limit. */ + ISACencLB_obj->frame_nb = 1; + return 0; + } else if ((frame_mode == 1) && (ISACencLB_obj->frame_nb == 1)) { + ISACencLB_obj->frame_nb = 0; + } + + if (err != -ISAC_DISALLOWED_BITSTREAM_LENGTH) { + return -ISAC_PAYLOAD_LARGER_THAN_LIMIT; + } else { + return status; + } + } + + if (err == -ISAC_DISALLOWED_BITSTREAM_LENGTH) { + bytesSpecCoderUsed = STREAM_SIZE_MAX; + /* Being conservative */ + transcodeScale = bytesLeftSpecCoding / bytesSpecCoderUsed * 0.5; + } else { + bytesSpecCoderUsed = ISACencLB_obj->bitstr_obj.stream_index - + transcodingParam.stream_index; + transcodeScale = bytesLeftSpecCoding / bytesSpecCoderUsed; + } + + /* To be safe, we reduce the scale depending on + the number of iterations. */ + transcodeScale *= (1.0 - (0.9 * (double)iterCntr / + (double)MAX_PAYLOAD_LIMIT_ITERATION)); + + /* Scale the LPC Gains. */ + for (k = 0; k < SUBFRAMES; k++) { + lofilt_coef[(LPC_LOBAND_ORDER + 1) * k] = + transcodingParam.loFiltGain[k] * transcodeScale; + hifilt_coef[(LPC_HIBAND_ORDER + 1) * k] = + transcodingParam.hiFiltGain[k] * transcodeScale; + transcodingParam.loFiltGain[k] = lofilt_coef[(LPC_LOBAND_ORDER + 1) * k]; + transcodingParam.hiFiltGain[k] = hifilt_coef[(LPC_HIBAND_ORDER + 1) * k]; + } + + /* Scale DFT coefficients. */ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + fre[k] = (int16_t)(fre[k] * transcodeScale); + fim[k] = (int16_t)(fim[k] * transcodeScale); + } + + /* Save data for multiple packets memory. */ + my_index = ISACencLB_obj->SaveEnc_obj.startIdx * FRAMESAMPLES_HALF; + memcpy(&ISACencLB_obj->SaveEnc_obj.fre[my_index], fre, sizeof(fre)); + memcpy(&ISACencLB_obj->SaveEnc_obj.fim[my_index], fim, sizeof(fim)); + + /* Re-store the state of arithmetic coder before coding LPC gains. */ + ISACencLB_obj->bitstr_obj.W_upper = transcodingParam.W_upper; + ISACencLB_obj->bitstr_obj.stream_index = transcodingParam.stream_index; + ISACencLB_obj->bitstr_obj.streamval = transcodingParam.streamval; + ISACencLB_obj->bitstr_obj.stream[transcodingParam.stream_index - 2] = + transcodingParam.stream[0]; + ISACencLB_obj->bitstr_obj.stream[transcodingParam.stream_index - 1] = + transcodingParam.stream[1]; + ISACencLB_obj->bitstr_obj.stream[transcodingParam.stream_index] = + transcodingParam.stream[2]; + + /* Code gains. */ + WebRtcIsac_EncodeLpcGainLb(lofilt_coef, hifilt_coef, + &ISACencLB_obj->bitstr_obj, + &ISACencLB_obj->SaveEnc_obj); + + /* Update the number of bytes left for encoding the spectrum. */ + bytesLeftSpecCoding = payloadLimitBytes - transcodingParam.stream_index; + + /* Encode the spectrum. */ + err = WebRtcIsac_EncodeSpec(fre, fim, AvgPitchGain_Q12, kIsacLowerBand, + &ISACencLB_obj->bitstr_obj); + + if ((err < 0) && (err != -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + /* There has been an error but it was not too large + payload (we can cure too large payload). */ + if (frame_mode == 1 && ISACencLB_obj->frame_nb == 1) { + /* If this is the second 30 ms of a 60 ms frame reset + this such that in the next call encoder starts fresh. */ + ISACencLB_obj->frame_nb = 0; + } + return err; + } + iterCntr++; + } + + /* If 60 ms frame-size and just processed the first 30 ms, */ + /* go back to main function to buffer the other 30 ms speech frame. */ + if (frame_mode == 1) { + if (ISACencLB_obj->frame_nb == 0) { + ISACencLB_obj->frame_nb = 1; + return 0; + } else if (ISACencLB_obj->frame_nb == 1) { + ISACencLB_obj->frame_nb = 0; + /* Also update the frame-length for next packet, + in Adaptive mode only. */ + if (codingMode == 0 && (ISACencLB_obj->enforceFrameSize == 0)) { + ISACencLB_obj->new_framelength = + WebRtcIsac_GetNewFrameLength(ISACencLB_obj->bottleneck, + ISACencLB_obj->current_framesamples); + } + } + } else { + ISACencLB_obj->frame_nb = 0; + } + + /* Complete arithmetic coding. */ + stream_length = WebRtcIsac_EncTerminate(&ISACencLB_obj->bitstr_obj); + return stream_length; +} + + + +static int LimitPayloadUb(ISACUBEncStruct* ISACencUB_obj, + uint16_t payloadLimitBytes, + double bytesLeftSpecCoding, + transcode_obj* transcodingParam, + int16_t* fre, int16_t* fim, + double* lpcGains, enum ISACBand band, int status) { + + int iterCntr = 0; + int k; + double bytesSpecCoderUsed; + double transcodeScale; + const int16_t kAveragePitchGain = 0.0; + + do { + if (iterCntr >= MAX_PAYLOAD_LIMIT_ITERATION) { + /* We were not able to limit the payload size. */ + return -ISAC_PAYLOAD_LARGER_THAN_LIMIT; + } + + if (status == -ISAC_DISALLOWED_BITSTREAM_LENGTH) { + bytesSpecCoderUsed = STREAM_SIZE_MAX; + /* Being conservative. */ + transcodeScale = bytesLeftSpecCoding / bytesSpecCoderUsed * 0.5; + } else { + bytesSpecCoderUsed = ISACencUB_obj->bitstr_obj.stream_index - + transcodingParam->stream_index; + transcodeScale = bytesLeftSpecCoding / bytesSpecCoderUsed; + } + + /* To be safe, we reduce the scale depending on the + number of iterations. */ + transcodeScale *= (1.0 - (0.9 * (double)iterCntr / + (double)MAX_PAYLOAD_LIMIT_ITERATION)); + + /* Scale the LPC Gains. */ + if (band == kIsacUpperBand16) { + /* Two sets of coefficients if 16 kHz. */ + for (k = 0; k < SUBFRAMES; k++) { + transcodingParam->loFiltGain[k] *= transcodeScale; + transcodingParam->hiFiltGain[k] *= transcodeScale; + } + } else { + /* One sets of coefficients if 12 kHz. */ + for (k = 0; k < SUBFRAMES; k++) { + transcodingParam->loFiltGain[k] *= transcodeScale; + } + } + + /* Scale DFT coefficients. */ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + fre[k] = (int16_t)(fre[k] * transcodeScale + 0.5); + fim[k] = (int16_t)(fim[k] * transcodeScale + 0.5); + } + /* Store FFT coefficients for multiple encoding. */ + memcpy(ISACencUB_obj->SaveEnc_obj.realFFT, fre, + sizeof(ISACencUB_obj->SaveEnc_obj.realFFT)); + memcpy(ISACencUB_obj->SaveEnc_obj.imagFFT, fim, + sizeof(ISACencUB_obj->SaveEnc_obj.imagFFT)); + + /* Store the state of arithmetic coder before coding LPC gains */ + ISACencUB_obj->bitstr_obj.W_upper = transcodingParam->W_upper; + ISACencUB_obj->bitstr_obj.stream_index = transcodingParam->stream_index; + ISACencUB_obj->bitstr_obj.streamval = transcodingParam->streamval; + ISACencUB_obj->bitstr_obj.stream[transcodingParam->stream_index - 2] = + transcodingParam->stream[0]; + ISACencUB_obj->bitstr_obj.stream[transcodingParam->stream_index - 1] = + transcodingParam->stream[1]; + ISACencUB_obj->bitstr_obj.stream[transcodingParam->stream_index] = + transcodingParam->stream[2]; + + /* Store the gains for multiple encoding. */ + memcpy(ISACencUB_obj->SaveEnc_obj.lpcGain, lpcGains, + SUBFRAMES * sizeof(double)); + /* Entropy Code lpc-gains, indices are stored for a later use.*/ + WebRtcIsac_EncodeLpcGainUb(transcodingParam->loFiltGain, + &ISACencUB_obj->bitstr_obj, + ISACencUB_obj->SaveEnc_obj.lpcGainIndex); + + /* If 16kHz should do one more set. */ + if (band == kIsacUpperBand16) { + /* Store the gains for multiple encoding. */ + memcpy(&ISACencUB_obj->SaveEnc_obj.lpcGain[SUBFRAMES], + &lpcGains[SUBFRAMES], SUBFRAMES * sizeof(double)); + /* Entropy Code lpc-gains, indices are stored for a later use.*/ + WebRtcIsac_EncodeLpcGainUb( + transcodingParam->hiFiltGain, &ISACencUB_obj->bitstr_obj, + &ISACencUB_obj->SaveEnc_obj.lpcGainIndex[SUBFRAMES]); + } + + /* Update the number of bytes left for encoding the spectrum. */ + bytesLeftSpecCoding = payloadLimitBytes - + ISACencUB_obj->bitstr_obj.stream_index; + + /* Save the bit-stream object at this point for FEC. */ + memcpy(&ISACencUB_obj->SaveEnc_obj.bitStreamObj, + &ISACencUB_obj->bitstr_obj, sizeof(Bitstr)); + + /* Encode the spectrum. */ + status = WebRtcIsac_EncodeSpec(fre, fim, kAveragePitchGain, + band, &ISACencUB_obj->bitstr_obj); + if ((status < 0) && (status != -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + /* There has been an error but it was not too large payload + (we can cure too large payload). */ + return status; + } + iterCntr++; + } while ((ISACencUB_obj->bitstr_obj.stream_index > payloadLimitBytes) || + (status == -ISAC_DISALLOWED_BITSTREAM_LENGTH)); + return 0; +} + +int WebRtcIsac_EncodeUb16(const TransformTables* transform_tables, + float* in, ISACUBEncStruct* ISACencUB_obj, + int32_t jitterInfo) { + int err; + int k; + + double lpcVecs[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + double percepFilterParams[(1 + UB_LPC_ORDER) * (SUBFRAMES << 1) + + (1 + UB_LPC_ORDER)]; + + double LP_lookahead[FRAMESAMPLES]; + int16_t fre[FRAMESAMPLES_HALF]; /* Q7 */ + int16_t fim[FRAMESAMPLES_HALF]; /* Q7 */ + + int status = 0; + + double varscale[2]; + double corr[SUBFRAMES << 1][UB_LPC_ORDER + 1]; + double lpcGains[SUBFRAMES << 1]; + transcode_obj transcodingParam; + uint16_t payloadLimitBytes; + double s2nr; + const int16_t kAveragePitchGain = 0.0; + int bytesLeftSpecCoding; + + /* Buffer speech samples (by 10ms packet) until the frame-length is */ + /* reached (30 ms). */ + /*********************************************************************/ + + /* fill the buffer with 10ms input data */ + memcpy(&ISACencUB_obj->data_buffer_float[ISACencUB_obj->buffer_index], in, + FRAMESAMPLES_10ms * sizeof(float)); + + /* If buffer size is not equal to current frame-size, and end of file is + * not reached yet, we don't do encoding unless we have the whole frame. */ + if (ISACencUB_obj->buffer_index + FRAMESAMPLES_10ms < FRAMESAMPLES) { + ISACencUB_obj->buffer_index += FRAMESAMPLES_10ms; + return 0; + } + + /* End of buffer function. */ + /**************************/ + + /* Encoding */ + /************/ + + /* Reset bit-stream */ + WebRtcIsac_ResetBitstream(&(ISACencUB_obj->bitstr_obj)); + + /* Encoding of bandwidth information. */ + WebRtcIsac_EncodeJitterInfo(jitterInfo, &ISACencUB_obj->bitstr_obj); + + status = WebRtcIsac_EncodeBandwidth(isac16kHz, &ISACencUB_obj->bitstr_obj); + if (status < 0) { + return status; + } + + s2nr = WebRtcIsac_GetSnr(ISACencUB_obj->bottleneck, FRAMESAMPLES); + + memcpy(lpcVecs, ISACencUB_obj->lastLPCVec, UB_LPC_ORDER * sizeof(double)); + + for (k = 0; k < FRAMESAMPLES; k++) { + LP_lookahead[k] = ISACencUB_obj->data_buffer_float[UB_LOOKAHEAD + k]; + } + + /* Find coefficients for perceptual pre-filters. */ + WebRtcIsac_GetLpcCoefUb(LP_lookahead, &ISACencUB_obj->maskfiltstr_obj, + &lpcVecs[UB_LPC_ORDER], corr, varscale, isac16kHz); + + memcpy(ISACencUB_obj->lastLPCVec, + &lpcVecs[(UB16_LPC_VEC_PER_FRAME - 1) * (UB_LPC_ORDER)], + sizeof(double) * UB_LPC_ORDER); + + /* Code LPC model and shape - gains not quantized yet. */ + WebRtcIsac_EncodeLpcUB(lpcVecs, &ISACencUB_obj->bitstr_obj, + percepFilterParams, isac16kHz, + &ISACencUB_obj->SaveEnc_obj); + + /* the first set of lpc parameters are from the last sub-frame of + * the previous frame. so we don't care about them. */ + WebRtcIsac_GetLpcGain(s2nr, &percepFilterParams[UB_LPC_ORDER + 1], + (SUBFRAMES << 1), lpcGains, corr, varscale); + + /* Store the state of arithmetic coder before coding LPC gains */ + transcodingParam.stream_index = ISACencUB_obj->bitstr_obj.stream_index; + transcodingParam.W_upper = ISACencUB_obj->bitstr_obj.W_upper; + transcodingParam.streamval = ISACencUB_obj->bitstr_obj.streamval; + transcodingParam.stream[0] = + ISACencUB_obj->bitstr_obj.stream[ISACencUB_obj->bitstr_obj.stream_index - + 2]; + transcodingParam.stream[1] = + ISACencUB_obj->bitstr_obj.stream[ISACencUB_obj->bitstr_obj.stream_index - + 1]; + transcodingParam.stream[2] = + ISACencUB_obj->bitstr_obj.stream[ISACencUB_obj->bitstr_obj.stream_index]; + + /* Store LPC Gains before encoding them. */ + for (k = 0; k < SUBFRAMES; k++) { + transcodingParam.loFiltGain[k] = lpcGains[k]; + transcodingParam.hiFiltGain[k] = lpcGains[SUBFRAMES + k]; + } + + /* Store the gains for multiple encoding. */ + memcpy(ISACencUB_obj->SaveEnc_obj.lpcGain, lpcGains, + (SUBFRAMES << 1) * sizeof(double)); + + WebRtcIsac_EncodeLpcGainUb(lpcGains, &ISACencUB_obj->bitstr_obj, + ISACencUB_obj->SaveEnc_obj.lpcGainIndex); + WebRtcIsac_EncodeLpcGainUb( + &lpcGains[SUBFRAMES], &ISACencUB_obj->bitstr_obj, + &ISACencUB_obj->SaveEnc_obj.lpcGainIndex[SUBFRAMES]); + + /* Get the correct value for the payload limit and calculate the number of + bytes left for coding the spectrum. It is a 30ms frame + Subract 3 because termination process may add 3 bytes */ + payloadLimitBytes = ISACencUB_obj->maxPayloadSizeBytes - + ISACencUB_obj->numBytesUsed - 3; + bytesLeftSpecCoding = payloadLimitBytes - + ISACencUB_obj->bitstr_obj.stream_index; + + for (k = 0; k < (SUBFRAMES << 1); k++) { + percepFilterParams[k * (UB_LPC_ORDER + 1) + (UB_LPC_ORDER + 1)] = + lpcGains[k]; + } + + /* LPC filtering (using normalized lattice filter), */ + /* first half-frame. */ + WebRtcIsac_NormLatticeFilterMa(UB_LPC_ORDER, + ISACencUB_obj->maskfiltstr_obj.PreStateLoF, + ISACencUB_obj->maskfiltstr_obj.PreStateLoG, + &ISACencUB_obj->data_buffer_float[0], + &percepFilterParams[UB_LPC_ORDER + 1], + &LP_lookahead[0]); + + /* Second half-frame filtering. */ + WebRtcIsac_NormLatticeFilterMa( + UB_LPC_ORDER, ISACencUB_obj->maskfiltstr_obj.PreStateLoF, + ISACencUB_obj->maskfiltstr_obj.PreStateLoG, + &ISACencUB_obj->data_buffer_float[FRAMESAMPLES_HALF], + &percepFilterParams[(UB_LPC_ORDER + 1) + SUBFRAMES * (UB_LPC_ORDER + 1)], + &LP_lookahead[FRAMESAMPLES_HALF]); + + WebRtcIsac_Time2Spec(transform_tables, + &LP_lookahead[0], &LP_lookahead[FRAMESAMPLES_HALF], + fre, fim, &ISACencUB_obj->fftstr_obj); + + /* Store FFT coefficients for multiple encoding. */ + memcpy(ISACencUB_obj->SaveEnc_obj.realFFT, fre, sizeof(fre)); + memcpy(ISACencUB_obj->SaveEnc_obj.imagFFT, fim, sizeof(fim)); + + /* Prepare the audio buffer for the next packet + * move the last 3 ms to the beginning of the buffer. */ + memcpy(ISACencUB_obj->data_buffer_float, + &ISACencUB_obj->data_buffer_float[FRAMESAMPLES], + LB_TOTAL_DELAY_SAMPLES * sizeof(float)); + /* start writing with 3 ms delay to compensate for the delay + * of the lower-band. */ + ISACencUB_obj->buffer_index = LB_TOTAL_DELAY_SAMPLES; + + /* Save the bit-stream object at this point for FEC. */ + memcpy(&ISACencUB_obj->SaveEnc_obj.bitStreamObj, &ISACencUB_obj->bitstr_obj, + sizeof(Bitstr)); + + /* Qantization and lossless coding */ + /* Note that there is no pitch-gain for this band so kAveragePitchGain = 0 + * is passed to the function. In fact, the function ignores the 3rd parameter + * for this band. */ + err = WebRtcIsac_EncodeSpec(fre, fim, kAveragePitchGain, kIsacUpperBand16, + &ISACencUB_obj->bitstr_obj); + if ((err < 0) && (err != -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + return err; + } + + if ((ISACencUB_obj->bitstr_obj.stream_index > payloadLimitBytes) || + (err == -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + err = LimitPayloadUb(ISACencUB_obj, payloadLimitBytes, bytesLeftSpecCoding, + &transcodingParam, fre, fim, lpcGains, + kIsacUpperBand16, err); + } + if (err < 0) { + return err; + } + /* Complete arithmetic coding. */ + return WebRtcIsac_EncTerminate(&ISACencUB_obj->bitstr_obj); +} + + +int WebRtcIsac_EncodeUb12(const TransformTables* transform_tables, + float* in, ISACUBEncStruct* ISACencUB_obj, + int32_t jitterInfo) { + int err; + int k; + + double lpcVecs[UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME]; + + double percepFilterParams[(1 + UB_LPC_ORDER) * SUBFRAMES]; + float LP[FRAMESAMPLES_HALF]; + float HP[FRAMESAMPLES_HALF]; + + double LP_lookahead[FRAMESAMPLES_HALF]; + double HP_lookahead[FRAMESAMPLES_HALF]; + double LPw[FRAMESAMPLES_HALF]; + + double HPw[FRAMESAMPLES_HALF]; + int16_t fre[FRAMESAMPLES_HALF]; /* Q7 */ + int16_t fim[FRAMESAMPLES_HALF]; /* Q7 */ + + int status = 0; + + double varscale[1]; + + double corr[UB_LPC_GAIN_DIM][UB_LPC_ORDER + 1]; + double lpcGains[SUBFRAMES]; + transcode_obj transcodingParam; + uint16_t payloadLimitBytes; + double s2nr; + const int16_t kAveragePitchGain = 0.0; + double bytesLeftSpecCoding; + + /* Buffer speech samples (by 10ms packet) until the framelength is */ + /* reached (30 ms). */ + /********************************************************************/ + + /* Fill the buffer with 10ms input data. */ + memcpy(&ISACencUB_obj->data_buffer_float[ISACencUB_obj->buffer_index], in, + FRAMESAMPLES_10ms * sizeof(float)); + + /* if buffer-size is not equal to current frame-size then increase the + index and return. We do the encoding when we have enough audio. */ + if (ISACencUB_obj->buffer_index + FRAMESAMPLES_10ms < FRAMESAMPLES) { + ISACencUB_obj->buffer_index += FRAMESAMPLES_10ms; + return 0; + } + /* If buffer reached the right size, reset index and continue + with encoding the frame */ + ISACencUB_obj->buffer_index = 0; + + /* End of buffer function */ + /**************************/ + + /* Encoding */ + /************/ + + /* Reset bit-stream. */ + WebRtcIsac_ResetBitstream(&(ISACencUB_obj->bitstr_obj)); + + /* Encoding bandwidth information. */ + WebRtcIsac_EncodeJitterInfo(jitterInfo, &ISACencUB_obj->bitstr_obj); + status = WebRtcIsac_EncodeBandwidth(isac12kHz, &ISACencUB_obj->bitstr_obj); + if (status < 0) { + return status; + } + + s2nr = WebRtcIsac_GetSnr(ISACencUB_obj->bottleneck, FRAMESAMPLES); + + /* Split signal in two bands. */ + WebRtcIsac_SplitAndFilterFloat(ISACencUB_obj->data_buffer_float, HP, LP, + HP_lookahead, LP_lookahead, + &ISACencUB_obj->prefiltbankstr_obj); + + /* Find coefficients for perceptual pre-filters. */ + WebRtcIsac_GetLpcCoefUb(LP_lookahead, &ISACencUB_obj->maskfiltstr_obj, + lpcVecs, corr, varscale, isac12kHz); + + /* Code LPC model and shape - gains not quantized yet. */ + WebRtcIsac_EncodeLpcUB(lpcVecs, &ISACencUB_obj->bitstr_obj, + percepFilterParams, isac12kHz, + &ISACencUB_obj->SaveEnc_obj); + + WebRtcIsac_GetLpcGain(s2nr, percepFilterParams, SUBFRAMES, lpcGains, corr, + varscale); + + /* Store the state of arithmetic coder before coding LPC gains. */ + transcodingParam.W_upper = ISACencUB_obj->bitstr_obj.W_upper; + transcodingParam.stream_index = ISACencUB_obj->bitstr_obj.stream_index; + transcodingParam.streamval = ISACencUB_obj->bitstr_obj.streamval; + transcodingParam.stream[0] = + ISACencUB_obj->bitstr_obj.stream[ISACencUB_obj->bitstr_obj.stream_index - + 2]; + transcodingParam.stream[1] = + ISACencUB_obj->bitstr_obj.stream[ISACencUB_obj->bitstr_obj.stream_index - + 1]; + transcodingParam.stream[2] = + ISACencUB_obj->bitstr_obj.stream[ISACencUB_obj->bitstr_obj.stream_index]; + + /* Store LPC Gains before encoding them. */ + for (k = 0; k < SUBFRAMES; k++) { + transcodingParam.loFiltGain[k] = lpcGains[k]; + } + + /* Store the gains for multiple encoding. */ + memcpy(ISACencUB_obj->SaveEnc_obj.lpcGain, lpcGains, SUBFRAMES * + sizeof(double)); + + WebRtcIsac_EncodeLpcGainUb(lpcGains, &ISACencUB_obj->bitstr_obj, + ISACencUB_obj->SaveEnc_obj.lpcGainIndex); + + for (k = 0; k < SUBFRAMES; k++) { + percepFilterParams[k * (UB_LPC_ORDER + 1)] = lpcGains[k]; + } + + /* perceptual pre-filtering (using normalized lattice filter) */ + /* low-band filtering */ + WebRtcIsac_NormLatticeFilterMa(UB_LPC_ORDER, + ISACencUB_obj->maskfiltstr_obj.PreStateLoF, + ISACencUB_obj->maskfiltstr_obj.PreStateLoG, LP, + percepFilterParams, LPw); + + /* Get the correct value for the payload limit and calculate the number + of bytes left for coding the spectrum. It is a 30ms frame Subract 3 + because termination process may add 3 bytes */ + payloadLimitBytes = ISACencUB_obj->maxPayloadSizeBytes - + ISACencUB_obj->numBytesUsed - 3; + bytesLeftSpecCoding = payloadLimitBytes - + ISACencUB_obj->bitstr_obj.stream_index; + + memset(HPw, 0, sizeof(HPw)); + + /* Transform */ + WebRtcIsac_Time2Spec(transform_tables, + LPw, HPw, fre, fim, &ISACencUB_obj->fftstr_obj); + + /* Store FFT coefficients for multiple encoding. */ + memcpy(ISACencUB_obj->SaveEnc_obj.realFFT, fre, + sizeof(ISACencUB_obj->SaveEnc_obj.realFFT)); + memcpy(ISACencUB_obj->SaveEnc_obj.imagFFT, fim, + sizeof(ISACencUB_obj->SaveEnc_obj.imagFFT)); + + /* Save the bit-stream object at this point for FEC. */ + memcpy(&ISACencUB_obj->SaveEnc_obj.bitStreamObj, + &ISACencUB_obj->bitstr_obj, sizeof(Bitstr)); + + /* Quantization and loss-less coding */ + /* The 4th parameter to this function is pitch-gain, which is only used + * when encoding 0-8 kHz band, and irrelevant in this function, therefore, + * we insert zero here. */ + err = WebRtcIsac_EncodeSpec(fre, fim, kAveragePitchGain, kIsacUpperBand12, + &ISACencUB_obj->bitstr_obj); + if ((err < 0) && (err != -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + /* There has been an error but it was not too large + payload (we can cure too large payload) */ + return err; + } + + if ((ISACencUB_obj->bitstr_obj.stream_index > payloadLimitBytes) || + (err == -ISAC_DISALLOWED_BITSTREAM_LENGTH)) { + err = LimitPayloadUb(ISACencUB_obj, payloadLimitBytes, bytesLeftSpecCoding, + &transcodingParam, fre, fim, lpcGains, + kIsacUpperBand12, err); + } + if (err < 0) { + return err; + } + /* Complete arithmetic coding. */ + return WebRtcIsac_EncTerminate(&ISACencUB_obj->bitstr_obj); +} + + + + + + +/* This function is used to create a new bit-stream with new BWE. + The same data as previously encoded with the function WebRtcIsac_Encoder(). + The data needed is taken from the structure, where it was stored + when calling the encoder. */ + +int WebRtcIsac_EncodeStoredDataLb(const IsacSaveEncoderData* ISACSavedEnc_obj, + Bitstr* ISACBitStr_obj, int BWnumber, + float scale) { + int ii; + int status; + int BWno = BWnumber; + + const uint16_t* WebRtcIsac_kQPitchGainCdf_ptr[1]; + const uint16_t** cdf; + + double tmpLPCcoeffs_lo[(ORDERLO + 1)*SUBFRAMES * 2]; + double tmpLPCcoeffs_hi[(ORDERHI + 1)*SUBFRAMES * 2]; + int tmpLPCindex_g[12 * 2]; + int16_t tmp_fre[FRAMESAMPLES], tmp_fim[FRAMESAMPLES]; + const int kModel = 0; + + /* Sanity Check - possible values for BWnumber is 0 - 23. */ + if ((BWnumber < 0) || (BWnumber > 23)) { + return -ISAC_RANGE_ERROR_BW_ESTIMATOR; + } + + /* Reset bit-stream. */ + WebRtcIsac_ResetBitstream(ISACBitStr_obj); + + /* Encode frame length */ + status = WebRtcIsac_EncodeFrameLen(ISACSavedEnc_obj->framelength, + ISACBitStr_obj); + if (status < 0) { + /* Wrong frame size. */ + return status; + } + + /* Transcoding */ + if ((scale > 0.0) && (scale < 1.0)) { + /* Compensate LPC gain. */ + for (ii = 0; + ii < ((ORDERLO + 1)* SUBFRAMES * (1 + ISACSavedEnc_obj->startIdx)); + ii++) { + tmpLPCcoeffs_lo[ii] = scale * ISACSavedEnc_obj->LPCcoeffs_lo[ii]; + } + for (ii = 0; + ii < ((ORDERHI + 1) * SUBFRAMES * (1 + ISACSavedEnc_obj->startIdx)); + ii++) { + tmpLPCcoeffs_hi[ii] = scale * ISACSavedEnc_obj->LPCcoeffs_hi[ii]; + } + /* Scale DFT. */ + for (ii = 0; + ii < (FRAMESAMPLES_HALF * (1 + ISACSavedEnc_obj->startIdx)); + ii++) { + tmp_fre[ii] = (int16_t)((scale) * (float)ISACSavedEnc_obj->fre[ii]); + tmp_fim[ii] = (int16_t)((scale) * (float)ISACSavedEnc_obj->fim[ii]); + } + } else { + for (ii = 0; + ii < (KLT_ORDER_GAIN * (1 + ISACSavedEnc_obj->startIdx)); + ii++) { + tmpLPCindex_g[ii] = ISACSavedEnc_obj->LPCindex_g[ii]; + } + for (ii = 0; + ii < (FRAMESAMPLES_HALF * (1 + ISACSavedEnc_obj->startIdx)); + ii++) { + tmp_fre[ii] = ISACSavedEnc_obj->fre[ii]; + tmp_fim[ii] = ISACSavedEnc_obj->fim[ii]; + } + } + + /* Encode bandwidth estimate. */ + WebRtcIsac_EncodeReceiveBw(&BWno, ISACBitStr_obj); + + /* Loop over number of 30 msec */ + for (ii = 0; ii <= ISACSavedEnc_obj->startIdx; ii++) { + /* Encode pitch gains. */ + *WebRtcIsac_kQPitchGainCdf_ptr = WebRtcIsac_kQPitchGainCdf; + WebRtcIsac_EncHistMulti(ISACBitStr_obj, + &ISACSavedEnc_obj->pitchGain_index[ii], + WebRtcIsac_kQPitchGainCdf_ptr, 1); + + /* Entropy coding of quantization pitch lags */ + /* Voicing classification. */ + if (ISACSavedEnc_obj->meanGain[ii] < 0.2) { + cdf = WebRtcIsac_kQPitchLagCdfPtrLo; + } else if (ISACSavedEnc_obj->meanGain[ii] < 0.4) { + cdf = WebRtcIsac_kQPitchLagCdfPtrMid; + } else { + cdf = WebRtcIsac_kQPitchLagCdfPtrHi; + } + WebRtcIsac_EncHistMulti(ISACBitStr_obj, + &ISACSavedEnc_obj->pitchIndex[PITCH_SUBFRAMES * ii], + cdf, PITCH_SUBFRAMES); + + /* LPC */ + /* Only one model exists. The entropy coding is done only for backward + * compatibility. */ + WebRtcIsac_EncHistMulti(ISACBitStr_obj, &kModel, + WebRtcIsac_kQKltModelCdfPtr, 1); + /* Entropy coding of quantization indices - LPC shape only. */ + WebRtcIsac_EncHistMulti(ISACBitStr_obj, + &ISACSavedEnc_obj->LPCindex_s[KLT_ORDER_SHAPE * ii], + WebRtcIsac_kQKltCdfPtrShape, + KLT_ORDER_SHAPE); + + /* If transcoding, get new LPC gain indices */ + if (scale < 1.0) { + WebRtcIsac_TranscodeLPCCoef( + &tmpLPCcoeffs_lo[(ORDERLO + 1) * SUBFRAMES * ii], + &tmpLPCcoeffs_hi[(ORDERHI + 1)*SUBFRAMES * ii], + &tmpLPCindex_g[KLT_ORDER_GAIN * ii]); + } + + /* Entropy coding of quantization indices - LPC gain. */ + WebRtcIsac_EncHistMulti(ISACBitStr_obj, &tmpLPCindex_g[KLT_ORDER_GAIN * ii], + WebRtcIsac_kQKltCdfPtrGain, KLT_ORDER_GAIN); + + /* Quantization and loss-less coding. */ + status = WebRtcIsac_EncodeSpec(&tmp_fre[ii * FRAMESAMPLES_HALF], + &tmp_fim[ii * FRAMESAMPLES_HALF], + ISACSavedEnc_obj->AvgPitchGain[ii], + kIsacLowerBand, ISACBitStr_obj); + if (status < 0) { + return status; + } + } + /* Complete arithmetic coding. */ + return WebRtcIsac_EncTerminate(ISACBitStr_obj); +} + + +int WebRtcIsac_EncodeStoredDataUb( + const ISACUBSaveEncDataStruct* ISACSavedEnc_obj, + Bitstr* bitStream, + int32_t jitterInfo, + float scale, + enum ISACBandwidth bandwidth) { + int n; + int err; + double lpcGain[SUBFRAMES]; + int16_t realFFT[FRAMESAMPLES_HALF]; + int16_t imagFFT[FRAMESAMPLES_HALF]; + const uint16_t** shape_cdf; + int shape_len; + const int16_t kAveragePitchGain = 0.0; + enum ISACBand band; + /* Reset bitstream. */ + WebRtcIsac_ResetBitstream(bitStream); + + /* Encode jitter index. */ + WebRtcIsac_EncodeJitterInfo(jitterInfo, bitStream); + + err = WebRtcIsac_EncodeBandwidth(bandwidth, bitStream); + if (err < 0) { + return err; + } + + /* Encode LPC-shape. */ + if (bandwidth == isac12kHz) { + shape_cdf = WebRtcIsac_kLpcShapeCdfMatUb12; + shape_len = UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME; + band = kIsacUpperBand12; + } else { + shape_cdf = WebRtcIsac_kLpcShapeCdfMatUb16; + shape_len = UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME; + band = kIsacUpperBand16; + } + WebRtcIsac_EncHistMulti(bitStream, ISACSavedEnc_obj->indexLPCShape, + shape_cdf, shape_len); + + if ((scale <= 0.0) || (scale >= 1.0)) { + /* We only consider scales between zero and one. */ + WebRtcIsac_EncHistMulti(bitStream, ISACSavedEnc_obj->lpcGainIndex, + WebRtcIsac_kLpcGainCdfMat, UB_LPC_GAIN_DIM); + if (bandwidth == isac16kHz) { + /* Store gain indices of the second half. */ + WebRtcIsac_EncHistMulti(bitStream, + &ISACSavedEnc_obj->lpcGainIndex[SUBFRAMES], + WebRtcIsac_kLpcGainCdfMat, UB_LPC_GAIN_DIM); + } + /* Store FFT coefficients. */ + err = WebRtcIsac_EncodeSpec(ISACSavedEnc_obj->realFFT, + ISACSavedEnc_obj->imagFFT, kAveragePitchGain, + band, bitStream); + } else { + /* Scale LPC gain and FFT coefficients. */ + for (n = 0; n < SUBFRAMES; n++) { + lpcGain[n] = scale * ISACSavedEnc_obj->lpcGain[n]; + } + /* Store LPC gains. */ + WebRtcIsac_StoreLpcGainUb(lpcGain, bitStream); + + if (bandwidth == isac16kHz) { + /* Scale and code the gains of the second half of the frame, if 16kHz. */ + for (n = 0; n < SUBFRAMES; n++) { + lpcGain[n] = scale * ISACSavedEnc_obj->lpcGain[n + SUBFRAMES]; + } + WebRtcIsac_StoreLpcGainUb(lpcGain, bitStream); + } + + for (n = 0; n < FRAMESAMPLES_HALF; n++) { + realFFT[n] = (int16_t)(scale * (float)ISACSavedEnc_obj->realFFT[n] + + 0.5f); + imagFFT[n] = (int16_t)(scale * (float)ISACSavedEnc_obj->imagFFT[n] + + 0.5f); + } + /* Store FFT coefficients. */ + err = WebRtcIsac_EncodeSpec(realFFT, imagFFT, kAveragePitchGain, + band, bitStream); + } + if (err < 0) { + /* Error happened while encoding FFT coefficients. */ + return err; + } + + /* Complete arithmetic coding. */ + return WebRtcIsac_EncTerminate(bitStream); +} + +int16_t WebRtcIsac_GetRedPayloadUb( + const ISACUBSaveEncDataStruct* ISACSavedEncObj, + Bitstr* bitStreamObj, + enum ISACBandwidth bandwidth) { + int n; + int16_t status; + int16_t realFFT[FRAMESAMPLES_HALF]; + int16_t imagFFT[FRAMESAMPLES_HALF]; + enum ISACBand band; + const int16_t kAveragePitchGain = 0.0; + /* Store bit-stream object. */ + memcpy(bitStreamObj, &ISACSavedEncObj->bitStreamObj, sizeof(Bitstr)); + + /* Scale FFT coefficients. */ + for (n = 0; n < FRAMESAMPLES_HALF; n++) { + realFFT[n] = (int16_t)((float)ISACSavedEncObj->realFFT[n] * + RCU_TRANSCODING_SCALE_UB + 0.5); + imagFFT[n] = (int16_t)((float)ISACSavedEncObj->imagFFT[n] * + RCU_TRANSCODING_SCALE_UB + 0.5); + } + + band = (bandwidth == isac12kHz) ? kIsacUpperBand12 : kIsacUpperBand16; + status = WebRtcIsac_EncodeSpec(realFFT, imagFFT, kAveragePitchGain, band, + bitStreamObj); + if (status < 0) { + return status; + } else { + /* Terminate entropy coding */ + return WebRtcIsac_EncTerminate(bitStreamObj); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.c new file mode 100644 index 0000000000..64cd4a75ec --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.c @@ -0,0 +1,708 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * code_LPC_UB.c + * + * This file contains definition of functions used to + * encode LPC parameters (Shape & gain) of the upper band. + * + */ + +#include "encode_lpc_swb.h" + +#include +#include +#include + +#include "lpc_gain_swb_tables.h" +#include "lpc_shape_swb12_tables.h" +#include "lpc_shape_swb16_tables.h" +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +/****************************************************************************** + * WebRtcIsac_RemoveLarMean() + * + * Remove the means from LAR coefficients. + * + * Input: + * -lar : pointer to lar vectors. LAR vectors are + * concatenated. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -lar : pointer to mean-removed LAR:s. + * + * + */ +int16_t +WebRtcIsac_RemoveLarMean( + double* lar, + int16_t bandwidth) +{ + int16_t coeffCntr; + int16_t vecCntr; + int16_t numVec; + const double* meanLAR; + switch(bandwidth) + { + case isac12kHz: + { + numVec = UB_LPC_VEC_PER_FRAME; + meanLAR = WebRtcIsac_kMeanLarUb12; + break; + } + case isac16kHz: + { + numVec = UB16_LPC_VEC_PER_FRAME; + meanLAR = WebRtcIsac_kMeanLarUb16; + break; + } + default: + return -1; + } + + for(vecCntr = 0; vecCntr < numVec; vecCntr++) + { + for(coeffCntr = 0; coeffCntr < UB_LPC_ORDER; coeffCntr++) + { + // REMOVE MEAN + *lar++ -= meanLAR[coeffCntr]; + } + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_DecorrelateIntraVec() + * + * Remove the correlation amonge the components of LAR vectors. If LAR vectors + * of one frame are put in a matrix where each column is a LAR vector of a + * sub-frame, then this is equivalent to multiplying the LAR matrix with + * a decorrelting mtrix from left. + * + * Input: + * -inLar : pointer to mean-removed LAR vecrtors. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : decorrelated LAR vectors. + */ +int16_t +WebRtcIsac_DecorrelateIntraVec( + const double* data, + double* out, + int16_t bandwidth) +{ + const double* ptrData; + const double* ptrRow; + int16_t rowCntr; + int16_t colCntr; + int16_t larVecCntr; + int16_t numVec; + const double* decorrMat; + switch(bandwidth) + { + case isac12kHz: + { + decorrMat = &WebRtcIsac_kIntraVecDecorrMatUb12[0][0]; + numVec = UB_LPC_VEC_PER_FRAME; + break; + } + case isac16kHz: + { + decorrMat = &WebRtcIsac_kIintraVecDecorrMatUb16[0][0]; + numVec = UB16_LPC_VEC_PER_FRAME; + break; + } + default: + return -1; + } + + // + // decorrMat * data + // + // data is assumed to contain 'numVec' of LAR + // vectors (mean removed) each of dimension 'UB_LPC_ORDER' + // concatenated one after the other. + // + + ptrData = data; + for(larVecCntr = 0; larVecCntr < numVec; larVecCntr++) + { + for(rowCntr = 0; rowCntr < UB_LPC_ORDER; rowCntr++) + { + ptrRow = &decorrMat[rowCntr * UB_LPC_ORDER]; + *out = 0; + for(colCntr = 0; colCntr < UB_LPC_ORDER; colCntr++) + { + *out += ptrData[colCntr] * ptrRow[colCntr]; + } + out++; + } + ptrData += UB_LPC_ORDER; + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_DecorrelateInterVec() + * + * Remover the correlation among mean-removed LAR vectors. If LAR vectors + * of one frame are put in a matrix where each column is a LAR vector of a + * sub-frame, then this is equivalent to multiplying the LAR matrix with + * a decorrelting mtrix from right. + * + * Input: + * -data : pointer to matrix of LAR vectors. The matrix + * is stored column-wise. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : decorrelated LAR vectors. + */ +int16_t +WebRtcIsac_DecorrelateInterVec( + const double* data, + double* out, + int16_t bandwidth) +{ + int16_t coeffCntr; + int16_t rowCntr; + int16_t colCntr; + const double* decorrMat; + int16_t interVecDim; + + switch(bandwidth) + { + case isac12kHz: + { + decorrMat = &WebRtcIsac_kInterVecDecorrMatUb12[0][0]; + interVecDim = UB_LPC_VEC_PER_FRAME; + break; + } + case isac16kHz: + { + decorrMat = &WebRtcIsac_kInterVecDecorrMatUb16[0][0]; + interVecDim = UB16_LPC_VEC_PER_FRAME; + break; + } + default: + return -1; + } + + // + // data * decorrMat + // + // data is of size 'interVecDim' * 'UB_LPC_ORDER' + // That is 'interVecDim' of LAR vectors (mean removed) + // in columns each of dimension 'UB_LPC_ORDER'. + // matrix is stored column-wise. + // + + for(coeffCntr = 0; coeffCntr < UB_LPC_ORDER; coeffCntr++) + { + for(colCntr = 0; colCntr < interVecDim; colCntr++) + { + out[coeffCntr + colCntr * UB_LPC_ORDER] = 0; + for(rowCntr = 0; rowCntr < interVecDim; rowCntr++) + { + out[coeffCntr + colCntr * UB_LPC_ORDER] += + data[coeffCntr + rowCntr * UB_LPC_ORDER] * + decorrMat[rowCntr * interVecDim + colCntr]; + } + } + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_QuantizeUncorrLar() + * + * Quantize the uncorrelated parameters. + * + * Input: + * -data : uncorrelated LAR vectors. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -data : quantized version of the input. + * -idx : pointer to quantization indices. + */ +double +WebRtcIsac_QuantizeUncorrLar( + double* data, + int* recIdx, + int16_t bandwidth) +{ + int16_t cntr; + int32_t idx; + int16_t interVecDim; + const double* leftRecPoint; + double quantizationStepSize; + const int16_t* numQuantCell; + switch(bandwidth) + { + case isac12kHz: + { + leftRecPoint = WebRtcIsac_kLpcShapeLeftRecPointUb12; + quantizationStepSize = WebRtcIsac_kLpcShapeQStepSizeUb12; + numQuantCell = WebRtcIsac_kLpcShapeNumRecPointUb12; + interVecDim = UB_LPC_VEC_PER_FRAME; + break; + } + case isac16kHz: + { + leftRecPoint = WebRtcIsac_kLpcShapeLeftRecPointUb16; + quantizationStepSize = WebRtcIsac_kLpcShapeQStepSizeUb16; + numQuantCell = WebRtcIsac_kLpcShapeNumRecPointUb16; + interVecDim = UB16_LPC_VEC_PER_FRAME; + break; + } + default: + return -1; + } + + // + // Quantize the parametrs. + // + for(cntr = 0; cntr < UB_LPC_ORDER * interVecDim; cntr++) + { + idx = (int32_t)floor((*data - leftRecPoint[cntr]) / + quantizationStepSize + 0.5); + if(idx < 0) + { + idx = 0; + } + else if(idx >= numQuantCell[cntr]) + { + idx = numQuantCell[cntr] - 1; + } + + *data++ = leftRecPoint[cntr] + idx * quantizationStepSize; + *recIdx++ = idx; + } + return 0; +} + + +/****************************************************************************** + * WebRtcIsac_DequantizeLpcParam() + * + * Get the quantized value of uncorrelated LARs given the quantization indices. + * + * Input: + * -idx : pointer to quantiztion indices. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : pointer to quantized values. + */ +int16_t +WebRtcIsac_DequantizeLpcParam( + const int* idx, + double* out, + int16_t bandwidth) +{ + int16_t cntr; + int16_t interVecDim; + const double* leftRecPoint; + double quantizationStepSize; + + switch(bandwidth) + { + case isac12kHz: + { + leftRecPoint = WebRtcIsac_kLpcShapeLeftRecPointUb12; + quantizationStepSize = WebRtcIsac_kLpcShapeQStepSizeUb12; + interVecDim = UB_LPC_VEC_PER_FRAME; + break; + } + case isac16kHz: + { + leftRecPoint = WebRtcIsac_kLpcShapeLeftRecPointUb16; + quantizationStepSize = WebRtcIsac_kLpcShapeQStepSizeUb16; + interVecDim = UB16_LPC_VEC_PER_FRAME; + break; + } + default: + return -1; + } + + // + // Dequantize given the quantization indices + // + + for(cntr = 0; cntr < UB_LPC_ORDER * interVecDim; cntr++) + { + *out++ = leftRecPoint[cntr] + *idx++ * quantizationStepSize; + } + return 0; +} + + +/****************************************************************************** + * WebRtcIsac_CorrelateIntraVec() + * + * This is the inverse of WebRtcIsac_DecorrelateIntraVec(). + * + * Input: + * -data : uncorrelated parameters. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : correlated parametrs. + */ +int16_t +WebRtcIsac_CorrelateIntraVec( + const double* data, + double* out, + int16_t bandwidth) +{ + int16_t vecCntr; + int16_t rowCntr; + int16_t colCntr; + int16_t numVec; + const double* ptrData; + const double* intraVecDecorrMat; + + switch(bandwidth) + { + case isac12kHz: + { + numVec = UB_LPC_VEC_PER_FRAME; + intraVecDecorrMat = &WebRtcIsac_kIntraVecDecorrMatUb12[0][0]; + break; + } + case isac16kHz: + { + numVec = UB16_LPC_VEC_PER_FRAME; + intraVecDecorrMat = &WebRtcIsac_kIintraVecDecorrMatUb16[0][0]; + break; + } + default: + return -1; + } + + + ptrData = data; + for(vecCntr = 0; vecCntr < numVec; vecCntr++) + { + for(colCntr = 0; colCntr < UB_LPC_ORDER; colCntr++) + { + *out = 0; + for(rowCntr = 0; rowCntr < UB_LPC_ORDER; rowCntr++) + { + *out += ptrData[rowCntr] * + intraVecDecorrMat[rowCntr * UB_LPC_ORDER + colCntr]; + } + out++; + } + ptrData += UB_LPC_ORDER; + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_CorrelateInterVec() + * + * This is the inverse of WebRtcIsac_DecorrelateInterVec(). + * + * Input: + * -data + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : correlated parametrs. + */ +int16_t +WebRtcIsac_CorrelateInterVec( + const double* data, + double* out, + int16_t bandwidth) +{ + int16_t coeffCntr; + int16_t rowCntr; + int16_t colCntr; + int16_t interVecDim; + double myVec[UB16_LPC_VEC_PER_FRAME] = {0.0}; + const double* interVecDecorrMat; + + switch(bandwidth) + { + case isac12kHz: + { + interVecDim = UB_LPC_VEC_PER_FRAME; + interVecDecorrMat = &WebRtcIsac_kInterVecDecorrMatUb12[0][0]; + break; + } + case isac16kHz: + { + interVecDim = UB16_LPC_VEC_PER_FRAME; + interVecDecorrMat = &WebRtcIsac_kInterVecDecorrMatUb16[0][0]; + break; + } + default: + return -1; + } + + for(coeffCntr = 0; coeffCntr < UB_LPC_ORDER; coeffCntr++) + { + for(rowCntr = 0; rowCntr < interVecDim; rowCntr++) + { + myVec[rowCntr] = 0; + for(colCntr = 0; colCntr < interVecDim; colCntr++) + { + myVec[rowCntr] += data[coeffCntr + colCntr * UB_LPC_ORDER] * //*ptrData * + interVecDecorrMat[rowCntr * interVecDim + colCntr]; + //ptrData += UB_LPC_ORDER; + } + } + + for(rowCntr = 0; rowCntr < interVecDim; rowCntr++) + { + out[coeffCntr + rowCntr * UB_LPC_ORDER] = myVec[rowCntr]; + } + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_AddLarMean() + * + * This is the inverse of WebRtcIsac_RemoveLarMean() + * + * Input: + * -data : pointer to mean-removed LAR:s. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -data : pointer to LARs. + */ +int16_t +WebRtcIsac_AddLarMean( + double* data, + int16_t bandwidth) +{ + int16_t coeffCntr; + int16_t vecCntr; + int16_t numVec; + const double* meanLAR; + + switch(bandwidth) + { + case isac12kHz: + { + numVec = UB_LPC_VEC_PER_FRAME; + meanLAR = WebRtcIsac_kMeanLarUb12; + break; + } + case isac16kHz: + { + numVec = UB16_LPC_VEC_PER_FRAME; + meanLAR = WebRtcIsac_kMeanLarUb16; + break; + } + default: + return -1; + } + + for(vecCntr = 0; vecCntr < numVec; vecCntr++) + { + for(coeffCntr = 0; coeffCntr < UB_LPC_ORDER; coeffCntr++) + { + *data++ += meanLAR[coeffCntr]; + } + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_ToLogDomainRemoveMean() + * + * Transform the LPC gain to log domain then remove the mean value. + * + * Input: + * -lpcGain : pointer to LPC Gain, expecting 6 LPC gains + * + * Output: + * -lpcGain : mean-removed in log domain. + */ +int16_t +WebRtcIsac_ToLogDomainRemoveMean( + double* data) +{ + int16_t coeffCntr; + for(coeffCntr = 0; coeffCntr < UB_LPC_GAIN_DIM; coeffCntr++) + { + data[coeffCntr] = log(data[coeffCntr]) - WebRtcIsac_kMeanLpcGain; + } + return 0; +} + + +/****************************************************************************** + * WebRtcIsac_DecorrelateLPGain() + * + * Decorrelate LPC gains. There are 6 LPC Gains per frame. This is like + * multiplying gain vector with decorrelating matrix. + * + * Input: + * -data : LPC gain in log-domain with mean removed. + * + * Output: + * -out : decorrelated parameters. + */ +int16_t WebRtcIsac_DecorrelateLPGain( + const double* data, + double* out) +{ + int16_t rowCntr; + int16_t colCntr; + + for(colCntr = 0; colCntr < UB_LPC_GAIN_DIM; colCntr++) + { + *out = 0; + for(rowCntr = 0; rowCntr < UB_LPC_GAIN_DIM; rowCntr++) + { + *out += data[rowCntr] * WebRtcIsac_kLpcGainDecorrMat[rowCntr][colCntr]; + } + out++; + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_QuantizeLpcGain() + * + * Quantize the decorrelated log-domain gains. + * + * Input: + * -lpcGain : uncorrelated LPC gains. + * + * Output: + * -idx : quantization indices + * -lpcGain : quantized value of the inpt. + */ +double WebRtcIsac_QuantizeLpcGain( + double* data, + int* idx) +{ + int16_t coeffCntr; + for(coeffCntr = 0; coeffCntr < UB_LPC_GAIN_DIM; coeffCntr++) + { + *idx = (int)floor((*data - WebRtcIsac_kLeftRecPointLpcGain[coeffCntr]) / + WebRtcIsac_kQSizeLpcGain + 0.5); + + if(*idx < 0) + { + *idx = 0; + } + else if(*idx >= WebRtcIsac_kNumQCellLpcGain[coeffCntr]) + { + *idx = WebRtcIsac_kNumQCellLpcGain[coeffCntr] - 1; + } + *data = WebRtcIsac_kLeftRecPointLpcGain[coeffCntr] + *idx * + WebRtcIsac_kQSizeLpcGain; + + data++; + idx++; + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_DequantizeLpcGain() + * + * Get the quantized values given the quantization indices. + * + * Input: + * -idx : pointer to quantization indices. + * + * Output: + * -lpcGains : quantized values of the given parametes. + */ +int16_t WebRtcIsac_DequantizeLpcGain( + const int* idx, + double* out) +{ + int16_t coeffCntr; + for(coeffCntr = 0; coeffCntr < UB_LPC_GAIN_DIM; coeffCntr++) + { + *out = WebRtcIsac_kLeftRecPointLpcGain[coeffCntr] + *idx * + WebRtcIsac_kQSizeLpcGain; + out++; + idx++; + } + return 0; +} + +/****************************************************************************** + * WebRtcIsac_CorrelateLpcGain() + * + * This is the inverse of WebRtcIsac_DecorrelateLPGain(). + * + * Input: + * -data : decorrelated parameters. + * + * Output: + * -out : correlated parameters. + */ +int16_t WebRtcIsac_CorrelateLpcGain( + const double* data, + double* out) +{ + int16_t rowCntr; + int16_t colCntr; + + for(rowCntr = 0; rowCntr < UB_LPC_GAIN_DIM; rowCntr++) + { + *out = 0; + for(colCntr = 0; colCntr < UB_LPC_GAIN_DIM; colCntr++) + { + *out += WebRtcIsac_kLpcGainDecorrMat[rowCntr][colCntr] * data[colCntr]; + } + out++; + } + + return 0; +} + + +/****************************************************************************** + * WebRtcIsac_AddMeanToLinearDomain() + * + * This is the inverse of WebRtcIsac_ToLogDomainRemoveMean(). + * + * Input: + * -lpcGain : LPC gain in log-domain & mean removed + * + * Output: + * -lpcGain : LPC gain in normal domain. + */ +int16_t WebRtcIsac_AddMeanToLinearDomain( + double* lpcGains) +{ + int16_t coeffCntr; + for(coeffCntr = 0; coeffCntr < UB_LPC_GAIN_DIM; coeffCntr++) + { + lpcGains[coeffCntr] = exp(lpcGains[coeffCntr] + WebRtcIsac_kMeanLpcGain); + } + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.h new file mode 100644 index 0000000000..6609644b0c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.h @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * encode_lpc_swb.h + * + * This file contains declaration of functions used to + * encode LPC parameters (Shape & gain) of the upper band. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ENCODE_LPC_SWB_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ENCODE_LPC_SWB_H_ + +#include "settings.h" +#include "structs.h" +#include "typedefs.h" // NOLINT(build/include) + +/****************************************************************************** + * WebRtcIsac_RemoveLarMean() + * + * Remove the means from LAR coefficients. + * + * Input: + * -lar : pointer to lar vectors. LAR vectors are + * concatenated. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -lar : pointer to mean-removed LAR:s. + * + * + */ +int16_t WebRtcIsac_RemoveLarMean( + double* lar, + int16_t bandwidth); + +/****************************************************************************** + * WebRtcIsac_DecorrelateIntraVec() + * + * Remove the correlation amonge the components of LAR vectors. If LAR vectors + * of one frame are put in a matrix where each column is a LAR vector of a + * sub-frame, then this is equivalent to multiplying the LAR matrix with + * a decorrelting mtrix from left. + * + * Input: + * -inLar : pointer to mean-removed LAR vecrtors. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : decorrelated LAR vectors. + */ +int16_t WebRtcIsac_DecorrelateIntraVec( + const double* inLAR, + double* out, + int16_t bandwidth); + + +/****************************************************************************** + * WebRtcIsac_DecorrelateInterVec() + * + * Remover the correlation among mean-removed LAR vectors. If LAR vectors + * of one frame are put in a matrix where each column is a LAR vector of a + * sub-frame, then this is equivalent to multiplying the LAR matrix with + * a decorrelting mtrix from right. + * + * Input: + * -data : pointer to matrix of LAR vectors. The matrix + * is stored column-wise. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : decorrelated LAR vectors. + */ +int16_t WebRtcIsac_DecorrelateInterVec( + const double* data, + double* out, + int16_t bandwidth); + + +/****************************************************************************** + * WebRtcIsac_QuantizeUncorrLar() + * + * Quantize the uncorrelated parameters. + * + * Input: + * -data : uncorrelated LAR vectors. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -data : quantized version of the input. + * -idx : pointer to quantization indices. + */ +double WebRtcIsac_QuantizeUncorrLar( + double* data, + int* idx, + int16_t bandwidth); + + +/****************************************************************************** + * WebRtcIsac_CorrelateIntraVec() + * + * This is the inverse of WebRtcIsac_DecorrelateIntraVec(). + * + * Input: + * -data : uncorrelated parameters. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : correlated parametrs. + */ +int16_t WebRtcIsac_CorrelateIntraVec( + const double* data, + double* out, + int16_t bandwidth); + + +/****************************************************************************** + * WebRtcIsac_CorrelateInterVec() + * + * This is the inverse of WebRtcIsac_DecorrelateInterVec(). + * + * Input: + * -data + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : correlated parametrs. + */ +int16_t WebRtcIsac_CorrelateInterVec( + const double* data, + double* out, + int16_t bandwidth); + + +/****************************************************************************** + * WebRtcIsac_AddLarMean() + * + * This is the inverse of WebRtcIsac_RemoveLarMean() + * + * Input: + * -data : pointer to mean-removed LAR:s. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -data : pointer to LARs. + */ +int16_t WebRtcIsac_AddLarMean( + double* data, + int16_t bandwidth); + + +/****************************************************************************** + * WebRtcIsac_DequantizeLpcParam() + * + * Get the quantized value of uncorrelated LARs given the quantization indices. + * + * Input: + * -idx : pointer to quantiztion indices. + * -bandwidth : indicates if the given LAR vectors belong + * to SWB-12kHz or SWB-16kHz. + * + * Output: + * -out : pointer to quantized values. + */ +int16_t WebRtcIsac_DequantizeLpcParam( + const int* idx, + double* out, + int16_t bandwidth); + + +/****************************************************************************** + * WebRtcIsac_ToLogDomainRemoveMean() + * + * Transform the LPC gain to log domain then remove the mean value. + * + * Input: + * -lpcGain : pointer to LPC Gain, expecting 6 LPC gains + * + * Output: + * -lpcGain : mean-removed in log domain. + */ +int16_t WebRtcIsac_ToLogDomainRemoveMean( + double* lpGains); + + +/****************************************************************************** + * WebRtcIsac_DecorrelateLPGain() + * + * Decorrelate LPC gains. There are 6 LPC Gains per frame. This is like + * multiplying gain vector with decorrelating matrix. + * + * Input: + * -data : LPC gain in log-domain with mean removed. + * + * Output: + * -out : decorrelated parameters. + */ +int16_t WebRtcIsac_DecorrelateLPGain( + const double* data, + double* out); + + +/****************************************************************************** + * WebRtcIsac_QuantizeLpcGain() + * + * Quantize the decorrelated log-domain gains. + * + * Input: + * -lpcGain : uncorrelated LPC gains. + * + * Output: + * -idx : quantization indices + * -lpcGain : quantized value of the inpt. + */ +double WebRtcIsac_QuantizeLpcGain( + double* lpGains, + int* idx); + + +/****************************************************************************** + * WebRtcIsac_DequantizeLpcGain() + * + * Get the quantized values given the quantization indices. + * + * Input: + * -idx : pointer to quantization indices. + * + * Output: + * -lpcGains : quantized values of the given parametes. + */ +int16_t WebRtcIsac_DequantizeLpcGain( + const int* idx, + double* lpGains); + + +/****************************************************************************** + * WebRtcIsac_CorrelateLpcGain() + * + * This is the inverse of WebRtcIsac_DecorrelateLPGain(). + * + * Input: + * -data : decorrelated parameters. + * + * Output: + * -out : correlated parameters. + */ +int16_t WebRtcIsac_CorrelateLpcGain( + const double* data, + double* out); + + +/****************************************************************************** + * WebRtcIsac_AddMeanToLinearDomain() + * + * This is the inverse of WebRtcIsac_ToLogDomainRemoveMean(). + * + * Input: + * -lpcGain : LPC gain in log-domain & mean removed + * + * Output: + * -lpcGain : LPC gain in normal domain. + */ +int16_t WebRtcIsac_AddMeanToLinearDomain( + double* lpcGains); + + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ENCODE_LPC_SWB_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.c new file mode 100644 index 0000000000..b494cc287a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.c @@ -0,0 +1,2066 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * entropy_coding.c + * + * This header file defines all of the functions used to arithmetically + * encode the iSAC bistream + * + */ + + +#include "entropy_coding.h" +#include "settings.h" +#include "arith_routines.h" +#include "signal_processing_library.h" +#include "spectrum_ar_model_tables.h" +#include "lpc_tables.h" +#include "pitch_gain_tables.h" +#include "pitch_lag_tables.h" +#include "encode_lpc_swb.h" +#include "lpc_shape_swb12_tables.h" +#include "lpc_shape_swb16_tables.h" +#include "lpc_gain_swb_tables.h" +#include "os_specific_inline.h" + +#include +#include + +static const uint16_t kLpcVecPerSegmentUb12 = 5; +static const uint16_t kLpcVecPerSegmentUb16 = 4; + +/* CDF array for encoder bandwidth (12 vs 16 kHz) indicator. */ +static const uint16_t kOneBitEqualProbCdf[3] = { + 0, 32768, 65535 }; + +/* Pointer to cdf array for encoder bandwidth (12 vs 16 kHz) indicator. */ +static const uint16_t* const kOneBitEqualProbCdf_ptr[1] = { + kOneBitEqualProbCdf }; + +/* + * Initial cdf index for decoder of encoded bandwidth + * (12 vs 16 kHz) indicator. + */ +static const uint16_t kOneBitEqualProbInitIndex[1] = { 1 }; + + +static const int kIsSWB12 = 1; + +/* compute correlation from power spectrum */ +static void FindCorrelation(int32_t* PSpecQ12, int32_t* CorrQ7) { + int32_t summ[FRAMESAMPLES / 8]; + int32_t diff[FRAMESAMPLES / 8]; + const int16_t* CS_ptrQ9; + int32_t sum; + int k, n; + + for (k = 0; k < FRAMESAMPLES / 8; k++) { + summ[k] = (PSpecQ12[k] + PSpecQ12[FRAMESAMPLES_QUARTER - 1 - k] + 16) >> 5; + diff[k] = (PSpecQ12[k] - PSpecQ12[FRAMESAMPLES_QUARTER - 1 - k] + 16) >> 5; + } + + sum = 2; + for (n = 0; n < FRAMESAMPLES / 8; n++) { + sum += summ[n]; + } + CorrQ7[0] = sum; + + for (k = 0; k < AR_ORDER; k += 2) { + sum = 0; + CS_ptrQ9 = WebRtcIsac_kCos[k]; + for (n = 0; n < FRAMESAMPLES / 8; n++) + sum += (CS_ptrQ9[n] * diff[n] + 256) >> 9; + CorrQ7[k + 1] = sum; + } + + for (k = 1; k < AR_ORDER; k += 2) { + sum = 0; + CS_ptrQ9 = WebRtcIsac_kCos[k]; + for (n = 0; n < FRAMESAMPLES / 8; n++) + sum += (CS_ptrQ9[n] * summ[n] + 256) >> 9; + CorrQ7[k + 1] = sum; + } +} + +/* compute inverse AR power spectrum */ +/* Changed to the function used in iSAC FIX for compatibility reasons */ +static void FindInvArSpec(const int16_t* ARCoefQ12, + const int32_t gainQ10, + int32_t* CurveQ16) { + int32_t CorrQ11[AR_ORDER + 1]; + int32_t sum, tmpGain; + int32_t diffQ16[FRAMESAMPLES / 8]; + const int16_t* CS_ptrQ9; + int k, n; + int16_t round, shftVal = 0, sh; + + sum = 0; + for (n = 0; n < AR_ORDER + 1; n++) { + sum += WEBRTC_SPL_MUL(ARCoefQ12[n], ARCoefQ12[n]); /* Q24 */ + } + sum = ((sum >> 6) * 65 + 32768) >> 16; /* Q8 */ + CorrQ11[0] = (sum * gainQ10 + 256) >> 9; + + /* To avoid overflow, we shift down gainQ10 if it is large. + * We will not lose any precision */ + if (gainQ10 > 400000) { + tmpGain = gainQ10 >> 3; + round = 32; + shftVal = 6; + } else { + tmpGain = gainQ10; + round = 256; + shftVal = 9; + } + + for (k = 1; k < AR_ORDER + 1; k++) { + sum = 16384; + for (n = k; n < AR_ORDER + 1; n++) + sum += WEBRTC_SPL_MUL(ARCoefQ12[n - k], ARCoefQ12[n]); /* Q24 */ + sum >>= 15; + CorrQ11[k] = (sum * tmpGain + round) >> shftVal; + } + sum = CorrQ11[0] << 7; + for (n = 0; n < FRAMESAMPLES / 8; n++) { + CurveQ16[n] = sum; + } + for (k = 1; k < AR_ORDER; k += 2) { + for (n = 0; n < FRAMESAMPLES / 8; n++) { + CurveQ16[n] += (WebRtcIsac_kCos[k][n] * CorrQ11[k + 1] + 2) >> 2; + } + } + + CS_ptrQ9 = WebRtcIsac_kCos[0]; + + /* If CorrQ11[1] too large we avoid getting overflow in the + * calculation by shifting */ + sh = WebRtcSpl_NormW32(CorrQ11[1]); + if (CorrQ11[1] == 0) { /* Use next correlation */ + sh = WebRtcSpl_NormW32(CorrQ11[2]); + } + if (sh < 9) { + shftVal = 9 - sh; + } else { + shftVal = 0; + } + for (n = 0; n < FRAMESAMPLES / 8; n++) { + diffQ16[n] = (CS_ptrQ9[n] * (CorrQ11[1] >> shftVal) + 2) >> 2; + } + for (k = 2; k < AR_ORDER; k += 2) { + CS_ptrQ9 = WebRtcIsac_kCos[k]; + for (n = 0; n < FRAMESAMPLES / 8; n++) { + diffQ16[n] += (CS_ptrQ9[n] * (CorrQ11[k + 1] >> shftVal) + 2) >> 2; + } + } + + for (k = 0; k < FRAMESAMPLES / 8; k++) { + int32_t diff_q16_shifted = (int32_t)((uint32_t)(diffQ16[k]) << shftVal); + CurveQ16[FRAMESAMPLES_QUARTER - 1 - k] = CurveQ16[k] - diff_q16_shifted; + CurveQ16[k] += diff_q16_shifted; + } +} + +/* Generate array of dither samples in Q7. */ +static void GenerateDitherQ7Lb(int16_t* bufQ7, uint32_t seed, + int length, int16_t AvgPitchGain_Q12) { + int k, shft; + int16_t dither1_Q7, dither2_Q7, dither_gain_Q14; + + /* This threshold should be equal to that in decode_spec(). */ + if (AvgPitchGain_Q12 < 614) { + for (k = 0; k < length - 2; k += 3) { + /* New random unsigned int. */ + seed = (seed * 196314165) + 907633515; + + /* Fixed-point dither sample between -64 and 64 (Q7). */ + /* dither = seed * 128 / 4294967295 */ + dither1_Q7 = (int16_t)(((int32_t)(seed + 16777216)) >> 25); + + /* New random unsigned int. */ + seed = (seed * 196314165) + 907633515; + + /* Fixed-point dither sample between -64 and 64. */ + dither2_Q7 = (int16_t)(((int32_t)(seed + 16777216)) >> 25); + + shft = (seed >> 25) & 15; + if (shft < 5) { + bufQ7[k] = dither1_Q7; + bufQ7[k + 1] = dither2_Q7; + bufQ7[k + 2] = 0; + } else if (shft < 10) { + bufQ7[k] = dither1_Q7; + bufQ7[k + 1] = 0; + bufQ7[k + 2] = dither2_Q7; + } else { + bufQ7[k] = 0; + bufQ7[k + 1] = dither1_Q7; + bufQ7[k + 2] = dither2_Q7; + } + } + } else { + dither_gain_Q14 = (int16_t)(22528 - 10 * AvgPitchGain_Q12); + + /* Dither on half of the coefficients. */ + for (k = 0; k < length - 1; k += 2) { + /* New random unsigned int */ + seed = (seed * 196314165) + 907633515; + + /* Fixed-point dither sample between -64 and 64. */ + dither1_Q7 = (int16_t)(((int32_t)(seed + 16777216)) >> 25); + + /* Dither sample is placed in either even or odd index. */ + shft = (seed >> 25) & 1; /* Either 0 or 1 */ + + bufQ7[k + shft] = (((dither_gain_Q14 * dither1_Q7) + 8192) >> 14); + bufQ7[k + 1 - shft] = 0; + } + } +} + + + +/****************************************************************************** + * GenerateDitherQ7LbUB() + * + * generate array of dither samples in Q7 There are less zeros in dither + * vector compared to GenerateDitherQ7Lb. + * + * A uniform random number generator with the range of [-64 64] is employed + * but the generated dithers are scaled by 0.35, a heuristic scaling. + * + * Input: + * -seed : the initial seed for the random number generator. + * -length : the number of dither values to be generated. + * + * Output: + * -bufQ7 : pointer to a buffer where dithers are written to. + */ +static void GenerateDitherQ7LbUB( + int16_t* bufQ7, + uint32_t seed, + int length) { + int k; + for (k = 0; k < length; k++) { + /* new random unsigned int */ + seed = (seed * 196314165) + 907633515; + + /* Fixed-point dither sample between -64 and 64 (Q7). */ + /* bufQ7 = seed * 128 / 4294967295 */ + bufQ7[k] = (int16_t)(((int32_t)(seed + 16777216)) >> 25); + + /* Scale by 0.35. */ + bufQ7[k] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(bufQ7[k], 2048, 13); + } +} + +/* + * Function to decode the complex spectrum from the bit stream + * returns the total number of bytes in the stream. + */ +int WebRtcIsac_DecodeSpec(Bitstr* streamdata, int16_t AvgPitchGain_Q12, + enum ISACBand band, double* fr, double* fi) { + int16_t DitherQ7[FRAMESAMPLES]; + int16_t data[FRAMESAMPLES]; + int32_t invARSpec2_Q16[FRAMESAMPLES_QUARTER]; + uint16_t invARSpecQ8[FRAMESAMPLES_QUARTER]; + int16_t ARCoefQ12[AR_ORDER + 1]; + int16_t RCQ15[AR_ORDER]; + int16_t gainQ10; + int32_t gain2_Q10, res; + int32_t in_sqrt; + int32_t newRes; + int k, len, i; + int is_12khz = !kIsSWB12; + int num_dft_coeff = FRAMESAMPLES; + /* Create dither signal. */ + if (band == kIsacLowerBand) { + GenerateDitherQ7Lb(DitherQ7, streamdata->W_upper, FRAMESAMPLES, + AvgPitchGain_Q12); + } else { + GenerateDitherQ7LbUB(DitherQ7, streamdata->W_upper, FRAMESAMPLES); + if (band == kIsacUpperBand12) { + is_12khz = kIsSWB12; + num_dft_coeff = FRAMESAMPLES_HALF; + } + } + + /* Decode model parameters. */ + if (WebRtcIsac_DecodeRc(streamdata, RCQ15) < 0) + return -ISAC_RANGE_ERROR_DECODE_SPECTRUM; + + WebRtcSpl_ReflCoefToLpc(RCQ15, AR_ORDER, ARCoefQ12); + + if (WebRtcIsac_DecodeGain2(streamdata, &gain2_Q10) < 0) + return -ISAC_RANGE_ERROR_DECODE_SPECTRUM; + + /* Compute inverse AR power spectrum. */ + FindInvArSpec(ARCoefQ12, gain2_Q10, invARSpec2_Q16); + + /* Convert to magnitude spectrum, + * by doing square-roots (modified from SPLIB). */ + res = 1 << (WebRtcSpl_GetSizeInBits(invARSpec2_Q16[0]) >> 1); + for (k = 0; k < FRAMESAMPLES_QUARTER; k++) { + in_sqrt = invARSpec2_Q16[k]; + i = 10; + + /* Negative values make no sense for a real sqrt-function. */ + if (in_sqrt < 0) + in_sqrt = -in_sqrt; + + newRes = (in_sqrt / res + res) >> 1; + do { + res = newRes; + newRes = (in_sqrt / res + res) >> 1; + } while (newRes != res && i-- > 0); + + invARSpecQ8[k] = (int16_t)newRes; + } + + len = WebRtcIsac_DecLogisticMulti2(data, streamdata, invARSpecQ8, DitherQ7, + num_dft_coeff, is_12khz); + /* Arithmetic decoding of spectrum. */ + if (len < 1) { + return -ISAC_RANGE_ERROR_DECODE_SPECTRUM; + } + + switch (band) { + case kIsacLowerBand: { + /* Scale down spectral samples with low SNR. */ + int32_t p1; + int32_t p2; + if (AvgPitchGain_Q12 <= 614) { + p1 = 30 << 10; + p2 = 32768 + (33 << 16); + } else { + p1 = 36 << 10; + p2 = 32768 + (40 << 16); + } + for (k = 0; k < FRAMESAMPLES; k += 4) { + gainQ10 = WebRtcSpl_DivW32W16ResW16(p1, (int16_t)( + (invARSpec2_Q16[k >> 2] + p2) >> 16)); + *fr++ = (double)((data[ k ] * gainQ10 + 512) >> 10) / 128.0; + *fi++ = (double)((data[k + 1] * gainQ10 + 512) >> 10) / 128.0; + *fr++ = (double)((data[k + 2] * gainQ10 + 512) >> 10) / 128.0; + *fi++ = (double)((data[k + 3] * gainQ10 + 512) >> 10) / 128.0; + } + break; + } + case kIsacUpperBand12: { + for (k = 0, i = 0; k < FRAMESAMPLES_HALF; k += 4) { + fr[i] = (double)data[ k ] / 128.0; + fi[i] = (double)data[k + 1] / 128.0; + i++; + fr[i] = (double)data[k + 2] / 128.0; + fi[i] = (double)data[k + 3] / 128.0; + i++; + } + /* The second half of real and imaginary coefficients is zero. This is + * due to using the old FFT module which requires two signals as input + * while in 0-12 kHz mode we only have 8-12 kHz band, and the second + * signal is set to zero. */ + memset(&fr[FRAMESAMPLES_QUARTER], 0, FRAMESAMPLES_QUARTER * + sizeof(double)); + memset(&fi[FRAMESAMPLES_QUARTER], 0, FRAMESAMPLES_QUARTER * + sizeof(double)); + break; + } + case kIsacUpperBand16: { + for (i = 0, k = 0; k < FRAMESAMPLES; k += 4, i++) { + fr[i] = (double)data[ k ] / 128.0; + fi[i] = (double)data[k + 1] / 128.0; + fr[(FRAMESAMPLES_HALF) - 1 - i] = (double)data[k + 2] / 128.0; + fi[(FRAMESAMPLES_HALF) - 1 - i] = (double)data[k + 3] / 128.0; + } + break; + } + } + return len; +} + + +int WebRtcIsac_EncodeSpec(const int16_t* fr, const int16_t* fi, + int16_t AvgPitchGain_Q12, enum ISACBand band, + Bitstr* streamdata) { + int16_t ditherQ7[FRAMESAMPLES]; + int16_t dataQ7[FRAMESAMPLES]; + int32_t PSpec[FRAMESAMPLES_QUARTER]; + int32_t invARSpec2_Q16[FRAMESAMPLES_QUARTER]; + uint16_t invARSpecQ8[FRAMESAMPLES_QUARTER]; + int32_t CorrQ7[AR_ORDER + 1]; + int32_t CorrQ7_norm[AR_ORDER + 1]; + int16_t RCQ15[AR_ORDER]; + int16_t ARCoefQ12[AR_ORDER + 1]; + int32_t gain2_Q10; + int16_t val; + int32_t nrg, res; + uint32_t sum; + int32_t in_sqrt; + int32_t newRes; + int16_t err; + uint32_t nrg_u32; + int shift_var; + int k, n, j, i; + int is_12khz = !kIsSWB12; + int num_dft_coeff = FRAMESAMPLES; + + /* Create dither signal. */ + if (band == kIsacLowerBand) { + GenerateDitherQ7Lb(ditherQ7, streamdata->W_upper, FRAMESAMPLES, + AvgPitchGain_Q12); + } else { + GenerateDitherQ7LbUB(ditherQ7, streamdata->W_upper, FRAMESAMPLES); + if (band == kIsacUpperBand12) { + is_12khz = kIsSWB12; + num_dft_coeff = FRAMESAMPLES_HALF; + } + } + + /* add dither and quantize, and compute power spectrum */ + switch (band) { + case kIsacLowerBand: { + for (k = 0; k < FRAMESAMPLES; k += 4) { + val = ((*fr++ + ditherQ7[k] + 64) & 0xFF80) - ditherQ7[k]; + dataQ7[k] = val; + sum = val * val; + + val = ((*fi++ + ditherQ7[k + 1] + 64) & 0xFF80) - ditherQ7[k + 1]; + dataQ7[k + 1] = val; + sum += val * val; + + val = ((*fr++ + ditherQ7[k + 2] + 64) & 0xFF80) - ditherQ7[k + 2]; + dataQ7[k + 2] = val; + sum += val * val; + + val = ((*fi++ + ditherQ7[k + 3] + 64) & 0xFF80) - ditherQ7[k + 3]; + dataQ7[k + 3] = val; + sum += val * val; + + PSpec[k >> 2] = sum >> 2; + } + break; + } + case kIsacUpperBand12: { + for (k = 0, j = 0; k < FRAMESAMPLES_HALF; k += 4) { + val = ((*fr++ + ditherQ7[k] + 64) & 0xFF80) - ditherQ7[k]; + dataQ7[k] = val; + sum = val * val; + + val = ((*fi++ + ditherQ7[k + 1] + 64) & 0xFF80) - ditherQ7[k + 1]; + dataQ7[k + 1] = val; + sum += val * val; + + PSpec[j++] = sum >> 1; + + val = ((*fr++ + ditherQ7[k + 2] + 64) & 0xFF80) - ditherQ7[k + 2]; + dataQ7[k + 2] = val; + sum = val * val; + + val = ((*fi++ + ditherQ7[k + 3] + 64) & 0xFF80) - ditherQ7[k + 3]; + dataQ7[k + 3] = val; + sum += val * val; + + PSpec[j++] = sum >> 1; + } + break; + } + case kIsacUpperBand16: { + for (j = 0, k = 0; k < FRAMESAMPLES; k += 4, j++) { + val = ((fr[j] + ditherQ7[k] + 64) & 0xFF80) - ditherQ7[k]; + dataQ7[k] = val; + sum = val * val; + + val = ((fi[j] + ditherQ7[k + 1] + 64) & 0xFF80) - ditherQ7[k + 1]; + dataQ7[k + 1] = val; + sum += val * val; + + val = ((fr[(FRAMESAMPLES_HALF) - 1 - j] + ditherQ7[k + 2] + 64) & + 0xFF80) - ditherQ7[k + 2]; + dataQ7[k + 2] = val; + sum += val * val; + + val = ((fi[(FRAMESAMPLES_HALF) - 1 - j] + ditherQ7[k + 3] + 64) & + 0xFF80) - ditherQ7[k + 3]; + dataQ7[k + 3] = val; + sum += val * val; + + PSpec[k >> 2] = sum >> 2; + } + break; + } + } + + /* compute correlation from power spectrum */ + FindCorrelation(PSpec, CorrQ7); + + /* Find AR coefficients */ + /* Aumber of bit shifts to 14-bit normalize CorrQ7[0] + * (leaving room for sign) */ + shift_var = WebRtcSpl_NormW32(CorrQ7[0]) - 18; + + if (shift_var > 0) { + for (k = 0; k < AR_ORDER + 1; k++) { + CorrQ7_norm[k] = CorrQ7[k] << shift_var; + } + } else { + for (k = 0; k < AR_ORDER + 1; k++) { + CorrQ7_norm[k] = CorrQ7[k] >> (-shift_var); + } + } + + /* Find RC coefficients. */ + WebRtcSpl_AutoCorrToReflCoef(CorrQ7_norm, AR_ORDER, RCQ15); + + /* Quantize & code RC Coefficient. */ + WebRtcIsac_EncodeRc(RCQ15, streamdata); + + /* RC -> AR coefficients */ + WebRtcSpl_ReflCoefToLpc(RCQ15, AR_ORDER, ARCoefQ12); + + /* Compute ARCoef' * Corr * ARCoef in Q19. */ + nrg = 0; + for (j = 0; j <= AR_ORDER; j++) { + for (n = 0; n <= j; n++) { + nrg += (ARCoefQ12[j] * ((CorrQ7_norm[j - n] * ARCoefQ12[n] + 256) >> 9) + + 4) >> 3; + } + for (n = j + 1; n <= AR_ORDER; n++) { + nrg += (ARCoefQ12[j] * ((CorrQ7_norm[n - j] * ARCoefQ12[n] + 256) >> 9) + + 4) >> 3; + } + } + + nrg_u32 = (uint32_t)nrg; + if (shift_var > 0) { + nrg_u32 = nrg_u32 >> shift_var; + } else { + nrg_u32 = nrg_u32 << (-shift_var); + } + if (nrg_u32 > 0x7FFFFFFF) { + nrg = 0x7FFFFFFF; + } else { + nrg = (int32_t)nrg_u32; + } + /* Also shifts 31 bits to the left! */ + gain2_Q10 = WebRtcSpl_DivResultInQ31(FRAMESAMPLES_QUARTER, nrg); + + /* Quantize & code gain2_Q10. */ + if (WebRtcIsac_EncodeGain2(&gain2_Q10, streamdata)) { + return -1; + } + + /* Compute inverse AR power spectrum. */ + FindInvArSpec(ARCoefQ12, gain2_Q10, invARSpec2_Q16); + /* Convert to magnitude spectrum, by doing square-roots + * (modified from SPLIB). */ + res = 1 << (WebRtcSpl_GetSizeInBits(invARSpec2_Q16[0]) >> 1); + for (k = 0; k < FRAMESAMPLES_QUARTER; k++) { + in_sqrt = invARSpec2_Q16[k]; + i = 10; + /* Negative values make no sense for a real sqrt-function. */ + if (in_sqrt < 0) { + in_sqrt = -in_sqrt; + } + newRes = (in_sqrt / res + res) >> 1; + do { + res = newRes; + newRes = (in_sqrt / res + res) >> 1; + } while (newRes != res && i-- > 0); + + invARSpecQ8[k] = (int16_t)newRes; + } + /* arithmetic coding of spectrum */ + err = WebRtcIsac_EncLogisticMulti2(streamdata, dataQ7, invARSpecQ8, + num_dft_coeff, is_12khz); + if (err < 0) { + return (err); + } + return 0; +} + + +/* step-up */ +void WebRtcIsac_Rc2Poly(double* RC, int N, double* a) { + int m, k; + double tmp[MAX_AR_MODEL_ORDER]; + + a[0] = 1.0; + tmp[0] = 1.0; + for (m = 1; m <= N; m++) { + /* copy */ + memcpy(&tmp[1], &a[1], (m - 1) * sizeof(double)); + a[m] = RC[m - 1]; + for (k = 1; k < m; k++) { + a[k] += RC[m - 1] * tmp[m - k]; + } + } + return; +} + +/* step-down */ +void WebRtcIsac_Poly2Rc(double* a, int N, double* RC) { + int m, k; + double tmp[MAX_AR_MODEL_ORDER]; + double tmp_inv; + + RC[N - 1] = a[N]; + for (m = N - 1; m > 0; m--) { + tmp_inv = 1.0 / (1.0 - RC[m] * RC[m]); + for (k = 1; k <= m; k++) { + tmp[k] = (a[k] - RC[m] * a[m - k + 1]) * tmp_inv; + } + + memcpy(&a[1], &tmp[1], (m - 1) * sizeof(double)); + RC[m - 1] = tmp[m]; + } + return; +} + + +#define MAX_ORDER 100 + +/* Matlab's LAR definition */ +void WebRtcIsac_Rc2Lar(const double* refc, double* lar, int order) { + int k; + for (k = 0; k < order; k++) { + lar[k] = log((1 + refc[k]) / (1 - refc[k])); + } +} + + +void WebRtcIsac_Lar2Rc(const double* lar, double* refc, int order) { + int k; + double tmp; + + for (k = 0; k < order; k++) { + tmp = exp(lar[k]); + refc[k] = (tmp - 1) / (tmp + 1); + } +} + +void WebRtcIsac_Poly2Lar(double* lowband, int orderLo, double* hiband, + int orderHi, int Nsub, double* lars) { + int k; + double rc[MAX_ORDER], *inpl, *inph, *outp; + + inpl = lowband; + inph = hiband; + outp = lars; + for (k = 0; k < Nsub; k++) { + /* gains */ + outp[0] = inpl[0]; + outp[1] = inph[0]; + outp += 2; + + /* Low band */ + inpl[0] = 1.0; + WebRtcIsac_Poly2Rc(inpl, orderLo, rc); + WebRtcIsac_Rc2Lar(rc, outp, orderLo); + outp += orderLo; + + /* High band */ + inph[0] = 1.0; + WebRtcIsac_Poly2Rc(inph, orderHi, rc); + WebRtcIsac_Rc2Lar(rc, outp, orderHi); + outp += orderHi; + + inpl += orderLo + 1; + inph += orderHi + 1; + } +} + + +int16_t WebRtcIsac_Poly2LarUB(double* lpcVecs, int16_t bandwidth) { + double poly[MAX_ORDER]; + double rc[MAX_ORDER]; + double* ptrIO; + int16_t vecCntr; + int16_t vecSize; + int16_t numVec; + + vecSize = UB_LPC_ORDER; + switch (bandwidth) { + case isac12kHz: { + numVec = UB_LPC_VEC_PER_FRAME; + break; + } + case isac16kHz: { + numVec = UB16_LPC_VEC_PER_FRAME; + break; + } + default: + return -1; + } + + ptrIO = lpcVecs; + poly[0] = 1.0; + for (vecCntr = 0; vecCntr < numVec; vecCntr++) { + memcpy(&poly[1], ptrIO, sizeof(double) * vecSize); + WebRtcIsac_Poly2Rc(poly, vecSize, rc); + WebRtcIsac_Rc2Lar(rc, ptrIO, vecSize); + ptrIO += vecSize; + } + return 0; +} + + +void WebRtcIsac_Lar2Poly(double* lars, double* lowband, int orderLo, + double* hiband, int orderHi, int Nsub) { + int k, orderTot; + double rc[MAX_ORDER], *outpl, *outph, *inp; + + orderTot = (orderLo + orderHi + 2); + outpl = lowband; + outph = hiband; + /* First two elements of 'inp' store gains*/ + inp = lars; + for (k = 0; k < Nsub; k++) { + /* Low band */ + WebRtcIsac_Lar2Rc(&inp[2], rc, orderLo); + WebRtcIsac_Rc2Poly(rc, orderLo, outpl); + + /* High band */ + WebRtcIsac_Lar2Rc(&inp[orderLo + 2], rc, orderHi); + WebRtcIsac_Rc2Poly(rc, orderHi, outph); + + /* gains */ + outpl[0] = inp[0]; + outph[0] = inp[1]; + + outpl += orderLo + 1; + outph += orderHi + 1; + inp += orderTot; + } +} + +/* + * assumes 2 LAR vectors interpolates to 'numPolyVec' A-polynomials + * Note: 'numPolyVecs' includes the first and the last point of the interval + */ +void WebRtcIsac_Lar2PolyInterpolUB(double* larVecs, double* percepFilterParams, + int numPolyVecs) { + int polyCntr, coeffCntr; + double larInterpol[UB_LPC_ORDER]; + double rc[UB_LPC_ORDER]; + double delta[UB_LPC_ORDER]; + + /* calculate the step-size for linear interpolation coefficients */ + for (coeffCntr = 0; coeffCntr < UB_LPC_ORDER; coeffCntr++) { + delta[coeffCntr] = (larVecs[UB_LPC_ORDER + coeffCntr] - + larVecs[coeffCntr]) / (numPolyVecs - 1); + } + + for (polyCntr = 0; polyCntr < numPolyVecs; polyCntr++) { + for (coeffCntr = 0; coeffCntr < UB_LPC_ORDER; coeffCntr++) { + larInterpol[coeffCntr] = larVecs[coeffCntr] + + delta[coeffCntr] * polyCntr; + } + WebRtcIsac_Lar2Rc(larInterpol, rc, UB_LPC_ORDER); + + /* convert to A-polynomial, the following function returns A[0] = 1; + * which is written where gains had to be written. Then we write the + * gain (outside this function). This way we say a memcpy. */ + WebRtcIsac_Rc2Poly(rc, UB_LPC_ORDER, percepFilterParams); + percepFilterParams += (UB_LPC_ORDER + 1); + } +} + +int WebRtcIsac_DecodeLpc(Bitstr* streamdata, double* LPCCoef_lo, + double* LPCCoef_hi) { + double lars[KLT_ORDER_GAIN + KLT_ORDER_SHAPE]; + int err; + + err = WebRtcIsac_DecodeLpcCoef(streamdata, lars); + if (err < 0) { + return -ISAC_RANGE_ERROR_DECODE_LPC; + } + WebRtcIsac_Lar2Poly(lars, LPCCoef_lo, ORDERLO, LPCCoef_hi, ORDERHI, + SUBFRAMES); + return 0; +} + +int16_t WebRtcIsac_DecodeInterpolLpcUb(Bitstr* streamdata, + double* percepFilterParams, + int16_t bandwidth) { + double lpcCoeff[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + int err; + int interpolCntr; + int subframeCntr; + int16_t numSegments; + int16_t numVecPerSegment; + int16_t numGains; + + double percepFilterGains[SUBFRAMES << 1]; + double* ptrOutParam = percepFilterParams; + + err = WebRtcIsac_DecodeLpcCoefUB(streamdata, lpcCoeff, percepFilterGains, + bandwidth); + if (err < 0) { + return -ISAC_RANGE_ERROR_DECODE_LPC; + } + + switch (bandwidth) { + case isac12kHz: { + numGains = SUBFRAMES; + numSegments = UB_LPC_VEC_PER_FRAME - 1; + numVecPerSegment = kLpcVecPerSegmentUb12; + break; + } + case isac16kHz: { + numGains = SUBFRAMES << 1; + numSegments = UB16_LPC_VEC_PER_FRAME - 1; + numVecPerSegment = kLpcVecPerSegmentUb16; + break; + } + default: + return -1; + } + + for (interpolCntr = 0; interpolCntr < numSegments; interpolCntr++) { + WebRtcIsac_Lar2PolyInterpolUB(&lpcCoeff[interpolCntr * UB_LPC_ORDER], + ptrOutParam, numVecPerSegment + 1); + ptrOutParam += (numVecPerSegment * (UB_LPC_ORDER + 1)); + } + + ptrOutParam = percepFilterParams; + + if (bandwidth == isac16kHz) { + ptrOutParam += (1 + UB_LPC_ORDER); + } + + for (subframeCntr = 0; subframeCntr < numGains; subframeCntr++) { + *ptrOutParam = percepFilterGains[subframeCntr]; + ptrOutParam += (1 + UB_LPC_ORDER); + } + return 0; +} + + +/* decode & dequantize LPC Coef */ +int WebRtcIsac_DecodeLpcCoef(Bitstr* streamdata, double* LPCCoef) { + int j, k, n, pos, pos2, posg, poss, offsg, offss, offs2; + int index_g[KLT_ORDER_GAIN], index_s[KLT_ORDER_SHAPE]; + double tmpcoeffs_g[KLT_ORDER_GAIN], tmpcoeffs_s[KLT_ORDER_SHAPE]; + double tmpcoeffs2_g[KLT_ORDER_GAIN], tmpcoeffs2_s[KLT_ORDER_SHAPE]; + double sum; + int err; + int model = 1; + + /* entropy decoding of model number */ + /* We are keeping this for backward compatibility of bit-streams. */ + err = WebRtcIsac_DecHistOneStepMulti(&model, streamdata, + WebRtcIsac_kQKltModelCdfPtr, + WebRtcIsac_kQKltModelInitIndex, 1); + if (err < 0) { + return err; + } + /* Only accepted value of model is 0. It is kept in bit-stream for backward + * compatibility. */ + if (model != 0) { + return -ISAC_DISALLOWED_LPC_MODEL; + } + + /* entropy decoding of quantization indices */ + err = WebRtcIsac_DecHistOneStepMulti( + index_s, streamdata, WebRtcIsac_kQKltCdfPtrShape, + WebRtcIsac_kQKltInitIndexShape, KLT_ORDER_SHAPE); + if (err < 0) { + return err; + } + err = WebRtcIsac_DecHistOneStepMulti( + index_g, streamdata, WebRtcIsac_kQKltCdfPtrGain, + WebRtcIsac_kQKltInitIndexGain, KLT_ORDER_GAIN); + if (err < 0) { + return err; + } + + /* find quantization levels for coefficients */ + for (k = 0; k < KLT_ORDER_SHAPE; k++) { + tmpcoeffs_s[k] = + WebRtcIsac_kQKltLevelsShape[WebRtcIsac_kQKltOffsetShape[k] + + index_s[k]]; + } + for (k = 0; k < KLT_ORDER_GAIN; k++) { + tmpcoeffs_g[k] = WebRtcIsac_kQKltLevelsGain[WebRtcIsac_kQKltOffsetGain[k] + + index_g[k]]; + } + + /* Inverse KLT */ + + /* Left transform, transpose matrix! */ + offsg = 0; + offss = 0; + posg = 0; + poss = 0; + for (j = 0; j < SUBFRAMES; j++) { + offs2 = 0; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = offsg; + pos2 = offs2; + for (n = 0; n < LPC_GAIN_ORDER; n++) { + sum += tmpcoeffs_g[pos++] * WebRtcIsac_kKltT1Gain[pos2++]; + } + tmpcoeffs2_g[posg++] = sum; + offs2 += LPC_GAIN_ORDER; + } + offs2 = 0; + for (k = 0; k < LPC_SHAPE_ORDER; k++) { + sum = 0; + pos = offss; + pos2 = offs2; + for (n = 0; n < LPC_SHAPE_ORDER; n++) { + sum += tmpcoeffs_s[pos++] * WebRtcIsac_kKltT1Shape[pos2++]; + } + tmpcoeffs2_s[poss++] = sum; + offs2 += LPC_SHAPE_ORDER; + } + offsg += LPC_GAIN_ORDER; + offss += LPC_SHAPE_ORDER; + } + + /* Right transform, transpose matrix */ + offsg = 0; + offss = 0; + posg = 0; + poss = 0; + for (j = 0; j < SUBFRAMES; j++) { + posg = offsg; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = k; + pos2 = j; + for (n = 0; n < SUBFRAMES; n++) { + sum += tmpcoeffs2_g[pos] * WebRtcIsac_kKltT2Gain[pos2]; + pos += LPC_GAIN_ORDER; + pos2 += SUBFRAMES; + + } + tmpcoeffs_g[posg++] = sum; + } + poss = offss; + for (k = 0; k < LPC_SHAPE_ORDER; k++) { + sum = 0; + pos = k; + pos2 = j; + for (n = 0; n < SUBFRAMES; n++) { + sum += tmpcoeffs2_s[pos] * WebRtcIsac_kKltT2Shape[pos2]; + pos += LPC_SHAPE_ORDER; + pos2 += SUBFRAMES; + } + tmpcoeffs_s[poss++] = sum; + } + offsg += LPC_GAIN_ORDER; + offss += LPC_SHAPE_ORDER; + } + + /* scaling, mean addition, and gain restoration */ + posg = 0; + poss = 0; + pos = 0; + for (k = 0; k < SUBFRAMES; k++) { + /* log gains */ + LPCCoef[pos] = tmpcoeffs_g[posg] / LPC_GAIN_SCALE; + LPCCoef[pos] += WebRtcIsac_kLpcMeansGain[posg]; + LPCCoef[pos] = exp(LPCCoef[pos]); + pos++; + posg++; + LPCCoef[pos] = tmpcoeffs_g[posg] / LPC_GAIN_SCALE; + LPCCoef[pos] += WebRtcIsac_kLpcMeansGain[posg]; + LPCCoef[pos] = exp(LPCCoef[pos]); + pos++; + posg++; + + /* Low-band LAR coefficients. */ + for (n = 0; n < LPC_LOBAND_ORDER; n++, pos++, poss++) { + LPCCoef[pos] = tmpcoeffs_s[poss] / LPC_LOBAND_SCALE; + LPCCoef[pos] += WebRtcIsac_kLpcMeansShape[poss]; + } + + /* High-band LAR coefficients. */ + for (n = 0; n < LPC_HIBAND_ORDER; n++, pos++, poss++) { + LPCCoef[pos] = tmpcoeffs_s[poss] / LPC_HIBAND_SCALE; + LPCCoef[pos] += WebRtcIsac_kLpcMeansShape[poss]; + } + } + return 0; +} + +/* Encode LPC in LAR domain. */ +void WebRtcIsac_EncodeLar(double* LPCCoef, Bitstr* streamdata, + IsacSaveEncoderData* encData) { + int j, k, n, pos, pos2, poss, offss, offs2; + int index_s[KLT_ORDER_SHAPE]; + int index_ovr_s[KLT_ORDER_SHAPE]; + double tmpcoeffs_s[KLT_ORDER_SHAPE]; + double tmpcoeffs2_s[KLT_ORDER_SHAPE]; + double sum; + const int kModel = 0; + + /* Mean removal and scaling. */ + poss = 0; + pos = 0; + for (k = 0; k < SUBFRAMES; k++) { + /* First two element are gains, move over them. */ + pos += 2; + + /* Low-band LAR coefficients. */ + for (n = 0; n < LPC_LOBAND_ORDER; n++, poss++, pos++) { + tmpcoeffs_s[poss] = LPCCoef[pos] - WebRtcIsac_kLpcMeansShape[poss]; + tmpcoeffs_s[poss] *= LPC_LOBAND_SCALE; + } + + /* High-band LAR coefficients. */ + for (n = 0; n < LPC_HIBAND_ORDER; n++, poss++, pos++) { + tmpcoeffs_s[poss] = LPCCoef[pos] - WebRtcIsac_kLpcMeansShape[poss]; + tmpcoeffs_s[poss] *= LPC_HIBAND_SCALE; + } + } + + /* KLT */ + + /* Left transform. */ + offss = 0; + for (j = 0; j < SUBFRAMES; j++) { + poss = offss; + for (k = 0; k < LPC_SHAPE_ORDER; k++) { + sum = 0; + pos = offss; + pos2 = k; + for (n = 0; n < LPC_SHAPE_ORDER; n++) { + sum += tmpcoeffs_s[pos++] * WebRtcIsac_kKltT1Shape[pos2]; + pos2 += LPC_SHAPE_ORDER; + } + tmpcoeffs2_s[poss++] = sum; + } + offss += LPC_SHAPE_ORDER; + } + + /* Right transform. */ + offss = 0; + offs2 = 0; + for (j = 0; j < SUBFRAMES; j++) { + poss = offss; + for (k = 0; k < LPC_SHAPE_ORDER; k++) { + sum = 0; + pos = k; + pos2 = offs2; + for (n = 0; n < SUBFRAMES; n++) { + sum += tmpcoeffs2_s[pos] * WebRtcIsac_kKltT2Shape[pos2++]; + pos += LPC_SHAPE_ORDER; + } + tmpcoeffs_s[poss++] = sum; + } + offs2 += SUBFRAMES; + offss += LPC_SHAPE_ORDER; + } + + /* Quantize coefficients. */ + for (k = 0; k < KLT_ORDER_SHAPE; k++) { + index_s[k] = (WebRtcIsac_lrint(tmpcoeffs_s[k] / KLT_STEPSIZE)) + + WebRtcIsac_kQKltQuantMinShape[k]; + if (index_s[k] < 0) { + index_s[k] = 0; + } else if (index_s[k] > WebRtcIsac_kQKltMaxIndShape[k]) { + index_s[k] = WebRtcIsac_kQKltMaxIndShape[k]; + } + index_ovr_s[k] = WebRtcIsac_kQKltOffsetShape[k] + index_s[k]; + } + + + /* Only one model remains in this version of the code, kModel = 0. We + * are keeping for bit-streams to be backward compatible. */ + /* entropy coding of model number */ + WebRtcIsac_EncHistMulti(streamdata, &kModel, WebRtcIsac_kQKltModelCdfPtr, 1); + + /* Save data for creation of multiple bit streams */ + /* Entropy coding of quantization indices - shape only. */ + WebRtcIsac_EncHistMulti(streamdata, index_s, WebRtcIsac_kQKltCdfPtrShape, + KLT_ORDER_SHAPE); + + /* Save data for creation of multiple bit streams. */ + for (k = 0; k < KLT_ORDER_SHAPE; k++) { + encData->LPCindex_s[KLT_ORDER_SHAPE * encData->startIdx + k] = index_s[k]; + } + + /* Find quantization levels for shape coefficients. */ + for (k = 0; k < KLT_ORDER_SHAPE; k++) { + tmpcoeffs_s[k] = WebRtcIsac_kQKltLevelsShape[index_ovr_s[k]]; + } + /* Inverse KLT. */ + /* Left transform, transpose matrix.! */ + offss = 0; + poss = 0; + for (j = 0; j < SUBFRAMES; j++) { + offs2 = 0; + for (k = 0; k < LPC_SHAPE_ORDER; k++) { + sum = 0; + pos = offss; + pos2 = offs2; + for (n = 0; n < LPC_SHAPE_ORDER; n++) { + sum += tmpcoeffs_s[pos++] * WebRtcIsac_kKltT1Shape[pos2++]; + } + tmpcoeffs2_s[poss++] = sum; + offs2 += LPC_SHAPE_ORDER; + } + offss += LPC_SHAPE_ORDER; + } + + /* Right transform, Transpose matrix */ + offss = 0; + poss = 0; + for (j = 0; j < SUBFRAMES; j++) { + poss = offss; + for (k = 0; k < LPC_SHAPE_ORDER; k++) { + sum = 0; + pos = k; + pos2 = j; + for (n = 0; n < SUBFRAMES; n++) { + sum += tmpcoeffs2_s[pos] * WebRtcIsac_kKltT2Shape[pos2]; + pos += LPC_SHAPE_ORDER; + pos2 += SUBFRAMES; + } + tmpcoeffs_s[poss++] = sum; + } + offss += LPC_SHAPE_ORDER; + } + + /* Scaling, mean addition, and gain restoration. */ + poss = 0; + pos = 0; + for (k = 0; k < SUBFRAMES; k++) { + /* Ignore gains. */ + pos += 2; + + /* Low band LAR coefficients. */ + for (n = 0; n < LPC_LOBAND_ORDER; n++, pos++, poss++) { + LPCCoef[pos] = tmpcoeffs_s[poss] / LPC_LOBAND_SCALE; + LPCCoef[pos] += WebRtcIsac_kLpcMeansShape[poss]; + } + + /* High band LAR coefficients. */ + for (n = 0; n < LPC_HIBAND_ORDER; n++, pos++, poss++) { + LPCCoef[pos] = tmpcoeffs_s[poss] / LPC_HIBAND_SCALE; + LPCCoef[pos] += WebRtcIsac_kLpcMeansShape[poss]; + } + } +} + + +void WebRtcIsac_EncodeLpcLb(double* LPCCoef_lo, double* LPCCoef_hi, + Bitstr* streamdata, IsacSaveEncoderData* encData) { + double lars[KLT_ORDER_GAIN + KLT_ORDER_SHAPE]; + int k; + + WebRtcIsac_Poly2Lar(LPCCoef_lo, ORDERLO, LPCCoef_hi, ORDERHI, SUBFRAMES, + lars); + WebRtcIsac_EncodeLar(lars, streamdata, encData); + WebRtcIsac_Lar2Poly(lars, LPCCoef_lo, ORDERLO, LPCCoef_hi, ORDERHI, + SUBFRAMES); + /* Save data for creation of multiple bit streams (and transcoding). */ + for (k = 0; k < (ORDERLO + 1)*SUBFRAMES; k++) { + encData->LPCcoeffs_lo[(ORDERLO + 1)*SUBFRAMES * encData->startIdx + k] = + LPCCoef_lo[k]; + } + for (k = 0; k < (ORDERHI + 1)*SUBFRAMES; k++) { + encData->LPCcoeffs_hi[(ORDERHI + 1)*SUBFRAMES * encData->startIdx + k] = + LPCCoef_hi[k]; + } +} + + +int16_t WebRtcIsac_EncodeLpcUB(double* lpcVecs, Bitstr* streamdata, + double* interpolLPCCoeff, + int16_t bandwidth, + ISACUBSaveEncDataStruct* encData) { + double U[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + int idx[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + int interpolCntr; + + WebRtcIsac_Poly2LarUB(lpcVecs, bandwidth); + WebRtcIsac_RemoveLarMean(lpcVecs, bandwidth); + WebRtcIsac_DecorrelateIntraVec(lpcVecs, U, bandwidth); + WebRtcIsac_DecorrelateInterVec(U, lpcVecs, bandwidth); + WebRtcIsac_QuantizeUncorrLar(lpcVecs, idx, bandwidth); + + WebRtcIsac_CorrelateInterVec(lpcVecs, U, bandwidth); + WebRtcIsac_CorrelateIntraVec(U, lpcVecs, bandwidth); + WebRtcIsac_AddLarMean(lpcVecs, bandwidth); + + switch (bandwidth) { + case isac12kHz: { + /* Store the indices to be used for multiple encoding. */ + memcpy(encData->indexLPCShape, idx, UB_LPC_ORDER * + UB_LPC_VEC_PER_FRAME * sizeof(int)); + WebRtcIsac_EncHistMulti(streamdata, idx, WebRtcIsac_kLpcShapeCdfMatUb12, + UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME); + for (interpolCntr = 0; interpolCntr < UB_INTERPOL_SEGMENTS; + interpolCntr++) { + WebRtcIsac_Lar2PolyInterpolUB(lpcVecs, interpolLPCCoeff, + kLpcVecPerSegmentUb12 + 1); + lpcVecs += UB_LPC_ORDER; + interpolLPCCoeff += (kLpcVecPerSegmentUb12 * (UB_LPC_ORDER + 1)); + } + break; + } + case isac16kHz: { + /* Store the indices to be used for multiple encoding. */ + memcpy(encData->indexLPCShape, idx, UB_LPC_ORDER * + UB16_LPC_VEC_PER_FRAME * sizeof(int)); + WebRtcIsac_EncHistMulti(streamdata, idx, WebRtcIsac_kLpcShapeCdfMatUb16, + UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME); + for (interpolCntr = 0; interpolCntr < UB16_INTERPOL_SEGMENTS; + interpolCntr++) { + WebRtcIsac_Lar2PolyInterpolUB(lpcVecs, interpolLPCCoeff, + kLpcVecPerSegmentUb16 + 1); + lpcVecs += UB_LPC_ORDER; + interpolLPCCoeff += (kLpcVecPerSegmentUb16 * (UB_LPC_ORDER + 1)); + } + break; + } + default: + return -1; + } + return 0; +} + +void WebRtcIsac_EncodeLpcGainLb(double* LPCCoef_lo, double* LPCCoef_hi, + Bitstr* streamdata, + IsacSaveEncoderData* encData) { + int j, k, n, pos, pos2, posg, offsg, offs2; + int index_g[KLT_ORDER_GAIN]; + int index_ovr_g[KLT_ORDER_GAIN]; + double tmpcoeffs_g[KLT_ORDER_GAIN]; + double tmpcoeffs2_g[KLT_ORDER_GAIN]; + double sum; + /* log gains, mean removal and scaling */ + posg = 0; + for (k = 0; k < SUBFRAMES; k++) { + tmpcoeffs_g[posg] = log(LPCCoef_lo[(LPC_LOBAND_ORDER + 1) * k]); + tmpcoeffs_g[posg] -= WebRtcIsac_kLpcMeansGain[posg]; + tmpcoeffs_g[posg] *= LPC_GAIN_SCALE; + posg++; + tmpcoeffs_g[posg] = log(LPCCoef_hi[(LPC_HIBAND_ORDER + 1) * k]); + tmpcoeffs_g[posg] -= WebRtcIsac_kLpcMeansGain[posg]; + tmpcoeffs_g[posg] *= LPC_GAIN_SCALE; + posg++; + } + + /* KLT */ + + /* Left transform. */ + offsg = 0; + for (j = 0; j < SUBFRAMES; j++) { + posg = offsg; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = offsg; + pos2 = k; + for (n = 0; n < LPC_GAIN_ORDER; n++) { + sum += tmpcoeffs_g[pos++] * WebRtcIsac_kKltT1Gain[pos2]; + pos2 += LPC_GAIN_ORDER; + } + tmpcoeffs2_g[posg++] = sum; + } + offsg += LPC_GAIN_ORDER; + } + + /* Right transform. */ + offsg = 0; + offs2 = 0; + for (j = 0; j < SUBFRAMES; j++) { + posg = offsg; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = k; + pos2 = offs2; + for (n = 0; n < SUBFRAMES; n++) { + sum += tmpcoeffs2_g[pos] * WebRtcIsac_kKltT2Gain[pos2++]; + pos += LPC_GAIN_ORDER; + } + tmpcoeffs_g[posg++] = sum; + } + offs2 += SUBFRAMES; + offsg += LPC_GAIN_ORDER; + } + + /* Quantize coefficients. */ + for (k = 0; k < KLT_ORDER_GAIN; k++) { + /* Get index. */ + pos2 = WebRtcIsac_lrint(tmpcoeffs_g[k] / KLT_STEPSIZE); + index_g[k] = (pos2) + WebRtcIsac_kQKltQuantMinGain[k]; + if (index_g[k] < 0) { + index_g[k] = 0; + } else if (index_g[k] > WebRtcIsac_kQKltMaxIndGain[k]) { + index_g[k] = WebRtcIsac_kQKltMaxIndGain[k]; + } + index_ovr_g[k] = WebRtcIsac_kQKltOffsetGain[k] + index_g[k]; + + /* Find quantization levels for coefficients. */ + tmpcoeffs_g[k] = WebRtcIsac_kQKltLevelsGain[index_ovr_g[k]]; + + /* Save data for creation of multiple bit streams. */ + encData->LPCindex_g[KLT_ORDER_GAIN * encData->startIdx + k] = index_g[k]; + } + + /* Entropy coding of quantization indices - gain. */ + WebRtcIsac_EncHistMulti(streamdata, index_g, WebRtcIsac_kQKltCdfPtrGain, + KLT_ORDER_GAIN); + + /* Find quantization levels for coefficients. */ + /* Left transform. */ + offsg = 0; + posg = 0; + for (j = 0; j < SUBFRAMES; j++) { + offs2 = 0; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = offsg; + pos2 = offs2; + for (n = 0; n < LPC_GAIN_ORDER; n++) + sum += tmpcoeffs_g[pos++] * WebRtcIsac_kKltT1Gain[pos2++]; + tmpcoeffs2_g[posg++] = sum; + offs2 += LPC_GAIN_ORDER; + } + offsg += LPC_GAIN_ORDER; + } + + /* Right transform, transpose matrix. */ + offsg = 0; + posg = 0; + for (j = 0; j < SUBFRAMES; j++) { + posg = offsg; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = k; + pos2 = j; + for (n = 0; n < SUBFRAMES; n++) { + sum += tmpcoeffs2_g[pos] * WebRtcIsac_kKltT2Gain[pos2]; + pos += LPC_GAIN_ORDER; + pos2 += SUBFRAMES; + } + tmpcoeffs_g[posg++] = sum; + } + offsg += LPC_GAIN_ORDER; + } + + + /* Scaling, mean addition, and gain restoration. */ + posg = 0; + for (k = 0; k < SUBFRAMES; k++) { + sum = tmpcoeffs_g[posg] / LPC_GAIN_SCALE; + sum += WebRtcIsac_kLpcMeansGain[posg]; + LPCCoef_lo[k * (LPC_LOBAND_ORDER + 1)] = exp(sum); + pos++; + posg++; + sum = tmpcoeffs_g[posg] / LPC_GAIN_SCALE; + sum += WebRtcIsac_kLpcMeansGain[posg]; + LPCCoef_hi[k * (LPC_HIBAND_ORDER + 1)] = exp(sum); + pos++; + posg++; + } + +} + +void WebRtcIsac_EncodeLpcGainUb(double* lpGains, Bitstr* streamdata, + int* lpcGainIndex) { + double U[UB_LPC_GAIN_DIM]; + int idx[UB_LPC_GAIN_DIM]; + WebRtcIsac_ToLogDomainRemoveMean(lpGains); + WebRtcIsac_DecorrelateLPGain(lpGains, U); + WebRtcIsac_QuantizeLpcGain(U, idx); + /* Store the index for re-encoding for FEC. */ + memcpy(lpcGainIndex, idx, UB_LPC_GAIN_DIM * sizeof(int)); + WebRtcIsac_CorrelateLpcGain(U, lpGains); + WebRtcIsac_AddMeanToLinearDomain(lpGains); + WebRtcIsac_EncHistMulti(streamdata, idx, WebRtcIsac_kLpcGainCdfMat, + UB_LPC_GAIN_DIM); +} + + +void WebRtcIsac_StoreLpcGainUb(double* lpGains, Bitstr* streamdata) { + double U[UB_LPC_GAIN_DIM]; + int idx[UB_LPC_GAIN_DIM]; + WebRtcIsac_ToLogDomainRemoveMean(lpGains); + WebRtcIsac_DecorrelateLPGain(lpGains, U); + WebRtcIsac_QuantizeLpcGain(U, idx); + WebRtcIsac_EncHistMulti(streamdata, idx, WebRtcIsac_kLpcGainCdfMat, + UB_LPC_GAIN_DIM); +} + + + +int16_t WebRtcIsac_DecodeLpcGainUb(double* lpGains, Bitstr* streamdata) { + double U[UB_LPC_GAIN_DIM]; + int idx[UB_LPC_GAIN_DIM]; + int err; + err = WebRtcIsac_DecHistOneStepMulti(idx, streamdata, + WebRtcIsac_kLpcGainCdfMat, + WebRtcIsac_kLpcGainEntropySearch, + UB_LPC_GAIN_DIM); + if (err < 0) { + return -1; + } + WebRtcIsac_DequantizeLpcGain(idx, U); + WebRtcIsac_CorrelateLpcGain(U, lpGains); + WebRtcIsac_AddMeanToLinearDomain(lpGains); + return 0; +} + + + +/* decode & dequantize RC */ +int WebRtcIsac_DecodeRc(Bitstr* streamdata, int16_t* RCQ15) { + int k, err; + int index[AR_ORDER]; + + /* entropy decoding of quantization indices */ + err = WebRtcIsac_DecHistOneStepMulti(index, streamdata, + WebRtcIsac_kQArRcCdfPtr, + WebRtcIsac_kQArRcInitIndex, AR_ORDER); + if (err < 0) + return err; + + /* find quantization levels for reflection coefficients */ + for (k = 0; k < AR_ORDER; k++) { + RCQ15[k] = *(WebRtcIsac_kQArRcLevelsPtr[k] + index[k]); + } + return 0; +} + + +/* quantize & code RC */ +void WebRtcIsac_EncodeRc(int16_t* RCQ15, Bitstr* streamdata) { + int k; + int index[AR_ORDER]; + + /* quantize reflection coefficients (add noise feedback?) */ + for (k = 0; k < AR_ORDER; k++) { + index[k] = WebRtcIsac_kQArRcInitIndex[k]; + // The safe-guards in following while conditions are to suppress gcc 4.8.3 + // warnings, Issue 2888. Otherwise, first and last elements of + // |WebRtcIsac_kQArBoundaryLevels| are such that the following search + // *never* cause an out-of-boundary read. + if (RCQ15[k] > WebRtcIsac_kQArBoundaryLevels[index[k]]) { + while (index[k] + 1 < NUM_AR_RC_QUANT_BAUNDARY && + RCQ15[k] > WebRtcIsac_kQArBoundaryLevels[index[k] + 1]) { + index[k]++; + } + } else { + while (index[k] > 0 && + RCQ15[k] < WebRtcIsac_kQArBoundaryLevels[--index[k]]) ; + } + RCQ15[k] = *(WebRtcIsac_kQArRcLevelsPtr[k] + index[k]); + } + + /* entropy coding of quantization indices */ + WebRtcIsac_EncHistMulti(streamdata, index, WebRtcIsac_kQArRcCdfPtr, AR_ORDER); +} + + +/* decode & dequantize squared Gain */ +int WebRtcIsac_DecodeGain2(Bitstr* streamdata, int32_t* gainQ10) { + int index, err; + + /* entropy decoding of quantization index */ + err = WebRtcIsac_DecHistOneStepMulti(&index, streamdata, + WebRtcIsac_kQGainCdf_ptr, + WebRtcIsac_kQGainInitIndex, 1); + if (err < 0) { + return err; + } + /* find quantization level */ + *gainQ10 = WebRtcIsac_kQGain2Levels[index]; + return 0; +} + + +/* quantize & code squared Gain */ +int WebRtcIsac_EncodeGain2(int32_t* gainQ10, Bitstr* streamdata) { + int index; + + /* find quantization index */ + index = WebRtcIsac_kQGainInitIndex[0]; + if (*gainQ10 > WebRtcIsac_kQGain2BoundaryLevels[index]) { + while (*gainQ10 > WebRtcIsac_kQGain2BoundaryLevels[index + 1]) { + index++; + } + } else { + while (*gainQ10 < WebRtcIsac_kQGain2BoundaryLevels[--index]) ; + } + /* De-quantize */ + *gainQ10 = WebRtcIsac_kQGain2Levels[index]; + + /* entropy coding of quantization index */ + WebRtcIsac_EncHistMulti(streamdata, &index, WebRtcIsac_kQGainCdf_ptr, 1); + return 0; +} + + +/* code and decode Pitch Gains and Lags functions */ + +/* decode & dequantize Pitch Gains */ +int WebRtcIsac_DecodePitchGain(Bitstr* streamdata, + int16_t* PitchGains_Q12) { + int index_comb, err; + const uint16_t* WebRtcIsac_kQPitchGainCdf_ptr[1]; + + /* Entropy decoding of quantization indices */ + *WebRtcIsac_kQPitchGainCdf_ptr = WebRtcIsac_kQPitchGainCdf; + err = WebRtcIsac_DecHistBisectMulti(&index_comb, streamdata, + WebRtcIsac_kQPitchGainCdf_ptr, + WebRtcIsac_kQCdfTableSizeGain, 1); + /* Error check, Q_mean_Gain.. tables are of size 144 */ + if ((err < 0) || (index_comb < 0) || (index_comb >= 144)) { + return -ISAC_RANGE_ERROR_DECODE_PITCH_GAIN; + } + /* De-quantize back to pitch gains by table look-up. */ + PitchGains_Q12[0] = WebRtcIsac_kQMeanGain1Q12[index_comb]; + PitchGains_Q12[1] = WebRtcIsac_kQMeanGain2Q12[index_comb]; + PitchGains_Q12[2] = WebRtcIsac_kQMeanGain3Q12[index_comb]; + PitchGains_Q12[3] = WebRtcIsac_kQMeanGain4Q12[index_comb]; + return 0; +} + + +/* Quantize & code Pitch Gains. */ +void WebRtcIsac_EncodePitchGain(int16_t* PitchGains_Q12, + Bitstr* streamdata, + IsacSaveEncoderData* encData) { + int k, j; + double C; + double S[PITCH_SUBFRAMES]; + int index[3]; + int index_comb; + const uint16_t* WebRtcIsac_kQPitchGainCdf_ptr[1]; + double PitchGains[PITCH_SUBFRAMES] = {0, 0, 0, 0}; + + /* Take the asin. */ + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchGains[k] = ((float)PitchGains_Q12[k]) / 4096; + S[k] = asin(PitchGains[k]); + } + + /* Find quantization index; only for the first three + * transform coefficients. */ + for (k = 0; k < 3; k++) { + /* transform */ + C = 0.0; + for (j = 0; j < PITCH_SUBFRAMES; j++) { + C += WebRtcIsac_kTransform[k][j] * S[j]; + } + /* Quantize */ + index[k] = WebRtcIsac_lrint(C / PITCH_GAIN_STEPSIZE); + + /* Check that the index is not outside the boundaries of the table. */ + if (index[k] < WebRtcIsac_kIndexLowerLimitGain[k]) { + index[k] = WebRtcIsac_kIndexLowerLimitGain[k]; + } else if (index[k] > WebRtcIsac_kIndexUpperLimitGain[k]) { + index[k] = WebRtcIsac_kIndexUpperLimitGain[k]; + } + index[k] -= WebRtcIsac_kIndexLowerLimitGain[k]; + } + + /* Calculate unique overall index. */ + index_comb = WebRtcIsac_kIndexMultsGain[0] * index[0] + + WebRtcIsac_kIndexMultsGain[1] * index[1] + index[2]; + + /* unquantize back to pitch gains by table look-up */ + PitchGains_Q12[0] = WebRtcIsac_kQMeanGain1Q12[index_comb]; + PitchGains_Q12[1] = WebRtcIsac_kQMeanGain2Q12[index_comb]; + PitchGains_Q12[2] = WebRtcIsac_kQMeanGain3Q12[index_comb]; + PitchGains_Q12[3] = WebRtcIsac_kQMeanGain4Q12[index_comb]; + + /* entropy coding of quantization pitch gains */ + *WebRtcIsac_kQPitchGainCdf_ptr = WebRtcIsac_kQPitchGainCdf; + WebRtcIsac_EncHistMulti(streamdata, &index_comb, + WebRtcIsac_kQPitchGainCdf_ptr, 1); + encData->pitchGain_index[encData->startIdx] = index_comb; +} + + + +/* Pitch LAG */ +/* Decode & de-quantize Pitch Lags. */ +int WebRtcIsac_DecodePitchLag(Bitstr* streamdata, int16_t* PitchGain_Q12, + double* PitchLags) { + int k, err; + double StepSize; + double C; + int index[PITCH_SUBFRAMES]; + double mean_gain; + const double* mean_val2, *mean_val3, *mean_val4; + const int16_t* lower_limit; + const uint16_t* init_index; + const uint16_t* cdf_size; + const uint16_t** cdf; + double PitchGain[4] = {0, 0, 0, 0}; + + /* compute mean pitch gain */ + mean_gain = 0.0; + for (k = 0; k < 4; k++) { + PitchGain[k] = ((float)PitchGain_Q12[k]) / 4096; + mean_gain += PitchGain[k]; + } + mean_gain /= 4.0; + + /* voicing classification. */ + if (mean_gain < 0.2) { + StepSize = WebRtcIsac_kQPitchLagStepsizeLo; + cdf = WebRtcIsac_kQPitchLagCdfPtrLo; + cdf_size = WebRtcIsac_kQPitchLagCdfSizeLo; + mean_val2 = WebRtcIsac_kQMeanLag2Lo; + mean_val3 = WebRtcIsac_kQMeanLag3Lo; + mean_val4 = WebRtcIsac_kQMeanLag4Lo; + lower_limit = WebRtcIsac_kQIndexLowerLimitLagLo; + init_index = WebRtcIsac_kQInitIndexLagLo; + } else if (mean_gain < 0.4) { + StepSize = WebRtcIsac_kQPitchLagStepsizeMid; + cdf = WebRtcIsac_kQPitchLagCdfPtrMid; + cdf_size = WebRtcIsac_kQPitchLagCdfSizeMid; + mean_val2 = WebRtcIsac_kQMeanLag2Mid; + mean_val3 = WebRtcIsac_kQMeanLag3Mid; + mean_val4 = WebRtcIsac_kQMeanLag4Mid; + lower_limit = WebRtcIsac_kQIndexLowerLimitLagMid; + init_index = WebRtcIsac_kQInitIndexLagMid; + } else { + StepSize = WebRtcIsac_kQPitchLagStepsizeHi; + cdf = WebRtcIsac_kQPitchLagCdfPtrHi; + cdf_size = WebRtcIsac_kQPitchLagCdfSizeHi; + mean_val2 = WebRtcIsac_kQMeanLag2Hi; + mean_val3 = WebRtcIsac_kQMeanLag3Hi; + mean_val4 = WebRtcIsac_kQMeanLag4Hi; + lower_limit = WebRtcIsac_kQindexLowerLimitLagHi; + init_index = WebRtcIsac_kQInitIndexLagHi; + } + + /* Entropy decoding of quantization indices. */ + err = WebRtcIsac_DecHistBisectMulti(index, streamdata, cdf, cdf_size, 1); + if ((err < 0) || (index[0] < 0)) { + return -ISAC_RANGE_ERROR_DECODE_PITCH_LAG; + } + err = WebRtcIsac_DecHistOneStepMulti(index + 1, streamdata, cdf + 1, + init_index, 3); + if (err < 0) { + return -ISAC_RANGE_ERROR_DECODE_PITCH_LAG; + } + + /* Unquantize back to transform coefficients and do the inverse transform: + * S = T'*C. */ + C = (index[0] + lower_limit[0]) * StepSize; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] = WebRtcIsac_kTransformTranspose[k][0] * C; + } + C = mean_val2[index[1]]; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] += WebRtcIsac_kTransformTranspose[k][1] * C; + } + C = mean_val3[index[2]]; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] += WebRtcIsac_kTransformTranspose[k][2] * C; + } + C = mean_val4[index[3]]; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] += WebRtcIsac_kTransformTranspose[k][3] * C; + } + return 0; +} + + + +/* Quantize & code pitch lags. */ +void WebRtcIsac_EncodePitchLag(double* PitchLags, int16_t* PitchGain_Q12, + Bitstr* streamdata, + IsacSaveEncoderData* encData) { + int k, j; + double StepSize; + double C; + int index[PITCH_SUBFRAMES]; + double mean_gain; + const double* mean_val2, *mean_val3, *mean_val4; + const int16_t* lower_limit, *upper_limit; + const uint16_t** cdf; + double PitchGain[4] = {0, 0, 0, 0}; + + /* compute mean pitch gain */ + mean_gain = 0.0; + for (k = 0; k < 4; k++) { + PitchGain[k] = ((float)PitchGain_Q12[k]) / 4096; + mean_gain += PitchGain[k]; + } + mean_gain /= 4.0; + + /* Save data for creation of multiple bit streams */ + encData->meanGain[encData->startIdx] = mean_gain; + + /* Voicing classification. */ + if (mean_gain < 0.2) { + StepSize = WebRtcIsac_kQPitchLagStepsizeLo; + cdf = WebRtcIsac_kQPitchLagCdfPtrLo; + mean_val2 = WebRtcIsac_kQMeanLag2Lo; + mean_val3 = WebRtcIsac_kQMeanLag3Lo; + mean_val4 = WebRtcIsac_kQMeanLag4Lo; + lower_limit = WebRtcIsac_kQIndexLowerLimitLagLo; + upper_limit = WebRtcIsac_kQIndexUpperLimitLagLo; + } else if (mean_gain < 0.4) { + StepSize = WebRtcIsac_kQPitchLagStepsizeMid; + cdf = WebRtcIsac_kQPitchLagCdfPtrMid; + mean_val2 = WebRtcIsac_kQMeanLag2Mid; + mean_val3 = WebRtcIsac_kQMeanLag3Mid; + mean_val4 = WebRtcIsac_kQMeanLag4Mid; + lower_limit = WebRtcIsac_kQIndexLowerLimitLagMid; + upper_limit = WebRtcIsac_kQIndexUpperLimitLagMid; + } else { + StepSize = WebRtcIsac_kQPitchLagStepsizeHi; + cdf = WebRtcIsac_kQPitchLagCdfPtrHi; + mean_val2 = WebRtcIsac_kQMeanLag2Hi; + mean_val3 = WebRtcIsac_kQMeanLag3Hi; + mean_val4 = WebRtcIsac_kQMeanLag4Hi; + lower_limit = WebRtcIsac_kQindexLowerLimitLagHi; + upper_limit = WebRtcIsac_kQindexUpperLimitLagHi; + } + + /* find quantization index */ + for (k = 0; k < 4; k++) { + /* transform */ + C = 0.0; + for (j = 0; j < PITCH_SUBFRAMES; j++) { + C += WebRtcIsac_kTransform[k][j] * PitchLags[j]; + } + /* quantize */ + index[k] = WebRtcIsac_lrint(C / StepSize); + + /* check that the index is not outside the boundaries of the table */ + if (index[k] < lower_limit[k]) { + index[k] = lower_limit[k]; + } else if (index[k] > upper_limit[k]) index[k] = upper_limit[k]; { + index[k] -= lower_limit[k]; + } + /* Save data for creation of multiple bit streams */ + encData->pitchIndex[PITCH_SUBFRAMES * encData->startIdx + k] = index[k]; + } + + /* Un-quantize back to transform coefficients and do the inverse transform: + * S = T'*C */ + C = (index[0] + lower_limit[0]) * StepSize; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] = WebRtcIsac_kTransformTranspose[k][0] * C; + } + C = mean_val2[index[1]]; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] += WebRtcIsac_kTransformTranspose[k][1] * C; + } + C = mean_val3[index[2]]; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] += WebRtcIsac_kTransformTranspose[k][2] * C; + } + C = mean_val4[index[3]]; + for (k = 0; k < PITCH_SUBFRAMES; k++) { + PitchLags[k] += WebRtcIsac_kTransformTranspose[k][3] * C; + } + /* entropy coding of quantization pitch lags */ + WebRtcIsac_EncHistMulti(streamdata, index, cdf, PITCH_SUBFRAMES); +} + + + +/* Routines for in-band signaling of bandwidth estimation */ +/* Histograms based on uniform distribution of indices */ +/* Move global variables later! */ + + +/* cdf array for frame length indicator */ +const uint16_t WebRtcIsac_kFrameLengthCdf[4] = { + 0, 21845, 43690, 65535 }; + +/* pointer to cdf array for frame length indicator */ +const uint16_t* WebRtcIsac_kFrameLengthCdf_ptr[1] = { + WebRtcIsac_kFrameLengthCdf }; + +/* initial cdf index for decoder of frame length indicator */ +const uint16_t WebRtcIsac_kFrameLengthInitIndex[1] = { 1 }; + + +int WebRtcIsac_DecodeFrameLen(Bitstr* streamdata, int16_t* framesamples) { + int frame_mode, err; + err = 0; + /* entropy decoding of frame length [1:30ms,2:60ms] */ + err = WebRtcIsac_DecHistOneStepMulti(&frame_mode, streamdata, + WebRtcIsac_kFrameLengthCdf_ptr, + WebRtcIsac_kFrameLengthInitIndex, 1); + if (err < 0) + return -ISAC_RANGE_ERROR_DECODE_FRAME_LENGTH; + + switch (frame_mode) { + case 1: + *framesamples = 480; /* 30ms */ + break; + case 2: + *framesamples = 960; /* 60ms */ + break; + default: + err = -ISAC_DISALLOWED_FRAME_MODE_DECODER; + } + return err; +} + +int WebRtcIsac_EncodeFrameLen(int16_t framesamples, Bitstr* streamdata) { + int frame_mode, status; + + status = 0; + frame_mode = 0; + /* entropy coding of frame length [1:480 samples,2:960 samples] */ + switch (framesamples) { + case 480: + frame_mode = 1; + break; + case 960: + frame_mode = 2; + break; + default: + status = - ISAC_DISALLOWED_FRAME_MODE_ENCODER; + } + + if (status < 0) + return status; + + WebRtcIsac_EncHistMulti(streamdata, &frame_mode, + WebRtcIsac_kFrameLengthCdf_ptr, 1); + return status; +} + +/* cdf array for estimated bandwidth */ +static const uint16_t kBwCdf[25] = { + 0, 2731, 5461, 8192, 10923, 13653, 16384, 19114, 21845, 24576, 27306, 30037, + 32768, 35498, 38229, 40959, 43690, 46421, 49151, 51882, 54613, 57343, 60074, + 62804, 65535 }; + +/* pointer to cdf array for estimated bandwidth */ +static const uint16_t* const kBwCdfPtr[1] = { kBwCdf }; + +/* initial cdf index for decoder of estimated bandwidth*/ +static const uint16_t kBwInitIndex[1] = { 7 }; + + +int WebRtcIsac_DecodeSendBW(Bitstr* streamdata, int16_t* BWno) { + int BWno32, err; + + /* entropy decoding of sender's BW estimation [0..23] */ + err = WebRtcIsac_DecHistOneStepMulti(&BWno32, streamdata, kBwCdfPtr, + kBwInitIndex, 1); + if (err < 0) { + return -ISAC_RANGE_ERROR_DECODE_BANDWIDTH; + } + *BWno = (int16_t)BWno32; + return err; +} + +void WebRtcIsac_EncodeReceiveBw(int* BWno, Bitstr* streamdata) { + /* entropy encoding of receiver's BW estimation [0..23] */ + WebRtcIsac_EncHistMulti(streamdata, BWno, kBwCdfPtr, 1); +} + + +/* estimate code length of LPC Coef */ +void WebRtcIsac_TranscodeLPCCoef(double* LPCCoef_lo, double* LPCCoef_hi, + int* index_g) { + int j, k, n, pos, pos2, posg, offsg, offs2; + int index_ovr_g[KLT_ORDER_GAIN]; + double tmpcoeffs_g[KLT_ORDER_GAIN]; + double tmpcoeffs2_g[KLT_ORDER_GAIN]; + double sum; + + /* log gains, mean removal and scaling */ + posg = 0; + for (k = 0; k < SUBFRAMES; k++) { + tmpcoeffs_g[posg] = log(LPCCoef_lo[(LPC_LOBAND_ORDER + 1) * k]); + tmpcoeffs_g[posg] -= WebRtcIsac_kLpcMeansGain[posg]; + tmpcoeffs_g[posg] *= LPC_GAIN_SCALE; + posg++; + tmpcoeffs_g[posg] = log(LPCCoef_hi[(LPC_HIBAND_ORDER + 1) * k]); + tmpcoeffs_g[posg] -= WebRtcIsac_kLpcMeansGain[posg]; + tmpcoeffs_g[posg] *= LPC_GAIN_SCALE; + posg++; + } + + /* KLT */ + + /* Left transform. */ + offsg = 0; + for (j = 0; j < SUBFRAMES; j++) { + posg = offsg; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = offsg; + pos2 = k; + for (n = 0; n < LPC_GAIN_ORDER; n++) { + sum += tmpcoeffs_g[pos++] * WebRtcIsac_kKltT1Gain[pos2]; + pos2 += LPC_GAIN_ORDER; + } + tmpcoeffs2_g[posg++] = sum; + } + offsg += LPC_GAIN_ORDER; + } + + /* Right transform. */ + offsg = 0; + offs2 = 0; + for (j = 0; j < SUBFRAMES; j++) { + posg = offsg; + for (k = 0; k < LPC_GAIN_ORDER; k++) { + sum = 0; + pos = k; + pos2 = offs2; + for (n = 0; n < SUBFRAMES; n++) { + sum += tmpcoeffs2_g[pos] * WebRtcIsac_kKltT2Gain[pos2++]; + pos += LPC_GAIN_ORDER; + } + tmpcoeffs_g[posg++] = sum; + } + offs2 += SUBFRAMES; + offsg += LPC_GAIN_ORDER; + } + + + /* quantize coefficients */ + for (k = 0; k < KLT_ORDER_GAIN; k++) { + /* Get index. */ + pos2 = WebRtcIsac_lrint(tmpcoeffs_g[k] / KLT_STEPSIZE); + index_g[k] = (pos2) + WebRtcIsac_kQKltQuantMinGain[k]; + if (index_g[k] < 0) { + index_g[k] = 0; + } else if (index_g[k] > WebRtcIsac_kQKltMaxIndGain[k]) { + index_g[k] = WebRtcIsac_kQKltMaxIndGain[k]; + } + index_ovr_g[k] = WebRtcIsac_kQKltOffsetGain[k] + index_g[k]; + + /* find quantization levels for coefficients */ + tmpcoeffs_g[k] = WebRtcIsac_kQKltLevelsGain[index_ovr_g[k]]; + } +} + + +/* Decode & de-quantize LPC Coefficients. */ +int WebRtcIsac_DecodeLpcCoefUB(Bitstr* streamdata, double* lpcVecs, + double* percepFilterGains, + int16_t bandwidth) { + int index_s[KLT_ORDER_SHAPE]; + + double U[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + int err; + + /* Entropy decoding of quantization indices. */ + switch (bandwidth) { + case isac12kHz: { + err = WebRtcIsac_DecHistOneStepMulti( + index_s, streamdata, WebRtcIsac_kLpcShapeCdfMatUb12, + WebRtcIsac_kLpcShapeEntropySearchUb12, UB_LPC_ORDER * + UB_LPC_VEC_PER_FRAME); + break; + } + case isac16kHz: { + err = WebRtcIsac_DecHistOneStepMulti( + index_s, streamdata, WebRtcIsac_kLpcShapeCdfMatUb16, + WebRtcIsac_kLpcShapeEntropySearchUb16, UB_LPC_ORDER * + UB16_LPC_VEC_PER_FRAME); + break; + } + default: + return -1; + } + + if (err < 0) { + return err; + } + + WebRtcIsac_DequantizeLpcParam(index_s, lpcVecs, bandwidth); + WebRtcIsac_CorrelateInterVec(lpcVecs, U, bandwidth); + WebRtcIsac_CorrelateIntraVec(U, lpcVecs, bandwidth); + WebRtcIsac_AddLarMean(lpcVecs, bandwidth); + WebRtcIsac_DecodeLpcGainUb(percepFilterGains, streamdata); + + if (bandwidth == isac16kHz) { + /* Decode another set of Gains. */ + WebRtcIsac_DecodeLpcGainUb(&percepFilterGains[SUBFRAMES], streamdata); + } + return 0; +} + +int16_t WebRtcIsac_EncodeBandwidth(enum ISACBandwidth bandwidth, + Bitstr* streamData) { + int bandwidthMode; + switch (bandwidth) { + case isac12kHz: { + bandwidthMode = 0; + break; + } + case isac16kHz: { + bandwidthMode = 1; + break; + } + default: + return -ISAC_DISALLOWED_ENCODER_BANDWIDTH; + } + WebRtcIsac_EncHistMulti(streamData, &bandwidthMode, kOneBitEqualProbCdf_ptr, + 1); + return 0; +} + +int16_t WebRtcIsac_DecodeBandwidth(Bitstr* streamData, + enum ISACBandwidth* bandwidth) { + int bandwidthMode; + if (WebRtcIsac_DecHistOneStepMulti(&bandwidthMode, streamData, + kOneBitEqualProbCdf_ptr, + kOneBitEqualProbInitIndex, 1) < 0) { + return -ISAC_RANGE_ERROR_DECODE_BANDWITH; + } + switch (bandwidthMode) { + case 0: { + *bandwidth = isac12kHz; + break; + } + case 1: { + *bandwidth = isac16kHz; + break; + } + default: + return -ISAC_DISALLOWED_BANDWIDTH_MODE_DECODER; + } + return 0; +} + +int16_t WebRtcIsac_EncodeJitterInfo(int32_t jitterIndex, + Bitstr* streamData) { + /* This is to avoid LINUX warning until we change 'int' to 'Word32'. */ + int intVar; + + if ((jitterIndex < 0) || (jitterIndex > 1)) { + return -1; + } + intVar = (int)(jitterIndex); + /* Use the same CDF table as for bandwidth + * both take two values with equal probability.*/ + WebRtcIsac_EncHistMulti(streamData, &intVar, kOneBitEqualProbCdf_ptr, 1); + return 0; +} + +int16_t WebRtcIsac_DecodeJitterInfo(Bitstr* streamData, + int32_t* jitterInfo) { + int intVar; + /* Use the same CDF table as for bandwidth + * both take two values with equal probability. */ + if (WebRtcIsac_DecHistOneStepMulti(&intVar, streamData, + kOneBitEqualProbCdf_ptr, + kOneBitEqualProbInitIndex, 1) < 0) { + return -ISAC_RANGE_ERROR_DECODE_BANDWITH; + } + *jitterInfo = (int16_t)(intVar); + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.h new file mode 100644 index 0000000000..2ccd96b620 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.h @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * entropy_coding.h + * + * This header file declares all of the functions used to arithmetically + * encode the iSAC bistream + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ENTROPY_CODING_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ENTROPY_CODING_H_ + +#include "settings.h" +#include "structs.h" + +/****************************************************************************** + * WebRtcIsac_DecodeSpec() + * Decode real and imaginary part of the DFT coefficients, given a bit-stream. + * The decoded DFT coefficient can be transformed to time domain by + * WebRtcIsac_Time2Spec(). + * + * Input: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * - AvgPitchGain_Q12 : average pitch-gain of the frame. This is only + * relevant for 0-4 kHz band, and the input value is + * not used in other bands. + * - band : specifies which band's DFT should be decoded. + * + * Output: + * - *fr : pointer to a buffer where the real part of DFT + * coefficients are written to. + * - *fi : pointer to a buffer where the imaginary part + * of DFT coefficients are written to. + * + * Return value : < 0 if an error occures + * 0 if succeeded. + */ +int WebRtcIsac_DecodeSpec(Bitstr* streamdata, int16_t AvgPitchGain_Q12, + enum ISACBand band, double* fr, double* fi); + +/****************************************************************************** + * WebRtcIsac_EncodeSpec() + * Encode real and imaginary part of the DFT coefficients into the given + * bit-stream. + * + * Input: + * - *fr : pointer to a buffer where the real part of DFT + * coefficients are written to. + * - *fi : pointer to a buffer where the imaginary part + * of DFT coefficients are written to. + * - AvgPitchGain_Q12 : average pitch-gain of the frame. This is only + * relevant for 0-4 kHz band, and the input value is + * not used in other bands. + * - band : specifies which band's DFT should be decoded. + * + * Output: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Return value : < 0 if an error occures + * 0 if succeeded. + */ +int WebRtcIsac_EncodeSpec(const int16_t* fr, const int16_t* fi, + int16_t AvgPitchGain_Q12, enum ISACBand band, + Bitstr* streamdata); + +/* decode & dequantize LPC Coef */ +int WebRtcIsac_DecodeLpcCoef(Bitstr* streamdata, double* LPCCoef); +int WebRtcIsac_DecodeLpcCoefUB(Bitstr* streamdata, double* lpcVecs, + double* percepFilterGains, + int16_t bandwidth); + +int WebRtcIsac_DecodeLpc(Bitstr* streamdata, double* LPCCoef_lo, + double* LPCCoef_hi); + +/* quantize & code LPC Coef */ +void WebRtcIsac_EncodeLpcLb(double* LPCCoef_lo, double* LPCCoef_hi, + Bitstr* streamdata, IsacSaveEncoderData* encData); + +void WebRtcIsac_EncodeLpcGainLb(double* LPCCoef_lo, double* LPCCoef_hi, + Bitstr* streamdata, + IsacSaveEncoderData* encData); + +/****************************************************************************** + * WebRtcIsac_EncodeLpcUB() + * Encode LPC parameters, given as A-polynomial, of upper-band. The encoding + * is performed in LAR domain. + * For the upper-band, we compute and encode LPC of some sub-frames, LPC of + * other sub-frames are computed by linear interpolation, in LAR domain. This + * function performs the interpolation and returns the LPC of all sub-frames. + * + * Inputs: + * - lpcCoef : a buffer containing A-polynomials of sub-frames + * (excluding first coefficient that is 1). + * - bandwidth : specifies if the codec is operating at 0-12 kHz + * or 0-16 kHz mode. + * + * Input/output: + * - streamdata : pointer to a structure containing the encoded + * data and the parameters needed for entropy + * coding. + * + * Output: + * - interpolLPCCoeff : Decoded and interpolated LPC (A-polynomial) + * of all sub-frames. + * If LP analysis is of order K, and there are N + * sub-frames then this is a buffer of size + * (k + 1) * N, each vector starts with the LPC gain + * of the corresponding sub-frame. The LPC gains + * are encoded and inserted after this function is + * called. The first A-coefficient which is 1 is not + * included. + * + * Return value : 0 if encoding is successful, + * <0 if failed to encode. + */ +int16_t WebRtcIsac_EncodeLpcUB(double* lpcCoeff, Bitstr* streamdata, + double* interpolLPCCoeff, + int16_t bandwidth, + ISACUBSaveEncDataStruct* encData); + +/****************************************************************************** + * WebRtcIsac_DecodeInterpolLpcUb() + * Decode LPC coefficients and interpolate to get the coefficients fo all + * sub-frmaes. + * + * Inputs: + * - bandwidth : spepecifies if the codec is in 0-12 kHz or + * 0-16 kHz mode. + * + * Input/output: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Output: + * - percepFilterParam : Decoded and interpolated LPC (A-polynomial) of + * all sub-frames. + * If LP analysis is of order K, and there are N + * sub-frames then this is a buffer of size + * (k + 1) * N, each vector starts with the LPC gain + * of the corresponding sub-frame. The LPC gains + * are encoded and inserted after this function is + * called. The first A-coefficient which is 1 is not + * included. + * + * Return value : 0 if encoding is successful, + * <0 if failed to encode. + */ +int16_t WebRtcIsac_DecodeInterpolLpcUb(Bitstr* streamdata, + double* percepFilterParam, + int16_t bandwidth); + +/* Decode & dequantize RC */ +int WebRtcIsac_DecodeRc(Bitstr* streamdata, int16_t* RCQ15); + +/* Quantize & code RC */ +void WebRtcIsac_EncodeRc(int16_t* RCQ15, Bitstr* streamdata); + +/* Decode & dequantize squared Gain */ +int WebRtcIsac_DecodeGain2(Bitstr* streamdata, int32_t* Gain2); + +/* Quantize & code squared Gain (input is squared gain) */ +int WebRtcIsac_EncodeGain2(int32_t* gain2, Bitstr* streamdata); + +void WebRtcIsac_EncodePitchGain(int16_t* PitchGains_Q12, + Bitstr* streamdata, + IsacSaveEncoderData* encData); + +void WebRtcIsac_EncodePitchLag(double* PitchLags, + int16_t* PitchGain_Q12, + Bitstr* streamdata, + IsacSaveEncoderData* encData); + +int WebRtcIsac_DecodePitchGain(Bitstr* streamdata, + int16_t* PitchGain_Q12); +int WebRtcIsac_DecodePitchLag(Bitstr* streamdata, int16_t* PitchGain_Q12, + double* PitchLag); + +int WebRtcIsac_DecodeFrameLen(Bitstr* streamdata, int16_t* framelength); +int WebRtcIsac_EncodeFrameLen(int16_t framelength, Bitstr* streamdata); +int WebRtcIsac_DecodeSendBW(Bitstr* streamdata, int16_t* BWno); +void WebRtcIsac_EncodeReceiveBw(int* BWno, Bitstr* streamdata); + +/* Step-down */ +void WebRtcIsac_Poly2Rc(double* a, int N, double* RC); + +/* Step-up */ +void WebRtcIsac_Rc2Poly(double* RC, int N, double* a); + +void WebRtcIsac_TranscodeLPCCoef(double* LPCCoef_lo, double* LPCCoef_hi, + int* index_g); + + +/****************************************************************************** + * WebRtcIsac_EncodeLpcGainUb() + * Encode LPC gains of sub-Frames. + * + * Input/outputs: + * - lpGains : a buffer which contains 'SUBFRAME' number of + * LP gains to be encoded. The input values are + * overwritten by the quantized values. + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Output: + * - lpcGainIndex : quantization indices for lpc gains, these will + * be stored to be used for FEC. + */ +void WebRtcIsac_EncodeLpcGainUb(double* lpGains, Bitstr* streamdata, + int* lpcGainIndex); + + +/****************************************************************************** + * WebRtcIsac_EncodeLpcGainUb() + * Store LPC gains of sub-Frames in 'streamdata'. + * + * Input: + * - lpGains : a buffer which contains 'SUBFRAME' number of + * LP gains to be encoded. + * Input/outputs: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + */ +void WebRtcIsac_StoreLpcGainUb(double* lpGains, Bitstr* streamdata); + + +/****************************************************************************** + * WebRtcIsac_DecodeLpcGainUb() + * Decode the LPC gain of sub-frames. + * + * Input/output: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Output: + * - lpGains : a buffer where decoded LPC gians will be stored. + * + * Return value : 0 if succeeded. + * <0 if failed. + */ +int16_t WebRtcIsac_DecodeLpcGainUb(double* lpGains, Bitstr* streamdata); + + +/****************************************************************************** + * WebRtcIsac_EncodeBandwidth() + * Encode if the bandwidth of encoded audio is 0-12 kHz or 0-16 kHz. + * + * Input: + * - bandwidth : an enumerator specifying if the codec in is + * 0-12 kHz or 0-16 kHz mode. + * + * Input/output: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Return value : 0 if succeeded. + * <0 if failed. + */ +int16_t WebRtcIsac_EncodeBandwidth(enum ISACBandwidth bandwidth, + Bitstr* streamData); + + +/****************************************************************************** + * WebRtcIsac_DecodeBandwidth() + * Decode the bandwidth of the encoded audio, i.e. if the bandwidth is 0-12 kHz + * or 0-16 kHz. + * + * Input/output: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Output: + * - bandwidth : an enumerator specifying if the codec is in + * 0-12 kHz or 0-16 kHz mode. + * + * Return value : 0 if succeeded. + * <0 if failed. + */ +int16_t WebRtcIsac_DecodeBandwidth(Bitstr* streamData, + enum ISACBandwidth* bandwidth); + + +/****************************************************************************** + * WebRtcIsac_EncodeJitterInfo() + * Decode the jitter information. + * + * Input/output: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Input: + * - jitterInfo : one bit of info specifying if the channel is + * in high/low jitter. Zero indicates low jitter + * and one indicates high jitter. + * + * Return value : 0 if succeeded. + * <0 if failed. + */ +int16_t WebRtcIsac_EncodeJitterInfo(int32_t jitterIndex, + Bitstr* streamData); + + +/****************************************************************************** + * WebRtcIsac_DecodeJitterInfo() + * Decode the jitter information. + * + * Input/output: + * - streamdata : pointer to a stucture containg the encoded + * data and theparameters needed for entropy + * coding. + * + * Output: + * - jitterInfo : one bit of info specifying if the channel is + * in high/low jitter. Zero indicates low jitter + * and one indicates high jitter. + * + * Return value : 0 if succeeded. + * <0 if failed. + */ +int16_t WebRtcIsac_DecodeJitterInfo(Bitstr* streamData, + int32_t* jitterInfo); + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ENTROPY_CODING_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.c new file mode 100644 index 0000000000..c854d8c27c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.c @@ -0,0 +1,943 @@ +/* + * Copyright(c)1995,97 Mark Olesen + * Queen's Univ at Kingston (Canada) + * + * Permission to use, copy, modify, and distribute this software for + * any purpose without fee is hereby granted, provided that this + * entire notice is included in all copies of any software which is + * or includes a copy or modification of this software and in all + * copies of the supporting documentation for such software. + * + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR QUEEN'S + * UNIVERSITY AT KINGSTON MAKES ANY REPRESENTATION OR WARRANTY OF ANY + * KIND CONCERNING THE MERCHANTABILITY OF THIS SOFTWARE OR ITS + * FITNESS FOR ANY PARTICULAR PURPOSE. + * + * All of which is to say that you can do what you like with this + * source code provided you don't try to sell it as your own and you + * include an unaltered copy of this message (including the + * copyright). + * + * It is also implicitly understood that bug fixes and improvements + * should make their way back to the general Internet community so + * that everyone benefits. + * + * Changes: + * Trivial type modifications by the WebRTC authors. + */ + + +/* + * File: + * WebRtcIsac_Fftn.c + * + * Public: + * WebRtcIsac_Fftn / fftnf (); + * + * Private: + * WebRtcIsac_Fftradix / fftradixf (); + * + * Descript: + * multivariate complex Fourier transform, computed in place + * using mixed-radix Fast Fourier Transform algorithm. + * + * Fortran code by: + * RC Singleton, Stanford Research Institute, Sept. 1968 + * + * translated by f2c (version 19950721). + * + * int WebRtcIsac_Fftn (int ndim, const int dims[], REAL Re[], REAL Im[], + * int iSign, double scaling); + * + * NDIM = the total number dimensions + * DIMS = a vector of array sizes + * if NDIM is zero then DIMS must be zero-terminated + * + * RE and IM hold the real and imaginary components of the data, and return + * the resulting real and imaginary Fourier coefficients. Multidimensional + * data *must* be allocated contiguously. There is no limit on the number + * of dimensions. + * + * ISIGN = the sign of the complex exponential (ie, forward or inverse FFT) + * the magnitude of ISIGN (normally 1) is used to determine the + * correct indexing increment (see below). + * + * SCALING = normalizing constant by which the final result is *divided* + * if SCALING == -1, normalize by total dimension of the transform + * if SCALING < -1, normalize by the square-root of the total dimension + * + * example: + * tri-variate transform with Re[n1][n2][n3], Im[n1][n2][n3] + * + * int dims[3] = {n1,n2,n3} + * WebRtcIsac_Fftn (3, dims, Re, Im, 1, scaling); + * + *-----------------------------------------------------------------------* + * int WebRtcIsac_Fftradix (REAL Re[], REAL Im[], size_t nTotal, size_t nPass, + * size_t nSpan, int iSign, size_t max_factors, + * size_t max_perm); + * + * RE, IM - see above documentation + * + * Although there is no limit on the number of dimensions, WebRtcIsac_Fftradix() must + * be called once for each dimension, but the calls may be in any order. + * + * NTOTAL = the total number of complex data values + * NPASS = the dimension of the current variable + * NSPAN/NPASS = the spacing of consecutive data values while indexing the + * current variable + * ISIGN - see above documentation + * + * example: + * tri-variate transform with Re[n1][n2][n3], Im[n1][n2][n3] + * + * WebRtcIsac_Fftradix (Re, Im, n1*n2*n3, n1, n1, 1, maxf, maxp); + * WebRtcIsac_Fftradix (Re, Im, n1*n2*n3, n2, n1*n2, 1, maxf, maxp); + * WebRtcIsac_Fftradix (Re, Im, n1*n2*n3, n3, n1*n2*n3, 1, maxf, maxp); + * + * single-variate transform, + * NTOTAL = N = NSPAN = (number of complex data values), + * + * WebRtcIsac_Fftradix (Re, Im, n, n, n, 1, maxf, maxp); + * + * The data can also be stored in a single array with alternating real and + * imaginary parts, the magnitude of ISIGN is changed to 2 to give correct + * indexing increment, and data [0] and data [1] used to pass the initial + * addresses for the sequences of real and imaginary values, + * + * example: + * REAL data [2*NTOTAL]; + * WebRtcIsac_Fftradix ( &data[0], &data[1], NTOTAL, nPass, nSpan, 2, maxf, maxp); + * + * for temporary allocation: + * + * MAX_FACTORS >= the maximum prime factor of NPASS + * MAX_PERM >= the number of prime factors of NPASS. In addition, + * if the square-free portion K of NPASS has two or more prime + * factors, then MAX_PERM >= (K-1) + * + * storage in FACTOR for a maximum of 15 prime factors of NPASS. if NPASS + * has more than one square-free factor, the product of the square-free + * factors must be <= 210 array storage for maximum prime factor of 23 the + * following two constants should agree with the array dimensions. + * + *----------------------------------------------------------------------*/ +#include "fft.h" + +#include +#include + + + +/* double precision routine */ +static int +WebRtcIsac_Fftradix (double Re[], double Im[], + size_t nTotal, size_t nPass, size_t nSpan, int isign, + int max_factors, unsigned int max_perm, + FFTstr *fftstate); + + + +#ifndef M_PI +# define M_PI 3.14159265358979323846264338327950288 +#endif + +#ifndef SIN60 +# define SIN60 0.86602540378443865 /* sin(60 deg) */ +# define COS72 0.30901699437494742 /* cos(72 deg) */ +# define SIN72 0.95105651629515357 /* sin(72 deg) */ +#endif + +# define REAL double +# define FFTN WebRtcIsac_Fftn +# define FFTNS "fftn" +# define FFTRADIX WebRtcIsac_Fftradix +# define FFTRADIXS "fftradix" + + +int WebRtcIsac_Fftns(unsigned int ndim, const int dims[], + double Re[], + double Im[], + int iSign, + double scaling, + FFTstr *fftstate) +{ + + size_t nSpan, nPass, nTotal; + unsigned int i; + int ret, max_factors, max_perm; + + /* + * tally the number of elements in the data array + * and determine the number of dimensions + */ + nTotal = 1; + if (ndim && dims [0]) + { + for (i = 0; i < ndim; i++) + { + if (dims [i] <= 0) + { + return -1; + } + nTotal *= dims [i]; + } + } + else + { + ndim = 0; + for (i = 0; dims [i]; i++) + { + if (dims [i] <= 0) + { + return -1; + } + nTotal *= dims [i]; + ndim++; + } + } + + /* determine maximum number of factors and permuations */ +#if 1 + /* + * follow John Beale's example, just use the largest dimension and don't + * worry about excess allocation. May be someone else will do it? + */ + max_factors = max_perm = 1; + for (i = 0; i < ndim; i++) + { + nSpan = dims [i]; + if ((int)nSpan > max_factors) + { + max_factors = (int)nSpan; + } + if ((int)nSpan > max_perm) + { + max_perm = (int)nSpan; + } + } +#else + /* use the constants used in the original Fortran code */ + max_factors = 23; + max_perm = 209; +#endif + /* loop over the dimensions: */ + nPass = 1; + for (i = 0; i < ndim; i++) + { + nSpan = dims [i]; + nPass *= nSpan; + ret = FFTRADIX (Re, Im, nTotal, nSpan, nPass, iSign, + max_factors, max_perm, fftstate); + /* exit, clean-up already done */ + if (ret) + return ret; + } + + /* Divide through by the normalizing constant: */ + if (scaling && scaling != 1.0) + { + if (iSign < 0) iSign = -iSign; + if (scaling < 0.0) + { + scaling = (double)nTotal; + if (scaling < -1.0) + scaling = sqrt (scaling); + } + scaling = 1.0 / scaling; /* multiply is often faster */ + for (i = 0; i < nTotal; i += iSign) + { + Re [i] *= scaling; + Im [i] *= scaling; + } + } + return 0; +} + +/* + * singleton's mixed radix routine + * + * could move allocation out to WebRtcIsac_Fftn(), but leave it here so that it's + * possible to make this a standalone function + */ + +static int FFTRADIX (REAL Re[], + REAL Im[], + size_t nTotal, + size_t nPass, + size_t nSpan, + int iSign, + int max_factors, + unsigned int max_perm, + FFTstr *fftstate) +{ + int ii, mfactor, kspan, ispan, inc; + int j, jc, jf, jj, k, k1, k2, k3, k4, kk, kt, nn, ns, nt; + + + REAL radf; + REAL c1, c2, c3, cd, aa, aj, ak, ajm, ajp, akm, akp; + REAL s1, s2, s3, sd, bb, bj, bk, bjm, bjp, bkm, bkp; + + REAL *Rtmp = NULL; /* temp space for real part*/ + REAL *Itmp = NULL; /* temp space for imaginary part */ + REAL *Cos = NULL; /* Cosine values */ + REAL *Sin = NULL; /* Sine values */ + + REAL s60 = SIN60; /* sin(60 deg) */ + REAL c72 = COS72; /* cos(72 deg) */ + REAL s72 = SIN72; /* sin(72 deg) */ + REAL pi2 = M_PI; /* use PI first, 2 PI later */ + + + fftstate->SpaceAlloced = 0; + fftstate->MaxPermAlloced = 0; + + + // initialize to avoid warnings + k3 = c2 = c3 = s2 = s3 = 0.0; + + if (nPass < 2) + return 0; + + /* allocate storage */ + if (fftstate->SpaceAlloced < max_factors * sizeof (REAL)) + { +#ifdef SUN_BROKEN_REALLOC + if (!fftstate->SpaceAlloced) /* first time */ + { + fftstate->SpaceAlloced = max_factors * sizeof (REAL); + } + else + { +#endif + fftstate->SpaceAlloced = max_factors * sizeof (REAL); +#ifdef SUN_BROKEN_REALLOC + } +#endif + } + else + { + /* allow full use of alloc'd space */ + max_factors = fftstate->SpaceAlloced / sizeof (REAL); + } + if (fftstate->MaxPermAlloced < max_perm) + { +#ifdef SUN_BROKEN_REALLOC + if (!fftstate->MaxPermAlloced) /* first time */ + else +#endif + fftstate->MaxPermAlloced = max_perm; + } + else + { + /* allow full use of alloc'd space */ + max_perm = fftstate->MaxPermAlloced; + } + + /* assign pointers */ + Rtmp = (REAL *) fftstate->Tmp0; + Itmp = (REAL *) fftstate->Tmp1; + Cos = (REAL *) fftstate->Tmp2; + Sin = (REAL *) fftstate->Tmp3; + + /* + * Function Body + */ + inc = iSign; + if (iSign < 0) { + s72 = -s72; + s60 = -s60; + pi2 = -pi2; + inc = -inc; /* absolute value */ + } + + /* adjust for strange increments */ + nt = inc * (int)nTotal; + ns = inc * (int)nSpan; + kspan = ns; + + nn = nt - inc; + jc = ns / (int)nPass; + radf = pi2 * (double) jc; + pi2 *= 2.0; /* use 2 PI from here on */ + + ii = 0; + jf = 0; + /* determine the factors of n */ + mfactor = 0; + k = (int)nPass; + while (k % 16 == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = 4; + k /= 16; + } + j = 3; + jj = 9; + do { + while (k % jj == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = j; + k /= jj; + } + j += 2; + jj = j * j; + } while (jj <= k); + if (k <= 4) { + kt = mfactor; + fftstate->factor [mfactor] = k; + if (k != 1) + mfactor++; + } else { + if (k - (k / 4 << 2) == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = 2; + k /= 4; + } + kt = mfactor; + j = 2; + do { + if (k % j == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = j; + k /= j; + } + j = ((j + 1) / 2 << 1) + 1; + } while (j <= k); + } + if (kt) { + j = kt; + do { + mfactor++; + fftstate->factor [mfactor - 1] = fftstate->factor [j - 1]; + j--; + } while (j); + } + + /* test that mfactors is in range */ + if (mfactor > NFACTOR) + { + return -1; + } + + /* compute fourier transform */ + for (;;) { + sd = radf / (double) kspan; + cd = sin(sd); + cd = 2.0 * cd * cd; + sd = sin(sd + sd); + kk = 0; + ii++; + + switch (fftstate->factor [ii - 1]) { + case 2: + /* transform for factor of 2 (including rotation factor) */ + kspan /= 2; + k1 = kspan + 2; + do { + do { + k2 = kk + kspan; + ak = Re [k2]; + bk = Im [k2]; + Re [k2] = Re [kk] - ak; + Im [k2] = Im [kk] - bk; + Re [kk] += ak; + Im [kk] += bk; + kk = k2 + kspan; + } while (kk < nn); + kk -= nn; + } while (kk < jc); + if (kk >= kspan) + goto Permute_Results_Label; /* exit infinite loop */ + do { + c1 = 1.0 - cd; + s1 = sd; + do { + do { + do { + k2 = kk + kspan; + ak = Re [kk] - Re [k2]; + bk = Im [kk] - Im [k2]; + Re [kk] += Re [k2]; + Im [kk] += Im [k2]; + Re [k2] = c1 * ak - s1 * bk; + Im [k2] = s1 * ak + c1 * bk; + kk = k2 + kspan; + } while (kk < (nt-1)); + k2 = kk - nt; + c1 = -c1; + kk = k1 - k2; + } while (kk > k2); + ak = c1 - (cd * c1 + sd * s1); + s1 = sd * c1 - cd * s1 + s1; + c1 = 2.0 - (ak * ak + s1 * s1); + s1 *= c1; + c1 *= ak; + kk += jc; + } while (kk < k2); + k1 += inc + inc; + kk = (k1 - kspan + 1) / 2 + jc - 1; + } while (kk < (jc + jc)); + break; + + case 4: /* transform for factor of 4 */ + ispan = kspan; + kspan /= 4; + + do { + c1 = 1.0; + s1 = 0.0; + do { + do { + k1 = kk + kspan; + k2 = k1 + kspan; + k3 = k2 + kspan; + akp = Re [kk] + Re [k2]; + akm = Re [kk] - Re [k2]; + ajp = Re [k1] + Re [k3]; + ajm = Re [k1] - Re [k3]; + bkp = Im [kk] + Im [k2]; + bkm = Im [kk] - Im [k2]; + bjp = Im [k1] + Im [k3]; + bjm = Im [k1] - Im [k3]; + Re [kk] = akp + ajp; + Im [kk] = bkp + bjp; + ajp = akp - ajp; + bjp = bkp - bjp; + if (iSign < 0) { + akp = akm + bjm; + bkp = bkm - ajm; + akm -= bjm; + bkm += ajm; + } else { + akp = akm - bjm; + bkp = bkm + ajm; + akm += bjm; + bkm -= ajm; + } + /* avoid useless multiplies */ + if (s1 == 0.0) { + Re [k1] = akp; + Re [k2] = ajp; + Re [k3] = akm; + Im [k1] = bkp; + Im [k2] = bjp; + Im [k3] = bkm; + } else { + Re [k1] = akp * c1 - bkp * s1; + Re [k2] = ajp * c2 - bjp * s2; + Re [k3] = akm * c3 - bkm * s3; + Im [k1] = akp * s1 + bkp * c1; + Im [k2] = ajp * s2 + bjp * c2; + Im [k3] = akm * s3 + bkm * c3; + } + kk = k3 + kspan; + } while (kk < nt); + + c2 = c1 - (cd * c1 + sd * s1); + s1 = sd * c1 - cd * s1 + s1; + c1 = 2.0 - (c2 * c2 + s1 * s1); + s1 *= c1; + c1 *= c2; + /* values of c2, c3, s2, s3 that will get used next time */ + c2 = c1 * c1 - s1 * s1; + s2 = 2.0 * c1 * s1; + c3 = c2 * c1 - s2 * s1; + s3 = c2 * s1 + s2 * c1; + kk = kk - nt + jc; + } while (kk < kspan); + kk = kk - kspan + inc; + } while (kk < jc); + if (kspan == jc) + goto Permute_Results_Label; /* exit infinite loop */ + break; + + default: + /* transform for odd factors */ +#ifdef FFT_RADIX4 + return -1; + break; +#else /* FFT_RADIX4 */ + k = fftstate->factor [ii - 1]; + ispan = kspan; + kspan /= k; + + switch (k) { + case 3: /* transform for factor of 3 (optional code) */ + do { + do { + k1 = kk + kspan; + k2 = k1 + kspan; + ak = Re [kk]; + bk = Im [kk]; + aj = Re [k1] + Re [k2]; + bj = Im [k1] + Im [k2]; + Re [kk] = ak + aj; + Im [kk] = bk + bj; + ak -= 0.5 * aj; + bk -= 0.5 * bj; + aj = (Re [k1] - Re [k2]) * s60; + bj = (Im [k1] - Im [k2]) * s60; + Re [k1] = ak - bj; + Re [k2] = ak + bj; + Im [k1] = bk + aj; + Im [k2] = bk - aj; + kk = k2 + kspan; + } while (kk < (nn - 1)); + kk -= nn; + } while (kk < kspan); + break; + + case 5: /* transform for factor of 5 (optional code) */ + c2 = c72 * c72 - s72 * s72; + s2 = 2.0 * c72 * s72; + do { + do { + k1 = kk + kspan; + k2 = k1 + kspan; + k3 = k2 + kspan; + k4 = k3 + kspan; + akp = Re [k1] + Re [k4]; + akm = Re [k1] - Re [k4]; + bkp = Im [k1] + Im [k4]; + bkm = Im [k1] - Im [k4]; + ajp = Re [k2] + Re [k3]; + ajm = Re [k2] - Re [k3]; + bjp = Im [k2] + Im [k3]; + bjm = Im [k2] - Im [k3]; + aa = Re [kk]; + bb = Im [kk]; + Re [kk] = aa + akp + ajp; + Im [kk] = bb + bkp + bjp; + ak = akp * c72 + ajp * c2 + aa; + bk = bkp * c72 + bjp * c2 + bb; + aj = akm * s72 + ajm * s2; + bj = bkm * s72 + bjm * s2; + Re [k1] = ak - bj; + Re [k4] = ak + bj; + Im [k1] = bk + aj; + Im [k4] = bk - aj; + ak = akp * c2 + ajp * c72 + aa; + bk = bkp * c2 + bjp * c72 + bb; + aj = akm * s2 - ajm * s72; + bj = bkm * s2 - bjm * s72; + Re [k2] = ak - bj; + Re [k3] = ak + bj; + Im [k2] = bk + aj; + Im [k3] = bk - aj; + kk = k4 + kspan; + } while (kk < (nn-1)); + kk -= nn; + } while (kk < kspan); + break; + + default: + if (k != jf) { + jf = k; + s1 = pi2 / (double) k; + c1 = cos(s1); + s1 = sin(s1); + if (jf > max_factors){ + return -1; + } + Cos [jf - 1] = 1.0; + Sin [jf - 1] = 0.0; + j = 1; + do { + Cos [j - 1] = Cos [k - 1] * c1 + Sin [k - 1] * s1; + Sin [j - 1] = Cos [k - 1] * s1 - Sin [k - 1] * c1; + k--; + Cos [k - 1] = Cos [j - 1]; + Sin [k - 1] = -Sin [j - 1]; + j++; + } while (j < k); + } + do { + do { + k1 = kk; + k2 = kk + ispan; + ak = aa = Re [kk]; + bk = bb = Im [kk]; + j = 1; + k1 += kspan; + do { + k2 -= kspan; + j++; + Rtmp [j - 1] = Re [k1] + Re [k2]; + ak += Rtmp [j - 1]; + Itmp [j - 1] = Im [k1] + Im [k2]; + bk += Itmp [j - 1]; + j++; + Rtmp [j - 1] = Re [k1] - Re [k2]; + Itmp [j - 1] = Im [k1] - Im [k2]; + k1 += kspan; + } while (k1 < k2); + Re [kk] = ak; + Im [kk] = bk; + k1 = kk; + k2 = kk + ispan; + j = 1; + do { + k1 += kspan; + k2 -= kspan; + jj = j; + ak = aa; + bk = bb; + aj = 0.0; + bj = 0.0; + k = 1; + do { + k++; + ak += Rtmp [k - 1] * Cos [jj - 1]; + bk += Itmp [k - 1] * Cos [jj - 1]; + k++; + aj += Rtmp [k - 1] * Sin [jj - 1]; + bj += Itmp [k - 1] * Sin [jj - 1]; + jj += j; + if (jj > jf) { + jj -= jf; + } + } while (k < jf); + k = jf - j; + Re [k1] = ak - bj; + Im [k1] = bk + aj; + Re [k2] = ak + bj; + Im [k2] = bk - aj; + j++; + } while (j < k); + kk += ispan; + } while (kk < nn); + kk -= nn; + } while (kk < kspan); + break; + } + + /* multiply by rotation factor (except for factors of 2 and 4) */ + if (ii == mfactor) + goto Permute_Results_Label; /* exit infinite loop */ + kk = jc; + do { + c2 = 1.0 - cd; + s1 = sd; + do { + c1 = c2; + s2 = s1; + kk += kspan; + do { + do { + ak = Re [kk]; + Re [kk] = c2 * ak - s2 * Im [kk]; + Im [kk] = s2 * ak + c2 * Im [kk]; + kk += ispan; + } while (kk < nt); + ak = s1 * s2; + s2 = s1 * c2 + c1 * s2; + c2 = c1 * c2 - ak; + kk = kk - nt + kspan; + } while (kk < ispan); + c2 = c1 - (cd * c1 + sd * s1); + s1 += sd * c1 - cd * s1; + c1 = 2.0 - (c2 * c2 + s1 * s1); + s1 *= c1; + c2 *= c1; + kk = kk - ispan + jc; + } while (kk < kspan); + kk = kk - kspan + jc + inc; + } while (kk < (jc + jc)); + break; +#endif /* FFT_RADIX4 */ + } + } + + /* permute the results to normal order---done in two stages */ + /* permutation for square factors of n */ +Permute_Results_Label: + fftstate->Perm [0] = ns; + if (kt) { + k = kt + kt + 1; + if (mfactor < k) + k--; + j = 1; + fftstate->Perm [k] = jc; + do { + fftstate->Perm [j] = fftstate->Perm [j - 1] / fftstate->factor [j - 1]; + fftstate->Perm [k - 1] = fftstate->Perm [k] * fftstate->factor [j - 1]; + j++; + k--; + } while (j < k); + k3 = fftstate->Perm [k]; + kspan = fftstate->Perm [1]; + kk = jc; + k2 = kspan; + j = 1; + if (nPass != nTotal) { + /* permutation for multivariate transform */ + Permute_Multi_Label: + do { + do { + k = kk + jc; + do { + /* swap Re [kk] <> Re [k2], Im [kk] <> Im [k2] */ + ak = Re [kk]; Re [kk] = Re [k2]; Re [k2] = ak; + bk = Im [kk]; Im [kk] = Im [k2]; Im [k2] = bk; + kk += inc; + k2 += inc; + } while (kk < (k-1)); + kk += ns - jc; + k2 += ns - jc; + } while (kk < (nt-1)); + k2 = k2 - nt + kspan; + kk = kk - nt + jc; + } while (k2 < (ns-1)); + do { + do { + k2 -= fftstate->Perm [j - 1]; + j++; + k2 = fftstate->Perm [j] + k2; + } while (k2 > fftstate->Perm [j - 1]); + j = 1; + do { + if (kk < (k2-1)) + goto Permute_Multi_Label; + kk += jc; + k2 += kspan; + } while (k2 < (ns-1)); + } while (kk < (ns-1)); + } else { + /* permutation for single-variate transform (optional code) */ + Permute_Single_Label: + do { + /* swap Re [kk] <> Re [k2], Im [kk] <> Im [k2] */ + ak = Re [kk]; Re [kk] = Re [k2]; Re [k2] = ak; + bk = Im [kk]; Im [kk] = Im [k2]; Im [k2] = bk; + kk += inc; + k2 += kspan; + } while (k2 < (ns-1)); + do { + do { + k2 -= fftstate->Perm [j - 1]; + j++; + k2 = fftstate->Perm [j] + k2; + } while (k2 >= fftstate->Perm [j - 1]); + j = 1; + do { + if (kk < k2) + goto Permute_Single_Label; + kk += inc; + k2 += kspan; + } while (k2 < (ns-1)); + } while (kk < (ns-1)); + } + jc = k3; + } + + if ((kt << 1) + 1 >= mfactor) + return 0; + ispan = fftstate->Perm [kt]; + /* permutation for square-free factors of n */ + j = mfactor - kt; + fftstate->factor [j] = 1; + do { + fftstate->factor [j - 1] *= fftstate->factor [j]; + j--; + } while (j != kt); + kt++; + nn = fftstate->factor [kt - 1] - 1; + if (nn > (int) max_perm) { + return -1; + } + j = jj = 0; + for (;;) { + k = kt + 1; + k2 = fftstate->factor [kt - 1]; + kk = fftstate->factor [k - 1]; + j++; + if (j > nn) + break; /* exit infinite loop */ + jj += kk; + while (jj >= k2) { + jj -= k2; + k2 = kk; + k++; + kk = fftstate->factor [k - 1]; + jj += kk; + } + fftstate->Perm [j - 1] = jj; + } + /* determine the permutation cycles of length greater than 1 */ + j = 0; + for (;;) { + do { + j++; + kk = fftstate->Perm [j - 1]; + } while (kk < 0); + if (kk != j) { + do { + k = kk; + kk = fftstate->Perm [k - 1]; + fftstate->Perm [k - 1] = -kk; + } while (kk != j); + k3 = kk; + } else { + fftstate->Perm [j - 1] = -j; + if (j == nn) + break; /* exit infinite loop */ + } + } + max_factors *= inc; + /* reorder a and b, following the permutation cycles */ + for (;;) { + j = k3 + 1; + nt -= ispan; + ii = nt - inc + 1; + if (nt < 0) + break; /* exit infinite loop */ + do { + do { + j--; + } while (fftstate->Perm [j - 1] < 0); + jj = jc; + do { + kspan = jj; + if (jj > max_factors) { + kspan = max_factors; + } + jj -= kspan; + k = fftstate->Perm [j - 1]; + kk = jc * k + ii + jj; + k1 = kk + kspan - 1; + k2 = 0; + do { + k2++; + Rtmp [k2 - 1] = Re [k1]; + Itmp [k2 - 1] = Im [k1]; + k1 -= inc; + } while (k1 != (kk-1)); + do { + k1 = kk + kspan - 1; + k2 = k1 - jc * (k + fftstate->Perm [k - 1]); + k = -fftstate->Perm [k - 1]; + do { + Re [k1] = Re [k2]; + Im [k1] = Im [k2]; + k1 -= inc; + k2 -= inc; + } while (k1 != (kk-1)); + kk = k2 + 1; + } while (k != j); + k1 = kk + kspan - 1; + k2 = 0; + do { + k2++; + Re [k1] = Rtmp [k2 - 1]; + Im [k1] = Itmp [k2 - 1]; + k1 -= inc; + } while (k1 != (kk-1)); + } while (jj); + } while (j != 1); + } + return 0; /* exit point here */ +} +/* ---------------------- end-of-file (c source) ---------------------- */ + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.h new file mode 100644 index 0000000000..b583b50268 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/*--------------------------------*-C-*---------------------------------* + * File: + * fftn.h + * ---------------------------------------------------------------------* + * Re[]: real value array + * Im[]: imaginary value array + * nTotal: total number of complex values + * nPass: number of elements involved in this pass of transform + * nSpan: nspan/nPass = number of bytes to increment pointer + * in Re[] and Im[] + * isign: exponent: +1 = forward -1 = reverse + * scaling: normalizing constant by which the final result is *divided* + * scaling == -1, normalize by total dimension of the transform + * scaling < -1, normalize by the square-root of the total dimension + * + * ---------------------------------------------------------------------- + * See the comments in the code for correct usage! + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FFT_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FFT_H_ + + +#include "structs.h" + + +/* double precision routine */ + + +int WebRtcIsac_Fftns (unsigned int ndim, const int dims[], double Re[], double Im[], + int isign, double scaling, FFTstr *fftstate); + + + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FFT_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c new file mode 100644 index 0000000000..d47eb1fa66 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#ifdef WEBRTC_ANDROID +#include +#endif +#include "pitch_estimator.h" +#include "lpc_analysis.h" +#include "codec.h" + + + +void WebRtcIsac_AllPoleFilter(double* InOut, + double* Coef, + size_t lengthInOut, + int orderCoef) { + /* the state of filter is assumed to be in InOut[-1] to InOut[-orderCoef] */ + double scal; + double sum; + size_t n; + int k; + + //if (fabs(Coef[0]-1.0)<0.001) { + if ( (Coef[0] > 0.9999) && (Coef[0] < 1.0001) ) + { + for(n = 0; n < lengthInOut; n++) + { + sum = Coef[1] * InOut[-1]; + for(k = 2; k <= orderCoef; k++){ + sum += Coef[k] * InOut[-k]; + } + *InOut++ -= sum; + } + } + else + { + scal = 1.0 / Coef[0]; + for(n=0;nbuffer, sizeof(double) * PITCH_WLPCBUFLEN); + memcpy(tmpbuffer+PITCH_WLPCBUFLEN, in, sizeof(double) * PITCH_FRAME_LEN); + memcpy(wfdata->buffer, tmpbuffer+PITCH_FRAME_LEN, sizeof(double) * PITCH_WLPCBUFLEN); + + dp=weoutbuf; + dp2=whoutbuf; + for (k=0;kweostate[k]; + *dp2++ = wfdata->whostate[k]; + opol[k]=0.0; + } + opol[0]=1.0; + opol[PITCH_WLPCORDER]=0.0; + weo=dp; + who=dp2; + + endpos=PITCH_WLPCBUFLEN + PITCH_SUBFRAME_LEN; + inp=tmpbuffer + PITCH_WLPCBUFLEN; + + for (n=0; nwindow[k]*tmpbuffer[start+k]; + } + + /* Get LPC polynomial */ + WebRtcIsac_AutoCorr(corr, ext, PITCH_WLPCWINLEN, PITCH_WLPCORDER); + corr[0]=1.01*corr[0]+1.0; /* White noise correction */ + WebRtcIsac_LevDurb(apol, rc, corr, PITCH_WLPCORDER); + WebRtcIsac_BwExpand(apolr, apol, rho, PITCH_WLPCORDER+1); + + /* Filtering */ + WebRtcIsac_ZeroPoleFilter(inp, apol, apolr, PITCH_SUBFRAME_LEN, PITCH_WLPCORDER, weo); + WebRtcIsac_ZeroPoleFilter(inp, apolr, opol, PITCH_SUBFRAME_LEN, PITCH_WLPCORDER, who); + + inp+=PITCH_SUBFRAME_LEN; + endpos+=PITCH_SUBFRAME_LEN; + weo+=PITCH_SUBFRAME_LEN; + who+=PITCH_SUBFRAME_LEN; + } + + /* Export filter states */ + for (k=0;kweostate[k]=weoutbuf[PITCH_FRAME_LEN+k]; + wfdata->whostate[k]=whoutbuf[PITCH_FRAME_LEN+k]; + } + + /* Export output data */ + memcpy(weiout, weoutbuf+PITCH_WLPCORDER, sizeof(double) * PITCH_FRAME_LEN); + memcpy(whiout, whoutbuf+PITCH_WLPCORDER, sizeof(double) * PITCH_FRAME_LEN); +} + + +static const double APupper[ALLPASSSECTIONS] = {0.0347, 0.3826}; +static const double APlower[ALLPASSSECTIONS] = {0.1544, 0.744}; + + +void WebRtcIsac_AllpassFilterForDec(double* InOut, + const double* APSectionFactors, + size_t lengthInOut, + double* FilterState) { + //This performs all-pass filtering--a series of first order all-pass sections are used + //to filter the input in a cascade manner. + size_t n,j; + double temp; + for (j=0; jINLABUFx arrays + each of length QLOOKAHEAD. + The remaining FRAMESAMPLES_HALF-QLOOKAHEAD samples are based + on the first FRAMESAMPLES_HALF-QLOOKAHEAD samples of the input + array in[]. + HP: a FRAMESAMPLES_HALF array of high-pass filtered samples that + have been phase equalized. The first QLOOKAHEAD samples are + based on the samples in the two prefiltdata->INLABUFx arrays + each of length QLOOKAHEAD. + The remaining FRAMESAMPLES_HALF-QLOOKAHEAD samples are based + on the first FRAMESAMPLES_HALF-QLOOKAHEAD samples of the input + array in[]. + + LP_la: a FRAMESAMPLES_HALF array of low-pass filtered samples. + These samples are not phase equalized. They are computed + from the samples in the in[] array. + HP_la: a FRAMESAMPLES_HALF array of high-pass filtered samples + that are not phase equalized. They are computed from + the in[] vector. + prefiltdata: this input data structure's filterbank state and + lookahead sample buffers are updated for the next + encoding iteration. +*/ +void WebRtcIsac_SplitAndFilterFloat(float *pin, float *LP, float *HP, + double *LP_la, double *HP_la, + PreFiltBankstr *prefiltdata) +{ + int k,n; + float CompositeAPFilterState[NUMBEROFCOMPOSITEAPSECTIONS]; + float ForTransform_CompositeAPFilterState[NUMBEROFCOMPOSITEAPSECTIONS]; + float ForTransform_CompositeAPFilterState2[NUMBEROFCOMPOSITEAPSECTIONS]; + float tempinoutvec[FRAMESAMPLES+MAX_AR_MODEL_ORDER]; + float tempin_ch1[FRAMESAMPLES+MAX_AR_MODEL_ORDER]; + float tempin_ch2[FRAMESAMPLES+MAX_AR_MODEL_ORDER]; + float in[FRAMESAMPLES]; + float ftmp; + + + /* High pass filter */ + + for (k=0;kHPstates_float[0] + + kHpStCoefInFloat[3] * prefiltdata->HPstates_float[1]; + ftmp = pin[k] - kHpStCoefInFloat[0] * prefiltdata->HPstates_float[0] - + kHpStCoefInFloat[1] * prefiltdata->HPstates_float[1]; + prefiltdata->HPstates_float[1] = prefiltdata->HPstates_float[0]; + prefiltdata->HPstates_float[0] = ftmp; + } + + /* + % backwards all-pass filtering to obtain zero-phase + [tmp1(N2+LA:-1:LA+1, 1), state1] = filter(Q.coef, Q.coef(end:-1:1), in(N:-2:2)); + tmp1(LA:-1:1) = filter(Q.coef, Q.coef(end:-1:1), Q.LookAheadBuf1, state1); + Q.LookAheadBuf1 = in(N:-2:N-2*LA+2); + */ + /*Backwards all-pass filter the odd samples of the input (upper channel) + to eventually obtain zero phase. The composite all-pass filter (comprised of both + the upper and lower channel all-pass filsters in series) is used for the + filtering. */ + + /* First Channel */ + + /*initial state of composite filter is zero */ + for (k=0;kINLABUF1_float, + WebRtcIsac_kCompositeApFactorsFloat, QLOOKAHEAD, + NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + /* save the output, but write it in forward order */ + /* write the lookahead samples for the next encoding iteration. Every other + sample at the end of the input frame is written in reverse order for the + lookahead length. Exported in the prefiltdata structure. */ + for (k=0;kINLABUF1_float[k]; + prefiltdata->INLABUF1_float[k]=in[FRAMESAMPLES-1-2*k]; + } + + /* Second Channel. This is exactly like the first channel, except that the + even samples are now filtered instead (lower channel). */ + for (k=0;kINLABUF2_float, + WebRtcIsac_kCompositeApFactorsFloat, QLOOKAHEAD,NUMBEROFCOMPOSITEAPSECTIONS, + CompositeAPFilterState); + + for (k=0;kINLABUF2_float[k]; + prefiltdata->INLABUF2_float[k]=in[FRAMESAMPLES-2-2*k]; + } + + /* Transform filter states from backward to forward */ + /*At this point, each of the states of the backwards composite filters for the + two channels are transformed into forward filtering states for the corresponding + forward channel filters. Each channel's forward filtering state from the previous + encoding iteration is added to the transformed state to get a proper forward state */ + + /* So the existing NUMBEROFCOMPOSITEAPSECTIONS x 1 (4x1) state vector is multiplied by a + NUMBEROFCHANNELAPSECTIONSxNUMBEROFCOMPOSITEAPSECTIONS (2x4) transform matrix to get the + new state that is added to the previous 2x1 input state */ + + for (k=0;kINSTAT1_float[k] += ForTransform_CompositeAPFilterState[n]* + WebRtcIsac_kTransform1Float[k*NUMBEROFCHANNELAPSECTIONS+n]; + prefiltdata->INSTAT2_float[k] += ForTransform_CompositeAPFilterState2[n]* + WebRtcIsac_kTransform2Float[k*NUMBEROFCHANNELAPSECTIONS+n]; + } + } + + /*obtain polyphase components by forward all-pass filtering through each channel */ + /* the backward filtered samples are now forward filtered with the corresponding channel filters */ + /* The all pass filtering automatically updates the filter states which are exported in the + prefiltdata structure */ + WebRtcIsac_AllPassFilter2Float(tempin_ch1,WebRtcIsac_kUpperApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, prefiltdata->INSTAT1_float); + WebRtcIsac_AllPassFilter2Float(tempin_ch2,WebRtcIsac_kLowerApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, prefiltdata->INSTAT2_float); + + /* Now Construct low-pass and high-pass signals as combinations of polyphase components */ + for (k=0; kINSTATLA1_float); + WebRtcIsac_AllPassFilter2Float(tempin_ch2,WebRtcIsac_kLowerApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, prefiltdata->INSTATLA2_float); + + for (k=0; kSTATE_0_UPPER_float); + + /* Now, all-pass filter the new lower channel signal. But since all-pass filter factors + at the decoder are swapped from the ones at the encoder, the 'upper' channel + all-pass filter factors (WebRtcIsac_kUpperApFactorsFloat) are used to filter this new + lower channel signal */ + WebRtcIsac_AllPassFilter2Float(tempin_ch2, WebRtcIsac_kUpperApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS,postfiltdata->STATE_0_LOWER_float); + + + /* Merge outputs to form the full length output signal.*/ + for (k=0;kHPstates1_float[0] + + kHpStCoefOut1Float[3] * postfiltdata->HPstates1_float[1]; + ftmp = Out[k] - kHpStCoefOut1Float[0] * postfiltdata->HPstates1_float[0] - + kHpStCoefOut1Float[1] * postfiltdata->HPstates1_float[1]; + postfiltdata->HPstates1_float[1] = postfiltdata->HPstates1_float[0]; + postfiltdata->HPstates1_float[0] = ftmp; + Out[k] = ftmp2; + } + + for (k=0;kHPstates2_float[0] + + kHpStCoefOut2Float[3] * postfiltdata->HPstates2_float[1]; + ftmp = Out[k] - kHpStCoefOut2Float[0] * postfiltdata->HPstates2_float[0] - + kHpStCoefOut2Float[1] * postfiltdata->HPstates2_float[1]; + postfiltdata->HPstates2_float[1] = postfiltdata->HPstates2_float[0]; + postfiltdata->HPstates2_float[0] = ftmp; + Out[k] = ftmp2; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/intialize.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/intialize.c new file mode 100644 index 0000000000..01e683c573 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/intialize.c @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* encode.c - Encoding function for the iSAC coder */ + +#include "structs.h" +#include "codec.h" +#include "pitch_estimator.h" + +#include + +void WebRtcIsac_InitMasking(MaskFiltstr *maskdata) { + + int k; + + for (k = 0; k < WINLEN; k++) { + maskdata->DataBufferLo[k] = 0.0; + maskdata->DataBufferHi[k] = 0.0; + } + for (k = 0; k < ORDERLO+1; k++) { + maskdata->CorrBufLo[k] = 0.0; + maskdata->PreStateLoF[k] = 0.0; + maskdata->PreStateLoG[k] = 0.0; + maskdata->PostStateLoF[k] = 0.0; + maskdata->PostStateLoG[k] = 0.0; + } + for (k = 0; k < ORDERHI+1; k++) { + maskdata->CorrBufHi[k] = 0.0; + maskdata->PreStateHiF[k] = 0.0; + maskdata->PreStateHiG[k] = 0.0; + maskdata->PostStateHiF[k] = 0.0; + maskdata->PostStateHiG[k] = 0.0; + } + + maskdata->OldEnergy = 10.0; + return; +} + +void WebRtcIsac_InitPreFilterbank(PreFiltBankstr *prefiltdata) +{ + int k; + + for (k = 0; k < QLOOKAHEAD; k++) { + prefiltdata->INLABUF1[k] = 0; + prefiltdata->INLABUF2[k] = 0; + + prefiltdata->INLABUF1_float[k] = 0; + prefiltdata->INLABUF2_float[k] = 0; + } + for (k = 0; k < 2*(QORDER-1); k++) { + prefiltdata->INSTAT1[k] = 0; + prefiltdata->INSTAT2[k] = 0; + prefiltdata->INSTATLA1[k] = 0; + prefiltdata->INSTATLA2[k] = 0; + + prefiltdata->INSTAT1_float[k] = 0; + prefiltdata->INSTAT2_float[k] = 0; + prefiltdata->INSTATLA1_float[k] = 0; + prefiltdata->INSTATLA2_float[k] = 0; + } + + /* High pass filter states */ + prefiltdata->HPstates[0] = 0.0; + prefiltdata->HPstates[1] = 0.0; + + prefiltdata->HPstates_float[0] = 0.0f; + prefiltdata->HPstates_float[1] = 0.0f; + + return; +} + +void WebRtcIsac_InitPostFilterbank(PostFiltBankstr *postfiltdata) +{ + int k; + + for (k = 0; k < 2*POSTQORDER; k++) { + postfiltdata->STATE_0_LOWER[k] = 0; + postfiltdata->STATE_0_UPPER[k] = 0; + + postfiltdata->STATE_0_LOWER_float[k] = 0; + postfiltdata->STATE_0_UPPER_float[k] = 0; + } + + /* High pass filter states */ + postfiltdata->HPstates1[0] = 0.0; + postfiltdata->HPstates1[1] = 0.0; + + postfiltdata->HPstates2[0] = 0.0; + postfiltdata->HPstates2[1] = 0.0; + + postfiltdata->HPstates1_float[0] = 0.0f; + postfiltdata->HPstates1_float[1] = 0.0f; + + postfiltdata->HPstates2_float[0] = 0.0f; + postfiltdata->HPstates2_float[1] = 0.0f; + + return; +} + + +void WebRtcIsac_InitPitchFilter(PitchFiltstr *pitchfiltdata) +{ + int k; + + for (k = 0; k < PITCH_BUFFSIZE; k++) { + pitchfiltdata->ubuf[k] = 0.0; + } + pitchfiltdata->ystate[0] = 0.0; + for (k = 1; k < (PITCH_DAMPORDER); k++) { + pitchfiltdata->ystate[k] = 0.0; + } + pitchfiltdata->oldlagp[0] = 50.0; + pitchfiltdata->oldgainp[0] = 0.0; +} + +void WebRtcIsac_InitWeightingFilter(WeightFiltstr *wfdata) +{ + int k; + double t, dtmp, dtmp2, denum, denum2; + + for (k=0;kbuffer[k]=0.0; + + for (k=0;kistate[k]=0.0; + wfdata->weostate[k]=0.0; + wfdata->whostate[k]=0.0; + } + + /* next part should be in Matlab, writing to a global table */ + t = 0.5; + denum = 1.0 / ((double) PITCH_WLPCWINLEN); + denum2 = denum * denum; + for (k=0;kwindow[k] = dtmp2 * dtmp2; + t++; + } +} + +/* clear all buffers */ +void WebRtcIsac_InitPitchAnalysis(PitchAnalysisStruct *State) +{ + int k; + + for (k = 0; k < PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2; k++) + State->dec_buffer[k] = 0.0; + for (k = 0; k < 2*ALLPASSSECTIONS+1; k++) + State->decimator_state[k] = 0.0; + for (k = 0; k < 2; k++) + State->hp_state[k] = 0.0; + for (k = 0; k < QLOOKAHEAD; k++) + State->whitened_buf[k] = 0.0; + for (k = 0; k < QLOOKAHEAD; k++) + State->inbuf[k] = 0.0; + + WebRtcIsac_InitPitchFilter(&(State->PFstr_wght)); + + WebRtcIsac_InitPitchFilter(&(State->PFstr)); + + WebRtcIsac_InitWeightingFilter(&(State->Wghtstr)); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c new file mode 100644 index 0000000000..79dc7e2b39 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c @@ -0,0 +1,2374 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * isac.c + * + * This C file contains the functions for the ISAC API + * + */ + +#include "modules/audio_coding/codecs/isac/main/include/isac.h" + +#include +#include +#include +#include + +#include "rtc_base/checks.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h" +#include "modules/audio_coding/codecs/isac/main/source/codec.h" +#include "modules/audio_coding/codecs/isac/main/source/crc.h" +#include "modules/audio_coding/codecs/isac/main/source/entropy_coding.h" +#include "modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.h" +#include "modules/audio_coding/codecs/isac/main/source/os_specific_inline.h" +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +#define BIT_MASK_DEC_INIT 0x0001 +#define BIT_MASK_ENC_INIT 0x0002 + +#define LEN_CHECK_SUM_WORD8 4 +#define MAX_NUM_LAYERS 10 + + +/**************************************************************************** + * UpdatePayloadSizeLimit(...) + * + * Call this function to update the limit on the payload size. The limit on + * payload size might change i) if a user ''directly changes the limit by + * calling xxx_setMaxPayloadSize() or xxx_setMaxRate(), or ii) indirectly + * when bandwidth is changing. The latter might be the result of bandwidth + * adaptation, or direct change of the bottleneck in instantaneous mode. + * + * This function takes the current overall limit on payload, and translates it + * to the limits on lower and upper-band. If the codec is in wideband mode, + * then the overall limit and the limit on the lower-band is the same. + * Otherwise, a fraction of the limit should be allocated to lower-band + * leaving some room for the upper-band bit-stream. That is why an update + * of limit is required every time that the bandwidth is changing. + * + */ +static void UpdatePayloadSizeLimit(ISACMainStruct* instISAC) { + int16_t lim30MsPayloadBytes = WEBRTC_SPL_MIN( + (instISAC->maxPayloadSizeBytes), + (instISAC->maxRateBytesPer30Ms)); + int16_t lim60MsPayloadBytes = WEBRTC_SPL_MIN( + (instISAC->maxPayloadSizeBytes), + (instISAC->maxRateBytesPer30Ms << 1)); + + /* The only time that iSAC will have 60 ms + * frame-size is when operating in wideband, so + * there is no upper-band bit-stream. */ + + if (instISAC->bandwidthKHz == isac8kHz) { + /* At 8 kHz there is no upper-band bit-stream, + * therefore, the lower-band limit is the overall limit. */ + instISAC->instLB.ISACencLB_obj.payloadLimitBytes60 = + lim60MsPayloadBytes; + instISAC->instLB.ISACencLB_obj.payloadLimitBytes30 = + lim30MsPayloadBytes; + } else { + /* When in super-wideband, we only have 30 ms frames. + * Do a rate allocation for the given limit. */ + if (lim30MsPayloadBytes > 250) { + /* 4/5 to lower-band the rest for upper-band. */ + instISAC->instLB.ISACencLB_obj.payloadLimitBytes30 = + (lim30MsPayloadBytes << 2) / 5; + } else if (lim30MsPayloadBytes > 200) { + /* For the interval of 200 to 250 the share of + * upper-band linearly grows from 20 to 50. */ + instISAC->instLB.ISACencLB_obj.payloadLimitBytes30 = + (lim30MsPayloadBytes << 1) / 5 + 100; + } else { + /* Allocate only 20 for upper-band. */ + instISAC->instLB.ISACencLB_obj.payloadLimitBytes30 = + lim30MsPayloadBytes - 20; + } + instISAC->instUB.ISACencUB_obj.maxPayloadSizeBytes = + lim30MsPayloadBytes; + } +} + + +/**************************************************************************** + * UpdateBottleneck(...) + * + * This function updates the bottleneck only if the codec is operating in + * channel-adaptive mode. Furthermore, as the update of bottleneck might + * result in an update of bandwidth, therefore, the bottlenech should be + * updated just right before the first 10ms of a frame is pushed into encoder. + * + */ +static void UpdateBottleneck(ISACMainStruct* instISAC) { + /* Read the bottleneck from bandwidth estimator for the + * first 10 ms audio. This way, if there is a change + * in bandwidth, upper and lower-band will be in sync. */ + if ((instISAC->codingMode == 0) && + (instISAC->instLB.ISACencLB_obj.buffer_index == 0) && + (instISAC->instLB.ISACencLB_obj.frame_nb == 0)) { + int32_t bottleneck = + WebRtcIsac_GetUplinkBandwidth(&instISAC->bwestimator_obj); + + /* Adding hysteresis when increasing signal bandwidth. */ + if ((instISAC->bandwidthKHz == isac8kHz) + && (bottleneck > 37000) + && (bottleneck < 41000)) { + bottleneck = 37000; + } + + /* Switching from 12 kHz to 16 kHz is not allowed at this revision. + * If we let this happen, we have to take care of buffer_index and + * the last LPC vector. */ + if ((instISAC->bandwidthKHz != isac16kHz) && + (bottleneck > 46000)) { + bottleneck = 46000; + } + + /* We might need a rate allocation. */ + if (instISAC->encoderSamplingRateKHz == kIsacWideband) { + /* Wideband is the only choice we have here. */ + instISAC->instLB.ISACencLB_obj.bottleneck = + (bottleneck > 32000) ? 32000 : bottleneck; + instISAC->bandwidthKHz = isac8kHz; + } else { + /* Do the rate-allocation and get the new bandwidth. */ + enum ISACBandwidth bandwidth; + WebRtcIsac_RateAllocation(bottleneck, + &(instISAC->instLB.ISACencLB_obj.bottleneck), + &(instISAC->instUB.ISACencUB_obj.bottleneck), + &bandwidth); + if (bandwidth != isac8kHz) { + instISAC->instLB.ISACencLB_obj.new_framelength = 480; + } + if (bandwidth != instISAC->bandwidthKHz) { + /* Bandwidth is changing. */ + instISAC->bandwidthKHz = bandwidth; + UpdatePayloadSizeLimit(instISAC); + if (bandwidth == isac12kHz) { + instISAC->instLB.ISACencLB_obj.buffer_index = 0; + } + /* Currently we don't let the bandwidth to switch to 16 kHz + * if in adaptive mode. If we let this happen, we have to take + * care of buffer_index and the last LPC vector. */ + } + } + } +} + + +/**************************************************************************** + * GetSendBandwidthInfo(...) + * + * This is called to get the bandwidth info. This info is the bandwidth and + * the jitter of 'there-to-here' channel, estimated 'here.' These info + * is signaled in an in-band fashion to the other side. + * + * The call to the bandwidth estimator triggers a recursive averaging which + * has to be synchronized between encoder & decoder, therefore, the call to + * BWE should be once per packet. As the BWE info is inserted into bit-stream + * We need a valid info right before the encodeLB function is going to + * generate a bit-stream. That is when lower-band buffer has already 20ms + * of audio, and the 3rd block of 10ms is going to be injected into encoder. + * + * Inputs: + * - instISAC : iSAC instance. + * + * Outputs: + * - bandwidthIndex : an index which has to be encoded in + * lower-band bit-stream, indicating the + * bandwidth of there-to-here channel. + * - jitterInfo : this indicates if the jitter is high + * or low and it is encoded in upper-band + * bit-stream. + * + */ +static void GetSendBandwidthInfo(ISACMainStruct* instISAC, + int16_t* bandwidthIndex, + int16_t* jitterInfo) { + if ((instISAC->instLB.ISACencLB_obj.buffer_index == + (FRAMESAMPLES_10ms << 1)) && + (instISAC->instLB.ISACencLB_obj.frame_nb == 0)) { + /* Bandwidth estimation and coding. */ + WebRtcIsac_GetDownlinkBwJitIndexImpl(&(instISAC->bwestimator_obj), + bandwidthIndex, jitterInfo, + instISAC->decoderSamplingRateKHz); + } +} + + +/**************************************************************************** + * WebRtcIsac_AssignSize(...) + * + * This function returns the size of the ISAC instance, so that the instance + * can be created out side iSAC. + * + * Output: + * - sizeinbytes : number of bytes needed to allocate for the + * instance. + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcIsac_AssignSize(int* sizeInBytes) { + *sizeInBytes = sizeof(ISACMainStruct) * 2 / sizeof(int16_t); + return 0; +} + + +/**************************************************************************** + * WebRtcIsac_Assign(...) + * + * This function assigns the memory already created to the ISAC instance. + * + * Input: + * - ISAC_main_inst : address of the pointer to the coder instance. + * - instISAC_Addr : the already allocated memory, where we put the + * iSAC structure. + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcIsac_Assign(ISACStruct** ISAC_main_inst, + void* instISAC_Addr) { + if (instISAC_Addr != NULL) { + ISACMainStruct* instISAC = (ISACMainStruct*)instISAC_Addr; + instISAC->errorCode = 0; + instISAC->initFlag = 0; + + /* Assign the address. */ + *ISAC_main_inst = (ISACStruct*)instISAC_Addr; + + /* Default is wideband. */ + instISAC->encoderSamplingRateKHz = kIsacWideband; + instISAC->decoderSamplingRateKHz = kIsacWideband; + instISAC->bandwidthKHz = isac8kHz; + instISAC->in_sample_rate_hz = 16000; + + WebRtcIsac_InitTransform(&instISAC->transform_tables); + return 0; + } else { + return -1; + } +} + + +/**************************************************************************** + * WebRtcIsac_Create(...) + * + * This function creates an ISAC instance, which will contain the state + * information for one coding/decoding channel. + * + * Input: + * - ISAC_main_inst : address of the pointer to the coder instance. + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcIsac_Create(ISACStruct** ISAC_main_inst) { + ISACMainStruct* instISAC; + + if (ISAC_main_inst != NULL) { + instISAC = (ISACMainStruct*)malloc(sizeof(ISACMainStruct)); + *ISAC_main_inst = (ISACStruct*)instISAC; + if (*ISAC_main_inst != NULL) { + instISAC->errorCode = 0; + instISAC->initFlag = 0; + /* Default is wideband. */ + instISAC->bandwidthKHz = isac8kHz; + instISAC->encoderSamplingRateKHz = kIsacWideband; + instISAC->decoderSamplingRateKHz = kIsacWideband; + instISAC->in_sample_rate_hz = 16000; + + WebRtcIsac_InitTransform(&instISAC->transform_tables); + return 0; + } else { + return -1; + } + } else { + return -1; + } +} + + +/**************************************************************************** + * WebRtcIsac_Free(...) + * + * This function frees the ISAC instance created at the beginning. + * + * Input: + * - ISAC_main_inst : a ISAC instance. + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcIsac_Free(ISACStruct* ISAC_main_inst) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + free(instISAC); + return 0; +} + + +/**************************************************************************** + * EncoderInitLb(...) - internal function for initialization of + * Lower Band + * EncoderInitUb(...) - internal function for initialization of + * Upper Band + * WebRtcIsac_EncoderInit(...) - API function + * + * This function initializes a ISAC instance prior to the encoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - CodingMode : 0 -> Bit rate and frame length are automatically + * adjusted to available bandwidth on + * transmission channel, applicable just to + * wideband mode. + * 1 -> User sets a frame length and a target bit + * rate which is taken as the maximum + * short-term average bit rate. + * + * Return value : 0 - Ok + * -1 - Error + */ +static int16_t EncoderInitLb(ISACLBStruct* instLB, + int16_t codingMode, + enum IsacSamplingRate sampRate) { + int16_t statusInit = 0; + int k; + + /* Init stream vector to zero */ + for (k = 0; k < STREAM_SIZE_MAX_60; k++) { + instLB->ISACencLB_obj.bitstr_obj.stream[k] = 0; + } + + if ((codingMode == 1) || (sampRate == kIsacSuperWideband)) { + /* 30 ms frame-size if either in super-wideband or + * instantaneous mode (I-mode). */ + instLB->ISACencLB_obj.new_framelength = 480; + } else { + instLB->ISACencLB_obj.new_framelength = INITIAL_FRAMESAMPLES; + } + + WebRtcIsac_InitMasking(&instLB->ISACencLB_obj.maskfiltstr_obj); + WebRtcIsac_InitPreFilterbank(&instLB->ISACencLB_obj.prefiltbankstr_obj); + WebRtcIsac_InitPitchFilter(&instLB->ISACencLB_obj.pitchfiltstr_obj); + WebRtcIsac_InitPitchAnalysis( + &instLB->ISACencLB_obj.pitchanalysisstr_obj); + + instLB->ISACencLB_obj.buffer_index = 0; + instLB->ISACencLB_obj.frame_nb = 0; + /* Default for I-mode. */ + instLB->ISACencLB_obj.bottleneck = 32000; + instLB->ISACencLB_obj.current_framesamples = 0; + instLB->ISACencLB_obj.s2nr = 0; + instLB->ISACencLB_obj.payloadLimitBytes30 = STREAM_SIZE_MAX_30; + instLB->ISACencLB_obj.payloadLimitBytes60 = STREAM_SIZE_MAX_60; + instLB->ISACencLB_obj.maxPayloadBytes = STREAM_SIZE_MAX_60; + instLB->ISACencLB_obj.maxRateInBytes = STREAM_SIZE_MAX_30; + instLB->ISACencLB_obj.enforceFrameSize = 0; + /* Invalid value prevents getRedPayload to + run before encoder is called. */ + instLB->ISACencLB_obj.lastBWIdx = -1; + return statusInit; +} + +static int16_t EncoderInitUb(ISACUBStruct* instUB, + int16_t bandwidth) { + int16_t statusInit = 0; + int k; + + /* Init stream vector to zero. */ + for (k = 0; k < STREAM_SIZE_MAX_60; k++) { + instUB->ISACencUB_obj.bitstr_obj.stream[k] = 0; + } + + WebRtcIsac_InitMasking(&instUB->ISACencUB_obj.maskfiltstr_obj); + WebRtcIsac_InitPreFilterbank(&instUB->ISACencUB_obj.prefiltbankstr_obj); + + if (bandwidth == isac16kHz) { + instUB->ISACencUB_obj.buffer_index = LB_TOTAL_DELAY_SAMPLES; + } else { + instUB->ISACencUB_obj.buffer_index = 0; + } + /* Default for I-mode. */ + instUB->ISACencUB_obj.bottleneck = 32000; + /* These store the limits for the wideband + super-wideband bit-stream. */ + instUB->ISACencUB_obj.maxPayloadSizeBytes = STREAM_SIZE_MAX_30 << 1; + /* This has to be updated after each lower-band encoding to guarantee + * a correct payload-limitation. */ + instUB->ISACencUB_obj.numBytesUsed = 0; + memset(instUB->ISACencUB_obj.data_buffer_float, 0, + (MAX_FRAMESAMPLES + LB_TOTAL_DELAY_SAMPLES) * sizeof(float)); + + memcpy(&(instUB->ISACencUB_obj.lastLPCVec), + WebRtcIsac_kMeanLarUb16, sizeof(double) * UB_LPC_ORDER); + + return statusInit; +} + + +int16_t WebRtcIsac_EncoderInit(ISACStruct* ISAC_main_inst, + int16_t codingMode) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + int16_t status; + + if ((codingMode != 0) && (codingMode != 1)) { + instISAC->errorCode = ISAC_DISALLOWED_CODING_MODE; + return -1; + } + /* Default bottleneck. */ + instISAC->bottleneck = MAX_ISAC_BW; + + if (instISAC->encoderSamplingRateKHz == kIsacWideband) { + instISAC->bandwidthKHz = isac8kHz; + instISAC->maxPayloadSizeBytes = STREAM_SIZE_MAX_60; + instISAC->maxRateBytesPer30Ms = STREAM_SIZE_MAX_30; + } else { + instISAC->bandwidthKHz = isac16kHz; + instISAC->maxPayloadSizeBytes = STREAM_SIZE_MAX; + instISAC->maxRateBytesPer30Ms = STREAM_SIZE_MAX; + } + + /* Channel-adaptive = 0; Instantaneous (Channel-independent) = 1. */ + instISAC->codingMode = codingMode; + + WebRtcIsac_InitBandwidthEstimator(&instISAC->bwestimator_obj, + instISAC->encoderSamplingRateKHz, + instISAC->decoderSamplingRateKHz); + + WebRtcIsac_InitRateModel(&instISAC->rate_data_obj); + /* Default for I-mode. */ + instISAC->MaxDelay = 10.0; + + status = EncoderInitLb(&instISAC->instLB, codingMode, + instISAC->encoderSamplingRateKHz); + if (status < 0) { + instISAC->errorCode = -status; + return -1; + } + + if (instISAC->encoderSamplingRateKHz == kIsacSuperWideband) { + /* Initialize encoder filter-bank. */ + memset(instISAC->analysisFBState1, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + memset(instISAC->analysisFBState2, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + + status = EncoderInitUb(&(instISAC->instUB), + instISAC->bandwidthKHz); + if (status < 0) { + instISAC->errorCode = -status; + return -1; + } + } + /* Initialization is successful, set the flag. */ + instISAC->initFlag |= BIT_MASK_ENC_INIT; + return 0; +} + + +/**************************************************************************** + * WebRtcIsac_Encode(...) + * + * This function encodes 10ms frame(s) and inserts it into a package. + * Input speech length has to be 160 samples (10ms). The encoder buffers those + * 10ms frames until it reaches the chosen Framesize (480 or 960 samples + * corresponding to 30 or 60 ms frames), and then proceeds to the encoding. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - speechIn : input speech vector. + * + * Output: + * - encoded : the encoded data vector + * + * Return value: + * : >0 - Length (in bytes) of coded data + * : 0 - The buffer didn't reach the chosen + * frameSize so it keeps buffering speech + * samples. + * : -1 - Error + */ +int WebRtcIsac_Encode(ISACStruct* ISAC_main_inst, + const int16_t* speechIn, + uint8_t* encoded) { + float inFrame[FRAMESAMPLES_10ms]; + int16_t speechInLB[FRAMESAMPLES_10ms]; + int16_t speechInUB[FRAMESAMPLES_10ms]; + int streamLenLB = 0; + int streamLenUB = 0; + int streamLen = 0; + size_t k = 0; + uint8_t garbageLen = 0; + int32_t bottleneck = 0; + int16_t bottleneckIdx = 0; + int16_t jitterInfo = 0; + + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + ISACLBStruct* instLB = &(instISAC->instLB); + ISACUBStruct* instUB = &(instISAC->instUB); + + /* Check if encoder initiated. */ + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + return -1; + } + + if (instISAC->encoderSamplingRateKHz == kIsacSuperWideband) { + WebRtcSpl_AnalysisQMF(speechIn, SWBFRAMESAMPLES_10ms, speechInLB, + speechInUB, instISAC->analysisFBState1, + instISAC->analysisFBState2); + + /* Convert from fixed to floating point. */ + for (k = 0; k < FRAMESAMPLES_10ms; k++) { + inFrame[k] = (float)speechInLB[k]; + } + } else { + for (k = 0; k < FRAMESAMPLES_10ms; k++) { + inFrame[k] = (float) speechIn[k]; + } + } + + /* Add some noise to avoid denormal numbers. */ + inFrame[0] += (float)1.23455334e-3; + inFrame[1] -= (float)2.04324239e-3; + inFrame[2] += (float)1.90854954e-3; + inFrame[9] += (float)1.84854878e-3; + + /* This function will update the bottleneck if required. */ + UpdateBottleneck(instISAC); + + /* Get the bandwith information which has to be sent to the other side. */ + GetSendBandwidthInfo(instISAC, &bottleneckIdx, &jitterInfo); + + /* Encode lower-band. */ + streamLenLB = WebRtcIsac_EncodeLb(&instISAC->transform_tables, + inFrame, &instLB->ISACencLB_obj, + instISAC->codingMode, bottleneckIdx); + if (streamLenLB < 0) { + return -1; + } + + if (instISAC->encoderSamplingRateKHz == kIsacSuperWideband) { + instUB = &(instISAC->instUB); + + /* Convert to float. */ + for (k = 0; k < FRAMESAMPLES_10ms; k++) { + inFrame[k] = (float) speechInUB[k]; + } + + /* Add some noise to avoid denormal numbers. */ + inFrame[0] += (float)1.23455334e-3; + inFrame[1] -= (float)2.04324239e-3; + inFrame[2] += (float)1.90854954e-3; + inFrame[9] += (float)1.84854878e-3; + + /* Tell to upper-band the number of bytes used so far. + * This is for payload limitation. */ + instUB->ISACencUB_obj.numBytesUsed = + (int16_t)(streamLenLB + 1 + LEN_CHECK_SUM_WORD8); + /* Encode upper-band. */ + switch (instISAC->bandwidthKHz) { + case isac12kHz: { + streamLenUB = WebRtcIsac_EncodeUb12(&instISAC->transform_tables, + inFrame, &instUB->ISACencUB_obj, + jitterInfo); + break; + } + case isac16kHz: { + streamLenUB = WebRtcIsac_EncodeUb16(&instISAC->transform_tables, + inFrame, &instUB->ISACencUB_obj, + jitterInfo); + break; + } + case isac8kHz: { + streamLenUB = 0; + break; + } + } + + if ((streamLenUB < 0) && (streamLenUB != -ISAC_PAYLOAD_LARGER_THAN_LIMIT)) { + /* An error has happened but this is not the error due to a + * bit-stream larger than the limit. */ + return -1; + } + + if (streamLenLB == 0) { + return 0; + } + + /* One byte is allocated for the length. According to older decoders + so the length bit-stream plus one byte for size and + LEN_CHECK_SUM_WORD8 for the checksum should be less than or equal + to 255. */ + if ((streamLenUB > (255 - (LEN_CHECK_SUM_WORD8 + 1))) || + (streamLenUB == -ISAC_PAYLOAD_LARGER_THAN_LIMIT)) { + /* We have got a too long bit-stream we skip the upper-band + * bit-stream for this frame. */ + streamLenUB = 0; + } + + memcpy(encoded, instLB->ISACencLB_obj.bitstr_obj.stream, streamLenLB); + streamLen = streamLenLB; + if (streamLenUB > 0) { + encoded[streamLenLB] = (uint8_t)(streamLenUB + 1 + LEN_CHECK_SUM_WORD8); + memcpy(&encoded[streamLenLB + 1], + instUB->ISACencUB_obj.bitstr_obj.stream, + streamLenUB); + streamLen += encoded[streamLenLB]; + } else { + encoded[streamLenLB] = 0; + } + } else { + if (streamLenLB == 0) { + return 0; + } + memcpy(encoded, instLB->ISACencLB_obj.bitstr_obj.stream, streamLenLB); + streamLenUB = 0; + streamLen = streamLenLB; + } + + /* Add Garbage if required. */ + bottleneck = WebRtcIsac_GetUplinkBandwidth(&instISAC->bwestimator_obj); + if (instISAC->codingMode == 0) { + int minBytes; + int limit; + uint8_t* ptrGarbage; + + instISAC->MaxDelay = (double)WebRtcIsac_GetUplinkMaxDelay( + &instISAC->bwestimator_obj); + + /* Update rate model and get minimum number of bytes in this packet. */ + minBytes = WebRtcIsac_GetMinBytes( + &(instISAC->rate_data_obj), streamLen, + instISAC->instLB.ISACencLB_obj.current_framesamples, bottleneck, + instISAC->MaxDelay, instISAC->bandwidthKHz); + + /* Make sure MinBytes does not exceed packet size limit. */ + if (instISAC->bandwidthKHz == isac8kHz) { + if (instLB->ISACencLB_obj.current_framesamples == FRAMESAMPLES) { + limit = instLB->ISACencLB_obj.payloadLimitBytes30; + } else { + limit = instLB->ISACencLB_obj.payloadLimitBytes60; + } + } else { + limit = instUB->ISACencUB_obj.maxPayloadSizeBytes; + } + minBytes = (minBytes > limit) ? limit : minBytes; + + /* Make sure we don't allow more than 255 bytes of garbage data. + * We store the length of the garbage data in 8 bits in the bitstream, + * 255 is the max garbage length we can signal using 8 bits. */ + if ((instISAC->bandwidthKHz == isac8kHz) || + (streamLenUB == 0)) { + ptrGarbage = &encoded[streamLenLB]; + limit = streamLen + 255; + } else { + ptrGarbage = &encoded[streamLenLB + 1 + streamLenUB]; + limit = streamLen + (255 - encoded[streamLenLB]); + } + minBytes = (minBytes > limit) ? limit : minBytes; + + garbageLen = (minBytes > streamLen) ? (uint8_t)(minBytes - streamLen) : 0; + + /* Save data for creation of multiple bit-streams. */ + /* If bit-stream too short then add garbage at the end. */ + if (garbageLen > 0) { + /* Overwrite the garbage area to avoid leaking possibly sensitive data + over the network. This also makes the output deterministic. */ + memset(ptrGarbage, 0, garbageLen); + + /* For a correct length of the upper-band bit-stream together + * with the garbage. Garbage is embeded in upper-band bit-stream. + * That is the only way to preserve backward compatibility. */ + if ((instISAC->bandwidthKHz == isac8kHz) || + (streamLenUB == 0)) { + encoded[streamLenLB] = garbageLen; + } else { + encoded[streamLenLB] += garbageLen; + /* Write the length of the garbage at the end of the upper-band + * bit-stream, if exists. This helps for sanity check. */ + encoded[streamLenLB + 1 + streamLenUB] = garbageLen; + + } + streamLen += garbageLen; + } + } else { + /* update rate model */ + WebRtcIsac_UpdateRateModel( + &instISAC->rate_data_obj, streamLen, + instISAC->instLB.ISACencLB_obj.current_framesamples, bottleneck); + garbageLen = 0; + } + + /* Generate CRC if required. */ + if ((instISAC->bandwidthKHz != isac8kHz) && (streamLenUB > 0)) { + uint32_t crc; + + WebRtcIsac_GetCrc((int16_t*)(&(encoded[streamLenLB + 1])), + streamLenUB + garbageLen, &crc); +#ifndef WEBRTC_ARCH_BIG_ENDIAN + for (k = 0; k < LEN_CHECK_SUM_WORD8; k++) { + encoded[streamLen - LEN_CHECK_SUM_WORD8 + k] = + (uint8_t)(crc >> (24 - k * 8)); + } +#else + memcpy(&encoded[streamLenLB + streamLenUB + 1], &crc, LEN_CHECK_SUM_WORD8); +#endif + } + return streamLen; +} + + +/****************************************************************************** + * WebRtcIsac_GetNewBitStream(...) + * + * This function returns encoded data, with the recieved bwe-index in the + * stream. If the rate is set to a value less than bottleneck of codec + * the new bistream will be re-encoded with the given target rate. + * It should always return a complete packet, i.e. only called once + * even for 60 msec frames. + * + * NOTE 1! This function does not write in the ISACStruct, it is not allowed. + * NOTE 2! Rates larger than the bottleneck of the codec will be limited + * to the current bottleneck. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - bweIndex : Index of bandwidth estimate to put in new + * bitstream + * - rate : target rate of the transcoder is bits/sec. + * Valid values are the accepted rate in iSAC, + * i.e. 10000 to 56000. + * + * Output: + * - encoded : The encoded data vector + * + * Return value : >0 - Length (in bytes) of coded data + * -1 - Error or called in SWB mode + * NOTE! No error code is written to + * the struct since it is only allowed to read + * the struct. + */ +int16_t WebRtcIsac_GetNewBitStream(ISACStruct* ISAC_main_inst, + int16_t bweIndex, + int16_t jitterInfo, + int32_t rate, + uint8_t* encoded, + int16_t isRCU) { + Bitstr iSACBitStreamInst; /* Local struct for bitstream handling */ + int16_t streamLenLB; + int16_t streamLenUB; + int16_t totalStreamLen; + double gain2; + double gain1; + float scale; + enum ISACBandwidth bandwidthKHz; + double rateLB; + double rateUB; + int32_t currentBN; + uint32_t crc; +#ifndef WEBRTC_ARCH_BIG_ENDIAN + int16_t k; +#endif + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + return -1; + } + + /* Get the bottleneck of this iSAC and limit the + * given rate to the current bottleneck. */ + WebRtcIsac_GetUplinkBw(ISAC_main_inst, ¤tBN); + if (rate > currentBN) { + rate = currentBN; + } + + if (WebRtcIsac_RateAllocation(rate, &rateLB, &rateUB, &bandwidthKHz) < 0) { + return -1; + } + + /* Cannot transcode from 16 kHz to 12 kHz. */ + if ((bandwidthKHz == isac12kHz) && + (instISAC->bandwidthKHz == isac16kHz)) { + return -1; + } + + /* A gain [dB] for the given rate. */ + gain1 = WebRtcIsac_GetSnr( + rateLB, instISAC->instLB.ISACencLB_obj.current_framesamples); + /* The gain [dB] of this iSAC. */ + gain2 = WebRtcIsac_GetSnr( + instISAC->instLB.ISACencLB_obj.bottleneck, + instISAC->instLB.ISACencLB_obj.current_framesamples); + + /* Scale is the ratio of two gains in normal domain. */ + scale = (float)pow(10, (gain1 - gain2) / 20.0); + /* Change the scale if this is a RCU bit-stream. */ + scale = (isRCU) ? (scale * RCU_TRANSCODING_SCALE) : scale; + + streamLenLB = WebRtcIsac_EncodeStoredDataLb( + &instISAC->instLB.ISACencLB_obj.SaveEnc_obj, + &iSACBitStreamInst, bweIndex, scale); + + if (streamLenLB < 0) { + return -1; + } + + /* Convert from bytes to int16_t. */ + memcpy(encoded, iSACBitStreamInst.stream, streamLenLB); + + if (bandwidthKHz == isac8kHz) { + return streamLenLB; + } + + totalStreamLen = streamLenLB; + /* super-wideband is always at 30ms. + * These gains are in dB. + * Gain for the given rate. */ + gain1 = WebRtcIsac_GetSnr(rateUB, FRAMESAMPLES); + /* Gain of this iSAC */ + gain2 = WebRtcIsac_GetSnr(instISAC->instUB.ISACencUB_obj.bottleneck, + FRAMESAMPLES); + + /* Scale is the ratio of two gains in normal domain. */ + scale = (float)pow(10, (gain1 - gain2) / 20.0); + + /* Change the scale if this is a RCU bit-stream. */ + scale = (isRCU)? (scale * RCU_TRANSCODING_SCALE_UB) : scale; + + streamLenUB = WebRtcIsac_EncodeStoredDataUb( + &(instISAC->instUB.ISACencUB_obj.SaveEnc_obj), + &iSACBitStreamInst, jitterInfo, scale, + instISAC->bandwidthKHz); + + if (streamLenUB < 0) { + return -1; + } + + if (streamLenUB + 1 + LEN_CHECK_SUM_WORD8 > 255) { + return streamLenLB; + } + + totalStreamLen = streamLenLB + streamLenUB + 1 + LEN_CHECK_SUM_WORD8; + encoded[streamLenLB] = streamLenUB + 1 + LEN_CHECK_SUM_WORD8; + + memcpy(&encoded[streamLenLB + 1], iSACBitStreamInst.stream, + streamLenUB); + + WebRtcIsac_GetCrc((int16_t*)(&(encoded[streamLenLB + 1])), + streamLenUB, &crc); +#ifndef WEBRTC_ARCH_BIG_ENDIAN + for (k = 0; k < LEN_CHECK_SUM_WORD8; k++) { + encoded[totalStreamLen - LEN_CHECK_SUM_WORD8 + k] = + (uint8_t)((crc >> (24 - k * 8)) & 0xFF); + } +#else + memcpy(&encoded[streamLenLB + streamLenUB + 1], &crc, + LEN_CHECK_SUM_WORD8); +#endif + return totalStreamLen; +} + + +/**************************************************************************** + * DecoderInitLb(...) - internal function for initialization of + * Lower Band + * DecoderInitUb(...) - internal function for initialization of + * Upper Band + * WebRtcIsac_DecoderInit(...) - API function + * + * This function initializes a ISAC instance prior to the decoder calls. + * + * Input: + * - ISAC_main_inst : ISAC instance. + */ +static void DecoderInitLb(ISACLBStruct* instISAC) { + int i; + /* Initialize stream vector to zero. */ + for (i = 0; i < STREAM_SIZE_MAX_60; i++) { + instISAC->ISACdecLB_obj.bitstr_obj.stream[i] = 0; + } + + WebRtcIsac_InitMasking(&instISAC->ISACdecLB_obj.maskfiltstr_obj); + WebRtcIsac_InitPostFilterbank( + &instISAC->ISACdecLB_obj.postfiltbankstr_obj); + WebRtcIsac_InitPitchFilter(&instISAC->ISACdecLB_obj.pitchfiltstr_obj); +} + +static void DecoderInitUb(ISACUBStruct* instISAC) { + int i; + /* Init stream vector to zero */ + for (i = 0; i < STREAM_SIZE_MAX_60; i++) { + instISAC->ISACdecUB_obj.bitstr_obj.stream[i] = 0; + } + + WebRtcIsac_InitMasking(&instISAC->ISACdecUB_obj.maskfiltstr_obj); + WebRtcIsac_InitPostFilterbank( + &instISAC->ISACdecUB_obj.postfiltbankstr_obj); +} + +void WebRtcIsac_DecoderInit(ISACStruct* ISAC_main_inst) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + + DecoderInitLb(&instISAC->instLB); + if (instISAC->decoderSamplingRateKHz == kIsacSuperWideband) { + memset(instISAC->synthesisFBState1, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + memset(instISAC->synthesisFBState2, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + DecoderInitUb(&(instISAC->instUB)); + } + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != BIT_MASK_ENC_INIT) { + WebRtcIsac_InitBandwidthEstimator(&instISAC->bwestimator_obj, + instISAC->encoderSamplingRateKHz, + instISAC->decoderSamplingRateKHz); + } + instISAC->initFlag |= BIT_MASK_DEC_INIT; + instISAC->resetFlag_8kHz = 0; +} + + +/**************************************************************************** + * WebRtcIsac_UpdateBwEstimate(...) + * + * This function updates the estimate of the bandwidth. + * + * NOTE: + * The estimates of bandwidth is not valid if the sample rate of the far-end + * encoder is set to 48 kHz and send timestamps are increamented according to + * 48 kHz sampling rate. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s). + * - packet_size : size of the packet. + * - rtp_seq_number : the RTP number of the packet. + * - arr_ts : the arrival time of the packet (from NetEq) + * in samples. + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcIsac_UpdateBwEstimate(ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + Bitstr streamdata; +#ifndef WEBRTC_ARCH_BIG_ENDIAN + int k; +#endif + int16_t err; + + /* Check if decoder initiated. */ + if ((instISAC->initFlag & BIT_MASK_DEC_INIT) != BIT_MASK_DEC_INIT) { + instISAC->errorCode = ISAC_DECODER_NOT_INITIATED; + return -1; + } + + /* Check that the size of the packet is valid, and if not return without + * updating the bandwidth estimate. A valid size is at least 10 bytes. */ + if (packet_size < 10) { + /* Return error code if the packet length is null. */ + instISAC->errorCode = ISAC_EMPTY_PACKET; + return -1; + } + + WebRtcIsac_ResetBitstream(&(streamdata)); + +#ifndef WEBRTC_ARCH_BIG_ENDIAN + for (k = 0; k < 10; k++) { + uint16_t ek = ((const uint16_t*)encoded)[k >> 1]; + streamdata.stream[k] = (uint8_t)((ek >> ((k & 1) << 3)) & 0xff); + } +#else + memcpy(streamdata.stream, encoded, 10); +#endif + + err = WebRtcIsac_EstimateBandwidth(&instISAC->bwestimator_obj, &streamdata, + packet_size, rtp_seq_number, send_ts, + arr_ts, instISAC->encoderSamplingRateKHz, + instISAC->decoderSamplingRateKHz); + if (err < 0) { + /* Return error code if something went wrong. */ + instISAC->errorCode = -err; + return -1; + } + return 0; +} + +static int Decode(ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t lenEncodedBytes, + int16_t* decoded, + int16_t* speechType, + int16_t isRCUPayload) { + /* Number of samples (480 or 960), output from decoder + that were actually used in the encoder/decoder + (determined on the fly). */ + int16_t numSamplesLB; + int16_t numSamplesUB; + int16_t speechIdx; + float outFrame[MAX_FRAMESAMPLES]; + int16_t outFrameLB[MAX_FRAMESAMPLES]; + int16_t outFrameUB[MAX_FRAMESAMPLES]; + int numDecodedBytesLBint; + size_t numDecodedBytesLB; + int numDecodedBytesUB; + size_t lenEncodedLBBytes; + int16_t validChecksum = 1; + int16_t k; + uint16_t numLayer; + size_t totSizeBytes; + int16_t err; + + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + ISACUBDecStruct* decInstUB = &(instISAC->instUB.ISACdecUB_obj); + ISACLBDecStruct* decInstLB = &(instISAC->instLB.ISACdecLB_obj); + + /* Check if decoder initiated. */ + if ((instISAC->initFlag & BIT_MASK_DEC_INIT) != + BIT_MASK_DEC_INIT) { + instISAC->errorCode = ISAC_DECODER_NOT_INITIATED; + return -1; + } + + if (lenEncodedBytes == 0) { + /* return error code if the packet length is null. */ + instISAC->errorCode = ISAC_EMPTY_PACKET; + return -1; + } + + /* The size of the encoded lower-band is bounded by + * STREAM_SIZE_MAX. If a payload with the size larger than STREAM_SIZE_MAX + * is received, it is not considered erroneous. */ + lenEncodedLBBytes = (lenEncodedBytes > STREAM_SIZE_MAX) ? + STREAM_SIZE_MAX : lenEncodedBytes; + + /* Copy to lower-band bit-stream structure. */ + memcpy(instISAC->instLB.ISACdecLB_obj.bitstr_obj.stream, encoded, + lenEncodedLBBytes); + + /* We need to initialize numSamplesLB to something; otherwise, in the test + for whether we should return -1 below, the compiler might generate code + that fools Memcheck (Valgrind) into thinking that the control flow depends + on the uninitialized value in numSamplesLB (since WebRtcIsac_DecodeLb will + not fill it in if it fails and returns -1). */ + numSamplesLB = 0; + + /* Regardless of that the current codec is setup to work in + * wideband or super-wideband, the decoding of the lower-band + * has to be performed. */ + numDecodedBytesLBint = WebRtcIsac_DecodeLb(&instISAC->transform_tables, + outFrame, decInstLB, + &numSamplesLB, isRCUPayload); + numDecodedBytesLB = (size_t)numDecodedBytesLBint; + if ((numDecodedBytesLBint < 0) || + (numDecodedBytesLB > lenEncodedLBBytes) || + (numSamplesLB > MAX_FRAMESAMPLES)) { + instISAC->errorCode = ISAC_LENGTH_MISMATCH; + return -1; + } + + /* Error Check, we accept multi-layer bit-stream This will limit number + * of iterations of the while loop. Even without this the number + * of iterations is limited. */ + numLayer = 1; + totSizeBytes = numDecodedBytesLB; + while (totSizeBytes != lenEncodedBytes) { + if ((totSizeBytes > lenEncodedBytes) || + (encoded[totSizeBytes] == 0) || + (numLayer > MAX_NUM_LAYERS)) { + instISAC->errorCode = ISAC_LENGTH_MISMATCH; + return -1; + } + totSizeBytes += encoded[totSizeBytes]; + numLayer++; + } + + if (instISAC->decoderSamplingRateKHz == kIsacWideband) { + for (k = 0; k < numSamplesLB; k++) { + if (outFrame[k] > 32767) { + decoded[k] = 32767; + } else if (outFrame[k] < -32768) { + decoded[k] = -32768; + } else { + decoded[k] = (int16_t)WebRtcIsac_lrint(outFrame[k]); + } + } + numSamplesUB = 0; + } else { + uint32_t crc; + /* We don't accept larger than 30ms (480 samples at lower-band) + * frame-size. */ + for (k = 0; k < numSamplesLB; k++) { + if (outFrame[k] > 32767) { + outFrameLB[k] = 32767; + } else if (outFrame[k] < -32768) { + outFrameLB[k] = -32768; + } else { + outFrameLB[k] = (int16_t)WebRtcIsac_lrint(outFrame[k]); + } + } + + /* Check for possible error, and if upper-band stream exists. */ + if (numDecodedBytesLB == lenEncodedBytes) { + /* Decoding was successful. No super-wideband bit-stream exists. */ + numSamplesUB = numSamplesLB; + memset(outFrameUB, 0, sizeof(int16_t) * numSamplesUB); + + /* Prepare for the potential increase of signal bandwidth. */ + instISAC->resetFlag_8kHz = 2; + } else { + /* This includes the checksum and the bytes that stores the length. */ + int16_t lenNextStream = encoded[numDecodedBytesLB]; + + /* Is this garbage or valid super-wideband bit-stream? + * Check if checksum is valid. */ + if (lenNextStream <= (LEN_CHECK_SUM_WORD8 + 1)) { + /* Such a small second layer cannot be super-wideband layer. + * It must be a short garbage. */ + validChecksum = 0; + } else { + /* Run CRC to see if the checksum match. */ + WebRtcIsac_GetCrc((int16_t*)(&encoded[numDecodedBytesLB + 1]), + lenNextStream - LEN_CHECK_SUM_WORD8 - 1, &crc); + + validChecksum = 1; + for (k = 0; k < LEN_CHECK_SUM_WORD8; k++) { + validChecksum &= (((crc >> (24 - k * 8)) & 0xFF) == + encoded[numDecodedBytesLB + lenNextStream - + LEN_CHECK_SUM_WORD8 + k]); + } + } + + if (!validChecksum) { + /* This is a garbage, we have received a wideband + * bit-stream with garbage. */ + numSamplesUB = numSamplesLB; + memset(outFrameUB, 0, sizeof(int16_t) * numSamplesUB); + } else { + /* A valid super-wideband biststream exists. */ + enum ISACBandwidth bandwidthKHz; + int32_t maxDelayBit; + + /* If we have super-wideband bit-stream, we cannot + * have 60 ms frame-size. */ + if (numSamplesLB > FRAMESAMPLES) { + instISAC->errorCode = ISAC_LENGTH_MISMATCH; + return -1; + } + + /* The rest of the bit-stream contains the upper-band + * bit-stream curently this is the only thing there, + * however, we might add more layers. */ + + /* Have to exclude one byte where the length is stored + * and last 'LEN_CHECK_SUM_WORD8' bytes where the + * checksum is stored. */ + lenNextStream -= (LEN_CHECK_SUM_WORD8 + 1); + + memcpy(decInstUB->bitstr_obj.stream, + &encoded[numDecodedBytesLB + 1], lenNextStream); + + /* Reset bit-stream object, this is the first decoding. */ + WebRtcIsac_ResetBitstream(&(decInstUB->bitstr_obj)); + + /* Decode jitter information. */ + err = WebRtcIsac_DecodeJitterInfo(&decInstUB->bitstr_obj, &maxDelayBit); + if (err < 0) { + instISAC->errorCode = -err; + return -1; + } + + /* Update jitter info which is in the upper-band bit-stream + * only if the encoder is in super-wideband. Otherwise, + * the jitter info is already embedded in bandwidth index + * and has been updated. */ + if (instISAC->encoderSamplingRateKHz == kIsacSuperWideband) { + err = WebRtcIsac_UpdateUplinkJitter( + &(instISAC->bwestimator_obj), maxDelayBit); + if (err < 0) { + instISAC->errorCode = -err; + return -1; + } + } + + /* Decode bandwidth information. */ + err = WebRtcIsac_DecodeBandwidth(&decInstUB->bitstr_obj, + &bandwidthKHz); + if (err < 0) { + instISAC->errorCode = -err; + return -1; + } + + switch (bandwidthKHz) { + case isac12kHz: { + numDecodedBytesUB = WebRtcIsac_DecodeUb12( + &instISAC->transform_tables, outFrame, decInstUB, isRCUPayload); + + /* Hang-over for transient alleviation - + * wait two frames to add the upper band going up from 8 kHz. */ + if (instISAC->resetFlag_8kHz > 0) { + if (instISAC->resetFlag_8kHz == 2) { + /* Silence first and a half frame. */ + memset(outFrame, 0, MAX_FRAMESAMPLES * + sizeof(float)); + } else { + const float rampStep = 2.0f / MAX_FRAMESAMPLES; + float rampVal = 0; + memset(outFrame, 0, (MAX_FRAMESAMPLES >> 1) * + sizeof(float)); + + /* Ramp up second half of second frame. */ + for (k = MAX_FRAMESAMPLES / 2; k < MAX_FRAMESAMPLES; k++) { + outFrame[k] *= rampVal; + rampVal += rampStep; + } + } + instISAC->resetFlag_8kHz -= 1; + } + + break; + } + case isac16kHz: { + numDecodedBytesUB = WebRtcIsac_DecodeUb16( + &instISAC->transform_tables, outFrame, decInstUB, isRCUPayload); + break; + } + default: + return -1; + } + + if (numDecodedBytesUB < 0) { + instISAC->errorCode = numDecodedBytesUB; + return -1; + } + if (numDecodedBytesLB + numDecodedBytesUB > lenEncodedBytes) { + // We have supposedly decoded more bytes than we were given. Likely + // caused by bad input data. + instISAC->errorCode = ISAC_LENGTH_MISMATCH; + return -1; + } + + /* It might be less due to garbage. */ + if ((numDecodedBytesUB != lenNextStream) && + (numDecodedBytesUB != (lenNextStream - + encoded[numDecodedBytesLB + 1 + numDecodedBytesUB]))) { + instISAC->errorCode = ISAC_LENGTH_MISMATCH; + return -1; + } + + /* If there is no error Upper-band always decodes + * 30 ms (480 samples). */ + numSamplesUB = FRAMESAMPLES; + + /* Convert to W16. */ + for (k = 0; k < numSamplesUB; k++) { + if (outFrame[k] > 32767) { + outFrameUB[k] = 32767; + } else if (outFrame[k] < -32768) { + outFrameUB[k] = -32768; + } else { + outFrameUB[k] = (int16_t)WebRtcIsac_lrint( + outFrame[k]); + } + } + } + } + + speechIdx = 0; + while (speechIdx < numSamplesLB) { + WebRtcSpl_SynthesisQMF(&outFrameLB[speechIdx], &outFrameUB[speechIdx], + FRAMESAMPLES_10ms, &decoded[(speechIdx << 1)], + instISAC->synthesisFBState1, + instISAC->synthesisFBState2); + + speechIdx += FRAMESAMPLES_10ms; + } + } + *speechType = 0; + return (numSamplesLB + numSamplesUB); +} + + + + + + + +/**************************************************************************** + * WebRtcIsac_Decode(...) + * + * This function decodes a ISAC frame. Output speech length + * will be a multiple of 480 samples: 480 or 960 samples, + * depending on the frameSize (30 or 60 ms). + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC frame(s) + * - len : bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded vector + * -1 - Error + */ + +int WebRtcIsac_Decode(ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t lenEncodedBytes, + int16_t* decoded, + int16_t* speechType) { + int16_t isRCUPayload = 0; + return Decode(ISAC_main_inst, encoded, lenEncodedBytes, decoded, + speechType, isRCUPayload); +} + +/**************************************************************************** + * WebRtcIsac_DecodeRcu(...) + * + * This function decodes a redundant (RCU) iSAC frame. Function is called in + * NetEq with a stored RCU payload in case of packet loss. Output speech length + * will be a multiple of 480 samples: 480 or 960 samples, + * depending on the framesize (30 or 60 ms). + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - encoded : encoded ISAC RCU frame(s) + * - len : bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded vector + * -1 - Error + */ + + + +int WebRtcIsac_DecodeRcu(ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + size_t lenEncodedBytes, + int16_t* decoded, + int16_t* speechType) { + int16_t isRCUPayload = 1; + return Decode(ISAC_main_inst, encoded, lenEncodedBytes, decoded, + speechType, isRCUPayload); +} + + +/**************************************************************************** + * WebRtcIsac_DecodePlc(...) + * + * This function conducts PLC for ISAC frame(s). Output speech length + * will be a multiple of 480 samples: 480 or 960 samples, + * depending on the frameSize (30 or 60 ms). + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - noOfLostFrames : Number of PLC frames to produce + * + * Output: + * - decoded : The decoded vector + * + * Return value : Number of samples in decoded PLC vector + */ +size_t WebRtcIsac_DecodePlc(ISACStruct* ISAC_main_inst, + int16_t* decoded, + size_t noOfLostFrames) { + size_t numSamples = 0; + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + + /* Limit number of frames to two = 60 millisecond. + * Otherwise we exceed data vectors. */ + if (noOfLostFrames > 2) { + noOfLostFrames = 2; + } + + /* Get the number of samples per frame */ + switch (instISAC->decoderSamplingRateKHz) { + case kIsacWideband: { + numSamples = 480 * noOfLostFrames; + break; + } + case kIsacSuperWideband: { + numSamples = 960 * noOfLostFrames; + break; + } + } + + /* Set output samples to zero. */ + memset(decoded, 0, numSamples * sizeof(int16_t)); + return numSamples; +} + + +/**************************************************************************** + * ControlLb(...) - Internal function for controlling Lower Band + * ControlUb(...) - Internal function for controlling Upper Band + * WebRtcIsac_Control(...) - API function + * + * This function sets the limit on the short-term average bit rate and the + * frame length. Should be used only in Instantaneous mode. + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - rate : limit on the short-term average bit rate, + * in bits/second (between 10000 and 32000) + * - frameSize : number of milliseconds per frame (30 or 60) + * + * Return value : 0 - ok + * -1 - Error + */ +static int16_t ControlLb(ISACLBStruct* instISAC, double rate, + int16_t frameSize) { + if ((rate >= 10000) && (rate <= 32000)) { + instISAC->ISACencLB_obj.bottleneck = rate; + } else { + return -ISAC_DISALLOWED_BOTTLENECK; + } + + if ((frameSize == 30) || (frameSize == 60)) { + instISAC->ISACencLB_obj.new_framelength = (FS / 1000) * frameSize; + } else { + return -ISAC_DISALLOWED_FRAME_LENGTH; + } + + return 0; +} + +static int16_t ControlUb(ISACUBStruct* instISAC, double rate) { + if ((rate >= 10000) && (rate <= 32000)) { + instISAC->ISACencUB_obj.bottleneck = rate; + } else { + return -ISAC_DISALLOWED_BOTTLENECK; + } + return 0; +} + +int16_t WebRtcIsac_Control(ISACStruct* ISAC_main_inst, + int32_t bottleneckBPS, + int frameSize) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + int16_t status; + double rateLB; + double rateUB; + enum ISACBandwidth bandwidthKHz; + + if (instISAC->codingMode == 0) { + /* In adaptive mode. */ + instISAC->errorCode = ISAC_MODE_MISMATCH; + return -1; + } + + /* Check if encoder initiated */ + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + return -1; + } + + if (instISAC->encoderSamplingRateKHz == kIsacWideband) { + /* If the sampling rate is 16kHz then bandwith should be 8kHz, + * regardless of bottleneck. */ + bandwidthKHz = isac8kHz; + rateLB = (bottleneckBPS > 32000) ? 32000 : bottleneckBPS; + rateUB = 0; + } else { + if (WebRtcIsac_RateAllocation(bottleneckBPS, &rateLB, &rateUB, + &bandwidthKHz) < 0) { + return -1; + } + } + + if ((instISAC->encoderSamplingRateKHz == kIsacSuperWideband) && + (frameSize != 30) && + (bandwidthKHz != isac8kHz)) { + /* Cannot have 60 ms in super-wideband. */ + instISAC->errorCode = ISAC_DISALLOWED_FRAME_LENGTH; + return -1; + } + + status = ControlLb(&instISAC->instLB, rateLB, (int16_t)frameSize); + if (status < 0) { + instISAC->errorCode = -status; + return -1; + } + if (bandwidthKHz != isac8kHz) { + status = ControlUb(&(instISAC->instUB), rateUB); + if (status < 0) { + instISAC->errorCode = -status; + return -1; + } + } + + + /* Check if bandwidth is changing from wideband to super-wideband + * then we have to synch data buffer of lower & upper-band. Also + * clean up the upper-band data buffer. */ + + if ((instISAC->bandwidthKHz == isac8kHz) && (bandwidthKHz != isac8kHz)) { + memset(instISAC->instUB.ISACencUB_obj.data_buffer_float, 0, + sizeof(float) * (MAX_FRAMESAMPLES + LB_TOTAL_DELAY_SAMPLES)); + + if (bandwidthKHz == isac12kHz) { + instISAC->instUB.ISACencUB_obj.buffer_index = + instISAC->instLB.ISACencLB_obj.buffer_index; + } else { + instISAC->instUB.ISACencUB_obj.buffer_index = + LB_TOTAL_DELAY_SAMPLES + instISAC->instLB.ISACencLB_obj.buffer_index; + + memcpy(&(instISAC->instUB.ISACencUB_obj.lastLPCVec), + WebRtcIsac_kMeanLarUb16, sizeof(double) * UB_LPC_ORDER); + } + } + + /* Update the payload limit if the bandwidth is changing. */ + if (instISAC->bandwidthKHz != bandwidthKHz) { + instISAC->bandwidthKHz = bandwidthKHz; + UpdatePayloadSizeLimit(instISAC); + } + instISAC->bottleneck = bottleneckBPS; + return 0; +} + +void WebRtcIsac_SetInitialBweBottleneck(ISACStruct* ISAC_main_inst, + int bottleneck_bits_per_second) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + RTC_DCHECK_GE(bottleneck_bits_per_second, 10000); + RTC_DCHECK_LE(bottleneck_bits_per_second, 32000); + instISAC->bwestimator_obj.send_bw_avg = (float)bottleneck_bits_per_second; +} + +/**************************************************************************** + * WebRtcIsac_ControlBwe(...) + * + * This function sets the initial values of bottleneck and frame-size if + * iSAC is used in channel-adaptive mode. Through this API, users can + * enforce a frame-size for all values of bottleneck. Then iSAC will not + * automatically change the frame-size. + * + * + * Input: + * - ISAC_main_inst : ISAC instance. + * - rateBPS : initial value of bottleneck in bits/second + * 10000 <= rateBPS <= 32000 is accepted + * For default bottleneck set rateBPS = 0 + * - frameSizeMs : number of milliseconds per frame (30 or 60) + * - enforceFrameSize : 1 to enforce the given frame-size through out + * the adaptation process, 0 to let iSAC change + * the frame-size if required. + * + * Return value : 0 - ok + * -1 - Error + */ +int16_t WebRtcIsac_ControlBwe(ISACStruct* ISAC_main_inst, + int32_t bottleneckBPS, + int frameSizeMs, + int16_t enforceFrameSize) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + enum ISACBandwidth bandwidth; + + /* Check if encoder initiated */ + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + return -1; + } + + /* Check that we are in channel-adaptive mode, otherwise, return (-1) */ + if (instISAC->codingMode != 0) { + instISAC->errorCode = ISAC_MODE_MISMATCH; + return -1; + } + if ((frameSizeMs != 30) && + (instISAC->encoderSamplingRateKHz == kIsacSuperWideband)) { + return -1; + } + + /* Set structure variable if enforceFrameSize is set. ISAC will then + * keep the chosen frame size. */ + if (enforceFrameSize != 0) { + instISAC->instLB.ISACencLB_obj.enforceFrameSize = 1; + } else { + instISAC->instLB.ISACencLB_obj.enforceFrameSize = 0; + } + + /* Set the initial rate. If the input value is zero then the default intial + * rate is used. Otehrwise, values between 10 to 32 kbps are accepted. */ + if (bottleneckBPS != 0) { + double rateLB; + double rateUB; + if (WebRtcIsac_RateAllocation(bottleneckBPS, &rateLB, &rateUB, + &bandwidth) < 0) { + return -1; + } + instISAC->bwestimator_obj.send_bw_avg = (float)bottleneckBPS; + instISAC->bandwidthKHz = bandwidth; + } + + /* Set the initial frame-size. If 'enforceFrameSize' is set, the frame-size + * will not change */ + if (frameSizeMs != 0) { + if ((frameSizeMs == 30) || (frameSizeMs == 60)) { + instISAC->instLB.ISACencLB_obj.new_framelength = + (int16_t)((FS / 1000) * frameSizeMs); + } else { + instISAC->errorCode = ISAC_DISALLOWED_FRAME_LENGTH; + return -1; + } + } + return 0; +} + + +/**************************************************************************** + * WebRtcIsac_GetDownLinkBwIndex(...) + * + * This function returns index representing the Bandwidth estimate from + * the other side to this side. + * + * Input: + * - ISAC_main_inst : iSAC structure + * + * Output: + * - bweIndex : Bandwidth estimate to transmit to other side. + * + */ +int16_t WebRtcIsac_GetDownLinkBwIndex(ISACStruct* ISAC_main_inst, + int16_t* bweIndex, + int16_t* jitterInfo) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + + /* Check if encoder initialized. */ + if ((instISAC->initFlag & BIT_MASK_DEC_INIT) != + BIT_MASK_DEC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + return -1; + } + + /* Call function to get Bandwidth Estimate. */ + WebRtcIsac_GetDownlinkBwJitIndexImpl(&(instISAC->bwestimator_obj), bweIndex, + jitterInfo, + instISAC->decoderSamplingRateKHz); + return 0; +} + + +/**************************************************************************** + * WebRtcIsac_UpdateUplinkBw(...) + * + * This function takes an index representing the Bandwidth estimate from + * this side to other side and updates BWE. + * + * Input: + * - ISAC_main_inst : iSAC structure + * - rateIndex : Bandwidth estimate from other side. + * + * Return value : 0 - ok + * -1 - index out of range + */ +int16_t WebRtcIsac_UpdateUplinkBw(ISACStruct* ISAC_main_inst, + int16_t bweIndex) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + int16_t returnVal; + + /* Check if encoder initiated. */ + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + return -1; + } + + /* Call function to get Bandwidth Estimate. */ + returnVal = WebRtcIsac_UpdateUplinkBwImpl( + &(instISAC->bwestimator_obj), bweIndex, + instISAC->encoderSamplingRateKHz); + + if (returnVal < 0) { + instISAC->errorCode = -returnVal; + return -1; + } else { + return 0; + } +} + + +/**************************************************************************** + * WebRtcIsac_ReadBwIndex(...) + * + * This function returns the index of the Bandwidth estimate from the + * bit-stream. + * + * Input: + * - encoded : Encoded bit-stream + * + * Output: + * - frameLength : Length of frame in packet (in samples) + * - bweIndex : Bandwidth estimate in bit-stream + * + */ +int16_t WebRtcIsac_ReadBwIndex(const uint8_t* encoded, + int16_t* bweIndex) { + Bitstr streamdata; +#ifndef WEBRTC_ARCH_BIG_ENDIAN + int k; +#endif + int16_t err; + + WebRtcIsac_ResetBitstream(&(streamdata)); + +#ifndef WEBRTC_ARCH_BIG_ENDIAN + for (k = 0; k < 10; k++) { + int16_t ek2 = ((const int16_t*)encoded)[k >> 1]; + streamdata.stream[k] = (uint8_t)((ek2 >> ((k & 1) << 3)) & 0xff); + } +#else + memcpy(streamdata.stream, encoded, 10); +#endif + + /* Decode frame length. */ + err = WebRtcIsac_DecodeFrameLen(&streamdata, bweIndex); + if (err < 0) { + return err; + } + + /* Decode BW estimation. */ + err = WebRtcIsac_DecodeSendBW(&streamdata, bweIndex); + if (err < 0) { + return err; + } + + return 0; +} + + +/**************************************************************************** + * WebRtcIsac_ReadFrameLen(...) + * + * This function returns the number of samples the decoder will generate if + * the given payload is decoded. + * + * Input: + * - encoded : Encoded bitstream + * + * Output: + * - frameLength : Length of frame in packet (in samples) + * + */ +int16_t WebRtcIsac_ReadFrameLen(ISACStruct* ISAC_main_inst, + const uint8_t* encoded, + int16_t* frameLength) { + Bitstr streamdata; +#ifndef WEBRTC_ARCH_BIG_ENDIAN + int k; +#endif + int16_t err; + ISACMainStruct* instISAC; + + WebRtcIsac_ResetBitstream(&(streamdata)); + +#ifndef WEBRTC_ARCH_BIG_ENDIAN + for (k = 0; k < 10; k++) { + int16_t ek2 = ((const int16_t*)encoded)[k >> 1]; + streamdata.stream[k] = (uint8_t)((ek2 >> ((k & 1) << 3)) & 0xff); + } +#else + memcpy(streamdata.stream, encoded, 10); +#endif + + /* Decode frame length. */ + err = WebRtcIsac_DecodeFrameLen(&streamdata, frameLength); + if (err < 0) { + return -1; + } + instISAC = (ISACMainStruct*)ISAC_main_inst; + + if (instISAC->decoderSamplingRateKHz == kIsacSuperWideband) { + /* The decoded frame length indicates the number of samples in + * lower-band in this case, multiply by 2 to get the total number + * of samples. */ + *frameLength <<= 1; + } + return 0; +} + + +/******************************************************************************* + * WebRtcIsac_GetNewFrameLen(...) + * + * This function returns the frame length (in samples) of the next packet. + * In the case of channel-adaptive mode, iSAC decides on its frame length based + * on the estimated bottleneck, this AOI allows a user to prepare for the next + * packet (at the encoder). + * + * The primary usage is in CE to make the iSAC works in channel-adaptive mode + * + * Input: + * - ISAC_main_inst : iSAC struct + * + * Return Value : frame lenght in samples + * + */ +int16_t WebRtcIsac_GetNewFrameLen(ISACStruct* ISAC_main_inst) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + + /* Return new frame length. */ + if (instISAC->in_sample_rate_hz == 16000) + return (instISAC->instLB.ISACencLB_obj.new_framelength); + else /* 32000 Hz */ + return ((instISAC->instLB.ISACencLB_obj.new_framelength) * 2); +} + + +/**************************************************************************** + * WebRtcIsac_GetErrorCode(...) + * + * This function can be used to check the error code of an iSAC instance. + * When a function returns -1 an error code will be set for that instance. + * The function below extracts the code of the last error that occurred in + * the specified instance. + * + * Input: + * - ISAC_main_inst : ISAC instance + * + * Return value : Error code + */ +int16_t WebRtcIsac_GetErrorCode(ISACStruct* ISAC_main_inst) { + return ((ISACMainStruct*)ISAC_main_inst)->errorCode; +} + + +/**************************************************************************** + * WebRtcIsac_GetUplinkBw(...) + * + * This function outputs the target bottleneck of the codec. In + * channel-adaptive mode, the target bottleneck is specified through an in-band + * signalling retrieved by bandwidth estimator. + * In channel-independent, also called instantaneous mode, the target + * bottleneck is provided to the encoder by calling xxx_control(...) (if + * xxx_control is never called, the default values are used.). + * Note that the output is the iSAC internal operating bottleneck which might + * differ slightly from the one provided through xxx_control(). + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Output: + * - *bottleneck : bottleneck in bits/sec + * + * Return value : -1 if error happens + * 0 bit-rates computed correctly. + */ +int16_t WebRtcIsac_GetUplinkBw(ISACStruct* ISAC_main_inst, + int32_t* bottleneck) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + + if (instISAC->codingMode == 0) { + /* We are in adaptive mode then get the bottleneck from BWE. */ + *bottleneck = (int32_t)instISAC->bwestimator_obj.send_bw_avg; + } else { + *bottleneck = instISAC->bottleneck; + } + + if ((*bottleneck > 32000) && (*bottleneck < 38000)) { + *bottleneck = 32000; + } else if ((*bottleneck > 45000) && (*bottleneck < 50000)) { + *bottleneck = 45000; + } else if (*bottleneck > 56000) { + *bottleneck = 56000; + } + return 0; +} + + +/****************************************************************************** + * WebRtcIsac_SetMaxPayloadSize(...) + * + * This function sets a limit for the maximum payload size of iSAC. The same + * value is used both for 30 and 60 ms packets. If the encoder sampling rate + * is 16 kHz the maximum payload size is between 120 and 400 bytes. If the + * encoder sampling rate is 32 kHz the maximum payload size is between 120 + * and 600 bytes. + * + * --------------- + * IMPORTANT NOTES + * --------------- + * The size of a packet is limited to the minimum of 'max-payload-size' and + * 'max-rate.' For instance, let's assume the max-payload-size is set to + * 170 bytes, and max-rate is set to 40 kbps. Note that a limit of 40 kbps + * translates to 150 bytes for 30ms frame-size & 300 bytes for 60ms + * frame-size. Then a packet with a frame-size of 30 ms is limited to 150, + * i.e. min(170, 150), and a packet with 60 ms frame-size is limited to + * 170 bytes, i.e. min(170, 300). + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxPayloadBytes : maximum size of the payload in bytes + * valid values are between 100 and 400 bytes + * if encoder sampling rate is 16 kHz. For + * 32 kHz encoder sampling rate valid values + * are between 100 and 600 bytes. + * + * Return value : 0 if successful + * -1 if error happens + */ +int16_t WebRtcIsac_SetMaxPayloadSize(ISACStruct* ISAC_main_inst, + int16_t maxPayloadBytes) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + int16_t status = 0; + + /* Check if encoder initiated */ + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + return -1; + } + + if (instISAC->encoderSamplingRateKHz == kIsacSuperWideband) { + /* Sanity check. */ + if (maxPayloadBytes < 120) { + /* 'maxRate' is out of valid range + * set to the acceptable value and return -1. */ + maxPayloadBytes = 120; + status = -1; + } + + /* sanity check */ + if (maxPayloadBytes > STREAM_SIZE_MAX) { + /* maxRate is out of valid range, + * set to the acceptable value and return -1. */ + maxPayloadBytes = STREAM_SIZE_MAX; + status = -1; + } + } else { + if (maxPayloadBytes < 120) { + /* Max payload-size is out of valid range + * set to the acceptable value and return -1. */ + maxPayloadBytes = 120; + status = -1; + } + if (maxPayloadBytes > STREAM_SIZE_MAX_60) { + /* Max payload-size is out of valid range + * set to the acceptable value and return -1. */ + maxPayloadBytes = STREAM_SIZE_MAX_60; + status = -1; + } + } + instISAC->maxPayloadSizeBytes = maxPayloadBytes; + UpdatePayloadSizeLimit(instISAC); + return status; +} + + +/****************************************************************************** + * WebRtcIsac_SetMaxRate(...) + * + * This function sets the maximum rate which the codec may not exceed for + * any signal packet. The maximum rate is defined and payload-size per + * frame-size in bits per second. + * + * The codec has a maximum rate of 53400 bits per second (200 bytes per 30 + * ms) if the encoder sampling rate is 16kHz, and 160 kbps (600 bytes/30 ms) + * if the encoder sampling rate is 32 kHz. + * + * It is possible to set a maximum rate between 32000 and 53400 bits/sec + * in wideband mode, and 32000 to 160000 bits/sec in super-wideband mode. + * + * --------------- + * IMPORTANT NOTES + * --------------- + * The size of a packet is limited to the minimum of 'max-payload-size' and + * 'max-rate.' For instance, let's assume the max-payload-size is set to + * 170 bytes, and max-rate is set to 40 kbps. Note that a limit of 40 kbps + * translates to 150 bytes for 30ms frame-size & 300 bytes for 60ms + * frame-size. Then a packet with a frame-size of 30 ms is limited to 150, + * i.e. min(170, 150), and a packet with 60 ms frame-size is limited to + * 170 bytes, min(170, 300). + * + * Input: + * - ISAC_main_inst : iSAC instance + * - maxRate : maximum rate in bits per second, + * valid values are 32000 to 53400 bits/sec in + * wideband mode, and 32000 to 160000 bits/sec in + * super-wideband mode. + * + * Return value : 0 if successful + * -1 if error happens + */ +int16_t WebRtcIsac_SetMaxRate(ISACStruct* ISAC_main_inst, + int32_t maxRate) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + int16_t maxRateInBytesPer30Ms; + int16_t status = 0; + + /* check if encoder initiated */ + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != BIT_MASK_ENC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + return -1; + } + /* Calculate maximum number of bytes per 30 msec packets for the + given maximum rate. Multiply with 30/1000 to get number of + bits per 30 ms, divide by 8 to get number of bytes per 30 ms: + maxRateInBytes = floor((maxRate * 30/1000) / 8); */ + maxRateInBytesPer30Ms = (int16_t)(maxRate * 3 / 800); + + if (instISAC->encoderSamplingRateKHz == kIsacWideband) { + if (maxRate < 32000) { + /* 'maxRate' is out of valid range. + * Set to the acceptable value and return -1. */ + maxRateInBytesPer30Ms = 120; + status = -1; + } + + if (maxRate > 53400) { + /* 'maxRate' is out of valid range. + * Set to the acceptable value and return -1. */ + maxRateInBytesPer30Ms = 200; + status = -1; + } + } else { + if (maxRateInBytesPer30Ms < 120) { + /* 'maxRate' is out of valid range + * Set to the acceptable value and return -1. */ + maxRateInBytesPer30Ms = 120; + status = -1; + } + + if (maxRateInBytesPer30Ms > STREAM_SIZE_MAX) { + /* 'maxRate' is out of valid range. + * Set to the acceptable value and return -1. */ + maxRateInBytesPer30Ms = STREAM_SIZE_MAX; + status = -1; + } + } + instISAC->maxRateBytesPer30Ms = maxRateInBytesPer30Ms; + UpdatePayloadSizeLimit(instISAC); + return status; +} + + +/**************************************************************************** + * WebRtcIsac_GetRedPayload(...) + * + * This function populates "encoded" with the redundant payload of the recently + * encodedframe. This function has to be called once that WebRtcIsac_Encode(...) + * returns a positive value. Regardless of the frame-size this function will + * be called only once after encoding is completed. The bit-stream is + * targeted for 16000 bit/sec. + * + * Input: + * - ISAC_main_inst : iSAC struct + * + * Output: + * - encoded : the encoded data vector + * + * + * Return value : >0 - Length (in bytes) of coded data + * : -1 - Error + */ +int16_t WebRtcIsac_GetRedPayload(ISACStruct* ISAC_main_inst, + uint8_t* encoded) { + Bitstr iSACBitStreamInst; + int16_t streamLenLB; + int16_t streamLenUB; + int16_t streamLen; + int16_t totalLenUB; + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; +#ifndef WEBRTC_ARCH_BIG_ENDIAN + int k; +#endif + + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + instISAC->errorCode = ISAC_ENCODER_NOT_INITIATED; + } + + WebRtcIsac_ResetBitstream(&(iSACBitStreamInst)); + + streamLenLB = WebRtcIsac_EncodeStoredDataLb( + &instISAC->instLB.ISACencLB_obj.SaveEnc_obj, + &iSACBitStreamInst, + instISAC->instLB.ISACencLB_obj.lastBWIdx, + RCU_TRANSCODING_SCALE); + if (streamLenLB < 0) { + return -1; + } + + /* convert from bytes to int16_t. */ + memcpy(encoded, iSACBitStreamInst.stream, streamLenLB); + streamLen = streamLenLB; + if (instISAC->bandwidthKHz == isac8kHz) { + return streamLenLB; + } + + streamLenUB = WebRtcIsac_GetRedPayloadUb( + &instISAC->instUB.ISACencUB_obj.SaveEnc_obj, + &iSACBitStreamInst, instISAC->bandwidthKHz); + if (streamLenUB < 0) { + /* An error has happened but this is not the error due to a + * bit-stream larger than the limit. */ + return -1; + } + + /* We have one byte to write the total length of the upper-band. + * The length includes the bit-stream length, check-sum and the + * single byte where the length is written to. This is according to + * iSAC wideband and how the "garbage" is dealt. */ + totalLenUB = streamLenUB + 1 + LEN_CHECK_SUM_WORD8; + if (totalLenUB > 255) { + streamLenUB = 0; + } + + /* Generate CRC if required. */ + if ((instISAC->bandwidthKHz != isac8kHz) && + (streamLenUB > 0)) { + uint32_t crc; + streamLen += totalLenUB; + encoded[streamLenLB] = (uint8_t)totalLenUB; + memcpy(&encoded[streamLenLB + 1], iSACBitStreamInst.stream, + streamLenUB); + + WebRtcIsac_GetCrc((int16_t*)(&(encoded[streamLenLB + 1])), + streamLenUB, &crc); +#ifndef WEBRTC_ARCH_BIG_ENDIAN + for (k = 0; k < LEN_CHECK_SUM_WORD8; k++) { + encoded[streamLen - LEN_CHECK_SUM_WORD8 + k] = + (uint8_t)((crc >> (24 - k * 8)) & 0xFF); + } +#else + memcpy(&encoded[streamLenLB + streamLenUB + 1], &crc, + LEN_CHECK_SUM_WORD8); +#endif + } + return streamLen; +} + + +/**************************************************************************** + * WebRtcIsac_version(...) + * + * This function returns the version number. + * + * Output: + * - version : Pointer to character string + * + */ +void WebRtcIsac_version(char* version) { + strcpy(version, "4.3.0"); +} + + +/****************************************************************************** + * WebRtcIsac_SetEncSampRate() + * This function sets the sampling rate of the encoder. Initialization of the + * encoder WILL NOT overwrite the sampling rate of the encoder. The default + * value is 16 kHz which is set when the instance is created. The encoding-mode + * and the bottleneck remain unchanged by this call, however, the maximum rate + * and maximum payload-size will be reset to their default values. + * + * Input: + * - ISAC_main_inst : iSAC instance + * - sample_rate_hz : sampling rate in Hertz, valid values are 16000 + * and 32000. + * + * Return value : 0 if successful + * -1 if failed. + */ +int16_t WebRtcIsac_SetEncSampRate(ISACStruct* ISAC_main_inst, + uint16_t sample_rate_hz) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + enum IsacSamplingRate encoder_operational_rate; + + if ((sample_rate_hz != 16000) && (sample_rate_hz != 32000)) { + /* Sampling Frequency is not supported. */ + instISAC->errorCode = ISAC_UNSUPPORTED_SAMPLING_FREQUENCY; + return -1; + } + if (sample_rate_hz == 16000) { + encoder_operational_rate = kIsacWideband; + } else { + encoder_operational_rate = kIsacSuperWideband; + } + + if ((instISAC->initFlag & BIT_MASK_ENC_INIT) != + BIT_MASK_ENC_INIT) { + if (encoder_operational_rate == kIsacWideband) { + instISAC->bandwidthKHz = isac8kHz; + } else { + instISAC->bandwidthKHz = isac16kHz; + } + } else { + ISACUBStruct* instUB = &(instISAC->instUB); + ISACLBStruct* instLB = &(instISAC->instLB); + int32_t bottleneck = instISAC->bottleneck; + int16_t codingMode = instISAC->codingMode; + int16_t frameSizeMs = instLB->ISACencLB_obj.new_framelength / + (FS / 1000); + + if ((encoder_operational_rate == kIsacWideband) && + (instISAC->encoderSamplingRateKHz == kIsacSuperWideband)) { + /* Changing from super-wideband to wideband. + * we don't need to re-initialize the encoder of the lower-band. */ + instISAC->bandwidthKHz = isac8kHz; + if (codingMode == 1) { + ControlLb(instLB, + (bottleneck > 32000) ? 32000 : bottleneck, FRAMESIZE); + } + instISAC->maxPayloadSizeBytes = STREAM_SIZE_MAX_60; + instISAC->maxRateBytesPer30Ms = STREAM_SIZE_MAX_30; + } else if ((encoder_operational_rate == kIsacSuperWideband) && + (instISAC->encoderSamplingRateKHz == kIsacWideband)) { + double bottleneckLB = 0; + double bottleneckUB = 0; + if (codingMode == 1) { + WebRtcIsac_RateAllocation(bottleneck, &bottleneckLB, &bottleneckUB, + &(instISAC->bandwidthKHz)); + } + + instISAC->bandwidthKHz = isac16kHz; + instISAC->maxPayloadSizeBytes = STREAM_SIZE_MAX; + instISAC->maxRateBytesPer30Ms = STREAM_SIZE_MAX; + + EncoderInitLb(instLB, codingMode, encoder_operational_rate); + EncoderInitUb(instUB, instISAC->bandwidthKHz); + + memset(instISAC->analysisFBState1, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + memset(instISAC->analysisFBState2, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + + if (codingMode == 1) { + instISAC->bottleneck = bottleneck; + ControlLb(instLB, bottleneckLB, + (instISAC->bandwidthKHz == isac8kHz) ? frameSizeMs:FRAMESIZE); + if (instISAC->bandwidthKHz > isac8kHz) { + ControlUb(instUB, bottleneckUB); + } + } else { + instLB->ISACencLB_obj.enforceFrameSize = 0; + instLB->ISACencLB_obj.new_framelength = FRAMESAMPLES; + } + } + } + instISAC->encoderSamplingRateKHz = encoder_operational_rate; + instISAC->in_sample_rate_hz = sample_rate_hz; + return 0; +} + + +/****************************************************************************** + * WebRtcIsac_SetDecSampRate() + * This function sets the sampling rate of the decoder. Initialization of the + * decoder WILL NOT overwrite the sampling rate of the encoder. The default + * value is 16 kHz which is set when the instance is created. + * + * Input: + * - ISAC_main_inst : iSAC instance + * - sample_rate_hz : sampling rate in Hertz, valid values are 16000 + * and 32000. + * + * Return value : 0 if successful + * -1 if failed. + */ +int16_t WebRtcIsac_SetDecSampRate(ISACStruct* ISAC_main_inst, + uint16_t sample_rate_hz) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + enum IsacSamplingRate decoder_operational_rate; + + if (sample_rate_hz == 16000) { + decoder_operational_rate = kIsacWideband; + } else if (sample_rate_hz == 32000) { + decoder_operational_rate = kIsacSuperWideband; + } else { + /* Sampling Frequency is not supported. */ + instISAC->errorCode = ISAC_UNSUPPORTED_SAMPLING_FREQUENCY; + return -1; + } + + if ((instISAC->decoderSamplingRateKHz == kIsacWideband) && + (decoder_operational_rate == kIsacSuperWideband)) { + /* Switching from wideband to super-wideband at the decoder + * we need to reset the filter-bank and initialize upper-band decoder. */ + memset(instISAC->synthesisFBState1, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + memset(instISAC->synthesisFBState2, 0, + FB_STATE_SIZE_WORD32 * sizeof(int32_t)); + + DecoderInitUb(&instISAC->instUB); + } + instISAC->decoderSamplingRateKHz = decoder_operational_rate; + return 0; +} + + +/****************************************************************************** + * WebRtcIsac_EncSampRate() + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Return value : sampling rate in Hertz. The input to encoder + * is expected to be sampled in this rate. + * + */ +uint16_t WebRtcIsac_EncSampRate(ISACStruct* ISAC_main_inst) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + return instISAC->in_sample_rate_hz; +} + + +/****************************************************************************** + * WebRtcIsac_DecSampRate() + * Return the sampling rate of the decoded audio. + * + * Input: + * - ISAC_main_inst : iSAC instance + * + * Return value : sampling rate in Hertz. Decoder output is + * sampled at this rate. + * + */ +uint16_t WebRtcIsac_DecSampRate(ISACStruct* ISAC_main_inst) { + ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; + return instISAC->decoderSamplingRateKHz == kIsacWideband ? 16000 : 32000; +} + +void WebRtcIsac_GetBandwidthInfo(ISACStruct* inst, + IsacBandwidthInfo* bwinfo) { + ISACMainStruct* instISAC = (ISACMainStruct*)inst; + RTC_DCHECK_NE(0, instISAC->initFlag & BIT_MASK_DEC_INIT); + WebRtcIsacBw_GetBandwidthInfo(&instISAC->bwestimator_obj, + instISAC->decoderSamplingRateKHz, bwinfo); +} + +void WebRtcIsac_SetBandwidthInfo(ISACStruct* inst, + const IsacBandwidthInfo* bwinfo) { + ISACMainStruct* instISAC = (ISACMainStruct*)inst; + RTC_DCHECK_NE(0, instISAC->initFlag & BIT_MASK_ENC_INIT); + WebRtcIsacBw_SetBandwidthInfo(&instISAC->bwestimator_obj, bwinfo); +} + +void WebRtcIsac_SetEncSampRateInDecoder(ISACStruct* inst, + int sample_rate_hz) { + ISACMainStruct* instISAC = (ISACMainStruct*)inst; + RTC_DCHECK_NE(0, instISAC->initFlag & BIT_MASK_DEC_INIT); + RTC_DCHECK(!(instISAC->initFlag & BIT_MASK_ENC_INIT)); + RTC_DCHECK(sample_rate_hz == 16000 || sample_rate_hz == 32000); + instISAC->encoderSamplingRateKHz = sample_rate_hz / 1000; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_float_type.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_float_type.h new file mode 100644 index 0000000000..59a8805bd5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_float_type.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_FLOAT_TYPE_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_FLOAT_TYPE_H_ + +#include "modules/audio_coding/codecs/isac/main/include/isac.h" + +namespace webrtc { + +struct IsacFloat { + using instance_type = ISACStruct; + static const bool has_swb = true; + static inline int16_t Control(instance_type* inst, + int32_t rate, + int framesize) { + return WebRtcIsac_Control(inst, rate, framesize); + } + static inline int16_t ControlBwe(instance_type* inst, + int32_t rate_bps, + int frame_size_ms, + int16_t enforce_frame_size) { + return WebRtcIsac_ControlBwe(inst, rate_bps, frame_size_ms, + enforce_frame_size); + } + static inline int16_t Create(instance_type** inst) { + return WebRtcIsac_Create(inst); + } + static inline int DecodeInternal(instance_type* inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speech_type) { + return WebRtcIsac_Decode(inst, encoded, len, decoded, speech_type); + } + static inline size_t DecodePlc(instance_type* inst, + int16_t* decoded, + size_t num_lost_frames) { + return WebRtcIsac_DecodePlc(inst, decoded, num_lost_frames); + } + + static inline void DecoderInit(instance_type* inst) { + WebRtcIsac_DecoderInit(inst); + } + static inline int Encode(instance_type* inst, + const int16_t* speech_in, + uint8_t* encoded) { + return WebRtcIsac_Encode(inst, speech_in, encoded); + } + static inline int16_t EncoderInit(instance_type* inst, int16_t coding_mode) { + return WebRtcIsac_EncoderInit(inst, coding_mode); + } + static inline uint16_t EncSampRate(instance_type* inst) { + return WebRtcIsac_EncSampRate(inst); + } + + static inline int16_t Free(instance_type* inst) { + return WebRtcIsac_Free(inst); + } + static inline void GetBandwidthInfo(instance_type* inst, + IsacBandwidthInfo* bwinfo) { + WebRtcIsac_GetBandwidthInfo(inst, bwinfo); + } + static inline int16_t GetErrorCode(instance_type* inst) { + return WebRtcIsac_GetErrorCode(inst); + } + + static inline int16_t GetNewFrameLen(instance_type* inst) { + return WebRtcIsac_GetNewFrameLen(inst); + } + static inline void SetBandwidthInfo(instance_type* inst, + const IsacBandwidthInfo* bwinfo) { + WebRtcIsac_SetBandwidthInfo(inst, bwinfo); + } + static inline int16_t SetDecSampRate(instance_type* inst, + uint16_t sample_rate_hz) { + return WebRtcIsac_SetDecSampRate(inst, sample_rate_hz); + } + static inline int16_t SetEncSampRate(instance_type* inst, + uint16_t sample_rate_hz) { + return WebRtcIsac_SetEncSampRate(inst, sample_rate_hz); + } + static inline void SetEncSampRateInDecoder(instance_type* inst, + uint16_t sample_rate_hz) { + WebRtcIsac_SetEncSampRateInDecoder(inst, sample_rate_hz); + } + static inline void SetInitialBweBottleneck(instance_type* inst, + int bottleneck_bits_per_second) { + WebRtcIsac_SetInitialBweBottleneck(inst, bottleneck_bits_per_second); + } + static inline int16_t UpdateBwEstimate(instance_type* inst, + const uint8_t* encoded, + size_t packet_size, + uint16_t rtp_seq_number, + uint32_t send_ts, + uint32_t arr_ts) { + return WebRtcIsac_UpdateBwEstimate(inst, encoded, packet_size, + rtp_seq_number, send_ts, arr_ts); + } + static inline int16_t SetMaxPayloadSize(instance_type* inst, + int16_t max_payload_size_bytes) { + return WebRtcIsac_SetMaxPayloadSize(inst, max_payload_size_bytes); + } + static inline int16_t SetMaxRate(instance_type* inst, int32_t max_bit_rate) { + return WebRtcIsac_SetMaxRate(inst, max_bit_rate); + } +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_FLOAT_TYPE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_unittest.cc new file mode 100644 index 0000000000..727f0f61e0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac_unittest.cc @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "modules/audio_coding/codecs/isac/main/include/isac.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +struct WebRtcISACStruct; + +namespace webrtc { + +// Number of samples in a 60 ms, sampled at 32 kHz. +const int kIsacNumberOfSamples = 320 * 6; +// Maximum number of bytes in output bitstream. +const size_t kMaxBytes = 1000; + +class IsacTest : public ::testing::Test { + protected: + IsacTest(); + virtual void SetUp(); + + WebRtcISACStruct* isac_codec_; + + int16_t speech_data_[kIsacNumberOfSamples]; + int16_t output_data_[kIsacNumberOfSamples]; + uint8_t bitstream_[kMaxBytes]; + uint8_t bitstream_small_[7]; // Simulate sync packets. +}; + +IsacTest::IsacTest() + : isac_codec_(NULL) { +} + +void IsacTest::SetUp() { + // Read some samples from a speech file, to be used in the encode test. + FILE* input_file; + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + input_file = fopen(file_name.c_str(), "rb"); + ASSERT_TRUE(input_file != NULL); + ASSERT_EQ(kIsacNumberOfSamples, + static_cast(fread(speech_data_, sizeof(int16_t), + kIsacNumberOfSamples, input_file))); + fclose(input_file); + input_file = NULL; +} + +// Test failing Create. +TEST_F(IsacTest, IsacCreateFail) { + // Test to see that an invalid pointer is caught. + EXPECT_EQ(-1, WebRtcIsac_Create(NULL)); +} + +// Test failing Free. +TEST_F(IsacTest, IsacFreeFail) { + // Test to see that free function doesn't crash. + EXPECT_EQ(0, WebRtcIsac_Free(NULL)); +} + +// Test normal Create and Free. +TEST_F(IsacTest, IsacCreateFree) { + EXPECT_EQ(0, WebRtcIsac_Create(&isac_codec_)); + EXPECT_TRUE(isac_codec_ != NULL); + EXPECT_EQ(0, WebRtcIsac_Free(isac_codec_));} + +TEST_F(IsacTest, IsacUpdateBWE) { + // Create encoder memory. + EXPECT_EQ(0, WebRtcIsac_Create(&isac_codec_)); + + // Init encoder (adaptive mode) and decoder. + WebRtcIsac_EncoderInit(isac_codec_, 0); + WebRtcIsac_DecoderInit(isac_codec_); + + int encoded_bytes; + + // Test with call with a small packet (sync packet). + EXPECT_EQ(-1, WebRtcIsac_UpdateBwEstimate(isac_codec_, bitstream_small_, 7, 1, + 12345, 56789)); + + // Encode 60 ms of data (needed to create a first packet). + encoded_bytes = WebRtcIsac_Encode(isac_codec_, speech_data_, bitstream_); + EXPECT_EQ(0, encoded_bytes); + encoded_bytes = WebRtcIsac_Encode(isac_codec_, speech_data_, bitstream_); + EXPECT_EQ(0, encoded_bytes); + encoded_bytes = WebRtcIsac_Encode(isac_codec_, speech_data_, bitstream_); + EXPECT_EQ(0, encoded_bytes); + encoded_bytes = WebRtcIsac_Encode(isac_codec_, speech_data_, bitstream_); + EXPECT_EQ(0, encoded_bytes); + encoded_bytes = WebRtcIsac_Encode(isac_codec_, speech_data_, bitstream_); + EXPECT_EQ(0, encoded_bytes); + encoded_bytes = WebRtcIsac_Encode(isac_codec_, speech_data_, bitstream_); + EXPECT_GT(encoded_bytes, 0); + + // Call to update bandwidth estimator with real data. + EXPECT_EQ(0, WebRtcIsac_UpdateBwEstimate(isac_codec_, bitstream_, + static_cast(encoded_bytes), + 1, 12345, 56789)); + + // Free memory. + EXPECT_EQ(0, WebRtcIsac_Free(isac_codec_)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lattice.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lattice.c new file mode 100644 index 0000000000..eabe708045 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lattice.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lattice.c + * + * contains the normalized lattice filter routines (MA and AR) for iSAC codec + * + */ +#include "settings.h" +#include "codec.h" + +#include +#include +#include +#ifdef WEBRTC_ANDROID +#include +#endif + +/* filter the signal using normalized lattice filter */ +/* MA filter */ +void WebRtcIsac_NormLatticeFilterMa(int orderCoef, + float *stateF, + float *stateG, + float *lat_in, + double *filtcoeflo, + double *lat_out) +{ + int n,k,i,u,temp1; + int ord_1 = orderCoef+1; + float sth[MAX_AR_MODEL_ORDER]; + float cth[MAX_AR_MODEL_ORDER]; + float inv_cth[MAX_AR_MODEL_ORDER]; + double a[MAX_AR_MODEL_ORDER+1]; + float f[MAX_AR_MODEL_ORDER+1][HALF_SUBFRAMELEN], g[MAX_AR_MODEL_ORDER+1][HALF_SUBFRAMELEN]; + float gain1; + + for (u=0;u=0;i--) //get the state of f&g for the first input, for all orders + { + ARf[i][0] = cth[i]*ARf[i+1][0] - sth[i]*stateG[i]; + ARg[i+1][0] = sth[i]*ARf[i+1][0] + cth[i]* stateG[i]; + } + ARg[0][0] = ARf[0][0]; + + for(n=0;n<(HALF_SUBFRAMELEN-1);n++) + { + for(k=orderCoef-1;k>=0;k--) + { + ARf[k][n+1] = cth[k]*ARf[k+1][n+1] - sth[k]*ARg[k][n]; + ARg[k+1][n+1] = sth[k]*ARf[k+1][n+1] + cth[k]* ARg[k][n]; + } + ARg[0][n+1] = ARf[0][n+1]; + } + + memcpy(lat_out+u * HALF_SUBFRAMELEN, &(ARf[0][0]), sizeof(float) * HALF_SUBFRAMELEN); + + /* cannot use memcpy in the following */ + for (i=0;i0; m--) + { + tmp_inv = 1.0f / cth2; + for (k=1; k<=m; k++) + { + tmp[k] = ((float)a[k] - sth[m] * (float)a[m-k+1]) * tmp_inv; + } + + for (k=1; k +#include + +#define LEVINSON_EPS 1.0e-10 + + +/* window */ +/* Matlab generation code: + * t = (1:256)/257; r = 1-(1-t).^.45; w = sin(r*pi).^3; w = w/sum(w); plot((1:256)/8, w); grid; + * for k=1:16, fprintf(1, '%.8f, ', w(k*16 + (-15:0))); fprintf(1, '\n'); end + */ +static const double kLpcCorrWindow[WINLEN] = { + 0.00000000, 0.00000001, 0.00000004, 0.00000010, 0.00000020, + 0.00000035, 0.00000055, 0.00000083, 0.00000118, 0.00000163, + 0.00000218, 0.00000283, 0.00000361, 0.00000453, 0.00000558, 0.00000679, + 0.00000817, 0.00000973, 0.00001147, 0.00001342, 0.00001558, + 0.00001796, 0.00002058, 0.00002344, 0.00002657, 0.00002997, + 0.00003365, 0.00003762, 0.00004190, 0.00004651, 0.00005144, 0.00005673, + 0.00006236, 0.00006837, 0.00007476, 0.00008155, 0.00008875, + 0.00009636, 0.00010441, 0.00011290, 0.00012186, 0.00013128, + 0.00014119, 0.00015160, 0.00016252, 0.00017396, 0.00018594, 0.00019846, + 0.00021155, 0.00022521, 0.00023946, 0.00025432, 0.00026978, + 0.00028587, 0.00030260, 0.00031998, 0.00033802, 0.00035674, + 0.00037615, 0.00039626, 0.00041708, 0.00043863, 0.00046092, 0.00048396, + 0.00050775, 0.00053233, 0.00055768, 0.00058384, 0.00061080, + 0.00063858, 0.00066720, 0.00069665, 0.00072696, 0.00075813, + 0.00079017, 0.00082310, 0.00085692, 0.00089164, 0.00092728, 0.00096384, + 0.00100133, 0.00103976, 0.00107914, 0.00111947, 0.00116077, + 0.00120304, 0.00124630, 0.00129053, 0.00133577, 0.00138200, + 0.00142924, 0.00147749, 0.00152676, 0.00157705, 0.00162836, 0.00168070, + 0.00173408, 0.00178850, 0.00184395, 0.00190045, 0.00195799, + 0.00201658, 0.00207621, 0.00213688, 0.00219860, 0.00226137, + 0.00232518, 0.00239003, 0.00245591, 0.00252284, 0.00259079, 0.00265977, + 0.00272977, 0.00280078, 0.00287280, 0.00294582, 0.00301984, + 0.00309484, 0.00317081, 0.00324774, 0.00332563, 0.00340446, + 0.00348421, 0.00356488, 0.00364644, 0.00372889, 0.00381220, 0.00389636, + 0.00398135, 0.00406715, 0.00415374, 0.00424109, 0.00432920, + 0.00441802, 0.00450754, 0.00459773, 0.00468857, 0.00478001, + 0.00487205, 0.00496464, 0.00505775, 0.00515136, 0.00524542, 0.00533990, + 0.00543476, 0.00552997, 0.00562548, 0.00572125, 0.00581725, + 0.00591342, 0.00600973, 0.00610612, 0.00620254, 0.00629895, + 0.00639530, 0.00649153, 0.00658758, 0.00668341, 0.00677894, 0.00687413, + 0.00696891, 0.00706322, 0.00715699, 0.00725016, 0.00734266, + 0.00743441, 0.00752535, 0.00761540, 0.00770449, 0.00779254, + 0.00787947, 0.00796519, 0.00804963, 0.00813270, 0.00821431, 0.00829437, + 0.00837280, 0.00844949, 0.00852436, 0.00859730, 0.00866822, + 0.00873701, 0.00880358, 0.00886781, 0.00892960, 0.00898884, + 0.00904542, 0.00909923, 0.00915014, 0.00919805, 0.00924283, 0.00928436, + 0.00932252, 0.00935718, 0.00938821, 0.00941550, 0.00943890, + 0.00945828, 0.00947351, 0.00948446, 0.00949098, 0.00949294, + 0.00949020, 0.00948262, 0.00947005, 0.00945235, 0.00942938, 0.00940099, + 0.00936704, 0.00932738, 0.00928186, 0.00923034, 0.00917268, + 0.00910872, 0.00903832, 0.00896134, 0.00887763, 0.00878706, + 0.00868949, 0.00858478, 0.00847280, 0.00835343, 0.00822653, 0.00809199, + 0.00794970, 0.00779956, 0.00764145, 0.00747530, 0.00730103, + 0.00711857, 0.00692787, 0.00672888, 0.00652158, 0.00630597, + 0.00608208, 0.00584994, 0.00560962, 0.00536124, 0.00510493, 0.00484089, + 0.00456935, 0.00429062, 0.00400505, 0.00371310, 0.00341532, + 0.00311238, 0.00280511, 0.00249452, 0.00218184, 0.00186864, + 0.00155690, 0.00124918, 0.00094895, 0.00066112, 0.00039320, 0.00015881 +}; + +double WebRtcIsac_LevDurb(double *a, double *k, double *r, size_t order) +{ + + double sum, alpha; + size_t m, m_h, i; + alpha = 0; //warning -DH + a[0] = 1.0; + if (r[0] < LEVINSON_EPS) { /* if r[0] <= 0, set LPC coeff. to zero */ + for (i = 0; i < order; i++) { + k[i] = 0; + a[i+1] = 0; + } + } else { + a[1] = k[0] = -r[1]/r[0]; + alpha = r[0] + r[1] * k[0]; + for (m = 1; m < order; m++){ + sum = r[m + 1]; + for (i = 0; i < m; i++){ + sum += a[i+1] * r[m - i]; + } + k[m] = -sum / alpha; + alpha += k[m] * sum; + m_h = (m + 1) >> 1; + for (i = 0; i < m_h; i++){ + sum = a[i+1] + k[m] * a[m - i]; + a[m - i] += k[m] * a[i+1]; + a[i+1] = sum; + } + a[m+1] = k[m]; + } + } + return alpha; +} + + +//was static before, but didn't work with MEX file +void WebRtcIsac_GetVars(const double *input, const int16_t *pitchGains_Q12, + double *oldEnergy, double *varscale) +{ + double nrg[4], chng, pg; + int k; + + double pitchGains[4]={0,0,0,0};; + + /* Calculate energies of first and second frame halfs */ + nrg[0] = 0.0001; + for (k = QLOOKAHEAD/2; k < (FRAMESAMPLES_QUARTER + QLOOKAHEAD) / 2; k++) { + nrg[0] += input[k]*input[k]; + } + nrg[1] = 0.0001; + for ( ; k < (FRAMESAMPLES_HALF + QLOOKAHEAD) / 2; k++) { + nrg[1] += input[k]*input[k]; + } + nrg[2] = 0.0001; + for ( ; k < (FRAMESAMPLES*3/4 + QLOOKAHEAD) / 2; k++) { + nrg[2] += input[k]*input[k]; + } + nrg[3] = 0.0001; + for ( ; k < (FRAMESAMPLES + QLOOKAHEAD) / 2; k++) { + nrg[3] += input[k]*input[k]; + } + + /* Calculate average level change */ + chng = 0.25 * (fabs(10.0 * log10(nrg[3] / nrg[2])) + + fabs(10.0 * log10(nrg[2] / nrg[1])) + + fabs(10.0 * log10(nrg[1] / nrg[0])) + + fabs(10.0 * log10(nrg[0] / *oldEnergy))); + + + /* Find average pitch gain */ + pg = 0.0; + for (k=0; k<4; k++) + { + pitchGains[k] = ((float)pitchGains_Q12[k])/4096; + pg += pitchGains[k]; + } + pg *= 0.25; + + /* If pitch gain is low and energy constant - increase noise level*/ + /* Matlab code: + pg = 0:.01:.45; plot(pg, 0.0 + 1.0 * exp( -1.0 * exp(-200.0 * pg.*pg.*pg) / (1.0 + 0.4 * 0) )) + */ + *varscale = 0.0 + 1.0 * exp( -1.4 * exp(-200.0 * pg*pg*pg) / (1.0 + 0.4 * chng) ); + + *oldEnergy = nrg[3]; +} + +void +WebRtcIsac_GetVarsUB( + const double* input, + double* oldEnergy, + double* varscale) +{ + double nrg[4], chng; + int k; + + /* Calculate energies of first and second frame halfs */ + nrg[0] = 0.0001; + for (k = 0; k < (FRAMESAMPLES_QUARTER) / 2; k++) { + nrg[0] += input[k]*input[k]; + } + nrg[1] = 0.0001; + for ( ; k < (FRAMESAMPLES_HALF) / 2; k++) { + nrg[1] += input[k]*input[k]; + } + nrg[2] = 0.0001; + for ( ; k < (FRAMESAMPLES*3/4) / 2; k++) { + nrg[2] += input[k]*input[k]; + } + nrg[3] = 0.0001; + for ( ; k < (FRAMESAMPLES) / 2; k++) { + nrg[3] += input[k]*input[k]; + } + + /* Calculate average level change */ + chng = 0.25 * (fabs(10.0 * log10(nrg[3] / nrg[2])) + + fabs(10.0 * log10(nrg[2] / nrg[1])) + + fabs(10.0 * log10(nrg[1] / nrg[0])) + + fabs(10.0 * log10(nrg[0] / *oldEnergy))); + + + /* If pitch gain is low and energy constant - increase noise level*/ + /* Matlab code: + pg = 0:.01:.45; plot(pg, 0.0 + 1.0 * exp( -1.0 * exp(-200.0 * pg.*pg.*pg) / (1.0 + 0.4 * 0) )) + */ + *varscale = exp( -1.4 / (1.0 + 0.4 * chng) ); + + *oldEnergy = nrg[3]; +} + +void WebRtcIsac_GetLpcCoefLb(double *inLo, double *inHi, MaskFiltstr *maskdata, + double signal_noise_ratio, const int16_t *pitchGains_Q12, + double *lo_coeff, double *hi_coeff) +{ + int k, n, j, pos1, pos2; + double varscale; + + double DataLo[WINLEN], DataHi[WINLEN]; + double corrlo[ORDERLO+2], corrlo2[ORDERLO+1]; + double corrhi[ORDERHI+1]; + double k_veclo[ORDERLO], k_vechi[ORDERHI]; + + double a_LO[ORDERLO+1], a_HI[ORDERHI+1]; + double tmp, res_nrg; + + double FwdA, FwdB; + + /* hearing threshold level in dB; higher value gives more noise */ + const double HearThresOffset = -28.0; + + /* bandwdith expansion factors for low- and high band */ + const double gammaLo = 0.9; + const double gammaHi = 0.8; + + /* less-noise-at-low-frequencies factor */ + double aa; + + + /* convert from dB to signal level */ + const double H_T_H = pow(10.0, 0.05 * HearThresOffset); + double S_N_R = pow(10.0, 0.05 * signal_noise_ratio) / 3.46; /* divide by sqrt(12) */ + + /* change quallevel depending on pitch gains and level fluctuations */ + WebRtcIsac_GetVars(inLo, pitchGains_Q12, &(maskdata->OldEnergy), &varscale); + + /* less-noise-at-low-frequencies factor */ + aa = 0.35 * (0.5 + 0.5 * varscale); + + /* replace data in buffer by new look-ahead data */ + for (pos1 = 0; pos1 < QLOOKAHEAD; pos1++) + maskdata->DataBufferLo[pos1 + WINLEN - QLOOKAHEAD] = inLo[pos1]; + + for (k = 0; k < SUBFRAMES; k++) { + + /* Update input buffer and multiply signal with window */ + for (pos1 = 0; pos1 < WINLEN - UPDATE/2; pos1++) { + maskdata->DataBufferLo[pos1] = maskdata->DataBufferLo[pos1 + UPDATE/2]; + maskdata->DataBufferHi[pos1] = maskdata->DataBufferHi[pos1 + UPDATE/2]; + DataLo[pos1] = maskdata->DataBufferLo[pos1] * kLpcCorrWindow[pos1]; + DataHi[pos1] = maskdata->DataBufferHi[pos1] * kLpcCorrWindow[pos1]; + } + pos2 = k * UPDATE/2; + for (n = 0; n < UPDATE/2; n++, pos1++) { + maskdata->DataBufferLo[pos1] = inLo[QLOOKAHEAD + pos2]; + maskdata->DataBufferHi[pos1] = inHi[pos2++]; + DataLo[pos1] = maskdata->DataBufferLo[pos1] * kLpcCorrWindow[pos1]; + DataHi[pos1] = maskdata->DataBufferHi[pos1] * kLpcCorrWindow[pos1]; + } + + /* Get correlation coefficients */ + WebRtcIsac_AutoCorr(corrlo, DataLo, WINLEN, ORDERLO+1); /* computing autocorrelation */ + WebRtcIsac_AutoCorr(corrhi, DataHi, WINLEN, ORDERHI); + + + /* less noise for lower frequencies, by filtering/scaling autocorrelation sequences */ + corrlo2[0] = (1.0+aa*aa) * corrlo[0] - 2.0*aa * corrlo[1]; + tmp = (1.0 + aa*aa); + for (n = 1; n <= ORDERLO; n++) { + corrlo2[n] = tmp * corrlo[n] - aa * (corrlo[n-1] + corrlo[n+1]); + } + tmp = (1.0+aa) * (1.0+aa); + for (n = 0; n <= ORDERHI; n++) { + corrhi[n] = tmp * corrhi[n]; + } + + /* add white noise floor */ + corrlo2[0] += 1e-6; + corrhi[0] += 1e-6; + + + FwdA = 0.01; + FwdB = 0.01; + + /* recursive filtering of correlation over subframes */ + for (n = 0; n <= ORDERLO; n++) { + maskdata->CorrBufLo[n] = FwdA * maskdata->CorrBufLo[n] + corrlo2[n]; + corrlo2[n] = ((1.0-FwdA)*FwdB) * maskdata->CorrBufLo[n] + (1.0-FwdB) * corrlo2[n]; + } + for (n = 0; n <= ORDERHI; n++) { + maskdata->CorrBufHi[n] = FwdA * maskdata->CorrBufHi[n] + corrhi[n]; + corrhi[n] = ((1.0-FwdA)*FwdB) * maskdata->CorrBufHi[n] + (1.0-FwdB) * corrhi[n]; + } + + /* compute prediction coefficients */ + WebRtcIsac_LevDurb(a_LO, k_veclo, corrlo2, ORDERLO); + WebRtcIsac_LevDurb(a_HI, k_vechi, corrhi, ORDERHI); + + /* bandwidth expansion */ + tmp = gammaLo; + for (n = 1; n <= ORDERLO; n++) { + a_LO[n] *= tmp; + tmp *= gammaLo; + } + + /* residual energy */ + res_nrg = 0.0; + for (j = 0; j <= ORDERLO; j++) { + for (n = 0; n <= j; n++) { + res_nrg += a_LO[j] * corrlo2[j-n] * a_LO[n]; + } + for (n = j+1; n <= ORDERLO; n++) { + res_nrg += a_LO[j] * corrlo2[n-j] * a_LO[n]; + } + } + + /* add hearing threshold and compute the gain */ + *lo_coeff++ = S_N_R / (sqrt(res_nrg) / varscale + H_T_H); + + /* copy coefficients to output array */ + for (n = 1; n <= ORDERLO; n++) { + *lo_coeff++ = a_LO[n]; + } + + + /* bandwidth expansion */ + tmp = gammaHi; + for (n = 1; n <= ORDERHI; n++) { + a_HI[n] *= tmp; + tmp *= gammaHi; + } + + /* residual energy */ + res_nrg = 0.0; + for (j = 0; j <= ORDERHI; j++) { + for (n = 0; n <= j; n++) { + res_nrg += a_HI[j] * corrhi[j-n] * a_HI[n]; + } + for (n = j+1; n <= ORDERHI; n++) { + res_nrg += a_HI[j] * corrhi[n-j] * a_HI[n]; + } + } + + /* add hearing threshold and compute of the gain */ + *hi_coeff++ = S_N_R / (sqrt(res_nrg) / varscale + H_T_H); + + /* copy coefficients to output array */ + for (n = 1; n <= ORDERHI; n++) { + *hi_coeff++ = a_HI[n]; + } + } +} + + + +/****************************************************************************** + * WebRtcIsac_GetLpcCoefUb() + * + * Compute LP coefficients and correlation coefficients. At 12 kHz LP + * coefficients of the first and the last sub-frame is computed. At 16 kHz + * LP coefficients of 4th, 8th and 12th sub-frames are computed. We always + * compute correlation coefficients of all sub-frames. + * + * Inputs: + * -inSignal : Input signal + * -maskdata : a structure keeping signal from previous frame. + * -bandwidth : specifies if the codec is in 0-16 kHz mode or + * 0-12 kHz mode. + * + * Outputs: + * -lpCoeff : pointer to a buffer where A-polynomials are + * written to (first coeff is 1 and it is not + * written) + * -corrMat : a matrix where correlation coefficients of each + * sub-frame are written to one row. + * -varscale : a scale used to compute LPC gains. + */ +void +WebRtcIsac_GetLpcCoefUb( + double* inSignal, + MaskFiltstr* maskdata, + double* lpCoeff, + double corrMat[][UB_LPC_ORDER + 1], + double* varscale, + int16_t bandwidth) +{ + int frameCntr, activeFrameCntr, n, pos1, pos2; + int16_t criterion1; + int16_t criterion2; + int16_t numSubFrames = SUBFRAMES * (1 + (bandwidth == isac16kHz)); + double data[WINLEN]; + double corrSubFrame[UB_LPC_ORDER+2]; + double reflecCoeff[UB_LPC_ORDER]; + + double aPolynom[UB_LPC_ORDER+1]; + double tmp; + + /* bandwdith expansion factors */ + const double gamma = 0.9; + + /* change quallevel depending on pitch gains and level fluctuations */ + WebRtcIsac_GetVarsUB(inSignal, &(maskdata->OldEnergy), varscale); + + /* replace data in buffer by new look-ahead data */ + for(frameCntr = 0, activeFrameCntr = 0; frameCntr < numSubFrames; + frameCntr++) + { + if(frameCntr == SUBFRAMES) + { + // we are in 16 kHz + varscale++; + WebRtcIsac_GetVarsUB(&inSignal[FRAMESAMPLES_HALF], + &(maskdata->OldEnergy), varscale); + } + /* Update input buffer and multiply signal with window */ + for(pos1 = 0; pos1 < WINLEN - UPDATE/2; pos1++) + { + maskdata->DataBufferLo[pos1] = maskdata->DataBufferLo[pos1 + + UPDATE/2]; + data[pos1] = maskdata->DataBufferLo[pos1] * kLpcCorrWindow[pos1]; + } + pos2 = frameCntr * UPDATE/2; + for(n = 0; n < UPDATE/2; n++, pos1++, pos2++) + { + maskdata->DataBufferLo[pos1] = inSignal[pos2]; + data[pos1] = maskdata->DataBufferLo[pos1] * kLpcCorrWindow[pos1]; + } + + /* Get correlation coefficients */ + /* computing autocorrelation */ + WebRtcIsac_AutoCorr(corrSubFrame, data, WINLEN, UB_LPC_ORDER+1); + memcpy(corrMat[frameCntr], corrSubFrame, + (UB_LPC_ORDER+1)*sizeof(double)); + + criterion1 = ((frameCntr == 0) || (frameCntr == (SUBFRAMES - 1))) && + (bandwidth == isac12kHz); + criterion2 = (((frameCntr+1) % 4) == 0) && + (bandwidth == isac16kHz); + if(criterion1 || criterion2) + { + /* add noise */ + corrSubFrame[0] += 1e-6; + /* compute prediction coefficients */ + WebRtcIsac_LevDurb(aPolynom, reflecCoeff, corrSubFrame, + UB_LPC_ORDER); + + /* bandwidth expansion */ + tmp = gamma; + for (n = 1; n <= UB_LPC_ORDER; n++) + { + *lpCoeff++ = aPolynom[n] * tmp; + tmp *= gamma; + } + activeFrameCntr++; + } + } +} + + + +/****************************************************************************** + * WebRtcIsac_GetLpcGain() + * + * Compute the LPC gains for each sub-frame, given the LPC of each sub-frame + * and the corresponding correlation coefficients. + * + * Inputs: + * -signal_noise_ratio : the desired SNR in dB. + * -numVecs : number of sub-frames + * -corrMat : a matrix of correlation coefficients where + * each row is a set of correlation coefficients of + * one sub-frame. + * -varscale : a scale computed when WebRtcIsac_GetLpcCoefUb() + * is called. + * + * Outputs: + * -gain : pointer to a buffer where LP gains are written. + * + */ +void +WebRtcIsac_GetLpcGain( + double signal_noise_ratio, + const double* filtCoeffVecs, + int numVecs, + double* gain, + double corrMat[][UB_LPC_ORDER + 1], + const double* varscale) +{ + int16_t j, n; + int16_t subFrameCntr; + double aPolynom[ORDERLO + 1]; + double res_nrg; + + const double HearThresOffset = -28.0; + const double H_T_H = pow(10.0, 0.05 * HearThresOffset); + /* divide by sqrt(12) = 3.46 */ + const double S_N_R = pow(10.0, 0.05 * signal_noise_ratio) / 3.46; + + aPolynom[0] = 1; + for(subFrameCntr = 0; subFrameCntr < numVecs; subFrameCntr++) + { + if(subFrameCntr == SUBFRAMES) + { + // we are in second half of a SWB frame. use new varscale + varscale++; + } + memcpy(&aPolynom[1], &filtCoeffVecs[(subFrameCntr * (UB_LPC_ORDER + 1)) + + 1], sizeof(double) * UB_LPC_ORDER); + + /* residual energy */ + res_nrg = 0.0; + for(j = 0; j <= UB_LPC_ORDER; j++) + { + for(n = 0; n <= j; n++) + { + res_nrg += aPolynom[j] * corrMat[subFrameCntr][j-n] * + aPolynom[n]; + } + for(n = j+1; n <= UB_LPC_ORDER; n++) + { + res_nrg += aPolynom[j] * corrMat[subFrameCntr][n-j] * + aPolynom[n]; + } + } + + /* add hearing threshold and compute the gain */ + gain[subFrameCntr] = S_N_R / (sqrt(res_nrg) / *varscale + H_T_H); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_analysis.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_analysis.h new file mode 100644 index 0000000000..1566ab0d40 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_analysis.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_analysis.h + * + * LPC functions + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_ANALYSIS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_ANALYSIS_H_ + +#include "settings.h" +#include "structs.h" + +double WebRtcIsac_LevDurb(double *a, double *k, double *r, size_t order); + +void WebRtcIsac_GetVars(const double *input, const int16_t *pitchGains_Q12, + double *oldEnergy, double *varscale); + +void WebRtcIsac_GetLpcCoefLb(double *inLo, double *inHi, MaskFiltstr *maskdata, + double signal_noise_ratio, const int16_t *pitchGains_Q12, + double *lo_coeff, double *hi_coeff); + + +void WebRtcIsac_GetLpcGain( + double signal_noise_ratio, + const double* filtCoeffVecs, + int numVecs, + double* gain, + double corrLo[][UB_LPC_ORDER + 1], + const double* varscale); + +void WebRtcIsac_GetLpcCoefUb( + double* inSignal, + MaskFiltstr* maskdata, + double* lpCoeff, + double corr[][UB_LPC_ORDER + 1], + double* varscale, + int16_t bandwidth); + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_ANALYIS_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.c new file mode 100644 index 0000000000..d9073d5e9d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * SWB_KLT_Tables_LPCGain.c + * + * This file defines tables used for entropy coding of LPC Gain + * of upper-band. + * + */ + +#include "lpc_gain_swb_tables.h" +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +const double WebRtcIsac_kQSizeLpcGain = 0.100000; + +const double WebRtcIsac_kMeanLpcGain = -3.3822; + +/* +* The smallest reconstruction points for quantiztion of +* LPC gains. +*/ +const double WebRtcIsac_kLeftRecPointLpcGain[SUBFRAMES] = +{ + -0.800000, -1.000000, -1.200000, -2.200000, -3.000000, -12.700000 +}; + +/* +* Number of reconstruction points of quantizers for LPC Gains. +*/ +const int16_t WebRtcIsac_kNumQCellLpcGain[SUBFRAMES] = +{ + 17, 20, 25, 45, 77, 170 +}; +/* +* Starting index for entropy decoder to search for the right interval, +* one entry per LAR coefficient +*/ +const uint16_t WebRtcIsac_kLpcGainEntropySearch[SUBFRAMES] = +{ + 8, 10, 12, 22, 38, 85 +}; + +/* +* The following 6 vectors define CDF of 6 decorrelated LPC +* gains. +*/ +const uint16_t WebRtcIsac_kLpcGainCdfVec0[18] = +{ + 0, 10, 27, 83, 234, 568, 1601, 4683, 16830, 57534, 63437, + 64767, 65229, 65408, 65483, 65514, 65527, 65535 +}; + +const uint16_t WebRtcIsac_kLpcGainCdfVec1[21] = +{ + 0, 15, 33, 84, 185, 385, 807, 1619, 3529, 7850, 19488, + 51365, 62437, 64548, 65088, 65304, 65409, 65484, 65507, 65522, 65535 +}; + +const uint16_t WebRtcIsac_kLpcGainCdfVec2[26] = +{ + 0, 15, 29, 54, 89, 145, 228, 380, 652, 1493, 4260, + 12359, 34133, 50749, 57224, 60814, 62927, 64078, 64742, 65103, 65311, 65418, + 65473, 65509, 65521, 65535 +}; + +const uint16_t WebRtcIsac_kLpcGainCdfVec3[46] = +{ + 0, 8, 12, 16, 26, 42, 56, 76, 111, 164, 247, + 366, 508, 693, 1000, 1442, 2155, 3188, 4854, 7387, 11249, 17617, + 30079, 46711, 56291, 60127, 62140, 63258, 63954, 64384, 64690, 64891, 65031, + 65139, 65227, 65293, 65351, 65399, 65438, 65467, 65492, 65504, 65510, 65518, + 65523, 65535 +}; + +const uint16_t WebRtcIsac_kLpcGainCdfVec4[78] = +{ + 0, 17, 29, 39, 51, 70, 104, 154, 234, 324, 443, + 590, 760, 971, 1202, 1494, 1845, 2274, 2797, 3366, 4088, 4905, + 5899, 7142, 8683, 10625, 12983, 16095, 20637, 28216, 38859, 47237, 51537, + 54150, 56066, 57583, 58756, 59685, 60458, 61103, 61659, 62144, 62550, 62886, + 63186, 63480, 63743, 63954, 64148, 64320, 64467, 64600, 64719, 64837, 64939, + 65014, 65098, 65160, 65211, 65250, 65290, 65325, 65344, 65366, 65391, 65410, + 65430, 65447, 65460, 65474, 65487, 65494, 65501, 65509, 65513, 65518, 65520, + 65535 +}; + +const uint16_t WebRtcIsac_kLpcGainCdfVec5[171] = +{ + 0, 10, 12, 14, 16, 18, 23, 29, 35, 42, 51, + 58, 65, 72, 78, 87, 96, 103, 111, 122, 134, 150, + 167, 184, 202, 223, 244, 265, 289, 315, 346, 379, 414, + 450, 491, 532, 572, 613, 656, 700, 751, 802, 853, 905, + 957, 1021, 1098, 1174, 1250, 1331, 1413, 1490, 1565, 1647, 1730, + 1821, 1913, 2004, 2100, 2207, 2314, 2420, 2532, 2652, 2783, 2921, + 3056, 3189, 3327, 3468, 3640, 3817, 3993, 4171, 4362, 4554, 4751, + 4948, 5142, 5346, 5566, 5799, 6044, 6301, 6565, 6852, 7150, 7470, + 7797, 8143, 8492, 8835, 9181, 9547, 9919, 10315, 10718, 11136, 11566, + 12015, 12482, 12967, 13458, 13953, 14432, 14903, 15416, 15936, 16452, 16967, + 17492, 18024, 18600, 19173, 19736, 20311, 20911, 21490, 22041, 22597, 23157, + 23768, 24405, 25034, 25660, 26280, 26899, 27614, 28331, 29015, 29702, 30403, + 31107, 31817, 32566, 33381, 34224, 35099, 36112, 37222, 38375, 39549, 40801, + 42074, 43350, 44626, 45982, 47354, 48860, 50361, 51845, 53312, 54739, 56026, + 57116, 58104, 58996, 59842, 60658, 61488, 62324, 63057, 63769, 64285, 64779, + 65076, 65344, 65430, 65500, 65517, 65535 +}; + +/* +* An array of pointers to CDFs of decorrelated LPC Gains +*/ +const uint16_t* WebRtcIsac_kLpcGainCdfMat[SUBFRAMES] = +{ + WebRtcIsac_kLpcGainCdfVec0, WebRtcIsac_kLpcGainCdfVec1, + WebRtcIsac_kLpcGainCdfVec2, WebRtcIsac_kLpcGainCdfVec3, + WebRtcIsac_kLpcGainCdfVec4, WebRtcIsac_kLpcGainCdfVec5 +}; + +/* +* A matrix to decorrellate LPC gains of subframes. +*/ +const double WebRtcIsac_kLpcGainDecorrMat[SUBFRAMES][SUBFRAMES] = +{ + {-0.150860, 0.327872, 0.367220, 0.504613, 0.559270, 0.409234}, + { 0.457128, -0.613591, -0.289283, -0.029734, 0.393760, 0.418240}, + {-0.626043, 0.136489, -0.439118, -0.448323, 0.135987, 0.420869}, + { 0.526617, 0.480187, 0.242552, -0.488754, -0.158713, 0.411331}, + {-0.302587, -0.494953, 0.588112, -0.063035, -0.404290, 0.387510}, + { 0.086378, 0.147714, -0.428875, 0.548300, -0.570121, 0.401391} +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.h new file mode 100644 index 0000000000..5ea537575d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * SWB_KLT_Tables_LPCGain.h + * + * This file declares tables used for entropy coding of LPC Gain + * of upper-band. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_GAIN_SWB_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_GAIN_SWB_TABLES_H_ + +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +extern const double WebRtcIsac_kQSizeLpcGain; + +extern const double WebRtcIsac_kLeftRecPointLpcGain[SUBFRAMES]; + +extern const int16_t WebRtcIsac_kNumQCellLpcGain[SUBFRAMES]; + +extern const uint16_t WebRtcIsac_kLpcGainEntropySearch[SUBFRAMES]; + +extern const uint16_t WebRtcIsac_kLpcGainCdfVec0[18]; + +extern const uint16_t WebRtcIsac_kLpcGainCdfVec1[21]; + +extern const uint16_t WebRtcIsac_kLpcGainCdfVec2[26]; + +extern const uint16_t WebRtcIsac_kLpcGainCdfVec3[46]; + +extern const uint16_t WebRtcIsac_kLpcGainCdfVec4[78]; + +extern const uint16_t WebRtcIsac_kLpcGainCdfVec5[171]; + +extern const uint16_t* WebRtcIsac_kLpcGainCdfMat[SUBFRAMES]; + +extern const double WebRtcIsac_kLpcGainDecorrMat[SUBFRAMES][SUBFRAMES]; + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_GAIN_SWB_TABLES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.c new file mode 100644 index 0000000000..6e64e41479 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * SWB_KLT_Tables.c + * + * This file defines tables used for entropy coding of LPC shape of + * upper-band signal if the bandwidth is 12 kHz. + * + */ + +#include "lpc_shape_swb12_tables.h" +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +/* +* Mean value of LAR +*/ +const double WebRtcIsac_kMeanLarUb12[UB_LPC_ORDER] = +{ + 0.03748928306641, 0.09453441192543, -0.01112522344398, 0.03800237516842 +}; + +/* +* A rotation matrix to decorrelate intra-vector correlation, +* i.e. correlation among components of LAR vector. +*/ +const double WebRtcIsac_kIntraVecDecorrMatUb12[UB_LPC_ORDER][UB_LPC_ORDER] = +{ + {-0.00075365493856, -0.05809964887743, -0.23397966154116, 0.97050367376411}, + { 0.00625021257734, -0.17299965610679, 0.95977735920651, 0.22104179375008}, + { 0.20543384258374, -0.96202143495696, -0.15301870801552, -0.09432375099565}, + {-0.97865075648479, -0.20300322280841, -0.02581111653779, -0.01913568980258} +}; + +/* +* A rotation matrix to remove correlation among LAR coefficients +* of different LAR vectors. One might guess that decorrelation matrix +* for the first component should differ from the second component +* but we haven't observed a significant benefit of having different +* decorrelation matrices for different components. +*/ +const double WebRtcIsac_kInterVecDecorrMatUb12 +[UB_LPC_VEC_PER_FRAME][UB_LPC_VEC_PER_FRAME] = +{ + { 0.70650597970460, -0.70770707262373}, + {-0.70770707262373, -0.70650597970460} +}; + +/* +* LAR quantization step-size. +*/ +const double WebRtcIsac_kLpcShapeQStepSizeUb12 = 0.150000; + +/* +* The smallest reconstruction points for quantiztion of LAR coefficients. +*/ +const double WebRtcIsac_kLpcShapeLeftRecPointUb12 +[UB_LPC_ORDER*UB_LPC_VEC_PER_FRAME] = +{ + -0.900000, -1.050000, -1.350000, -1.800000, -1.350000, -1.650000, + -2.250000, -3.450000 +}; + +/* +* Number of reconstruction points of quantizers for LAR coefficients. +*/ +const int16_t WebRtcIsac_kLpcShapeNumRecPointUb12 +[UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME] = +{ + 13, 15, 19, 27, 19, 24, 32, 48 +}; + +/* +* Starting index for entropy decoder to search for the right interval, +* one entry per LAR coefficient +*/ +const uint16_t WebRtcIsac_kLpcShapeEntropySearchUb12 +[UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME] = +{ + 6, 7, 9, 13, 9, 12, 16, 24 +}; + +/* +* The following 8 vectors define CDF of 8 decorrelated LAR +* coefficients. +*/ +const uint16_t WebRtcIsac_kLpcShapeCdfVec0Ub12[14] = +{ + 0, 13, 95, 418, 1687, 6498, 21317, 44200, 59029, 63849, 65147, + 65449, 65525, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec1Ub12[16] = +{ + 0, 10, 59, 255, 858, 2667, 8200, 22609, 42988, 57202, 62947, + 64743, 65308, 65476, 65522, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec2Ub12[20] = +{ + 0, 18, 40, 118, 332, 857, 2017, 4822, 11321, 24330, 41279, + 54342, 60637, 63394, 64659, 65184, 65398, 65482, 65518, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec3Ub12[28] = +{ + 0, 21, 38, 90, 196, 398, 770, 1400, 2589, 4650, 8211, + 14933, 26044, 39592, 50814, 57452, 60971, 62884, 63995, 64621, 65019, 65273, + 65410, 65480, 65514, 65522, 65531, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec4Ub12[20] = +{ + 0, 7, 46, 141, 403, 969, 2132, 4649, 10633, 24902, 43254, + 54665, 59928, 62674, 64173, 64938, 65293, 65464, 65523, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec5Ub12[25] = +{ + 0, 7, 22, 72, 174, 411, 854, 1737, 3545, 6774, 13165, + 25221, 40980, 52821, 58714, 61706, 63472, 64437, 64989, 65287, 65430, 65503, + 65525, 65529, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec6Ub12[33] = +{ + 0, 11, 21, 36, 65, 128, 228, 401, 707, 1241, 2126, + 3589, 6060, 10517, 18853, 31114, 42477, 49770, 54271, 57467, 59838, 61569, + 62831, 63772, 64433, 64833, 65123, 65306, 65419, 65466, 65499, 65519, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec7Ub12[49] = +{ + 0, 14, 34, 67, 107, 167, 245, 326, 449, 645, 861, + 1155, 1508, 2003, 2669, 3544, 4592, 5961, 7583, 9887, 13256, 18765, + 26519, 34077, 40034, 44349, 47795, 50663, 53262, 55473, 57458, 59122, 60592, + 61742, 62690, 63391, 63997, 64463, 64794, 65045, 65207, 65309, 65394, 65443, + 65478, 65504, 65514, 65523, 65535 +}; + +/* +* An array of pointers to CDFs of decorrelated LARs +*/ +const uint16_t* WebRtcIsac_kLpcShapeCdfMatUb12 +[UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME] = +{ + WebRtcIsac_kLpcShapeCdfVec0Ub12, WebRtcIsac_kLpcShapeCdfVec1Ub12, + WebRtcIsac_kLpcShapeCdfVec2Ub12, WebRtcIsac_kLpcShapeCdfVec3Ub12, + WebRtcIsac_kLpcShapeCdfVec4Ub12, WebRtcIsac_kLpcShapeCdfVec5Ub12, + WebRtcIsac_kLpcShapeCdfVec6Ub12, WebRtcIsac_kLpcShapeCdfVec7Ub12 +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.h new file mode 100644 index 0000000000..c703a6cb4b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_shape_swb12_tables.h + * + * This file declares tables used for entropy coding of LPC shape of + * upper-band signal if the bandwidth is 12 kHz. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_SHAPE_SWB12_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_SHAPE_SWB12_TABLES_H_ + +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +extern const double WebRtcIsac_kMeanLarUb12[UB_LPC_ORDER]; + +extern const double WebRtcIsac_kMeanLpcGain; + +extern const double WebRtcIsac_kIntraVecDecorrMatUb12[UB_LPC_ORDER][UB_LPC_ORDER]; + +extern const double WebRtcIsac_kInterVecDecorrMatUb12 +[UB_LPC_VEC_PER_FRAME][UB_LPC_VEC_PER_FRAME]; + +extern const double WebRtcIsac_kLpcShapeQStepSizeUb12; + +extern const double WebRtcIsac_kLpcShapeLeftRecPointUb12 +[UB_LPC_ORDER*UB_LPC_VEC_PER_FRAME]; + + +extern const int16_t WebRtcIsac_kLpcShapeNumRecPointUb12 +[UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME]; + +extern const uint16_t WebRtcIsac_kLpcShapeEntropySearchUb12 +[UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec0Ub12[14]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec1Ub12[16]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec2Ub12[20]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec3Ub12[28]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec4Ub12[20]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec5Ub12[25]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec6Ub12[33]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec7Ub12[49]; + +extern const uint16_t* WebRtcIsac_kLpcShapeCdfMatUb12 +[UB_LPC_ORDER * UB_LPC_VEC_PER_FRAME]; + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_SHAPE_SWB12_TABLES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.c new file mode 100644 index 0000000000..852d07fd73 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * SWB16_KLT_Tables.c + * + * This file defines tables used for entropy coding of LPC shape of + * upper-band signal if the bandwidth is 16 kHz. + * + */ + +#include "lpc_shape_swb16_tables.h" +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +/* +* Mean value of LAR +*/ +const double WebRtcIsac_kMeanLarUb16[UB_LPC_ORDER] = +{ +0.454978, 0.364747, 0.102999, 0.104523 +}; + +/* +* A rotation matrix to decorrelate intra-vector correlation, +* i.e. correlation among components of LAR vector. +*/ +const double WebRtcIsac_kIintraVecDecorrMatUb16[UB_LPC_ORDER][UB_LPC_ORDER] = +{ + {-0.020528, -0.085858, -0.002431, 0.996093}, + {-0.033155, 0.036102, 0.998786, 0.004866}, + { 0.202627, 0.974853, -0.028940, 0.088132}, + {-0.978479, 0.202454, -0.039785, -0.002811} +}; + +/* +* A rotation matrix to remove correlation among LAR coefficients +* of different LAR vectors. One might guess that decorrelation matrix +* for the first component should differ from the second component +* but we haven't observed a significant benefit of having different +* decorrelation matrices for different components. +*/ +const double WebRtcIsac_kInterVecDecorrMatUb16 +[UB16_LPC_VEC_PER_FRAME][UB16_LPC_VEC_PER_FRAME] = +{ + { 0.291675, -0.515786, 0.644927, 0.482658}, + {-0.647220, 0.479712, 0.289556, 0.516856}, + { 0.643084, 0.485489, -0.289307, 0.516763}, + {-0.287185, -0.517823, -0.645389, 0.482553} +}; + +/* +* The following 16 vectors define CDF of 16 decorrelated LAR +* coefficients. +*/ +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub16[14] = +{ + 0, 2, 20, 159, 1034, 5688, 20892, 44653, + 59849, 64485, 65383, 65518, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec1Ub16[16] = +{ + 0, 1, 7, 43, 276, 1496, 6681, 21653, + 43891, 58859, 64022, 65248, 65489, 65529, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec2Ub16[18] = +{ + 0, 1, 9, 54, 238, 933, 3192, 9461, + 23226, 42146, 56138, 62413, 64623, 65300, 65473, 65521, + 65533, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec3Ub16[30] = +{ + 0, 2, 4, 8, 17, 36, 75, 155, + 329, 683, 1376, 2662, 5047, 9508, 17526, 29027, + 40363, 48997, 55096, 59180, 61789, 63407, 64400, 64967, + 65273, 65429, 65497, 65526, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec4Ub16[16] = +{ + 0, 1, 10, 63, 361, 1785, 7407, 22242, + 43337, 58125, 63729, 65181, 65472, 65527, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec5Ub16[17] = +{ + 0, 1, 7, 29, 134, 599, 2443, 8590, + 22962, 42635, 56911, 63060, 64940, 65408, 65513, 65531, + 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec6Ub16[21] = +{ + 0, 1, 5, 16, 57, 191, 611, 1808, + 4847, 11755, 24612, 40910, 53789, 60698, 63729, 64924, + 65346, 65486, 65523, 65532, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec7Ub16[36] = +{ + 0, 1, 4, 12, 25, 55, 104, 184, + 314, 539, 926, 1550, 2479, 3861, 5892, 8845, + 13281, 20018, 29019, 38029, 45581, 51557, 56057, 59284, + 61517, 63047, 64030, 64648, 65031, 65261, 65402, 65480, + 65518, 65530, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec8Ub16[21] = +{ + 0, 1, 2, 7, 26, 103, 351, 1149, + 3583, 10204, 23846, 41711, 55361, 61917, 64382, 65186, + 65433, 65506, 65528, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub160[21] = +{ + 0, 6, 19, 63, 205, 638, 1799, 4784, + 11721, 24494, 40803, 53805, 60886, 63822, 64931, 65333, + 65472, 65517, 65530, 65533, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub161[28] = +{ + 0, 1, 3, 11, 31, 86, 221, 506, + 1101, 2296, 4486, 8477, 15356, 26079, 38941, 49952, + 57165, 61257, 63426, 64549, 65097, 65351, 65463, 65510, + 65526, 65532, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub162[55] = +{ + 0, 3, 12, 23, 42, 65, 89, 115, + 150, 195, 248, 327, 430, 580, 784, 1099, + 1586, 2358, 3651, 5899, 9568, 14312, 19158, 23776, + 28267, 32663, 36991, 41153, 45098, 48680, 51870, 54729, + 57141, 59158, 60772, 62029, 63000, 63761, 64322, 64728, + 65000, 65192, 65321, 65411, 65463, 65496, 65514, 65523, + 65527, 65529, 65531, 65532, 65533, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub163[26] = +{ + 0, 2, 4, 10, 21, 48, 114, 280, + 701, 1765, 4555, 11270, 24267, 41213, 54285, 61003, + 63767, 64840, 65254, 65421, 65489, 65514, 65526, 65532, + 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub164[28] = +{ + 0, 1, 3, 6, 15, 36, 82, 196, + 453, 1087, 2557, 5923, 13016, 25366, 40449, 52582, + 59539, 62896, 64389, 65033, 65316, 65442, 65494, 65519, + 65529, 65533, 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub165[34] = +{ + 0, 2, 4, 8, 18, 35, 73, 146, + 279, 524, 980, 1789, 3235, 5784, 10040, 16998, + 27070, 38543, 48499, 55421, 59712, 62257, 63748, 64591, + 65041, 65278, 65410, 65474, 65508, 65522, 65530, 65533, + 65534, 65535 +}; + +const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub166[71] = +{ + 0, 1, 2, 6, 13, 26, 55, 92, + 141, 191, 242, 296, 355, 429, 522, 636, + 777, 947, 1162, 1428, 1753, 2137, 2605, 3140, + 3743, 4409, 5164, 6016, 6982, 8118, 9451, 10993, + 12754, 14810, 17130, 19780, 22864, 26424, 30547, 35222, + 40140, 44716, 48698, 52056, 54850, 57162, 59068, 60643, + 61877, 62827, 63561, 64113, 64519, 64807, 65019, 65167, + 65272, 65343, 65399, 65440, 65471, 65487, 65500, 65509, + 65518, 65524, 65527, 65531, 65533, 65534, 65535 +}; + +/* +* An array of pointers to CDFs of decorrelated LARs +*/ +const uint16_t* WebRtcIsac_kLpcShapeCdfMatUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME] = { + WebRtcIsac_kLpcShapeCdfVec01Ub16, + WebRtcIsac_kLpcShapeCdfVec1Ub16, + WebRtcIsac_kLpcShapeCdfVec2Ub16, + WebRtcIsac_kLpcShapeCdfVec3Ub16, + WebRtcIsac_kLpcShapeCdfVec4Ub16, + WebRtcIsac_kLpcShapeCdfVec5Ub16, + WebRtcIsac_kLpcShapeCdfVec6Ub16, + WebRtcIsac_kLpcShapeCdfVec7Ub16, + WebRtcIsac_kLpcShapeCdfVec8Ub16, + WebRtcIsac_kLpcShapeCdfVec01Ub160, + WebRtcIsac_kLpcShapeCdfVec01Ub161, + WebRtcIsac_kLpcShapeCdfVec01Ub162, + WebRtcIsac_kLpcShapeCdfVec01Ub163, + WebRtcIsac_kLpcShapeCdfVec01Ub164, + WebRtcIsac_kLpcShapeCdfVec01Ub165, + WebRtcIsac_kLpcShapeCdfVec01Ub166 +}; + +/* +* The smallest reconstruction points for quantiztion of LAR coefficients. +*/ +const double WebRtcIsac_kLpcShapeLeftRecPointUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME] = +{ + -0.8250, -0.9750, -1.1250, -2.1750, -0.9750, -1.1250, -1.4250, + -2.6250, -1.4250, -1.2750, -1.8750, -3.6750, -1.7250, -1.8750, + -2.3250, -5.4750 +}; + +/* +* Number of reconstruction points of quantizers for LAR coefficients. +*/ +const int16_t WebRtcIsac_kLpcShapeNumRecPointUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME] = +{ + 13, 15, 17, 29, 15, 16, 20, 35, 20, + 20, 27, 54, 25, 27, 33, 70 +}; + +/* +* Starting index for entropy decoder to search for the right interval, +* one entry per LAR coefficient +*/ +const uint16_t WebRtcIsac_kLpcShapeEntropySearchUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME] = +{ + 6, 7, 8, 14, 7, 8, 10, 17, 10, + 10, 13, 27, 12, 13, 16, 35 +}; + +/* +* LAR quantization step-size. +*/ +const double WebRtcIsac_kLpcShapeQStepSizeUb16 = 0.150000; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.h new file mode 100644 index 0000000000..17a77d7c1d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_shape_swb16_tables.h + * + * This file declares tables used for entropy coding of LPC shape of + * upper-band signal if the bandwidth is 16 kHz. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_SHAPE_SWB16_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_SHAPE_SWB16_TABLES_H_ + +#include "settings.h" +#include "typedefs.h" // NOLINT(build/include) + +extern const double WebRtcIsac_kMeanLarUb16[UB_LPC_ORDER]; + +extern const double WebRtcIsac_kIintraVecDecorrMatUb16[UB_LPC_ORDER][UB_LPC_ORDER]; + +extern const double WebRtcIsac_kInterVecDecorrMatUb16 +[UB16_LPC_VEC_PER_FRAME][UB16_LPC_VEC_PER_FRAME]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub16[14]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec1Ub16[16]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec2Ub16[18]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec3Ub16[30]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec4Ub16[16]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec5Ub16[17]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec6Ub16[21]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec7Ub16[36]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec8Ub16[21]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub160[21]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub161[28]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub162[55]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub163[26]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub164[28]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub165[34]; + +extern const uint16_t WebRtcIsac_kLpcShapeCdfVec01Ub166[71]; + +extern const uint16_t* WebRtcIsac_kLpcShapeCdfMatUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + +extern const double WebRtcIsac_kLpcShapeLeftRecPointUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + +extern const int16_t WebRtcIsac_kLpcShapeNumRecPointUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + +extern const uint16_t WebRtcIsac_kLpcShapeEntropySearchUb16 +[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + +extern const double WebRtcIsac_kLpcShapeQStepSizeUb16; + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_SHAPE_SWB16_TABLES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.c new file mode 100644 index 0000000000..909809bc67 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.c @@ -0,0 +1,601 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* coding tables for the KLT coefficients */ + +#include "lpc_tables.h" +#include "settings.h" + +/* cdf array for model indicator */ +const uint16_t WebRtcIsac_kQKltModelCdf[4] = { + 0, 15434, 37548, 65535 }; + +/* pointer to cdf array for model indicator */ +const uint16_t *WebRtcIsac_kQKltModelCdfPtr[1] = { + WebRtcIsac_kQKltModelCdf }; + +/* initial cdf index for decoder of model indicator */ +const uint16_t WebRtcIsac_kQKltModelInitIndex[1] = { 1 }; + +/* offset to go from rounded value to quantization index */ +const short WebRtcIsac_kQKltQuantMinGain[12] = { + 3, 6, 4, 6, 6, 9, 5, 16, 11, 34, 32, 47 }; + + +const short WebRtcIsac_kQKltQuantMinShape[108] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 2, 2, 2, 3, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, + 1, 1, 1, 2, 2, 3, 0, 0, 0, 0, + 1, 0, 1, 1, 1, 1, 1, 1, 1, 2, + 2, 4, 3, 5, 0, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 2, 3, 4, + 4, 7, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 2, 3, 2, 3, 4, 4, 5, 7, 13, + 0, 1, 1, 2, 3, 2, 2, 2, 4, 4, + 5, 6, 7, 11, 9, 13, 12, 26 }; + +/* maximum quantization index */ +const uint16_t WebRtcIsac_kQKltMaxIndGain[12] = { + 6, 12, 8, 14, 10, 19, 12, 31, 22, 56, 52, 138 }; + +const uint16_t WebRtcIsac_kQKltMaxIndShape[108] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 2, 2, 2, 2, 4, 4, 5, 6, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 1, 2, 2, + 2, 2, 3, 4, 5, 7, 0, 0, 0, 0, + 2, 0, 2, 2, 2, 2, 3, 2, 2, 4, + 4, 6, 6, 9, 0, 0, 0, 0, 2, 2, + 2, 2, 2, 2, 3, 2, 4, 4, 7, 7, + 9, 13, 0, 0, 2, 2, 2, 2, 2, 2, + 3, 4, 5, 4, 6, 8, 8, 10, 16, 25, + 0, 2, 2, 4, 5, 4, 4, 4, 7, 8, + 9, 10, 13, 19, 17, 23, 25, 49 }; + +/* index offset */ +const uint16_t WebRtcIsac_kQKltOffsetGain[12] = { + 0, 7, 20, 29, 44, 55, 75, 88, 120, 143, 200, 253 }; + +const uint16_t WebRtcIsac_kQKltOffsetShape[108] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 11, 14, 17, 20, 23, 28, 33, 39, 46, 47, + 48, 49, 50, 52, 53, 54, 55, 56, 58, 61, + 64, 67, 70, 74, 79, 85, 93, 94, 95, 96, + 97, 100, 101, 104, 107, 110, 113, 117, 120, 123, + 128, 133, 140, 147, 157, 158, 159, 160, 161, 164, + 167, 170, 173, 176, 179, 183, 186, 191, 196, 204, + 212, 222, 236, 237, 238, 241, 244, 247, 250, 253, + 256, 260, 265, 271, 276, 283, 292, 301, 312, 329, + 355, 356, 359, 362, 367, 373, 378, 383, 388, 396, + 405, 415, 426, 440, 460, 478, 502, 528 }; + +/* initial cdf index for KLT coefficients */ +const uint16_t WebRtcIsac_kQKltInitIndexGain[12] = { + 3, 6, 4, 7, 5, 10, 6, 16, 11, 28, 26, 69}; + +const uint16_t WebRtcIsac_kQKltInitIndexShape[108] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 2, 2, 3, 3, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, + 1, 1, 2, 2, 3, 4, 0, 0, 0, 0, + 1, 0, 1, 1, 1, 1, 2, 1, 1, 2, + 2, 3, 3, 5, 0, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 2, 4, 4, + 5, 7, 0, 0, 1, 1, 1, 1, 1, 1, + 2, 2, 3, 2, 3, 4, 4, 5, 8, 13, + 0, 1, 1, 2, 3, 2, 2, 2, 4, 4, + 5, 5, 7, 10, 9, 12, 13, 25 }; + + +/* quantizer representation levels */ +const double WebRtcIsac_kQKltLevelsGain[392] = { + -2.78127126, -1.76745590, -0.77913790, -0.00437329, 0.79961206, + 1.81775776, 2.81389782, -5.78753143, -4.88384084, -3.89320940, + -2.88133610, -1.92859977, -0.86347396, 0.02003888, 0.86140400, + 1.89667156, 2.97134967, 3.98781964, 4.91727277, 5.82865898, + -4.11195874, -2.80898424, -1.87547977, -0.80943825, -0.00679084, + 0.79573851, 1.83953397, 2.67586037, 3.76274082, -6.10933968, + -4.93034581, -3.89281296, -2.91530625, -1.89684163, -0.85319130, + -0.02275767, 0.86862017, 1.91578276, 2.96107339, 3.96543056, + 4.91369908, 5.91058154, 6.83848343, 8.07136925, -5.87470395, + -4.84703049, -3.84284597, -2.86168446, -1.89290192, -0.82798145, + -0.00080013, 0.82594974, 1.85754329, 2.88351798, 3.96172628, + -8.85684885, -7.87387461, -6.97811862, -5.93256270, -4.94301439, + -3.95513701, -2.96041544, -1.94031192, -0.87961478, -0.00456201, + 0.89911505, 1.91723376, 2.94011511, 3.93302540, 4.97990967, + 5.93133404, 7.02181199, 7.92407762, 8.80155440, 10.04665814, + -4.82396678, -3.85612158, -2.89482244, -1.89558408, -0.90036978, + -0.00677823, 0.90607989, 1.90937981, 2.91175777, 3.91637730, + 4.97565723, 5.84771228, 7.11145863, -16.07879840, -15.03776309, + -13.93905670, -12.95671800, -11.89171202, -10.95820934, -9.95923714, + -8.94357334, -7.99068299, -6.97481009, -5.94826231, -4.96673988, + -3.97490466, -2.97846970, -1.95130435, -0.94215262, -0.01444043, + 0.96770704, 1.95848598, 2.94107862, 3.95666119, 4.97253085, + 5.97191122, 6.93277360, 7.96608727, 8.87958779, 10.00264269, + 10.86560820, 12.07449071, 13.04491775, 13.97507061, 14.91845261, + -10.85696295, -9.83365357, -9.01245635, -7.95915145, -6.95625003, + -5.95362618, -4.93468444, -3.98760978, -2.95044407, -1.97041277, + -0.97701799, -0.00840234, 0.97834289, 1.98361415, 2.97802439, + 3.96415871, 4.95369042, 5.94101770, 6.92756798, 7.94063998, + 8.85951828, 9.97077022, 11.00068503, -33.92030406, -32.81426422, + -32.00000000, -31.13243639, -30.11886909, -29.06017570, -28.12598824, + -27.22045482, -25.81215858, -25.07849962, -23.93018013, -23.02097643, + -21.89529725, -20.99091085, -19.98889048, -18.94327044, -17.96562071, + -16.96126218, -15.95054062, -14.98516200, -13.97101012, -13.02106500, + -11.98438006, -11.03216748, -9.95930286, -8.97043946, -7.98085082, + -6.98360995, -5.98998802, -4.98668173, -4.00032906, -3.00420619, + -1.98701132, -0.99324682, -0.00609324, 0.98297834, 1.99483076, + 3.00305044, 3.97142097, 4.97525759, 5.98612258, 6.97448236, + 7.97575900, 9.01086211, 9.98665542, 11.00541438, 11.98078628, + 12.92352471, 14.06849675, 14.99949430, 15.94904834, 16.97440321, + 18.04040916, 18.88987609, 20.05312391, 21.00000000, 21.79443341, + -31.98578825, -31.00000000, -29.89060567, -28.98555686, -27.97114102, + -26.84935410, -26.02402230, -24.94195278, -23.92336849, -22.95552382, + -21.97932836, -20.96055470, -19.99649553, -19.03436122, -17.96706525, + -17.01139515, -16.01363516, -14.99154248, -14.00298333, -12.99630613, + -11.99955519, -10.99000421, -10.00819092, -8.99763648, -7.98431793, + -7.01769025, -5.99604690, -4.99980697, -3.99334671, -3.01748192, + -2.02051217, -1.00848371, -0.01942358, 1.00477757, 1.95477872, + 2.98593031, 3.98779079, 4.96862849, 6.02694771, 6.93983733, + 7.89874717, 8.99615862, 10.02367921, 10.96293452, 11.84351528, + 12.92207187, 13.85122329, 15.05146877, 15.99371264, 17.00000000, + 18.00000000, 19.00000000, 19.82763573, -47.00000000, -46.00000000, + -44.87138498, -44.00000000, -43.00000000, -42.00000000, -41.00000000, + -39.88966612, -38.98913239, -37.80306486, -37.23584325, -35.94200288, + -34.99881301, -34.11361858, -33.06507360, -32.13129135, -30.90891364, + -29.81511907, -28.99250380, -28.04535391, -26.99767800, -26.04418164, + -24.95687851, -24.04865595, -23.03392645, -21.89366707, -20.93517364, + -19.99388660, -18.91620943, -18.03749683, -16.99532379, -15.98683813, + -15.06421479, -13.99359211, -12.99714098, -11.97022520, -10.98500279, + -9.98834422, -8.95729330, -8.01232284, -7.00253661, -5.99681626, + -5.01207817, -3.95914904, -3.01232178, -1.96615919, -0.97687670, + 0.01228030, 0.98412288, 2.01753544, 3.00580570, 3.97783510, + 4.98846894, 6.01321400, 7.00867732, 8.00416375, 9.01771966, + 9.98637729, 10.98255180, 11.99194163, 13.01807333, 14.00999545, + 15.00118556, 16.00089224, 17.00584148, 17.98251763, 18.99942091, + 19.96917690, 20.97839265, 21.98207297, 23.00171271, 23.99930737, + 24.99746061, 26.00936304, 26.98240132, 28.01126868, 29.01395915, + 29.98153507, 31.01376711, 31.99876818, 33.00475317, 33.99753994, + 34.99493913, 35.98933585, 36.95620160, 37.98428461, 38.99317544, + 40.01832073, 40.98048133, 41.95999283, 42.98232091, 43.96523612, + 44.99574268, 45.99524194, 47.05464025, 48.03821548, 48.99354366, + 49.96400411, 50.98017973, 51.95184408, 52.96291806, 54.00194392, + 54.96603783, 55.95623778, 57.03076595, 58.05889901, 58.99081551, + 59.97928121, 61.05071612, 62.03971580, 63.01286038, 64.01290338, + 65.02074503, 65.99454594, 67.00399425, 67.96571257, 68.95305727, + 69.92030664, 70.95594862, 71.98088567, 73.04764124, 74.00285480, + 75.02696330, 75.89837673, 76.93459997, 78.16266309, 78.83317543, + 80.00000000, 80.87251574, 82.09803524, 83.10671664, 84.00000000, + 84.77023523, 86.00000000, 87.00000000, 87.92946897, 88.69159118, + 90.00000000, 90.90535270 }; + +const double WebRtcIsac_kQKltLevelsShape[578] = { + 0.00032397, 0.00008053, -0.00061202, -0.00012620, 0.00030437, + 0.00054764, -0.00027902, 0.00069360, 0.00029449, -0.80219239, + 0.00091089, -0.74514927, -0.00094283, 0.64030631, -0.60509119, + 0.00035575, 0.61851665, -0.62129957, 0.00375219, 0.60054900, + -0.61554359, 0.00054977, 0.63362016, -1.73118727, -0.65422341, + 0.00524568, 0.66165298, 1.76785515, -1.83182018, -0.65997434, + -0.00011887, 0.67524299, 1.79933938, -1.76344480, -0.72547708, + -0.00133017, 0.73104704, 1.75305377, 2.85164534, -2.80423916, + -1.71959639, -0.75419722, -0.00329945, 0.77196760, 1.72211069, + 2.87339653, 0.00031089, -0.00015311, 0.00018201, -0.00035035, + -0.77357251, 0.00154647, -0.00047625, -0.00045299, 0.00086590, + 0.00044762, -0.83383829, 0.00024787, -0.68526258, -0.00122472, + 0.64643255, -0.60904942, -0.00448987, 0.62309184, -0.59626442, + -0.00574132, 0.62296546, -0.63222115, 0.00013441, 0.63609545, + -0.66911055, -0.00369971, 0.66346095, 2.07281301, -1.77184694, + -0.67640425, -0.00010145, 0.64818392, 1.74948973, -1.69420224, + -0.71943894, -0.00004680, 0.75303493, 1.81075983, 2.80610041, + -2.80005755, -1.79866753, -0.77409777, -0.00084220, 0.80141293, + 1.78291081, 2.73954236, 3.82994169, 0.00015140, -0.00012766, + -0.00034241, -0.00119125, -0.76113497, 0.00069246, 0.76722027, + 0.00132862, -0.69107530, 0.00010656, 0.77061578, -0.78012970, + 0.00095947, 0.77828502, -0.64787758, 0.00217168, 0.63050167, + -0.58601125, 0.00306596, 0.59466308, -0.58603410, 0.00059779, + 0.64257970, 1.76512766, -0.61193600, -0.00259517, 0.59767574, + -0.61026273, 0.00315811, 0.61725479, -1.69169719, -0.65816029, + 0.00067575, 0.65576890, 2.00000000, -1.72689193, -0.69780808, + -0.00040990, 0.70668487, 1.74198458, -3.79028154, -3.00000000, + -1.73194459, -0.70179341, -0.00106695, 0.71302629, 1.76849782, + -2.89332364, -1.78585007, -0.78731491, -0.00132610, 0.79692976, + 1.75247009, 2.97828682, -5.26238694, -3.69559829, -2.87286122, + -1.84908818, -0.84434577, -0.01167975, 0.84641753, 1.84087672, + 2.87628156, 3.83556679, -0.00190204, 0.00092642, 0.00354385, + -0.00012982, -0.67742785, 0.00229509, 0.64935672, -0.58444751, + 0.00470733, 0.57299534, -0.58456202, -0.00097715, 0.64593607, + -0.64060330, -0.00638534, 0.59680157, -0.59287537, 0.00490772, + 0.58919707, -0.60306173, -0.00417464, 0.60562100, -1.75218757, + -0.63018569, -0.00225922, 0.63863300, -0.63949939, -0.00126421, + 0.64268914, -1.75851182, -0.68318060, 0.00510418, 0.69049211, + 1.88178506, -1.71136148, -0.72710534, -0.00815559, 0.73412917, + 1.79996711, -2.77111145, -1.73940498, -0.78212945, 0.01074476, + 0.77688916, 1.76873972, 2.87281379, 3.77554698, -3.75832725, + -2.95463235, -1.80451491, -0.80017226, 0.00149902, 0.80729206, + 1.78265046, 2.89391793, -3.78236148, -2.83640598, -1.82532067, + -0.88844327, -0.00620952, 0.88208030, 1.85757631, 2.81712391, + 3.88430176, 5.16179367, -7.00000000, -5.93805408, -4.87172597, + -3.87524433, -2.89399744, -1.92359563, -0.92136341, -0.00172725, + 0.93087018, 1.90528280, 2.89809686, 3.88085708, 4.89147740, + 5.89078692, -0.00239502, 0.00312564, -1.00000000, 0.00178325, + 1.00000000, -0.62198029, 0.00143254, 0.65344051, -0.59851220, + -0.00676987, 0.61510140, -0.58894151, 0.00385055, 0.59794203, + -0.59808568, -0.00038214, 0.57625703, -0.63009713, -0.01107985, + 0.61278758, -0.64206758, -0.00154369, 0.65480598, 1.80604162, + -1.80909286, -0.67810514, 0.00205762, 0.68571097, 1.79453891, + -3.22682422, -1.73808453, -0.71870305, -0.00738594, 0.71486172, + 1.73005326, -1.66891897, -0.73689615, -0.00616203, 0.74262409, + 1.73807899, -2.92417482, -1.73866741, -0.78133871, 0.00764425, + 0.80027264, 1.78668732, 2.74992588, -4.00000000, -2.75578740, + -1.83697516, -0.83117035, -0.00355191, 0.83527172, 1.82814700, + 2.77377675, 3.80718693, -3.81667698, -2.83575471, -1.83372350, + -0.86579471, 0.00547578, 0.87582281, 1.82858793, 2.87265007, + 3.91405377, -4.87521600, -3.78999094, -2.86437014, -1.86964365, + -0.90618018, 0.00128243, 0.91497811, 1.87374952, 2.83199819, + 3.91519130, 4.76632822, -6.68713448, -6.01252467, -4.94587936, + -3.88795368, -2.91299088, -1.92592211, -0.95504570, -0.00089980, + 0.94565200, 1.93239633, 2.91832808, 3.91363475, 4.88920034, + 5.96471415, 6.83905252, 7.86195009, 8.81571018,-12.96141759, + -11.73039516,-10.96459719, -9.97382433, -9.04414433, -7.89460619, + -6.96628608, -5.93236595, -4.93337924, -3.95479990, -2.96451499, + -1.96635876, -0.97271229, -0.00402238, 0.98343930, 1.98348291, + 2.96641164, 3.95456471, 4.95517089, 5.98975714, 6.90322073, + 7.90468849, 8.85639467, 9.97255498, 10.79006309, 11.81988596, + 0.04950500, -1.00000000, -0.01226628, 1.00000000, -0.59479469, + -0.10438305, 0.59822144, -2.00000000, -0.67109149, -0.09256692, + 0.65171621, 2.00000000, -3.00000000, -1.68391999, -0.76681039, + -0.03354151, 0.71509146, 1.77615472, -2.00000000, -0.68661511, + -0.02497881, 0.66478398, 2.00000000, -2.00000000, -0.67032784, + -0.00920582, 0.64892756, 2.00000000, -2.00000000, -0.68561894, + 0.03641869, 0.73021611, 1.68293863, -4.00000000, -2.72024184, + -1.80096059, -0.81696185, 0.03604685, 0.79232033, 1.70070730, + 3.00000000, -4.00000000, -2.71795670, -1.80482986, -0.86001162, + 0.03764903, 0.87723968, 1.79970771, 2.72685932, 3.67589143, + -5.00000000, -4.00000000, -2.85492548, -1.78996365, -0.83250358, + -0.01376828, 0.84195506, 1.78161105, 2.76754458, 4.00000000, + -6.00000000, -5.00000000, -3.82268811, -2.77563624, -1.82608163, + -0.86486114, -0.02671886, 0.86693165, 1.88422879, 2.86248347, + 3.95632216, -7.00000000, -6.00000000, -5.00000000, -3.77533988, + -2.86391432, -1.87052039, -0.90513658, 0.06271236, 0.91083620, + 1.85734756, 2.86031688, 3.82019418, 4.94420394, 6.00000000, + -11.00000000,-10.00000000, -9.00000000, -8.00000000, -6.91952415, + -6.00000000, -4.92044374, -3.87845165, -2.87392362, -1.88413020, + -0.91915740, 0.00318517, 0.91602800, 1.89664838, 2.88925058, + 3.84123856, 4.78988651, 5.94526812, 6.81953917, 8.00000000, + -9.00000000, -8.00000000, -7.03319143, -5.94530963, -4.86669720, + -3.92438007, -2.88620396, -1.92848070, -0.94365985, 0.01671855, + 0.97349410, 1.93419878, 2.89740109, 3.89662823, 4.83235583, + 5.88106535, 6.80328232, 8.00000000,-13.00000000,-12.00000000, + -11.00000000,-10.00000000, -9.00000000, -7.86033489, -6.83344055, + -5.89844215, -4.90811454, -3.94841298, -2.95820490, -1.98627966, + -0.99161468, -0.02286136, 0.96055651, 1.95052433, 2.93969396, + 3.94304346, 4.88522624, 5.87434241, 6.78309433, 7.87244101, + 9.00000000, 10.00000000,-12.09117356,-11.00000000,-10.00000000, + -8.84766108, -7.86934236, -6.98544896, -5.94233429, -4.95583292, + -3.95575986, -2.97085529, -1.98955811, -0.99359873, -0.00485413, + 0.98298870, 1.98093258, 2.96430203, 3.95540216, 4.96915010, + 5.96775124, 6.99236918, 7.96503302, 8.99864542, 9.85857723, + 10.96541926, 11.91647197, 12.71060069,-26.00000000,-25.00000000, + -24.00585596,-23.11642573,-22.14271284,-20.89800711,-19.87815799, + -19.05036354,-17.88555651,-16.86471209,-15.97711073,-14.94012359, + -14.02661226,-12.98243228,-11.97489256,-10.97402777, -9.96425624, + -9.01085220, -7.97372506, -6.98795002, -5.97271328, -5.00191694, + -3.98055849, -2.98458048, -1.99470442, -0.99656768, -0.00825666, + 1.00272004, 1.99922218, 2.99357669, 4.01407905, 5.01003897, + 5.98115528, 7.00018958, 8.00338125, 8.98981046, 9.98990318, + 10.96341479, 11.96866930, 12.99175139, 13.94580443, 14.95745083, + 15.98992869, 16.97484646, 17.99630043, 18.93396897, 19.88347741, + 20.96532482, 21.92191032, 23.22314702 }; + + +/* cdf tables for quantizer indices */ +const uint16_t WebRtcIsac_kQKltCdfGain[404] = { + 0, 13, 301, 3730, 61784, 65167, 65489, 65535, 0, 17, + 142, 314, 929, 2466, 7678, 56450, 63463, 64740, 65204, 65426, + 65527, 65535, 0, 8, 100, 724, 6301, 60105, 65125, 65510, + 65531, 65535, 0, 13, 117, 368, 1068, 3010, 11928, 53603, + 61177, 63404, 64505, 65108, 65422, 65502, 65531, 65535, 0, 4, + 17, 96, 410, 1859, 12125, 54361, 64103, 65305, 65497, 65535, + 0, 4, 88, 230, 469, 950, 1746, 3228, 6092, 16592, + 44756, 56848, 61256, 63308, 64325, 64920, 65309, 65460, 65502, + 65522, 65535, 0, 88, 352, 1675, 6339, 20749, 46686, 59284, 63525, + 64949, 65359, 65502, 65527, 65535, 0, 13, 38, 63, 117, + 234, 381, 641, 929, 1407, 2043, 2809, 4032, 5753, 8792, + 14407, 24308, 38941, 48947, 55403, 59293, 61411, 62688, 63630, + 64329, 64840, 65188, 65376, 65472, 65506, 65527, 65531, 65535, + 0, 8, 29, 75, 222, 615, 1327, 2801, 5623, 9931, 16094, 24966, + 34419, 43458, 50676, 56186, 60055, 62500, 63936, 64765, 65225, + 65435, 65514, 65535, 0, 8, 13, 15, 17, 21, 33, 59, + 71, 92, 151, 243, 360, 456, 674, 934, 1223, 1583, + 1989, 2504, 3031, 3617, 4354, 5154, 6163, 7411, 8780, 10747, + 12874, 15591, 18974, 23027, 27436, 32020, 36948, 41830, 46205, + 49797, 53042, 56094, 58418, 60360, 61763, 62818, 63559, 64103, + 64509, 64798, 65045, 65162, 65288, 65363, 65447, 65506, 65522, + 65531, 65533, 65535, 0, 4, 6, 25, 38, 71, 138, 264, 519, 808, + 1227, 1825, 2516, 3408, 4279, 5560, 7092, 9197, 11420, 14108, + 16947, 20300, 23926, 27459, 31164, 34827, 38575, 42178, 45540, + 48747, 51444, 54090, 56426, 58460, 60080, 61595, 62734, 63668, + 64275, 64673, 64936, 65112, 65217, 65334, 65426, 65464, 65477, + 65489, 65518, 65527, 65529, 65531, 65533, 65535, 0, 2, 4, 8, 10, + 12, 14, 16, 21, 33, 50, 71, 84, 92, 105, 138, 180, 255, 318, + 377, 435, 473, 511, 590, 682, 758, 913, 1097, 1256, 1449, 1671, + 1884, 2169, 2445, 2772, 3157, 3563, 3944, 4375, 4848, 5334, 5820, + 6448, 7101, 7716, 8378, 9102, 9956, 10752, 11648, 12707, 13670, + 14758, 15910, 17187, 18472, 19627, 20649, 21951, 23169, 24283, + 25552, 26862, 28227, 29391, 30764, 31882, 33213, 34432, 35600, + 36910, 38116, 39464, 40729, 41872, 43144, 44371, 45514, 46762, + 47813, 48968, 50069, 51032, 51974, 52908, 53737, 54603, 55445, + 56282, 56990, 57572, 58191, 58840, 59410, 59887, 60264, 60607, + 60946, 61269, 61516, 61771, 61960, 62198, 62408, 62558, 62776, + 62985, 63207, 63408, 63546, 63739, 63906, 64070, 64237, 64371, + 64551, 64677, 64836, 64999, 65095, 65213, 65284, 65338, 65380, + 65426, 65447, 65472, 65485, 65487, 65489, 65502, 65510, 65512, + 65514, 65516, 65518, 65522, 65531, 65533, 65535 }; + + +const uint16_t WebRtcIsac_kQKltCdfShape[686] = { + 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65535, + 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, 4, + 65535, 0, 8, 65514, 65535, 0, 29, 65481, 65535, 0, + 121, 65439, 65535, 0, 239, 65284, 65535, 0, 8, 779, + 64999, 65527, 65535, 0, 8, 888, 64693, 65522, 65535, 0, + 29, 2604, 62843, 65497, 65531, 65535, 0, 25, 176, 4576, + 61164, 65275, 65527, 65535, 0, 65535, 0, 65535, 0, 65535, + 0, 65535, 0, 4, 65535, 0, 65535, 0, 65535, 0, + 65535, 0, 65535, 0, 4, 65535, 0, 33, 65502, 65535, + 0, 54, 65481, 65535, 0, 251, 65309, 65535, 0, 611, + 65074, 65535, 0, 1273, 64292, 65527, 65535, 0, 4, 1809, + 63940, 65518, 65535, 0, 88, 4392, 60603, 65426, 65531, 65535, + 0, 25, 419, 7046, 57756, 64961, 65514, 65531, 65535, 0, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 4, 65531, + 65535, 0, 65535, 0, 8, 65531, 65535, 0, 4, 65527, + 65535, 0, 17, 65510, 65535, 0, 42, 65481, 65535, 0, + 197, 65342, 65531, 65535, 0, 385, 65154, 65535, 0, 1005, + 64522, 65535, 0, 8, 1985, 63469, 65533, 65535, 0, 38, + 3119, 61884, 65514, 65535, 0, 4, 6, 67, 4961, 60804, + 65472, 65535, 0, 17, 565, 9182, 56538, 65087, 65514, 65535, + 0, 8, 63, 327, 2118, 14490, 52774, 63839, 65376, 65522, + 65535, 0, 65535, 0, 65535, 0, 65535, 0, 65535, 0, + 17, 65522, 65535, 0, 59, 65489, 65535, 0, 50, 65522, + 65535, 0, 54, 65489, 65535, 0, 310, 65179, 65535, 0, + 615, 64836, 65535, 0, 4, 1503, 63965, 65535, 0, 2780, + 63383, 65535, 0, 21, 3919, 61051, 65527, 65535, 0, 84, + 6674, 59929, 65435, 65535, 0, 4, 255, 7976, 55784, 65150, + 65518, 65531, 65535, 0, 4, 8, 582, 10726, 53465, 64949, + 65518, 65535, 0, 29, 339, 3006, 17555, 49517, 62956, 65200, + 65497, 65531, 65535, 0, 2, 33, 138, 565, 2324, 7670, + 22089, 45966, 58949, 63479, 64966, 65380, 65518, 65535, 0, 65535, + 0, 65535, 0, 2, 65533, 65535, 0, 46, 65514, 65535, + 0, 414, 65091, 65535, 0, 540, 64911, 65535, 0, 419, + 65162, 65535, 0, 976, 64790, 65535, 0, 2977, 62495, 65531, + 65535, 0, 4, 3852, 61034, 65527, 65535, 0, 4, 29, + 6021, 60243, 65468, 65535, 0, 84, 6711, 58066, 65418, 65535, + 0, 13, 281, 9550, 54917, 65125, 65506, 65535, 0, 2, + 63, 984, 12108, 52644, 64342, 65435, 65527, 65535, 0, 29, + 251, 2014, 14871, 47553, 62881, 65229, 65518, 65535, 0, 13, + 142, 749, 4220, 18497, 45200, 60913, 64823, 65426, 65527, 65535, + 0, 13, 71, 264, 1176, 3789, 10500, 24480, 43488, 56324, + 62315, 64493, 65242, 65464, 65514, 65522, 65531, 65535, 0, 4, + 13, 38, 109, 205, 448, 850, 1708, 3429, 6276, 11371, + 19221, 29734, 40955, 49391, 55411, 59460, 62102, 63793, 64656, + 65150, 65401, 65485, 65522, 65531, 65535, 0, 65535, 0, 2, 65533, + 65535, 0, 1160, 65476, 65535, 0, 2, 6640, 64763, 65533, + 65535, 0, 2, 38, 9923, 61009, 65527, 65535, 0, 2, + 4949, 63092, 65533, 65535, 0, 2, 3090, 63398, 65533, 65535, + 0, 2, 2520, 58744, 65510, 65535, 0, 2, 13, 544, + 8784, 51403, 65148, 65533, 65535, 0, 2, 25, 1017, 10412, + 43550, 63651, 65489, 65527, 65535, 0, 2, 4, 29, 783, + 13377, 52462, 64524, 65495, 65533, 65535, 0, 2, 4, 6, + 100, 1817, 18451, 52590, 63559, 65376, 65531, 65535, 0, 2, + 4, 6, 46, 385, 2562, 11225, 37416, 60488, 65026, 65487, + 65529, 65533, 65535, 0, 2, 4, 6, 8, 10, 12, + 42, 222, 971, 5221, 19811, 45048, 60312, 64486, 65294, 65474, + 65525, 65529, 65533, 65535, 0, 2, 4, 8, 71, 167, + 666, 2533, 7875, 19622, 38082, 54359, 62108, 64633, 65290, 65495, + 65529, 65533, 65535, 0, 2, 4, 6, 8, 10, 13, + 109, 586, 1930, 4949, 11600, 22641, 36125, 48312, 56899, 61495, + 63927, 64932, 65389, 65489, 65518, 65531, 65533, 65535, 0, 4, + 6, 8, 67, 209, 712, 1838, 4195, 8432, 14432, 22834, + 31723, 40523, 48139, 53929, 57865, 60657, 62403, 63584, 64363, + 64907, 65167, 65372, 65472, 65514, 65535, 0, 2, 4, 13, 25, + 42, 46, 50, 75, 113, 147, 281, 448, 657, 909, + 1185, 1591, 1976, 2600, 3676, 5317, 7398, 9914, 12941, 16169, + 19477, 22885, 26464, 29851, 33360, 37228, 41139, 44802, 48654, + 52058, 55181, 57676, 59581, 61022, 62190, 63107, 63676, 64199, + 64547, 64924, 65158, 65313, 65430, 65481, 65518, 65535 }; + + +/* pointers to cdf tables for quantizer indices */ +const uint16_t *WebRtcIsac_kQKltCdfPtrGain[12] = { + WebRtcIsac_kQKltCdfGain +0 +0, WebRtcIsac_kQKltCdfGain +0 +8, + WebRtcIsac_kQKltCdfGain +0 +22, WebRtcIsac_kQKltCdfGain +0 +32, + WebRtcIsac_kQKltCdfGain +0 +48, WebRtcIsac_kQKltCdfGain +0 +60, + WebRtcIsac_kQKltCdfGain +0 +81, WebRtcIsac_kQKltCdfGain +0 +95, + WebRtcIsac_kQKltCdfGain +0 +128, WebRtcIsac_kQKltCdfGain +0 +152, + WebRtcIsac_kQKltCdfGain +0 +210, WebRtcIsac_kQKltCdfGain +0 +264 }; + +const uint16_t *WebRtcIsac_kQKltCdfPtrShape[108] = { + WebRtcIsac_kQKltCdfShape +0 +0, WebRtcIsac_kQKltCdfShape +0 +2, + WebRtcIsac_kQKltCdfShape +0 +4, WebRtcIsac_kQKltCdfShape +0 +6, + WebRtcIsac_kQKltCdfShape +0 +8, WebRtcIsac_kQKltCdfShape +0 +10, + WebRtcIsac_kQKltCdfShape +0 +12, WebRtcIsac_kQKltCdfShape +0 +14, + WebRtcIsac_kQKltCdfShape +0 +16, WebRtcIsac_kQKltCdfShape +0 +18, + WebRtcIsac_kQKltCdfShape +0 +21, WebRtcIsac_kQKltCdfShape +0 +25, + WebRtcIsac_kQKltCdfShape +0 +29, WebRtcIsac_kQKltCdfShape +0 +33, + WebRtcIsac_kQKltCdfShape +0 +37, WebRtcIsac_kQKltCdfShape +0 +43, + WebRtcIsac_kQKltCdfShape +0 +49, WebRtcIsac_kQKltCdfShape +0 +56, + WebRtcIsac_kQKltCdfShape +0 +64, WebRtcIsac_kQKltCdfShape +0 +66, + WebRtcIsac_kQKltCdfShape +0 +68, WebRtcIsac_kQKltCdfShape +0 +70, + WebRtcIsac_kQKltCdfShape +0 +72, WebRtcIsac_kQKltCdfShape +0 +75, + WebRtcIsac_kQKltCdfShape +0 +77, WebRtcIsac_kQKltCdfShape +0 +79, + WebRtcIsac_kQKltCdfShape +0 +81, WebRtcIsac_kQKltCdfShape +0 +83, + WebRtcIsac_kQKltCdfShape +0 +86, WebRtcIsac_kQKltCdfShape +0 +90, + WebRtcIsac_kQKltCdfShape +0 +94, WebRtcIsac_kQKltCdfShape +0 +98, + WebRtcIsac_kQKltCdfShape +0 +102, WebRtcIsac_kQKltCdfShape +0 +107, + WebRtcIsac_kQKltCdfShape +0 +113, WebRtcIsac_kQKltCdfShape +0 +120, + WebRtcIsac_kQKltCdfShape +0 +129, WebRtcIsac_kQKltCdfShape +0 +131, + WebRtcIsac_kQKltCdfShape +0 +133, WebRtcIsac_kQKltCdfShape +0 +135, + WebRtcIsac_kQKltCdfShape +0 +137, WebRtcIsac_kQKltCdfShape +0 +141, + WebRtcIsac_kQKltCdfShape +0 +143, WebRtcIsac_kQKltCdfShape +0 +147, + WebRtcIsac_kQKltCdfShape +0 +151, WebRtcIsac_kQKltCdfShape +0 +155, + WebRtcIsac_kQKltCdfShape +0 +159, WebRtcIsac_kQKltCdfShape +0 +164, + WebRtcIsac_kQKltCdfShape +0 +168, WebRtcIsac_kQKltCdfShape +0 +172, + WebRtcIsac_kQKltCdfShape +0 +178, WebRtcIsac_kQKltCdfShape +0 +184, + WebRtcIsac_kQKltCdfShape +0 +192, WebRtcIsac_kQKltCdfShape +0 +200, + WebRtcIsac_kQKltCdfShape +0 +211, WebRtcIsac_kQKltCdfShape +0 +213, + WebRtcIsac_kQKltCdfShape +0 +215, WebRtcIsac_kQKltCdfShape +0 +217, + WebRtcIsac_kQKltCdfShape +0 +219, WebRtcIsac_kQKltCdfShape +0 +223, + WebRtcIsac_kQKltCdfShape +0 +227, WebRtcIsac_kQKltCdfShape +0 +231, + WebRtcIsac_kQKltCdfShape +0 +235, WebRtcIsac_kQKltCdfShape +0 +239, + WebRtcIsac_kQKltCdfShape +0 +243, WebRtcIsac_kQKltCdfShape +0 +248, + WebRtcIsac_kQKltCdfShape +0 +252, WebRtcIsac_kQKltCdfShape +0 +258, + WebRtcIsac_kQKltCdfShape +0 +264, WebRtcIsac_kQKltCdfShape +0 +273, + WebRtcIsac_kQKltCdfShape +0 +282, WebRtcIsac_kQKltCdfShape +0 +293, + WebRtcIsac_kQKltCdfShape +0 +308, WebRtcIsac_kQKltCdfShape +0 +310, + WebRtcIsac_kQKltCdfShape +0 +312, WebRtcIsac_kQKltCdfShape +0 +316, + WebRtcIsac_kQKltCdfShape +0 +320, WebRtcIsac_kQKltCdfShape +0 +324, + WebRtcIsac_kQKltCdfShape +0 +328, WebRtcIsac_kQKltCdfShape +0 +332, + WebRtcIsac_kQKltCdfShape +0 +336, WebRtcIsac_kQKltCdfShape +0 +341, + WebRtcIsac_kQKltCdfShape +0 +347, WebRtcIsac_kQKltCdfShape +0 +354, + WebRtcIsac_kQKltCdfShape +0 +360, WebRtcIsac_kQKltCdfShape +0 +368, + WebRtcIsac_kQKltCdfShape +0 +378, WebRtcIsac_kQKltCdfShape +0 +388, + WebRtcIsac_kQKltCdfShape +0 +400, WebRtcIsac_kQKltCdfShape +0 +418, + WebRtcIsac_kQKltCdfShape +0 +445, WebRtcIsac_kQKltCdfShape +0 +447, + WebRtcIsac_kQKltCdfShape +0 +451, WebRtcIsac_kQKltCdfShape +0 +455, + WebRtcIsac_kQKltCdfShape +0 +461, WebRtcIsac_kQKltCdfShape +0 +468, + WebRtcIsac_kQKltCdfShape +0 +474, WebRtcIsac_kQKltCdfShape +0 +480, + WebRtcIsac_kQKltCdfShape +0 +486, WebRtcIsac_kQKltCdfShape +0 +495, + WebRtcIsac_kQKltCdfShape +0 +505, WebRtcIsac_kQKltCdfShape +0 +516, + WebRtcIsac_kQKltCdfShape +0 +528, WebRtcIsac_kQKltCdfShape +0 +543, + WebRtcIsac_kQKltCdfShape +0 +564, WebRtcIsac_kQKltCdfShape +0 +583, + WebRtcIsac_kQKltCdfShape +0 +608, WebRtcIsac_kQKltCdfShape +0 +635 }; + + +/* left KLT transforms */ +const double WebRtcIsac_kKltT1Gain[4] = { + -0.79742827, 0.60341375, 0.60341375, 0.79742827 }; + +const double WebRtcIsac_kKltT1Shape[324] = { + 0.00159597, 0.00049320, 0.00513821, 0.00021066, 0.01338581, + -0.00422367, -0.00272072, 0.00935107, 0.02047622, 0.02691189, + 0.00478236, 0.03969702, 0.00886698, 0.04877604, -0.10898362, + -0.05930891, -0.03415047, 0.98889721, 0.00293558, -0.00035282, + 0.01156321, -0.00195341, -0.00937631, 0.01052213, -0.02551163, + 0.01644059, 0.03189927, 0.07754773, -0.08742313, -0.03026338, + 0.05136248, -0.14395974, 0.17725040, 0.22664856, 0.93380230, + 0.07076411, 0.00557890, -0.00222834, 0.01377569, 0.01466808, + 0.02847361, -0.00603178, 0.02382480, -0.01210452, 0.03797267, + -0.02371480, 0.11260335, -0.07366682, 0.00453436, -0.04136941, + -0.07912843, -0.95031418, 0.25295337, -0.05302216, -0.00617554, + -0.00044040, -0.00653778, 0.01097838, 0.01529174, 0.01374431, + -0.00748512, -0.00020034, 0.02432713, 0.11101570, -0.08556891, + 0.09282249, -0.01029446, 0.67556443, -0.67454300, 0.06910063, + 0.20866865, -0.10318050, 0.00932175, 0.00524058, 0.00803610, + -0.00594676, -0.01082578, 0.01069906, 0.00546768, 0.01565291, + 0.06816200, 0.10201227, 0.16812734, 0.22984074, 0.58213170, + -0.54138651, -0.51379962, 0.06847390, -0.01920037, -0.04592324, + -0.00467394, 0.00328858, 0.00377424, -0.00987448, 0.08222096, + -0.00377301, 0.04551941, -0.02592517, 0.16317082, 0.13077530, + 0.22702921, -0.31215289, -0.69645962, -0.38047101, -0.39339411, + 0.11124777, 0.02508035, -0.00708074, 0.00400344, 0.00040331, + 0.01142402, 0.01725406, 0.01635170, 0.14285366, 0.03949233, + -0.05905676, 0.05877154, -0.17497577, -0.32479440, 0.80754464, + -0.38085603, -0.17055430, -0.03168622, -0.07531451, 0.02942002, + -0.02148095, -0.00754114, -0.00322372, 0.00567812, -0.01701521, + -0.12358320, 0.11473564, 0.09070136, 0.06533068, -0.22560802, + 0.19209022, 0.81605094, 0.36592275, -0.09919829, 0.16667122, + 0.16300725, 0.04803807, 0.06739263, -0.00156752, -0.01685302, + -0.00905240, -0.02297836, -0.00469939, 0.06310613, -0.16391930, + 0.10919511, 0.12529293, 0.85581322, -0.32145522, 0.24539076, + 0.07181839, 0.07289591, 0.14066759, 0.10406711, 0.05815518, + 0.01072680, -0.00759339, 0.00053486, -0.00044865, 0.03407361, + 0.01645348, 0.08758579, 0.27722240, 0.53665485, -0.74853376, + -0.01118192, -0.19805430, 0.06130619, -0.09675299, 0.08978480, + 0.03405255, -0.00706867, 0.05102045, 0.03250746, 0.01849966, + -0.01216314, -0.01184187, -0.01579288, 0.00114807, 0.11376166, + 0.88342114, -0.36425379, 0.13863190, 0.12524180, -0.13553892, + 0.04715856, -0.12341103, 0.04531568, 0.01899360, -0.00206897, + 0.00567768, -0.01444163, 0.00411946, -0.00855896, 0.00381663, + -0.01664861, -0.05534280, 0.21328278, 0.20161162, 0.72360394, + 0.59130708, -0.08043791, 0.08757349, -0.13893918, -0.05147377, + 0.02680690, -0.01144070, 0.00625162, -0.00634215, -0.01248947, + -0.00329455, -0.00609625, -0.00136305, -0.05097048, -0.01029851, + 0.25065384, -0.16856837, -0.07123372, 0.15992623, -0.39487617, + -0.79972301, 0.18118185, -0.04826639, -0.01805578, -0.02927253, + -0.16400618, 0.07472763, 0.10376449, 0.01705406, 0.01065801, + -0.01500498, 0.02039914, 0.37776349, -0.84484186, 0.10434286, + 0.15616990, 0.13474456, -0.00906238, -0.25238368, -0.03820885, + -0.10650905, -0.03880833, -0.03660028, -0.09640894, 0.00583314, + 0.01922097, 0.01489911, -0.02431117, -0.09372217, 0.39404721, + -0.84786223, -0.31277121, 0.03193850, 0.01974060, 0.01887901, + 0.00337911, -0.11359599, -0.02792521, -0.03220184, -0.01533311, + 0.00015962, -0.04225043, -0.00933965, 0.00675311, 0.00206060, + 0.15926771, 0.40199829, -0.80792558, -0.35591604, -0.17169764, + 0.02830436, 0.02459982, -0.03438589, 0.00718705, -0.01798329, + -0.01594508, -0.00702430, -0.00952419, -0.00962701, -0.01307212, + -0.01749740, 0.01299602, 0.00587270, -0.36103108, -0.82039266, + -0.43092844, -0.08500097, -0.04361674, -0.00333482, 0.01250434, + -0.02538295, -0.00921797, 0.01645071, -0.01400872, 0.00317607, + 0.00003277, -0.01617646, -0.00616863, -0.00882661, 0.00466157, + 0.00353237, 0.91803104, -0.39503305, -0.02048964, 0.00060125, + 0.01980634, 0.00300109, 0.00313880, 0.00657337, 0.00715163, + 0.00000261, 0.00854276, -0.00154825, -0.00516128, 0.00909527, + 0.00095609, 0.00701196, -0.00221867, -0.00156741 }; + +/* right KLT transforms */ +const double WebRtcIsac_kKltT2Gain[36] = { + 0.14572837, -0.45446306, 0.61990621, -0.52197033, 0.32145074, + -0.11026900, -0.20698282, 0.48962182, -0.27127933, -0.33627476, + 0.65094037, -0.32715751, 0.40262573, -0.47844405, -0.33876075, + 0.44130653, 0.37383966, -0.39964662, -0.51730480, 0.06611973, + 0.49030187, 0.47512886, -0.02141226, -0.51129451, -0.58578569, + -0.39132064, -0.13187771, 0.15649421, 0.40735596, 0.54396897, + 0.40381276, 0.40904942, 0.41179766, 0.41167576, 0.40840251, + 0.40468132 }; + +const double WebRtcIsac_kKltT2Shape[36] = { + 0.13427386, -0.35132558, 0.52506528, -0.59419077, 0.45075085, + -0.16312057, 0.29857439, -0.58660147, 0.34265431, 0.20879510, + -0.56063262, 0.30238345, 0.43308283, -0.41186999, -0.35288681, + 0.42768996, 0.36094634, -0.45284910, -0.47116680, 0.02893449, + 0.54326135, 0.45249040, -0.06264420, -0.52283830, 0.57137758, + 0.44298139, 0.12617554, -0.20819946, -0.42324603, -0.48876443, + 0.39597050, 0.40713935, 0.41389880, 0.41512486, 0.41130400, + 0.40575001 }; + +/* means of log gains and LAR coefficients*/ +const double WebRtcIsac_kLpcMeansGain[12] = { + -6.86881911, -5.35075273, -6.86792680, -5.36200897, -6.86401538, + -5.36921533, -6.86802969, -5.36893966, -6.86538097, -5.36315063, + -6.85535304, -5.35155315 }; + +const double WebRtcIsac_kLpcMeansShape[108] = { + -0.91232981, 0.26258634, -0.33716701, 0.08477430, -0.03378426, + 0.14423909, 0.07036185, 0.06155019, 0.01490385, 0.04138740, + 0.01427317, 0.01288970, 0.83872106, 0.25750199, 0.07988929, + -0.01957923, 0.00831390, 0.01770300, -0.90957164, 0.25732216, + -0.33385344, 0.08735740, -0.03715332, 0.14584917, 0.06998990, + 0.06131968, 0.01504379, 0.04067339, 0.01428039, 0.01406460, + 0.83846243, 0.26169862, 0.08109025, -0.01767055, 0.00970539, + 0.01954310, -0.90490803, 0.24656405, -0.33578607, 0.08843286, + -0.03749139, 0.14443959, 0.07214669, 0.06170993, 0.01449947, + 0.04134309, 0.01314762, 0.01413471, 0.83895203, 0.26748062, + 0.08197507, -0.01781298, 0.00885967, 0.01922394, -0.90922472, + 0.24495889, -0.33921540, 0.08877169, -0.03581332, 0.14199172, + 0.07444032, 0.06185940, 0.01502054, 0.04185113, 0.01276579, + 0.01355457, 0.83645358, 0.26631720, 0.08119697, -0.01835449, + 0.00788512, 0.01846446, -0.90482253, 0.24658310, -0.34019734, + 0.08281090, -0.03486038, 0.14359248, 0.07401336, 0.06001471, + 0.01528421, 0.04254560, 0.01321472, 0.01240799, 0.83857127, + 0.26281654, 0.08174380, -0.02099842, 0.00755176, 0.01699448, + -0.90132307, 0.25174308, -0.33838268, 0.07883863, -0.02877906, + 0.14105407, 0.07220290, 0.06000352, 0.01684879, 0.04226844, + 0.01331331, 0.01269244, 0.83832138, 0.25467485, 0.08118028, + -0.02120528, 0.00747832, 0.01567212 }; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.h new file mode 100644 index 0000000000..9412f2e389 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * lpc_tables.h + * + * header file for coding tables for the LPC coefficients + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_TABLES_H_ + +#include "structs.h" + +#include "settings.h" + +#define KLT_STEPSIZE 1.00000000 +#define KLT_NUM_AVG_GAIN 0 +#define KLT_NUM_AVG_SHAPE 0 +#define KLT_NUM_MODELS 3 +#define LPC_GAIN_SCALE 4.000f +#define LPC_LOBAND_SCALE 2.100f +#define LPC_LOBAND_ORDER ORDERLO +#define LPC_HIBAND_SCALE 0.450f +#define LPC_HIBAND_ORDER ORDERHI +#define LPC_GAIN_ORDER 2 + +#define LPC_SHAPE_ORDER (LPC_LOBAND_ORDER + LPC_HIBAND_ORDER) + +#define KLT_ORDER_GAIN (LPC_GAIN_ORDER * SUBFRAMES) +#define KLT_ORDER_SHAPE (LPC_SHAPE_ORDER * SUBFRAMES) + +/* cdf array for model indicator */ +extern const uint16_t WebRtcIsac_kQKltModelCdf[KLT_NUM_MODELS+1]; + +/* pointer to cdf array for model indicator */ +extern const uint16_t *WebRtcIsac_kQKltModelCdfPtr[1]; + +/* initial cdf index for decoder of model indicator */ +extern const uint16_t WebRtcIsac_kQKltModelInitIndex[1]; + +/* offset to go from rounded value to quantization index */ +extern const short WebRtcIsac_kQKltQuantMinGain[12]; + +extern const short WebRtcIsac_kQKltQuantMinShape[108]; + +/* maximum quantization index */ +extern const uint16_t WebRtcIsac_kQKltMaxIndGain[12]; + +extern const uint16_t WebRtcIsac_kQKltMaxIndShape[108]; + +/* index offset */ +extern const uint16_t WebRtcIsac_kQKltOffsetGain[12]; + +extern const uint16_t WebRtcIsac_kQKltOffsetShape[108]; + +/* initial cdf index for KLT coefficients */ +extern const uint16_t WebRtcIsac_kQKltInitIndexGain[12]; + +extern const uint16_t WebRtcIsac_kQKltInitIndexShape[108]; + +/* quantizer representation levels */ +extern const double WebRtcIsac_kQKltLevelsGain[392]; + +extern const double WebRtcIsac_kQKltLevelsShape[578]; + +/* cdf tables for quantizer indices */ +extern const uint16_t WebRtcIsac_kQKltCdfGain[404]; + +extern const uint16_t WebRtcIsac_kQKltCdfShape[686]; + +/* pointers to cdf tables for quantizer indices */ +extern const uint16_t *WebRtcIsac_kQKltCdfPtrGain[12]; + +extern const uint16_t *WebRtcIsac_kQKltCdfPtrShape[108]; + +/* left KLT transforms */ +extern const double WebRtcIsac_kKltT1Gain[4]; + +extern const double WebRtcIsac_kKltT1Shape[324]; + +/* right KLT transforms */ +extern const double WebRtcIsac_kKltT2Gain[36]; + +extern const double WebRtcIsac_kKltT2Shape[36]; + +/* means of log gains and LAR coefficients */ +extern const double WebRtcIsac_kLpcMeansGain[12]; + +extern const double WebRtcIsac_kLpcMeansShape[108]; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_LPC_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h new file mode 100644 index 0000000000..597dc21d1c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ + +#include +#include "typedefs.h" // NOLINT(build/include) + +#if defined(WEBRTC_POSIX) +#define WebRtcIsac_lrint lrint +#elif (defined(WEBRTC_ARCH_X86) && defined(WIN32)) +static __inline long int WebRtcIsac_lrint(double x_dbl) { + long int x_int; + + __asm { + fld x_dbl + fistp x_int + }; + + return x_int; +} +#else // Do a slow but correct implementation of lrint + +static __inline long int WebRtcIsac_lrint(double x_dbl) { + long int x_int; + x_int = (long int)floor(x_dbl + 0.499999999999); + return x_int; +} + +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c new file mode 100644 index 0000000000..090b94ca00 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c @@ -0,0 +1,623 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "pitch_estimator.h" + +#include +#include +#include +#ifdef WEBRTC_ANDROID +#include +#endif + +static const double kInterpolWin[8] = {-0.00067556028640, 0.02184247643159, -0.12203175715679, 0.60086484101160, + 0.60086484101160, -0.12203175715679, 0.02184247643159, -0.00067556028640}; + +/* interpolation filter */ +__inline static void IntrepolFilter(double *data_ptr, double *intrp) +{ + *intrp = kInterpolWin[0] * data_ptr[-3]; + *intrp += kInterpolWin[1] * data_ptr[-2]; + *intrp += kInterpolWin[2] * data_ptr[-1]; + *intrp += kInterpolWin[3] * data_ptr[0]; + *intrp += kInterpolWin[4] * data_ptr[1]; + *intrp += kInterpolWin[5] * data_ptr[2]; + *intrp += kInterpolWin[6] * data_ptr[3]; + *intrp += kInterpolWin[7] * data_ptr[4]; +} + + +/* 2D parabolic interpolation */ +/* probably some 0.5 factors can be eliminated, and the square-roots can be removed from the Cholesky fact. */ +__inline static void Intrpol2D(double T[3][3], double *x, double *y, double *peak_val) +{ + double c, b[2], A[2][2]; + double t1, t2, d; + double delta1, delta2; + + + // double T[3][3] = {{-1.25, -.25,-.25}, {-.25, .75, .75}, {-.25, .75, .75}}; + // should result in: delta1 = 0.5; delta2 = 0.0; peak_val = 1.0 + + c = T[1][1]; + b[0] = 0.5 * (T[1][2] + T[2][1] - T[0][1] - T[1][0]); + b[1] = 0.5 * (T[1][0] + T[2][1] - T[0][1] - T[1][2]); + A[0][1] = -0.5 * (T[0][1] + T[2][1] - T[1][0] - T[1][2]); + t1 = 0.5 * (T[0][0] + T[2][2]) - c; + t2 = 0.5 * (T[2][0] + T[0][2]) - c; + d = (T[0][1] + T[1][2] + T[1][0] + T[2][1]) - 4.0 * c - t1 - t2; + A[0][0] = -t1 - 0.5 * d; + A[1][1] = -t2 - 0.5 * d; + + /* deal with singularities or ill-conditioned cases */ + if ( (A[0][0] < 1e-7) || ((A[0][0] * A[1][1] - A[0][1] * A[0][1]) < 1e-7) ) { + *peak_val = T[1][1]; + return; + } + + /* Cholesky decomposition: replace A by upper-triangular factor */ + A[0][0] = sqrt(A[0][0]); + A[0][1] = A[0][1] / A[0][0]; + A[1][1] = sqrt(A[1][1] - A[0][1] * A[0][1]); + + /* compute [x; y] = -0.5 * inv(A) * b */ + t1 = b[0] / A[0][0]; + t2 = (b[1] - t1 * A[0][1]) / A[1][1]; + delta2 = t2 / A[1][1]; + delta1 = 0.5 * (t1 - delta2 * A[0][1]) / A[0][0]; + delta2 *= 0.5; + + /* limit norm */ + t1 = delta1 * delta1 + delta2 * delta2; + if (t1 > 1.0) { + delta1 /= t1; + delta2 /= t1; + } + + *peak_val = 0.5 * (b[0] * delta1 + b[1] * delta2) + c; + + *x += delta1; + *y += delta2; +} + + +static void PCorr(const double *in, double *outcorr) +{ + double sum, ysum, prod; + const double *x, *inptr; + int k, n; + + //ysum = 1e-6; /* use this with float (i.s.o. double)! */ + ysum = 1e-13; + sum = 0.0; + x = in + PITCH_MAX_LAG/2 + 2; + for (n = 0; n < PITCH_CORR_LEN2; n++) { + ysum += in[n] * in[n]; + sum += x[n] * in[n]; + } + + outcorr += PITCH_LAG_SPAN2 - 1; /* index of last element in array */ + *outcorr = sum / sqrt(ysum); + + for (k = 1; k < PITCH_LAG_SPAN2; k++) { + ysum -= in[k-1] * in[k-1]; + ysum += in[PITCH_CORR_LEN2 + k - 1] * in[PITCH_CORR_LEN2 + k - 1]; + sum = 0.0; + inptr = &in[k]; + prod = x[0] * inptr[0]; + for (n = 1; n < PITCH_CORR_LEN2; n++) { + sum += prod; + prod = x[n] * inptr[n]; + } + sum += prod; + outcorr--; + *outcorr = sum / sqrt(ysum); + } +} + + +void WebRtcIsac_InitializePitch(const double *in, + const double old_lag, + const double old_gain, + PitchAnalysisStruct *State, + double *lags) +{ + double buf_dec[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2+2]; + double ratio, log_lag, gain_bias; + double bias; + double corrvec1[PITCH_LAG_SPAN2]; + double corrvec2[PITCH_LAG_SPAN2]; + int m, k; + // Allocating 10 extra entries at the begining of the CorrSurf + double corrSurfBuff[10 + (2*PITCH_BW+3)*(PITCH_LAG_SPAN2+4)]; + double* CorrSurf[2*PITCH_BW+3]; + double *CorrSurfPtr1, *CorrSurfPtr2; + double LagWin[3] = {0.2, 0.5, 0.98}; + int ind1, ind2, peaks_ind, peak, max_ind; + int peaks[PITCH_MAX_NUM_PEAKS]; + double adj, gain_tmp; + double corr, corr_max; + double intrp_a, intrp_b, intrp_c, intrp_d; + double peak_vals[PITCH_MAX_NUM_PEAKS]; + double lags1[PITCH_MAX_NUM_PEAKS]; + double lags2[PITCH_MAX_NUM_PEAKS]; + double T[3][3]; + int row; + + for(k = 0; k < 2*PITCH_BW+3; k++) + { + CorrSurf[k] = &corrSurfBuff[10 + k * (PITCH_LAG_SPAN2+4)]; + } + /* reset CorrSurf matrix */ + memset(corrSurfBuff, 0, sizeof(double) * (10 + (2*PITCH_BW+3) * (PITCH_LAG_SPAN2+4))); + + //warnings -DH + max_ind = 0; + peak = 0; + + /* copy old values from state buffer */ + memcpy(buf_dec, State->dec_buffer, sizeof(double) * (PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2)); + + /* decimation; put result after the old values */ + WebRtcIsac_DecimateAllpass(in, State->decimator_state, PITCH_FRAME_LEN, + &buf_dec[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2]); + + /* low-pass filtering */ + for (k = PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2; k < PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2+2; k++) + buf_dec[k] += 0.75 * buf_dec[k-1] - 0.25 * buf_dec[k-2]; + + /* copy end part back into state buffer */ + memcpy(State->dec_buffer, buf_dec+PITCH_FRAME_LEN/2, sizeof(double) * (PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2)); + + /* compute correlation for first and second half of the frame */ + PCorr(buf_dec, corrvec1); + PCorr(buf_dec + PITCH_CORR_STEP2, corrvec2); + + /* bias towards pitch lag of previous frame */ + log_lag = log(0.5 * old_lag); + gain_bias = 4.0 * old_gain * old_gain; + if (gain_bias > 0.8) gain_bias = 0.8; + for (k = 0; k < PITCH_LAG_SPAN2; k++) + { + ratio = log((double) (k + (PITCH_MIN_LAG/2-2))) - log_lag; + bias = 1.0 + gain_bias * exp(-5.0 * ratio * ratio); + corrvec1[k] *= bias; + } + + /* taper correlation functions */ + for (k = 0; k < 3; k++) { + gain_tmp = LagWin[k]; + corrvec1[k] *= gain_tmp; + corrvec2[k] *= gain_tmp; + corrvec1[PITCH_LAG_SPAN2-1-k] *= gain_tmp; + corrvec2[PITCH_LAG_SPAN2-1-k] *= gain_tmp; + } + + corr_max = 0.0; + /* fill middle row of correlation surface */ + ind1 = 0; + ind2 = 0; + CorrSurfPtr1 = &CorrSurf[PITCH_BW][2]; + for (k = 0; k < PITCH_LAG_SPAN2; k++) { + corr = corrvec1[ind1++] + corrvec2[ind2++]; + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + } + /* fill first and last rows of correlation surface */ + ind1 = 0; + ind2 = PITCH_BW; + CorrSurfPtr1 = &CorrSurf[0][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW][PITCH_BW+2]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = 0.2 * ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + /* fill second and next to last rows of correlation surface */ + ind1 = 0; + ind2 = PITCH_BW-1; + CorrSurfPtr1 = &CorrSurf[1][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW-1][PITCH_BW+1]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW+1; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = 0.9 * ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + /* fill remainder of correlation surface */ + for (m = 2; m < PITCH_BW; m++) { + ind1 = 0; + ind2 = PITCH_BW - m; /* always larger than ind1 */ + CorrSurfPtr1 = &CorrSurf[m][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW-m][PITCH_BW+2-m]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW+m; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + } + + /* threshold value to qualify as a peak */ + corr_max *= 0.6; + + peaks_ind = 0; + /* find peaks */ + for (m = 1; m < PITCH_BW+1; m++) { + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + CorrSurfPtr1 = &CorrSurf[m][2]; + for (k = 2; k < PITCH_LAG_SPAN2-PITCH_BW-2+m; k++) { + corr = CorrSurfPtr1[k]; + if (corr > corr_max) { + if ( (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+5)]) && (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+4)]) ) { + if ( (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+4)]) && (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+5)]) ) { + /* found a peak; store index into matrix */ + peaks[peaks_ind++] = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + } + } + } + } + } + for (m = PITCH_BW+1; m < 2*PITCH_BW; m++) { + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + CorrSurfPtr1 = &CorrSurf[m][2]; + for (k = 2+m-PITCH_BW; k < PITCH_LAG_SPAN2-2; k++) { + corr = CorrSurfPtr1[k]; + if (corr > corr_max) { + if ( (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+5)]) && (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+4)]) ) { + if ( (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+4)]) && (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+5)]) ) { + /* found a peak; store index into matrix */ + peaks[peaks_ind++] = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + } + } + } + } + } + + if (peaks_ind > 0) { + /* examine each peak */ + CorrSurfPtr1 = &CorrSurf[0][0]; + for (k = 0; k < peaks_ind; k++) { + peak = peaks[k]; + + /* compute four interpolated values around current peak */ + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)], &intrp_a); + IntrepolFilter(&CorrSurfPtr1[peak - 1 ], &intrp_b); + IntrepolFilter(&CorrSurfPtr1[peak ], &intrp_c); + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)], &intrp_d); + + /* determine maximum of the interpolated values */ + corr = CorrSurfPtr1[peak]; + corr_max = intrp_a; + if (intrp_b > corr_max) corr_max = intrp_b; + if (intrp_c > corr_max) corr_max = intrp_c; + if (intrp_d > corr_max) corr_max = intrp_d; + + /* determine where the peak sits and fill a 3x3 matrix around it */ + row = peak / (PITCH_LAG_SPAN2+4); + lags1[k] = (double) ((peak - row * (PITCH_LAG_SPAN2+4)) + PITCH_MIN_LAG/2 - 4); + lags2[k] = (double) (lags1[k] + PITCH_BW - row); + if ( corr > corr_max ) { + T[0][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[2][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[1][1] = corr; + T[0][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + T[2][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + T[1][0] = intrp_a; + T[0][1] = intrp_b; + T[2][1] = intrp_c; + T[1][2] = intrp_d; + } else { + if (intrp_a == corr_max) { + lags1[k] -= 0.5; + lags2[k] += 0.5; + IntrepolFilter(&CorrSurfPtr1[peak - 2*(PITCH_LAG_SPAN2+5)], &T[0][0]); + IntrepolFilter(&CorrSurfPtr1[peak - (2*PITCH_LAG_SPAN2+9)], &T[2][0]); + T[1][1] = intrp_a; + T[0][2] = intrp_b; + T[2][2] = intrp_c; + T[1][0] = CorrSurfPtr1[peak - (2*PITCH_LAG_SPAN2+9)]; + T[0][1] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[2][1] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[1][2] = corr; + } else if (intrp_b == corr_max) { + lags1[k] -= 0.5; + lags2[k] -= 0.5; + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+6)], &T[0][0]); + T[2][0] = intrp_a; + T[1][1] = intrp_b; + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+3)], &T[0][2]); + T[2][2] = intrp_d; + T[1][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[0][1] = CorrSurfPtr1[peak - 1]; + T[2][1] = corr; + T[1][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + } else if (intrp_c == corr_max) { + lags1[k] += 0.5; + lags2[k] += 0.5; + T[0][0] = intrp_a; + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)], &T[2][0]); + T[1][1] = intrp_c; + T[0][2] = intrp_d; + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)], &T[2][2]); + T[1][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[0][1] = corr; + T[2][1] = CorrSurfPtr1[peak + 1]; + T[1][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + } else { + lags1[k] += 0.5; + lags2[k] -= 0.5; + T[0][0] = intrp_b; + T[2][0] = intrp_c; + T[1][1] = intrp_d; + IntrepolFilter(&CorrSurfPtr1[peak + 2*(PITCH_LAG_SPAN2+4)], &T[0][2]); + IntrepolFilter(&CorrSurfPtr1[peak + (2*PITCH_LAG_SPAN2+9)], &T[2][2]); + T[1][0] = corr; + T[0][1] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + T[2][1] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + T[1][2] = CorrSurfPtr1[peak + (2*PITCH_LAG_SPAN2+9)]; + } + } + + /* 2D parabolic interpolation gives more accurate lags and peak value */ + Intrpol2D(T, &lags1[k], &lags2[k], &peak_vals[k]); + } + + /* determine the highest peak, after applying a bias towards short lags */ + corr_max = 0.0; + for (k = 0; k < peaks_ind; k++) { + corr = peak_vals[k] * pow(PITCH_PEAK_DECAY, log(lags1[k] + lags2[k])); + if (corr > corr_max) { + corr_max = corr; + peak = k; + } + } + + lags1[peak] *= 2.0; + lags2[peak] *= 2.0; + + if (lags1[peak] < (double) PITCH_MIN_LAG) lags1[peak] = (double) PITCH_MIN_LAG; + if (lags2[peak] < (double) PITCH_MIN_LAG) lags2[peak] = (double) PITCH_MIN_LAG; + if (lags1[peak] > (double) PITCH_MAX_LAG) lags1[peak] = (double) PITCH_MAX_LAG; + if (lags2[peak] > (double) PITCH_MAX_LAG) lags2[peak] = (double) PITCH_MAX_LAG; + + /* store lags of highest peak in output array */ + lags[0] = lags1[peak]; + lags[1] = lags1[peak]; + lags[2] = lags2[peak]; + lags[3] = lags2[peak]; + } + else + { + row = max_ind / (PITCH_LAG_SPAN2+4); + lags1[0] = (double) ((max_ind - row * (PITCH_LAG_SPAN2+4)) + PITCH_MIN_LAG/2 - 4); + lags2[0] = (double) (lags1[0] + PITCH_BW - row); + + if (lags1[0] < (double) PITCH_MIN_LAG) lags1[0] = (double) PITCH_MIN_LAG; + if (lags2[0] < (double) PITCH_MIN_LAG) lags2[0] = (double) PITCH_MIN_LAG; + if (lags1[0] > (double) PITCH_MAX_LAG) lags1[0] = (double) PITCH_MAX_LAG; + if (lags2[0] > (double) PITCH_MAX_LAG) lags2[0] = (double) PITCH_MAX_LAG; + + /* store lags of highest peak in output array */ + lags[0] = lags1[0]; + lags[1] = lags1[0]; + lags[2] = lags2[0]; + lags[3] = lags2[0]; + } +} + + + +/* create weighting matrix by orthogonalizing a basis of polynomials of increasing order + * t = (0:4)'; + * A = [t.^0, t.^1, t.^2, t.^3, t.^4]; + * [Q, dummy] = qr(A); + * P.Weight = Q * diag([0, .1, .5, 1, 1]) * Q'; */ +static const double kWeight[5][5] = { + { 0.29714285714286, -0.30857142857143, -0.05714285714286, 0.05142857142857, 0.01714285714286}, + {-0.30857142857143, 0.67428571428571, -0.27142857142857, -0.14571428571429, 0.05142857142857}, + {-0.05714285714286, -0.27142857142857, 0.65714285714286, -0.27142857142857, -0.05714285714286}, + { 0.05142857142857, -0.14571428571429, -0.27142857142857, 0.67428571428571, -0.30857142857143}, + { 0.01714285714286, 0.05142857142857, -0.05714285714286, -0.30857142857143, 0.29714285714286} +}; + + +void WebRtcIsac_PitchAnalysis(const double *in, /* PITCH_FRAME_LEN samples */ + double *out, /* PITCH_FRAME_LEN+QLOOKAHEAD samples */ + PitchAnalysisStruct *State, + double *lags, + double *gains) +{ + double HPin[PITCH_FRAME_LEN]; + double Weighted[PITCH_FRAME_LEN]; + double Whitened[PITCH_FRAME_LEN + QLOOKAHEAD]; + double inbuf[PITCH_FRAME_LEN + QLOOKAHEAD]; + double out_G[PITCH_FRAME_LEN + QLOOKAHEAD]; // could be removed by using out instead + double out_dG[4][PITCH_FRAME_LEN + QLOOKAHEAD]; + double old_lag, old_gain; + double nrg_wht, tmp; + double Wnrg, Wfluct, Wgain; + double H[4][4]; + double grad[4]; + double dG[4]; + int k, m, n, iter; + + /* high pass filtering using second order pole-zero filter */ + WebRtcIsac_Highpass(in, HPin, State->hp_state, PITCH_FRAME_LEN); + + /* copy from state into buffer */ + memcpy(Whitened, State->whitened_buf, sizeof(double) * QLOOKAHEAD); + + /* compute weighted and whitened signals */ + WebRtcIsac_WeightingFilter(HPin, &Weighted[0], &Whitened[QLOOKAHEAD], &(State->Wghtstr)); + + /* copy from buffer into state */ + memcpy(State->whitened_buf, Whitened+PITCH_FRAME_LEN, sizeof(double) * QLOOKAHEAD); + + old_lag = State->PFstr_wght.oldlagp[0]; + old_gain = State->PFstr_wght.oldgainp[0]; + + /* inital pitch estimate */ + WebRtcIsac_InitializePitch(Weighted, old_lag, old_gain, State, lags); + + + /* Iterative optimization of lags - to be done */ + + /* compute energy of whitened signal */ + nrg_wht = 0.0; + for (k = 0; k < PITCH_FRAME_LEN + QLOOKAHEAD; k++) + nrg_wht += Whitened[k] * Whitened[k]; + + + /* Iterative optimization of gains */ + + /* set weights for energy, gain fluctiation, and spectral gain penalty functions */ + Wnrg = 1.0 / nrg_wht; + Wgain = 0.005; + Wfluct = 3.0; + + /* set initial gains */ + for (k = 0; k < 4; k++) + gains[k] = PITCH_MAX_GAIN_06; + + /* two iterations should be enough */ + for (iter = 0; iter < 2; iter++) { + /* compute Jacobian of pre-filter output towards gains */ + WebRtcIsac_PitchfilterPre_gains(Whitened, out_G, out_dG, &(State->PFstr_wght), lags, gains); + + /* gradient and approximate Hessian (lower triangle) for minimizing the filter's output power */ + for (k = 0; k < 4; k++) { + tmp = 0.0; + for (n = 0; n < PITCH_FRAME_LEN + QLOOKAHEAD; n++) + tmp += out_G[n] * out_dG[k][n]; + grad[k] = tmp * Wnrg; + } + for (k = 0; k < 4; k++) { + for (m = 0; m <= k; m++) { + tmp = 0.0; + for (n = 0; n < PITCH_FRAME_LEN + QLOOKAHEAD; n++) + tmp += out_dG[m][n] * out_dG[k][n]; + H[k][m] = tmp * Wnrg; + } + } + + /* add gradient and Hessian (lower triangle) for dampening fast gain changes */ + for (k = 0; k < 4; k++) { + tmp = kWeight[k+1][0] * old_gain; + for (m = 0; m < 4; m++) + tmp += kWeight[k+1][m+1] * gains[m]; + grad[k] += tmp * Wfluct; + } + for (k = 0; k < 4; k++) { + for (m = 0; m <= k; m++) { + H[k][m] += kWeight[k+1][m+1] * Wfluct; + } + } + + /* add gradient and Hessian for dampening gain */ + for (k = 0; k < 3; k++) { + tmp = 1.0 / (1 - gains[k]); + grad[k] += tmp * tmp * Wgain; + H[k][k] += 2.0 * tmp * (tmp * tmp * Wgain); + } + tmp = 1.0 / (1 - gains[3]); + grad[3] += 1.33 * (tmp * tmp * Wgain); + H[3][3] += 2.66 * tmp * (tmp * tmp * Wgain); + + + /* compute Cholesky factorization of Hessian + * by overwritting the upper triangle; scale factors on diagonal + * (for non pc-platforms store the inverse of the diagonals seperately to minimize divisions) */ + H[0][1] = H[1][0] / H[0][0]; + H[0][2] = H[2][0] / H[0][0]; + H[0][3] = H[3][0] / H[0][0]; + H[1][1] -= H[0][0] * H[0][1] * H[0][1]; + H[1][2] = (H[2][1] - H[0][1] * H[2][0]) / H[1][1]; + H[1][3] = (H[3][1] - H[0][1] * H[3][0]) / H[1][1]; + H[2][2] -= H[0][0] * H[0][2] * H[0][2] + H[1][1] * H[1][2] * H[1][2]; + H[2][3] = (H[3][2] - H[0][2] * H[3][0] - H[1][2] * H[1][1] * H[1][3]) / H[2][2]; + H[3][3] -= H[0][0] * H[0][3] * H[0][3] + H[1][1] * H[1][3] * H[1][3] + H[2][2] * H[2][3] * H[2][3]; + + /* Compute update as delta_gains = -inv(H) * grad */ + /* copy and negate */ + for (k = 0; k < 4; k++) + dG[k] = -grad[k]; + /* back substitution */ + dG[1] -= dG[0] * H[0][1]; + dG[2] -= dG[0] * H[0][2] + dG[1] * H[1][2]; + dG[3] -= dG[0] * H[0][3] + dG[1] * H[1][3] + dG[2] * H[2][3]; + /* scale */ + for (k = 0; k < 4; k++) + dG[k] /= H[k][k]; + /* back substitution */ + dG[2] -= dG[3] * H[2][3]; + dG[1] -= dG[3] * H[1][3] + dG[2] * H[1][2]; + dG[0] -= dG[3] * H[0][3] + dG[2] * H[0][2] + dG[1] * H[0][1]; + + /* update gains and check range */ + for (k = 0; k < 4; k++) { + gains[k] += dG[k]; + if (gains[k] > PITCH_MAX_GAIN) + gains[k] = PITCH_MAX_GAIN; + else if (gains[k] < 0.0) + gains[k] = 0.0; + } + } + + /* update state for next frame */ + WebRtcIsac_PitchfilterPre(Whitened, out, &(State->PFstr_wght), lags, gains); + + /* concatenate previous input's end and current input */ + memcpy(inbuf, State->inbuf, sizeof(double) * QLOOKAHEAD); + memcpy(inbuf+QLOOKAHEAD, in, sizeof(double) * PITCH_FRAME_LEN); + + /* lookahead pitch filtering for masking analysis */ + WebRtcIsac_PitchfilterPre_la(inbuf, out, &(State->PFstr), lags, gains); + + /* store last part of input */ + for (k = 0; k < QLOOKAHEAD; k++) + State->inbuf[k] = inbuf[k + PITCH_FRAME_LEN]; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h new file mode 100644 index 0000000000..96e1e17ee4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_estimator.h + * + * Pitch functions + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ + +#include "structs.h" + + + +void WebRtcIsac_PitchAnalysis(const double *in, /* PITCH_FRAME_LEN samples */ + double *out, /* PITCH_FRAME_LEN+QLOOKAHEAD samples */ + PitchAnalysisStruct *State, + double *lags, + double *gains); + +void WebRtcIsac_InitializePitch(const double *in, + const double old_lag, + const double old_gain, + PitchAnalysisStruct *State, + double *lags); + +void WebRtcIsac_PitchfilterPre(double *indat, + double *outdat, + PitchFiltstr *pfp, + double *lags, + double *gains); + +void WebRtcIsac_PitchfilterPost(double *indat, + double *outdat, + PitchFiltstr *pfp, + double *lags, + double *gains); + +void WebRtcIsac_PitchfilterPre_la(double *indat, + double *outdat, + PitchFiltstr *pfp, + double *lags, + double *gains); + +void WebRtcIsac_PitchfilterPre_gains(double *indat, + double *outdat, + double out_dG[][PITCH_FRAME_LEN + QLOOKAHEAD], + PitchFiltstr *pfp, + double *lags, + double *gains); + +void WebRtcIsac_WeightingFilter(const double *in, double *weiout, double *whiout, WeightFiltstr *wfdata); + +void WebRtcIsac_Highpass(const double *in, + double *out, + double *state, + size_t N); + +void WebRtcIsac_DecimateAllpass(const double *in, + double *state_in, /* array of size: + * 2*ALLPASSSECTIONS+1 */ + size_t N, /* number of input samples */ + double *out); /* array of size N/2 */ + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c new file mode 100644 index 0000000000..24ec63acdf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "pitch_estimator.h" + +#include +#include +#include + +#include "os_specific_inline.h" + +#include "rtc_base/compile_assert_c.h" + +/* + * We are implementing the following filters; + * + * Pre-filtering: + * y(z) = x(z) + damper(z) * gain * (x(z) + y(z)) * z ^ (-lag); + * + * Post-filtering: + * y(z) = x(z) - damper(z) * gain * (x(z) + y(z)) * z ^ (-lag); + * + * Note that |lag| is a floating number so we perform an interpolation to + * obtain the correct |lag|. + * + */ + +static const double kDampFilter[PITCH_DAMPORDER] = {-0.07, 0.25, 0.64, 0.25, + -0.07}; + +/* interpolation coefficients; generated by design_pitch_filter.m */ +static const double kIntrpCoef[PITCH_FRACS][PITCH_FRACORDER] = { + {-0.02239172458614, 0.06653315052934, -0.16515880017569, 0.60701333734125, + 0.64671399919202, -0.20249000396417, 0.09926548334755, -0.04765933793109, + 0.01754159521746}, + {-0.01985640750434, 0.05816126837866, -0.13991265473714, 0.44560418147643, + 0.79117042386876, -0.20266133815188, 0.09585268418555, -0.04533310458084, + 0.01654127246314}, + {-0.01463300534216, 0.04229888475060, -0.09897034715253, 0.28284326017787, + 0.90385267956632, -0.16976950138649, 0.07704272393639, -0.03584218578311, + 0.01295781500709}, + {-0.00764851320885, 0.02184035544377, -0.04985561057281, 0.13083306574393, + 0.97545011664662, -0.10177807997561, 0.04400901776474, -0.02010737175166, + 0.00719783432422}, + {-0.00000000000000, 0.00000000000000, -0.00000000000001, 0.00000000000001, + 0.99999999999999, 0.00000000000001, -0.00000000000001, 0.00000000000000, + -0.00000000000000}, + {0.00719783432422, -0.02010737175166, 0.04400901776474, -0.10177807997562, + 0.97545011664663, 0.13083306574393, -0.04985561057280, 0.02184035544377, + -0.00764851320885}, + {0.01295781500710, -0.03584218578312, 0.07704272393640, -0.16976950138650, + 0.90385267956634, 0.28284326017785, -0.09897034715252, 0.04229888475059, + -0.01463300534216}, + {0.01654127246315, -0.04533310458085, 0.09585268418557, -0.20266133815190, + 0.79117042386878, 0.44560418147640, -0.13991265473712, 0.05816126837865, + -0.01985640750433} +}; + +/* + * Enumerating the operation of the filter. + * iSAC has 4 different pitch-filter which are very similar in their structure. + * + * kPitchFilterPre : In this mode the filter is operating as pitch + * pre-filter. This is used at the encoder. + * kPitchFilterPost : In this mode the filter is operating as pitch + * post-filter. This is the inverse of pre-filter and used + * in the decoder. + * kPitchFilterPreLa : This is, in structure, similar to pre-filtering but + * utilizing 3 millisecond lookahead. It is used to + * obtain the signal for LPC analysis. + * kPitchFilterPreGain : This is, in structure, similar to pre-filtering but + * differential changes in gain is considered. This is + * used to find the optimal gain. + */ +typedef enum { + kPitchFilterPre, kPitchFilterPost, kPitchFilterPreLa, kPitchFilterPreGain +} PitchFilterOperation; + +/* + * Structure with parameters used for pitch-filtering. + * buffer : a buffer where the sum of previous inputs and outputs + * are stored. + * damper_state : the state of the damping filter. The filter is defined by + * |kDampFilter|. + * interpol_coeff : pointer to a set of coefficient which are used to utilize + * fractional pitch by interpolation. + * gain : pitch-gain to be applied to the current segment of input. + * lag : pitch-lag for the current segment of input. + * lag_offset : the offset of lag w.r.t. current sample. + * sub_frame : sub-frame index, there are 4 pitch sub-frames in an iSAC + * frame. + * This specifies the usage of the filter. See + * 'PitchFilterOperation' for operational modes. + * num_samples : number of samples to be processed in each segment. + * index : index of the input and output sample. + * damper_state_dg : state of damping filter for different trial gains. + * gain_mult : differential changes to gain. + */ +typedef struct { + double buffer[PITCH_INTBUFFSIZE + QLOOKAHEAD]; + double damper_state[PITCH_DAMPORDER]; + const double *interpol_coeff; + double gain; + double lag; + int lag_offset; + + int sub_frame; + PitchFilterOperation mode; + int num_samples; + int index; + + double damper_state_dg[4][PITCH_DAMPORDER]; + double gain_mult[4]; +} PitchFilterParam; + +/********************************************************************** + * FilterSegment() + * Filter one segment, a quarter of a frame. + * + * Inputs + * in_data : pointer to the input signal of 30 ms at 8 kHz sample-rate. + * filter_param : pitch filter parameters. + * + * Outputs + * out_data : pointer to a buffer where the filtered signal is written to. + * out_dg : [only used in kPitchFilterPreGain] pointer to a buffer + * where the output of different gain values (differential + * change to gain) is written. + */ +static void FilterSegment(const double* in_data, PitchFilterParam* parameters, + double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD]) { + int n; + int m; + int j; + double sum; + double sum2; + /* Index of |parameters->buffer| where the output is written to. */ + int pos = parameters->index + PITCH_BUFFSIZE; + /* Index of |parameters->buffer| where samples are read for fractional-lag + * computation. */ + int pos_lag = pos - parameters->lag_offset; + + for (n = 0; n < parameters->num_samples; ++n) { + /* Shift low pass filter states. */ + for (m = PITCH_DAMPORDER - 1; m > 0; --m) { + parameters->damper_state[m] = parameters->damper_state[m - 1]; + } + /* Filter to get fractional pitch. */ + sum = 0.0; + for (m = 0; m < PITCH_FRACORDER; ++m) { + sum += parameters->buffer[pos_lag + m] * parameters->interpol_coeff[m]; + } + /* Multiply with gain. */ + parameters->damper_state[0] = parameters->gain * sum; + + if (parameters->mode == kPitchFilterPreGain) { + int lag_index = parameters->index - parameters->lag_offset; + int m_tmp = (lag_index < 0) ? -lag_index : 0; + /* Update the damper state for the new sample. */ + for (m = PITCH_DAMPORDER - 1; m > 0; --m) { + for (j = 0; j < 4; ++j) { + parameters->damper_state_dg[j][m] = + parameters->damper_state_dg[j][m - 1]; + } + } + + for (j = 0; j < parameters->sub_frame + 1; ++j) { + /* Filter for fractional pitch. */ + sum2 = 0.0; + for (m = PITCH_FRACORDER-1; m >= m_tmp; --m) { + /* |lag_index + m| is always larger than or equal to zero, see how + * m_tmp is computed. This is equivalent to assume samples outside + * |out_dg[j]| are zero. */ + sum2 += out_dg[j][lag_index + m] * parameters->interpol_coeff[m]; + } + /* Add the contribution of differential gain change. */ + parameters->damper_state_dg[j][0] = parameters->gain_mult[j] * sum + + parameters->gain * sum2; + } + + /* Filter with damping filter, and store the results. */ + for (j = 0; j < parameters->sub_frame + 1; ++j) { + sum = 0.0; + for (m = 0; m < PITCH_DAMPORDER; ++m) { + sum -= parameters->damper_state_dg[j][m] * kDampFilter[m]; + } + out_dg[j][parameters->index] = sum; + } + } + /* Filter with damping filter. */ + sum = 0.0; + for (m = 0; m < PITCH_DAMPORDER; ++m) { + sum += parameters->damper_state[m] * kDampFilter[m]; + } + + /* Subtract from input and update buffer. */ + out_data[parameters->index] = in_data[parameters->index] - sum; + parameters->buffer[pos] = in_data[parameters->index] + + out_data[parameters->index]; + + ++parameters->index; + ++pos; + ++pos_lag; + } + return; +} + +/* Update filter parameters based on the pitch-gains and pitch-lags. */ +static void Update(PitchFilterParam* parameters) { + double fraction; + int fraction_index; + /* Compute integer lag-offset. */ + parameters->lag_offset = WebRtcIsac_lrint(parameters->lag + PITCH_FILTDELAY + + 0.5); + /* Find correct set of coefficients for computing fractional pitch. */ + fraction = parameters->lag_offset - (parameters->lag + PITCH_FILTDELAY); + fraction_index = WebRtcIsac_lrint(PITCH_FRACS * fraction - 0.5); + parameters->interpol_coeff = kIntrpCoef[fraction_index]; + + if (parameters->mode == kPitchFilterPreGain) { + /* If in this mode make a differential change to pitch gain. */ + parameters->gain_mult[parameters->sub_frame] += 0.2; + if (parameters->gain_mult[parameters->sub_frame] > 1.0) { + parameters->gain_mult[parameters->sub_frame] = 1.0; + } + if (parameters->sub_frame > 0) { + parameters->gain_mult[parameters->sub_frame - 1] -= 0.2; + } + } +} + +/****************************************************************************** + * FilterFrame() + * Filter a frame of 30 millisecond, given pitch-lags and pitch-gains. + * + * Inputs + * in_data : pointer to the input signal of 30 ms at 8 kHz sample-rate. + * lags : pointer to pitch-lags, 4 lags per frame. + * gains : pointer to pitch-gians, 4 gains per frame. + * mode : defining the functionality of the filter. It takes the + * following values. + * kPitchFilterPre: Pitch pre-filter, used at encoder. + * kPitchFilterPost: Pitch post-filter, used at decoder. + * kPitchFilterPreLa: Pitch pre-filter with lookahead. + * kPitchFilterPreGain: Pitch pre-filter used to otain optimal + * pitch-gains. + * + * Outputs + * out_data : pointer to a buffer where the filtered signal is written to. + * out_dg : [only used in kPitchFilterPreGain] pointer to a buffer + * where the output of different gain values (differential + * change to gain) is written. + */ +static void FilterFrame(const double* in_data, PitchFiltstr* filter_state, + double* lags, double* gains, PitchFilterOperation mode, + double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD]) { + PitchFilterParam filter_parameters; + double gain_delta, lag_delta; + double old_lag, old_gain; + int n; + int m; + const double kEnhancer = 1.3; + + /* Set up buffer and states. */ + filter_parameters.index = 0; + filter_parameters.lag_offset = 0; + filter_parameters.mode = mode; + /* Copy states to local variables. */ + memcpy(filter_parameters.buffer, filter_state->ubuf, + sizeof(filter_state->ubuf)); + RTC_COMPILE_ASSERT(sizeof(filter_parameters.buffer) >= + sizeof(filter_state->ubuf)); + memset(filter_parameters.buffer + + sizeof(filter_state->ubuf) / sizeof(filter_state->ubuf[0]), + 0, sizeof(filter_parameters.buffer) - sizeof(filter_state->ubuf)); + memcpy(filter_parameters.damper_state, filter_state->ystate, + sizeof(filter_state->ystate)); + + if (mode == kPitchFilterPreGain) { + /* Clear buffers. */ + memset(filter_parameters.gain_mult, 0, sizeof(filter_parameters.gain_mult)); + memset(filter_parameters.damper_state_dg, 0, + sizeof(filter_parameters.damper_state_dg)); + for (n = 0; n < PITCH_SUBFRAMES; ++n) { + //memset(out_dg[n], 0, sizeof(double) * (PITCH_FRAME_LEN + QLOOKAHEAD)); + memset(out_dg[n], 0, sizeof(out_dg[n])); + } + } else if (mode == kPitchFilterPost) { + /* Make output more periodic. Negative sign is to change the structure + * of the filter. */ + for (n = 0; n < PITCH_SUBFRAMES; ++n) { + gains[n] *= -kEnhancer; + } + } + + old_lag = *filter_state->oldlagp; + old_gain = *filter_state->oldgainp; + + /* No interpolation if pitch lag step is big. */ + if ((lags[0] > (PITCH_UPSTEP * old_lag)) || + (lags[0] < (PITCH_DOWNSTEP * old_lag))) { + old_lag = lags[0]; + old_gain = gains[0]; + + if (mode == kPitchFilterPreGain) { + filter_parameters.gain_mult[0] = 1.0; + } + } + + filter_parameters.num_samples = PITCH_UPDATE; + for (m = 0; m < PITCH_SUBFRAMES; ++m) { + /* Set the sub-frame value. */ + filter_parameters.sub_frame = m; + /* Calculate interpolation steps for pitch-lag and pitch-gain. */ + lag_delta = (lags[m] - old_lag) / PITCH_GRAN_PER_SUBFRAME; + filter_parameters.lag = old_lag; + gain_delta = (gains[m] - old_gain) / PITCH_GRAN_PER_SUBFRAME; + filter_parameters.gain = old_gain; + /* Store for the next sub-frame. */ + old_lag = lags[m]; + old_gain = gains[m]; + + for (n = 0; n < PITCH_GRAN_PER_SUBFRAME; ++n) { + /* Step-wise interpolation of pitch gains and lags. As pitch-lag changes, + * some parameters of filter need to be update. */ + filter_parameters.gain += gain_delta; + filter_parameters.lag += lag_delta; + /* Update parameters according to new lag value. */ + Update(&filter_parameters); + /* Filter a segment of input. */ + FilterSegment(in_data, &filter_parameters, out_data, out_dg); + } + } + + if (mode != kPitchFilterPreGain) { + /* Export buffer and states. */ + memcpy(filter_state->ubuf, &filter_parameters.buffer[PITCH_FRAME_LEN], + sizeof(filter_state->ubuf)); + memcpy(filter_state->ystate, filter_parameters.damper_state, + sizeof(filter_state->ystate)); + + /* Store for the next frame. */ + *filter_state->oldlagp = old_lag; + *filter_state->oldgainp = old_gain; + } + + if ((mode == kPitchFilterPreGain) || (mode == kPitchFilterPreLa)) { + /* Filter the lookahead segment, this is treated as the last sub-frame. So + * set |pf_param| to last sub-frame. */ + filter_parameters.sub_frame = PITCH_SUBFRAMES - 1; + filter_parameters.num_samples = QLOOKAHEAD; + FilterSegment(in_data, &filter_parameters, out_data, out_dg); + } +} + +void WebRtcIsac_PitchfilterPre(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPre, out_data, NULL); +} + +void WebRtcIsac_PitchfilterPre_la(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPreLa, out_data, + NULL); +} + +void WebRtcIsac_PitchfilterPre_gains( + double* in_data, double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD], PitchFiltstr *pf_state, + double* lags, double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPreGain, out_data, + out_dg); +} + +void WebRtcIsac_PitchfilterPost(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPost, out_data, NULL); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.c new file mode 100644 index 0000000000..947d3e7636 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "pitch_gain_tables.h" + +#include "settings.h" + +/* header file for coding tables for the pitch filter side-info in the entropy coder */ +/********************* Pitch Filter Gain Coefficient Tables ************************/ +/* cdf for quantized pitch filter gains */ +const uint16_t WebRtcIsac_kQPitchGainCdf[255] = { + 0, 2, 4, 6, 64, 901, 903, 905, 16954, 16956, + 16961, 17360, 17362, 17364, 17366, 17368, 17370, 17372, 17374, 17411, + 17514, 17516, 17583, 18790, 18796, 18802, 20760, 20777, 20782, 21722, + 21724, 21728, 21738, 21740, 21742, 21744, 21746, 21748, 22224, 22227, + 22230, 23214, 23229, 23239, 25086, 25108, 25120, 26088, 26094, 26098, + 26175, 26177, 26179, 26181, 26183, 26185, 26484, 26507, 26522, 27705, + 27731, 27750, 29767, 29799, 29817, 30866, 30883, 30885, 31025, 31029, + 31031, 31033, 31035, 31037, 31114, 31126, 31134, 32687, 32722, 32767, + 35718, 35742, 35757, 36943, 36952, 36954, 37115, 37128, 37130, 37132, + 37134, 37136, 37143, 37145, 37152, 38843, 38863, 38897, 47458, 47467, + 47474, 49040, 49061, 49063, 49145, 49157, 49159, 49161, 49163, 49165, + 49167, 49169, 49171, 49757, 49770, 49782, 61333, 61344, 61346, 62860, + 62883, 62885, 62887, 62889, 62891, 62893, 62895, 62897, 62899, 62901, + 62903, 62905, 62907, 62909, 65496, 65498, 65500, 65521, 65523, 65525, + 65527, 65529, 65531, 65533, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535}; + +/* index limits and ranges */ +const int16_t WebRtcIsac_kIndexLowerLimitGain[3] = { + -7, -2, -1}; + +const int16_t WebRtcIsac_kIndexUpperLimitGain[3] = { + 0, 3, 1}; + +const uint16_t WebRtcIsac_kIndexMultsGain[2] = { + 18, 3}; + +/* size of cdf table */ +const uint16_t WebRtcIsac_kQCdfTableSizeGain[1] = { + 256}; + +///////////////////////////FIXED POINT +/* mean values of pitch filter gains in FIXED point */ +const int16_t WebRtcIsac_kQMeanGain1Q12[144] = { + 843, 1092, 1336, 1222, 1405, 1656, 1500, 1815, 1843, 1838, 1839, 1843, 1843, 1843, 1843, 1843, + 1843, 1843, 814, 846, 1092, 1013, 1174, 1383, 1391, 1511, 1584, 1734, 1753, 1843, 1843, 1843, + 1843, 1843, 1843, 1843, 524, 689, 777, 845, 947, 1069, 1090, 1263, 1380, 1447, 1559, 1676, + 1645, 1749, 1843, 1843, 1843, 1843, 81, 477, 563, 611, 706, 806, 849, 1012, 1192, 1128, + 1330, 1489, 1425, 1576, 1826, 1741, 1843, 1843, 0, 290, 305, 356, 488, 575, 602, 741, + 890, 835, 1079, 1196, 1182, 1376, 1519, 1506, 1680, 1843, 0, 47, 97, 69, 289, 381, + 385, 474, 617, 664, 803, 1079, 935, 1160, 1269, 1265, 1506, 1741, 0, 0, 0, 0, + 112, 120, 190, 283, 442, 343, 526, 809, 684, 935, 1134, 1020, 1265, 1506, 0, 0, + 0, 0, 0, 0, 0, 111, 256, 87, 373, 597, 430, 684, 935, 770, 1020, 1265}; + +const int16_t WebRtcIsac_kQMeanGain2Q12[144] = { + 1760, 1525, 1285, 1747, 1671, 1393, 1843, 1826, 1555, 1843, 1784, 1606, 1843, 1843, 1711, 1843, + 1843, 1814, 1389, 1275, 1040, 1564, 1414, 1252, 1610, 1495, 1343, 1753, 1592, 1405, 1804, 1720, + 1475, 1843, 1814, 1581, 1208, 1061, 856, 1349, 1148, 994, 1390, 1253, 1111, 1495, 1343, 1178, + 1770, 1465, 1234, 1814, 1581, 1342, 1040, 793, 713, 1053, 895, 737, 1128, 1003, 861, 1277, + 1094, 981, 1475, 1192, 1019, 1581, 1342, 1098, 855, 570, 483, 833, 648, 540, 948, 744, + 572, 1009, 844, 636, 1234, 934, 685, 1342, 1217, 984, 537, 318, 124, 603, 423, 350, + 687, 479, 322, 791, 581, 430, 987, 671, 488, 1098, 849, 597, 283, 27, 0, 397, + 222, 38, 513, 271, 124, 624, 325, 157, 737, 484, 233, 849, 597, 343, 27, 0, + 0, 141, 0, 0, 256, 69, 0, 370, 87, 0, 484, 229, 0, 597, 343, 87}; + +const int16_t WebRtcIsac_kQMeanGain3Q12[144] = { + 1843, 1843, 1711, 1843, 1818, 1606, 1843, 1827, 1511, 1814, 1639, 1393, 1760, 1525, 1285, 1656, + 1419, 1176, 1835, 1718, 1475, 1841, 1650, 1387, 1648, 1498, 1287, 1600, 1411, 1176, 1522, 1299, + 1040, 1419, 1176, 928, 1773, 1461, 1128, 1532, 1355, 1202, 1429, 1260, 1115, 1398, 1151, 1025, + 1172, 1080, 790, 1176, 928, 677, 1475, 1147, 1019, 1276, 1096, 922, 1214, 1010, 901, 1057, + 893, 800, 1040, 796, 734, 928, 677, 424, 1137, 897, 753, 1120, 830, 710, 875, 751, + 601, 795, 642, 583, 790, 544, 475, 677, 474, 140, 987, 750, 482, 697, 573, 450, + 691, 487, 303, 661, 394, 332, 537, 303, 220, 424, 168, 0, 737, 484, 229, 624, + 348, 153, 441, 261, 136, 397, 166, 51, 283, 27, 0, 168, 0, 0, 484, 229, + 0, 370, 57, 0, 256, 43, 0, 141, 0, 0, 27, 0, 0, 0, 0, 0}; + + +const int16_t WebRtcIsac_kQMeanGain4Q12[144] = { + 1843, 1843, 1843, 1843, 1841, 1843, 1500, 1821, 1843, 1222, 1434, 1656, 843, 1092, 1336, 504, + 757, 1007, 1843, 1843, 1843, 1838, 1791, 1843, 1265, 1505, 1599, 965, 1219, 1425, 730, 821, + 1092, 249, 504, 757, 1783, 1819, 1843, 1351, 1567, 1727, 1096, 1268, 1409, 805, 961, 1131, + 444, 670, 843, 0, 249, 504, 1425, 1655, 1743, 1096, 1324, 1448, 822, 1019, 1199, 490, + 704, 867, 81, 450, 555, 0, 0, 249, 1247, 1428, 1530, 881, 1073, 1283, 610, 759, + 939, 278, 464, 645, 0, 200, 270, 0, 0, 0, 935, 1163, 1410, 528, 790, 1068, + 377, 499, 717, 173, 240, 274, 0, 43, 62, 0, 0, 0, 684, 935, 1182, 343, + 551, 735, 161, 262, 423, 0, 55, 27, 0, 0, 0, 0, 0, 0, 430, 684, + 935, 87, 377, 597, 0, 46, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.h new file mode 100644 index 0000000000..fe506ee284 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_gain_tables.h + * + * This file contains tables for the pitch filter side-info in the entropy coder. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_GAIN_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_GAIN_TABLES_H_ + +#include "typedefs.h" // NOLINT(build/include) + +/* header file for coding tables for the pitch filter side-info in the entropy coder */ +/********************* Pitch Filter Gain Coefficient Tables ************************/ +/* cdf for quantized pitch filter gains */ +extern const uint16_t WebRtcIsac_kQPitchGainCdf[255]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsac_kIndexLowerLimitGain[3]; + +extern const int16_t WebRtcIsac_kIndexUpperLimitGain[3]; +extern const uint16_t WebRtcIsac_kIndexMultsGain[2]; + +/* mean values of pitch filter gains */ +//(Y) +extern const int16_t WebRtcIsac_kQMeanGain1Q12[144]; +extern const int16_t WebRtcIsac_kQMeanGain2Q12[144]; +extern const int16_t WebRtcIsac_kQMeanGain3Q12[144]; +extern const int16_t WebRtcIsac_kQMeanGain4Q12[144]; +//(Y) + +/* size of cdf table */ +extern const uint16_t WebRtcIsac_kQCdfTableSizeGain[1]; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_GAIN_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.c new file mode 100644 index 0000000000..f845a22550 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.c @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "pitch_lag_tables.h" +#include "settings.h" + +/* header file for coding tables for the pitch filter side-info in the entropy coder */ +/********************* Pitch Filter Gain Coefficient Tables ************************/ + +/* tables for use with small pitch gain */ + +/* cdf for quantized pitch filter lags */ +const uint16_t WebRtcIsac_kQPitchLagCdf1Lo[127] = { + 0, 134, 336, 549, 778, 998, 1264, 1512, 1777, 2070, + 2423, 2794, 3051, 3361, 3708, 3979, 4315, 4610, 4933, 5269, + 5575, 5896, 6155, 6480, 6816, 7129, 7477, 7764, 8061, 8358, + 8718, 9020, 9390, 9783, 10177, 10543, 10885, 11342, 11795, 12213, + 12680, 13096, 13524, 13919, 14436, 14903, 15349, 15795, 16267, 16734, + 17266, 17697, 18130, 18632, 19080, 19447, 19884, 20315, 20735, 21288, + 21764, 22264, 22723, 23193, 23680, 24111, 24557, 25022, 25537, 26082, + 26543, 27090, 27620, 28139, 28652, 29149, 29634, 30175, 30692, 31273, + 31866, 32506, 33059, 33650, 34296, 34955, 35629, 36295, 36967, 37726, + 38559, 39458, 40364, 41293, 42256, 43215, 44231, 45253, 46274, 47359, + 48482, 49678, 50810, 51853, 53016, 54148, 55235, 56263, 57282, 58363, + 59288, 60179, 61076, 61806, 62474, 63129, 63656, 64160, 64533, 64856, + 65152, 65535, 65535, 65535, 65535, 65535, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf2Lo[20] = { + 0, 429, 3558, 5861, 8558, 11639, 15210, 19502, 24773, 31983, + 42602, 48567, 52601, 55676, 58160, 60172, 61889, 63235, 65383, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf3Lo[2] = { + 0, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf4Lo[10] = { + 0, 2966, 6368, 11182, 19431, 37793, 48532, 55353, 60626, 65535}; + +const uint16_t *WebRtcIsac_kQPitchLagCdfPtrLo[4] = {WebRtcIsac_kQPitchLagCdf1Lo, WebRtcIsac_kQPitchLagCdf2Lo, WebRtcIsac_kQPitchLagCdf3Lo, WebRtcIsac_kQPitchLagCdf4Lo}; + +/* size of first cdf table */ +const uint16_t WebRtcIsac_kQPitchLagCdfSizeLo[1] = {128}; + +/* index limits and ranges */ +const int16_t WebRtcIsac_kQIndexLowerLimitLagLo[4] = { +-140, -9, 0, -4}; + +const int16_t WebRtcIsac_kQIndexUpperLimitLagLo[4] = { +-20, 9, 0, 4}; + +/* initial index for arithmetic decoder */ +const uint16_t WebRtcIsac_kQInitIndexLagLo[3] = { + 10, 1, 5}; + +/* mean values of pitch filter lags */ +const double WebRtcIsac_kQMeanLag2Lo[19] = { +-17.21385070, -15.82678944, -14.07123081, -12.03003877, -10.01311864, -8.00794627, -5.91162987, -3.89231876, -1.90220980, -0.01879275, + 1.89144232, 3.88123171, 5.92146992, 7.96435361, 9.98923648, 11.98266347, 13.96101002, 15.74855713, 17.10976611}; + +const double WebRtcIsac_kQMeanLag3Lo[1] = { + 0.00000000}; + +const double WebRtcIsac_kQMeanLag4Lo[9] = { +-7.76246496, -5.92083980, -3.94095226, -1.89502305, 0.03724681, 1.93054221, 3.96443467, 5.91726366, 7.78434291}; + +const double WebRtcIsac_kQPitchLagStepsizeLo = 2.000000; + + +/* tables for use with medium pitch gain */ + +/* cdf for quantized pitch filter lags */ +const uint16_t WebRtcIsac_kQPitchLagCdf1Mid[255] = { + 0, 28, 61, 88, 121, 149, 233, 331, 475, 559, + 624, 661, 689, 712, 745, 791, 815, 843, 866, 922, + 959, 1024, 1061, 1117, 1178, 1238, 1280, 1350, 1453, 1513, + 1564, 1625, 1671, 1741, 1788, 1904, 2072, 2421, 2626, 2770, + 2840, 2900, 2942, 3012, 3068, 3115, 3147, 3194, 3254, 3319, + 3366, 3520, 3678, 3780, 3850, 3911, 3957, 4032, 4106, 4185, + 4292, 4474, 4683, 4842, 5019, 5191, 5321, 5428, 5540, 5675, + 5763, 5847, 5959, 6127, 6304, 6564, 6839, 7090, 7263, 7421, + 7556, 7728, 7872, 7984, 8142, 8361, 8580, 8743, 8938, 9227, + 9409, 9539, 9674, 9795, 9930, 10060, 10177, 10382, 10614, 10861, + 11038, 11271, 11415, 11629, 11792, 12044, 12193, 12416, 12574, 12821, + 13007, 13235, 13445, 13654, 13901, 14134, 14488, 15000, 15703, 16285, + 16504, 16797, 17086, 17328, 17579, 17807, 17998, 18268, 18538, 18836, + 19087, 19274, 19474, 19716, 19935, 20270, 20833, 21303, 21532, 21741, + 21978, 22207, 22523, 22770, 23054, 23613, 23943, 24204, 24399, 24651, + 24832, 25074, 25270, 25549, 25759, 26015, 26150, 26424, 26713, 27048, + 27342, 27504, 27681, 27854, 28021, 28207, 28412, 28664, 28859, 29064, + 29278, 29548, 29748, 30107, 30377, 30656, 30856, 31164, 31452, 31755, + 32011, 32328, 32626, 32919, 33319, 33789, 34329, 34925, 35396, 35973, + 36443, 36964, 37551, 38156, 38724, 39357, 40023, 40908, 41587, 42602, + 43924, 45037, 45810, 46597, 47421, 48291, 49092, 50051, 51448, 52719, + 53440, 54241, 54944, 55977, 56676, 57299, 57872, 58389, 59059, 59688, + 60237, 60782, 61094, 61573, 61890, 62290, 62658, 63030, 63217, 63454, + 63622, 63882, 64003, 64273, 64427, 64529, 64581, 64697, 64758, 64902, + 65414, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf2Mid[36] = { + 0, 71, 335, 581, 836, 1039, 1323, 1795, 2258, 2608, + 3005, 3591, 4243, 5344, 7163, 10583, 16848, 28078, 49448, 57007, + 60357, 61850, 62837, 63437, 63872, 64188, 64377, 64614, 64774, 64949, + 65039, 65115, 65223, 65360, 65474, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf3Mid[2] = { + 0, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf4Mid[20] = { + 0, 28, 246, 459, 667, 1045, 1523, 2337, 4337, 11347, + 44231, 56709, 60781, 62243, 63161, 63969, 64608, 65062, 65502, 65535}; + +const uint16_t *WebRtcIsac_kQPitchLagCdfPtrMid[4] = {WebRtcIsac_kQPitchLagCdf1Mid, WebRtcIsac_kQPitchLagCdf2Mid, WebRtcIsac_kQPitchLagCdf3Mid, WebRtcIsac_kQPitchLagCdf4Mid}; + +/* size of first cdf table */ +const uint16_t WebRtcIsac_kQPitchLagCdfSizeMid[1] = {256}; + +/* index limits and ranges */ +const int16_t WebRtcIsac_kQIndexLowerLimitLagMid[4] = { +-280, -17, 0, -9}; + +const int16_t WebRtcIsac_kQIndexUpperLimitLagMid[4] = { +-40, 17, 0, 9}; + +/* initial index for arithmetic decoder */ +const uint16_t WebRtcIsac_kQInitIndexLagMid[3] = { + 18, 1, 10}; + +/* mean values of pitch filter lags */ +const double WebRtcIsac_kQMeanLag2Mid[35] = { +-16.89183900, -15.86949778, -15.05476653, -14.00664348, -13.02793036, -12.07324237, -11.00542532, -10.11250602, -8.90792971, -8.02474753, +-7.00426767, -5.94055287, -4.98251338, -3.91053158, -2.98820425, -1.93524245, -0.92978085, -0.01722509, 0.91317387, 1.92973955, + 2.96908851, 3.93728974, 4.96308471, 5.92244151, 7.08673497, 8.00993708, 9.04656316, 9.98538742, 10.97851694, 11.94772884, + 13.02426166, 14.00039951, 15.01347042, 15.80758023, 16.94086895}; + +const double WebRtcIsac_kQMeanLag3Mid[1] = { + 0.00000000}; + +const double WebRtcIsac_kQMeanLag4Mid[19] = { +-8.60409403, -7.89198395, -7.03450280, -5.86260421, -4.93822322, -3.93078706, -2.91302322, -1.91824007, -0.87003282, 0.02822649, + 0.89951758, 1.87495484, 2.91802604, 3.96874074, 5.06571703, 5.93618227, 7.00520185, 7.88497726, 8.64160364}; + +const double WebRtcIsac_kQPitchLagStepsizeMid = 1.000000; + + +/* tables for use with large pitch gain */ + +/* cdf for quantized pitch filter lags */ +const uint16_t WebRtcIsac_kQPitchLagCdf1Hi[511] = { + 0, 7, 18, 33, 69, 105, 156, 228, 315, 612, + 680, 691, 709, 724, 735, 738, 742, 746, 749, 753, + 756, 760, 764, 774, 782, 785, 789, 796, 800, 803, + 807, 814, 818, 822, 829, 832, 847, 854, 858, 869, + 876, 883, 898, 908, 934, 977, 1010, 1050, 1060, 1064, + 1075, 1078, 1086, 1089, 1093, 1104, 1111, 1122, 1133, 1136, + 1151, 1162, 1183, 1209, 1252, 1281, 1339, 1364, 1386, 1401, + 1411, 1415, 1426, 1430, 1433, 1440, 1448, 1455, 1462, 1477, + 1487, 1495, 1502, 1506, 1509, 1516, 1524, 1531, 1535, 1542, + 1553, 1556, 1578, 1589, 1611, 1625, 1639, 1643, 1654, 1665, + 1672, 1687, 1694, 1705, 1708, 1719, 1730, 1744, 1752, 1759, + 1791, 1795, 1820, 1867, 1886, 1915, 1936, 1943, 1965, 1987, + 2041, 2099, 2161, 2175, 2200, 2211, 2226, 2233, 2244, 2251, + 2266, 2280, 2287, 2298, 2309, 2316, 2331, 2342, 2356, 2378, + 2403, 2418, 2447, 2497, 2544, 2602, 2863, 2895, 2903, 2935, + 2950, 2971, 3004, 3011, 3018, 3029, 3040, 3062, 3087, 3127, + 3152, 3170, 3199, 3243, 3293, 3322, 3340, 3377, 3402, 3427, + 3474, 3518, 3543, 3579, 3601, 3637, 3659, 3706, 3731, 3760, + 3818, 3847, 3869, 3901, 3920, 3952, 4068, 4169, 4220, 4271, + 4524, 4571, 4604, 4632, 4672, 4730, 4777, 4806, 4857, 4904, + 4951, 5002, 5031, 5060, 5107, 5150, 5212, 5266, 5331, 5382, + 5432, 5490, 5544, 5610, 5700, 5762, 5812, 5874, 5972, 6022, + 6091, 6163, 6232, 6305, 6402, 6540, 6685, 6880, 7090, 7271, + 7379, 7452, 7542, 7625, 7687, 7770, 7843, 7911, 7966, 8024, + 8096, 8190, 8252, 8320, 8411, 8501, 8585, 8639, 8751, 8842, + 8918, 8986, 9066, 9127, 9203, 9269, 9345, 9406, 9464, 9536, + 9612, 9667, 9735, 9844, 9931, 10036, 10119, 10199, 10260, 10358, + 10441, 10514, 10666, 10734, 10872, 10951, 11053, 11125, 11223, 11324, + 11516, 11664, 11737, 11816, 11892, 12008, 12120, 12200, 12280, 12392, + 12490, 12576, 12685, 12812, 12917, 13003, 13108, 13210, 13300, 13384, + 13470, 13579, 13673, 13771, 13879, 13999, 14136, 14201, 14368, 14614, + 14759, 14867, 14958, 15030, 15121, 15189, 15280, 15385, 15461, 15555, + 15653, 15768, 15884, 15971, 16069, 16145, 16210, 16279, 16380, 16463, + 16539, 16615, 16688, 16818, 16919, 17017, 18041, 18338, 18523, 18649, + 18790, 18917, 19047, 19167, 19315, 19460, 19601, 19731, 19858, 20068, + 20173, 20318, 20466, 20625, 20741, 20911, 21045, 21201, 21396, 21588, + 21816, 22022, 22305, 22547, 22786, 23072, 23322, 23600, 23879, 24168, + 24433, 24769, 25120, 25511, 25895, 26289, 26792, 27219, 27683, 28077, + 28566, 29094, 29546, 29977, 30491, 30991, 31573, 32105, 32594, 33173, + 33788, 34497, 35181, 35833, 36488, 37255, 37921, 38645, 39275, 39894, + 40505, 41167, 41790, 42431, 43096, 43723, 44385, 45134, 45858, 46607, + 47349, 48091, 48768, 49405, 49955, 50555, 51167, 51985, 52611, 53078, + 53494, 53965, 54435, 54996, 55601, 56125, 56563, 56838, 57244, 57566, + 57967, 58297, 58771, 59093, 59419, 59647, 59886, 60143, 60461, 60693, + 60917, 61170, 61416, 61634, 61891, 62122, 62310, 62455, 62632, 62839, + 63103, 63436, 63639, 63805, 63906, 64015, 64192, 64355, 64475, 64558, + 64663, 64742, 64811, 64865, 64916, 64956, 64981, 65025, 65068, 65115, + 65195, 65314, 65419, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf2Hi[68] = { + 0, 7, 11, 22, 37, 52, 56, 59, 81, 85, + 89, 96, 115, 130, 137, 152, 170, 181, 193, 200, + 207, 233, 237, 259, 289, 318, 363, 433, 592, 992, + 1607, 3062, 6149, 12206, 25522, 48368, 58223, 61918, 63640, 64584, + 64943, 65098, 65206, 65268, 65294, 65335, 65350, 65372, 65387, 65402, + 65413, 65420, 65428, 65435, 65439, 65450, 65454, 65468, 65472, 65476, + 65483, 65491, 65498, 65505, 65516, 65520, 65528, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf3Hi[2] = { + 0, 65535}; + +const uint16_t WebRtcIsac_kQPitchLagCdf4Hi[35] = { + 0, 7, 19, 30, 41, 48, 63, 74, 82, 96, + 122, 152, 215, 330, 701, 2611, 10931, 48106, 61177, 64341, + 65112, 65238, 65309, 65338, 65364, 65379, 65401, 65427, 65453, 65465, + 65476, 65490, 65509, 65528, 65535}; + +const uint16_t *WebRtcIsac_kQPitchLagCdfPtrHi[4] = {WebRtcIsac_kQPitchLagCdf1Hi, WebRtcIsac_kQPitchLagCdf2Hi, WebRtcIsac_kQPitchLagCdf3Hi, WebRtcIsac_kQPitchLagCdf4Hi}; + +/* size of first cdf table */ +const uint16_t WebRtcIsac_kQPitchLagCdfSizeHi[1] = {512}; + +/* index limits and ranges */ +const int16_t WebRtcIsac_kQindexLowerLimitLagHi[4] = { +-552, -34, 0, -16}; + +const int16_t WebRtcIsac_kQindexUpperLimitLagHi[4] = { +-80, 32, 0, 17}; + +/* initial index for arithmetic decoder */ +const uint16_t WebRtcIsac_kQInitIndexLagHi[3] = { + 34, 1, 18}; + +/* mean values of pitch filter lags */ +const double WebRtcIsac_kQMeanLag2Hi[67] = { +-17.07263295, -16.50000000, -15.83966081, -15.55613708, -14.96948007, -14.50000000, -14.00000000, -13.48377986, -13.00000000, -12.50000000, +-11.93199636, -11.44530414, -11.04197641, -10.39910301, -10.15202337, -9.51322461, -8.93357741, -8.46456632, -8.10270672, -7.53751847, +-6.98686404, -6.50000000, -6.08463150, -5.46872991, -5.00864717, -4.50163760, -4.01382410, -3.43856708, -2.96898001, -2.46554810, +-1.96861004, -1.47106701, -0.97197237, -0.46561654, -0.00531409, 0.45767857, 0.96777907, 1.47507903, 1.97740425, 2.46695420, + 3.00695774, 3.47167185, 4.02712538, 4.49280007, 5.01087640, 5.48191963, 6.04916550, 6.51511058, 6.97297819, 7.46565499, + 8.01489405, 8.39912001, 8.91819757, 9.50000000, 10.11654065, 10.50000000, 11.03712583, 11.50000000, 12.00000000, 12.38964346, + 12.89466127, 13.43657881, 13.96013840, 14.46279912, 15.00000000, 15.39412269, 15.96662441}; + +const double WebRtcIsac_kQMeanLag3Hi[1] = { + 0.00000000}; + +const double WebRtcIsac_kQMeanLag4Hi[34] = { +-7.98331221, -7.47988769, -7.03626557, -6.52708003, -6.06982173, -5.51856292, -5.05827033, -4.45909878, -3.99125864, -3.45308135, +-3.02328139, -2.47297273, -1.94341995, -1.44699056, -0.93612243, -0.43012406, 0.01120357, 0.44054812, 0.93199883, 1.45669587, + 1.97218322, 2.50187419, 2.98748690, 3.49343202, 4.01660147, 4.50984306, 5.01402683, 5.58936797, 5.91787793, 6.59998900, + 6.85034315, 7.53503316, 7.87711194, 8.53631648}; + +const double WebRtcIsac_kQPitchLagStepsizeHi = 0.500000; + +/* transform matrix */ +const double WebRtcIsac_kTransform[4][4] = { +{-0.50000000, -0.50000000, -0.50000000, -0.50000000}, +{ 0.67082039, 0.22360680, -0.22360680, -0.67082039}, +{ 0.50000000, -0.50000000, -0.50000000, 0.50000000}, +{ 0.22360680, -0.67082039, 0.67082039, -0.22360680}}; + +/* transpose transform matrix */ +const double WebRtcIsac_kTransformTranspose[4][4] = { +{-0.50000000, 0.67082039, 0.50000000, 0.22360680}, +{-0.50000000, 0.22360680, -0.50000000, -0.67082039}, +{-0.50000000, -0.22360680, -0.50000000, 0.67082039}, +{-0.50000000, -0.67082039, 0.50000000, -0.22360680}}; + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.h new file mode 100644 index 0000000000..6a57c87264 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_lag_tables.h + * + * This file contains tables for the pitch filter side-info in the entropy coder. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_LAG_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_LAG_TABLES_H_ + +#include "typedefs.h" // NOLINT(build/include) +/* header file for coding tables for the pitch filter side-info in the entropy coder */ +/********************* Pitch Filter Lag Coefficient Tables ************************/ + +/* tables for use with small pitch gain */ + +/* cdfs for quantized pitch lags */ +extern const uint16_t WebRtcIsac_kQPitchLagCdf1Lo[127]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf2Lo[20]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf3Lo[2]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf4Lo[10]; + +extern const uint16_t *WebRtcIsac_kQPitchLagCdfPtrLo[4]; + +/* size of first cdf table */ +extern const uint16_t WebRtcIsac_kQPitchLagCdfSizeLo[1]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsac_kQIndexLowerLimitLagLo[4]; +extern const int16_t WebRtcIsac_kQIndexUpperLimitLagLo[4]; + +/* initial index for arithmetic decoder */ +extern const uint16_t WebRtcIsac_kQInitIndexLagLo[3]; + +/* mean values of pitch filter lags */ +extern const double WebRtcIsac_kQMeanLag2Lo[19]; +extern const double WebRtcIsac_kQMeanLag3Lo[1]; +extern const double WebRtcIsac_kQMeanLag4Lo[9]; + +extern const double WebRtcIsac_kQPitchLagStepsizeLo; + + +/* tables for use with medium pitch gain */ + +/* cdfs for quantized pitch lags */ +extern const uint16_t WebRtcIsac_kQPitchLagCdf1Mid[255]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf2Mid[36]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf3Mid[2]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf4Mid[20]; + +extern const uint16_t *WebRtcIsac_kQPitchLagCdfPtrMid[4]; + +/* size of first cdf table */ +extern const uint16_t WebRtcIsac_kQPitchLagCdfSizeMid[1]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsac_kQIndexLowerLimitLagMid[4]; +extern const int16_t WebRtcIsac_kQIndexUpperLimitLagMid[4]; + +/* initial index for arithmetic decoder */ +extern const uint16_t WebRtcIsac_kQInitIndexLagMid[3]; + +/* mean values of pitch filter lags */ +extern const double WebRtcIsac_kQMeanLag2Mid[35]; +extern const double WebRtcIsac_kQMeanLag3Mid[1]; +extern const double WebRtcIsac_kQMeanLag4Mid[19]; + +extern const double WebRtcIsac_kQPitchLagStepsizeMid; + + +/* tables for use with large pitch gain */ + +/* cdfs for quantized pitch lags */ +extern const uint16_t WebRtcIsac_kQPitchLagCdf1Hi[511]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf2Hi[68]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf3Hi[2]; +extern const uint16_t WebRtcIsac_kQPitchLagCdf4Hi[35]; + +extern const uint16_t *WebRtcIsac_kQPitchLagCdfPtrHi[4]; + +/* size of first cdf table */ +extern const uint16_t WebRtcIsac_kQPitchLagCdfSizeHi[1]; + +/* index limits and ranges */ +extern const int16_t WebRtcIsac_kQindexLowerLimitLagHi[4]; +extern const int16_t WebRtcIsac_kQindexUpperLimitLagHi[4]; + +/* initial index for arithmetic decoder */ +extern const uint16_t WebRtcIsac_kQInitIndexLagHi[3]; + +/* mean values of pitch filter lags */ +extern const double WebRtcIsac_kQMeanLag2Hi[67]; +extern const double WebRtcIsac_kQMeanLag3Hi[1]; +extern const double WebRtcIsac_kQMeanLag4Hi[34]; + +extern const double WebRtcIsac_kQPitchLagStepsizeHi; + +/* transform matrix */ +extern const double WebRtcIsac_kTransform[4][4]; + +/* transpose transform matrix */ +extern const double WebRtcIsac_kTransformTranspose[4][4]; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_LAG_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/settings.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/settings.h new file mode 100644 index 0000000000..c08d72ffe3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/settings.h @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * settings.h + * + * Declaration of #defines used in the iSAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ + +/* sampling frequency (Hz) */ +#define FS 16000 + +/* number of samples per frame (either 320 (20ms), 480 (30ms) or 960 (60ms)) */ +#define INITIAL_FRAMESAMPLES 960 + + +#define MAXFFTSIZE 2048 +#define NFACTOR 11 + + + +/* do not modify the following; this will have to be modified if we + * have a 20ms framesize option */ +/**********************************************************************/ +/* miliseconds */ +#define FRAMESIZE 30 +/* number of samples per frame processed in the encoder, 480 */ +#define FRAMESAMPLES 480 /* ((FRAMESIZE*FS)/1000) */ +#define FRAMESAMPLES_HALF 240 +#define FRAMESAMPLES_QUARTER 120 +/**********************************************************************/ + + + +/* max number of samples per frame (= 60 ms frame) */ +#define MAX_FRAMESAMPLES 960 +#define MAX_SWBFRAMESAMPLES (MAX_FRAMESAMPLES * 2) +/* number of samples per 10ms frame */ +#define FRAMESAMPLES_10ms ((10*FS)/1000) +#define SWBFRAMESAMPLES_10ms (FRAMESAMPLES_10ms * 2) +/* number of samples in 30 ms frame */ +#define FRAMESAMPLES_30ms 480 +/* number of subframes */ +#define SUBFRAMES 6 +/* length of a subframe */ +#define UPDATE 80 +/* length of half a subframe (low/high band) */ +#define HALF_SUBFRAMELEN (UPDATE/2) +/* samples of look ahead (in a half-band, so actually + * half the samples of look ahead @ FS) */ +#define QLOOKAHEAD 24 /* 3 ms */ +/* order of AR model in spectral entropy coder */ +#define AR_ORDER 6 +/* order of LP model in spectral entropy coder */ +#define LP_ORDER 0 + +/* window length (masking analysis) */ +#define WINLEN 256 +/* order of low-band pole filter used to approximate masking curve */ +#define ORDERLO 12 +/* order of hi-band pole filter used to approximate masking curve */ +#define ORDERHI 6 + +#define UB_LPC_ORDER 4 +#define UB_LPC_VEC_PER_FRAME 2 +#define UB16_LPC_VEC_PER_FRAME 4 +#define UB_ACTIVE_SUBFRAMES 2 +#define UB_MAX_LPC_ORDER 6 +#define UB_INTERPOL_SEGMENTS 1 +#define UB16_INTERPOL_SEGMENTS 3 +#define LB_TOTAL_DELAY_SAMPLES 48 +enum ISACBandwidth {isac8kHz = 8, isac12kHz = 12, isac16kHz = 16}; +enum ISACBand {kIsacLowerBand = 0, kIsacUpperBand12 = 1, kIsacUpperBand16 = 2}; +enum IsacSamplingRate {kIsacWideband = 16, kIsacSuperWideband = 32}; +#define UB_LPC_GAIN_DIM SUBFRAMES +#define FB_STATE_SIZE_WORD32 6 + + +/* order for post_filter_bank */ +#define POSTQORDER 3 +/* order for pre-filterbank */ +#define QORDER 3 +/* another order */ +#define QORDER_ALL (POSTQORDER+QORDER-1) +/* for decimator */ +#define ALLPASSSECTIONS 2 + + +/* array size for byte stream in number of bytes. */ +/* The old maximum size still needed for the decoding */ +#define STREAM_SIZE_MAX 600 +#define STREAM_SIZE_MAX_30 200 /* 200 bytes=53.4 kbps @ 30 ms.framelength */ +#define STREAM_SIZE_MAX_60 400 /* 400 bytes=53.4 kbps @ 60 ms.framelength */ + +/* storage size for bit counts */ +#define BIT_COUNTER_SIZE 30 +/* maximum order of any AR model or filter */ +#define MAX_AR_MODEL_ORDER 12//50 + + +/* For pitch analysis */ +#define PITCH_FRAME_LEN (FRAMESAMPLES_HALF) /* 30 ms */ +#define PITCH_MAX_LAG 140 /* 57 Hz */ +#define PITCH_MIN_LAG 20 /* 400 Hz */ +#define PITCH_MAX_GAIN 0.45 +#define PITCH_MAX_GAIN_06 0.27 /* PITCH_MAX_GAIN*0.6 */ +#define PITCH_MAX_GAIN_Q12 1843 +#define PITCH_LAG_SPAN2 (PITCH_MAX_LAG/2-PITCH_MIN_LAG/2+5) +#define PITCH_CORR_LEN2 60 /* 15 ms */ +#define PITCH_CORR_STEP2 (PITCH_FRAME_LEN/4) +#define PITCH_BW 11 /* half the band width of correlation surface */ +#define PITCH_SUBFRAMES 4 +#define PITCH_GRAN_PER_SUBFRAME 5 +#define PITCH_SUBFRAME_LEN (PITCH_FRAME_LEN/PITCH_SUBFRAMES) +#define PITCH_UPDATE (PITCH_SUBFRAME_LEN/PITCH_GRAN_PER_SUBFRAME) +/* maximum number of peaks to be examined in correlation surface */ +#define PITCH_MAX_NUM_PEAKS 10 +#define PITCH_PEAK_DECAY 0.85 +/* For weighting filter */ +#define PITCH_WLPCORDER 6 +#define PITCH_WLPCWINLEN PITCH_FRAME_LEN +#define PITCH_WLPCASYM 0.3 /* asymmetry parameter */ +#define PITCH_WLPCBUFLEN PITCH_WLPCWINLEN +/* For pitch filter */ +/* Extra 50 for fraction and LP filters */ +#define PITCH_BUFFSIZE (PITCH_MAX_LAG + 50) +#define PITCH_INTBUFFSIZE (PITCH_FRAME_LEN+PITCH_BUFFSIZE) +/* Max rel. step for interpolation */ +#define PITCH_UPSTEP 1.5 +/* Max rel. step for interpolation */ +#define PITCH_DOWNSTEP 0.67 +#define PITCH_FRACS 8 +#define PITCH_FRACORDER 9 +#define PITCH_DAMPORDER 5 +#define PITCH_FILTDELAY 1.5f +/* stepsize for quantization of the pitch Gain */ +#define PITCH_GAIN_STEPSIZE 0.125 + + + +/* Order of high pass filter */ +#define HPORDER 2 + +/* some mathematical constants */ +/* log2(exp) */ +#define LOG2EXP 1.44269504088896 +#define PI 3.14159265358979 + +/* Maximum number of iterations allowed to limit payload size */ +#define MAX_PAYLOAD_LIMIT_ITERATION 5 + +/* Redundant Coding */ +#define RCU_BOTTLENECK_BPS 16000 +#define RCU_TRANSCODING_SCALE 0.40f +#define RCU_TRANSCODING_SCALE_INVERSE 2.5f + +#define RCU_TRANSCODING_SCALE_UB 0.50f +#define RCU_TRANSCODING_SCALE_UB_INVERSE 2.0f + +/* Define Error codes */ +/* 6000 General */ +#define ISAC_MEMORY_ALLOCATION_FAILED 6010 +#define ISAC_MODE_MISMATCH 6020 +#define ISAC_DISALLOWED_BOTTLENECK 6030 +#define ISAC_DISALLOWED_FRAME_LENGTH 6040 +#define ISAC_UNSUPPORTED_SAMPLING_FREQUENCY 6050 + +/* 6200 Bandwidth estimator */ +#define ISAC_RANGE_ERROR_BW_ESTIMATOR 6240 +/* 6400 Encoder */ +#define ISAC_ENCODER_NOT_INITIATED 6410 +#define ISAC_DISALLOWED_CODING_MODE 6420 +#define ISAC_DISALLOWED_FRAME_MODE_ENCODER 6430 +#define ISAC_DISALLOWED_BITSTREAM_LENGTH 6440 +#define ISAC_PAYLOAD_LARGER_THAN_LIMIT 6450 +#define ISAC_DISALLOWED_ENCODER_BANDWIDTH 6460 +/* 6600 Decoder */ +#define ISAC_DECODER_NOT_INITIATED 6610 +#define ISAC_EMPTY_PACKET 6620 +#define ISAC_DISALLOWED_FRAME_MODE_DECODER 6630 +#define ISAC_RANGE_ERROR_DECODE_FRAME_LENGTH 6640 +#define ISAC_RANGE_ERROR_DECODE_BANDWIDTH 6650 +#define ISAC_RANGE_ERROR_DECODE_PITCH_GAIN 6660 +#define ISAC_RANGE_ERROR_DECODE_PITCH_LAG 6670 +#define ISAC_RANGE_ERROR_DECODE_LPC 6680 +#define ISAC_RANGE_ERROR_DECODE_SPECTRUM 6690 +#define ISAC_LENGTH_MISMATCH 6730 +#define ISAC_RANGE_ERROR_DECODE_BANDWITH 6740 +#define ISAC_DISALLOWED_BANDWIDTH_MODE_DECODER 6750 +#define ISAC_DISALLOWED_LPC_MODEL 6760 +/* 6800 Call setup formats */ +#define ISAC_INCOMPATIBLE_FORMATS 6810 + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.c new file mode 100644 index 0000000000..0f6d889225 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "spectrum_ar_model_tables.h" +#include "settings.h" + +/********************* AR Coefficient Tables ************************/ +/* cdf for quantized reflection coefficient 1 */ +const uint16_t WebRtcIsac_kQArRc1Cdf[NUM_AR_RC_QUANT_BAUNDARY] = { + 0, 2, 4, 129, 7707, 57485, 65495, 65527, 65529, 65531, + 65533, 65535}; + +/* cdf for quantized reflection coefficient 2 */ +const uint16_t WebRtcIsac_kQArRc2Cdf[NUM_AR_RC_QUANT_BAUNDARY] = { + 0, 2, 4, 7, 531, 25298, 64525, 65526, 65529, 65531, + 65533, 65535}; + +/* cdf for quantized reflection coefficient 3 */ +const uint16_t WebRtcIsac_kQArRc3Cdf[NUM_AR_RC_QUANT_BAUNDARY] = { + 0, 2, 4, 6, 620, 22898, 64843, 65527, 65529, 65531, + 65533, 65535}; + +/* cdf for quantized reflection coefficient 4 */ +const uint16_t WebRtcIsac_kQArRc4Cdf[NUM_AR_RC_QUANT_BAUNDARY] = { + 0, 2, 4, 6, 35, 10034, 60733, 65506, 65529, 65531, + 65533, 65535}; + +/* cdf for quantized reflection coefficient 5 */ +const uint16_t WebRtcIsac_kQArRc5Cdf[NUM_AR_RC_QUANT_BAUNDARY] = { + 0, 2, 4, 6, 36, 7567, 56727, 65385, 65529, 65531, + 65533, 65535}; + +/* cdf for quantized reflection coefficient 6 */ +const uint16_t WebRtcIsac_kQArRc6Cdf[NUM_AR_RC_QUANT_BAUNDARY] = { + 0, 2, 4, 6, 14, 6579, 57360, 65409, 65529, 65531, + 65533, 65535}; + +/* representation levels for quantized reflection coefficient 1 */ +const int16_t WebRtcIsac_kQArRc1Levels[NUM_AR_RC_QUANT_BAUNDARY - 1] = { + -32104, -29007, -23202, -15496, -9279, -2577, 5934, 17535, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 2 */ +const int16_t WebRtcIsac_kQArRc2Levels[NUM_AR_RC_QUANT_BAUNDARY - 1] = { + -32104, -29503, -23494, -15261, -7309, -1399, 6158, 16381, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 3 */ +const int16_t WebRtcIsac_kQArRc3Levels[NUM_AR_RC_QUANT_BAUNDARY - 1] = { +-32104, -29503, -23157, -15186, -7347, -1359, 5829, 17535, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 4 */ +const int16_t WebRtcIsac_kQArRc4Levels[NUM_AR_RC_QUANT_BAUNDARY - 1] = { +-32104, -29503, -24512, -15362, -6665, -342, 6596, 14585, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 5 */ +const int16_t WebRtcIsac_kQArRc5Levels[NUM_AR_RC_QUANT_BAUNDARY - 1] = { +-32104, -29503, -24512, -15005, -6564, -106, 7123, 14920, 24512, 29503, 32104 +}; + +/* representation levels for quantized reflection coefficient 6 */ +const int16_t WebRtcIsac_kQArRc6Levels[NUM_AR_RC_QUANT_BAUNDARY - 1] = { +-32104, -29503, -24512, -15096, -6656, -37, 7036, 14847, 24512, 29503, 32104 +}; + +/* quantization boundary levels for reflection coefficients */ +const int16_t WebRtcIsac_kQArBoundaryLevels[NUM_AR_RC_QUANT_BAUNDARY] = { +-32768, -31441, -27566, -21458, -13612, -4663, 4663, 13612, 21458, 27566, 31441, +32767 +}; + +/* initial index for AR reflection coefficient quantizer and cdf table search */ +const uint16_t WebRtcIsac_kQArRcInitIndex[6] = { + 5, 5, 5, 5, 5, 5}; + +/* pointers to AR cdf tables */ +const uint16_t *WebRtcIsac_kQArRcCdfPtr[AR_ORDER] = { + WebRtcIsac_kQArRc1Cdf, WebRtcIsac_kQArRc2Cdf, WebRtcIsac_kQArRc3Cdf, + WebRtcIsac_kQArRc4Cdf, WebRtcIsac_kQArRc5Cdf, WebRtcIsac_kQArRc6Cdf +}; + +/* pointers to AR representation levels tables */ +const int16_t *WebRtcIsac_kQArRcLevelsPtr[AR_ORDER] = { + WebRtcIsac_kQArRc1Levels, WebRtcIsac_kQArRc2Levels, WebRtcIsac_kQArRc3Levels, + WebRtcIsac_kQArRc4Levels, WebRtcIsac_kQArRc5Levels, WebRtcIsac_kQArRc6Levels +}; + + +/******************** GAIN Coefficient Tables ***********************/ +/* cdf for Gain coefficient */ +const uint16_t WebRtcIsac_kQGainCdf[19] = { + 0, 2, 4, 6, 8, 10, 12, 14, 16, 1172, + 11119, 29411, 51699, 64445, 65527, 65529, 65531, 65533, 65535}; + +/* representation levels for quantized squared Gain coefficient */ +const int32_t WebRtcIsac_kQGain2Levels[18] = { +// 17, 28, 46, 76, 128, 215, 364, 709, 1268, 1960, 3405, 6078, 11286, 17827, 51918, 134498, 487432, 2048000}; + 128, 128, 128, 128, 128, 215, 364, 709, 1268, 1960, 3405, 6078, 11286, 17827, 51918, 134498, 487432, 2048000}; +/* quantization boundary levels for squared Gain coefficient */ +const int32_t WebRtcIsac_kQGain2BoundaryLevels[19] = { +0, 21, 35, 59, 99, 166, 280, 475, 815, 1414, 2495, 4505, 8397, 16405, 34431, 81359, 240497, 921600, 0x7FFFFFFF}; + +/* pointers to Gain cdf table */ +const uint16_t *WebRtcIsac_kQGainCdf_ptr[1] = {WebRtcIsac_kQGainCdf}; + +/* Gain initial index for gain quantizer and cdf table search */ +const uint16_t WebRtcIsac_kQGainInitIndex[1] = {11}; + +/************************* Cosine Tables ****************************/ +/* Cosine table */ +const int16_t WebRtcIsac_kCos[6][60] = { +{512, 512, 511, 510, 508, 507, 505, 502, 499, 496, 493, 489, 485, 480, 476, 470, 465, 459, 453, 447, +440, 433, 426, 418, 410, 402, 394, 385, 376, 367, 357, 348, 338, 327, 317, 306, 295, 284, 273, 262, +250, 238, 226, 214, 202, 190, 177, 165, 152, 139, 126, 113, 100, 87, 73, 60, 47, 33, 20, 7}, +{512, 510, 508, 503, 498, 491, 483, 473, 462, 450, 437, 422, 406, 389, 371, 352, 333, 312, 290, 268, +244, 220, 196, 171, 145, 120, 93, 67, 40, 13, -13, -40, -67, -93, -120, -145, -171, -196, -220, -244, +-268, -290, -312, -333, -352, -371, -389, -406, -422, -437, -450, -462, -473, -483, -491, -498, -503, -508, -510, -512}, +{512, 508, 502, 493, 480, 465, 447, 426, 402, 376, 348, 317, 284, 250, 214, 177, 139, 100, 60, 20, +-20, -60, -100, -139, -177, -214, -250, -284, -317, -348, -376, -402, -426, -447, -465, -480, -493, -502, -508, -512, +-512, -508, -502, -493, -480, -465, -447, -426, -402, -376, -348, -317, -284, -250, -214, -177, -139, -100, -60, -20}, +{511, 506, 495, 478, 456, 429, 398, 362, 322, 279, 232, 183, 133, 80, 27, -27, -80, -133, -183, -232, +-279, -322, -362, -398, -429, -456, -478, -495, -506, -511, -511, -506, -495, -478, -456, -429, -398, -362, -322, -279, +-232, -183, -133, -80, -27, 27, 80, 133, 183, 232, 279, 322, 362, 398, 429, 456, 478, 495, 506, 511}, +{511, 502, 485, 459, 426, 385, 338, 284, 226, 165, 100, 33, -33, -100, -165, -226, -284, -338, -385, -426, +-459, -485, -502, -511, -511, -502, -485, -459, -426, -385, -338, -284, -226, -165, -100, -33, 33, 100, 165, 226, +284, 338, 385, 426, 459, 485, 502, 511, 511, 502, 485, 459, 426, 385, 338, 284, 226, 165, 100, 33}, +{510, 498, 473, 437, 389, 333, 268, 196, 120, 40, -40, -120, -196, -268, -333, -389, -437, -473, -498, -510, +-510, -498, -473, -437, -389, -333, -268, -196, -120, -40, 40, 120, 196, 268, 333, 389, 437, 473, 498, 510, +510, 498, 473, 437, 389, 333, 268, 196, 120, 40, -40, -120, -196, -268, -333, -389, -437, -473, -498, -510} +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.h new file mode 100644 index 0000000000..82ab363adb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * spectrum_ar_model_tables.h + * + * This file contains definitions of tables with AR coefficients, + * Gain coefficients and cosine tables. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SPECTRUM_AR_MODEL_TABLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SPECTRUM_AR_MODEL_TABLES_H_ + +#include "structs.h" + +#define NUM_AR_RC_QUANT_BAUNDARY 12 + +/********************* AR Coefficient Tables ************************/ +/* cdf for quantized reflection coefficient 1 */ +extern const uint16_t WebRtcIsac_kQArRc1Cdf[NUM_AR_RC_QUANT_BAUNDARY]; + +/* cdf for quantized reflection coefficient 2 */ +extern const uint16_t WebRtcIsac_kQArRc2Cdf[NUM_AR_RC_QUANT_BAUNDARY]; + +/* cdf for quantized reflection coefficient 3 */ +extern const uint16_t WebRtcIsac_kQArRc3Cdf[NUM_AR_RC_QUANT_BAUNDARY]; + +/* cdf for quantized reflection coefficient 4 */ +extern const uint16_t WebRtcIsac_kQArRc4Cdf[NUM_AR_RC_QUANT_BAUNDARY]; + +/* cdf for quantized reflection coefficient 5 */ +extern const uint16_t WebRtcIsac_kQArRc5Cdf[NUM_AR_RC_QUANT_BAUNDARY]; + +/* cdf for quantized reflection coefficient 6 */ +extern const uint16_t WebRtcIsac_kQArRc6Cdf[NUM_AR_RC_QUANT_BAUNDARY]; + +/* quantization boundary levels for reflection coefficients */ +extern const int16_t WebRtcIsac_kQArBoundaryLevels[NUM_AR_RC_QUANT_BAUNDARY]; + +/* initial indices for AR reflection coefficient quantizer and cdf table search */ +extern const uint16_t WebRtcIsac_kQArRcInitIndex[AR_ORDER]; + +/* pointers to AR cdf tables */ +extern const uint16_t *WebRtcIsac_kQArRcCdfPtr[AR_ORDER]; + +/* pointers to AR representation levels tables */ +extern const int16_t *WebRtcIsac_kQArRcLevelsPtr[AR_ORDER]; + + +/******************** GAIN Coefficient Tables ***********************/ +/* cdf for Gain coefficient */ +extern const uint16_t WebRtcIsac_kQGainCdf[19]; + +/* representation levels for quantized Gain coefficient */ +extern const int32_t WebRtcIsac_kQGain2Levels[18]; + +/* squared quantization boundary levels for Gain coefficient */ +extern const int32_t WebRtcIsac_kQGain2BoundaryLevels[19]; + +/* pointer to Gain cdf table */ +extern const uint16_t *WebRtcIsac_kQGainCdf_ptr[1]; + +/* Gain initial index for gain quantizer and cdf table search */ +extern const uint16_t WebRtcIsac_kQGainInitIndex[1]; + +/************************* Cosine Tables ****************************/ +/* Cosine table */ +extern const int16_t WebRtcIsac_kCos[6][60]; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SPECTRUM_AR_MODEL_TABLES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h new file mode 100644 index 0000000000..ef4282b521 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h @@ -0,0 +1,495 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * structs.h + * + * This header file contains all the structs used in the ISAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ + +#include "modules/audio_coding/codecs/isac/bandwidth_info.h" +#include "modules/audio_coding/codecs/isac/main/include/isac.h" +#include "modules/audio_coding/codecs/isac/main/source/settings.h" +#include "typedefs.h" // NOLINT(build/include) + +typedef struct Bitstreamstruct { + + uint8_t stream[STREAM_SIZE_MAX]; + uint32_t W_upper; + uint32_t streamval; + uint32_t stream_index; + +} Bitstr; + +typedef struct { + + double DataBufferLo[WINLEN]; + double DataBufferHi[WINLEN]; + + double CorrBufLo[ORDERLO+1]; + double CorrBufHi[ORDERHI+1]; + + float PreStateLoF[ORDERLO+1]; + float PreStateLoG[ORDERLO+1]; + float PreStateHiF[ORDERHI+1]; + float PreStateHiG[ORDERHI+1]; + float PostStateLoF[ORDERLO+1]; + float PostStateLoG[ORDERLO+1]; + float PostStateHiF[ORDERHI+1]; + float PostStateHiG[ORDERHI+1]; + + double OldEnergy; + +} MaskFiltstr; + + +typedef struct { + + //state vectors for each of the two analysis filters + double INSTAT1[2*(QORDER-1)]; + double INSTAT2[2*(QORDER-1)]; + double INSTATLA1[2*(QORDER-1)]; + double INSTATLA2[2*(QORDER-1)]; + double INLABUF1[QLOOKAHEAD]; + double INLABUF2[QLOOKAHEAD]; + + float INSTAT1_float[2*(QORDER-1)]; + float INSTAT2_float[2*(QORDER-1)]; + float INSTATLA1_float[2*(QORDER-1)]; + float INSTATLA2_float[2*(QORDER-1)]; + float INLABUF1_float[QLOOKAHEAD]; + float INLABUF2_float[QLOOKAHEAD]; + + /* High pass filter */ + double HPstates[HPORDER]; + float HPstates_float[HPORDER]; + +} PreFiltBankstr; + + +typedef struct { + + //state vectors for each of the two analysis filters + double STATE_0_LOWER[2*POSTQORDER]; + double STATE_0_UPPER[2*POSTQORDER]; + + /* High pass filter */ + double HPstates1[HPORDER]; + double HPstates2[HPORDER]; + + float STATE_0_LOWER_float[2*POSTQORDER]; + float STATE_0_UPPER_float[2*POSTQORDER]; + + float HPstates1_float[HPORDER]; + float HPstates2_float[HPORDER]; + +} PostFiltBankstr; + +typedef struct { + + //data buffer for pitch filter + double ubuf[PITCH_BUFFSIZE]; + + //low pass state vector + double ystate[PITCH_DAMPORDER]; + + //old lag and gain + double oldlagp[1]; + double oldgainp[1]; + +} PitchFiltstr; + +typedef struct { + + //data buffer + double buffer[PITCH_WLPCBUFLEN]; + + //state vectors + double istate[PITCH_WLPCORDER]; + double weostate[PITCH_WLPCORDER]; + double whostate[PITCH_WLPCORDER]; + + //LPC window -> should be a global array because constant + double window[PITCH_WLPCWINLEN]; + +} WeightFiltstr; + +typedef struct { + + //for inital estimator + double dec_buffer[PITCH_CORR_LEN2 + PITCH_CORR_STEP2 + + PITCH_MAX_LAG/2 - PITCH_FRAME_LEN/2+2]; + double decimator_state[2*ALLPASSSECTIONS+1]; + double hp_state[2]; + + double whitened_buf[QLOOKAHEAD]; + + double inbuf[QLOOKAHEAD]; + + PitchFiltstr PFstr_wght; + PitchFiltstr PFstr; + WeightFiltstr Wghtstr; + +} PitchAnalysisStruct; + + + +/* Have instance of struct together with other iSAC structs */ +typedef struct { + + /* Previous frame length (in ms) */ + int32_t prev_frame_length; + + /* Previous RTP timestamp from received + packet (in samples relative beginning) */ + int32_t prev_rec_rtp_number; + + /* Send timestamp for previous packet (in ms using timeGetTime()) */ + uint32_t prev_rec_send_ts; + + /* Arrival time for previous packet (in ms using timeGetTime()) */ + uint32_t prev_rec_arr_ts; + + /* rate of previous packet, derived from RTP timestamps (in bits/s) */ + float prev_rec_rtp_rate; + + /* Time sinse the last update of the BN estimate (in ms) */ + uint32_t last_update_ts; + + /* Time sinse the last reduction (in ms) */ + uint32_t last_reduction_ts; + + /* How many times the estimate was update in the beginning */ + int32_t count_tot_updates_rec; + + /* The estimated bottle neck rate from there to here (in bits/s) */ + int32_t rec_bw; + float rec_bw_inv; + float rec_bw_avg; + float rec_bw_avg_Q; + + /* The estimated mean absolute jitter value, + as seen on this side (in ms) */ + float rec_jitter; + float rec_jitter_short_term; + float rec_jitter_short_term_abs; + float rec_max_delay; + float rec_max_delay_avg_Q; + + /* (assumed) bitrate for headers (bps) */ + float rec_header_rate; + + /* The estimated bottle neck rate from here to there (in bits/s) */ + float send_bw_avg; + + /* The estimated mean absolute jitter value, as seen on + the other siee (in ms) */ + float send_max_delay_avg; + + // number of packets received since last update + int num_pkts_rec; + + int num_consec_rec_pkts_over_30k; + + // flag for marking that a high speed network has been + // detected downstream + int hsn_detect_rec; + + int num_consec_snt_pkts_over_30k; + + // flag for marking that a high speed network has + // been detected upstream + int hsn_detect_snd; + + uint32_t start_wait_period; + + int in_wait_period; + + int change_to_WB; + + uint32_t senderTimestamp; + uint32_t receiverTimestamp; + //enum IsacSamplingRate incomingStreamSampFreq; + uint16_t numConsecLatePkts; + float consecLatency; + int16_t inWaitLatePkts; + + IsacBandwidthInfo external_bw_info; +} BwEstimatorstr; + + +typedef struct { + + /* boolean, flags if previous packet exceeded B.N. */ + int PrevExceed; + /* ms */ + int ExceedAgo; + /* packets left to send in current burst */ + int BurstCounter; + /* packets */ + int InitCounter; + /* ms remaining in buffer when next packet will be sent */ + double StillBuffered; + +} RateModel; + + +typedef struct { + + unsigned int SpaceAlloced; + unsigned int MaxPermAlloced; + double Tmp0[MAXFFTSIZE]; + double Tmp1[MAXFFTSIZE]; + double Tmp2[MAXFFTSIZE]; + double Tmp3[MAXFFTSIZE]; + int Perm[MAXFFTSIZE]; + int factor [NFACTOR]; + +} FFTstr; + + +/* The following strutc is used to store data from encoding, to make it + fast and easy to construct a new bitstream with a different Bandwidth + estimate. All values (except framelength and minBytes) is double size to + handle 60 ms of data. +*/ +typedef struct { + + /* Used to keep track of if it is first or second part of 60 msec packet */ + int startIdx; + + /* Frame length in samples */ + int16_t framelength; + + /* Pitch Gain */ + int pitchGain_index[2]; + + /* Pitch Lag */ + double meanGain[2]; + int pitchIndex[PITCH_SUBFRAMES*2]; + + /* LPC */ + int LPCindex_s[108*2]; /* KLT_ORDER_SHAPE = 108 */ + int LPCindex_g[12*2]; /* KLT_ORDER_GAIN = 12 */ + double LPCcoeffs_lo[(ORDERLO+1)*SUBFRAMES*2]; + double LPCcoeffs_hi[(ORDERHI+1)*SUBFRAMES*2]; + + /* Encode Spec */ + int16_t fre[FRAMESAMPLES]; + int16_t fim[FRAMESAMPLES]; + int16_t AvgPitchGain[2]; + + /* Used in adaptive mode only */ + int minBytes; + +} IsacSaveEncoderData; + + +typedef struct { + + int indexLPCShape[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + double lpcGain[SUBFRAMES<<1]; + int lpcGainIndex[SUBFRAMES<<1]; + + Bitstr bitStreamObj; + + int16_t realFFT[FRAMESAMPLES_HALF]; + int16_t imagFFT[FRAMESAMPLES_HALF]; +} ISACUBSaveEncDataStruct; + + + +typedef struct { + + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PreFiltBankstr prefiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + PitchAnalysisStruct pitchanalysisstr_obj; + FFTstr fftstr_obj; + IsacSaveEncoderData SaveEnc_obj; + + int buffer_index; + int16_t current_framesamples; + + float data_buffer_float[FRAMESAMPLES_30ms]; + + int frame_nb; + double bottleneck; + int16_t new_framelength; + double s2nr; + + /* Maximum allowed number of bits for a 30 msec packet */ + int16_t payloadLimitBytes30; + /* Maximum allowed number of bits for a 30 msec packet */ + int16_t payloadLimitBytes60; + /* Maximum allowed number of bits for both 30 and 60 msec packet */ + int16_t maxPayloadBytes; + /* Maximum allowed rate in bytes per 30 msec packet */ + int16_t maxRateInBytes; + + /*--- + If set to 1 iSAC will not addapt the frame-size, if used in + channel-adaptive mode. The initial value will be used for all rates. + ---*/ + int16_t enforceFrameSize; + + /*----- + This records the BWE index the encoder injected into the bit-stream. + It will be used in RCU. The same BWE index of main payload will be in + the redundant payload. We can not retrive it from BWE because it is + a recursive procedure (WebRtcIsac_GetDownlinkBwJitIndexImpl) and has to be + called only once per each encode. + -----*/ + int16_t lastBWIdx; +} ISACLBEncStruct; + +typedef struct { + + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PreFiltBankstr prefiltbankstr_obj; + FFTstr fftstr_obj; + ISACUBSaveEncDataStruct SaveEnc_obj; + + int buffer_index; + float data_buffer_float[MAX_FRAMESAMPLES + + LB_TOTAL_DELAY_SAMPLES]; + double bottleneck; + /* Maximum allowed number of bits for a 30 msec packet */ + //int16_t payloadLimitBytes30; + /* Maximum allowed number of bits for both 30 and 60 msec packet */ + //int16_t maxPayloadBytes; + int16_t maxPayloadSizeBytes; + + double lastLPCVec[UB_LPC_ORDER]; + int16_t numBytesUsed; + int16_t lastJitterInfo; +} ISACUBEncStruct; + + + +typedef struct { + + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PostFiltBankstr postfiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + FFTstr fftstr_obj; + +} ISACLBDecStruct; + +typedef struct { + + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PostFiltBankstr postfiltbankstr_obj; + FFTstr fftstr_obj; + +} ISACUBDecStruct; + + + +typedef struct { + + ISACLBEncStruct ISACencLB_obj; + ISACLBDecStruct ISACdecLB_obj; +} ISACLBStruct; + + +typedef struct { + + ISACUBEncStruct ISACencUB_obj; + ISACUBDecStruct ISACdecUB_obj; +} ISACUBStruct; + +/* + This struct is used to take a snapshot of the entropy coder and LPC gains + right before encoding LPC gains. This allows us to go back to that state + if we like to limit the payload size. +*/ +typedef struct { + /* 6 lower-band & 6 upper-band */ + double loFiltGain[SUBFRAMES]; + double hiFiltGain[SUBFRAMES]; + /* Upper boundary of interval W */ + uint32_t W_upper; + uint32_t streamval; + /* Index to the current position in bytestream */ + uint32_t stream_index; + uint8_t stream[3]; +} transcode_obj; + +typedef struct { + // TODO(kwiberg): The size of these tables could be reduced by storing floats + // instead of doubles, and by making use of the identity cos(x) = + // sin(x+pi/2). They could also be made global constants that we fill in at + // compile time. + double costab1[FRAMESAMPLES_HALF]; + double sintab1[FRAMESAMPLES_HALF]; + double costab2[FRAMESAMPLES_QUARTER]; + double sintab2[FRAMESAMPLES_QUARTER]; +} TransformTables; + +typedef struct { + // lower-band codec instance + ISACLBStruct instLB; + // upper-band codec instance + ISACUBStruct instUB; + + // Bandwidth Estimator and model for the rate. + BwEstimatorstr bwestimator_obj; + RateModel rate_data_obj; + double MaxDelay; + + /* 0 = adaptive; 1 = instantaneous */ + int16_t codingMode; + + // overall bottleneck of the codec + int32_t bottleneck; + + // QMF Filter state + int32_t analysisFBState1[FB_STATE_SIZE_WORD32]; + int32_t analysisFBState2[FB_STATE_SIZE_WORD32]; + int32_t synthesisFBState1[FB_STATE_SIZE_WORD32]; + int32_t synthesisFBState2[FB_STATE_SIZE_WORD32]; + + // Error Code + int16_t errorCode; + + // bandwidth of the encoded audio 8, 12 or 16 kHz + enum ISACBandwidth bandwidthKHz; + // Sampling rate of audio, encoder and decode, 8 or 16 kHz + enum IsacSamplingRate encoderSamplingRateKHz; + enum IsacSamplingRate decoderSamplingRateKHz; + // Flag to keep track of initializations, lower & upper-band + // encoder and decoder. + int16_t initFlag; + + // Flag to to indicate signal bandwidth switch + int16_t resetFlag_8kHz; + + // Maximum allowed rate, measured in Bytes per 30 ms. + int16_t maxRateBytesPer30Ms; + // Maximum allowed payload-size, measured in Bytes. + int16_t maxPayloadSizeBytes; + /* The expected sampling rate of the input signal. Valid values are 16000 + * and 32000. This is not the operation sampling rate of the codec. */ + uint16_t in_sample_rate_hz; + + // Trig tables for WebRtcIsac_Time2Spec and WebRtcIsac_Spec2time. + TransformTables transform_tables; +} ISACMainStruct; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/transform.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/transform.c new file mode 100644 index 0000000000..8992897f45 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/transform.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "settings.h" +#include "fft.h" +#include "codec.h" +#include "os_specific_inline.h" +#include + +void WebRtcIsac_InitTransform(TransformTables* tables) { + int k; + double fact, phase; + + fact = PI / (FRAMESAMPLES_HALF); + phase = 0.0; + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tables->costab1[k] = cos(phase); + tables->sintab1[k] = sin(phase); + phase += fact; + } + + fact = PI * ((double) (FRAMESAMPLES_HALF - 1)) / ((double) FRAMESAMPLES_HALF); + phase = 0.5 * fact; + for (k = 0; k < FRAMESAMPLES_QUARTER; k++) { + tables->costab2[k] = cos(phase); + tables->sintab2[k] = sin(phase); + phase += fact; + } +} + +void WebRtcIsac_Time2Spec(const TransformTables* tables, + double* inre1, + double* inre2, + int16_t* outreQ7, + int16_t* outimQ7, + FFTstr* fftstr_obj) { + int k; + int dims[1]; + double tmp1r, tmp1i, xr, xi, yr, yi, fact; + double tmpre[FRAMESAMPLES_HALF], tmpim[FRAMESAMPLES_HALF]; + + + dims[0] = FRAMESAMPLES_HALF; + + + /* Multiply with complex exponentials and combine into one complex vector */ + fact = 0.5 / sqrt(FRAMESAMPLES_HALF); + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tmp1r = tables->costab1[k]; + tmp1i = tables->sintab1[k]; + tmpre[k] = (inre1[k] * tmp1r + inre2[k] * tmp1i) * fact; + tmpim[k] = (inre2[k] * tmp1r - inre1[k] * tmp1i) * fact; + } + + + /* Get DFT */ + WebRtcIsac_Fftns(1, dims, tmpre, tmpim, -1, 1.0, fftstr_obj); + + /* Use symmetry to separate into two complex vectors and center frames in time around zero */ + for (k = 0; k < FRAMESAMPLES_QUARTER; k++) { + xr = tmpre[k] + tmpre[FRAMESAMPLES_HALF - 1 - k]; + yi = -tmpre[k] + tmpre[FRAMESAMPLES_HALF - 1 - k]; + xi = tmpim[k] - tmpim[FRAMESAMPLES_HALF - 1 - k]; + yr = tmpim[k] + tmpim[FRAMESAMPLES_HALF - 1 - k]; + + tmp1r = tables->costab2[k]; + tmp1i = tables->sintab2[k]; + outreQ7[k] = (int16_t)WebRtcIsac_lrint((xr * tmp1r - xi * tmp1i) * 128.0); + outimQ7[k] = (int16_t)WebRtcIsac_lrint((xr * tmp1i + xi * tmp1r) * 128.0); + outreQ7[FRAMESAMPLES_HALF - 1 - k] = (int16_t)WebRtcIsac_lrint((-yr * tmp1i - yi * tmp1r) * 128.0); + outimQ7[FRAMESAMPLES_HALF - 1 - k] = (int16_t)WebRtcIsac_lrint((-yr * tmp1r + yi * tmp1i) * 128.0); + } +} + +void WebRtcIsac_Spec2time(const TransformTables* tables, + double* inre, + double* inim, + double* outre1, + double* outre2, + FFTstr* fftstr_obj) { + int k; + double tmp1r, tmp1i, xr, xi, yr, yi, fact; + + int dims; + + dims = FRAMESAMPLES_HALF; + + for (k = 0; k < FRAMESAMPLES_QUARTER; k++) { + /* Move zero in time to beginning of frames */ + tmp1r = tables->costab2[k]; + tmp1i = tables->sintab2[k]; + xr = inre[k] * tmp1r + inim[k] * tmp1i; + xi = inim[k] * tmp1r - inre[k] * tmp1i; + yr = -inim[FRAMESAMPLES_HALF - 1 - k] * tmp1r - inre[FRAMESAMPLES_HALF - 1 - k] * tmp1i; + yi = -inre[FRAMESAMPLES_HALF - 1 - k] * tmp1r + inim[FRAMESAMPLES_HALF - 1 - k] * tmp1i; + + /* Combine into one vector, z = x + j * y */ + outre1[k] = xr - yi; + outre1[FRAMESAMPLES_HALF - 1 - k] = xr + yi; + outre2[k] = xi + yr; + outre2[FRAMESAMPLES_HALF - 1 - k] = -xi + yr; + } + + + /* Get IDFT */ + WebRtcIsac_Fftns(1, &dims, outre1, outre2, 1, FRAMESAMPLES_HALF, fftstr_obj); + + + /* Demodulate and separate */ + fact = sqrt(FRAMESAMPLES_HALF); + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tmp1r = tables->costab1[k]; + tmp1i = tables->sintab1[k]; + xr = (outre1[k] * tmp1r - outre2[k] * tmp1i) * fact; + outre2[k] = (outre2[k] * tmp1r + outre1[k] * tmp1i) * fact; + outre1[k] = xr; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACLongtest.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACLongtest.txt new file mode 100644 index 0000000000..3f05224a0e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACLongtest.txt @@ -0,0 +1,433 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + + + +if [ "$1" = "x64" ] || [ "$2" = "x64" ] || [ "$#" -eq 0 ] + then + PLATFORM=_X64 + ISAC=../x64/Release/ReleaseTest-API_2005.exe +elif [ "$1" = "LINUX" ] || [ "$2" = "LINUX" ] + then + PLATFORM=_linux + ISAC=../ReleaseTest-API/isacswtest +else + PLATFORM=_2005 + ISAC=../win32/Release/ReleaseTest-API_2005.exe +fi + +if [ "$#" -eq 0 ] || [ "$1" = "all" ] || [ "$1" = "wb" ] + then + LOGFILE=logNormal"$PLATFORM".txt + echo "START ISAC WB TEST" > $LOGFILE + echo >> $LOGFILE + + INFILES=$(cat InputFiles.txt) + SUBSET=$(cat InputFilesFew.txt) + CHANNELFILES=$(cat ChannelFiles.txt) + CHANNELLIST=($(cat ChannelFiles.txt)) + INDIR=../data/orig + OUTDIR=../dataqa"$PLATFORM" + mkdir -p $OUTDIR + rm -f $OUTDIR/* + + idx=0 + RATE=10000 + FRAMESIZE=30 + + + for file in $INFILES # loop over all input files + do + + echo "Input file: " $file + echo "-----------------------------------" + echo "Instantaneous with RATE " $RATE ", and Frame-size " $FRAMESIZE + $ISAC -I -B $RATE -FL $FRAMESIZE -FS 16 $INDIR/"$file" $OUTDIR/i_"$FRAMESIZE"_"$RATE"_"$file" >> $LOGFILE + echo + + name="${CHANNELLIST[$idx]}" + echo "Adaptive with channel file: " $name + + $ISAC -B $INDIR/${CHANNELLIST[$idx]} -FS 16 $INDIR/"$file" $OUTDIR/a_${name%.*}_"$file" >> $LOGFILE + + echo + echo + +# alternate between 30 & 60 ms. + if [ $FRAMESIZE -eq 30 ] + then + FRAMESIZE=60 + else + FRAMESIZE=30 + fi + +# rate between 10000 to 32000 bits/sec + if [ $RATE -le 30000 ] + then + let "RATE=RATE+2000" + else + let "RATE=10000" + fi + +# there are only three channel file + if [ $idx -ge 2 ]; then + idx=0 + else + let "idx=idx+1" + fi + + done + + idx=0 + +# loop over the subset of input files + for file in $SUBSET + do + + if [ $idx -eq 0 ]; then + $ISAC -B $INDIR/${CHANNELLIST[0]} -FL 30 -FIXED_FL -FS 16 $INDIR/"$file" $OUTDIR/a30_"$file" >> $LOGFILE + idx=1 + else + $ISAC -B $INDIR/${CHANNELLIST[0]} -FL 60 -FIXED_FL -FS 16 $INDIR/"$file" $OUTDIR/a60_"$file" >> $LOGFILE + idx=0 + fi + done + + $ISAC -B $INDIR/${CHANNELLIST[0]} -INITRATE 25000 -FL 30 -FS 16 $INDIR/"$file" $OUTDIR/a60_Init25kbps_"$file" >> $LOGFILE + + echo + echo WIDEBAND DONE! + echo + echo +fi + +if [ "$#" -eq 0 ] || [ "$1" = "all" ] || [ "$1" = "swb" ] + then + + LOGFILE=logNormal_SWB"$PLATFORM".txt + echo "START ISAC SWB TEST" > $LOGFILE + echo >> $LOGFILE + + echo STARTING TO TEST SUPER-WIDEBAND + + INFILES=$(cat InputFilesSWB.txt) + INDIR=../data/origswb + OUTDIR=../dataqaswb"$PLATFORM" + mkdir -p $OUTDIR + rm -f $OUTDIR/* + + for file in $INFILES + do + echo + echo "Input file: " $file + echo "--------------------------------" + for RATE in 12000 20000 32000 38000 45000 50000 56000 + do + + echo "Rate " $RATE + $ISAC -I -B $RATE -FL 30 -FS 32 $INDIR/"$file" $OUTDIR/swb_"$RATE"_"$file" >> $LOGFILE + echo + + done + + done +fi + +if [ "$#" -eq 0 ] || [ "$1" = "all" ] || [ "$1" = "API" ] + then + + LOGFILE_API=logNormal_API"$PLATFORM".txt + echo + echo + echo "START ISAC API TEST" > $LOGFILE_API + echo >> $LOGFILE_API + idx=1 + echo " Test Enforcement of frame-size" + echo "========================================================================================" + mkdir -p ../FrameSizeLim"$PLATFORM" + rm -f ../FrameSizeLim"$PLATFORM"/* + echo + echo "-- No enforcement; BN 10000" + echo + $ISAC -B 10000 -FS 16 ../data/orig/speech_and_misc_WB.pcm \ + ../FrameSizeLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Now Enforce 30 ms frame size with the same bottleneck" + echo "There should not be any 60 ms frame" + echo + $ISAC -B 10000 -FL 30 -FIXED_FL -FS 16 ../data/orig/speech_and_misc_WB.pcm \ + ../FrameSizeLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- No enforcement; BN 32000" + echo + $ISAC -B 32000 -FS 16 ../data/orig/speech_and_misc_WB.pcm \ + ../FrameSizeLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Now Enforce 60 ms frame size with the same bottleneck" + echo "There should not be any 30 ms frame" + echo + $ISAC -B 32000 -FL 60 -FIXED_FL -FS 16 ../data/orig/speech_and_misc_WB.pcm \ + ../FrameSizeLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo + echo + echo + + echo " Test Limiting of Payload Size and Rate" + echo "========================================================================================" + mkdir -p ../PayloadLim"$PLATFORM" + rm -f ../PayloadLim"$PLATFORM"/* + echo + echo + echo "-- No Limit, frame-size 60 ms, WIDEBAND" + echo + $ISAC -I -B 32000 -FL 60 -FS 16 ../data/orig/speech_and_misc_WB.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Payload-size limit of 250, frame-size 60 ms, WIDEBAND" + echo + $ISAC -I -B 32000 -FL 60 -FS 16 -MAX 250 ../data/orig/speech_and_misc_WB.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Rate limit of 33 kbps for 60 ms frame-size" + echo + $ISAC -I -B 32000 -FL 60 -FS 16 -MAXRATE 33000 ../data/orig/speech_and_misc_WB.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + echo + echo + + echo "-- No Limit, frame-size 30 ms, WIDEBAND" + echo + $ISAC -I -B 32000 -FL 30 -FS 16 ../data/orig/speech_and_misc_WB.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Payload-size limit of 130, frame-size 30 ms, WIDEBAND" + echo + $ISAC -I -B 32000 -FL 30 -FS 16 -MAX 130 ../data/orig/speech_and_misc_WB.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Rate limit of 33 kbps for 30 ms frame-size, wideband" + echo + $ISAC -I -B 32000 -FL 30 -FS 16 -MAXRATE 33000 ../data/orig/speech_and_misc_WB.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + echo + echo + + echo "-- No limit for 32 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 32000 -FL 30 -FS 32 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Payload limit of 130 bytes for 32 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 32000 -FL 30 -FS 32 -MAX 130 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + echo + echo + + echo "-- No limit, Rate 45 kbps, 30 ms, SUPER-WIDEBAND, 12 kHz" + echo + $ISAC -I -B 45000 -FL 30 -FS 32 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Rate limit of 46 kbps for 42 kbps, 30 ms, SUPER-WIDEBAND, 12 kHz" + echo + $ISAC -I -B 45000 -FL 30 -FS 32 -MAXRATE 46000 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Payload limit of 170 bytes for 45 kbps, 30 ms, SUPER-WIDEBAND, 12 kHz" + echo + $ISAC -I -B 45000 -FL 30 -FS 32 -MAX 170 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + echo + echo + + echo "-- No limit for 56 kbps, 30 ms, SUPER-WIDEBAND, 16 kHz" + echo + $ISAC -I -B 56000 -FL 30 -FS 32 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Payload limit of 200 bytes for 56 kbps 30 ms, SUPER-WIDEBAND, 16 kHz" + echo + $ISAC -I -B 56000 -FL 30 -FS 32 -MAX 200 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + echo "-- Rate limit of 57 kbps for 56 kbps 30 ms, SUPER-WIDEBAND, 16 kHz" + echo + $ISAC -I -B 56000 -FL 30 -FS 32 -MAXRATE 57000 ../data/origswb/jstest_32.pcm \ + ../PayloadLim"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + echo + echo + echo + echo + echo + + echo " Test Trans-Coding" + echo "========================================================================================" + mkdir -p ../Transcoding"$PLATFORM" + rm -f ../Transcoding"$PLATFORM"/* + echo + echo + echo "-- 20 kbps, 30 ms, WIDEBAND" + echo + $ISAC -I -B 20000 -FL 30 -FS 16 ../data/orig/speech_and_misc_WB.pcm \ + ../Transcoding"$PLATFORM"/APITest_refTrans20WB.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 32 kbps trans-coding to 20 kbps, 30 ms, WIDEBAND" + echo + $ISAC -I -B 32000 -FL 30 -FS 16 -T 20000 ../Transcoding"$PLATFORM"/APITest_32T20.pcm \ + ../data/orig/speech_and_misc_WB.pcm ../Transcoding"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + + echo + echo + echo "-- 38 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 38000 -FL 30 -FS 32 ../data/origswb/jstest_32.pcm \ + ../Transcoding"$PLATFORM"/APITest_refTrans38.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 45 kbps trans-coding to 38 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 45000 -FL 30 -FS 32 -T 38000 ../Transcoding"$PLATFORM"/APITest_45T38.pcm \ + ../data/origswb/jstest_32.pcm ../Transcoding"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + + echo + echo + echo "-- 20 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 20000 -FL 30 -FS 32 ../data/origswb/jstest_32.pcm \ + ../Transcoding"$PLATFORM"/APITest_refTrans20SWB.pcm >> $LOGFILE_API + let "idx=idx+1" + + echo + echo + + echo "-- 45 kbps trans-coding to 20 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 45000 -FL 30 -FS 32 -T 20000 ../Transcoding"$PLATFORM"/APITest_45T20.pcm \ + ../data/origswb/jstest_32.pcm ../Transcoding"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + echo + echo + echo "-- 50 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 50000 -FL 30 -FS 32 ../data/origswb/jstest_32.pcm \ + ../Transcoding"$PLATFORM"/APITest_refTrans50.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 56 kbps trans-coding to 50 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 56000 -FL 30 -FS 32 -T 50000 ../Transcoding"$PLATFORM"/APITest_56T50.pcm \ + ../data/origswb/jstest_32.pcm ../Transcoding"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 56 kbps trans-coding to 20 kbps, 30 ms, SUPER-WIDEBAND" + echo + $ISAC -I -B 56000 -FL 30 -FS 32 -T 20000 ../Transcoding"$PLATFORM"/APITest_56T20.pcm \ + ../data/origswb/jstest_32.pcm ../Transcoding"$PLATFORM"/APITest_"$idx".pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo "________________________________________________________" + echo + echo + echo + echo + echo + + echo " Test FEC" + echo "========================================================================================" + mkdir -p ../FEC"$PLATFORM" + rm -f ../FEC"$PLATFORM"/* + echo + echo + echo "-- 32 kbps with transcoding to 20kbps, 30 ms, WIDEBAND, 10% packet loss" + $ISAC -I -B 32000 -FL 30 -FS 16 -PL 10 -T 20000 ../FEC"$PLATFORM"/APITest_PL10_WB30_T20.pcm \ + ../data/orig/speech_and_misc_WB.pcm ../FEC"$PLATFORM"/APITest_PL10_WB30.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 32 kbps, 60 ms, WIDEBAND, 10% packet loss" + $ISAC -I -B 32000 -FL 60 -FS 16 -PL 10 ../data/orig/speech_and_misc_WB.pcm \ + ../FEC"$PLATFORM"/APITest_PL10_WB60.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 32 kbps with transcoding to 20 kbps, 30 ms, SUPER-WIDEBAND, 10% packet loss" + $ISAC -I -B 32000 -FL 30 -FS 32 -PL 10 -T 20000 ../FEC"$PLATFORM"/APITest_PL10_SWB_8kHz_T20.pcm \ + ../data/origswb/jstest_32.pcm ../FEC"$PLATFORM"/APITest_PL10_SWB_8kHz.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 45 kbps with Trascoding to 38 kbps, 30 ms, SUPER-WIDEBAND, 10% packet loss" + $ISAC -I -B 45000 -FL 30 -FS 32 -PL 10 -T 38000 ../FEC"$PLATFORM"/APITest_PL10_SWB_12kHz_T38.pcm \ + ../data/origswb/jstest_32.pcm ../FEC"$PLATFORM"/APITest_PL10_SWB_12kHz.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo + + echo "-- 56 kbps with transcoding to 50 kbps, 30 ms, SUPER-WIDEBAND, 10% packet loss" + $ISAC -I -B 56000 -FL 30 -FS 32 -PL 10 -T 50000 ../FEC"$PLATFORM"/APITest_PL10_SWB_16kHz_T50.pcm \ + ../data/origswb/jstest_32.pcm ../FEC"$PLATFORM"/APITest_PL10_SWB_16kHz.pcm >> $LOGFILE_API + let "idx=idx+1" + echo + echo +fi diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfault.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfault.txt new file mode 100644 index 0000000000..63829a4b98 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfault.txt @@ -0,0 +1,80 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character????? ?????? +if [ "$1" = "x64" ] || [ "$#" -eq 0 ] + then + PLATFORM=_X64 + ISAC=../x64/Release/ReleaseTest-API_2005.exe +elif [ "$1" = "2005" ] + then + PLATFORM=_2005 + ISAC=../win32/Release/ReleaseTest-API_2005.exe +elif [ "$1" == "LINUX" ] + then + PLATFORM=_linux + ISAC=../ReleaseTest-API/isacswtest +else + echo Unknown Platform + exit 2 +fi + +LOGFILE=logfault$PLATFORM.txt +echo "START FAULT TEST" > $LOGFILE + + +INFILES=$(cat InputFiles.txt) +SUBSET=$(cat InputFilesFew.txt) +CHANNELFILES=$(cat ChannelFiles.txt) +CHANNELLIST=($(cat ChannelFiles.txt)) +INDIR=../data/orig +INDIRSWB=../data/origswb +OUTDIR=../dataqaft$PLATFORM +mkdir -p $OUTDIR + +#maximum Target rate for different bandwidth +TARGETRATE=( 32000 32000 44000 56000 ) +SAMPFREQ=( 16 32 32 32 ) +FAULTTEST=(1 2 3 4 5 6 7 9) + +index1=0 + +file_wb=../data/orig/16kHz.pcm +file_swb=../data/origswb/32kHz.pcm + +for idx in 0 1 2 3 + do +# Fault test + echo + echo "Sampling Frequency " ${SAMPFREQ[idx]} "kHz, Rate " ${TARGETRATE[idx]} "bps." + echo "---------------------------------------------------" + if [ ${SAMPFREQ[idx]} -eq 16 ]; then + file=$file_wb + else + file=$file_swb + fi + + for testnr in ${FAULTTEST[*]} + do + echo "Running Fault Test " $testnr + $ISAC -I -B "${TARGETRATE[idx]}" -F $testnr -FS "${SAMPFREQ[idx]}" "$file" \ + $OUTDIR/ft"$testnr"_"${TARGETRATE[idx]}"_"${SAMPFREQ[idx]}".pcm >> LOGFILE + echo + + done + +# Fault test number 10, error in bitstream + echo "Running Fault Test 10" + $ISAC -I -B "${TARGETRATE[idx]}" -F 10 -FS "${SAMPFREQ[idx]}" "$file" \ + $OUTDIR/ft10_"${TARGETRATE[idx]}"_"${SAMPFREQ[idx]}".pcm >> LOGFILE + echo + echo "Running Fault Test 10 with packetloss" + $ISAC -I -B "${TARGETRATE[idx]}" -F 10 -PL 10 -FS "${SAMPFREQ[idx]}" "$file" \ + $OUTDIR/ft10plc_"${TARGETRATE[idx]}"_"${SAMPFREQ[idx]}".pcm >> LOGFILE + echo +done + +echo +echo +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfixfloat.txt b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfixfloat.txt new file mode 100644 index 0000000000..4cda78e4fe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/QA/runiSACfixfloat.txt @@ -0,0 +1,47 @@ +#!/bin/bash +(set -o igncr) 2>/dev/null && set -o igncr; # force bash to ignore \r character + +LOGFILE=logfxfl.txt +echo "START FIX-FLOAT TEST" > $LOGFILE + + +ISACFIXFLOAT=../../../fix/test/testFixFloat.exe + +INFILES=$(cat InputFiles.txt) +SUBSET=$(cat InputFilesFew.txt) +CHANNELFILES=$(cat ChannelFiles.txt) +CHANNELLIST=($(cat ChannelFiles.txt)) +INDIR=../data/orig +OUTDIR=../dataqafxfl +mkdir -p $OUTDIR + +index1=0 + +for file in $INFILES # loop over all input files + do + + for channel in $CHANNELFILES + do + let "index1=index1+1" + + $ISACFIXFLOAT $INDIR/$channel -m 1 -PLC $INDIR/"$file" $OUTDIR/flfx$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 2 -PLC $INDIR/"$file" $OUTDIR/fxfl$index1"$file" >> $LOGFILE + done + +done + +index1=0 + +for file in $SUBSET # loop over the subset of input files + do + let "index1=index1+1" + $ISACFIXFLOAT $INDIR/$channel -m 1 -NB 1 $INDIR/"$file" $OUTDIR/flfxnb1_$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 2 -NB 1 $INDIR/"$file" $OUTDIR/fxflnb1_$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 1 -NB 2 -PLC $INDIR/"$file" $OUTDIR/flfxnb2_$index1"$file" >> $LOGFILE + $ISACFIXFLOAT $INDIR/$channel -m 2 -NB 2 -PLC $INDIR/"$file" $OUTDIR/fxflnb2_$index1"$file" >> $LOGFILE +done + +echo DONE! + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc new file mode 100644 index 0000000000..33d9efcda1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc @@ -0,0 +1,949 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// ReleaseTest-API.cpp : Defines the entry point for the console application. +// + +#include +#include +#include +#include +#include +#include + +/* include API */ +#include "isac.h" +#include "utility.h" +#include "rtc_base/format_macros.h" + +/* Defines */ +#define SEED_FILE "randseed.txt" /* Used when running decoder on garbage data */ +#define MAX_FRAMESAMPLES 960 /* max number of samples per frame + (= 60 ms frame & 16 kHz) or + (= 30 ms frame & 32 kHz) */ +#define FRAMESAMPLES_10ms 160 /* number of samples per 10ms frame */ +#define SWBFRAMESAMPLES_10ms 320 +//#define FS 16000 /* sampling frequency (Hz) */ + +#ifdef WIN32 +#ifndef CLOCKS_PER_SEC +#define CLOCKS_PER_SEC 1000 /* Runtime statistics */ +#endif +#endif + +using namespace std; + +int main(int argc, char* argv[]) { + char inname[100], outname[100], bottleneck_file[100], vadfile[100]; + FILE* inp, *outp, * f_bn = NULL, * vadp = NULL, *bandwidthp; + int framecnt, endfile; + + size_t i; + int errtype, VADusage = 0, packetLossPercent = 0; + int16_t CodingMode; + int32_t bottleneck = 0; + int framesize = 30; /* ms */ + int cur_framesmpls, err; + + /* Runtime statistics */ + double starttime, runtime, length_file; + + size_t stream_len = 0; + int declen = 0, declenTC = 0; + bool lostFrame = false; + + int16_t shortdata[SWBFRAMESAMPLES_10ms]; + int16_t vaddata[SWBFRAMESAMPLES_10ms * 3]; + int16_t decoded[MAX_FRAMESAMPLES << 1]; + int16_t decodedTC[MAX_FRAMESAMPLES << 1]; + uint16_t streamdata[500]; + int16_t speechType[1]; + int16_t rateBPS = 0; + int16_t fixedFL = 0; + int16_t payloadSize = 0; + int32_t payloadRate = 0; + int setControlBWE = 0; + short FL, testNum; + char version_number[20]; + FILE* plFile; + int32_t sendBN; + +#if !defined(NDEBUG) + FILE* fy; + double kbps; +#endif + size_t totalbits = 0; + int totalsmpls = 0; + + /* If use GNS file */ + FILE* fp_gns = NULL; + char gns_file[100]; + size_t maxStreamLen30 = 0; + size_t maxStreamLen60 = 0; + short sampFreqKHz = 32; + short samplesIn10Ms; + short useAssign = 0; + // FILE logFile; + bool doTransCoding = false; + int32_t rateTransCoding = 0; + uint8_t streamDataTransCoding[1200]; + size_t streamLenTransCoding = 0; + FILE* transCodingFile = NULL; + FILE* transcodingBitstream = NULL; + size_t numTransCodingBytes = 0; + + /* only one structure used for ISAC encoder */ + ISACStruct* ISAC_main_inst = NULL; + ISACStruct* decoderTransCoding = NULL; + + BottleNeckModel BN_data; + +#if !defined(NDEBUG) + fy = fopen("bit_rate.dat", "w"); + fclose(fy); + fy = fopen("bytes_frames.dat", "w"); + fclose(fy); +#endif + + /* Handling wrong input arguments in the command line */ + if ((argc < 3) || (argc > 17)) { + printf("\n\nWrong number of arguments or flag values.\n\n"); + + printf("\n"); + WebRtcIsac_version(version_number); + printf("iSAC-swb version %s \n\n", version_number); + + printf("Usage:\n\n"); + printf("%s [-I] bottleneck_value infile outfile \n\n", argv[0]); + printf("with:\n"); + printf("[-FS num] : sampling frequency in kHz, valid values are\n"); + printf(" 16 & 32, with 16 as default.\n"); + printf("[-I] : if -I option is specified, the coder will use\n"); + printf(" an instantaneous Bottleneck value. If not, it\n"); + printf(" will be an adaptive Bottleneck value.\n"); + printf("[-assign] : Use Assign API.\n"); + printf("[-B num] : the value of the bottleneck provided either\n"); + printf(" as a fixed value in bits/sec (e.g. 25000) or\n"); + printf(" read from a file (e.g. bottleneck.txt)\n"); + printf("[-INITRATE num] : Set a new value for initial rate. Note! Only\n"); + printf(" used in adaptive mode.\n"); + printf("[-FL num] : Set (initial) frame length in msec. Valid\n"); + printf(" lengths are 30 and 60 msec.\n"); + printf("[-FIXED_FL] : Frame length will be fixed to initial value.\n"); + printf("[-MAX num] : Set the limit for the payload size of iSAC\n"); + printf(" in bytes. Minimum 100 maximum 400.\n"); + printf("[-MAXRATE num] : Set the maxrate for iSAC in bits per second.\n"); + printf(" Minimum 32000, maximum 53400.\n"); + printf("[-F num] : if -F option is specified, the test function\n"); + printf(" will run the iSAC API fault scenario\n"); + printf(" specified by the supplied number.\n"); + printf(" F 1 - Call encoder prior to init encoder call\n"); + printf(" F 2 - Call decoder prior to init decoder call\n"); + printf(" F 3 - Call decoder prior to encoder call\n"); + printf(" F 4 - Call decoder with a too short coded\n"); + printf(" sequence\n"); + printf(" F 5 - Call decoder with a too long coded\n"); + printf(" sequence\n"); + printf(" F 6 - Call decoder with random bit stream\n"); + printf(" F 7 - Call init encoder/decoder at random\n"); + printf(" during a call\n"); + printf(" F 8 - Call encoder/decoder without having\n"); + printf(" allocated memory for encoder/decoder\n"); + printf(" instance\n"); + printf(" F 9 - Call decodeB without calling decodeA\n"); + printf(" F 10 - Call decodeB with garbage data\n"); + printf("[-PL num] : if -PL option is specified \n"); + printf("[-T rate file] : test trans-coding with target bottleneck\n"); + printf(" 'rate' bits/sec\n"); + printf(" the output file is written to 'file'\n"); + printf("[-LOOP num] : number of times to repeat coding the input\n"); + printf(" file for stress testing\n"); + // printf("[-CE num] : Test of APIs used by Conference Engine.\n"); + // printf(" CE 1 - getNewBitstream, getBWE \n"); + // printf(" (CE 2 - RESERVED for transcoding)\n"); + // printf(" CE 3 - getSendBWE, setSendBWE. \n"); + // printf("-L filename : write the logging info into file + // (appending)\n"); + printf("infile : Normal speech input file\n"); + printf("outfile : Speech output file\n"); + exit(0); + } + + /* Print version number */ + printf("-------------------------------------------------\n"); + WebRtcIsac_version(version_number); + printf("iSAC version %s \n\n", version_number); + + /* Loop over all command line arguments */ + CodingMode = 0; + testNum = 0; + useAssign = 0; + // logFile = NULL; + char transCodingFileName[500]; + int16_t totFileLoop = 0; + int16_t numFileLoop = 0; + for (i = 1; i + 2 < static_cast(argc); i++) { + if (!strcmp("-LOOP", argv[i])) { + i++; + totFileLoop = (int16_t)atol(argv[i]); + if (totFileLoop <= 0) { + fprintf(stderr, "Invalid number of runs for the given input file, %d.", + totFileLoop); + exit(0); + } + } + + if (!strcmp("-T", argv[i])) { + doTransCoding = true; + i++; + rateTransCoding = atoi(argv[i]); + i++; + strcpy(transCodingFileName, argv[i]); + } + + /*Should we use assign API*/ + if (!strcmp("-assign", argv[i])) { + useAssign = 1; + } + + /* Set Sampling Rate */ + if (!strcmp("-FS", argv[i])) { + i++; + sampFreqKHz = atoi(argv[i]); + } + + /* Instantaneous mode */ + if (!strcmp("-I", argv[i])) { + printf("Instantaneous BottleNeck\n"); + CodingMode = 1; + } + + /* Set (initial) bottleneck value */ + if (!strcmp("-INITRATE", argv[i])) { + rateBPS = atoi(argv[i + 1]); + setControlBWE = 1; + if ((rateBPS < 10000) || (rateBPS > 32000)) { + printf("\n%d is not a initial rate. Valid values are in the range " + "10000 to 32000.\n", rateBPS); + exit(0); + } + printf("New initial rate: %d\n", rateBPS); + i++; + } + + /* Set (initial) framelength */ + if (!strcmp("-FL", argv[i])) { + framesize = atoi(argv[i + 1]); + if ((framesize != 30) && (framesize != 60)) { + printf("\n%d is not a valid frame length. Valid length are 30 and 60 " + "msec.\n", framesize); + exit(0); + } + setControlBWE = 1; + printf("Frame Length: %d\n", framesize); + i++; + } + + /* Fixed frame length */ + if (!strcmp("-FIXED_FL", argv[i])) { + fixedFL = 1; + setControlBWE = 1; + printf("Fixed Frame Length\n"); + } + + /* Set maximum allowed payload size in bytes */ + if (!strcmp("-MAX", argv[i])) { + payloadSize = atoi(argv[i + 1]); + printf("Maximum Payload Size: %d\n", payloadSize); + i++; + } + + /* Set maximum rate in bytes */ + if (!strcmp("-MAXRATE", argv[i])) { + payloadRate = atoi(argv[i + 1]); + printf("Maximum Rate in kbps: %d\n", payloadRate); + i++; + } + + /* Test of fault scenarious */ + if (!strcmp("-F", argv[i])) { + testNum = atoi(argv[i + 1]); + printf("Fault test: %d\n", testNum); + if (testNum < 1 || testNum > 10) { + printf("\n%d is not a valid Fault Scenario number. Valid Fault " + "Scenarios are numbered 1-10.\n", testNum); + exit(0); + } + i++; + } + + /* Packet loss test */ + if (!strcmp("-PL", argv[i])) { + if (isdigit(*argv[i + 1])) { + packetLossPercent = atoi(argv[i + 1]); + if ((packetLossPercent < 0) | (packetLossPercent > 100)) { + printf("\nInvalid packet loss perentage \n"); + exit(0); + } + if (packetLossPercent > 0) { + printf("Simulating %d %% of independent packet loss\n", + packetLossPercent); + } else { + printf("\nNo Packet Loss Is Simulated \n"); + } + } else { + plFile = fopen(argv[i + 1], "rb"); + if (plFile == NULL) { + printf("\n couldn't open the frameloss file: %s\n", argv[i + 1]); + exit(0); + } + printf("Simulating packet loss through the given channel file: %s\n", + argv[i + 1]); + } + i++; + } + + /* Random packetlosses */ + if (!strcmp("-rnd", argv[i])) { + srand((unsigned int)time(NULL)); + printf("Random pattern in lossed packets \n"); + } + + /* Use gns file */ + if (!strcmp("-G", argv[i])) { + sscanf(argv[i + 1], "%s", gns_file); + fp_gns = fopen(gns_file, "rb"); + if (fp_gns == NULL) { + printf("Cannot read file %s.\n", gns_file); + exit(0); + } + i++; + } + + // make it with '-B' + /* Get Bottleneck value */ + if (!strcmp("-B", argv[i])) { + i++; + bottleneck = atoi(argv[i]); + if (bottleneck == 0) { + sscanf(argv[i], "%s", bottleneck_file); + f_bn = fopen(bottleneck_file, "rb"); + if (f_bn == NULL) { + printf("Error No value provided for BottleNeck and cannot read file " + "%s.\n", bottleneck_file); + exit(0); + } else { + printf("reading bottleneck rates from file %s\n\n", bottleneck_file); + if (fscanf(f_bn, "%d", &bottleneck) == EOF) { + /* Set pointer to beginning of file */ + fseek(f_bn, 0L, SEEK_SET); + if (fscanf(f_bn, "%d", &bottleneck) == EOF) { + exit(0); + } + } + + /* Bottleneck is a cosine function + * Matlab code for writing the bottleneck file: + * BottleNeck_10ms = 20e3 + 10e3 * cos((0:5999)/5999*2*pi); + * fid = fopen('bottleneck.txt', 'wb'); + * fprintf(fid, '%d\n', BottleNeck_10ms); fclose(fid); + */ + } + } else { + printf("\nfixed bottleneck rate of %d bits/s\n\n", bottleneck); + } + } + /* Run Conference Engine APIs */ + // Do not test it in the first release + // + // if(!strcmp ("-CE", argv[i])) + // { + // testCE = atoi(argv[i + 1]); + // if(testCE==1) + // { + // i++; + // scale = (float)atof( argv[i+1] ); + // } + // else if(testCE == 2) + // { + // printf("\nCE-test 2 (transcoding) not implemented.\n"); + // exit(0); + // } + // else if(testCE < 1 || testCE > 3) + // { + // printf("\n%d is not a valid CE-test number. Valid CE tests + // are 1-3.\n", testCE); + // exit(0); + // } + // printf("CE-test number: %d\n", testCE); + // i++; + // } + } + + if (CodingMode == 0) { + printf("\nAdaptive BottleNeck\n"); + } + + switch (sampFreqKHz) { + case 16: { + printf("iSAC Wideband.\n"); + samplesIn10Ms = FRAMESAMPLES_10ms; + break; + } + case 32: { + printf("iSAC Supper-Wideband.\n"); + samplesIn10Ms = SWBFRAMESAMPLES_10ms; + break; + } + default: + printf("Unsupported sampling frequency %d kHz", sampFreqKHz); + exit(0); + } + + /* Get Input and Output files */ + sscanf(argv[argc - 2], "%s", inname); + sscanf(argv[argc - 1], "%s", outname); + printf("\nInput file: %s\n", inname); + printf("Output file: %s\n\n", outname); + if ((inp = fopen(inname, "rb")) == NULL) { + printf(" Error iSAC Cannot read file %s.\n", inname); + cout << flush; + exit(1); + } + + if ((outp = fopen(outname, "wb")) == NULL) { + printf(" Error iSAC Cannot write file %s.\n", outname); + cout << flush; + getc(stdin); + exit(1); + } + if (VADusage) { + if ((vadp = fopen(vadfile, "rb")) == NULL) { + printf(" Error iSAC Cannot read file %s.\n", vadfile); + cout << flush; + exit(1); + } + } + + if ((bandwidthp = fopen("bwe.pcm", "wb")) == NULL) { + printf(" Error iSAC Cannot read file %s.\n", "bwe.pcm"); + cout << flush; + exit(1); + } + + starttime = clock() / (double)CLOCKS_PER_SEC; /* Runtime statistics */ + + /* Initialize the ISAC and BN structs */ + if (testNum != 8) { + if (!useAssign) { + err = WebRtcIsac_Create(&ISAC_main_inst); + WebRtcIsac_SetEncSampRate(ISAC_main_inst, sampFreqKHz * 1000); + WebRtcIsac_SetDecSampRate(ISAC_main_inst, + sampFreqKHz >= 32 ? 32000 : 16000); + } else { + /* Test the Assign functions */ + int sss; + void* ppp; + err = WebRtcIsac_AssignSize(&sss); + ppp = malloc(sss); + err = WebRtcIsac_Assign(&ISAC_main_inst, ppp); + WebRtcIsac_SetEncSampRate(ISAC_main_inst, sampFreqKHz * 1000); + WebRtcIsac_SetDecSampRate(ISAC_main_inst, + sampFreqKHz >= 32 ? 32000 : 16000); + } + /* Error check */ + if (err < 0) { + printf("\n\n Error in create.\n\n"); + cout << flush; + exit(EXIT_FAILURE); + } + } + BN_data.arrival_time = 0; + BN_data.sample_count = 0; + BN_data.rtp_number = 0; + + /* Initialize encoder and decoder */ + framecnt = 0; + endfile = 0; + + if (doTransCoding) { + WebRtcIsac_Create(&decoderTransCoding); + WebRtcIsac_SetEncSampRate(decoderTransCoding, sampFreqKHz * 1000); + WebRtcIsac_SetDecSampRate(decoderTransCoding, + sampFreqKHz >= 32 ? 32000 : 16000); + WebRtcIsac_DecoderInit(decoderTransCoding); + transCodingFile = fopen(transCodingFileName, "wb"); + if (transCodingFile == NULL) { + printf("Could not open %s to output trans-coding.\n", + transCodingFileName); + exit(0); + } + strcat(transCodingFileName, ".bit"); + transcodingBitstream = fopen(transCodingFileName, "wb"); + if (transcodingBitstream == NULL) { + printf("Could not open %s to write the bit-stream of transcoder.\n", + transCodingFileName); + exit(0); + } + } + + if (testNum != 1) { + if (WebRtcIsac_EncoderInit(ISAC_main_inst, CodingMode) < 0) { + printf("Error could not initialize the encoder \n"); + cout << flush; + return 0; + } + } + if (testNum != 2) + WebRtcIsac_DecoderInit(ISAC_main_inst); + if (CodingMode == 1) { + err = WebRtcIsac_Control(ISAC_main_inst, bottleneck, framesize); + if (err < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + printf("\n\n Error in initialization (control): %d.\n\n", errtype); + cout << flush; + if (testNum == 0) { + exit(EXIT_FAILURE); + } + } + } + + if ((setControlBWE) && (CodingMode == 0)) { + err = WebRtcIsac_ControlBwe(ISAC_main_inst, rateBPS, framesize, fixedFL); + if (err < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + + printf("\n\n Error in Control BWE: %d.\n\n", errtype); + cout << flush; + exit(EXIT_FAILURE); + } + } + + if (payloadSize != 0) { + err = WebRtcIsac_SetMaxPayloadSize(ISAC_main_inst, payloadSize); + if (err < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + printf("\n\n Error in SetMaxPayloadSize: %d.\n\n", errtype); + cout << flush; + exit(EXIT_FAILURE); + } + } + if (payloadRate != 0) { + err = WebRtcIsac_SetMaxRate(ISAC_main_inst, payloadRate); + if (err < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + printf("\n\n Error in SetMaxRateInBytes: %d.\n\n", errtype); + cout << flush; + exit(EXIT_FAILURE); + } + } + + *speechType = 1; + + cout << "\n" << flush; + + length_file = 0; + int16_t bnIdxTC = 0; + int16_t jitterInfoTC = 0; + while (endfile == 0) { + /* Call init functions at random, fault test number 7 */ + if (testNum == 7 && (rand() % 2 == 0)) { + err = WebRtcIsac_EncoderInit(ISAC_main_inst, CodingMode); + /* Error check */ + if (err < 0) { + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + printf("\n\n Error in encoderinit: %d.\n\n", errtype); + cout << flush; + } + + WebRtcIsac_DecoderInit(ISAC_main_inst); + } + + cur_framesmpls = 0; + while (1) { + int stream_len_int = 0; + + /* Read 10 ms speech block */ + endfile = readframe(shortdata, inp, samplesIn10Ms); + + if (endfile) { + numFileLoop++; + if (numFileLoop < totFileLoop) { + rewind(inp); + framecnt = 0; + fprintf(stderr, "\n"); + endfile = readframe(shortdata, inp, samplesIn10Ms); + } + } + + if (testNum == 7) { + srand((unsigned int)time(NULL)); + } + + /* iSAC encoding */ + if (!(testNum == 3 && framecnt == 0)) { + stream_len_int = + WebRtcIsac_Encode(ISAC_main_inst, shortdata, (uint8_t*)streamdata); + if ((payloadSize != 0) && (stream_len_int > payloadSize)) { + if (testNum == 0) { + printf("\n\n"); + } + + printf("\nError: Streamsize out of range %d\n", + stream_len_int - payloadSize); + cout << flush; + } + + WebRtcIsac_GetUplinkBw(ISAC_main_inst, &sendBN); + + if (stream_len_int > 0) { + if (doTransCoding) { + int16_t indexStream; + uint8_t auxUW8; + + /******************** Main Transcoding stream ********************/ + WebRtcIsac_GetDownLinkBwIndex(ISAC_main_inst, &bnIdxTC, + &jitterInfoTC); + int streamLenTransCoding_int = WebRtcIsac_GetNewBitStream( + ISAC_main_inst, bnIdxTC, jitterInfoTC, rateTransCoding, + streamDataTransCoding, false); + if (streamLenTransCoding_int < 0) { + fprintf(stderr, "Error in trans-coding\n"); + exit(0); + } + streamLenTransCoding = + static_cast(streamLenTransCoding_int); + auxUW8 = (uint8_t)(((streamLenTransCoding & 0xFF00) >> 8) & 0x00FF); + if (fwrite(&auxUW8, sizeof(uint8_t), 1, transcodingBitstream) != + 1) { + return -1; + } + + auxUW8 = (uint8_t)(streamLenTransCoding & 0x00FF); + if (fwrite(&auxUW8, sizeof(uint8_t), 1, transcodingBitstream) != + 1) { + return -1; + } + + if (fwrite(streamDataTransCoding, sizeof(uint8_t), + streamLenTransCoding, transcodingBitstream) != + streamLenTransCoding) { + return -1; + } + + WebRtcIsac_ReadBwIndex(streamDataTransCoding, &indexStream); + if (indexStream != bnIdxTC) { + fprintf(stderr, + "Error in inserting Bandwidth index into transcoding " + "stream.\n"); + exit(0); + } + numTransCodingBytes += streamLenTransCoding; + } + } + } else { + break; + } + + if (stream_len_int < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + fprintf(stderr, "Error in encoder: %d.\n", errtype); + cout << flush; + exit(0); + } + stream_len = static_cast(stream_len_int); + + cur_framesmpls += samplesIn10Ms; + /* exit encoder loop if the encoder returned a bitstream */ + if (stream_len != 0) + break; + } + + /* read next bottleneck rate */ + if (f_bn != NULL) { + if (fscanf(f_bn, "%d", &bottleneck) == EOF) { + /* Set pointer to beginning of file */ + fseek(f_bn, 0L, SEEK_SET); + if (fscanf(f_bn, "%d", &bottleneck) == EOF) { + exit(0); + } + } + if (CodingMode == 1) { + WebRtcIsac_Control(ISAC_main_inst, bottleneck, framesize); + } + } + + length_file += cur_framesmpls; + if (cur_framesmpls == (3 * samplesIn10Ms)) { + maxStreamLen30 = + (stream_len > maxStreamLen30) ? stream_len : maxStreamLen30; + } else { + maxStreamLen60 = + (stream_len > maxStreamLen60) ? stream_len : maxStreamLen60; + } + + if (!lostFrame) { + lostFrame = ((rand() % 100) < packetLossPercent); + } else { + lostFrame = false; + } + + // RED. + if (lostFrame) { + int stream_len_int = WebRtcIsac_GetRedPayload( + ISAC_main_inst, reinterpret_cast(streamdata)); + if (stream_len_int < 0) { + fprintf(stderr, "Error getting RED payload\n"); + exit(0); + } + stream_len = static_cast(stream_len_int); + + if (doTransCoding) { + int streamLenTransCoding_int = WebRtcIsac_GetNewBitStream( + ISAC_main_inst, bnIdxTC, jitterInfoTC, rateTransCoding, + streamDataTransCoding, true); + if (streamLenTransCoding_int < 0) { + fprintf(stderr, "Error in RED trans-coding\n"); + exit(0); + } + streamLenTransCoding = + static_cast(streamLenTransCoding_int); + } + } + + /* make coded sequence to short be inreasing */ + /* the length the decoder expects */ + if (testNum == 4) { + stream_len += 10; + } + + /* make coded sequence to long be decreasing */ + /* the length the decoder expects */ + if (testNum == 5) { + stream_len -= 10; + } + + if (testNum == 6) { + srand((unsigned int)time(NULL)); + for (i = 0; i < stream_len; i++) { + streamdata[i] = rand(); + } + } + + if (VADusage) { + readframe(vaddata, vadp, samplesIn10Ms * 3); + } + + /* simulate packet handling through NetEq and the modem */ + if (!(testNum == 3 && framecnt == 0)) { + get_arrival_time(cur_framesmpls, stream_len, bottleneck, &BN_data, + sampFreqKHz * 1000, sampFreqKHz * 1000); + } + + if (VADusage && (framecnt > 10 && vaddata[0] == 0)) { + BN_data.rtp_number--; + } else { + /* Error test number 10, garbage data */ + if (testNum == 10) { + /* Test to run decoder with garbage data */ + for (i = 0; i < stream_len; i++) { + streamdata[i] = (short)(streamdata[i]) + (short)rand(); + } + } + + if (testNum != 9) { + err = WebRtcIsac_UpdateBwEstimate( + ISAC_main_inst, reinterpret_cast(streamdata), + stream_len, BN_data.rtp_number, BN_data.sample_count, + BN_data.arrival_time); + + if (err < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + if (testNum == 0) { + printf("\n\n"); + } + + printf("Error: in decoder: %d.", errtype); + cout << flush; + if (testNum == 0) { + printf("\n\n"); + } + } + } + + /* Call getFramelen, only used here for function test */ + err = WebRtcIsac_ReadFrameLen( + ISAC_main_inst, reinterpret_cast(streamdata), &FL); + if (err < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + if (testNum == 0) { + printf("\n\n"); + } + printf(" Error: in getFrameLen %d.", errtype); + cout << flush; + if (testNum == 0) { + printf("\n\n"); + } + } + + // iSAC decoding + + if (lostFrame) { + declen = WebRtcIsac_DecodeRcu( + ISAC_main_inst, reinterpret_cast(streamdata), + stream_len, decoded, speechType); + + if (doTransCoding) { + declenTC = + WebRtcIsac_DecodeRcu(decoderTransCoding, streamDataTransCoding, + streamLenTransCoding, decodedTC, speechType); + } + } else { + declen = WebRtcIsac_Decode(ISAC_main_inst, + reinterpret_cast(streamdata), + stream_len, decoded, speechType); + if (doTransCoding) { + declenTC = + WebRtcIsac_Decode(decoderTransCoding, streamDataTransCoding, + streamLenTransCoding, decodedTC, speechType); + } + } + + if (declen < 0) { + /* exit if returned with error */ + errtype = WebRtcIsac_GetErrorCode(ISAC_main_inst); + if (testNum == 0) { + printf("\n\n"); + } + printf(" Error: in decoder %d.", errtype); + cout << flush; + if (testNum == 0) { + printf("\n\n"); + } + } + + if (declenTC < 0) { + if (testNum == 0) { + printf("\n\n"); + } + printf(" Error: in decoding the transcoded stream"); + cout << flush; + if (testNum == 0) { + printf("\n\n"); + } + } + } + /* Write decoded speech frame to file */ + if ((declen > 0) && (numFileLoop == 0)) { + if (fwrite(decoded, sizeof(int16_t), declen, outp) != + static_cast(declen)) { + return -1; + } + } + + if ((declenTC > 0) && (numFileLoop == 0)) { + if (fwrite(decodedTC, sizeof(int16_t), declen, transCodingFile) != + static_cast(declen)) { + return -1; + } + } + + fprintf(stderr, "\rframe = %5d ", framecnt); + fflush(stderr); + framecnt++; + + /* Error test number 10, garbage data */ + // if (testNum == 10) + // { + // /* Test to run decoder with garbage data */ + // if ((seedfile = fopen(SEED_FILE, "a+t")) == NULL) { + // fprintf(stderr, "Error: Could not open file %s\n", SEED_FILE); + // } else { + // fprintf(seedfile, "ok\n\n"); + // fclose(seedfile); + // } + // } + /* Error test number 10, garbage data */ + // if (testNum == 10) { + // /* Test to run decoder with garbage data */ + // for (i = 0; i < stream_len; i++) { + // streamdata[i] = (short) (streamdata[i] + (short) rand()); + // } + // } + + totalsmpls += declen; + totalbits += 8 * stream_len; +#if !defined(NDEBUG) + kbps = ((double)sampFreqKHz * 1000.) / ((double)cur_framesmpls) * 8.0 * + stream_len / 1000.0; // kbits/s + fy = fopen("bit_rate.dat", "a"); + fprintf(fy, "Frame %i = %0.14f\n", framecnt, kbps); + fclose(fy); + +#endif + } + printf("\n"); + printf("total bits = %" PRIuS " bits\n", totalbits); + printf("measured average bitrate = %0.3f kbits/s\n", + (double)totalbits * (sampFreqKHz) / totalsmpls); + if (doTransCoding) { + printf("Transcoding average bit-rate = %0.3f kbps\n", + (double)numTransCodingBytes * 8.0 * (sampFreqKHz) / totalsmpls); + fclose(transCodingFile); + } + printf("\n"); + + /* Runtime statistics */ + runtime = (double)(clock() / (double)CLOCKS_PER_SEC - starttime); + length_file = length_file / (sampFreqKHz * 1000.); + + printf("\n\nLength of speech file: %.1f s\n", length_file); + printf("Time to run iSAC: %.2f s (%.2f %% of realtime)\n\n", runtime, + (100 * runtime / length_file)); + + if (maxStreamLen30 != 0) { + printf("Maximum payload size 30ms Frames %" PRIuS " bytes (%0.3f kbps)\n", + maxStreamLen30, maxStreamLen30 * 8 / 30.); + } + if (maxStreamLen60 != 0) { + printf("Maximum payload size 60ms Frames %" PRIuS " bytes (%0.3f kbps)\n", + maxStreamLen60, maxStreamLen60 * 8 / 60.); + } + // fprintf(stderr, "\n"); + + fprintf(stderr, " %.1f s", length_file); + fprintf(stderr, " %0.1f kbps", + (double)totalbits * (sampFreqKHz) / totalsmpls); + if (maxStreamLen30 != 0) { + fprintf(stderr, " plmax-30ms %" PRIuS " bytes (%0.0f kbps)", + maxStreamLen30, maxStreamLen30 * 8 / 30.); + } + if (maxStreamLen60 != 0) { + fprintf(stderr, " plmax-60ms %" PRIuS " bytes (%0.0f kbps)", + maxStreamLen60, maxStreamLen60 * 8 / 60.); + } + if (doTransCoding) { + fprintf(stderr, " transcoding rate %.0f kbps", + (double)numTransCodingBytes * 8.0 * (sampFreqKHz) / totalsmpls); + } + + fclose(inp); + fclose(outp); + WebRtcIsac_Free(ISAC_main_inst); + + exit(0); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/SwitchingSampRate/SwitchingSampRate.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/SwitchingSampRate/SwitchingSampRate.cc new file mode 100644 index 0000000000..a53e7bd0b5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/SwitchingSampRate/SwitchingSampRate.cc @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// SwitchingSampRate.cpp : Defines the entry point for the console +// application. +// + +#include +#include "isac.h" +#include "utility.h" +#include "signal_processing_library.h" + +#define MAX_FILE_NAME 500 +#define MAX_NUM_CLIENTS 2 + + +#define NUM_CLIENTS 2 + +using namespace std; + +int main(int argc, char* argv[]) +{ + char fileNameWB[MAX_FILE_NAME]; + char fileNameSWB[MAX_FILE_NAME]; + + char outFileName[MAX_NUM_CLIENTS][MAX_FILE_NAME]; + + FILE* inFile[MAX_NUM_CLIENTS]; + FILE* outFile[MAX_NUM_CLIENTS]; + + ISACStruct* codecInstance[MAX_NUM_CLIENTS]; + int32_t resamplerState[MAX_NUM_CLIENTS][8]; + + int encoderSampRate[MAX_NUM_CLIENTS]; + + int minBn = 16000; + int maxBn = 56000; + + int bnWB = 32000; + int bnSWB = 56000; + + strcpy(outFileName[0], "switchSampRate_out1.pcm"); + strcpy(outFileName[1], "switchSampRate_out2.pcm"); + + short clientCntr; + + size_t lenEncodedInBytes[MAX_NUM_CLIENTS]; + unsigned int lenAudioIn10ms[MAX_NUM_CLIENTS]; + size_t lenEncodedInBytesTmp[MAX_NUM_CLIENTS]; + unsigned int lenAudioIn10msTmp[MAX_NUM_CLIENTS]; + BottleNeckModel* packetData[MAX_NUM_CLIENTS]; + + char versionNumber[100]; + short samplesIn10ms[MAX_NUM_CLIENTS]; + int bottleneck[MAX_NUM_CLIENTS]; + + printf("\n\n"); + printf("____________________________________________\n\n"); + WebRtcIsac_version(versionNumber); + printf(" iSAC-swb version %s\n", versionNumber); + printf("____________________________________________\n"); + + + fileNameWB[0] = '\0'; + fileNameSWB[0] = '\0'; + + char myFlag[20]; + strcpy(myFlag, "-wb"); + // READ THE WIDEBAND AND SUPER-WIDEBAND FILE NAMES + if(readParamString(argc, argv, myFlag, fileNameWB, MAX_FILE_NAME) <= 0) + { + printf("No wideband file is specified"); + } + + strcpy(myFlag, "-swb"); + if(readParamString(argc, argv, myFlag, fileNameSWB, MAX_FILE_NAME) <= 0) + { + printf("No super-wideband file is specified"); + } + + // THE FIRST CLIENT STARTS IN WIDEBAND + encoderSampRate[0] = 16000; + OPEN_FILE_RB(inFile[0], fileNameWB); + + // THE SECOND CLIENT STARTS IN SUPER-WIDEBAND + encoderSampRate[1] = 32000; + OPEN_FILE_RB(inFile[1], fileNameSWB); + + strcpy(myFlag, "-I"); + short codingMode = readSwitch(argc, argv, myFlag); + + for(clientCntr = 0; clientCntr < NUM_CLIENTS; clientCntr++) + { + codecInstance[clientCntr] = NULL; + + printf("\n"); + printf("Client %d\n", clientCntr + 1); + printf("---------\n"); + printf("Starting %s", + (encoderSampRate[clientCntr] == 16000) + ? "wideband":"super-wideband"); + + // Open output File Name + OPEN_FILE_WB(outFile[clientCntr], outFileName[clientCntr]); + printf("Output File...................... %s\n", outFileName[clientCntr]); + + samplesIn10ms[clientCntr] = encoderSampRate[clientCntr] * 10; + + if(codingMode == 1) + { + bottleneck[clientCntr] = (clientCntr)? bnSWB:bnWB; + } + else + { + bottleneck[clientCntr] = (clientCntr)? minBn:maxBn; + } + + printf("Bottleneck....................... %0.3f kbits/sec \n", + bottleneck[clientCntr] / 1000.0); + + // coding-mode + printf("Encoding Mode.................... %s\n", + (codingMode == 1)? "Channel-Independent (Instantaneous)":"Adaptive"); + + lenEncodedInBytes[clientCntr] = 0; + lenAudioIn10ms[clientCntr] = 0; + lenEncodedInBytesTmp[clientCntr] = 0; + lenAudioIn10msTmp[clientCntr] = 0; + + packetData[clientCntr] = (BottleNeckModel*)new(BottleNeckModel); + if(packetData[clientCntr] == NULL) + { + printf("Could not allocate memory for packetData \n"); + return -1; + } + memset(packetData[clientCntr], 0, sizeof(BottleNeckModel)); + memset(resamplerState[clientCntr], 0, sizeof(int32_t) * 8); + } + + for(clientCntr = 0; clientCntr < NUM_CLIENTS; clientCntr++) + { + // Create + if(WebRtcIsac_Create(&codecInstance[clientCntr])) + { + printf("Could not creat client %d\n", clientCntr + 1); + return -1; + } + + WebRtcIsac_SetEncSampRate(codecInstance[clientCntr], encoderSampRate[clientCntr]); + + WebRtcIsac_SetDecSampRate(codecInstance[clientCntr], + encoderSampRate[clientCntr + (1 - ((clientCntr & 1)<<1))]); + + // Initialize Encoder + if(WebRtcIsac_EncoderInit(codecInstance[clientCntr], + codingMode) < 0) + { + printf("Could not initialize client, %d\n", clientCntr + 1); + return -1; + } + + WebRtcIsac_DecoderInit(codecInstance[clientCntr]); + + // setup Rate if in Instantaneous mode + if(codingMode != 0) + { + // ONLY Clients who are not in Adaptive mode + if(WebRtcIsac_Control(codecInstance[clientCntr], + bottleneck[clientCntr], 30) < 0) + { + printf("Could not setup bottleneck and frame-size for client %d\n", + clientCntr + 1); + return -1; + } + } + } + + + size_t streamLen; + short numSamplesRead; + size_t lenDecodedAudio; + short senderIdx; + short receiverIdx; + + printf("\n"); + short num10ms[MAX_NUM_CLIENTS]; + memset(num10ms, 0, sizeof(short)*MAX_NUM_CLIENTS); + FILE* arrivalTimeFile1 = fopen("arrivalTime1.dat", "wb"); + FILE* arrivalTimeFile2 = fopen("arrivalTime2.dat", "wb"); + short numPrint[MAX_NUM_CLIENTS]; + memset(numPrint, 0, sizeof(short) * MAX_NUM_CLIENTS); + + // Audio Buffers + short silence10ms[10 * 32]; + memset(silence10ms, 0, 320 * sizeof(short)); + short audioBuff10ms[10 * 32]; + short audioBuff60ms[60 * 32]; + short resampledAudio60ms[60 * 32]; + + unsigned short bitStream[600+600]; + short speechType[1]; + + short numSampFreqChanged = 0; + while(numSampFreqChanged < 10) + { + for(clientCntr = 0; clientCntr < NUM_CLIENTS; clientCntr++) + { + // Encoding/decoding for this pair of clients, if there is + // audio for any of them + //if(audioLeft[clientCntr] || audioLeft[clientCntr + 1]) + //{ + //for(pairCntr = 0; pairCntr < 2; pairCntr++) + //{ + senderIdx = clientCntr; // + pairCntr; + receiverIdx = 1 - clientCntr;// + (1 - pairCntr); + + //if(num10ms[senderIdx] > 6) + //{ + // printf("Too many frames read for client %d", + // senderIdx + 1); + // return -1; + //} + + numSamplesRead = (short)fread(audioBuff10ms, sizeof(short), + samplesIn10ms[senderIdx], inFile[senderIdx]); + if(numSamplesRead != samplesIn10ms[senderIdx]) + { + // file finished switch encoder sampling frequency. + printf("Changing Encoder Sampling frequency in client %d to ", senderIdx+1); + fclose(inFile[senderIdx]); + numSampFreqChanged++; + if(encoderSampRate[senderIdx] == 16000) + { + printf("super-wideband.\n"); + OPEN_FILE_RB(inFile[senderIdx], fileNameSWB); + encoderSampRate[senderIdx] = 32000; + } + else + { + printf("wideband.\n"); + OPEN_FILE_RB(inFile[senderIdx], fileNameWB); + encoderSampRate[senderIdx] = 16000; + } + WebRtcIsac_SetEncSampRate(codecInstance[senderIdx], encoderSampRate[senderIdx]); + WebRtcIsac_SetDecSampRate(codecInstance[receiverIdx], encoderSampRate[senderIdx]); + + samplesIn10ms[clientCntr] = encoderSampRate[clientCntr] * 10; + + numSamplesRead = (short)fread(audioBuff10ms, sizeof(short), + samplesIn10ms[senderIdx], inFile[senderIdx]); + if(numSamplesRead != samplesIn10ms[senderIdx]) + { + printf(" File %s for client %d has not enough audio\n", + (encoderSampRate[senderIdx]==16000)? "wideband":"super-wideband", + senderIdx + 1); + return -1; + } + } + num10ms[senderIdx]++; + + // sanity check + //if(num10ms[senderIdx] > 6) + //{ + // printf("Client %d has got more than 60 ms audio and encoded no packet.\n", + // senderIdx); + // return -1; + //} + + // Encode + + + int streamLen_int = WebRtcIsac_Encode(codecInstance[senderIdx], + audioBuff10ms, + (uint8_t*)bitStream); + int16_t ggg; + if (streamLen_int > 0) { + if ((WebRtcIsac_ReadFrameLen( + codecInstance[receiverIdx], + reinterpret_cast(bitStream), + &ggg)) < 0) + printf("ERROR\n"); + } + + // Sanity check + if(streamLen_int < 0) + { + printf(" Encoder error in client %d \n", senderIdx + 1); + return -1; + } + streamLen = static_cast(streamLen_int); + + + if(streamLen > 0) + { + // Packet generated; model sending through a channel, do bandwidth + // estimation at the receiver and decode. + lenEncodedInBytes[senderIdx] += streamLen; + lenAudioIn10ms[senderIdx] += (unsigned int)num10ms[senderIdx]; + lenEncodedInBytesTmp[senderIdx] += streamLen; + lenAudioIn10msTmp[senderIdx] += (unsigned int)num10ms[senderIdx]; + + // Print after ~5 sec. + if(lenAudioIn10msTmp[senderIdx] >= 100) + { + numPrint[senderIdx]++; + printf(" %d, %6.3f => %6.3f ", senderIdx+1, + bottleneck[senderIdx] / 1000.0, + lenEncodedInBytesTmp[senderIdx] * 0.8 / + lenAudioIn10msTmp[senderIdx]); + + if(codingMode == 0) + { + int32_t bn; + WebRtcIsac_GetUplinkBw(codecInstance[senderIdx], &bn); + printf("[%d] ", bn); + } + //int16_t rateIndexLB; + //int16_t rateIndexUB; + //WebRtcIsac_GetDownLinkBwIndex(codecInstance[receiverIdx], + // &rateIndexLB, &rateIndexUB); + //printf(" (%2d, %2d) ", rateIndexLB, rateIndexUB); + + cout << flush; + lenEncodedInBytesTmp[senderIdx] = 0; + lenAudioIn10msTmp[senderIdx] = 0; + //if(senderIdx == (NUM_CLIENTS - 1)) + //{ + printf(" %0.1f \n", lenAudioIn10ms[senderIdx] * 10. /1000); + //} + + // After ~20 sec change the bottleneck. + // if((numPrint[senderIdx] == 4) && (codingMode == 0)) + // { + // numPrint[senderIdx] = 0; + // if(codingMode == 0) + // { + // int newBottleneck = bottleneck[senderIdx] + + // (bottleneckChange[senderIdx] * 1000); + + // if(bottleneckChange[senderIdx] > 0) + // { + // if(newBottleneck >maxBn) + // { + // bottleneckChange[senderIdx] = -1; + // newBottleneck = bottleneck[senderIdx] + + // (bottleneckChange[senderIdx] * 1000); + // if(newBottleneck > minBn) + // { + // bottleneck[senderIdx] = newBottleneck; + // } + // } + // else + // { + // bottleneck[senderIdx] = newBottleneck; + // } + // } + // else + // { + // if(newBottleneck < minBn) + // { + // bottleneckChange[senderIdx] = 1; + // newBottleneck = bottleneck[senderIdx] + + // (bottleneckChange[senderIdx] * 1000); + // if(newBottleneck < maxBn) + // { + // bottleneck[senderIdx] = newBottleneck; + // } + // } + // else + // { + // bottleneck[senderIdx] = newBottleneck; + // } + // } + // } + // } + } + + // model a channel of given bottleneck, to get the receive timestamp + get_arrival_time(num10ms[senderIdx] * samplesIn10ms[senderIdx], + streamLen, bottleneck[senderIdx], packetData[senderIdx], + encoderSampRate[senderIdx]*1000, encoderSampRate[senderIdx]*1000); + + // Write the arrival time. + if(senderIdx == 0) + { + if (fwrite(&(packetData[senderIdx]->arrival_time), + sizeof(unsigned int), + 1, arrivalTimeFile1) != 1) { + return -1; + } + } + else + { + if (fwrite(&(packetData[senderIdx]->arrival_time), + sizeof(unsigned int), + 1, arrivalTimeFile2) != 1) { + return -1; + } + } + + // BWE + if (WebRtcIsac_UpdateBwEstimate( + codecInstance[receiverIdx], + reinterpret_cast(bitStream), + streamLen, + packetData[senderIdx]->rtp_number, + packetData[senderIdx]->sample_count, + packetData[senderIdx]->arrival_time) < 0) { + printf(" BWE Error at client %d \n", receiverIdx + 1); + return -1; + } + /**/ + // Decode + int lenDecodedAudio_int = WebRtcIsac_Decode( + codecInstance[receiverIdx], + reinterpret_cast(bitStream), + streamLen, + audioBuff60ms, + speechType); + if(lenDecodedAudio_int < 0) + { + printf(" Decoder error in client %d \n", receiverIdx + 1); + return -1; + } + lenDecodedAudio = static_cast(lenDecodedAudio_int); + + if(encoderSampRate[senderIdx] == 16000) + { + WebRtcSpl_UpsampleBy2(audioBuff60ms, lenDecodedAudio, resampledAudio60ms, + resamplerState[receiverIdx]); + if (fwrite(resampledAudio60ms, sizeof(short), lenDecodedAudio << 1, + outFile[receiverIdx]) != + lenDecodedAudio << 1) { + return -1; + } + } + else + { + if (fwrite(audioBuff60ms, sizeof(short), lenDecodedAudio, + outFile[receiverIdx]) != + lenDecodedAudio) { + return -1; + } + } + num10ms[senderIdx] = 0; + } + //} + //} + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/simpleKenny.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/simpleKenny.c new file mode 100644 index 0000000000..0d2d2cb375 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/test/simpleKenny.c @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* kenny.c - Main function for the iSAC coder */ + +#include +#include +#include +#include + +#ifdef WIN32 +#include "windows.h" +#ifndef CLOCKS_PER_SEC +#define CLOCKS_PER_SEC 1000 +#endif +#endif + +#include +#include + +/* include API */ +#include "isac.h" +#include "utility.h" +#include "rtc_base/format_macros.h" + +/* max number of samples per frame (= 60 ms frame) */ +#define MAX_FRAMESAMPLES_SWB 1920 +/* number of samples per 10ms frame */ +#define FRAMESAMPLES_SWB_10ms 320 +#define FRAMESAMPLES_WB_10ms 160 + +/* sampling frequency (Hz) */ +#define FS_SWB 32000 +#define FS_WB 16000 + +unsigned long framecnt = 0; + +int main(int argc, char* argv[]) { + //--- File IO ---- + FILE* inp; + FILE* outp; + char inname[500]; + char outname[500]; + + /* Runtime statistics */ + double rate; + double rateRCU; + size_t totalbits = 0; + unsigned long totalBitsRCU = 0; + unsigned long totalsmpls = 0; + + int32_t bottleneck = 39; + int frameSize = 30; /* ms */ + int16_t codingMode = 1; + int16_t shortdata[FRAMESAMPLES_SWB_10ms]; + int16_t decoded[MAX_FRAMESAMPLES_SWB]; + int16_t speechType[1]; + int16_t payloadLimit; + int32_t rateLimit; + ISACStruct* ISAC_main_inst; + + size_t stream_len = 0; + int declen = 0; + int16_t err; + int cur_framesmpls; + int endfile; +#ifdef WIN32 + double length_file; + double runtime; + char outDrive[10]; + char outPath[500]; + char outPrefix[500]; + char outSuffix[500]; + char bitrateFileName[500]; + FILE* bitrateFile; + double starttime; + double rateLB = 0; + double rateUB = 0; +#endif + FILE* histFile; + FILE* averageFile; + int sampFreqKHz; + int samplesIn10Ms; + size_t maxStreamLen = 0; + char histFileName[500]; + char averageFileName[500]; + unsigned int hist[600]; + double tmpSumStreamLen = 0; + unsigned int packetCntr = 0; + unsigned int lostPacketCntr = 0; + uint8_t payload[1200]; + uint8_t payloadRCU[1200]; + uint16_t packetLossPercent = 0; + int16_t rcuStreamLen = 0; + int onlyEncode; + int onlyDecode; + + BottleNeckModel packetData; + packetData.arrival_time = 0; + packetData.sample_count = 0; + packetData.rtp_number = 0; + memset(hist, 0, sizeof(hist)); + + /* handling wrong input arguments in the command line */ + if (argc < 5) { + int size; + WebRtcIsac_AssignSize(&size); + + printf("\n\nWrong number of arguments or flag values.\n\n"); + + printf("Usage:\n\n"); + printf("%s infile outfile -bn bottleneck [options]\n\n", argv[0]); + printf("with:\n"); + printf("-I.............. indicates encoding in instantaneous mode.\n"); + printf("-bn bottleneck.. the value of the bottleneck in bit/sec, e.g.\n"); + printf(" 39742, in instantaneous (channel-independent)\n"); + printf(" mode.\n\n"); + printf("infile.......... Normal speech input file\n\n"); + printf("outfile......... Speech output file\n\n"); + printf("OPTIONS\n"); + printf("-------\n"); + printf("-fs sampFreq.... sampling frequency of codec 16 or 32 (default)\n"); + printf(" kHz.\n"); + printf("-plim payloadLim payload limit in bytes, default is the maximum\n"); + printf(" possible.\n"); + printf("-rlim rateLim... rate limit in bits/sec, default is the maximum\n"); + printf(" possible.\n"); + printf("-h file......... record histogram and *append* to 'file'.\n"); + printf("-ave file....... record average rate of 3 sec intervales and\n"); + printf(" *append* to 'file'.\n"); + printf("-ploss.......... packet-loss percentage.\n"); + printf("-enc............ do only encoding and store the bit-stream\n"); + printf("-dec............ the input file is a bit-stream, decode it.\n\n"); + printf("Example usage:\n\n"); + printf("%s speechIn.pcm speechOut.pcm -B 40000 -fs 32\n\n", argv[0]); + printf("structure size %d bytes\n", size); + + exit(0); + } + + /* Get Bottleneck value */ + bottleneck = readParamInt(argc, argv, "-bn", 50000); + fprintf(stderr, "\nfixed bottleneck rate of %d bits/s\n\n", bottleneck); + + /* Get Input and Output files */ + sscanf(argv[1], "%s", inname); + sscanf(argv[2], "%s", outname); + codingMode = readSwitch(argc, argv, "-I"); + sampFreqKHz = (int16_t)readParamInt(argc, argv, "-fs", 32); + if (readParamString(argc, argv, "-h", histFileName, 500) > 0) { + histFile = fopen(histFileName, "a"); + if (histFile == NULL) { + printf("cannot open hist file %s", histFileName); + exit(0); + } + } else { + // NO recording of hitstogram + histFile = NULL; + } + + packetLossPercent = readParamInt(argc, argv, "-ploss", 0); + + if (readParamString(argc, argv, "-ave", averageFileName, 500) > 0) { + averageFile = fopen(averageFileName, "a"); + if (averageFile == NULL) { + printf("cannot open file to write rate %s", averageFileName); + exit(0); + } + } else { + averageFile = NULL; + } + + onlyEncode = readSwitch(argc, argv, "-enc"); + onlyDecode = readSwitch(argc, argv, "-dec"); + + switch (sampFreqKHz) { + case 16: { + samplesIn10Ms = 160; + break; + } + case 32: { + samplesIn10Ms = 320; + break; + } + default: + printf("A sampling frequency of %d kHz is not supported, valid values are" + " 8 and 16.\n", sampFreqKHz); + exit(-1); + } + payloadLimit = (int16_t)readParamInt(argc, argv, "-plim", 400); + rateLimit = readParamInt(argc, argv, "-rlim", 106800); + + if ((inp = fopen(inname, "rb")) == NULL) { + printf(" iSAC: Cannot read file %s.\n", inname); + exit(1); + } + if ((outp = fopen(outname, "wb")) == NULL) { + printf(" iSAC: Cannot write file %s.\n", outname); + exit(1); + } + +#ifdef WIN32 + _splitpath(outname, outDrive, outPath, outPrefix, outSuffix); + _makepath(bitrateFileName, outDrive, outPath, "bitrate", ".txt"); + + bitrateFile = fopen(bitrateFileName, "a"); + fprintf(bitrateFile, "%% %s \n", inname); +#endif + + printf("\n"); + printf("Input.................... %s\n", inname); + printf("Output................... %s\n", outname); + printf("Encoding Mode............ %s\n", + (codingMode == 1) ? "Channel-Independent" : "Channel-Adaptive"); + printf("Bottleneck............... %d bits/sec\n", bottleneck); + printf("Packet-loss Percentage... %d\n", packetLossPercent); + printf("\n"); + +#ifdef WIN32 + starttime = clock() / (double)CLOCKS_PER_SEC; /* Runtime statistics */ +#endif + + /* Initialize the ISAC and BN structs */ + err = WebRtcIsac_Create(&ISAC_main_inst); + + WebRtcIsac_SetEncSampRate(ISAC_main_inst, sampFreqKHz * 1000); + WebRtcIsac_SetDecSampRate(ISAC_main_inst, sampFreqKHz >= 32 ? 32000 : 16000); + /* Error check */ + if (err < 0) { + fprintf(stderr, "\n\n Error in create.\n\n"); + exit(EXIT_FAILURE); + } + + framecnt = 0; + endfile = 0; + + /* Initialize encoder and decoder */ + if (WebRtcIsac_EncoderInit(ISAC_main_inst, codingMode) < 0) { + printf("cannot initialize encoder\n"); + return -1; + } + WebRtcIsac_DecoderInit(ISAC_main_inst); + + if (codingMode == 1) { + if (WebRtcIsac_Control(ISAC_main_inst, bottleneck, frameSize) < 0) { + printf("cannot set bottleneck\n"); + return -1; + } + } else { + if (WebRtcIsac_ControlBwe(ISAC_main_inst, 15000, 30, 1) < 0) { + printf("cannot configure BWE\n"); + return -1; + } + } + + if (WebRtcIsac_SetMaxPayloadSize(ISAC_main_inst, payloadLimit) < 0) { + printf("cannot set maximum payload size %d.\n", payloadLimit); + return -1; + } + + if (rateLimit < 106800) { + if (WebRtcIsac_SetMaxRate(ISAC_main_inst, rateLimit) < 0) { + printf("cannot set the maximum rate %d.\n", rateLimit); + return -1; + } + } + + while (endfile == 0) { + fprintf(stderr, " \rframe = %7li", framecnt); + + //============== Readind from the file and encoding ================= + cur_framesmpls = 0; + stream_len = 0; + + if (onlyDecode) { + uint8_t auxUW8; + if (fread(&auxUW8, sizeof(uint8_t), 1, inp) < 1) { + break; + } + stream_len = auxUW8 << 8; + if (fread(&auxUW8, sizeof(uint8_t), 1, inp) < 1) { + break; + } + stream_len |= auxUW8; + if (fread(payload, 1, stream_len, inp) < stream_len) { + printf("last payload is corrupted\n"); + break; + } + } else { + while (stream_len == 0) { + int stream_len_int; + + // Read 10 ms speech block + endfile = readframe(shortdata, inp, samplesIn10Ms); + if (endfile) { + break; + } + cur_framesmpls += samplesIn10Ms; + + //-------- iSAC encoding --------- + stream_len_int = WebRtcIsac_Encode(ISAC_main_inst, shortdata, payload); + + if (stream_len_int < 0) { + // exit if returned with error + // errType=WebRtcIsac_GetErrorCode(ISAC_main_inst); + fprintf(stderr, "\nError in encoder\n"); + getc(stdin); + exit(EXIT_FAILURE); + } + stream_len = (size_t)stream_len_int; + } + //=================================================================== + if (endfile) { + break; + } + + rcuStreamLen = WebRtcIsac_GetRedPayload(ISAC_main_inst, payloadRCU); + if (rcuStreamLen < 0) { + fprintf(stderr, "\nError getting RED payload\n"); + getc(stdin); + exit(EXIT_FAILURE); + } + + get_arrival_time(cur_framesmpls, stream_len, bottleneck, &packetData, + sampFreqKHz * 1000, sampFreqKHz * 1000); + if (WebRtcIsac_UpdateBwEstimate( + ISAC_main_inst, payload, stream_len, packetData.rtp_number, + packetData.sample_count, packetData.arrival_time) < 0) { + printf(" BWE Error at client\n"); + return -1; + } + } + + if (endfile) { + break; + } + + maxStreamLen = (stream_len > maxStreamLen) ? stream_len : maxStreamLen; + packetCntr++; + + hist[stream_len]++; + if (averageFile != NULL) { + tmpSumStreamLen += stream_len; + if (packetCntr == 100) { + // kbps + fprintf(averageFile, "%8.3f ", + tmpSumStreamLen * 8.0 / (30.0 * packetCntr)); + packetCntr = 0; + tmpSumStreamLen = 0; + } + } + + if (onlyEncode) { + uint8_t auxUW8; + auxUW8 = (uint8_t)(((stream_len & 0x7F00) >> 8) & 0xFF); + if (fwrite(&auxUW8, sizeof(uint8_t), 1, outp) != 1) { + return -1; + } + + auxUW8 = (uint8_t)(stream_len & 0xFF); + if (fwrite(&auxUW8, sizeof(uint8_t), 1, outp) != 1) { + return -1; + } + if (fwrite(payload, 1, stream_len, outp) != stream_len) { + return -1; + } + } else { + //======================= iSAC decoding =========================== + + if ((rand() % 100) < packetLossPercent) { + declen = WebRtcIsac_DecodeRcu(ISAC_main_inst, payloadRCU, + (size_t)rcuStreamLen, decoded, + speechType); + lostPacketCntr++; + } else { + declen = WebRtcIsac_Decode(ISAC_main_inst, payload, stream_len, decoded, + speechType); + } + if (declen <= 0) { + // errType=WebRtcIsac_GetErrorCode(ISAC_main_inst); + fprintf(stderr, "\nError in decoder.\n"); + getc(stdin); + exit(1); + } + + // Write decoded speech frame to file + if (fwrite(decoded, sizeof(int16_t), declen, outp) != (size_t)declen) { + return -1; + } + cur_framesmpls = declen; + } + // Update Statistics + framecnt++; + totalsmpls += cur_framesmpls; + if (stream_len > 0) { + totalbits += 8 * stream_len; + } + if (rcuStreamLen > 0) { + totalBitsRCU += 8 * rcuStreamLen; + } + } + + rate = ((double)totalbits * (sampFreqKHz)) / (double)totalsmpls; + rateRCU = ((double)totalBitsRCU * (sampFreqKHz)) / (double)totalsmpls; + + printf("\n\n"); + printf("Sampling Rate............... %d kHz\n", sampFreqKHz); + printf("Payload Limit............... %d bytes \n", payloadLimit); + printf("Rate Limit.................. %d bits/sec \n", rateLimit); + +#ifdef WIN32 + fprintf(bitrateFile, "%d %10u %d %6.3f %6.3f %6.3f\n", + sampFreqKHz, framecnt, bottleneck, rateLB, rateUB, rate); + fclose(bitrateFile); +#endif // WIN32 + + printf("\n"); + printf("Measured bit-rate........... %0.3f kbps\n", rate); + printf("Measured RCU bit-ratre...... %0.3f kbps\n", rateRCU); + printf("Maximum bit-rate/payloadsize %0.3f / %" PRIuS "\n", + maxStreamLen * 8 / 0.03, maxStreamLen); + printf("Measured packet-loss........ %0.1f%% \n", + 100.0f * (float)lostPacketCntr / (float)packetCntr); + + printf("\n"); + +/* Runtime statistics */ +#ifdef WIN32 + runtime = (double)(clock() / (double)CLOCKS_PER_SEC - starttime); + length_file = ((double)framecnt * (double)declen / (sampFreqKHz * 1000)); + printf("Length of speech file....... %.1f s\n", length_file); + printf("Time to run iSAC............ %.2f s (%.2f %% of realtime)\n\n", + runtime, (100 * runtime / length_file)); +#endif + printf("\n\n_______________________________________________\n"); + + if (histFile != NULL) { + int n; + for (n = 0; n < 600; n++) { + fprintf(histFile, "%6d ", hist[n]); + } + fprintf(histFile, "\n"); + fclose(histFile); + } + if (averageFile != NULL) { + if (packetCntr > 0) { + fprintf(averageFile, "%8.3f ", + tmpSumStreamLen * 8.0 / (30.0 * packetCntr)); + } + fprintf(averageFile, "\n"); + fclose(averageFile); + } + + fclose(inp); + fclose(outp); + + WebRtcIsac_Free(ISAC_main_inst); + + exit(0); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.c new file mode 100644 index 0000000000..d9c4332123 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include +#include "utility.h" + +/* function for reading audio data from PCM file */ +int +readframe( + short* data, + FILE* inp, + int length) +{ + short k, rlen, status = 0; + unsigned char* ptrUChar; + ptrUChar = (unsigned char*)data; + + rlen = (short)fread(data, sizeof(short), length, inp); + if (rlen < length) { + for (k = rlen; k < length; k++) + data[k] = 0; + status = 1; + } + + // Assuming that our PCM files are written in Intel machines + for(k = 0; k < length; k++) + { + data[k] = (short)ptrUChar[k<<1] | ((((short)ptrUChar[(k<<1) + 1]) << 8) & 0xFF00); + } + + return status; +} + +short +readSwitch( + int argc, + char* argv[], + char* strID) +{ + short n; + for(n = 0; n < argc; n++) + { + if(strcmp(argv[n], strID) == 0) + { + return 1; + } + } + return 0; +} + +double +readParamDouble( + int argc, + char* argv[], + char* strID, + double defaultVal) +{ + double returnVal = defaultVal; + short n; + for(n = 0; n < argc; n++) + { + if(strcmp(argv[n], strID) == 0) + { + n++; + if(n < argc) + { + returnVal = atof(argv[n]); + } + break; + } + } + return returnVal; +} + +int +readParamInt( + int argc, + char* argv[], + char* strID, + int defaultVal) +{ + int returnVal = defaultVal; + short n; + for(n = 0; n < argc; n++) + { + if(strcmp(argv[n], strID) == 0) + { + n++; + if(n < argc) + { + returnVal = atoi(argv[n]); + } + break; + } + } + return returnVal; +} + +int +readParamString( + int argc, + char* argv[], + char* strID, + char* stringParam, + int maxSize) +{ + int paramLenght = 0; + short n; + for(n = 0; n < argc; n++) + { + if(strcmp(argv[n], strID) == 0) + { + n++; + if(n < argc) + { + strncpy(stringParam, argv[n], maxSize); + paramLenght = (int)strlen(argv[n]); + } + break; + } + } + return paramLenght; +} + +void +get_arrival_time( + int current_framesamples, /* samples */ + size_t packet_size, /* bytes */ + int bottleneck, /* excluding headers; bits/s */ + BottleNeckModel* BN_data, + short senderSampFreqHz, + short receiverSampFreqHz) +{ + unsigned int travelTimeMs; + const int headerSizeByte = 35; + + int headerRate; + + BN_data->whenPackGeneratedMs += (current_framesamples / (senderSampFreqHz / 1000)); + + headerRate = headerSizeByte * 8 * senderSampFreqHz / current_framesamples; /* bits/s */ + + /* everything in samples */ + BN_data->sample_count = BN_data->sample_count + current_framesamples; + + //travelTimeMs = ((packet_size + HeaderSize) * 8 * sampFreqHz) / + // (bottleneck + HeaderRate) + travelTimeMs = (unsigned int)floor((double)((packet_size + headerSizeByte) * 8 * 1000) + / (double)(bottleneck + headerRate) + 0.5); + + if(BN_data->whenPrevPackLeftMs > BN_data->whenPackGeneratedMs) + { + BN_data->whenPrevPackLeftMs += travelTimeMs; + } + else + { + BN_data->whenPrevPackLeftMs = BN_data->whenPackGeneratedMs + + travelTimeMs; + } + + BN_data->arrival_time = (BN_data->whenPrevPackLeftMs * + (receiverSampFreqHz / 1000)); + +// if (BN_data->arrival_time < BN_data->sample_count) +// BN_data->arrival_time = BN_data->sample_count; + + BN_data->rtp_number++; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.h new file mode 100644 index 0000000000..b5882a57fb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/util/utility.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_UTIL_UTILITY_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_UTIL_UTILITY_H_ + +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +#define OPEN_FILE_WB(filePtr, fullPath) \ + do \ + { \ + if(fullPath != NULL) \ + { \ + filePtr = fopen(fullPath, "wb"); \ + if(filePtr == NULL) \ + { \ + printf("could not open %s to write to.", fullPath); \ + return -1; \ + } \ + } \ + else \ + { \ + filePtr = NULL; \ + } \ + }while(0) + +#define OPEN_FILE_AB(filePtr, fullPath) \ + do \ + { \ + if(fullPath != NULL) \ + { \ + filePtr = fopen(fullPath, "ab"); \ + if(filePtr == NULL) \ + { \ + printf("could not open %s to write to.", fullPath); \ + return -1; \ + } \ + } \ + else \ + { \ + filePtr = NULL; \ + } \ + }while(0) + +#define OPEN_FILE_RB(filePtr, fullPath) \ + do \ + { \ + if(fullPath != NULL) \ + { \ + filePtr = fopen(fullPath, "rb"); \ + if(filePtr == NULL) \ + { \ + printf("could not open %s to read from.", fullPath); \ + return -1; \ + } \ + } \ + else \ + { \ + filePtr = NULL; \ + } \ + }while(0) + +#define WRITE_FILE_D(bufferPtr, len, filePtr) \ + do \ + { \ + if(filePtr != NULL) \ + { \ + double dummy[1000]; \ + int cntr; \ + for(cntr = 0; cntr < (len); cntr++) \ + { \ + dummy[cntr] = (double)bufferPtr[cntr]; \ + } \ + fwrite(dummy, sizeof(double), len, filePtr); \ + fflush(filePtr); \ + } \ + } while(0) + + typedef struct { + unsigned int whenPackGeneratedMs; + unsigned int whenPrevPackLeftMs; + unsigned int sendTimeMs ; /* milisecond */ + unsigned int arrival_time; /* samples */ + unsigned int sample_count; /* samples, also used as "send time stamp" */ + unsigned int rtp_number; + } BottleNeckModel; + + void get_arrival_time( + int current_framesamples, /* samples */ + size_t packet_size, /* bytes */ + int bottleneck, /* excluding headers; bits/s */ + BottleNeckModel* BN_data, + short senderSampFreqHz, + short receiverSampFreqHz); + + /* function for reading audio data from PCM file */ + int readframe( + short* data, + FILE* inp, + int length); + + short readSwitch( + int argc, + char* argv[], + char* strID); + + double readParamDouble( + int argc, + char* argv[], + char* strID, + double defaultVal); + + int readParamInt( + int argc, + char* argv[], + char* strID, + int defaultVal); + + int readParamString( + int argc, + char* argv[], + char* strID, + char* stringParam, + int maxSize); + +#if defined(__cplusplus) +} +#endif + + + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/unittest.cc new file mode 100644 index 0000000000..4e76e0dbbd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/unittest.cc @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include + +#include "modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h" +#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "rtc_base/buffer.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +namespace { + +const int kIsacNumberOfSamples = 32 * 60; // 60 ms at 32 kHz + +std::vector LoadSpeechData() { + webrtc::test::InputAudioFile input_file( + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm")); + std::vector speech_data(kIsacNumberOfSamples); + input_file.Read(kIsacNumberOfSamples, speech_data.data()); + return speech_data; +} + +template +IsacBandwidthInfo GetBwInfo(typename T::instance_type* inst) { + IsacBandwidthInfo bi; + T::GetBandwidthInfo(inst, &bi); + EXPECT_TRUE(bi.in_use); + return bi; +} + +// Encodes one packet. Returns the packet duration in milliseconds. +template +int EncodePacket(typename T::instance_type* inst, + const IsacBandwidthInfo* bi, + const int16_t* speech_data, + rtc::Buffer* output) { + output->SetSize(1000); + for (int duration_ms = 10;; duration_ms += 10) { + if (bi) + T::SetBandwidthInfo(inst, bi); + int encoded_bytes = T::Encode(inst, speech_data, output->data()); + if (encoded_bytes > 0 || duration_ms >= 60) { + EXPECT_GT(encoded_bytes, 0); + EXPECT_LE(static_cast(encoded_bytes), output->size()); + output->SetSize(encoded_bytes); + return duration_ms; + } + } +} + +template +std::vector DecodePacket(typename T::instance_type* inst, + const rtc::Buffer& encoded) { + std::vector decoded(kIsacNumberOfSamples); + int16_t speech_type; + int nsamples = T::DecodeInternal(inst, encoded.data(), encoded.size(), + &decoded.front(), &speech_type); + EXPECT_GT(nsamples, 0); + EXPECT_LE(static_cast(nsamples), decoded.size()); + decoded.resize(nsamples); + return decoded; +} + +class BoundedCapacityChannel final { + public: + BoundedCapacityChannel(int sample_rate_hz, int rate_bits_per_second) + : current_time_rtp_(0), + channel_rate_bytes_per_sample_(rate_bits_per_second / + (8.0 * sample_rate_hz)) {} + + // Simulate sending the given number of bytes at the given RTP time. Returns + // the new current RTP time after the sending is done. + int Send(int send_time_rtp, int nbytes) { + current_time_rtp_ = std::max(current_time_rtp_, send_time_rtp) + + nbytes / channel_rate_bytes_per_sample_; + return current_time_rtp_; + } + + private: + int current_time_rtp_; + // The somewhat strange unit for channel rate, bytes per sample, is because + // RTP time is measured in samples: + const double channel_rate_bytes_per_sample_; +}; + +// Test that the iSAC encoder produces identical output whether or not we use a +// conjoined encoder+decoder pair or a separate encoder and decoder that +// communicate BW estimation info explicitly. +template +void TestGetSetBandwidthInfo(const int16_t* speech_data, + int rate_bits_per_second, + int sample_rate_hz, + int frame_size_ms) { + const int bit_rate = 32000; + + // Conjoined encoder/decoder pair: + typename T::instance_type* encdec; + ASSERT_EQ(0, T::Create(&encdec)); + ASSERT_EQ(0, T::EncoderInit(encdec, adaptive ? 0 : 1)); + T::DecoderInit(encdec); + ASSERT_EQ(0, T::SetEncSampRate(encdec, sample_rate_hz)); + if (adaptive) + ASSERT_EQ(0, T::ControlBwe(encdec, bit_rate, frame_size_ms, false)); + else + ASSERT_EQ(0, T::Control(encdec, bit_rate, frame_size_ms)); + + // Disjoint encoder/decoder pair: + typename T::instance_type* enc; + ASSERT_EQ(0, T::Create(&enc)); + ASSERT_EQ(0, T::EncoderInit(enc, adaptive ? 0 : 1)); + ASSERT_EQ(0, T::SetEncSampRate(enc, sample_rate_hz)); + if (adaptive) + ASSERT_EQ(0, T::ControlBwe(enc, bit_rate, frame_size_ms, false)); + else + ASSERT_EQ(0, T::Control(enc, bit_rate, frame_size_ms)); + typename T::instance_type* dec; + ASSERT_EQ(0, T::Create(&dec)); + T::DecoderInit(dec); + T::SetInitialBweBottleneck(dec, bit_rate); + T::SetEncSampRateInDecoder(dec, sample_rate_hz); + + // 0. Get initial BW info from decoder. + auto bi = GetBwInfo(dec); + + BoundedCapacityChannel channel1(sample_rate_hz, rate_bits_per_second), + channel2(sample_rate_hz, rate_bits_per_second); + + int elapsed_time_ms = 0; + for (int i = 0; elapsed_time_ms < 10000; ++i) { + std::ostringstream ss; + ss << " i = " << i; + SCOPED_TRACE(ss.str()); + + // 1. Encode 3 * 10 ms or 6 * 10 ms. The separate encoder is given the BW + // info before each encode call. + rtc::Buffer bitstream1, bitstream2; + int duration1_ms = + EncodePacket(encdec, nullptr, speech_data, &bitstream1); + int duration2_ms = EncodePacket(enc, &bi, speech_data, &bitstream2); + EXPECT_EQ(duration1_ms, duration2_ms); + if (adaptive) + EXPECT_TRUE(duration1_ms == 30 || duration1_ms == 60); + else + EXPECT_EQ(frame_size_ms, duration1_ms); + ASSERT_EQ(bitstream1.size(), bitstream2.size()); + EXPECT_EQ(bitstream1, bitstream2); + + // 2. Deliver the encoded data to the decoders. + const int send_time = elapsed_time_ms * (sample_rate_hz / 1000); + EXPECT_EQ(0, T::UpdateBwEstimate( + encdec, bitstream1.data(), bitstream1.size(), i, send_time, + channel1.Send(send_time, + rtc::checked_cast(bitstream1.size())))); + EXPECT_EQ(0, T::UpdateBwEstimate( + dec, bitstream2.data(), bitstream2.size(), i, send_time, + channel2.Send(send_time, + rtc::checked_cast(bitstream2.size())))); + + // 3. Decode, and get new BW info from the separate decoder. + ASSERT_EQ(0, T::SetDecSampRate(encdec, sample_rate_hz)); + ASSERT_EQ(0, T::SetDecSampRate(dec, sample_rate_hz)); + auto decoded1 = DecodePacket(encdec, bitstream1); + auto decoded2 = DecodePacket(dec, bitstream2); + EXPECT_EQ(decoded1, decoded2); + bi = GetBwInfo(dec); + + elapsed_time_ms += duration1_ms; + } + + EXPECT_EQ(0, T::Free(encdec)); + EXPECT_EQ(0, T::Free(enc)); + EXPECT_EQ(0, T::Free(dec)); +} + +enum class IsacType { Fix, Float }; + +std::ostream& operator<<(std::ostream& os, IsacType t) { + os << (t == IsacType::Fix ? "fix" : "float"); + return os; +} + +struct IsacTestParam { + IsacType isac_type; + bool adaptive; + int channel_rate_bits_per_second; + int sample_rate_hz; + int frame_size_ms; + + friend std::ostream& operator<<(std::ostream& os, const IsacTestParam& itp) { + os << '{' << itp.isac_type << ',' + << (itp.adaptive ? "adaptive" : "nonadaptive") << ',' + << itp.channel_rate_bits_per_second << ',' << itp.sample_rate_hz << ',' + << itp.frame_size_ms << '}'; + return os; + } +}; + +class IsacCommonTest : public testing::TestWithParam {}; + +} // namespace + +TEST_P(IsacCommonTest, GetSetBandwidthInfo) { + auto p = GetParam(); + auto test_fun = [p] { + if (p.isac_type == IsacType::Fix) { + if (p.adaptive) + return TestGetSetBandwidthInfo; + else + return TestGetSetBandwidthInfo; + } else { + if (p.adaptive) + return TestGetSetBandwidthInfo; + else + return TestGetSetBandwidthInfo; + } + }(); + test_fun(LoadSpeechData().data(), p.channel_rate_bits_per_second, + p.sample_rate_hz, p.frame_size_ms); +} + +std::vector TestCases() { + static const IsacType types[] = {IsacType::Fix, IsacType::Float}; + static const bool adaptives[] = {true, false}; + static const int channel_rates[] = {12000, 15000, 19000, 22000}; + static const int sample_rates[] = {16000, 32000}; + static const int frame_sizes[] = {30, 60}; + std::vector cases; + for (IsacType type : types) + for (bool adaptive : adaptives) + for (int channel_rate : channel_rates) + for (int sample_rate : sample_rates) + if (!(type == IsacType::Fix && sample_rate == 32000)) + for (int frame_size : frame_sizes) + if (!(sample_rate == 32000 && frame_size == 60)) + cases.push_back( + {type, adaptive, channel_rate, sample_rate, frame_size}); + return cases; +} + +INSTANTIATE_TEST_CASE_P(, IsacCommonTest, testing::ValuesIn(TestCases())); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc new file mode 100644 index 0000000000..341336ee14 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" + +#include +#include +#include + +namespace webrtc { + +LegacyEncodedAudioFrame::LegacyEncodedAudioFrame(AudioDecoder* decoder, + rtc::Buffer&& payload) + : decoder_(decoder), payload_(std::move(payload)) {} + +LegacyEncodedAudioFrame::~LegacyEncodedAudioFrame() = default; + +size_t LegacyEncodedAudioFrame::Duration() const { + const int ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + return (ret < 0) ? 0 : static_cast(ret); +} + +rtc::Optional +LegacyEncodedAudioFrame::Decode(rtc::ArrayView decoded) const { + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + const int ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + + if (ret < 0) + return rtc::nullopt; + + return DecodeResult{static_cast(ret), speech_type}; +} + +std::vector LegacyEncodedAudioFrame::SplitBySamples( + AudioDecoder* decoder, + rtc::Buffer&& payload, + uint32_t timestamp, + size_t bytes_per_ms, + uint32_t timestamps_per_ms) { + RTC_DCHECK(payload.data()); + std::vector results; + size_t split_size_bytes = payload.size(); + + // Find a "chunk size" >= 20 ms and < 40 ms. + const size_t min_chunk_size = bytes_per_ms * 20; + if (min_chunk_size >= payload.size()) { + std::unique_ptr frame( + new LegacyEncodedAudioFrame(decoder, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); + } else { + // Reduce the split size by half as long as |split_size_bytes| is at least + // twice the minimum chunk size (so that the resulting size is at least as + // large as the minimum chunk size). + while (split_size_bytes >= 2 * min_chunk_size) { + split_size_bytes /= 2; + } + + const uint32_t timestamps_per_chunk = static_cast( + split_size_bytes * timestamps_per_ms / bytes_per_ms); + size_t byte_offset; + uint32_t timestamp_offset; + for (byte_offset = 0, timestamp_offset = 0; + byte_offset < payload.size(); + byte_offset += split_size_bytes, + timestamp_offset += timestamps_per_chunk) { + split_size_bytes = + std::min(split_size_bytes, payload.size() - byte_offset); + rtc::Buffer new_payload(payload.data() + byte_offset, split_size_bytes); + std::unique_ptr frame( + new LegacyEncodedAudioFrame(decoder, std::move(new_payload))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); + } + } + + return results; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h new file mode 100644 index 0000000000..275576e628 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ +#define MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ + +#include + +#include "api/array_view.h" +#include "api/audio_codecs/audio_decoder.h" + +namespace webrtc { + +class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame { + public: + LegacyEncodedAudioFrame(AudioDecoder* decoder, rtc::Buffer&& payload); + ~LegacyEncodedAudioFrame() override; + + static std::vector SplitBySamples( + AudioDecoder* decoder, + rtc::Buffer&& payload, + uint32_t timestamp, + size_t bytes_per_ms, + uint32_t timestamps_per_ms); + + size_t Duration() const override; + + rtc::Optional Decode( + rtc::ArrayView decoded) const override; + + // For testing: + const rtc::Buffer& payload() const { return payload_; } + + private: + AudioDecoder* const decoder_; + const rtc::Buffer payload_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc new file mode 100644 index 0000000000..e2dd4454e1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" + +#include "modules/audio_coding/acm2/rent_a_codec.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { + +class SplitBySamplesTest : public ::testing::TestWithParam { + protected: + virtual void SetUp() { + decoder_type_ = GetParam(); + switch (decoder_type_) { + case NetEqDecoder::kDecoderPCMu: + case NetEqDecoder::kDecoderPCMa: + bytes_per_ms_ = 8; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCMu_2ch: + case NetEqDecoder::kDecoderPCMa_2ch: + bytes_per_ms_ = 2 * 8; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderG722: + bytes_per_ms_ = 8; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16B: + bytes_per_ms_ = 16; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCM16Bwb: + bytes_per_ms_ = 32; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16Bswb32kHz: + bytes_per_ms_ = 64; + samples_per_ms_ = 32; + break; + case NetEqDecoder::kDecoderPCM16Bswb48kHz: + bytes_per_ms_ = 96; + samples_per_ms_ = 48; + break; + case NetEqDecoder::kDecoderPCM16B_2ch: + bytes_per_ms_ = 2 * 16; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCM16Bwb_2ch: + bytes_per_ms_ = 2 * 32; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: + bytes_per_ms_ = 2 * 64; + samples_per_ms_ = 32; + break; + case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: + bytes_per_ms_ = 2 * 96; + samples_per_ms_ = 48; + break; + case NetEqDecoder::kDecoderPCM16B_5ch: + bytes_per_ms_ = 5 * 16; + samples_per_ms_ = 8; + break; + default: + assert(false); + break; + } + } + size_t bytes_per_ms_; + int samples_per_ms_; + NetEqDecoder decoder_type_; +}; + +// Test splitting sample-based payloads. +TEST_P(SplitBySamplesTest, PayloadSizes) { + constexpr uint32_t kBaseTimestamp = 0x12345678; + struct ExpectedSplit { + size_t payload_size_ms; + size_t num_frames; + // For simplicity. We only expect up to two packets per split. + size_t frame_sizes[2]; + }; + // The payloads are expected to be split as follows: + // 10 ms -> 10 ms + // 20 ms -> 20 ms + // 30 ms -> 30 ms + // 40 ms -> 20 + 20 ms + // 50 ms -> 25 + 25 ms + // 60 ms -> 30 + 30 ms + ExpectedSplit expected_splits[] = { + {10, 1, {10}}, + {20, 1, {20}}, + {30, 1, {30}}, + {40, 2, {20, 20}}, + {50, 2, {25, 25}}, + {60, 2, {30, 30}} + }; + + for (const auto& expected_split : expected_splits) { + // The payload values are set to steadily increase (modulo 256), so that the + // resulting frames can be checked and we can be reasonably certain no + // sample was missed or repeated. + const auto generate_payload = [] (size_t num_bytes) { + rtc::Buffer payload(num_bytes); + uint8_t value = 0; + // Allow wrap-around of value in counter below. + for (size_t i = 0; i != payload.size(); ++i, ++value) { + payload[i] = value; + } + return payload; + }; + + const auto results = LegacyEncodedAudioFrame::SplitBySamples( + nullptr, + generate_payload(expected_split.payload_size_ms * bytes_per_ms_), + kBaseTimestamp, bytes_per_ms_, samples_per_ms_); + + EXPECT_EQ(expected_split.num_frames, results.size()); + uint32_t expected_timestamp = kBaseTimestamp; + uint32_t expected_byte_offset = 0; + uint8_t value = 0; + for (size_t i = 0; i != expected_split.num_frames; ++i) { + const auto& result = results[i]; + const LegacyEncodedAudioFrame* frame = + static_cast(result.frame.get()); + const size_t length_bytes = expected_split.frame_sizes[i] * bytes_per_ms_; + EXPECT_EQ(length_bytes, frame->payload().size()); + EXPECT_EQ(expected_timestamp, result.timestamp); + const rtc::Buffer& payload = frame->payload(); + // Allow wrap-around of value in counter below. + for (size_t i = 0; i != payload.size(); ++i, ++value) { + ASSERT_EQ(value, payload[i]); + } + + expected_timestamp += rtc::checked_cast( + expected_split.frame_sizes[i] * samples_per_ms_); + expected_byte_offset += rtc::checked_cast(length_bytes); + } + } +} + +INSTANTIATE_TEST_CASE_P( + LegacyEncodedAudioFrame, + SplitBySamplesTest, + ::testing::Values(NetEqDecoder::kDecoderPCMu, + NetEqDecoder::kDecoderPCMa, + NetEqDecoder::kDecoderPCMu_2ch, + NetEqDecoder::kDecoderPCMa_2ch, + NetEqDecoder::kDecoderG722, + NetEqDecoder::kDecoderPCM16B, + NetEqDecoder::kDecoderPCM16Bwb, + NetEqDecoder::kDecoderPCM16Bswb32kHz, + NetEqDecoder::kDecoderPCM16Bswb48kHz, + NetEqDecoder::kDecoderPCM16B_2ch, + NetEqDecoder::kDecoderPCM16Bwb_2ch, + NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch, + NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch, + NetEqDecoder::kDecoderPCM16B_5ch)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc new file mode 100644 index 0000000000..3d10b6fdbe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/audio_decoder_opus.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { +class OpusFrame : public AudioDecoder::EncodedAudioFrame { + public: + OpusFrame(AudioDecoderOpusImpl* decoder, + rtc::Buffer&& payload, + bool is_primary_payload) + : decoder_(decoder), + payload_(std::move(payload)), + is_primary_payload_(is_primary_payload) {} + + size_t Duration() const override { + int ret; + if (is_primary_payload_) { + ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + } else { + ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); + } + return (ret < 0) ? 0 : static_cast(ret); + } + + rtc::Optional Decode( + rtc::ArrayView decoded) const override { + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + int ret; + if (is_primary_payload_) { + ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } else { + ret = decoder_->DecodeRedundant( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } + + if (ret < 0) + return rtc::nullopt; + + return DecodeResult{static_cast(ret), speech_type}; + } + + private: + AudioDecoderOpusImpl* const decoder_; + const rtc::Buffer payload_; + const bool is_primary_payload_; +}; + +} // namespace + +AudioDecoderOpusImpl::AudioDecoderOpusImpl(size_t num_channels) + : channels_(num_channels) { + RTC_DCHECK(num_channels == 1 || num_channels == 2); + WebRtcOpus_DecoderCreate(&dec_state_, channels_); + WebRtcOpus_DecoderInit(dec_state_); +} + +AudioDecoderOpusImpl::~AudioDecoderOpusImpl() { + WebRtcOpus_DecoderFree(dec_state_); +} + +std::vector AudioDecoderOpusImpl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + + if (PacketHasFec(payload.data(), payload.size())) { + const int duration = + PacketDurationRedundant(payload.data(), payload.size()); + RTC_DCHECK_GE(duration, 0); + rtc::Buffer payload_copy(payload.data(), payload.size()); + std::unique_ptr fec_frame( + new OpusFrame(this, std::move(payload_copy), false)); + results.emplace_back(timestamp - duration, 1, std::move(fec_frame)); + } + std::unique_ptr frame( + new OpusFrame(this, std::move(payload), true)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; +} + +int AudioDecoderOpusImpl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz, 48000); + int16_t temp_type = 1; // Default is speech. + int ret = + WebRtcOpus_Decode(dec_state_, encoded, encoded_len, decoded, &temp_type); + if (ret > 0) + ret *= static_cast(channels_); // Return total number of samples. + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +int AudioDecoderOpusImpl::DecodeRedundantInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return DecodeInternal(encoded, encoded_len, sample_rate_hz, decoded, + speech_type); + } + + RTC_DCHECK_EQ(sample_rate_hz, 48000); + int16_t temp_type = 1; // Default is speech. + int ret = WebRtcOpus_DecodeFec(dec_state_, encoded, encoded_len, decoded, + &temp_type); + if (ret > 0) + ret *= static_cast(channels_); // Return total number of samples. + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +void AudioDecoderOpusImpl::Reset() { + WebRtcOpus_DecoderInit(dec_state_); +} + +int AudioDecoderOpusImpl::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + return WebRtcOpus_DurationEst(dec_state_, encoded, encoded_len); +} + +int AudioDecoderOpusImpl::PacketDurationRedundant(const uint8_t* encoded, + size_t encoded_len) const { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return PacketDuration(encoded, encoded_len); + } + + return WebRtcOpus_FecDurationEst(encoded, encoded_len); +} + +bool AudioDecoderOpusImpl::PacketHasFec(const uint8_t* encoded, + size_t encoded_len) const { + int fec; + fec = WebRtcOpus_PacketHasFec(encoded, encoded_len); + return (fec == 1); +} + +int AudioDecoderOpusImpl::SampleRateHz() const { + return 48000; +} + +size_t AudioDecoderOpusImpl::Channels() const { + return channels_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h new file mode 100644 index 0000000000..70aa40bdc4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_OPUS_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_OPUS_H_ + +#include "api/audio_codecs/audio_decoder.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class AudioDecoderOpusImpl final : public AudioDecoder { + public: + explicit AudioDecoderOpusImpl(size_t num_channels); + ~AudioDecoderOpusImpl() override; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + void Reset() override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int PacketDurationRedundant(const uint8_t* encoded, + size_t encoded_len) const override; + bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + int DecodeRedundantInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + OpusDecInst* dec_state_; + const size_t channels_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderOpusImpl); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_OPUS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc new file mode 100644 index 0000000000..d3455b56e1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -0,0 +1,819 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/audio_encoder_opus.h" + +#include +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/protobuf_utils.h" +#include "rtc_base/ptr_util.h" +#include "rtc_base/string_to_number.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { + +// Codec parameters for Opus. +// draft-spittka-payload-rtp-opus-03 + +// Recommended bitrates: +// 8-12 kb/s for NB speech, +// 16-20 kb/s for WB speech, +// 28-40 kb/s for FB speech, +// 48-64 kb/s for FB mono music, and +// 64-128 kb/s for FB stereo music. +// The current implementation applies the following values to mono signals, +// and multiplies them by 2 for stereo. +constexpr int kOpusBitrateNbBps = 12000; +constexpr int kOpusBitrateWbBps = 20000; +constexpr int kOpusBitrateFbBps = 32000; + +constexpr int kSampleRateHz = 48000; +constexpr int kDefaultMaxPlaybackRate = 48000; + +// These two lists must be sorted from low to high +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME +constexpr int kANASupportedFrameLengths[] = {20, 60, 120}; +constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120}; +#else +constexpr int kANASupportedFrameLengths[] = {20, 60}; +constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60}; +#endif + +// PacketLossFractionSmoother uses an exponential filter with a time constant +// of -1.0 / ln(0.9999) = 10000 ms. +constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f; + +// Optimize the loss rate to configure Opus. Basically, optimized loss rate is +// the input loss rate rounded down to various levels, because a robustly good +// audio quality is achieved by lowering the packet loss down. +// Additionally, to prevent toggling, margins are used, i.e., when jumping to +// a loss rate from below, a higher threshold is used than jumping to the same +// level from above. +float OptimizePacketLossRate(float new_loss_rate, float old_loss_rate) { + RTC_DCHECK_GE(new_loss_rate, 0.0f); + RTC_DCHECK_LE(new_loss_rate, 1.0f); + RTC_DCHECK_GE(old_loss_rate, 0.0f); + RTC_DCHECK_LE(old_loss_rate, 1.0f); + constexpr float kPacketLossRate20 = 0.20f; + constexpr float kPacketLossRate10 = 0.10f; + constexpr float kPacketLossRate5 = 0.05f; + constexpr float kPacketLossRate1 = 0.01f; + constexpr float kLossRate20Margin = 0.02f; + constexpr float kLossRate10Margin = 0.01f; + constexpr float kLossRate5Margin = 0.01f; + if (new_loss_rate >= + kPacketLossRate20 + + kLossRate20Margin * + (kPacketLossRate20 - old_loss_rate > 0 ? 1 : -1)) { + return kPacketLossRate20; + } else if (new_loss_rate >= + kPacketLossRate10 + + kLossRate10Margin * + (kPacketLossRate10 - old_loss_rate > 0 ? 1 : -1)) { + return kPacketLossRate10; + } else if (new_loss_rate >= + kPacketLossRate5 + + kLossRate5Margin * + (kPacketLossRate5 - old_loss_rate > 0 ? 1 : -1)) { + return kPacketLossRate5; + } else if (new_loss_rate >= kPacketLossRate1) { + return kPacketLossRate1; + } else { + return 0.0f; + } +} + +rtc::Optional GetFormatParameter(const SdpAudioFormat& format, + const std::string& param) { + auto it = format.parameters.find(param); + if (it == format.parameters.end()) + return rtc::nullopt; + + return it->second; +} + +template +rtc::Optional GetFormatParameter(const SdpAudioFormat& format, + const std::string& param) { + return rtc::StringToNumber(GetFormatParameter(format, param).value_or("")); +} + +int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) { + const int bitrate = [&] { + if (max_playback_rate <= 8000) { + return kOpusBitrateNbBps * rtc::dchecked_cast(num_channels); + } else if (max_playback_rate <= 16000) { + return kOpusBitrateWbBps * rtc::dchecked_cast(num_channels); + } else { + return kOpusBitrateFbBps * rtc::dchecked_cast(num_channels); + } + }(); + RTC_DCHECK_GE(bitrate, AudioEncoderOpusConfig::kMinBitrateBps); + RTC_DCHECK_LE(bitrate, AudioEncoderOpusConfig::kMaxBitrateBps); + return bitrate; +} + +// Get the maxaveragebitrate parameter in string-form, so we can properly figure +// out how invalid it is and accurately log invalid values. +int CalculateBitrate(int max_playback_rate_hz, + size_t num_channels, + rtc::Optional bitrate_param) { + const int default_bitrate = + CalculateDefaultBitrate(max_playback_rate_hz, num_channels); + + if (bitrate_param) { + const auto bitrate = rtc::StringToNumber(*bitrate_param); + if (bitrate) { + const int chosen_bitrate = + std::max(AudioEncoderOpusConfig::kMinBitrateBps, + std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps)); + if (bitrate != chosen_bitrate) { + RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate + << " clamped to " << chosen_bitrate; + } + return chosen_bitrate; + } + RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param + << "\" replaced by default bitrate " << default_bitrate; + } + + return default_bitrate; +} + +int GetChannelCount(const SdpAudioFormat& format) { + const auto param = GetFormatParameter(format, "stereo"); + if (param == "1") { + return 2; + } else { + return 1; + } +} + +int GetMaxPlaybackRate(const SdpAudioFormat& format) { + const auto param = GetFormatParameter(format, "maxplaybackrate"); + if (param && *param >= 8000) { + return std::min(*param, kDefaultMaxPlaybackRate); + } + return kDefaultMaxPlaybackRate; +} + +int GetFrameSizeMs(const SdpAudioFormat& format) { + const auto ptime = GetFormatParameter(format, "ptime"); + if (ptime) { + // Pick the next highest supported frame length from + // kOpusSupportedFrameLengths. + for (const int supported_frame_length : kOpusSupportedFrameLengths) { + if (supported_frame_length >= *ptime) { + return supported_frame_length; + } + } + // If none was found, return the largest supported frame length. + return *(std::end(kOpusSupportedFrameLengths) - 1); + } + + return AudioEncoderOpusConfig::kDefaultFrameSizeMs; +} + +void FindSupportedFrameLengths(int min_frame_length_ms, + int max_frame_length_ms, + std::vector* out) { + out->clear(); + std::copy_if(std::begin(kANASupportedFrameLengths), + std::end(kANASupportedFrameLengths), std::back_inserter(*out), + [&](int frame_length_ms) { + return frame_length_ms >= min_frame_length_ms && + frame_length_ms <= max_frame_length_ms; + }); + RTC_DCHECK(std::is_sorted(out->begin(), out->end())); +} + +int GetBitrateBps(const AudioEncoderOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + return *config.bitrate_bps; +} + +} // namespace + +void AudioEncoderOpusImpl::AppendSupportedEncoders( + std::vector* specs) { + const SdpAudioFormat fmt = { + "opus", 48000, 2, {{"minptime", "10"}, {"useinbandfec", "1"}}}; + const AudioCodecInfo info = QueryAudioEncoder(*SdpToConfig(fmt)); + specs->push_back({fmt, info}); +} + +AudioCodecInfo AudioEncoderOpusImpl::QueryAudioEncoder( + const AudioEncoderOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + AudioCodecInfo info(48000, config.num_channels, *config.bitrate_bps, + AudioEncoderOpusConfig::kMinBitrateBps, + AudioEncoderOpusConfig::kMaxBitrateBps); + info.allow_comfort_noise = false; + info.supports_network_adaption = true; + return info; +} + +std::unique_ptr AudioEncoderOpusImpl::MakeAudioEncoder( + const AudioEncoderOpusConfig& config, + int payload_type) { + RTC_DCHECK(config.IsOk()); + return rtc::MakeUnique(config, payload_type); +} + +rtc::Optional AudioEncoderOpusImpl::QueryAudioEncoder( + const SdpAudioFormat& format) { + if (STR_CASE_CMP(format.name.c_str(), GetPayloadName()) == 0 && + format.clockrate_hz == 48000 && format.num_channels == 2) { + const size_t num_channels = GetChannelCount(format); + const int bitrate = + CalculateBitrate(GetMaxPlaybackRate(format), num_channels, + GetFormatParameter(format, "maxaveragebitrate")); + AudioCodecInfo info(48000, num_channels, bitrate, + AudioEncoderOpusConfig::kMinBitrateBps, + AudioEncoderOpusConfig::kMaxBitrateBps); + info.allow_comfort_noise = false; + info.supports_network_adaption = true; + + return info; + } + return rtc::nullopt; +} + +AudioEncoderOpusConfig AudioEncoderOpusImpl::CreateConfig( + const CodecInst& codec_inst) { + AudioEncoderOpusConfig config; + config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48); + config.num_channels = codec_inst.channels; + config.bitrate_bps = codec_inst.rate; + config.application = config.num_channels == 1 + ? AudioEncoderOpusConfig::ApplicationMode::kVoip + : AudioEncoderOpusConfig::ApplicationMode::kAudio; + config.supported_frame_lengths_ms.push_back(config.frame_size_ms); + return config; +} + +rtc::Optional AudioEncoderOpusImpl::SdpToConfig( + const SdpAudioFormat& format) { + if (STR_CASE_CMP(format.name.c_str(), "opus") != 0 || + format.clockrate_hz != 48000) { + return rtc::nullopt; + } + + AudioEncoderOpusConfig config; + config.num_channels = GetChannelCount(format); + config.frame_size_ms = GetFrameSizeMs(format); + config.max_playback_rate_hz = GetMaxPlaybackRate(format); + config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1"); + config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1"); + config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1"); + config.bitrate_bps = + CalculateBitrate(config.max_playback_rate_hz, config.num_channels, + GetFormatParameter(format, "maxaveragebitrate")); + config.application = config.num_channels == 1 + ? AudioEncoderOpusConfig::ApplicationMode::kVoip + : AudioEncoderOpusConfig::ApplicationMode::kAudio; + + constexpr int kMinANAFrameLength = kANASupportedFrameLengths[0]; + constexpr int kMaxANAFrameLength = + kANASupportedFrameLengths[arraysize(kANASupportedFrameLengths) - 1]; + + // For now, minptime and maxptime are only used with ANA. If ptime is outside + // of this range, it will get adjusted once ANA takes hold. Ideally, we'd know + // if ANA was to be used when setting up the config, and adjust accordingly. + const int min_frame_length_ms = + GetFormatParameter(format, "minptime").value_or(kMinANAFrameLength); + const int max_frame_length_ms = + GetFormatParameter(format, "maxptime").value_or(kMaxANAFrameLength); + + FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms, + &config.supported_frame_lengths_ms); + RTC_DCHECK(config.IsOk()); + return config; +} + +rtc::Optional AudioEncoderOpusImpl::GetNewComplexity( + const AudioEncoderOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + const int bitrate_bps = GetBitrateBps(config); + if (bitrate_bps >= config.complexity_threshold_bps - + config.complexity_threshold_window_bps && + bitrate_bps <= config.complexity_threshold_bps + + config.complexity_threshold_window_bps) { + // Within the hysteresis window; make no change. + return rtc::nullopt; + } else { + return bitrate_bps <= config.complexity_threshold_bps + ? config.low_rate_complexity + : config.complexity; + } +} + +rtc::Optional AudioEncoderOpusImpl::GetNewBandwidth( + const AudioEncoderOpusConfig& config, + OpusEncInst* inst) { + constexpr int kMinWidebandBitrate = 8000; + constexpr int kMaxNarrowbandBitrate = 9000; + constexpr int kAutomaticThreshold = 11000; + RTC_DCHECK(config.IsOk()); + const int bitrate = GetBitrateBps(config); + if (bitrate > kAutomaticThreshold) { + return rtc::Optional(OPUS_AUTO); + } + const int bandwidth = WebRtcOpus_GetBandwidth(inst); + RTC_DCHECK_GE(bandwidth, 0); + if (bitrate > kMaxNarrowbandBitrate && bandwidth < OPUS_BANDWIDTH_WIDEBAND) { + return rtc::Optional(OPUS_BANDWIDTH_WIDEBAND); + } else if (bitrate < kMinWidebandBitrate && + bandwidth > OPUS_BANDWIDTH_NARROWBAND) { + return rtc::Optional(OPUS_BANDWIDTH_NARROWBAND); + } + return rtc::Optional(); +} + +class AudioEncoderOpusImpl::PacketLossFractionSmoother { + public: + explicit PacketLossFractionSmoother() + : last_sample_time_ms_(rtc::TimeMillis()), + smoother_(kAlphaForPacketLossFractionSmoother) {} + + // Gets the smoothed packet loss fraction. + float GetAverage() const { + float value = smoother_.filtered(); + return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value; + } + + // Add new observation to the packet loss fraction smoother. + void AddSample(float packet_loss_fraction) { + int64_t now_ms = rtc::TimeMillis(); + smoother_.Apply(static_cast(now_ms - last_sample_time_ms_), + packet_loss_fraction); + last_sample_time_ms_ = now_ms; + } + + private: + int64_t last_sample_time_ms_; + + // An exponential filter is used to smooth the packet loss fraction. + rtc::ExpFilter smoother_; +}; + +AudioEncoderOpusImpl::AudioEncoderOpusImpl(const AudioEncoderOpusConfig& config, + int payload_type) + : AudioEncoderOpusImpl( + config, + payload_type, + [this](const ProtoString& config_string, RtcEventLog* event_log) { + return DefaultAudioNetworkAdaptorCreator(config_string, event_log); + }, + // We choose 5sec as initial time constant due to empirical data. + rtc::MakeUnique(5000)) {} + +AudioEncoderOpusImpl::AudioEncoderOpusImpl( + const AudioEncoderOpusConfig& config, + int payload_type, + const AudioNetworkAdaptorCreator& audio_network_adaptor_creator, + std::unique_ptr bitrate_smoother) + : payload_type_(payload_type), + send_side_bwe_with_overhead_( + webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), + adjust_bandwidth_( + webrtc::field_trial::IsEnabled("WebRTC-AdjustOpusBandwidth")), + bitrate_changed_(true), + packet_loss_rate_(0.0), + inst_(nullptr), + packet_loss_fraction_smoother_(new PacketLossFractionSmoother()), + audio_network_adaptor_creator_(audio_network_adaptor_creator), + bitrate_smoother_(std::move(bitrate_smoother)), + consecutive_dtx_frames_(0) { + RTC_DCHECK(0 <= payload_type && payload_type <= 127); + + // Sanity check of the redundant payload type field that we want to get rid + // of. See https://bugs.chromium.org/p/webrtc/issues/detail?id=7847 + RTC_CHECK(config.payload_type == -1 || config.payload_type == payload_type); + + RTC_CHECK(RecreateEncoderInstance(config)); +} + +AudioEncoderOpusImpl::AudioEncoderOpusImpl(const CodecInst& codec_inst) + : AudioEncoderOpusImpl(CreateConfig(codec_inst), codec_inst.pltype) {} + +AudioEncoderOpusImpl::AudioEncoderOpusImpl(int payload_type, + const SdpAudioFormat& format) + : AudioEncoderOpusImpl(*SdpToConfig(format), payload_type) {} + +AudioEncoderOpusImpl::~AudioEncoderOpusImpl() { + RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); +} + +int AudioEncoderOpusImpl::SampleRateHz() const { + return kSampleRateHz; +} + +size_t AudioEncoderOpusImpl::NumChannels() const { + return config_.num_channels; +} + +size_t AudioEncoderOpusImpl::Num10MsFramesInNextPacket() const { + return Num10msFramesPerPacket(); +} + +size_t AudioEncoderOpusImpl::Max10MsFramesInAPacket() const { + return Num10msFramesPerPacket(); +} + +int AudioEncoderOpusImpl::GetTargetBitrate() const { + return GetBitrateBps(config_); +} + +void AudioEncoderOpusImpl::Reset() { + RTC_CHECK(RecreateEncoderInstance(config_)); +} + +bool AudioEncoderOpusImpl::SetFec(bool enable) { + if (enable) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_)); + } + config_.fec_enabled = enable; + return true; +} + +bool AudioEncoderOpusImpl::SetDtx(bool enable) { + if (enable) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_)); + } + config_.dtx_enabled = enable; + return true; +} + +bool AudioEncoderOpusImpl::GetDtx() const { + return config_.dtx_enabled; +} + +bool AudioEncoderOpusImpl::SetApplication(Application application) { + auto conf = config_; + switch (application) { + case Application::kSpeech: + conf.application = AudioEncoderOpusConfig::ApplicationMode::kVoip; + break; + case Application::kAudio: + conf.application = AudioEncoderOpusConfig::ApplicationMode::kAudio; + break; + } + return RecreateEncoderInstance(conf); +} + +void AudioEncoderOpusImpl::SetMaxPlaybackRate(int frequency_hz) { + auto conf = config_; + conf.max_playback_rate_hz = frequency_hz; + RTC_CHECK(RecreateEncoderInstance(conf)); +} + +bool AudioEncoderOpusImpl::EnableAudioNetworkAdaptor( + const std::string& config_string, + RtcEventLog* event_log) { + audio_network_adaptor_ = + audio_network_adaptor_creator_(config_string, event_log); + return audio_network_adaptor_.get() != nullptr; +} + +void AudioEncoderOpusImpl::DisableAudioNetworkAdaptor() { + audio_network_adaptor_.reset(nullptr); +} + +void AudioEncoderOpusImpl::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + if (!audio_network_adaptor_) { + packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction); + float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage(); + return SetProjectedPacketLossRate(average_fraction_loss); + } + audio_network_adaptor_->SetUplinkPacketLossFraction( + uplink_packet_loss_fraction); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpusImpl::OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetUplinkRecoverablePacketLossFraction( + uplink_recoverable_packet_loss_fraction); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpusImpl::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional bwe_period_ms) { + if (audio_network_adaptor_) { + audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps); + // We give smoothed bitrate allocation to audio network adaptor as + // the uplink bandwidth. + // The BWE spikes should not affect the bitrate smoother more than 25%. + // To simplify the calculations we use a step response as input signal. + // The step response of an exponential filter is + // u(t) = 1 - e^(-t / time_constant). + // In order to limit the affect of a BWE spike within 25% of its value + // before + // the next BWE update, we would choose a time constant that fulfills + // 1 - e^(-bwe_period_ms / time_constant) < 0.25 + // Then 4 * bwe_period_ms is a good choice. + if (bwe_period_ms) + bitrate_smoother_->SetTimeConstantMs(*bwe_period_ms * 4); + bitrate_smoother_->AddSample(target_audio_bitrate_bps); + + ApplyAudioNetworkAdaptor(); + } else if (send_side_bwe_with_overhead_) { + if (!overhead_bytes_per_packet_) { + RTC_LOG(LS_INFO) + << "AudioEncoderOpusImpl: Overhead unknown, target audio bitrate " + << target_audio_bitrate_bps << " bps is ignored."; + return; + } + const int overhead_bps = static_cast( + *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket()); + SetTargetBitrate( + std::min(AudioEncoderOpusConfig::kMaxBitrateBps, + std::max(AudioEncoderOpusConfig::kMinBitrateBps, + target_audio_bitrate_bps - overhead_bps))); + } else { + SetTargetBitrate(target_audio_bitrate_bps); + } +} + +void AudioEncoderOpusImpl::OnReceivedRtt(int rtt_ms) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetRtt(rtt_ms); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpusImpl::OnReceivedOverhead( + size_t overhead_bytes_per_packet) { + if (audio_network_adaptor_) { + audio_network_adaptor_->SetOverhead(overhead_bytes_per_packet); + ApplyAudioNetworkAdaptor(); + } else { + overhead_bytes_per_packet_ = overhead_bytes_per_packet; + } +} + +void AudioEncoderOpusImpl::SetReceiverFrameLengthRange( + int min_frame_length_ms, + int max_frame_length_ms) { + // Ensure that |SetReceiverFrameLengthRange| is called before + // |EnableAudioNetworkAdaptor|, otherwise we need to recreate + // |audio_network_adaptor_|, which is not a needed use case. + RTC_DCHECK(!audio_network_adaptor_); + FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms, + &config_.supported_frame_lengths_ms); +} + +AudioEncoder::EncodedInfo AudioEncoderOpusImpl::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + MaybeUpdateUplinkBandwidth(); + + if (input_buffer_.empty()) + first_timestamp_in_buffer_ = rtp_timestamp; + + input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend()); + if (input_buffer_.size() < + (Num10msFramesPerPacket() * SamplesPer10msFrame())) { + return EncodedInfo(); + } + RTC_CHECK_EQ(input_buffer_.size(), + Num10msFramesPerPacket() * SamplesPer10msFrame()); + + const size_t max_encoded_bytes = SufficientOutputBufferSize(); + EncodedInfo info; + info.encoded_bytes = + encoded->AppendData( + max_encoded_bytes, [&] (rtc::ArrayView encoded) { + int status = WebRtcOpus_Encode( + inst_, &input_buffer_[0], + rtc::CheckedDivExact(input_buffer_.size(), + config_.num_channels), + rtc::saturated_cast(max_encoded_bytes), + encoded.data()); + + RTC_CHECK_GE(status, 0); // Fails only if fed invalid data. + + return static_cast(status); + }); + input_buffer_.clear(); + + bool dtx_frame = (info.encoded_bytes <= 2); + + // Will use new packet size for next encoding. + config_.frame_size_ms = next_frame_length_ms_; + + if (adjust_bandwidth_ && bitrate_changed_) { + const auto bandwidth = GetNewBandwidth(config_, inst_); + if (bandwidth) { + RTC_CHECK_EQ(0, WebRtcOpus_SetBandwidth(inst_, *bandwidth)); + } + bitrate_changed_ = false; + } + + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.send_even_if_empty = true; // Allows Opus to send empty packets. + // After 20 DTX frames (MAX_CONSECUTIVE_DTX) Opus will send a frame + // coding the background noise. Avoid flagging this frame as speech + // (even though there is a probability of the frame being speech). + info.speech = !dtx_frame && (consecutive_dtx_frames_ != 20); + info.encoder_type = CodecType::kOpus; + + // Increase or reset DTX counter. + consecutive_dtx_frames_ = (dtx_frame) ? (consecutive_dtx_frames_ + 1) : (0); + + return info; +} + +size_t AudioEncoderOpusImpl::Num10msFramesPerPacket() const { + return static_cast(rtc::CheckedDivExact(config_.frame_size_ms, 10)); +} + +size_t AudioEncoderOpusImpl::SamplesPer10msFrame() const { + return rtc::CheckedDivExact(kSampleRateHz, 100) * config_.num_channels; +} + +size_t AudioEncoderOpusImpl::SufficientOutputBufferSize() const { + // Calculate the number of bytes we expect the encoder to produce, + // then multiply by two to give a wide margin for error. + const size_t bytes_per_millisecond = + static_cast(GetBitrateBps(config_) / (1000 * 8) + 1); + const size_t approx_encoded_bytes = + Num10msFramesPerPacket() * 10 * bytes_per_millisecond; + return 2 * approx_encoded_bytes; +} + +// If the given config is OK, recreate the Opus encoder instance with those +// settings, save the config, and return true. Otherwise, do nothing and return +// false. +bool AudioEncoderOpusImpl::RecreateEncoderInstance( + const AudioEncoderOpusConfig& config) { + if (!config.IsOk()) + return false; + config_ = config; + if (inst_) + RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); + input_buffer_.clear(); + input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame()); + RTC_CHECK_EQ(0, WebRtcOpus_EncoderCreate( + &inst_, config.num_channels, + config.application == + AudioEncoderOpusConfig::ApplicationMode::kVoip + ? 0 + : 1)); + RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, GetBitrateBps(config))); + if (config.fec_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_)); + } + RTC_CHECK_EQ( + 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz)); + // Use the default complexity if the start bitrate is within the hysteresis + // window. + complexity_ = GetNewComplexity(config).value_or(config.complexity); + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); + bitrate_changed_ = true; + if (config.dtx_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_)); + } + RTC_CHECK_EQ(0, + WebRtcOpus_SetPacketLossRate( + inst_, static_cast(packet_loss_rate_ * 100 + .5))); + if (config.cbr_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_)); + } + num_channels_to_encode_ = NumChannels(); + next_frame_length_ms_ = config_.frame_size_ms; + return true; +} + +void AudioEncoderOpusImpl::SetFrameLength(int frame_length_ms) { + next_frame_length_ms_ = frame_length_ms; +} + +void AudioEncoderOpusImpl::SetNumChannelsToEncode( + size_t num_channels_to_encode) { + RTC_DCHECK_GT(num_channels_to_encode, 0); + RTC_DCHECK_LE(num_channels_to_encode, config_.num_channels); + + if (num_channels_to_encode_ == num_channels_to_encode) + return; + + RTC_CHECK_EQ(0, WebRtcOpus_SetForceChannels(inst_, num_channels_to_encode)); + num_channels_to_encode_ = num_channels_to_encode; +} + +void AudioEncoderOpusImpl::SetProjectedPacketLossRate(float fraction) { + float opt_loss_rate = OptimizePacketLossRate(fraction, packet_loss_rate_); + if (packet_loss_rate_ != opt_loss_rate) { + packet_loss_rate_ = opt_loss_rate; + RTC_CHECK_EQ( + 0, WebRtcOpus_SetPacketLossRate( + inst_, static_cast(packet_loss_rate_ * 100 + .5))); + } +} + +void AudioEncoderOpusImpl::SetTargetBitrate(int bits_per_second) { + config_.bitrate_bps = rtc::SafeClamp( + bits_per_second, AudioEncoderOpusConfig::kMinBitrateBps, + AudioEncoderOpusConfig::kMaxBitrateBps); + RTC_DCHECK(config_.IsOk()); + RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, GetBitrateBps(config_))); + const auto new_complexity = GetNewComplexity(config_); + if (new_complexity && complexity_ != *new_complexity) { + complexity_ = *new_complexity; + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); + } + bitrate_changed_ = true; +} + +void AudioEncoderOpusImpl::ApplyAudioNetworkAdaptor() { + auto config = audio_network_adaptor_->GetEncoderRuntimeConfig(); + + if (config.bitrate_bps) + SetTargetBitrate(*config.bitrate_bps); + if (config.frame_length_ms) + SetFrameLength(*config.frame_length_ms); + if (config.enable_fec) + SetFec(*config.enable_fec); + if (config.uplink_packet_loss_fraction) + SetProjectedPacketLossRate(*config.uplink_packet_loss_fraction); + if (config.enable_dtx) + SetDtx(*config.enable_dtx); + if (config.num_channels) + SetNumChannelsToEncode(*config.num_channels); +} + +std::unique_ptr +AudioEncoderOpusImpl::DefaultAudioNetworkAdaptorCreator( + const ProtoString& config_string, + RtcEventLog* event_log) const { + AudioNetworkAdaptorImpl::Config config; + config.event_log = event_log; + return std::unique_ptr(new AudioNetworkAdaptorImpl( + config, ControllerManagerImpl::Create( + config_string, NumChannels(), supported_frame_lengths_ms(), + AudioEncoderOpusConfig::kMinBitrateBps, + num_channels_to_encode_, next_frame_length_ms_, + GetTargetBitrate(), config_.fec_enabled, GetDtx()))); +} + +void AudioEncoderOpusImpl::MaybeUpdateUplinkBandwidth() { + if (audio_network_adaptor_) { + int64_t now_ms = rtc::TimeMillis(); + if (!bitrate_smoother_last_update_time_ || + now_ms - *bitrate_smoother_last_update_time_ >= + config_.uplink_bandwidth_update_interval_ms) { + rtc::Optional smoothed_bitrate = bitrate_smoother_->GetAverage(); + if (smoothed_bitrate) + audio_network_adaptor_->SetUplinkBandwidth(*smoothed_bitrate); + bitrate_smoother_last_update_time_ = now_ms; + } + } +} + +ANAStats AudioEncoderOpusImpl::GetANAStats() const { + if (audio_network_adaptor_) { + return audio_network_adaptor_->GetStats(); + } + return ANAStats(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h new file mode 100644 index 0000000000..49c5207740 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ + +#include +#include +#include +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/audio_format.h" +#include "api/audio_codecs/opus/audio_encoder_opus_config.h" +#include "api/optional.h" +#include "common_audio/smoothing_filter.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/protobuf_utils.h" + +namespace webrtc { + +class RtcEventLog; + +struct CodecInst; + +class AudioEncoderOpusImpl final : public AudioEncoder { + public: + static AudioEncoderOpusConfig CreateConfig(const CodecInst& codec_inst); + + // Returns empty if the current bitrate falls within the hysteresis window, + // defined by complexity_threshold_bps +/- complexity_threshold_window_bps. + // Otherwise, returns the current complexity depending on whether the + // current bitrate is above or below complexity_threshold_bps. + static rtc::Optional GetNewComplexity( + const AudioEncoderOpusConfig& config); + + // Returns OPUS_AUTO if the the current bitrate is above wideband threshold. + // Returns empty if it is below, but bandwidth coincides with the desired one. + // Otherwise returns the desired bandwidth. + static rtc::Optional GetNewBandwidth( + const AudioEncoderOpusConfig& config, + OpusEncInst* inst); + + using AudioNetworkAdaptorCreator = + std::function(const std::string&, + RtcEventLog*)>; + + AudioEncoderOpusImpl(const AudioEncoderOpusConfig& config, int payload_type); + + // Dependency injection for testing. + AudioEncoderOpusImpl( + const AudioEncoderOpusConfig& config, + int payload_type, + const AudioNetworkAdaptorCreator& audio_network_adaptor_creator, + std::unique_ptr bitrate_smoother); + + explicit AudioEncoderOpusImpl(const CodecInst& codec_inst); + AudioEncoderOpusImpl(int payload_type, const SdpAudioFormat& format); + ~AudioEncoderOpusImpl() override; + + // Static interface for use by BuiltinAudioEncoderFactory. + static constexpr const char* GetPayloadName() { return "opus"; } + static rtc::Optional QueryAudioEncoder( + const SdpAudioFormat& format); + + int SampleRateHz() const override; + size_t NumChannels() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + + void Reset() override; + bool SetFec(bool enable) override; + + // Set Opus DTX. Once enabled, Opus stops transmission, when it detects + // voice being inactive. During that, it still sends 2 packets (one for + // content, one for signaling) about every 400 ms. + bool SetDtx(bool enable) override; + bool GetDtx() const override; + + bool SetApplication(Application application) override; + void SetMaxPlaybackRate(int frequency_hz) override; + bool EnableAudioNetworkAdaptor(const std::string& config_string, + RtcEventLog* event_log) override; + void DisableAudioNetworkAdaptor() override; + void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) override; + void OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional bwe_period_ms) override; + void OnReceivedRtt(int rtt_ms) override; + void OnReceivedOverhead(size_t overhead_bytes_per_packet) override; + void SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) override; + ANAStats GetANAStats() const override; + rtc::ArrayView supported_frame_lengths_ms() const { + return config_.supported_frame_lengths_ms; + } + + // Getters for testing. + float packet_loss_rate() const { return packet_loss_rate_; } + AudioEncoderOpusConfig::ApplicationMode application() const { + return config_.application; + } + bool fec_enabled() const { return config_.fec_enabled; } + size_t num_channels_to_encode() const { return num_channels_to_encode_; } + int next_frame_length_ms() const { return next_frame_length_ms_; } + + protected: + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + private: + class PacketLossFractionSmoother; + + static rtc::Optional SdpToConfig( + const SdpAudioFormat& format); + static void AppendSupportedEncoders(std::vector* specs); + static AudioCodecInfo QueryAudioEncoder(const AudioEncoderOpusConfig& config); + static std::unique_ptr MakeAudioEncoder( + const AudioEncoderOpusConfig&, + int payload_type); + + size_t Num10msFramesPerPacket() const; + size_t SamplesPer10msFrame() const; + size_t SufficientOutputBufferSize() const; + bool RecreateEncoderInstance(const AudioEncoderOpusConfig& config); + void SetFrameLength(int frame_length_ms); + void SetNumChannelsToEncode(size_t num_channels_to_encode); + void SetProjectedPacketLossRate(float fraction); + + // TODO(minyue): remove "override" when we can deprecate + // |AudioEncoder::SetTargetBitrate|. + void SetTargetBitrate(int target_bps) override; + + void ApplyAudioNetworkAdaptor(); + std::unique_ptr DefaultAudioNetworkAdaptorCreator( + const ProtoString& config_string, + RtcEventLog* event_log) const; + + void MaybeUpdateUplinkBandwidth(); + + AudioEncoderOpusConfig config_; + const int payload_type_; + const bool send_side_bwe_with_overhead_; + const bool adjust_bandwidth_; + bool bitrate_changed_; + float packet_loss_rate_; + std::vector input_buffer_; + OpusEncInst* inst_; + uint32_t first_timestamp_in_buffer_; + size_t num_channels_to_encode_; + int next_frame_length_ms_; + int complexity_; + std::unique_ptr packet_loss_fraction_smoother_; + const AudioNetworkAdaptorCreator audio_network_adaptor_creator_; + std::unique_ptr audio_network_adaptor_; + rtc::Optional overhead_bytes_per_packet_; + const std::unique_ptr bitrate_smoother_; + rtc::Optional bitrate_smoother_last_update_time_; + int consecutive_dtx_frames_; + + friend struct AudioEncoderOpus; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderOpusImpl); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc new file mode 100644 index 0000000000..dfef6821ad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -0,0 +1,877 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "common_audio/mocks/mock_smoothing_filter.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h" +#include "modules/audio_coding/codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "rtc_base/checks.h" +#include "rtc_base/fakeclock.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +using ::testing::NiceMock; +using ::testing::Return; + +namespace { + +const CodecInst kDefaultOpusSettings = {105, "opus", 48000, 960, 1, 32000}; +constexpr int64_t kInitialTimeUs = 12345678; + +AudioEncoderOpusConfig CreateConfig(const CodecInst& codec_inst) { + AudioEncoderOpusConfig config; + config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48); + config.num_channels = codec_inst.channels; + config.bitrate_bps = codec_inst.rate; + config.application = config.num_channels == 1 + ? AudioEncoderOpusConfig::ApplicationMode::kVoip + : AudioEncoderOpusConfig::ApplicationMode::kAudio; + config.supported_frame_lengths_ms.push_back(config.frame_size_ms); + return config; +} + +AudioEncoderOpusConfig CreateConfigWithParameters( + const SdpAudioFormat::Parameters& params) { + const SdpAudioFormat format("opus", 48000, 2, params); + return *AudioEncoderOpus::SdpToConfig(format); +} + +struct AudioEncoderOpusStates { + std::shared_ptr mock_audio_network_adaptor; + MockSmoothingFilter* mock_bitrate_smoother; + std::unique_ptr encoder; + std::unique_ptr fake_clock; + AudioEncoderOpusConfig config; +}; + +AudioEncoderOpusStates CreateCodec(size_t num_channels) { + AudioEncoderOpusStates states; + states.mock_audio_network_adaptor = + std::make_shared(nullptr); + states.fake_clock.reset(new rtc::ScopedFakeClock()); + states.fake_clock->SetTimeMicros(kInitialTimeUs); + std::weak_ptr mock_ptr( + states.mock_audio_network_adaptor); + AudioEncoderOpusImpl::AudioNetworkAdaptorCreator creator = + [mock_ptr](const std::string&, RtcEventLog* event_log) { + std::unique_ptr adaptor( + new NiceMock()); + EXPECT_CALL(*adaptor, Die()); + if (auto sp = mock_ptr.lock()) { + *sp = adaptor.get(); + } else { + RTC_NOTREACHED(); + } + return adaptor; + }; + + CodecInst codec_inst = kDefaultOpusSettings; + codec_inst.channels = num_channels; + states.config = CreateConfig(codec_inst); + std::unique_ptr bitrate_smoother( + new MockSmoothingFilter()); + states.mock_bitrate_smoother = bitrate_smoother.get(); + + states.encoder.reset(new AudioEncoderOpusImpl( + states.config, codec_inst.pltype, std::move(creator), + std::move(bitrate_smoother))); + return states; +} + +AudioEncoderRuntimeConfig CreateEncoderRuntimeConfig() { + constexpr int kBitrate = 40000; + constexpr int kFrameLength = 60; + constexpr bool kEnableFec = true; + constexpr bool kEnableDtx = false; + constexpr size_t kNumChannels = 1; + constexpr float kPacketLossFraction = 0.1f; + AudioEncoderRuntimeConfig config; + config.bitrate_bps = kBitrate; + config.frame_length_ms = kFrameLength; + config.enable_fec = kEnableFec; + config.enable_dtx = kEnableDtx; + config.num_channels = kNumChannels; + config.uplink_packet_loss_fraction = kPacketLossFraction; + return config; +} + +void CheckEncoderRuntimeConfig(const AudioEncoderOpusImpl* encoder, + const AudioEncoderRuntimeConfig& config) { + EXPECT_EQ(*config.bitrate_bps, encoder->GetTargetBitrate()); + EXPECT_EQ(*config.frame_length_ms, encoder->next_frame_length_ms()); + EXPECT_EQ(*config.enable_fec, encoder->fec_enabled()); + EXPECT_EQ(*config.enable_dtx, encoder->GetDtx()); + EXPECT_EQ(*config.num_channels, encoder->num_channels_to_encode()); +} + +// Create 10ms audio data blocks for a total packet size of "packet_size_ms". +std::unique_ptr Create10msAudioBlocks( + const std::unique_ptr& encoder, + int packet_size_ms) { + const std::string file_name = + test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + + std::unique_ptr speech_data(new test::AudioLoop()); + int audio_samples_per_ms = + rtc::CheckedDivExact(encoder->SampleRateHz(), 1000); + if (!speech_data->Init( + file_name, + packet_size_ms * audio_samples_per_ms * + encoder->num_channels_to_encode(), + 10 * audio_samples_per_ms * encoder->num_channels_to_encode())) + return nullptr; + return speech_data; +} + +} // namespace + +TEST(AudioEncoderOpusTest, DefaultApplicationModeMono) { + auto states = CreateCodec(1); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states.encoder->application()); +} + +TEST(AudioEncoderOpusTest, DefaultApplicationModeStereo) { + auto states = CreateCodec(2); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kAudio, + states.encoder->application()); +} + +TEST(AudioEncoderOpusTest, ChangeApplicationMode) { + auto states = CreateCodec(2); + EXPECT_TRUE( + states.encoder->SetApplication(AudioEncoder::Application::kSpeech)); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states.encoder->application()); +} + +TEST(AudioEncoderOpusTest, ResetWontChangeApplicationMode) { + auto states = CreateCodec(2); + + // Trigger a reset. + states.encoder->Reset(); + // Verify that the mode is still kAudio. + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kAudio, + states.encoder->application()); + + // Now change to kVoip. + EXPECT_TRUE( + states.encoder->SetApplication(AudioEncoder::Application::kSpeech)); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states.encoder->application()); + + // Trigger a reset again. + states.encoder->Reset(); + // Verify that the mode is still kVoip. + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states.encoder->application()); +} + +TEST(AudioEncoderOpusTest, ToggleDtx) { + auto states = CreateCodec(2); + // Enable DTX + EXPECT_TRUE(states.encoder->SetDtx(true)); + EXPECT_TRUE(states.encoder->GetDtx()); + // Turn off DTX. + EXPECT_TRUE(states.encoder->SetDtx(false)); + EXPECT_FALSE(states.encoder->GetDtx()); +} + +TEST(AudioEncoderOpusTest, + OnReceivedUplinkBandwidthWithoutAudioNetworkAdaptor) { + auto states = CreateCodec(1); + // Constants are replicated from audio_states.encoderopus.cc. + const int kMinBitrateBps = 6000; + const int kMaxBitrateBps = 510000; + // Set a too low bitrate. + states.encoder->OnReceivedUplinkBandwidth(kMinBitrateBps - 1, rtc::nullopt); + EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); + // Set a too high bitrate. + states.encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps + 1, rtc::nullopt); + EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); + // Set the minimum rate. + states.encoder->OnReceivedUplinkBandwidth(kMinBitrateBps, rtc::nullopt); + EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); + // Set the maximum rate. + states.encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps, rtc::nullopt); + EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); + // Set rates from kMaxBitrateBps up to 32000 bps. + for (int rate = kMinBitrateBps; rate <= 32000; rate += 1000) { + states.encoder->OnReceivedUplinkBandwidth(rate, rtc::nullopt); + EXPECT_EQ(rate, states.encoder->GetTargetBitrate()); + } +} + +namespace { + +// Returns a vector with the n evenly-spaced numbers a, a + (b - a)/(n - 1), +// ..., b. +std::vector IntervalSteps(float a, float b, size_t n) { + RTC_DCHECK_GT(n, 1u); + const float step = (b - a) / (n - 1); + std::vector points; + points.push_back(a); + for (size_t i = 1; i < n - 1; ++i) + points.push_back(a + i * step); + points.push_back(b); + return points; +} + +// Sets the packet loss rate to each number in the vector in turn, and verifies +// that the loss rate as reported by the encoder is |expected_return| for all +// of them. +void TestSetPacketLossRate(AudioEncoderOpusStates* states, + const std::vector& losses, + float expected_return) { + // |kSampleIntervalMs| is chosen to ease the calculation since + // 0.9999 ^ 184198 = 1e-8. Which minimizes the effect of + // PacketLossFractionSmoother used in AudioEncoderOpus. + constexpr int64_t kSampleIntervalMs = 184198; + for (float loss : losses) { + states->encoder->OnReceivedUplinkPacketLossFraction(loss); + states->fake_clock->AdvanceTime( + rtc::TimeDelta::FromMilliseconds(kSampleIntervalMs)); + EXPECT_FLOAT_EQ(expected_return, states->encoder->packet_loss_rate()); + } +} + +} // namespace + +TEST(AudioEncoderOpusTest, PacketLossRateOptimized) { + auto states = CreateCodec(1); + auto I = [](float a, float b) { return IntervalSteps(a, b, 10); }; + constexpr float eps = 1e-8f; + + // Note that the order of the following calls is critical. + + // clang-format off + TestSetPacketLossRate(&states, I(0.00f , 0.01f - eps), 0.00f); + TestSetPacketLossRate(&states, I(0.01f + eps, 0.06f - eps), 0.01f); + TestSetPacketLossRate(&states, I(0.06f + eps, 0.11f - eps), 0.05f); + TestSetPacketLossRate(&states, I(0.11f + eps, 0.22f - eps), 0.10f); + TestSetPacketLossRate(&states, I(0.22f + eps, 1.00f ), 0.20f); + + TestSetPacketLossRate(&states, I(1.00f , 0.18f + eps), 0.20f); + TestSetPacketLossRate(&states, I(0.18f - eps, 0.09f + eps), 0.10f); + TestSetPacketLossRate(&states, I(0.09f - eps, 0.04f + eps), 0.05f); + TestSetPacketLossRate(&states, I(0.04f - eps, 0.01f + eps), 0.01f); + TestSetPacketLossRate(&states, I(0.01f - eps, 0.00f ), 0.00f); + // clang-format on +} + +TEST(AudioEncoderOpusTest, SetReceiverFrameLengthRange) { + auto states = CreateCodec(2); + // Before calling to |SetReceiverFrameLengthRange|, + // |supported_frame_lengths_ms| should contain only the frame length being + // used. + using ::testing::ElementsAre; + EXPECT_THAT(states.encoder->supported_frame_lengths_ms(), + ElementsAre(states.encoder->next_frame_length_ms())); + states.encoder->SetReceiverFrameLengthRange(0, 12345); + states.encoder->SetReceiverFrameLengthRange(21, 60); + EXPECT_THAT(states.encoder->supported_frame_lengths_ms(), ElementsAre(60)); + states.encoder->SetReceiverFrameLengthRange(20, 59); + EXPECT_THAT(states.encoder->supported_frame_lengths_ms(), ElementsAre(20)); +} + +TEST(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnReceivedUplinkPacketLossFraction) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any packet loss fraction is fine. + constexpr float kUplinkPacketLoss = 0.1f; + EXPECT_CALL(**states.mock_audio_network_adaptor, + SetUplinkPacketLossFraction(kUplinkPacketLoss)); + states.encoder->OnReceivedUplinkPacketLossFraction(kUplinkPacketLoss); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedUplinkBandwidth) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any target audio bitrate is fine. + constexpr int kTargetAudioBitrate = 30000; + constexpr int64_t kProbingIntervalMs = 3000; + EXPECT_CALL(**states.mock_audio_network_adaptor, + SetTargetAudioBitrate(kTargetAudioBitrate)); + EXPECT_CALL(*states.mock_bitrate_smoother, + SetTimeConstantMs(kProbingIntervalMs * 4)); + EXPECT_CALL(*states.mock_bitrate_smoother, AddSample(kTargetAudioBitrate)); + states.encoder->OnReceivedUplinkBandwidth(kTargetAudioBitrate, + kProbingIntervalMs); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedRtt) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any rtt is fine. + constexpr int kRtt = 30; + EXPECT_CALL(**states.mock_audio_network_adaptor, SetRtt(kRtt)); + states.encoder->OnReceivedRtt(kRtt); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedOverhead) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any overhead is fine. + constexpr size_t kOverhead = 64; + EXPECT_CALL(**states.mock_audio_network_adaptor, SetOverhead(kOverhead)); + states.encoder->OnReceivedOverhead(kOverhead); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, + PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) { + auto states = CreateCodec(2); + + // The values are carefully chosen so that if no smoothing is made, the test + // will fail. + constexpr float kPacketLossFraction_1 = 0.02f; + constexpr float kPacketLossFraction_2 = 0.198f; + // |kSecondSampleTimeMs| is chosen to ease the calculation since + // 0.9999 ^ 6931 = 0.5. + constexpr int64_t kSecondSampleTimeMs = 6931; + + // First time, no filtering. + states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1); + EXPECT_FLOAT_EQ(0.01f, states.encoder->packet_loss_rate()); + + states.fake_clock->AdvanceTime( + rtc::TimeDelta::FromMilliseconds(kSecondSampleTimeMs)); + states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2); + + // Now the output of packet loss fraction smoother should be + // (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized + // packet loss rate to increase to 0.05. If no smoothing has been made, the + // optimized packet loss rate should have been increase to 0.1. + EXPECT_FLOAT_EQ(0.05f, states.encoder->packet_loss_rate()); +} + +TEST(AudioEncoderOpusTest, DoNotInvokeSetTargetBitrateIfOverheadUnknown) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + + auto states = CreateCodec(2); + + states.encoder->OnReceivedUplinkBandwidth(kDefaultOpusSettings.rate * 2, + rtc::nullopt); + + // Since |OnReceivedOverhead| has not been called, the codec bitrate should + // not change. + EXPECT_EQ(kDefaultOpusSettings.rate, states.encoder->GetTargetBitrate()); +} + +TEST(AudioEncoderOpusTest, OverheadRemovedFromTargetAudioBitrate) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + + auto states = CreateCodec(2); + + constexpr size_t kOverheadBytesPerPacket = 64; + states.encoder->OnReceivedOverhead(kOverheadBytesPerPacket); + + constexpr int kTargetBitrateBps = 40000; + states.encoder->OnReceivedUplinkBandwidth(kTargetBitrateBps, rtc::nullopt); + + int packet_rate = rtc::CheckedDivExact(48000, kDefaultOpusSettings.pacsize); + EXPECT_EQ(kTargetBitrateBps - + 8 * static_cast(kOverheadBytesPerPacket) * packet_rate, + states.encoder->GetTargetBitrate()); +} + +TEST(AudioEncoderOpusTest, BitrateBounded) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + + constexpr int kMinBitrateBps = 6000; + constexpr int kMaxBitrateBps = 510000; + + auto states = CreateCodec(2); + + constexpr size_t kOverheadBytesPerPacket = 64; + states.encoder->OnReceivedOverhead(kOverheadBytesPerPacket); + + int packet_rate = rtc::CheckedDivExact(48000, kDefaultOpusSettings.pacsize); + + // Set a target rate that is smaller than |kMinBitrateBps| when overhead is + // subtracted. The eventual codec rate should be bounded by |kMinBitrateBps|. + int target_bitrate = + kOverheadBytesPerPacket * 8 * packet_rate + kMinBitrateBps - 1; + states.encoder->OnReceivedUplinkBandwidth(target_bitrate, rtc::nullopt); + EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); + + // Set a target rate that is greater than |kMaxBitrateBps| when overhead is + // subtracted. The eventual codec rate should be bounded by |kMaxBitrateBps|. + target_bitrate = + kOverheadBytesPerPacket * 8 * packet_rate + kMaxBitrateBps + 1; + states.encoder->OnReceivedUplinkBandwidth(target_bitrate, rtc::nullopt); + EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); +} + +// Verifies that the complexity adaptation in the config works as intended. +TEST(AudioEncoderOpusTest, ConfigComplexityAdaptation) { + AudioEncoderOpusConfig config; + config.low_rate_complexity = 8; + config.complexity = 6; + + // Bitrate within hysteresis window. Expect empty output. + config.bitrate_bps = 12500; + EXPECT_EQ(rtc::nullopt, AudioEncoderOpusImpl::GetNewComplexity(config)); + + // Bitrate below hysteresis window. Expect higher complexity. + config.bitrate_bps = 10999; + EXPECT_EQ(8, AudioEncoderOpusImpl::GetNewComplexity(config)); + + // Bitrate within hysteresis window. Expect empty output. + config.bitrate_bps = 12500; + EXPECT_EQ(rtc::nullopt, AudioEncoderOpusImpl::GetNewComplexity(config)); + + // Bitrate above hysteresis window. Expect lower complexity. + config.bitrate_bps = 14001; + EXPECT_EQ(6, AudioEncoderOpusImpl::GetNewComplexity(config)); +} + +// Verifies that the bandwidth adaptation in the config works as intended. +TEST(AudioEncoderOpusTest, ConfigBandwidthAdaptation) { + AudioEncoderOpusConfig config; + // Sample rate of Opus. + constexpr size_t kOpusRateKhz = 48; + std::vector silence( + kOpusRateKhz * config.frame_size_ms * config.num_channels, 0); + constexpr size_t kMaxBytes = 1000; + uint8_t bitstream[kMaxBytes]; + + OpusEncInst* inst; + EXPECT_EQ(0, WebRtcOpus_EncoderCreate( + &inst, config.num_channels, + config.application == + AudioEncoderOpusConfig::ApplicationMode::kVoip + ? 0 + : 1)); + + // Bitrate below minmum wideband. Expect narrowband. + config.bitrate_bps = rtc::Optional(7999); + auto bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(rtc::Optional(OPUS_BANDWIDTH_NARROWBAND), bandwidth); + WebRtcOpus_SetBandwidth(inst, *bandwidth); + // It is necessary to encode here because Opus has some logic in the encoder + // that goes from the user-set bandwidth to the used and returned one. + WebRtcOpus_Encode(inst, silence.data(), + rtc::CheckedDivExact(silence.size(), config.num_channels), + kMaxBytes, bitstream); + + // Bitrate not yet above maximum narrowband. Expect empty. + config.bitrate_bps = rtc::Optional(9000); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(rtc::Optional(), bandwidth); + + // Bitrate above maximum narrowband. Expect wideband. + config.bitrate_bps = rtc::Optional(9001); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(rtc::Optional(OPUS_BANDWIDTH_WIDEBAND), bandwidth); + WebRtcOpus_SetBandwidth(inst, *bandwidth); + // It is necessary to encode here because Opus has some logic in the encoder + // that goes from the user-set bandwidth to the used and returned one. + WebRtcOpus_Encode(inst, silence.data(), + rtc::CheckedDivExact(silence.size(), config.num_channels), + kMaxBytes, bitstream); + + // Bitrate not yet below minimum wideband. Expect empty. + config.bitrate_bps = rtc::Optional(8000); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(rtc::Optional(), bandwidth); + + // Bitrate above automatic threshold. Expect automatic. + config.bitrate_bps = rtc::Optional(12001); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(rtc::Optional(OPUS_AUTO), bandwidth); + + EXPECT_EQ(0, WebRtcOpus_EncoderFree(inst)); +} + +TEST(AudioEncoderOpusTest, EmptyConfigDoesNotAffectEncoderSettings) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + AudioEncoderRuntimeConfig empty_config; + + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)) + .WillOnce(Return(empty_config)); + + constexpr size_t kOverhead = 64; + EXPECT_CALL(**states.mock_audio_network_adaptor, SetOverhead(kOverhead)) + .Times(2); + states.encoder->OnReceivedOverhead(kOverhead); + states.encoder->OnReceivedOverhead(kOverhead); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, UpdateUplinkBandwidthInAudioNetworkAdaptor) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + std::array audio; + audio.fill(0); + rtc::Buffer encoded; + EXPECT_CALL(*states.mock_bitrate_smoother, GetAverage()) + .WillOnce(Return(50000)); + EXPECT_CALL(**states.mock_audio_network_adaptor, SetUplinkBandwidth(50000)); + states.encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + + // Repeat update uplink bandwidth tests. + for (int i = 0; i < 5; i++) { + // Don't update till it is time to update again. + states.fake_clock->AdvanceTime(rtc::TimeDelta::FromMilliseconds( + states.config.uplink_bandwidth_update_interval_ms - 1)); + states.encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + + // Update when it is time to update. + EXPECT_CALL(*states.mock_bitrate_smoother, GetAverage()) + .WillOnce(Return(40000)); + EXPECT_CALL(**states.mock_audio_network_adaptor, SetUplinkBandwidth(40000)); + states.fake_clock->AdvanceTime(rtc::TimeDelta::FromMilliseconds(1)); + states.encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + } +} + +TEST(AudioEncoderOpusTest, EncodeAtMinBitrate) { + auto states = CreateCodec(1); + constexpr int kNumPacketsToEncode = 2; + auto audio_frames = + Create10msAudioBlocks(states.encoder, kNumPacketsToEncode * 20); + ASSERT_TRUE(audio_frames) << "Create10msAudioBlocks failed"; + rtc::Buffer encoded; + uint32_t rtp_timestamp = 12345; // Just a number not important to this test. + + states.encoder->OnReceivedUplinkBandwidth(0, rtc::nullopt); + for (int packet_index = 0; packet_index < kNumPacketsToEncode; + packet_index++) { + // Make sure we are not encoding before we have enough data for + // a 20ms packet. + for (int index = 0; index < 1; index++) { + states.encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(), + &encoded); + EXPECT_EQ(0u, encoded.size()); + } + + // Should encode now. + states.encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(), + &encoded); + EXPECT_GT(encoded.size(), 0u); + encoded.Clear(); + } +} + +TEST(AudioEncoderOpusTest, TestConfigDefaults) { + const auto config_opt = AudioEncoderOpus::SdpToConfig({"opus", 48000, 2}); + ASSERT_TRUE(config_opt); + EXPECT_EQ(48000, config_opt->max_playback_rate_hz); + EXPECT_EQ(1u, config_opt->num_channels); + EXPECT_FALSE(config_opt->fec_enabled); + EXPECT_FALSE(config_opt->dtx_enabled); + EXPECT_EQ(20, config_opt->frame_size_ms); +} + +TEST(AudioEncoderOpusTest, TestConfigFromParams) { + const auto config1 = CreateConfigWithParameters({{"stereo", "0"}}); + EXPECT_EQ(1U, config1.num_channels); + + const auto config2 = CreateConfigWithParameters({{"stereo", "1"}}); + EXPECT_EQ(2U, config2.num_channels); + + const auto config3 = CreateConfigWithParameters({{"useinbandfec", "0"}}); + EXPECT_FALSE(config3.fec_enabled); + + const auto config4 = CreateConfigWithParameters({{"useinbandfec", "1"}}); + EXPECT_TRUE(config4.fec_enabled); + + const auto config5 = CreateConfigWithParameters({{"usedtx", "0"}}); + EXPECT_FALSE(config5.dtx_enabled); + + const auto config6 = CreateConfigWithParameters({{"usedtx", "1"}}); + EXPECT_TRUE(config6.dtx_enabled); + + const auto config7 = CreateConfigWithParameters({{"cbr", "0"}}); + EXPECT_FALSE(config7.cbr_enabled); + + const auto config8 = CreateConfigWithParameters({{"cbr", "1"}}); + EXPECT_TRUE(config8.cbr_enabled); + + const auto config9 = + CreateConfigWithParameters({{"maxplaybackrate", "12345"}}); + EXPECT_EQ(12345, config9.max_playback_rate_hz); + + const auto config10 = + CreateConfigWithParameters({{"maxaveragebitrate", "96000"}}); + EXPECT_EQ(96000, config10.bitrate_bps); + + const auto config11 = CreateConfigWithParameters({{"maxptime", "40"}}); + for (int frame_length : config11.supported_frame_lengths_ms) { + EXPECT_LE(frame_length, 40); + } + + const auto config12 = CreateConfigWithParameters({{"minptime", "40"}}); + for (int frame_length : config12.supported_frame_lengths_ms) { + EXPECT_GE(frame_length, 40); + } + + const auto config13 = CreateConfigWithParameters({{"ptime", "40"}}); + EXPECT_EQ(40, config13.frame_size_ms); + + constexpr int kMinSupportedFrameLength = 10; + constexpr int kMaxSupportedFrameLength = + WEBRTC_OPUS_SUPPORT_120MS_PTIME ? 120 : 60; + + const auto config14 = CreateConfigWithParameters({{"ptime", "1"}}); + EXPECT_EQ(kMinSupportedFrameLength, config14.frame_size_ms); + + const auto config15 = CreateConfigWithParameters({{"ptime", "2000"}}); + EXPECT_EQ(kMaxSupportedFrameLength, config15.frame_size_ms); +} + +TEST(AudioEncoderOpusTest, TestConfigFromInvalidParams) { + const webrtc::SdpAudioFormat format("opus", 48000, 2); + const auto default_config = *AudioEncoderOpus::SdpToConfig(format); +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME + const std::vector default_supported_frame_lengths_ms({20, 60, 120}); +#else + const std::vector default_supported_frame_lengths_ms({20, 60}); +#endif + + AudioEncoderOpusConfig config; + config = CreateConfigWithParameters({{"stereo", "invalid"}}); + EXPECT_EQ(default_config.num_channels, config.num_channels); + + config = CreateConfigWithParameters({{"useinbandfec", "invalid"}}); + EXPECT_EQ(default_config.fec_enabled, config.fec_enabled); + + config = CreateConfigWithParameters({{"usedtx", "invalid"}}); + EXPECT_EQ(default_config.dtx_enabled, config.dtx_enabled); + + config = CreateConfigWithParameters({{"cbr", "invalid"}}); + EXPECT_EQ(default_config.dtx_enabled, config.dtx_enabled); + + config = CreateConfigWithParameters({{"maxplaybackrate", "0"}}); + EXPECT_EQ(default_config.max_playback_rate_hz, config.max_playback_rate_hz); + + config = CreateConfigWithParameters({{"maxplaybackrate", "-23"}}); + EXPECT_EQ(default_config.max_playback_rate_hz, config.max_playback_rate_hz); + + config = CreateConfigWithParameters({{"maxplaybackrate", "not a number!"}}); + EXPECT_EQ(default_config.max_playback_rate_hz, config.max_playback_rate_hz); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "0"}}); + EXPECT_EQ(6000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "-1000"}}); + EXPECT_EQ(6000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "1024000"}}); + EXPECT_EQ(510000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "not a number!"}}); + EXPECT_EQ(default_config.bitrate_bps, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxptime", "invalid"}}); + EXPECT_EQ(default_supported_frame_lengths_ms, + config.supported_frame_lengths_ms); + + config = CreateConfigWithParameters({{"minptime", "invalid"}}); + EXPECT_EQ(default_supported_frame_lengths_ms, + config.supported_frame_lengths_ms); + + config = CreateConfigWithParameters({{"ptime", "invalid"}}); + EXPECT_EQ(default_supported_frame_lengths_ms, + config.supported_frame_lengths_ms); +} + +// Test that bitrate will be overridden by the "maxaveragebitrate" parameter. +// Also test that the "maxaveragebitrate" can't be set to values outside the +// range of 6000 and 510000 +TEST(AudioEncoderOpusTest, SetSendCodecOpusMaxAverageBitrate) { + // Ignore if less than 6000. + const auto config1 = AudioEncoderOpus::SdpToConfig( + {"opus", 48000, 2, {{"maxaveragebitrate", "5999"}}}); + EXPECT_EQ(6000, config1->bitrate_bps); + + // Ignore if larger than 510000. + const auto config2 = AudioEncoderOpus::SdpToConfig( + {"opus", 48000, 2, {{"maxaveragebitrate", "510001"}}}); + EXPECT_EQ(510000, config2->bitrate_bps); + + const auto config3 = AudioEncoderOpus::SdpToConfig( + {"opus", 48000, 2, {{"maxaveragebitrate", "200000"}}}); + EXPECT_EQ(200000, config3->bitrate_bps); +} + +// Test maxplaybackrate <= 8000 triggers Opus narrow band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateNb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "8000"}}); + EXPECT_EQ(8000, config.max_playback_rate_hz); + EXPECT_EQ(12000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxplaybackrate", "8000"}, + {"stereo", "1"}}); + EXPECT_EQ(8000, config.max_playback_rate_hz); + EXPECT_EQ(24000, config.bitrate_bps); +} + +// Test 8000 < maxplaybackrate <= 12000 triggers Opus medium band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateMb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "8001"}}); + EXPECT_EQ(8001, config.max_playback_rate_hz); + EXPECT_EQ(20000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxplaybackrate", "8001"}, + {"stereo", "1"}}); + EXPECT_EQ(8001, config.max_playback_rate_hz); + EXPECT_EQ(40000, config.bitrate_bps); +} + +// Test 12000 < maxplaybackrate <= 16000 triggers Opus wide band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateWb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "12001"}}); + EXPECT_EQ(12001, config.max_playback_rate_hz); + EXPECT_EQ(20000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxplaybackrate", "12001"}, + {"stereo", "1"}}); + EXPECT_EQ(12001, config.max_playback_rate_hz); + EXPECT_EQ(40000, config.bitrate_bps); +} + +// Test 16000 < maxplaybackrate <= 24000 triggers Opus super wide band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateSwb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "16001"}}); + EXPECT_EQ(16001, config.max_playback_rate_hz); + EXPECT_EQ(32000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxplaybackrate", "16001"}, + {"stereo", "1"}}); + EXPECT_EQ(16001, config.max_playback_rate_hz); + EXPECT_EQ(64000, config.bitrate_bps); +} + +// Test 24000 < maxplaybackrate triggers Opus full band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateFb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "24001"}}); + EXPECT_EQ(24001, config.max_playback_rate_hz); + EXPECT_EQ(32000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxplaybackrate", "24001"}, + {"stereo", "1"}}); + EXPECT_EQ(24001, config.max_playback_rate_hz); + EXPECT_EQ(64000, config.bitrate_bps); +} + +TEST(AudioEncoderOpusTest, OpusFlagDtxAsNonSpeech) { + // Create encoder with DTX enabled. + AudioEncoderOpusConfig config; + config.dtx_enabled = true; + constexpr int payload_type = 17; + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(config, payload_type); + + // Open file containing speech and silence. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + test::AudioLoop audio_loop; + // Use the file as if it were sampled at 48 kHz. + constexpr int kSampleRateHz = 48000; + EXPECT_EQ(kSampleRateHz, encoder->SampleRateHz()); + constexpr size_t kMaxLoopLengthSamples = + kSampleRateHz * 10; // Max 10 second loop. + constexpr size_t kInputBlockSizeSamples = + 10 * kSampleRateHz / 1000; // 10 ms. + EXPECT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)); + + // Encode. + AudioEncoder::EncodedInfo info; + rtc::Buffer encoded(500); + int nonspeech_frames = 0; + int max_nonspeech_frames = 0; + int dtx_frames = 0; + int max_dtx_frames = 0; + uint32_t rtp_timestamp = 0u; + for (size_t i = 0; i < 500; ++i) { + encoded.Clear(); + + // Every second call to the encoder will generate an Opus packet. + for (int j = 0; j < 2; j++) { + info = + encoder->Encode(rtp_timestamp, audio_loop.GetNextBlock(), &encoded); + rtp_timestamp += kInputBlockSizeSamples; + } + + // Bookkeeping of number of DTX frames. + if (info.encoded_bytes <= 2) { + ++dtx_frames; + } else { + if (dtx_frames > max_dtx_frames) + max_dtx_frames = dtx_frames; + dtx_frames = 0; + } + + // Bookkeeping of number of non-speech frames. + if (info.speech == 0) { + ++nonspeech_frames; + } else { + if (nonspeech_frames > max_nonspeech_frames) + max_nonspeech_frames = nonspeech_frames; + nonspeech_frames = 0; + } + } + + // Maximum number of consecutive non-speech packets should exceed 20. + EXPECT_GT(max_nonspeech_frames, 20); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc new file mode 100644 index 0000000000..4394949f96 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/opus/audio_decoder_opus.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "common_audio/include/audio_util.h" +#include "common_audio/lapped_transform.h" +#include "common_audio/window_generator.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace { + +constexpr size_t kNumChannels = 1u; +constexpr int kSampleRateHz = 48000; +constexpr size_t kMaxLoopLengthSamples = kSampleRateHz * 50; // 50 seconds. +constexpr size_t kInputBlockSizeSamples = 10 * kSampleRateHz / 1000; // 10 ms +constexpr size_t kOutputBlockSizeSamples = 20 * kSampleRateHz / 1000; // 20 ms +constexpr size_t kFftSize = 1024; +constexpr size_t kNarrowbandSize = 4000 * kFftSize / kSampleRateHz; +constexpr float kKbdAlpha = 1.5f; + +class PowerRatioEstimator : public LappedTransform::Callback { + public: + PowerRatioEstimator() : low_pow_(0.f), high_pow_(0.f) { + WindowGenerator::KaiserBesselDerived(kKbdAlpha, kFftSize, window_); + transform_.reset(new LappedTransform(kNumChannels, 0u, + kInputBlockSizeSamples, window_, + kFftSize, kFftSize / 2, this)); + } + + void ProcessBlock(float* data) { transform_->ProcessChunk(&data, nullptr); } + + float PowerRatio() { return high_pow_ / low_pow_; } + + protected: + void ProcessAudioBlock(const std::complex* const* input, + size_t num_input_channels, + size_t num_freq_bins, + size_t num_output_channels, + std::complex* const* output) override { + float low_pow = 0.f; + float high_pow = 0.f; + for (size_t i = 0u; i < num_input_channels; ++i) { + for (size_t j = 0u; j < kNarrowbandSize; ++j) { + float low_mag = std::abs(input[i][j]); + low_pow += low_mag * low_mag; + float high_mag = std::abs(input[i][j + kNarrowbandSize]); + high_pow += high_mag * high_mag; + } + } + low_pow_ += low_pow / (num_input_channels * kFftSize); + high_pow_ += high_pow / (num_input_channels * kFftSize); + } + + private: + std::unique_ptr transform_; + float window_[kFftSize]; + float low_pow_; + float high_pow_; +}; + +float EncodedPowerRatio(AudioEncoder* encoder, + AudioDecoder* decoder, + test::AudioLoop* audio_loop) { + // Encode and decode. + uint32_t rtp_timestamp = 0u; + constexpr size_t kBufferSize = 500; + rtc::Buffer encoded(kBufferSize); + std::vector decoded(kOutputBlockSizeSamples); + std::vector decoded_float(kOutputBlockSizeSamples); + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + PowerRatioEstimator power_ratio_estimator; + for (size_t i = 0; i < 1000; ++i) { + encoded.Clear(); + AudioEncoder::EncodedInfo encoder_info = + encoder->Encode(rtp_timestamp, audio_loop->GetNextBlock(), &encoded); + rtp_timestamp += kInputBlockSizeSamples; + if (encoded.size() > 0) { + int decoder_info = decoder->Decode( + encoded.data(), encoded.size(), kSampleRateHz, + decoded.size() * sizeof(decoded[0]), decoded.data(), &speech_type); + if (decoder_info > 0) { + S16ToFloat(decoded.data(), decoded.size(), decoded_float.data()); + power_ratio_estimator.ProcessBlock(decoded_float.data()); + } + } + } + return power_ratio_estimator.PowerRatio(); +} + +} // namespace + +TEST(BandwidthAdaptationTest, BandwidthAdaptationTest) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-AdjustOpusBandwidth/Enabled/"); + + constexpr float kMaxNarrowbandRatio = 0.003f; + constexpr float kMinWidebandRatio = 0.03f; + + // Create encoder. + AudioEncoderOpusConfig enc_config; + enc_config.bitrate_bps = rtc::Optional(7999); + enc_config.num_channels = kNumChannels; + constexpr int payload_type = 17; + auto encoder = AudioEncoderOpus::MakeAudioEncoder(enc_config, payload_type); + + // Create decoder. + AudioDecoderOpus::Config dec_config; + dec_config.num_channels = kNumChannels; + auto decoder = AudioDecoderOpus::MakeAudioDecoder(dec_config); + + // Open speech file. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/speech_mono_32_48kHz", "pcm"); + test::AudioLoop audio_loop; + EXPECT_EQ(kSampleRateHz, encoder->SampleRateHz()); + ASSERT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)); + + EXPECT_LT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMaxNarrowbandRatio); + + encoder->OnReceivedTargetAudioBitrate(9000); + EXPECT_LT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMaxNarrowbandRatio); + + encoder->OnReceivedTargetAudioBitrate(9001); + EXPECT_GT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMinWidebandRatio); + + encoder->OnReceivedTargetAudioBitrate(8000); + EXPECT_GT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMinWidebandRatio); + + encoder->OnReceivedTargetAudioBitrate(12001); + EXPECT_GT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMinWidebandRatio); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc new file mode 100644 index 0000000000..4c9174d92c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/timeutils.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { + +namespace { +int64_t RunComplexityTest(const AudioEncoderOpusConfig& config) { + // Create encoder. + constexpr int payload_type = 17; + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(config, payload_type); + // Open speech file. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/speech_mono_32_48kHz", "pcm"); + test::AudioLoop audio_loop; + constexpr int kSampleRateHz = 48000; + EXPECT_EQ(kSampleRateHz, encoder->SampleRateHz()); + constexpr size_t kMaxLoopLengthSamples = + kSampleRateHz * 10; // 10 second loop. + constexpr size_t kInputBlockSizeSamples = + 10 * kSampleRateHz / 1000; // 60 ms. + EXPECT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)); + // Encode. + const int64_t start_time_ms = rtc::TimeMillis(); + AudioEncoder::EncodedInfo info; + rtc::Buffer encoded(500); + uint32_t rtp_timestamp = 0u; + for (size_t i = 0; i < 10000; ++i) { + encoded.Clear(); + info = encoder->Encode(rtp_timestamp, audio_loop.GetNextBlock(), &encoded); + rtp_timestamp += kInputBlockSizeSamples; + } + return rtc::TimeMillis() - start_time_ms; +} +} // namespace + +// This test encodes an audio file using Opus twice with different bitrates +// (~11 kbps and 15.5 kbps). The runtime for each is measured, and the ratio +// between the two is calculated and tracked. This test explicitly sets the +// low_rate_complexity to 9. When running on desktop platforms, this is the same +// as the regular complexity, and the expectation is that the resulting ratio +// should be less than 100% (since the encoder runs faster at lower bitrates, +// given a fixed complexity setting). On the other hand, when running on +// mobiles, the regular complexity is 5, and we expect the resulting ratio to +// be higher, since we have explicitly asked for a higher complexity setting at +// the lower rate. +TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOn) { + // Create config. + AudioEncoderOpusConfig config; + // The limit -- including the hysteresis window -- at which the complexity + // shuold be increased. + config.bitrate_bps = 11000 - 1; + config.low_rate_complexity = 9; + int64_t runtime_10999bps = RunComplexityTest(config); + + config.bitrate_bps = 15500; + int64_t runtime_15500bps = RunComplexityTest(config); + + test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_on", + 100.0 * runtime_10999bps / runtime_15500bps, "percent", + true); +} + +// This test is identical to the one above, but without the complexity +// adaptation enabled (neither on desktop, nor on mobile). The expectation is +// that the resulting ratio is less than 100% at all times. +TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOff) { + // Create config. + AudioEncoderOpusConfig config; + // The limit -- including the hysteresis window -- at which the complexity + // shuold be increased (but not in this test since complexity adaptation is + // disabled). + config.bitrate_bps = 11000 - 1; + int64_t runtime_10999bps = RunComplexityTest(config); + + config.bitrate_bps = 15500; + int64_t runtime_15500bps = RunComplexityTest(config); + + test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_off", + 100.0 * runtime_10999bps / runtime_15500bps, "percent", + true); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc new file mode 100644 index 0000000000..4e0a17ecc1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "rtc_base/format_macros.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +using std::string; +using std::tuple; +using std::get; +using ::testing::TestWithParam; + +namespace webrtc { + +// Define coding parameter as . +typedef tuple coding_param; +typedef struct mode mode; + +struct mode { + bool fec; + uint8_t target_packet_loss_rate; +}; + +const int kOpusBlockDurationMs = 20; +const int kOpusSamplingKhz = 48; + +class OpusFecTest : public TestWithParam { + protected: + OpusFecTest(); + + virtual void SetUp(); + virtual void TearDown(); + + virtual void EncodeABlock(); + + virtual void DecodeABlock(bool lost_previous, bool lost_current); + + int block_duration_ms_; + int sampling_khz_; + size_t block_length_sample_; + + size_t channels_; + int bit_rate_; + + size_t data_pointer_; + size_t loop_length_samples_; + size_t max_bytes_; + size_t encoded_bytes_; + + WebRtcOpusEncInst* opus_encoder_; + WebRtcOpusDecInst* opus_decoder_; + + string in_filename_; + + std::unique_ptr in_data_; + std::unique_ptr out_data_; + std::unique_ptr bit_stream_; +}; + +void OpusFecTest::SetUp() { + channels_ = get<0>(GetParam()); + bit_rate_ = get<1>(GetParam()); + printf("Coding %" PRIuS " channel signal at %d bps.\n", channels_, bit_rate_); + + in_filename_ = test::ResourcePath(get<2>(GetParam()), get<3>(GetParam())); + + FILE* fp = fopen(in_filename_.c_str(), "rb"); + ASSERT_FALSE(fp == NULL); + + // Obtain file size. + fseek(fp, 0, SEEK_END); + loop_length_samples_ = ftell(fp) / sizeof(int16_t); + rewind(fp); + + // Allocate memory to contain the whole file. + in_data_.reset(new int16_t[loop_length_samples_ + + block_length_sample_ * channels_]); + + // Copy the file into the buffer. + ASSERT_EQ(fread(&in_data_[0], sizeof(int16_t), loop_length_samples_, fp), + loop_length_samples_); + fclose(fp); + + // The audio will be used in a looped manner. To ease the acquisition of an + // audio frame that crosses the end of the excerpt, we add an extra block + // length of samples to the end of the array, starting over again from the + // beginning of the array. Audio frames cross the end of the excerpt always + // appear as a continuum of memory. + memcpy(&in_data_[loop_length_samples_], &in_data_[0], + block_length_sample_ * channels_ * sizeof(int16_t)); + + // Maximum number of bytes in output bitstream. + max_bytes_ = block_length_sample_ * channels_ * sizeof(int16_t); + + out_data_.reset(new int16_t[2 * block_length_sample_ * channels_]); + bit_stream_.reset(new uint8_t[max_bytes_]); + + // If channels_ == 1, use Opus VOIP mode, otherwise, audio mode. + int app = channels_ == 1 ? 0 : 1; + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, app)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, bit_rate_)); +} + +void OpusFecTest::TearDown() { + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +OpusFecTest::OpusFecTest() + : block_duration_ms_(kOpusBlockDurationMs), + sampling_khz_(kOpusSamplingKhz), + block_length_sample_( + static_cast(block_duration_ms_ * sampling_khz_)), + data_pointer_(0), + max_bytes_(0), + encoded_bytes_(0), + opus_encoder_(NULL), + opus_decoder_(NULL) { +} + +void OpusFecTest::EncodeABlock() { + int value = WebRtcOpus_Encode(opus_encoder_, + &in_data_[data_pointer_], + block_length_sample_, + max_bytes_, &bit_stream_[0]); + EXPECT_GT(value, 0); + + encoded_bytes_ = static_cast(value); +} + +void OpusFecTest::DecodeABlock(bool lost_previous, bool lost_current) { + int16_t audio_type; + int value_1 = 0, value_2 = 0; + + if (lost_previous) { + // Decode previous frame. + if (!lost_current && + WebRtcOpus_PacketHasFec(&bit_stream_[0], encoded_bytes_) == 1) { + value_1 = WebRtcOpus_DecodeFec(opus_decoder_, &bit_stream_[0], + encoded_bytes_, &out_data_[0], + &audio_type); + } else { + value_1 = WebRtcOpus_DecodePlc(opus_decoder_, &out_data_[0], 1); + } + EXPECT_EQ(static_cast(block_length_sample_), value_1); + } + + if (!lost_current) { + // Decode current frame. + value_2 = WebRtcOpus_Decode(opus_decoder_, &bit_stream_[0], encoded_bytes_, + &out_data_[value_1 * channels_], &audio_type); + EXPECT_EQ(static_cast(block_length_sample_), value_2); + } +} + +TEST_P(OpusFecTest, RandomPacketLossTest) { + const int kDurationMs = 200000; + int time_now_ms, fec_frames; + int actual_packet_loss_rate; + bool lost_current, lost_previous; + mode mode_set[3] = {{true, 0}, + {false, 0}, + {true, 50}}; + + lost_current = false; + for (int i = 0; i < 3; i++) { + if (mode_set[i].fec) { + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_encoder_, + mode_set[i].target_packet_loss_rate)); + printf("FEC is ON, target at packet loss rate %d percent.\n", + mode_set[i].target_packet_loss_rate); + } else { + EXPECT_EQ(0, WebRtcOpus_DisableFec(opus_encoder_)); + printf("FEC is OFF.\n"); + } + // In this test, we let the target packet loss rate match the actual rate. + actual_packet_loss_rate = mode_set[i].target_packet_loss_rate; + // Run every mode a certain time. + time_now_ms = 0; + fec_frames = 0; + while (time_now_ms < kDurationMs) { + // Encode & decode. + EncodeABlock(); + + // Check if payload has FEC. + int fec = WebRtcOpus_PacketHasFec(&bit_stream_[0], encoded_bytes_); + + // If FEC is disabled or the target packet loss rate is set to 0, there + // should be no FEC in the bit stream. + if (!mode_set[i].fec || mode_set[i].target_packet_loss_rate == 0) { + EXPECT_EQ(fec, 0); + } else if (fec == 1) { + fec_frames++; + } + + lost_previous = lost_current; + lost_current = rand() < actual_packet_loss_rate * (RAND_MAX / 100); + DecodeABlock(lost_previous, lost_current); + + time_now_ms += block_duration_ms_; + + // |data_pointer_| is incremented and wrapped across + // |loop_length_samples_|. + data_pointer_ = (data_pointer_ + block_length_sample_ * channels_) % + loop_length_samples_; + } + if (mode_set[i].fec) { + printf("%.2f percent frames has FEC.\n", + static_cast(fec_frames) * block_duration_ms_ / 2000); + } + } +} + +const coding_param param_set[] = { + std::make_tuple(1, + 64000, + string("audio_coding/testfile32kHz"), + string("pcm")), + std::make_tuple(1, + 32000, + string("audio_coding/testfile32kHz"), + string("pcm")), + std::make_tuple(2, + 64000, + string("audio_coding/teststereo32kHz"), + string("pcm"))}; + +// 64 kbps, stereo +INSTANTIATE_TEST_CASE_P(AllTest, OpusFecTest, + ::testing::ValuesIn(param_set)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_inst.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_inst.h new file mode 100644 index 0000000000..066fa2267f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_inst.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INST_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INST_H_ + +#include + +#include "rtc_base/ignore_wundef.h" + +RTC_PUSH_IGNORING_WUNDEF() +#include "opus.h" +RTC_POP_IGNORING_WUNDEF() + +struct WebRtcOpusEncInst { + OpusEncoder* encoder; + size_t channels; + int in_dtx_mode; +}; + +struct WebRtcOpusDecInst { + OpusDecoder* decoder; + int prev_decoded_samples; + size_t channels; + int in_dtx_mode; +}; + + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.c new file mode 100644 index 0000000000..d219098b01 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.c @@ -0,0 +1,523 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/opus_interface.h" + +#include "rtc_base/checks.h" + +#include +#include + +enum { +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME + /* Maximum supported frame size in WebRTC is 120 ms. */ + kWebRtcOpusMaxEncodeFrameSizeMs = 120, +#else + /* Maximum supported frame size in WebRTC is 60 ms. */ + kWebRtcOpusMaxEncodeFrameSizeMs = 60, +#endif + + /* The format allows up to 120 ms frames. Since we don't control the other + * side, we must allow for packets of that size. NetEq is currently limited + * to 60 ms on the receive side. */ + kWebRtcOpusMaxDecodeFrameSizeMs = 120, + + /* Maximum sample count per channel is 48 kHz * maximum frame size in + * milliseconds. */ + kWebRtcOpusMaxFrameSizePerChannel = 48 * kWebRtcOpusMaxDecodeFrameSizeMs, + + /* Default frame size, 20 ms @ 48 kHz, in samples (for one channel). */ + kWebRtcOpusDefaultFrameSize = 960, +}; + +int16_t WebRtcOpus_EncoderCreate(OpusEncInst** inst, + size_t channels, + int32_t application) { + int opus_app; + if (!inst) + return -1; + + switch (application) { + case 0: + opus_app = OPUS_APPLICATION_VOIP; + break; + case 1: + opus_app = OPUS_APPLICATION_AUDIO; + break; + default: + return -1; + } + + OpusEncInst* state = calloc(1, sizeof(OpusEncInst)); + RTC_DCHECK(state); + + int error; + state->encoder = opus_encoder_create(48000, (int)channels, opus_app, + &error); + if (error != OPUS_OK || !state->encoder) { + WebRtcOpus_EncoderFree(state); + return -1; + } + + state->in_dtx_mode = 0; + state->channels = channels; + + *inst = state; + return 0; +} + +int16_t WebRtcOpus_EncoderFree(OpusEncInst* inst) { + if (inst) { + opus_encoder_destroy(inst->encoder); + free(inst); + return 0; + } else { + return -1; + } +} + +int WebRtcOpus_Encode(OpusEncInst* inst, + const int16_t* audio_in, + size_t samples, + size_t length_encoded_buffer, + uint8_t* encoded) { + int res; + + if (samples > 48 * kWebRtcOpusMaxEncodeFrameSizeMs) { + return -1; + } + + res = opus_encode(inst->encoder, + (const opus_int16*)audio_in, + (int)samples, + encoded, + (opus_int32)length_encoded_buffer); + + if (res <= 0) { + return -1; + } + + if (res <= 2) { + // Indicates DTX since the packet has nothing but a header. In principle, + // there is no need to send this packet. However, we do transmit the first + // occurrence to let the decoder know that the encoder enters DTX mode. + if (inst->in_dtx_mode) { + return 0; + } else { + inst->in_dtx_mode = 1; + return res; + } + } + + inst->in_dtx_mode = 0; + return res; +} + +int16_t WebRtcOpus_SetBitRate(OpusEncInst* inst, int32_t rate) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_BITRATE(rate)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetPacketLossRate(OpusEncInst* inst, int32_t loss_rate) { + if (inst) { + return opus_encoder_ctl(inst->encoder, + OPUS_SET_PACKET_LOSS_PERC(loss_rate)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetMaxPlaybackRate(OpusEncInst* inst, int32_t frequency_hz) { + opus_int32 set_bandwidth; + + if (!inst) + return -1; + + if (frequency_hz <= 8000) { + set_bandwidth = OPUS_BANDWIDTH_NARROWBAND; + } else if (frequency_hz <= 12000) { + set_bandwidth = OPUS_BANDWIDTH_MEDIUMBAND; + } else if (frequency_hz <= 16000) { + set_bandwidth = OPUS_BANDWIDTH_WIDEBAND; + } else if (frequency_hz <= 24000) { + set_bandwidth = OPUS_BANDWIDTH_SUPERWIDEBAND; + } else { + set_bandwidth = OPUS_BANDWIDTH_FULLBAND; + } + return opus_encoder_ctl(inst->encoder, + OPUS_SET_MAX_BANDWIDTH(set_bandwidth)); +} + +int16_t WebRtcOpus_EnableFec(OpusEncInst* inst) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_INBAND_FEC(1)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_DisableFec(OpusEncInst* inst) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_INBAND_FEC(0)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_EnableDtx(OpusEncInst* inst) { + if (!inst) { + return -1; + } + + // To prevent Opus from entering CELT-only mode by forcing signal type to + // voice to make sure that DTX behaves correctly. Currently, DTX does not + // last long during a pure silence, if the signal type is not forced. + // TODO(minyue): Remove the signal type forcing when Opus DTX works properly + // without it. + int ret = opus_encoder_ctl(inst->encoder, + OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + if (ret != OPUS_OK) + return ret; + + return opus_encoder_ctl(inst->encoder, OPUS_SET_DTX(1)); +} + +int16_t WebRtcOpus_DisableDtx(OpusEncInst* inst) { + if (inst) { + int ret = opus_encoder_ctl(inst->encoder, + OPUS_SET_SIGNAL(OPUS_AUTO)); + if (ret != OPUS_OK) + return ret; + return opus_encoder_ctl(inst->encoder, OPUS_SET_DTX(0)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_EnableCbr(OpusEncInst* inst) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_VBR(0)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_DisableCbr(OpusEncInst* inst) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_VBR(1)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetComplexity(OpusEncInst* inst, int32_t complexity) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_COMPLEXITY(complexity)); + } else { + return -1; + } +} + +int32_t WebRtcOpus_GetBandwidth(OpusEncInst* inst) { + if (!inst) { + return -1; + } + int32_t bandwidth; + if (opus_encoder_ctl(inst->encoder, OPUS_GET_BANDWIDTH(&bandwidth)) == 0) { + return bandwidth; + } else { + return -1; + } + +} + +int16_t WebRtcOpus_SetBandwidth(OpusEncInst* inst, int32_t bandwidth) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_BANDWIDTH(bandwidth)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, size_t num_channels) { + if (!inst) + return -1; + if (num_channels == 0) { + return opus_encoder_ctl(inst->encoder, + OPUS_SET_FORCE_CHANNELS(OPUS_AUTO)); + } else if (num_channels == 1 || num_channels == 2) { + return opus_encoder_ctl(inst->encoder, + OPUS_SET_FORCE_CHANNELS(num_channels)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_DecoderCreate(OpusDecInst** inst, size_t channels) { + int error; + OpusDecInst* state; + + if (inst != NULL) { + /* Create Opus decoder state. */ + state = (OpusDecInst*) calloc(1, sizeof(OpusDecInst)); + if (state == NULL) { + return -1; + } + + /* Create new memory, always at 48000 Hz. */ + state->decoder = opus_decoder_create(48000, (int)channels, &error); + if (error == OPUS_OK && state->decoder != NULL) { + /* Creation of memory all ok. */ + state->channels = channels; + state->prev_decoded_samples = kWebRtcOpusDefaultFrameSize; + state->in_dtx_mode = 0; + *inst = state; + return 0; + } + + /* If memory allocation was unsuccessful, free the entire state. */ + if (state->decoder) { + opus_decoder_destroy(state->decoder); + } + free(state); + } + return -1; +} + +int16_t WebRtcOpus_DecoderFree(OpusDecInst* inst) { + if (inst) { + opus_decoder_destroy(inst->decoder); + free(inst); + return 0; + } else { + return -1; + } +} + +size_t WebRtcOpus_DecoderChannels(OpusDecInst* inst) { + return inst->channels; +} + +void WebRtcOpus_DecoderInit(OpusDecInst* inst) { + opus_decoder_ctl(inst->decoder, OPUS_RESET_STATE); + inst->in_dtx_mode = 0; +} + +/* For decoder to determine if it is to output speech or comfort noise. */ +static int16_t DetermineAudioType(OpusDecInst* inst, size_t encoded_bytes) { + // Audio type becomes comfort noise if |encoded_byte| is 1 and keeps + // to be so if the following |encoded_byte| are 0 or 1. + if (encoded_bytes == 0 && inst->in_dtx_mode) { + return 2; // Comfort noise. + } else if (encoded_bytes == 1 || encoded_bytes == 2) { + // TODO(henrik.lundin): There is a slight risk that a 2-byte payload is in + // fact a 1-byte TOC with a 1-byte payload. That will be erroneously + // interpreted as comfort noise output, but such a payload is probably + // faulty anyway. + inst->in_dtx_mode = 1; + return 2; // Comfort noise. + } else { + inst->in_dtx_mode = 0; + return 0; // Speech. + } +} + +/* |frame_size| is set to maximum Opus frame size in the normal case, and + * is set to the number of samples needed for PLC in case of losses. + * It is up to the caller to make sure the value is correct. */ +static int DecodeNative(OpusDecInst* inst, const uint8_t* encoded, + size_t encoded_bytes, int frame_size, + int16_t* decoded, int16_t* audio_type, int decode_fec) { + int res = opus_decode(inst->decoder, encoded, (opus_int32)encoded_bytes, + (opus_int16*)decoded, frame_size, decode_fec); + + if (res <= 0) + return -1; + + *audio_type = DetermineAudioType(inst, encoded_bytes); + + return res; +} + +int WebRtcOpus_Decode(OpusDecInst* inst, const uint8_t* encoded, + size_t encoded_bytes, int16_t* decoded, + int16_t* audio_type) { + int decoded_samples; + + if (encoded_bytes == 0) { + *audio_type = DetermineAudioType(inst, encoded_bytes); + decoded_samples = WebRtcOpus_DecodePlc(inst, decoded, 1); + } else { + decoded_samples = DecodeNative(inst, + encoded, + encoded_bytes, + kWebRtcOpusMaxFrameSizePerChannel, + decoded, + audio_type, + 0); + } + if (decoded_samples < 0) { + return -1; + } + + /* Update decoded sample memory, to be used by the PLC in case of losses. */ + inst->prev_decoded_samples = decoded_samples; + + return decoded_samples; +} + +int WebRtcOpus_DecodePlc(OpusDecInst* inst, int16_t* decoded, + int number_of_lost_frames) { + int16_t audio_type = 0; + int decoded_samples; + int plc_samples; + + /* The number of samples we ask for is |number_of_lost_frames| times + * |prev_decoded_samples_|. Limit the number of samples to maximum + * |kWebRtcOpusMaxFrameSizePerChannel|. */ + plc_samples = number_of_lost_frames * inst->prev_decoded_samples; + plc_samples = (plc_samples <= kWebRtcOpusMaxFrameSizePerChannel) ? + plc_samples : kWebRtcOpusMaxFrameSizePerChannel; + decoded_samples = DecodeNative(inst, NULL, 0, plc_samples, + decoded, &audio_type, 0); + if (decoded_samples < 0) { + return -1; + } + + return decoded_samples; +} + +int WebRtcOpus_DecodeFec(OpusDecInst* inst, const uint8_t* encoded, + size_t encoded_bytes, int16_t* decoded, + int16_t* audio_type) { + int decoded_samples; + int fec_samples; + + if (WebRtcOpus_PacketHasFec(encoded, encoded_bytes) != 1) { + return 0; + } + + fec_samples = opus_packet_get_samples_per_frame(encoded, 48000); + + decoded_samples = DecodeNative(inst, encoded, encoded_bytes, + fec_samples, decoded, audio_type, 1); + if (decoded_samples < 0) { + return -1; + } + + return decoded_samples; +} + +int WebRtcOpus_DurationEst(OpusDecInst* inst, + const uint8_t* payload, + size_t payload_length_bytes) { + if (payload_length_bytes == 0) { + // WebRtcOpus_Decode calls PLC when payload length is zero. So we return + // PLC duration correspondingly. + return WebRtcOpus_PlcDuration(inst); + } + + int frames, samples; + frames = opus_packet_get_nb_frames(payload, (opus_int32)payload_length_bytes); + if (frames < 0) { + /* Invalid payload data. */ + return 0; + } + samples = frames * opus_packet_get_samples_per_frame(payload, 48000); + if (samples < 120 || samples > 5760) { + /* Invalid payload duration. */ + return 0; + } + return samples; +} + +int WebRtcOpus_PlcDuration(OpusDecInst* inst) { + /* The number of samples we ask for is |number_of_lost_frames| times + * |prev_decoded_samples_|. Limit the number of samples to maximum + * |kWebRtcOpusMaxFrameSizePerChannel|. */ + const int plc_samples = inst->prev_decoded_samples; + return (plc_samples <= kWebRtcOpusMaxFrameSizePerChannel) ? + plc_samples : kWebRtcOpusMaxFrameSizePerChannel; +} + +int WebRtcOpus_FecDurationEst(const uint8_t* payload, + size_t payload_length_bytes) { + int samples; + if (WebRtcOpus_PacketHasFec(payload, payload_length_bytes) != 1) { + return 0; + } + + samples = opus_packet_get_samples_per_frame(payload, 48000); + if (samples < 480 || samples > 5760) { + /* Invalid payload duration. */ + return 0; + } + return samples; +} + +int WebRtcOpus_PacketHasFec(const uint8_t* payload, + size_t payload_length_bytes) { + int frames, channels, payload_length_ms; + int n; + opus_int16 frame_sizes[48]; + const unsigned char *frame_data[48]; + + if (payload == NULL || payload_length_bytes == 0) + return 0; + + /* In CELT_ONLY mode, packets should not have FEC. */ + if (payload[0] & 0x80) + return 0; + + payload_length_ms = opus_packet_get_samples_per_frame(payload, 48000) / 48; + if (10 > payload_length_ms) + payload_length_ms = 10; + + channels = opus_packet_get_nb_channels(payload); + + switch (payload_length_ms) { + case 10: + case 20: { + frames = 1; + break; + } + case 40: { + frames = 2; + break; + } + case 60: { + frames = 3; + break; + } + default: { + return 0; // It is actually even an invalid packet. + } + } + + /* The following is to parse the LBRR flags. */ + if (opus_packet_parse(payload, (opus_int32)payload_length_bytes, NULL, + frame_data, frame_sizes, NULL) < 0) { + return 0; + } + + if (frame_sizes[0] <= 1) { + return 0; + } + + for (n = 0; n < channels; n++) { + if (frame_data[0][0] & (0x80 >> ((n + 1) * (frames + 1) - 1))) + return 1; + } + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.h new file mode 100644 index 0000000000..4b8e892344 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.h @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INTERFACE_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INTERFACE_H_ + +#include + +#include "modules/audio_coding/codecs/opus/opus_inst.h" +#include "typedefs.h" // NOLINT(build/include) + +#ifdef __cplusplus +extern "C" { +#endif + +// Opaque wrapper types for the codec state. +typedef struct WebRtcOpusEncInst OpusEncInst; +typedef struct WebRtcOpusDecInst OpusDecInst; + +/**************************************************************************** + * WebRtcOpus_EncoderCreate(...) + * + * This function create an Opus encoder. + * + * Input: + * - channels : number of channels. + * - application : 0 - VOIP applications. + * Favor speech intelligibility. + * 1 - Audio applications. + * Favor faithfulness to the original input. + * + * Output: + * - inst : a pointer to Encoder context that is created + * if success. + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EncoderCreate(OpusEncInst** inst, + size_t channels, + int32_t application); + +int16_t WebRtcOpus_EncoderFree(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_Encode(...) + * + * This function encodes audio as a series of Opus frames and inserts + * it into a packet. Input buffer can be any length. + * + * Input: + * - inst : Encoder context + * - audio_in : Input speech data buffer + * - samples : Samples per channel in audio_in + * - length_encoded_buffer : Output buffer size + * + * Output: + * - encoded : Output compressed data buffer + * + * Return value : >=0 - Length (in bytes) of coded data + * -1 - Error + */ +int WebRtcOpus_Encode(OpusEncInst* inst, + const int16_t* audio_in, + size_t samples, + size_t length_encoded_buffer, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcOpus_SetBitRate(...) + * + * This function adjusts the target bitrate of the encoder. + * + * Input: + * - inst : Encoder context + * - rate : New target bitrate + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetBitRate(OpusEncInst* inst, int32_t rate); + +/**************************************************************************** + * WebRtcOpus_SetPacketLossRate(...) + * + * This function configures the encoder's expected packet loss percentage. + * + * Input: + * - inst : Encoder context + * - loss_rate : loss percentage in the range 0-100, inclusive. + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetPacketLossRate(OpusEncInst* inst, int32_t loss_rate); + +/**************************************************************************** + * WebRtcOpus_SetMaxPlaybackRate(...) + * + * Configures the maximum playback rate for encoding. Due to hardware + * limitations, the receiver may render audio up to a playback rate. Opus + * encoder can use this information to optimize for network usage and encoding + * complexity. This will affect the audio bandwidth in the coded audio. However, + * the input/output sample rate is not affected. + * + * Input: + * - inst : Encoder context + * - frequency_hz : Maximum playback rate in Hz. + * This parameter can take any value. The relation + * between the value and the Opus internal mode is + * as following: + * frequency_hz <= 8000 narrow band + * 8000 < frequency_hz <= 12000 medium band + * 12000 < frequency_hz <= 16000 wide band + * 16000 < frequency_hz <= 24000 super wide band + * frequency_hz > 24000 full band + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetMaxPlaybackRate(OpusEncInst* inst, int32_t frequency_hz); + +/* TODO(minyue): Check whether an API to check the FEC and the packet loss rate + * is needed. It might not be very useful since there are not many use cases and + * the caller can always maintain the states. */ + +/**************************************************************************** + * WebRtcOpus_EnableFec() + * + * This function enables FEC for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EnableFec(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_DisableFec() + * + * This function disables FEC for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_DisableFec(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_EnableDtx() + * + * This function enables Opus internal DTX for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EnableDtx(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_DisableDtx() + * + * This function disables Opus internal DTX for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_DisableDtx(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_EnableCbr() + * + * This function enables CBR for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EnableCbr(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_DisableCbr() + * + * This function disables CBR for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_DisableCbr(OpusEncInst* inst); + +/* + * WebRtcOpus_SetComplexity(...) + * + * This function adjusts the computational complexity. The effect is the same as + * calling the complexity setting of Opus as an Opus encoder related CTL. + * + * Input: + * - inst : Encoder context + * - complexity : New target complexity (0-10, inclusive) + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetComplexity(OpusEncInst* inst, int32_t complexity); + +/* + * WebRtcOpus_GetBandwidth(...) + * + * This function returns the current bandwidth. + * + * Input: + * - inst : Encoder context + * + * Return value : Bandwidth - Success + * -1 - Error + */ +int32_t WebRtcOpus_GetBandwidth(OpusEncInst* inst); + +/* + * WebRtcOpus_SetBandwidth(...) + * + * By default Opus decides which bandwidth to encode the signal in depending on + * the the bitrate. This function overrules the previous setting and forces the + * encoder to encode in narrowband/wideband/fullband/etc. + * + * Input: + * - inst : Encoder context + * - bandwidth : New target bandwidth. Valid values are: + * OPUS_BANDWIDTH_NARROWBAND + * OPUS_BANDWIDTH_MEDIUMBAND + * OPUS_BANDWIDTH_WIDEBAND + * OPUS_BANDWIDTH_SUPERWIDEBAND + * OPUS_BANDWIDTH_FULLBAND + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetBandwidth(OpusEncInst* inst, int32_t bandwidth); + +/* + * WebRtcOpus_SetForceChannels(...) + * + * If the encoder is initialized as a stereo encoder, Opus will by default + * decide whether to encode in mono or stereo based on the bitrate. This + * function overrules the previous setting, and forces the encoder to encode + * in auto/mono/stereo. + * + * If the Encoder is initialized as a mono encoder, and one tries to force + * stereo, the function will return an error. + * + * Input: + * - inst : Encoder context + * - num_channels : 0 - Not forced + * 1 - Mono + * 2 - Stereo + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, size_t num_channels); + +int16_t WebRtcOpus_DecoderCreate(OpusDecInst** inst, size_t channels); +int16_t WebRtcOpus_DecoderFree(OpusDecInst* inst); + +/**************************************************************************** + * WebRtcOpus_DecoderChannels(...) + * + * This function returns the number of channels created for Opus decoder. + */ +size_t WebRtcOpus_DecoderChannels(OpusDecInst* inst); + +/**************************************************************************** + * WebRtcOpus_DecoderInit(...) + * + * This function resets state of the decoder. + * + * Input: + * - inst : Decoder context + */ +void WebRtcOpus_DecoderInit(OpusDecInst* inst); + +/**************************************************************************** + * WebRtcOpus_Decode(...) + * + * This function decodes an Opus packet into one or more audio frames at the + * ACM interface's sampling rate (32 kHz). + * + * Input: + * - inst : Decoder context + * - encoded : Encoded data + * - encoded_bytes : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - audio_type : 1 normal, 2 CNG (for Opus it should + * always return 1 since we're not using Opus's + * built-in DTX/CNG scheme) + * + * Return value : >0 - Samples per channel in decoded vector + * -1 - Error + */ +int WebRtcOpus_Decode(OpusDecInst* inst, const uint8_t* encoded, + size_t encoded_bytes, int16_t* decoded, + int16_t* audio_type); + +/**************************************************************************** + * WebRtcOpus_DecodePlc(...) + * + * This function processes PLC for opus frame(s). + * Input: + * - inst : Decoder context + * - number_of_lost_frames : Number of PLC frames to produce + * + * Output: + * - decoded : The decoded vector + * + * Return value : >0 - number of samples in decoded PLC vector + * -1 - Error + */ +int WebRtcOpus_DecodePlc(OpusDecInst* inst, int16_t* decoded, + int number_of_lost_frames); + +/**************************************************************************** + * WebRtcOpus_DecodeFec(...) + * + * This function decodes the FEC data from an Opus packet into one or more audio + * frames at the ACM interface's sampling rate (32 kHz). + * + * Input: + * - inst : Decoder context + * - encoded : Encoded data + * - encoded_bytes : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector (previous frame) + * + * Return value : >0 - Samples per channel in decoded vector + * 0 - No FEC data in the packet + * -1 - Error + */ +int WebRtcOpus_DecodeFec(OpusDecInst* inst, const uint8_t* encoded, + size_t encoded_bytes, int16_t* decoded, + int16_t* audio_type); + +/**************************************************************************** + * WebRtcOpus_DurationEst(...) + * + * This function calculates the duration of an opus packet. + * Input: + * - inst : Decoder context + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : The duration of the packet, in samples per + * channel. + */ +int WebRtcOpus_DurationEst(OpusDecInst* inst, + const uint8_t* payload, + size_t payload_length_bytes); + +/**************************************************************************** + * WebRtcOpus_PlcDuration(...) + * + * This function calculates the duration of a frame returned by packet loss + * concealment (PLC). + * + * Input: + * - inst : Decoder context + * + * Return value : The duration of a frame returned by PLC, in + * samples per channel. + */ +int WebRtcOpus_PlcDuration(OpusDecInst* inst); + +/* TODO(minyue): Check whether it is needed to add a decoder context to the + * arguments, like WebRtcOpus_DurationEst(...). In fact, the packet itself tells + * the duration. The decoder context in WebRtcOpus_DurationEst(...) is not used. + * So it may be advisable to remove it from WebRtcOpus_DurationEst(...). */ + +/**************************************************************************** + * WebRtcOpus_FecDurationEst(...) + * + * This function calculates the duration of the FEC data within an opus packet. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : >0 - The duration of the FEC data in the + * packet in samples per channel. + * 0 - No FEC data in the packet. + */ +int WebRtcOpus_FecDurationEst(const uint8_t* payload, + size_t payload_length_bytes); + +/**************************************************************************** + * WebRtcOpus_PacketHasFec(...) + * + * This function detects if an opus packet has FEC. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : 0 - the packet does NOT contain FEC. + * 1 - the packet contains FEC. + */ +int WebRtcOpus_PacketHasFec(const uint8_t* payload, + size_t payload_length_bytes); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INTERFACE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc new file mode 100644 index 0000000000..ca46aa1b16 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/codecs/tools/audio_codec_speed_test.h" + +using ::std::string; + +namespace webrtc { + +static const int kOpusBlockDurationMs = 20; +static const int kOpusSamplingKhz = 48; + +class OpusSpeedTest : public AudioCodecSpeedTest { + protected: + OpusSpeedTest(); + void SetUp() override; + void TearDown() override; + float EncodeABlock(int16_t* in_data, uint8_t* bit_stream, + size_t max_bytes, size_t* encoded_bytes) override; + float DecodeABlock(const uint8_t* bit_stream, size_t encoded_bytes, + int16_t* out_data) override; + WebRtcOpusEncInst* opus_encoder_; + WebRtcOpusDecInst* opus_decoder_; +}; + +OpusSpeedTest::OpusSpeedTest() + : AudioCodecSpeedTest(kOpusBlockDurationMs, + kOpusSamplingKhz, + kOpusSamplingKhz), + opus_encoder_(NULL), + opus_decoder_(NULL) { +} + +void OpusSpeedTest::SetUp() { + AudioCodecSpeedTest::SetUp(); + // If channels_ == 1, use Opus VOIP mode, otherwise, audio mode. + int app = channels_ == 1 ? 0 : 1; + /* Create encoder memory. */ + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, app)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + /* Set bitrate. */ + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, bit_rate_)); +} + +void OpusSpeedTest::TearDown() { + AudioCodecSpeedTest::TearDown(); + /* Free memory. */ + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +float OpusSpeedTest::EncodeABlock(int16_t* in_data, uint8_t* bit_stream, + size_t max_bytes, size_t* encoded_bytes) { + clock_t clocks = clock(); + int value = WebRtcOpus_Encode(opus_encoder_, in_data, + input_length_sample_, max_bytes, + bit_stream); + clocks = clock() - clocks; + EXPECT_GT(value, 0); + *encoded_bytes = static_cast(value); + return 1000.0 * clocks / CLOCKS_PER_SEC; +} + +float OpusSpeedTest::DecodeABlock(const uint8_t* bit_stream, + size_t encoded_bytes, int16_t* out_data) { + int value; + int16_t audio_type; + clock_t clocks = clock(); + value = WebRtcOpus_Decode(opus_decoder_, bit_stream, encoded_bytes, out_data, + &audio_type); + clocks = clock() - clocks; + EXPECT_EQ(output_length_sample_, static_cast(value)); + return 1000.0 * clocks / CLOCKS_PER_SEC; +} + +/* Test audio length in second. */ +constexpr size_t kDurationSec = 400; + +#define ADD_TEST(complexity) \ +TEST_P(OpusSpeedTest, OpusSetComplexityTest##complexity) { \ + /* Set complexity. */ \ + printf("Setting complexity to %d ...\n", complexity); \ + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, complexity)); \ + EncodeDecode(kDurationSec); \ +} + +ADD_TEST(10); +ADD_TEST(9); +ADD_TEST(8); +ADD_TEST(7); +ADD_TEST(6); +ADD_TEST(5); +ADD_TEST(4); +ADD_TEST(3); +ADD_TEST(2); +ADD_TEST(1); +ADD_TEST(0); + +#define ADD_BANDWIDTH_TEST(bandwidth) \ + TEST_P(OpusSpeedTest, OpusSetBandwidthTest##bandwidth) { \ + /* Set bandwidth. */ \ + printf("Setting bandwidth to %d ...\n", bandwidth); \ + EXPECT_EQ(0, WebRtcOpus_SetBandwidth(opus_encoder_, bandwidth)); \ + EncodeDecode(kDurationSec); \ + } + +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_NARROWBAND); +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_MEDIUMBAND); +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_WIDEBAND); +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_SUPERWIDEBAND); +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_FULLBAND); + +// List all test cases: (channel, bit rat, filename, extension). +const coding_param param_set[] = { + std::make_tuple(1, + 64000, + string("audio_coding/speech_mono_32_48kHz"), + string("pcm"), + true), + std::make_tuple(1, + 32000, + string("audio_coding/speech_mono_32_48kHz"), + string("pcm"), + true), + std::make_tuple(2, + 64000, + string("audio_coding/music_stereo_48kHz"), + string("pcm"), + true)}; + +INSTANTIATE_TEST_CASE_P(AllTest, OpusSpeedTest, + ::testing::ValuesIn(param_set)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc new file mode 100644 index 0000000000..12a15853c9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc @@ -0,0 +1,816 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_coding/codecs/opus/opus_inst.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +using test::AudioLoop; +using ::testing::TestWithParam; +using ::testing::Values; +using ::testing::Combine; + +// Maximum number of bytes in output bitstream. +const size_t kMaxBytes = 1000; +// Sample rate of Opus. +const size_t kOpusRateKhz = 48; +// Number of samples-per-channel in a 20 ms frame, sampled at 48 kHz. +const size_t kOpus20msFrameSamples = kOpusRateKhz * 20; +// Number of samples-per-channel in a 10 ms frame, sampled at 48 kHz. +const size_t kOpus10msFrameSamples = kOpusRateKhz * 10; + +class OpusTest : public TestWithParam<::testing::tuple> { + protected: + OpusTest(); + + void TestDtxEffect(bool dtx, int block_length_ms); + + void TestCbrEffect(bool dtx, int block_length_ms); + + // Prepare |speech_data_| for encoding, read from a hard-coded file. + // After preparation, |speech_data_.GetNextBlock()| returns a pointer to a + // block of |block_length_ms| milliseconds. The data is looped every + // |loop_length_ms| milliseconds. + void PrepareSpeechData(size_t channel, + int block_length_ms, + int loop_length_ms); + + int EncodeDecode(WebRtcOpusEncInst* encoder, + rtc::ArrayView input_audio, + WebRtcOpusDecInst* decoder, + int16_t* output_audio, + int16_t* audio_type); + + void SetMaxPlaybackRate(WebRtcOpusEncInst* encoder, + opus_int32 expect, int32_t set); + + void CheckAudioBounded(const int16_t* audio, size_t samples, size_t channels, + uint16_t bound) const; + + WebRtcOpusEncInst* opus_encoder_; + WebRtcOpusDecInst* opus_decoder_; + + AudioLoop speech_data_; + uint8_t bitstream_[kMaxBytes]; + size_t encoded_bytes_; + size_t channels_; + int application_; +}; + +OpusTest::OpusTest() + : opus_encoder_(NULL), + opus_decoder_(NULL), + encoded_bytes_(0), + channels_(static_cast(::testing::get<0>(GetParam()))), + application_(::testing::get<1>(GetParam())) { +} + +void OpusTest::PrepareSpeechData(size_t channel, int block_length_ms, + int loop_length_ms) { + const std::string file_name = + webrtc::test::ResourcePath((channel == 1) ? + "audio_coding/testfile32kHz" : + "audio_coding/teststereo32kHz", "pcm"); + if (loop_length_ms < block_length_ms) { + loop_length_ms = block_length_ms; + } + EXPECT_TRUE(speech_data_.Init(file_name, + loop_length_ms * kOpusRateKhz * channel, + block_length_ms * kOpusRateKhz * channel)); +} + +void OpusTest::SetMaxPlaybackRate(WebRtcOpusEncInst* encoder, + opus_int32 expect, + int32_t set) { + opus_int32 bandwidth; + EXPECT_EQ(0, WebRtcOpus_SetMaxPlaybackRate(opus_encoder_, set)); + opus_encoder_ctl(opus_encoder_->encoder, + OPUS_GET_MAX_BANDWIDTH(&bandwidth)); + EXPECT_EQ(expect, bandwidth); +} + +void OpusTest::CheckAudioBounded(const int16_t* audio, size_t samples, + size_t channels, uint16_t bound) const { + for (size_t i = 0; i < samples; ++i) { + for (size_t c = 0; c < channels; ++c) { + ASSERT_GE(audio[i * channels + c], -bound); + ASSERT_LE(audio[i * channels + c], bound); + } + } +} + +int OpusTest::EncodeDecode(WebRtcOpusEncInst* encoder, + rtc::ArrayView input_audio, + WebRtcOpusDecInst* decoder, + int16_t* output_audio, + int16_t* audio_type) { + int encoded_bytes_int = WebRtcOpus_Encode( + encoder, input_audio.data(), + rtc::CheckedDivExact(input_audio.size(), channels_), + kMaxBytes, bitstream_); + EXPECT_GE(encoded_bytes_int, 0); + encoded_bytes_ = static_cast(encoded_bytes_int); + int est_len = WebRtcOpus_DurationEst(decoder, bitstream_, encoded_bytes_); + int act_len = WebRtcOpus_Decode(decoder, bitstream_, + encoded_bytes_, output_audio, + audio_type); + EXPECT_EQ(est_len, act_len); + return act_len; +} + +// Test if encoder/decoder can enter DTX mode properly and do not enter DTX when +// they should not. This test is signal dependent. +void OpusTest::TestDtxEffect(bool dtx, int block_length_ms) { + PrepareSpeechData(channels_, block_length_ms, 2000); + const size_t samples = kOpusRateKhz * block_length_ms; + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, + channels_ == 1 ? 32000 : 64000)); + + // Set input audio as silence. + std::vector silence(samples * channels_, 0); + + // Setting DTX. + EXPECT_EQ(0, dtx ? WebRtcOpus_EnableDtx(opus_encoder_) : + WebRtcOpus_DisableDtx(opus_encoder_)); + + int16_t audio_type; + int16_t* output_data_decode = new int16_t[samples * channels_]; + + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(samples, + static_cast(EncodeDecode( + opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode, &audio_type))); + // If not DTX, it should never enter DTX mode. If DTX, we do not care since + // whether it enters DTX depends on the signal type. + if (!dtx) { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + } + + // We input some silent segments. In DTX mode, the encoder will stop sending. + // However, DTX may happen after a while. + for (int i = 0; i < 30; ++i) { + EXPECT_EQ(samples, + static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, output_data_decode, + &audio_type))); + if (!dtx) { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } else if (encoded_bytes_ == 1) { + EXPECT_EQ(1, opus_encoder_->in_dtx_mode); + EXPECT_EQ(1, opus_decoder_->in_dtx_mode); + EXPECT_EQ(2, audio_type); // Comfort noise. + break; + } + } + + // When Opus is in DTX, it wakes up in a regular basis. It sends two packets, + // one with an arbitrary size and the other of 1-byte, then stops sending for + // a certain number of frames. + + // |max_dtx_frames| is the maximum number of frames Opus can stay in DTX. + const int max_dtx_frames = 400 / block_length_ms + 1; + + // We run |kRunTimeMs| milliseconds of pure silence. + const int kRunTimeMs = 4500; + + // We check that, after a |kCheckTimeMs| milliseconds (given that the CNG in + // Opus needs time to adapt), the absolute values of DTX decoded signal are + // bounded by |kOutputValueBound|. + const int kCheckTimeMs = 4000; + +#if defined(OPUS_FIXED_POINT) + // Fixed-point Opus generates a random (comfort) noise, which has a less + // predictable value bound than its floating-point Opus. This value depends on + // input signal, and the time window for checking the output values (between + // |kCheckTimeMs| and |kRunTimeMs|). + const uint16_t kOutputValueBound = 30; + +#else + const uint16_t kOutputValueBound = 2; +#endif + + int time = 0; + while (time < kRunTimeMs) { + // DTX mode is maintained for maximum |max_dtx_frames| frames. + int i = 0; + for (; i < max_dtx_frames; ++i) { + time += block_length_ms; + EXPECT_EQ(samples, + static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, output_data_decode, + &audio_type))); + if (dtx) { + if (encoded_bytes_ > 1) + break; + EXPECT_EQ(0U, encoded_bytes_) // Send 0 byte. + << "Opus should have entered DTX mode."; + EXPECT_EQ(1, opus_encoder_->in_dtx_mode); + EXPECT_EQ(1, opus_decoder_->in_dtx_mode); + EXPECT_EQ(2, audio_type); // Comfort noise. + if (time >= kCheckTimeMs) { + CheckAudioBounded(output_data_decode, samples, channels_, + kOutputValueBound); + } + } else { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + } + + if (dtx) { + // With DTX, Opus must stop transmission for some time. + EXPECT_GT(i, 1); + } + + // We expect a normal payload. + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + + // Enters DTX again immediately. + time += block_length_ms; + EXPECT_EQ(samples, + static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, output_data_decode, + &audio_type))); + if (dtx) { + EXPECT_EQ(1U, encoded_bytes_); // Send 1 byte. + EXPECT_EQ(1, opus_encoder_->in_dtx_mode); + EXPECT_EQ(1, opus_decoder_->in_dtx_mode); + EXPECT_EQ(2, audio_type); // Comfort noise. + if (time >= kCheckTimeMs) { + CheckAudioBounded(output_data_decode, samples, channels_, + kOutputValueBound); + } + } else { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + } + + silence[0] = 10000; + if (dtx) { + // Verify that encoder/decoder can jump out from DTX mode. + EXPECT_EQ(samples, + static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, output_data_decode, + &audio_type))); + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + + // Free memory. + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +// Test if CBR does what we expect. +void OpusTest::TestCbrEffect(bool cbr, int block_length_ms) { + PrepareSpeechData(channels_, block_length_ms, 2000); + const size_t samples = kOpusRateKhz * block_length_ms; + + int32_t max_pkt_size_diff = 0; + int32_t prev_pkt_size = 0; + + // Create encoder memory. + EXPECT_EQ(0, + WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + + // Set bitrate. + EXPECT_EQ( + 0, WebRtcOpus_SetBitRate(opus_encoder_, channels_ == 1 ? 32000 : 64000)); + + // Setting CBR. + EXPECT_EQ(0, cbr ? WebRtcOpus_EnableCbr(opus_encoder_) + : WebRtcOpus_DisableCbr(opus_encoder_)); + + int16_t audio_type; + std::vector audio_out(samples * channels_); + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(samples, static_cast(EncodeDecode( + opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, audio_out.data(), &audio_type))); + + if (prev_pkt_size > 0) { + int32_t diff = std::abs((int32_t)encoded_bytes_ - prev_pkt_size); + max_pkt_size_diff = std::max(max_pkt_size_diff, diff); + } + prev_pkt_size = rtc::checked_cast(encoded_bytes_); + } + + if (cbr) { + EXPECT_EQ(max_pkt_size_diff, 0); + } else { + EXPECT_GT(max_pkt_size_diff, 0); + } + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +// Test failing Create. +TEST(OpusTest, OpusCreateFail) { + WebRtcOpusEncInst* opus_encoder; + WebRtcOpusDecInst* opus_decoder; + + // Test to see that an invalid pointer is caught. + EXPECT_EQ(-1, WebRtcOpus_EncoderCreate(NULL, 1, 0)); + // Invalid channel number. + EXPECT_EQ(-1, WebRtcOpus_EncoderCreate(&opus_encoder, 3, 0)); + // Invalid applciation mode. + EXPECT_EQ(-1, WebRtcOpus_EncoderCreate(&opus_encoder, 1, 2)); + + EXPECT_EQ(-1, WebRtcOpus_DecoderCreate(NULL, 1)); + // Invalid channel number. + EXPECT_EQ(-1, WebRtcOpus_DecoderCreate(&opus_decoder, 3)); +} + +// Test failing Free. +TEST(OpusTest, OpusFreeFail) { + // Test to see that an invalid pointer is caught. + EXPECT_EQ(-1, WebRtcOpus_EncoderFree(NULL)); + EXPECT_EQ(-1, WebRtcOpus_DecoderFree(NULL)); +} + +// Test normal Create and Free. +TEST_P(OpusTest, OpusCreateFree) { + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + EXPECT_TRUE(opus_encoder_ != NULL); + EXPECT_TRUE(opus_decoder_ != NULL); + // Free encoder and decoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusEncodeDecode) { + PrepareSpeechData(channels_, 20, 20); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, + channels_)); + + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, + channels_ == 1 ? 32000 : 64000)); + + // Check number of channels for decoder. + EXPECT_EQ(channels_, WebRtcOpus_DecoderChannels(opus_decoder_)); + + // Check application mode. + opus_int32 app; + opus_encoder_ctl(opus_encoder_->encoder, + OPUS_GET_APPLICATION(&app)); + EXPECT_EQ(application_ == 0 ? OPUS_APPLICATION_VOIP : OPUS_APPLICATION_AUDIO, + app); + + // Encode & decode. + int16_t audio_type; + int16_t* output_data_decode = new int16_t[kOpus20msFrameSamples * channels_]; + EXPECT_EQ(kOpus20msFrameSamples, + static_cast( + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, output_data_decode, &audio_type))); + + // Free memory. + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusSetBitRate) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetBitRate(opus_encoder_, 60000)); + + // Create encoder memory, try with different bitrates. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 30000)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 60000)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 300000)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 600000)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusSetComplexity) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetComplexity(opus_encoder_, 9)); + + // Create encoder memory, try with different complexities. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, 0)); + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, 10)); + EXPECT_EQ(-1, WebRtcOpus_SetComplexity(opus_encoder_, 11)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusSetBandwidth) { + PrepareSpeechData(channels_, 20, 20); + + int16_t audio_type; + std::unique_ptr output_data_decode( + new int16_t[kOpus20msFrameSamples * channels_]()); + + // Test without creating encoder memory. + EXPECT_EQ(-1, + WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND)); + EXPECT_EQ(-1, WebRtcOpus_GetBandwidth(opus_encoder_)); + + // Create encoder memory, try with different bandwidths. + EXPECT_EQ(0, + WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + + EXPECT_EQ(-1, WebRtcOpus_SetBandwidth(opus_encoder_, + OPUS_BANDWIDTH_NARROWBAND - 1)); + EXPECT_EQ(0, + WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND)); + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode.get(), &audio_type); + EXPECT_EQ(OPUS_BANDWIDTH_NARROWBAND, WebRtcOpus_GetBandwidth(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_FULLBAND)); + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode.get(), &audio_type); + EXPECT_EQ(OPUS_BANDWIDTH_FULLBAND, WebRtcOpus_GetBandwidth(opus_encoder_)); + EXPECT_EQ( + -1, WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_FULLBAND + 1)); + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode.get(), &audio_type); + EXPECT_EQ(OPUS_BANDWIDTH_FULLBAND, WebRtcOpus_GetBandwidth(opus_encoder_)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusForceChannels) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetForceChannels(opus_encoder_, 1)); + + ASSERT_EQ(0, + WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, application_)); + + if (channels_ == 2) { + EXPECT_EQ(-1, WebRtcOpus_SetForceChannels(opus_encoder_, 3)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 2)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 1)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 0)); + } else { + EXPECT_EQ(-1, WebRtcOpus_SetForceChannels(opus_encoder_, 2)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 1)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 0)); + } + + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +// Encode and decode one frame, initialize the decoder and +// decode once more. +TEST_P(OpusTest, OpusDecodeInit) { + PrepareSpeechData(channels_, 20, 20); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + + // Encode & decode. + int16_t audio_type; + int16_t* output_data_decode = new int16_t[kOpus20msFrameSamples * channels_]; + EXPECT_EQ(kOpus20msFrameSamples, + static_cast( + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, output_data_decode, &audio_type))); + + WebRtcOpus_DecoderInit(opus_decoder_); + + EXPECT_EQ(kOpus20msFrameSamples, + static_cast(WebRtcOpus_Decode( + opus_decoder_, bitstream_, encoded_bytes_, output_data_decode, + &audio_type))); + + // Free memory. + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusEnableDisableFec) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_EnableFec(opus_encoder_)); + EXPECT_EQ(-1, WebRtcOpus_DisableFec(opus_encoder_)); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DisableFec(opus_encoder_)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusEnableDisableDtx) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_EnableDtx(opus_encoder_)); + EXPECT_EQ(-1, WebRtcOpus_DisableDtx(opus_encoder_)); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + + opus_int32 dtx; + + // DTX is off by default. + opus_encoder_ctl(opus_encoder_->encoder, + OPUS_GET_DTX(&dtx)); + EXPECT_EQ(0, dtx); + + // Test to enable DTX. + EXPECT_EQ(0, WebRtcOpus_EnableDtx(opus_encoder_)); + opus_encoder_ctl(opus_encoder_->encoder, + OPUS_GET_DTX(&dtx)); + EXPECT_EQ(1, dtx); + + // Test to disable DTX. + EXPECT_EQ(0, WebRtcOpus_DisableDtx(opus_encoder_)); + opus_encoder_ctl(opus_encoder_->encoder, + OPUS_GET_DTX(&dtx)); + EXPECT_EQ(0, dtx); + + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusDtxOff) { + TestDtxEffect(false, 10); + TestDtxEffect(false, 20); + TestDtxEffect(false, 40); +} + +TEST_P(OpusTest, OpusDtxOn) { + TestDtxEffect(true, 10); + TestDtxEffect(true, 20); + TestDtxEffect(true, 40); +} + +TEST_P(OpusTest, OpusCbrOff) { + TestCbrEffect(false, 10); + TestCbrEffect(false, 20); + TestCbrEffect(false, 40); +} + +TEST_P(OpusTest, OpusCbrOn) { + TestCbrEffect(true, 10); + TestCbrEffect(true, 20); + TestCbrEffect(true, 40); +} + +TEST_P(OpusTest, OpusSetPacketLossRate) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_encoder_, 50)); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_encoder_, 50)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_encoder_, -1)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_encoder_, 101)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusSetMaxPlaybackRate) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetMaxPlaybackRate(opus_encoder_, 20000)); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_FULLBAND, 48000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_FULLBAND, 24001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_SUPERWIDEBAND, 24000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_SUPERWIDEBAND, 16001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_WIDEBAND, 16000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_WIDEBAND, 12001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_MEDIUMBAND, 12000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_MEDIUMBAND, 8001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND, 8000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND, 4000); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +// Test PLC. +TEST_P(OpusTest, OpusDecodePlc) { + PrepareSpeechData(channels_, 20, 20); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, + channels_== 1 ? 32000 : 64000)); + + // Check number of channels for decoder. + EXPECT_EQ(channels_, WebRtcOpus_DecoderChannels(opus_decoder_)); + + // Encode & decode. + int16_t audio_type; + int16_t* output_data_decode = new int16_t[kOpus20msFrameSamples * channels_]; + EXPECT_EQ(kOpus20msFrameSamples, + static_cast( + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, output_data_decode, &audio_type))); + + // Call decoder PLC. + int16_t* plc_buffer = new int16_t[kOpus20msFrameSamples * channels_]; + EXPECT_EQ(kOpus20msFrameSamples, + static_cast(WebRtcOpus_DecodePlc( + opus_decoder_, plc_buffer, 1))); + + // Free memory. + delete[] plc_buffer; + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +// Duration estimation. +TEST_P(OpusTest, OpusDurationEstimation) { + PrepareSpeechData(channels_, 20, 20); + + // Create. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + + // 10 ms. We use only first 10 ms of a 20 ms block. + auto speech_block = speech_data_.GetNextBlock(); + int encoded_bytes_int = WebRtcOpus_Encode( + opus_encoder_, speech_block.data(), + rtc::CheckedDivExact(speech_block.size(), 2 * channels_), + kMaxBytes, bitstream_); + EXPECT_GE(encoded_bytes_int, 0); + EXPECT_EQ(kOpus10msFrameSamples, + static_cast(WebRtcOpus_DurationEst( + opus_decoder_, bitstream_, + static_cast(encoded_bytes_int)))); + + // 20 ms + speech_block = speech_data_.GetNextBlock(); + encoded_bytes_int = WebRtcOpus_Encode( + opus_encoder_, speech_block.data(), + rtc::CheckedDivExact(speech_block.size(), channels_), + kMaxBytes, bitstream_); + EXPECT_GE(encoded_bytes_int, 0); + EXPECT_EQ(kOpus20msFrameSamples, + static_cast(WebRtcOpus_DurationEst( + opus_decoder_, bitstream_, + static_cast(encoded_bytes_int)))); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusDecodeRepacketized) { + constexpr size_t kPackets = 6; + + PrepareSpeechData(channels_, 20, 20 * kPackets); + + // Create encoder memory. + ASSERT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, + channels_, + application_)); + ASSERT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, + channels_)); + + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, + channels_ == 1 ? 32000 : 64000)); + + // Check number of channels for decoder. + EXPECT_EQ(channels_, WebRtcOpus_DecoderChannels(opus_decoder_)); + + // Encode & decode. + int16_t audio_type; + std::unique_ptr output_data_decode( + new int16_t[kPackets * kOpus20msFrameSamples * channels_]); + OpusRepacketizer* rp = opus_repacketizer_create(); + + size_t num_packets = 0; + constexpr size_t kMaxCycles = 100; + for (size_t idx = 0; idx < kMaxCycles; ++idx) { + auto speech_block = speech_data_.GetNextBlock(); + encoded_bytes_ = + WebRtcOpus_Encode(opus_encoder_, speech_block.data(), + rtc::CheckedDivExact(speech_block.size(), channels_), + kMaxBytes, bitstream_); + if (opus_repacketizer_cat( + rp, bitstream_, + rtc::checked_cast(encoded_bytes_)) == OPUS_OK) { + ++num_packets; + if (num_packets == kPackets) { + break; + } + } else { + // Opus repacketizer cannot guarantee a success. We try again if it fails. + opus_repacketizer_init(rp); + num_packets = 0; + } + } + EXPECT_EQ(kPackets, num_packets); + + encoded_bytes_ = opus_repacketizer_out(rp, bitstream_, kMaxBytes); + + EXPECT_EQ(kOpus20msFrameSamples * kPackets, + static_cast(WebRtcOpus_DurationEst( + opus_decoder_, bitstream_, encoded_bytes_))); + + EXPECT_EQ(kOpus20msFrameSamples * kPackets, + static_cast(WebRtcOpus_Decode( + opus_decoder_, bitstream_, encoded_bytes_, + output_data_decode.get(), &audio_type))); + + // Free memory. + opus_repacketizer_destroy(rp); + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +INSTANTIATE_TEST_CASE_P(VariousMode, + OpusTest, + Combine(Values(1, 2), Values(0, 1))); + + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc new file mode 100644 index 0000000000..b07624dffc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h" + +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioDecoderPcm16B::AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels) + : sample_rate_hz_(sample_rate_hz), num_channels_(num_channels) { + RTC_DCHECK(sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000) + << "Unsupported sample rate " << sample_rate_hz; + RTC_DCHECK_GE(num_channels, 1); +} + +void AudioDecoderPcm16B::Reset() {} + +int AudioDecoderPcm16B::SampleRateHz() const { + return sample_rate_hz_; +} + +size_t AudioDecoderPcm16B::Channels() const { + return num_channels_; +} + +int AudioDecoderPcm16B::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz_, sample_rate_hz); + size_t ret = WebRtcPcm16b_Decode(encoded, encoded_len, decoded); + *speech_type = ConvertSpeechType(1); + return static_cast(ret); +} + +std::vector AudioDecoderPcm16B::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz_, 1000); + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, samples_per_ms * 2 * num_channels_, + samples_per_ms); +} + +int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // Two encoded byte per sample per channel. + return static_cast(encoded_len / (2 * Channels())); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h new file mode 100644 index 0000000000..7d234224e3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_DECODER_PCM16B_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_DECODER_PCM16B_H_ + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class AudioDecoderPcm16B final : public AudioDecoder { + public: + AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels); + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + const int sample_rate_hz_; + const size_t num_channels_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoderPcm16B); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_DECODER_PCM16B_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc new file mode 100644 index 0000000000..831daedc63 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h" + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +size_t AudioEncoderPcm16B::EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) { + return WebRtcPcm16b_Encode(audio, input_len, encoded); +} + +size_t AudioEncoderPcm16B::BytesPerSample() const { + return 2; +} + +AudioEncoder::CodecType AudioEncoderPcm16B::GetCodecType() const { + return CodecType::kOther; +} + +namespace { + +AudioEncoderPcm16B::Config CreateConfig(const CodecInst& codec_inst) { + AudioEncoderPcm16B::Config config; + config.num_channels = codec_inst.channels; + config.sample_rate_hz = codec_inst.plfreq; + config.frame_size_ms = rtc::CheckedDivExact( + codec_inst.pacsize, rtc::CheckedDivExact(config.sample_rate_hz, 1000)); + config.payload_type = codec_inst.pltype; + return config; +} + +} // namespace + +bool AudioEncoderPcm16B::Config::IsOk() const { + if ((sample_rate_hz != 8000) && (sample_rate_hz != 16000) && + (sample_rate_hz != 32000) && (sample_rate_hz != 48000)) + return false; + return AudioEncoderPcm::Config::IsOk(); +} + +AudioEncoderPcm16B::AudioEncoderPcm16B(const CodecInst& codec_inst) + : AudioEncoderPcm16B(CreateConfig(codec_inst)) {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h new file mode 100644 index 0000000000..d6fd6e169e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_ENCODER_PCM16B_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_ENCODER_PCM16B_H_ + +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +struct CodecInst; + +class AudioEncoderPcm16B final : public AudioEncoderPcm { + public: + struct Config : public AudioEncoderPcm::Config { + public: + Config() : AudioEncoderPcm::Config(107), sample_rate_hz(8000) {} + bool IsOk() const; + + int sample_rate_hz; + }; + + explicit AudioEncoderPcm16B(const Config& config) + : AudioEncoderPcm(config, config.sample_rate_hz) {} + explicit AudioEncoderPcm16B(const CodecInst& codec_inst); + + protected: + size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) override; + + size_t BytesPerSample() const override; + + AudioEncoder::CodecType GetCodecType() const override; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderPcm16B); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_ENCODER_PCM16B_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c new file mode 100644 index 0000000000..54aa84d0e9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "pcm16b.h" + +#include "typedefs.h" // NOLINT(build/include) + +size_t WebRtcPcm16b_Encode(const int16_t* speech, + size_t len, + uint8_t* encoded) { + size_t i; + for (i = 0; i < len; ++i) { + uint16_t s = speech[i]; + encoded[2 * i] = s >> 8; + encoded[2 * i + 1] = s; + } + return 2 * len; +} + +size_t WebRtcPcm16b_Decode(const uint8_t* encoded, + size_t len, + int16_t* speech) { + size_t i; + for (i = 0; i < len / 2; ++i) + speech[i] = encoded[2 * i] << 8 | encoded[2 * i + 1]; + return len / 2; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h new file mode 100644 index 0000000000..041701a4b5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_H_ +/* + * Define the fixpoint numeric formats + */ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * WebRtcPcm16b_Encode(...) + * + * "Encode" a sample vector to 16 bit linear (Encoded standard is big endian) + * + * Input: + * - speech : Input speech vector + * - len : Number of samples in speech vector + * + * Output: + * - encoded : Encoded data vector (big endian 16 bit) + * + * Returned value : Length (in bytes) of coded data. + * Always equal to twice the len input parameter. + */ + +size_t WebRtcPcm16b_Encode(const int16_t* speech, + size_t len, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcPcm16b_Decode(...) + * + * "Decode" a vector to 16 bit linear (Encoded standard is big endian) + * + * Input: + * - encoded : Encoded data vector (big endian 16 bit) + * - len : Number of bytes in encoded + * + * Output: + * - speech : Decoded speech vector + * + * Returned value : Samples in speech + */ + +size_t WebRtcPcm16b_Decode(const uint8_t* encoded, + size_t len, + int16_t* speech); + +#ifdef __cplusplus +} +#endif + +#endif /* MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc new file mode 100644 index 0000000000..6d0fc2d015 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/pcm16b/pcm16b_common.h" + +namespace webrtc { + +void Pcm16BAppendSupportedCodecSpecs(std::vector* specs) { + for (uint8_t num_channels : {1, 2}) { + for (int sample_rate_hz : {8000, 16000, 32000}) { + specs->push_back( + {{"L16", sample_rate_hz, num_channels}, + {sample_rate_hz, num_channels, sample_rate_hz * num_channels * 16}}); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h new file mode 100644 index 0000000000..980a996408 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_COMMON_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_COMMON_H_ + +#include + +#include "api/audio_codecs/audio_decoder_factory.h" + +namespace webrtc { +void Pcm16BAppendSupportedCodecSpecs(std::vector* specs); +} + +#endif // MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc new file mode 100644 index 0000000000..4b9df6e0e6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" + +#include + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioEncoderCopyRed::Config::Config() = default; +AudioEncoderCopyRed::Config::Config(Config&&) = default; +AudioEncoderCopyRed::Config::~Config() = default; + +AudioEncoderCopyRed::AudioEncoderCopyRed(Config&& config) + : speech_encoder_(std::move(config.speech_encoder)), + red_payload_type_(config.payload_type) { + RTC_CHECK(speech_encoder_) << "Speech encoder not provided."; +} + +AudioEncoderCopyRed::~AudioEncoderCopyRed() = default; + +int AudioEncoderCopyRed::SampleRateHz() const { + return speech_encoder_->SampleRateHz(); +} + +size_t AudioEncoderCopyRed::NumChannels() const { + return speech_encoder_->NumChannels(); +} + +int AudioEncoderCopyRed::RtpTimestampRateHz() const { + return speech_encoder_->RtpTimestampRateHz(); +} + +size_t AudioEncoderCopyRed::Num10MsFramesInNextPacket() const { + return speech_encoder_->Num10MsFramesInNextPacket(); +} + +size_t AudioEncoderCopyRed::Max10MsFramesInAPacket() const { + return speech_encoder_->Max10MsFramesInAPacket(); +} + +int AudioEncoderCopyRed::GetTargetBitrate() const { + return speech_encoder_->GetTargetBitrate(); +} + +AudioEncoder::EncodedInfo AudioEncoderCopyRed::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + + const size_t primary_offset = encoded->size(); + EncodedInfo info = + speech_encoder_->Encode(rtp_timestamp, audio, encoded); + + RTC_CHECK(info.redundant.empty()) << "Cannot use nested redundant encoders."; + RTC_DCHECK_EQ(encoded->size() - primary_offset, info.encoded_bytes); + + if (info.encoded_bytes > 0) { + // |info| will be implicitly cast to an EncodedInfoLeaf struct, effectively + // discarding the (empty) vector of redundant information. This is + // intentional. + info.redundant.push_back(info); + RTC_DCHECK_EQ(info.redundant.size(), 1); + if (secondary_info_.encoded_bytes > 0) { + encoded->AppendData(secondary_encoded_); + info.redundant.push_back(secondary_info_); + RTC_DCHECK_EQ(info.redundant.size(), 2); + } + // Save primary to secondary. + secondary_encoded_.SetData(encoded->data() + primary_offset, + info.encoded_bytes); + secondary_info_ = info; + RTC_DCHECK_EQ(info.speech, info.redundant[0].speech); + } + // Update main EncodedInfo. + info.payload_type = red_payload_type_; + info.encoded_bytes = 0; + for (std::vector::const_iterator it = info.redundant.begin(); + it != info.redundant.end(); ++it) { + info.encoded_bytes += it->encoded_bytes; + } + return info; +} + +void AudioEncoderCopyRed::Reset() { + speech_encoder_->Reset(); + secondary_encoded_.Clear(); + secondary_info_.encoded_bytes = 0; +} + +bool AudioEncoderCopyRed::SetFec(bool enable) { + return speech_encoder_->SetFec(enable); +} + +bool AudioEncoderCopyRed::SetDtx(bool enable) { + return speech_encoder_->SetDtx(enable); +} + +bool AudioEncoderCopyRed::SetApplication(Application application) { + return speech_encoder_->SetApplication(application); +} + +void AudioEncoderCopyRed::SetMaxPlaybackRate(int frequency_hz) { + speech_encoder_->SetMaxPlaybackRate(frequency_hz); +} + +rtc::ArrayView> +AudioEncoderCopyRed::ReclaimContainedEncoders() { + return rtc::ArrayView>(&speech_encoder_, 1); +} + +void AudioEncoderCopyRed::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkPacketLossFraction( + uplink_packet_loss_fraction); +} + +void AudioEncoderCopyRed::OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkRecoverablePacketLossFraction( + uplink_recoverable_packet_loss_fraction); +} + +void AudioEncoderCopyRed::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional bwe_period_ms) { + speech_encoder_->OnReceivedUplinkBandwidth(target_audio_bitrate_bps, + bwe_period_ms); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h new file mode 100644 index 0000000000..e625c50d96 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_RED_AUDIO_ENCODER_COPY_RED_H_ +#define MODULES_AUDIO_CODING_CODECS_RED_AUDIO_ENCODER_COPY_RED_H_ + +#include +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "rtc_base/buffer.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// This class implements redundant audio coding. The class object will have an +// underlying AudioEncoder object that performs the actual encodings. The +// current class will gather the two latest encodings from the underlying codec +// into one packet. +class AudioEncoderCopyRed final : public AudioEncoder { + public: + struct Config { + Config(); + Config(Config&&); + ~Config(); + int payload_type; + std::unique_ptr speech_encoder; + }; + + explicit AudioEncoderCopyRed(Config&& config); + + ~AudioEncoderCopyRed() override; + + int SampleRateHz() const override; + size_t NumChannels() const override; + int RtpTimestampRateHz() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + void Reset() override; + bool SetFec(bool enable) override; + bool SetDtx(bool enable) override; + bool SetApplication(Application application) override; + void SetMaxPlaybackRate(int frequency_hz) override; + rtc::ArrayView> ReclaimContainedEncoders() + override; + void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) override; + void OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional bwe_period_ms) override; + + protected: + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + private: + std::unique_ptr speech_encoder_; + int red_payload_type_; + rtc::Buffer secondary_encoded_; + EncodedInfoLeaf secondary_info_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderCopyRed); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_RED_AUDIO_ENCODER_COPY_RED_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc new file mode 100644 index 0000000000..79d205e256 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/mock_audio_encoder.h" + +using ::testing::Return; +using ::testing::_; +using ::testing::SetArgPointee; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::MockFunction; + +namespace webrtc { + +namespace { +static const size_t kMaxNumSamples = 48 * 10 * 2; // 10 ms @ 48 kHz stereo. +} + +class AudioEncoderCopyRedTest : public ::testing::Test { + protected: + AudioEncoderCopyRedTest() + : mock_encoder_(new MockAudioEncoder), + timestamp_(4711), + sample_rate_hz_(16000), + num_audio_samples_10ms(sample_rate_hz_ / 100), + red_payload_type_(200) { + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type_; + config.speech_encoder = std::unique_ptr(mock_encoder_); + red_.reset(new AudioEncoderCopyRed(std::move(config))); + memset(audio_, 0, sizeof(audio_)); + EXPECT_CALL(*mock_encoder_, NumChannels()).WillRepeatedly(Return(1U)); + EXPECT_CALL(*mock_encoder_, SampleRateHz()) + .WillRepeatedly(Return(sample_rate_hz_)); + } + + void TearDown() override { + EXPECT_CALL(*mock_encoder_, Die()).Times(1); + red_.reset(); + } + + void Encode() { + ASSERT_TRUE(red_.get() != NULL); + encoded_.Clear(); + encoded_info_ = red_->Encode( + timestamp_, + rtc::ArrayView(audio_, num_audio_samples_10ms), + &encoded_); + timestamp_ += rtc::checked_cast(num_audio_samples_10ms); + } + + MockAudioEncoder* mock_encoder_; + std::unique_ptr red_; + uint32_t timestamp_; + int16_t audio_[kMaxNumSamples]; + const int sample_rate_hz_; + size_t num_audio_samples_10ms; + rtc::Buffer encoded_; + AudioEncoder::EncodedInfo encoded_info_; + const int red_payload_type_; +}; + +TEST_F(AudioEncoderCopyRedTest, CreateAndDestroy) { +} + +TEST_F(AudioEncoderCopyRedTest, CheckSampleRatePropagation) { + EXPECT_CALL(*mock_encoder_, SampleRateHz()).WillOnce(Return(17)); + EXPECT_EQ(17, red_->SampleRateHz()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckNumChannelsPropagation) { + EXPECT_CALL(*mock_encoder_, NumChannels()).WillOnce(Return(17U)); + EXPECT_EQ(17U, red_->NumChannels()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckFrameSizePropagation) { + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillOnce(Return(17U)); + EXPECT_EQ(17U, red_->Num10MsFramesInNextPacket()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckMaxFrameSizePropagation) { + EXPECT_CALL(*mock_encoder_, Max10MsFramesInAPacket()).WillOnce(Return(17U)); + EXPECT_EQ(17U, red_->Max10MsFramesInAPacket()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckTargetAudioBitratePropagation) { + EXPECT_CALL(*mock_encoder_, + OnReceivedUplinkBandwidth(4711, rtc::Optional())); + red_->OnReceivedUplinkBandwidth(4711, rtc::nullopt); +} + +TEST_F(AudioEncoderCopyRedTest, CheckPacketLossFractionPropagation) { + EXPECT_CALL(*mock_encoder_, OnReceivedUplinkPacketLossFraction(0.5)); + red_->OnReceivedUplinkPacketLossFraction(0.5); +} + +// Checks that the an Encode() call is immediately propagated to the speech +// encoder. +TEST_F(AudioEncoderCopyRedTest, CheckImmediateEncode) { + // Interleaving the EXPECT_CALL sequence with expectations on the MockFunction + // check ensures that exactly one call to EncodeImpl happens in each + // Encode call. + InSequence s; + MockFunction check; + for (int i = 1; i <= 6; ++i) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillRepeatedly(Return(AudioEncoder::EncodedInfo())); + EXPECT_CALL(check, Call(i)); + Encode(); + check.Call(i); + } +} + +// Checks that no output is produced if the underlying codec doesn't emit any +// new data, even if the RED codec is loaded with a secondary encoding. +TEST_F(AudioEncoderCopyRedTest, CheckNoOutput) { + static const size_t kEncodedSize = 17; + { + InSequence s; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(kEncodedSize))) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(0))) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(kEncodedSize))); + } + + // Start with one Encode() call that will produce output. + Encode(); + // First call is a special case, since it does not include a secondary + // payload. + EXPECT_EQ(1u, encoded_info_.redundant.size()); + EXPECT_EQ(kEncodedSize, encoded_info_.encoded_bytes); + + // Next call to the speech encoder will not produce any output. + Encode(); + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + + // Final call to the speech encoder will produce output. + Encode(); + EXPECT_EQ(2 * kEncodedSize, encoded_info_.encoded_bytes); + ASSERT_EQ(2u, encoded_info_.redundant.size()); +} + +// Checks that the correct payload sizes are populated into the redundancy +// information. +TEST_F(AudioEncoderCopyRedTest, CheckPayloadSizes) { + // Let the mock encoder return payload sizes 1, 2, 3, ..., 10 for the sequence + // of calls. + static const int kNumPackets = 10; + InSequence s; + for (int encode_size = 1; encode_size <= kNumPackets; ++encode_size) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(encode_size))); + } + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(1u, encoded_info_.redundant.size()); + EXPECT_EQ(1u, encoded_info_.encoded_bytes); + + for (size_t i = 2; i <= kNumPackets; ++i) { + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(i, encoded_info_.redundant[0].encoded_bytes); + EXPECT_EQ(i - 1, encoded_info_.redundant[1].encoded_bytes); + EXPECT_EQ(i + i - 1, encoded_info_.encoded_bytes); + } +} + +// Checks that the correct timestamps are returned. +TEST_F(AudioEncoderCopyRedTest, CheckTimestamps) { + uint32_t primary_timestamp = timestamp_; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 17; + info.encoded_timestamp = timestamp_; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(primary_timestamp, encoded_info_.encoded_timestamp); + + uint32_t secondary_timestamp = primary_timestamp; + primary_timestamp = timestamp_; + info.encoded_timestamp = timestamp_; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(primary_timestamp, encoded_info_.redundant[0].encoded_timestamp); + EXPECT_EQ(secondary_timestamp, encoded_info_.redundant[1].encoded_timestamp); + EXPECT_EQ(primary_timestamp, encoded_info_.encoded_timestamp); +} + +// Checks that the primary and secondary payloads are written correctly. +TEST_F(AudioEncoderCopyRedTest, CheckPayloads) { + // Let the mock encoder write payloads with increasing values. The first + // payload will have values 0, 1, 2, ..., kPayloadLenBytes - 1. + static const size_t kPayloadLenBytes = 5; + uint8_t payload[kPayloadLenBytes]; + for (uint8_t i = 0; i < kPayloadLenBytes; ++i) { + payload[i] = i; + } + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillRepeatedly(Invoke(MockAudioEncoder::CopyEncoding(payload))); + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(kPayloadLenBytes, encoded_info_.encoded_bytes); + for (size_t i = 0; i < kPayloadLenBytes; ++i) { + EXPECT_EQ(i, encoded_.data()[i]); + } + + for (int j = 0; j < 5; ++j) { + // Increment all values of the payload by 10. + for (size_t i = 0; i < kPayloadLenBytes; ++i) + payload[i] += 10; + + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(kPayloadLenBytes, encoded_info_.redundant[0].encoded_bytes); + EXPECT_EQ(kPayloadLenBytes, encoded_info_.redundant[1].encoded_bytes); + for (size_t i = 0; i < kPayloadLenBytes; ++i) { + // Check primary payload. + EXPECT_EQ((j + 1) * 10 + i, encoded_.data()[i]); + // Check secondary payload. + EXPECT_EQ(j * 10 + i, encoded_.data()[i + kPayloadLenBytes]); + } + } +} + +// Checks correct propagation of payload type. +// Checks that the correct timestamps are returned. +TEST_F(AudioEncoderCopyRedTest, CheckPayloadType) { + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 17; + info.payload_type = primary_payload_type; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + ASSERT_EQ(1u, encoded_info_.redundant.size()); + EXPECT_EQ(primary_payload_type, encoded_info_.redundant[0].payload_type); + EXPECT_EQ(red_payload_type_, encoded_info_.payload_type); + + const int secondary_payload_type = red_payload_type_ + 2; + info.payload_type = secondary_payload_type; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(secondary_payload_type, encoded_info_.redundant[0].payload_type); + EXPECT_EQ(primary_payload_type, encoded_info_.redundant[1].payload_type); + EXPECT_EQ(red_payload_type_, encoded_info_.payload_type); +} + +#if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// This test fixture tests various error conditions that makes the +// AudioEncoderCng die via CHECKs. +class AudioEncoderCopyRedDeathTest : public AudioEncoderCopyRedTest { + protected: + AudioEncoderCopyRedDeathTest() : AudioEncoderCopyRedTest() {} +}; + +TEST_F(AudioEncoderCopyRedDeathTest, WrongFrameSize) { + num_audio_samples_10ms *= 2; // 20 ms frame. + EXPECT_DEATH(Encode(), ""); + num_audio_samples_10ms = 0; // Zero samples. + EXPECT_DEATH(Encode(), ""); +} + +TEST_F(AudioEncoderCopyRedDeathTest, NullSpeechEncoder) { + AudioEncoderCopyRed* red = NULL; + AudioEncoderCopyRed::Config config; + config.speech_encoder = NULL; + EXPECT_DEATH(red = new AudioEncoderCopyRed(std::move(config)), + "Speech encoder not provided."); + // The delete operation is needed to avoid leak reports from memcheck. + delete red; +} + +#endif // GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc new file mode 100644 index 0000000000..d3749c142e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/tools/audio_codec_speed_test.h" + +#include "rtc_base/format_macros.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +using ::std::get; + +namespace webrtc { + +AudioCodecSpeedTest::AudioCodecSpeedTest(int block_duration_ms, + int input_sampling_khz, + int output_sampling_khz) + : block_duration_ms_(block_duration_ms), + input_sampling_khz_(input_sampling_khz), + output_sampling_khz_(output_sampling_khz), + input_length_sample_( + static_cast(block_duration_ms_ * input_sampling_khz_)), + output_length_sample_( + static_cast(block_duration_ms_ * output_sampling_khz_)), + data_pointer_(0), + loop_length_samples_(0), + max_bytes_(0), + encoded_bytes_(0), + encoding_time_ms_(0.0), + decoding_time_ms_(0.0), + out_file_(NULL) { +} + +void AudioCodecSpeedTest::SetUp() { + channels_ = get<0>(GetParam()); + bit_rate_ = get<1>(GetParam()); + in_filename_ = test::ResourcePath(get<2>(GetParam()), get<3>(GetParam())); + save_out_data_ = get<4>(GetParam()); + + FILE* fp = fopen(in_filename_.c_str(), "rb"); + assert(fp != NULL); + + // Obtain file size. + fseek(fp, 0, SEEK_END); + loop_length_samples_ = ftell(fp) / sizeof(int16_t); + rewind(fp); + + // Allocate memory to contain the whole file. + in_data_.reset(new int16_t[loop_length_samples_ + + input_length_sample_ * channels_]); + + data_pointer_ = 0; + + // Copy the file into the buffer. + ASSERT_EQ(fread(&in_data_[0], sizeof(int16_t), loop_length_samples_, fp), + loop_length_samples_); + fclose(fp); + + // Add an extra block length of samples to the end of the array, starting + // over again from the beginning of the array. This is done to simplify + // the reading process when reading over the end of the loop. + memcpy(&in_data_[loop_length_samples_], &in_data_[0], + input_length_sample_ * channels_ * sizeof(int16_t)); + + max_bytes_ = input_length_sample_ * channels_ * sizeof(int16_t); + out_data_.reset(new int16_t[output_length_sample_ * channels_]); + bit_stream_.reset(new uint8_t[max_bytes_]); + + if (save_out_data_) { + std::string out_filename = + ::testing::UnitTest::GetInstance()->current_test_info()->name(); + + // Erase '/' + size_t found; + while ((found = out_filename.find('/')) != std::string::npos) + out_filename.replace(found, 1, "_"); + + out_filename = test::OutputPath() + out_filename + ".pcm"; + + out_file_ = fopen(out_filename.c_str(), "wb"); + assert(out_file_ != NULL); + + printf("Output to be saved in %s.\n", out_filename.c_str()); + } +} + +void AudioCodecSpeedTest::TearDown() { + if (save_out_data_) { + fclose(out_file_); + } +} + +void AudioCodecSpeedTest::EncodeDecode(size_t audio_duration_sec) { + size_t time_now_ms = 0; + float time_ms; + + printf("Coding %d kHz-sampled %" PRIuS "-channel audio at %d bps ...\n", + input_sampling_khz_, channels_, bit_rate_); + + while (time_now_ms < audio_duration_sec * 1000) { + // Encode & decode. + time_ms = EncodeABlock(&in_data_[data_pointer_], &bit_stream_[0], + max_bytes_, &encoded_bytes_); + encoding_time_ms_ += time_ms; + time_ms = DecodeABlock(&bit_stream_[0], encoded_bytes_, &out_data_[0]); + decoding_time_ms_ += time_ms; + if (save_out_data_) { + fwrite(&out_data_[0], sizeof(int16_t), + output_length_sample_ * channels_, out_file_); + } + data_pointer_ = (data_pointer_ + input_length_sample_ * channels_) % + loop_length_samples_; + time_now_ms += block_duration_ms_; + } + + printf("Encoding: %.2f%% real time,\nDecoding: %.2f%% real time.\n", + (encoding_time_ms_ / audio_duration_sec) / 10.0, + (decoding_time_ms_ / audio_duration_sec) / 10.0); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h new file mode 100644 index 0000000000..9e616e789b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_TOOLS_AUDIO_CODEC_SPEED_TEST_H_ +#define MODULES_AUDIO_CODING_CODECS_TOOLS_AUDIO_CODEC_SPEED_TEST_H_ + +#include +#include + +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Define coding parameter as +// . +typedef std::tuple coding_param; + +class AudioCodecSpeedTest : public testing::TestWithParam { + protected: + AudioCodecSpeedTest(int block_duration_ms, + int input_sampling_khz, + int output_sampling_khz); + virtual void SetUp(); + virtual void TearDown(); + + // EncodeABlock(...) does the following: + // 1. encodes a block of audio, saved in |in_data|, + // 2. save the bit stream to |bit_stream| of |max_bytes| bytes in size, + // 3. assign |encoded_bytes| with the length of the bit stream (in bytes), + // 4. return the cost of time (in millisecond) spent on actual encoding. + virtual float EncodeABlock(int16_t* in_data, uint8_t* bit_stream, + size_t max_bytes, size_t* encoded_bytes) = 0; + + // DecodeABlock(...) does the following: + // 1. decodes the bit stream in |bit_stream| with a length of |encoded_bytes| + // (in bytes), + // 2. save the decoded audio in |out_data|, + // 3. return the cost of time (in millisecond) spent on actual decoding. + virtual float DecodeABlock(const uint8_t* bit_stream, size_t encoded_bytes, + int16_t* out_data) = 0; + + // Encoding and decode an audio of |audio_duration| (in seconds) and + // record the runtime for encoding and decoding separately. + void EncodeDecode(size_t audio_duration); + + int block_duration_ms_; + int input_sampling_khz_; + int output_sampling_khz_; + + // Number of samples-per-channel in a frame. + size_t input_length_sample_; + + // Expected output number of samples-per-channel in a frame. + size_t output_length_sample_; + + std::unique_ptr in_data_; + std::unique_ptr out_data_; + size_t data_pointer_; + size_t loop_length_samples_; + std::unique_ptr bit_stream_; + + // Maximum number of bytes in output bitstream for a frame of audio. + size_t max_bytes_; + + size_t encoded_bytes_; + float encoding_time_ms_; + float decoding_time_ms_; + FILE* out_file_; + + size_t channels_; + + // Bit rate is in bit-per-second. + int bit_rate_; + + std::string in_filename_; + + // Determines whether to save the output to file. + bool save_out_data_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_TOOLS_AUDIO_CODEC_SPEED_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/g711_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/g711_c_gn/moz.build new file mode 100644 index 0000000000..7ea6f552e0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/g711_c_gn/moz.build @@ -0,0 +1,210 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/g711_interface.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + +Library("g711_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/g711_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/g711_gn/moz.build new file mode 100644 index 0000000000..e15b0932fa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/g711_gn/moz.build @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("g711_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/g722_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/g722_c_gn/moz.build new file mode 100644 index 0000000000..6f03982968 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/g722_c_gn/moz.build @@ -0,0 +1,214 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_decode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_encode.c" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/g722_interface.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + +Library("g722_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/g722_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/g722_gn/moz.build new file mode 100644 index 0000000000..dd5afe2e65 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/g722_gn/moz.build @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("g722_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_c_gn/moz.build new file mode 100644 index 0000000000..d094f19133 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_c_gn/moz.build @@ -0,0 +1,296 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/bw_expand.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_construct.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/chebyshev.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/comp_corr.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/constants.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decode_residual.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/do_plc.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/encode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enh_upsample.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enhancer.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/frame_classify.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/gain_quant.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_input.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/hp_output.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/ilbc.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_decode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/init_encode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_check.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/my_corr.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/pack_bits.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/refiner.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/sort_sq.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/split_vq.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_construct.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/state_search.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/unpack_bits.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq3.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/vq4.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/window32_w32.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + +Library("ilbc_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_gn/moz.build new file mode 100644 index 0000000000..ac036193b3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/ilbc_gn/moz.build @@ -0,0 +1,229 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("ilbc_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module.h b/third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module.h new file mode 100644 index 0000000000..1a8ca8048b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module.h @@ -0,0 +1,821 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_H_ +#define MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_H_ + +#include +#include +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/optional.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/include/module.h" +#include "rtc_base/deprecation.h" +#include "rtc_base/function_view.h" +#include "system_wrappers/include/clock.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// forward declarations +struct CodecInst; +struct WebRtcRTPHeader; +class AudioDecoder; +class AudioEncoder; +class AudioFrame; +class RTPFragmentationHeader; + +#define WEBRTC_10MS_PCM_AUDIO 960 // 16 bits super wideband 48 kHz + +// Callback class used for sending data ready to be packetized +class AudioPacketizationCallback { + public: + virtual ~AudioPacketizationCallback() {} + + virtual int32_t SendData(FrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + const RTPFragmentationHeader* fragmentation) = 0; +}; + +// Callback class used for reporting VAD decision +class ACMVADCallback { + public: + virtual ~ACMVADCallback() {} + + virtual int32_t InFrameType(FrameType frame_type) = 0; +}; + +class AudioCodingModule { + protected: + AudioCodingModule() {} + + public: + struct Config { + Config(); + Config(const Config&); + ~Config(); + + NetEq::Config neteq_config; + Clock* clock; + rtc::scoped_refptr decoder_factory; + }; + + /////////////////////////////////////////////////////////////////////////// + // Creation and destruction of a ACM. + // + // The second method is used for testing where a simulated clock can be + // injected into ACM. ACM will take the ownership of the object clock and + // delete it when destroyed. + // + // TODO(solenberg): Remove once downstream projects are updated. + RTC_DEPRECATED static AudioCodingModule* Create(int id); + static AudioCodingModule* Create(); + static AudioCodingModule* Create(Clock* clock); + static AudioCodingModule* Create(const Config& config); + virtual ~AudioCodingModule() = default; + + /////////////////////////////////////////////////////////////////////////// + // Utility functions + // + + /////////////////////////////////////////////////////////////////////////// + // uint8_t NumberOfCodecs() + // Returns number of supported codecs. + // + // Return value: + // number of supported codecs. + /// + static int NumberOfCodecs(); + + /////////////////////////////////////////////////////////////////////////// + // int32_t Codec() + // Get supported codec with list number. + // + // Input: + // -list_id : list number. + // + // Output: + // -codec : a structure where the parameters of the codec, + // given by list number is written to. + // + // Return value: + // -1 if the list number (list_id) is invalid. + // 0 if succeeded. + // + static int Codec(int list_id, CodecInst* codec); + + /////////////////////////////////////////////////////////////////////////// + // int32_t Codec() + // Get supported codec with the given codec name, sampling frequency, and + // a given number of channels. + // + // Input: + // -payload_name : name of the codec. + // -sampling_freq_hz : sampling frequency of the codec. Note! for RED + // a sampling frequency of -1 is a valid input. + // -channels : number of channels ( 1 - mono, 2 - stereo). + // + // Output: + // -codec : a structure where the function returns the + // default parameters of the codec. + // + // Return value: + // -1 if no codec matches the given parameters. + // 0 if succeeded. + // + static int Codec(const char* payload_name, CodecInst* codec, + int sampling_freq_hz, size_t channels); + + /////////////////////////////////////////////////////////////////////////// + // int32_t Codec() + // + // Returns the list number of the given codec name, sampling frequency, and + // a given number of channels. + // + // Input: + // -payload_name : name of the codec. + // -sampling_freq_hz : sampling frequency of the codec. Note! for RED + // a sampling frequency of -1 is a valid input. + // -channels : number of channels ( 1 - mono, 2 - stereo). + // + // Return value: + // if the codec is found, the index of the codec in the list, + // -1 if the codec is not found. + // + static int Codec(const char* payload_name, int sampling_freq_hz, + size_t channels); + + /////////////////////////////////////////////////////////////////////////// + // bool IsCodecValid() + // Checks the validity of the parameters of the given codec. + // + // Input: + // -codec : the structure which keeps the parameters of the + // codec. + // + // Return value: + // true if the parameters are valid, + // false if any parameter is not valid. + // + static bool IsCodecValid(const CodecInst& codec); + + /////////////////////////////////////////////////////////////////////////// + // Sender + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t RegisterSendCodec() + // Registers a codec, specified by |send_codec|, as sending codec. + // This API can be called multiple of times to register Codec. The last codec + // registered overwrites the previous ones. + // The API can also be used to change payload type for CNG and RED, which are + // registered by default to default payload types. + // Note that registering CNG and RED won't overwrite speech codecs. + // This API can be called to set/change the send payload-type, frame-size + // or encoding rate (if applicable for the codec). + // + // Note: If a stereo codec is registered as send codec, VAD/DTX will + // automatically be turned off, since it is not supported for stereo sending. + // + // Note: If a secondary encoder is already registered, and the new send-codec + // has a sampling rate that does not match the secondary encoder, the + // secondary encoder will be unregistered. + // + // Input: + // -send_codec : Parameters of the codec to be registered, c.f. + // common_types.h for the definition of + // CodecInst. + // + // Return value: + // -1 if failed to initialize, + // 0 if succeeded. + // + virtual int32_t RegisterSendCodec(const CodecInst& send_codec) = 0; + + // Registers |external_speech_encoder| as encoder. The new encoder will + // replace any previously registered speech encoder (internal or external). + virtual void RegisterExternalSendCodec( + AudioEncoder* external_speech_encoder) = 0; + + // |modifier| is called exactly once with one argument: a pointer to the + // unique_ptr that holds the current encoder (which is null if there is no + // current encoder). For the duration of the call, |modifier| has exclusive + // access to the unique_ptr; it may call the encoder, steal the encoder and + // replace it with another encoder or with nullptr, etc. + virtual void ModifyEncoder( + rtc::FunctionView*)> modifier) = 0; + + // |modifier| is called exactly once with one argument: a const pointer to the + // current encoder (which is null if there is no current encoder). + virtual void QueryEncoder( + rtc::FunctionView query) = 0; + + // Utility method for simply replacing the existing encoder with a new one. + void SetEncoder(std::unique_ptr new_encoder) { + ModifyEncoder([&](std::unique_ptr* encoder) { + *encoder = std::move(new_encoder); + }); + } + + /////////////////////////////////////////////////////////////////////////// + // int32_t SendCodec() + // Get parameters for the codec currently registered as send codec. + // + // Return value: + // The send codec, or nothing if we don't have one + // + virtual rtc::Optional SendCodec() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t SendFrequency() + // Get the sampling frequency of the current encoder in Hertz. + // + // Return value: + // positive; sampling frequency [Hz] of the current encoder. + // -1 if an error has happened. + // + virtual int32_t SendFrequency() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // Sets the bitrate to the specified value in bits/sec. If the value is not + // supported by the codec, it will choose another appropriate value. + // + // This is only used in test code that rely on old ACM APIs. + // TODO(minyue): Remove it when possible. + virtual void SetBitRate(int bitrate_bps) = 0; + + // int32_t RegisterTransportCallback() + // Register a transport callback which will be called to deliver + // the encoded buffers whenever Process() is called and a + // bit-stream is ready. + // + // Input: + // -transport : pointer to the callback class + // transport->SendData() is called whenever + // Process() is called and bit-stream is ready + // to deliver. + // + // Return value: + // -1 if the transport callback could not be registered + // 0 if registration is successful. + // + virtual int32_t RegisterTransportCallback( + AudioPacketizationCallback* transport) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t Add10MsData() + // Add 10MS of raw (PCM) audio data and encode it. If the sampling + // frequency of the audio does not match the sampling frequency of the + // current encoder ACM will resample the audio. If an encoded packet was + // produced, it will be delivered via the callback object registered using + // RegisterTransportCallback, and the return value from this function will + // be the number of bytes encoded. + // + // Input: + // -audio_frame : the input audio frame, containing raw audio + // sampling frequency etc., + // c.f. module_common_types.h for definition of + // AudioFrame. + // + // Return value: + // >= 0 number of bytes encoded. + // -1 some error occurred. + // + virtual int32_t Add10MsData(const AudioFrame& audio_frame) = 0; + + /////////////////////////////////////////////////////////////////////////// + // (RED) Redundant Coding + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t SetREDStatus() + // configure RED status i.e. on/off. + // + // RFC 2198 describes a solution which has a single payload type which + // signifies a packet with redundancy. That packet then becomes a container, + // encapsulating multiple payloads into a single RTP packet. + // Such a scheme is flexible, since any amount of redundancy may be + // encapsulated within a single packet. There is, however, a small overhead + // since each encapsulated payload must be preceded by a header indicating + // the type of data enclosed. + // + // Input: + // -enable_red : if true RED is enabled, otherwise RED is + // disabled. + // + // Return value: + // -1 if failed to set RED status, + // 0 if succeeded. + // + virtual int32_t SetREDStatus(bool enable_red) = 0; + + /////////////////////////////////////////////////////////////////////////// + // bool REDStatus() + // Get RED status + // + // Return value: + // true if RED is enabled, + // false if RED is disabled. + // + virtual bool REDStatus() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // (FEC) Forward Error Correction (codec internal) + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t SetCodecFEC() + // Configures codec internal FEC status i.e. on/off. No effects on codecs that + // do not provide internal FEC. + // + // Input: + // -enable_fec : if true FEC will be enabled otherwise the FEC is + // disabled. + // + // Return value: + // -1 if failed, or the codec does not support FEC + // 0 if succeeded. + // + virtual int SetCodecFEC(bool enable_codec_fec) = 0; + + /////////////////////////////////////////////////////////////////////////// + // bool CodecFEC() + // Gets status of codec internal FEC. + // + // Return value: + // true if FEC is enabled, + // false if FEC is disabled. + // + virtual bool CodecFEC() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int SetPacketLossRate() + // Sets expected packet loss rate for encoding. Some encoders provide packet + // loss gnostic encoding to make stream less sensitive to packet losses, + // through e.g., FEC. No effects on codecs that do not provide such encoding. + // + // Input: + // -packet_loss_rate : expected packet loss rate (0 -- 100 inclusive). + // + // Return value + // -1 if failed to set packet loss rate, + // 0 if succeeded. + // + // This is only used in test code that rely on old ACM APIs. + // TODO(minyue): Remove it when possible. + virtual int SetPacketLossRate(int packet_loss_rate) = 0; + + /////////////////////////////////////////////////////////////////////////// + // (VAD) Voice Activity Detection + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t SetVAD() + // If DTX is enabled & the codec does not have internal DTX/VAD + // WebRtc VAD will be automatically enabled and |enable_vad| is ignored. + // + // If DTX is disabled but VAD is enabled no DTX packets are send, + // regardless of whether the codec has internal DTX/VAD or not. In this + // case, WebRtc VAD is running to label frames as active/in-active. + // + // NOTE! VAD/DTX is not supported when sending stereo. + // + // Inputs: + // -enable_dtx : if true DTX is enabled, + // otherwise DTX is disabled. + // -enable_vad : if true VAD is enabled, + // otherwise VAD is disabled. + // -vad_mode : determines the aggressiveness of VAD. A more + // aggressive mode results in more frames labeled + // as in-active, c.f. definition of + // ACMVADMode in audio_coding_module_typedefs.h + // for valid values. + // + // Return value: + // -1 if failed to set up VAD/DTX, + // 0 if succeeded. + // + virtual int32_t SetVAD(const bool enable_dtx = true, + const bool enable_vad = false, + const ACMVADMode vad_mode = VADNormal) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t VAD() + // Get VAD status. + // + // Outputs: + // -dtx_enabled : is set to true if DTX is enabled, otherwise + // is set to false. + // -vad_enabled : is set to true if VAD is enabled, otherwise + // is set to false. + // -vad_mode : is set to the current aggressiveness of VAD. + // + // Return value: + // -1 if fails to retrieve the setting of DTX/VAD, + // 0 if succeeded. + // + virtual int32_t VAD(bool* dtx_enabled, bool* vad_enabled, + ACMVADMode* vad_mode) const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t RegisterVADCallback() + // Call this method to register a callback function which is called + // any time that ACM encounters an empty frame. That is a frame which is + // recognized inactive. Depending on the codec WebRtc VAD or internal codec + // VAD is employed to identify a frame as active/inactive. + // + // Input: + // -vad_callback : pointer to a callback function. + // + // Return value: + // -1 if failed to register the callback function. + // 0 if the callback function is registered successfully. + // + virtual int32_t RegisterVADCallback(ACMVADCallback* vad_callback) = 0; + + /////////////////////////////////////////////////////////////////////////// + // Receiver + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t InitializeReceiver() + // Any decoder-related state of ACM will be initialized to the + // same state when ACM is created. This will not interrupt or + // effect encoding functionality of ACM. ACM would lose all the + // decoding-related settings by calling this function. + // For instance, all registered codecs are deleted and have to be + // registered again. + // + // Return value: + // -1 if failed to initialize, + // 0 if succeeded. + // + virtual int32_t InitializeReceiver() = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t ReceiveFrequency() + // Get sampling frequency of the last received payload. + // + // Return value: + // non-negative the sampling frequency in Hertz. + // -1 if an error has occurred. + // + virtual int32_t ReceiveFrequency() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t PlayoutFrequency() + // Get sampling frequency of audio played out. + // + // Return value: + // the sampling frequency in Hertz. + // + virtual int32_t PlayoutFrequency() const = 0; + + // Replace any existing decoders with the given payload type -> decoder map. + virtual void SetReceiveCodecs( + const std::map& codecs) = 0; + + // Registers a decoder for the given payload type. Returns true iff + // successful. + virtual bool RegisterReceiveCodec(int rtp_payload_type, + const SdpAudioFormat& audio_format) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t RegisterReceiveCodec() + // Register possible decoders, can be called multiple times for + // codecs, CNG-NB, CNG-WB, CNG-SWB, AVT and RED. + // + // Input: + // -receive_codec : parameters of the codec to be registered, c.f. + // common_types.h for the definition of + // CodecInst. + // + // Return value: + // -1 if failed to register the codec + // 0 if the codec registered successfully. + // + virtual int RegisterReceiveCodec(const CodecInst& receive_codec) = 0; + + // Register a decoder; call repeatedly to register multiple decoders. |df| is + // a decoder factory that returns an iSAC decoder; it will be called once if + // the decoder being registered is iSAC. + virtual int RegisterReceiveCodec( + const CodecInst& receive_codec, + rtc::FunctionView()> isac_factory) = 0; + + // Registers an external decoder. The name is only used to provide information + // back to the caller about the decoder. Hence, the name is arbitrary, and may + // be empty. + virtual int RegisterExternalReceiveCodec(int rtp_payload_type, + AudioDecoder* external_decoder, + int sample_rate_hz, + int num_channels, + const std::string& name) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t UnregisterReceiveCodec() + // Unregister the codec currently registered with a specific payload type + // from the list of possible receive codecs. + // + // Input: + // -payload_type : The number representing the payload type to + // unregister. + // + // Output: + // -1 if fails to unregister. + // 0 if the given codec is successfully unregistered. + // + virtual int UnregisterReceiveCodec( + uint8_t payload_type) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t ReceiveCodec() + // Get the codec associated with last received payload. + // + // Output: + // -curr_receive_codec : parameters of the codec associated with the last + // received payload, c.f. common_types.h for + // the definition of CodecInst. + // + // Return value: + // -1 if failed to retrieve the codec, + // 0 if the codec is successfully retrieved. + // + virtual int32_t ReceiveCodec(CodecInst* curr_receive_codec) const = 0; + + /////////////////////////////////////////////////////////////////////////// + // rtc::Optional ReceiveFormat() + // Get the format associated with last received payload. + // + // Return value: + // An SdpAudioFormat describing the format associated with the last + // received payload. + // An empty Optional if no payload has yet been received. + // + virtual rtc::Optional ReceiveFormat() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int ReceiveSampleRate() + // + // Mozilla extension. + // Return the sample-rate of the inbound audio stream. + // + // Return value: + // 0 if no audio has been received, the sample-rate of the inbound audio + // otherwise. + virtual int ReceiveSampleRate() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t IncomingPacket() + // Call this function to insert a parsed RTP packet into ACM. + // + // Inputs: + // -incoming_payload : received payload. + // -payload_len_bytes : the length of payload in bytes. + // -rtp_info : the relevant information retrieved from RTP + // header. + // + // Return value: + // -1 if failed to push in the payload + // 0 if payload is successfully pushed in. + // + virtual int32_t IncomingPacket(const uint8_t* incoming_payload, + const size_t payload_len_bytes, + const WebRtcRTPHeader& rtp_info) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int SetMinimumPlayoutDelay() + // Set a minimum for the playout delay, used for lip-sync. NetEq maintains + // such a delay unless channel condition yields to a higher delay. + // + // Input: + // -time_ms : minimum delay in milliseconds. + // + // Return value: + // -1 if failed to set the delay, + // 0 if the minimum delay is set. + // + virtual int SetMinimumPlayoutDelay(int time_ms) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int SetMaximumPlayoutDelay() + // Set a maximum for the playout delay + // + // Input: + // -time_ms : maximum delay in milliseconds. + // + // Return value: + // -1 if failed to set the delay, + // 0 if the maximum delay is set. + // + virtual int SetMaximumPlayoutDelay(int time_ms) = 0; + + // TODO(kwiberg): Consider if this is needed anymore, now that voe::Channel + // doesn't use it. + // The shortest latency, in milliseconds, required by jitter buffer. This + // is computed based on inter-arrival times and playout mode of NetEq. The + // actual delay is the maximum of least-required-delay and the minimum-delay + // specified by SetMinumumPlayoutDelay() API. + // + virtual int LeastRequiredDelayMs() const = 0; + + // int32_t PlayoutTimestamp() + // The send timestamp of an RTP packet is associated with the decoded + // audio of the packet in question. This function returns the timestamp of + // the latest audio obtained by calling PlayoutData10ms(). + // + // Input: + // -timestamp : a reference to a uint32_t to receive the + // timestamp. + // Return value: + // 0 if the output is a correct timestamp. + // -1 if failed to output the correct timestamp. + // + RTC_DEPRECATED virtual int32_t PlayoutTimestamp(uint32_t* timestamp) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t PlayoutTimestamp() + // The send timestamp of an RTP packet is associated with the decoded + // audio of the packet in question. This function returns the timestamp of + // the latest audio obtained by calling PlayoutData10ms(), or empty if no + // valid timestamp is available. + // + virtual rtc::Optional PlayoutTimestamp() = 0; + + /////////////////////////////////////////////////////////////////////////// + // int FilteredCurrentDelayMs() + // Returns the current total delay from NetEq (packet buffer and sync buffer) + // in ms, with smoothing applied to even out short-time fluctuations due to + // jitter. The packet buffer part of the delay is not updated during DTX/CNG + // periods. + // + virtual int FilteredCurrentDelayMs() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int FilteredCurrentDelayMs() + // Returns the current target delay for NetEq in ms. + // + virtual int TargetDelayMs() const = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t PlayoutData10Ms( + // Get 10 milliseconds of raw audio data for playout, at the given sampling + // frequency. ACM will perform a resampling if required. + // + // Input: + // -desired_freq_hz : the desired sampling frequency, in Hertz, of the + // output audio. If set to -1, the function returns + // the audio at the current sampling frequency. + // + // Output: + // -audio_frame : output audio frame which contains raw audio data + // and other relevant parameters, c.f. + // module_common_types.h for the definition of + // AudioFrame. + // -muted : if true, the sample data in audio_frame is not + // populated, and must be interpreted as all zero. + // + // Return value: + // -1 if the function fails, + // 0 if the function succeeds. + // + virtual int32_t PlayoutData10Ms(int32_t desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) = 0; + + ///////////////////////////////////////////////////////////////////////////// + // Same as above, but without the muted parameter. This methods should not be + // used if enable_fast_accelerate was set to true in NetEq::Config. + // TODO(henrik.lundin) Remove this method when downstream dependencies are + // ready. + virtual int32_t PlayoutData10Ms(int32_t desired_freq_hz, + AudioFrame* audio_frame) = 0; + + /////////////////////////////////////////////////////////////////////////// + // Codec specific + // + + /////////////////////////////////////////////////////////////////////////// + // int SetOpusApplication() + // Sets the intended application if current send codec is Opus. Opus uses this + // to optimize the encoding for applications like VOIP and music. Currently, + // two modes are supported: kVoip and kAudio. + // + // Input: + // - application : intended application. + // + // Return value: + // -1 if current send codec is not Opus or error occurred in setting the + // Opus application mode. + // 0 if the Opus application mode is successfully set. + // + virtual int SetOpusApplication(OpusApplicationMode application) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int SetOpusMaxPlaybackRate() + // If current send codec is Opus, informs it about maximum playback rate the + // receiver will render. Opus can use this information to optimize the bit + // rate and increase the computation efficiency. + // + // Input: + // -frequency_hz : maximum playback rate in Hz. + // + // Return value: + // -1 if current send codec is not Opus or + // error occurred in setting the maximum playback rate, + // 0 if maximum bandwidth is set successfully. + // + virtual int SetOpusMaxPlaybackRate(int frequency_hz) = 0; + + /////////////////////////////////////////////////////////////////////////// + // EnableOpusDtx() + // Enable the DTX, if current send codec is Opus. + // + // Return value: + // -1 if current send codec is not Opus or error occurred in enabling the + // Opus DTX. + // 0 if Opus DTX is enabled successfully. + // + virtual int EnableOpusDtx() = 0; + + /////////////////////////////////////////////////////////////////////////// + // int DisableOpusDtx() + // If current send codec is Opus, disables its internal DTX. + // + // Return value: + // -1 if current send codec is not Opus or error occurred in disabling DTX. + // 0 if Opus DTX is disabled successfully. + // + virtual int DisableOpusDtx() = 0; + + /////////////////////////////////////////////////////////////////////////// + // statistics + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t GetNetworkStatistics() + // Get network statistics. Note that the internal statistics of NetEq are + // reset by this call. + // + // Input: + // -network_statistics : a structure that contains network statistics. + // + // Return value: + // -1 if failed to set the network statistics, + // 0 if statistics are set successfully. + // + virtual int32_t GetNetworkStatistics( + NetworkStatistics* network_statistics) = 0; + + // + // Enable NACK and set the maximum size of the NACK list. If NACK is already + // enable then the maximum NACK list size is modified accordingly. + // + // If the sequence number of last received packet is N, the sequence numbers + // of NACK list are in the range of [N - |max_nack_list_size|, N). + // + // |max_nack_list_size| should be positive (none zero) and less than or + // equal to |Nack::kNackListSizeLimit|. Otherwise, No change is applied and -1 + // is returned. 0 is returned at success. + // + virtual int EnableNack(size_t max_nack_list_size) = 0; + + // Disable NACK. + virtual void DisableNack() = 0; + + // + // Get a list of packets to be retransmitted. |round_trip_time_ms| is an + // estimate of the round-trip-time (in milliseconds). Missing packets which + // will be playout in a shorter time than the round-trip-time (with respect + // to the time this API is called) will not be included in the list. + // + // Negative |round_trip_time_ms| results is an error message and empty list + // is returned. + // + virtual std::vector GetNackList( + int64_t round_trip_time_ms) const = 0; + + virtual void GetDecodingCallStatistics( + AudioDecodingCallStats* call_stats) const = 0; + + virtual ANAStats GetANAStats() const = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module_typedefs.h b/third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module_typedefs.h new file mode 100644 index 0000000000..ad71ef1f97 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/include/audio_coding_module_typedefs.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_TYPEDEFS_H_ +#define MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_TYPEDEFS_H_ + +#include + +#include "modules/include/module_common_types.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +/////////////////////////////////////////////////////////////////////////// +// enum ACMVADMode +// An enumerator for aggressiveness of VAD +// -VADNormal : least aggressive mode. +// -VADLowBitrate : more aggressive than "VADNormal" to save on +// bit-rate. +// -VADAggr : an aggressive mode. +// -VADVeryAggr : the most agressive mode. +// +enum ACMVADMode { + VADNormal = 0, + VADLowBitrate = 1, + VADAggr = 2, + VADVeryAggr = 3 +}; + +/////////////////////////////////////////////////////////////////////////// +// +// Enumeration of Opus mode for intended application. +// +// kVoip : optimized for voice signals. +// kAudio : optimized for non-voice signals like music. +// +enum OpusApplicationMode { + kVoip = 0, + kAudio = 1, +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_TYPEDEFS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_c_gn/moz.build new file mode 100644 index 0000000000..d8a4ba7011 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_c_gn/moz.build @@ -0,0 +1,257 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_hist.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/arith_routines_logist.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/crc.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/decode_bwe.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/encode_lpc_swb.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/entropy_coding.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/fft.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filterbank_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/filterbanks.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/intialize.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lattice.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_analysis.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_gain_swb_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb12_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_shape_swb16_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/lpc_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_gain_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_lag_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/spectrum_ar_model_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/transform.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "m", + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + +Library("isac_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_common_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_common_gn/moz.build new file mode 100644 index 0000000000..0913500d4f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_common_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/locked_bandwidth_info.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("isac_common_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_arm_asm_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_arm_asm_gn/moz.build new file mode 100644 index 0000000000..de9188eab3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_arm_asm_gn/moz.build @@ -0,0 +1,100 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ARCH_ARM"] = True +DEFINES["WEBRTC_ARCH_ARM_V7"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_HAS_NEON"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_POSIX"] = True +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True +DEFINES["_FILE_OFFSET_BITS"] = "64" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_armv7.S", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["__GNU_SOURCE"] = "1" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +Library("isac_fix_c_arm_asm_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_gn/moz.build new file mode 100644 index 0000000000..926ee9b762 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_c_gn/moz.build @@ -0,0 +1,438 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_plc.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_masking_model.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_hist.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/arith_routines_logist.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/decode_bwe.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/encode.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbank_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/initialize.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lpc_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_estimator_c.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_gain_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_lag_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/spectrum_ar_model_tables.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_tables.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "ppc64": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_c.c" + ] + +Library("isac_fix_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_common_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_common_gn/moz.build new file mode 100644 index 0000000000..a700308401 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_common_gn/moz.build @@ -0,0 +1,210 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/fft.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + +Library("isac_fix_common_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_gn/moz.build new file mode 100644 index 0000000000..c6f9cdf1b6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_fix_gn/moz.build @@ -0,0 +1,116 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +CXXFLAGS += [ + "-mfpu=neon" +] + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ARCH_ARM"] = True +DEFINES["WEBRTC_ARCH_ARM_V7"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_HAS_NEON"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_POSIX"] = True +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True +DEFINES["_FILE_OFFSET_BITS"] = "64" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_decoder_isacfix.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/audio_encoder_isacfix.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +Library("isac_fix_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_gn/moz.build new file mode 100644 index 0000000000..5a09b8178b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_gn/moz.build @@ -0,0 +1,228 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_decoder_isac.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/source/audio_encoder_isac.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "m", + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("isac_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/isac_neon_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_neon_gn/moz.build new file mode 100644 index 0000000000..153967d1af --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/isac_neon_gn/moz.build @@ -0,0 +1,199 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_HAS_NEON"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/entropy_coding_neon.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filterbanks_neon.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/filters_neon.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/lattice_neon.c", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/source/transform_neon.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +Library("isac_neon_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build new file mode 100644 index 0000000000..9f3ce8193b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("legacy_encoded_audio_frame_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.cc new file mode 100644 index 0000000000..183ad7b2b5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/accelerate.h" + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +namespace webrtc { + +Accelerate::ReturnCodes Accelerate::Process(const int16_t* input, + size_t input_length, + bool fast_accelerate, + AudioMultiVector* output, + size_t* length_change_samples) { + // Input length must be (almost) 30 ms. + static const size_t k15ms = 120; // 15 ms = 120 samples at 8 kHz sample rate. + if (num_channels_ == 0 || + input_length / num_channels_ < (2 * k15ms - 1) * fs_mult_) { + // Length of input data too short to do accelerate. Simply move all data + // from input to output. + output->PushBackInterleaved(input, input_length); + return kError; + } + return TimeStretch::Process(input, input_length, fast_accelerate, output, + length_change_samples); +} + +void Accelerate::SetParametersForPassiveSpeech(size_t /*len*/, + int16_t* best_correlation, + size_t* /*peak_index*/) const { + // When the signal does not contain any active speech, the correlation does + // not matter. Simply set it to zero. + *best_correlation = 0; +} + +Accelerate::ReturnCodes Accelerate::CheckCriteriaAndStretch( + const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool fast_mode, + AudioMultiVector* output) const { + // Check for strong correlation or passive speech. + // Use 8192 (0.5 in Q14) in fast mode. + const int correlation_threshold = fast_mode ? 8192 : kCorrelationThreshold; + if ((best_correlation > correlation_threshold) || !active_speech) { + // Do accelerate operation by overlap add. + + // Pre-calculate common multiplication with |fs_mult_|. + // 120 corresponds to 15 ms. + size_t fs_mult_120 = fs_mult_ * 120; + + if (fast_mode) { + // Fit as many multiples of |peak_index| as possible in fs_mult_120. + // TODO(henrik.lundin) Consider finding multiple correlation peaks and + // pick the one with the longest correlation lag in this case. + peak_index = (fs_mult_120 / peak_index) * peak_index; + } + + assert(fs_mult_120 >= peak_index); // Should be handled in Process(). + // Copy first part; 0 to 15 ms. + output->PushBackInterleaved(input, fs_mult_120 * num_channels_); + // Copy the |peak_index| starting at 15 ms to |temp_vector|. + AudioMultiVector temp_vector(num_channels_); + temp_vector.PushBackInterleaved(&input[fs_mult_120 * num_channels_], + peak_index * num_channels_); + // Cross-fade |temp_vector| onto the end of |output|. + output->CrossFade(temp_vector, peak_index); + // Copy the last unmodified part, 15 ms + pitch period until the end. + output->PushBackInterleaved( + &input[(fs_mult_120 + peak_index) * num_channels_], + input_length - (fs_mult_120 + peak_index) * num_channels_); + + if (active_speech) { + return kSuccess; + } else { + return kSuccessLowEnergy; + } + } else { + // Accelerate not allowed. Simply move all data from decoded to outData. + output->PushBackInterleaved(input, input_length); + return kNoStretch; + } +} + +Accelerate* AccelerateFactory::Create( + int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise) const { + return new Accelerate(sample_rate_hz, num_channels, background_noise); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.h new file mode 100644 index 0000000000..bf4f0f7266 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_ACCELERATE_H_ +#define MODULES_AUDIO_CODING_NETEQ_ACCELERATE_H_ + +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/time_stretch.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class BackgroundNoise; + +// This class implements the Accelerate operation. Most of the work is done +// in the base class TimeStretch, which is shared with the PreemptiveExpand +// operation. In the Accelerate class, the operations that are specific to +// Accelerate are implemented. +class Accelerate : public TimeStretch { + public: + Accelerate(int sample_rate_hz, size_t num_channels, + const BackgroundNoise& background_noise) + : TimeStretch(sample_rate_hz, num_channels, background_noise) { + } + + // This method performs the actual Accelerate operation. The samples are + // read from |input|, of length |input_length| elements, and are written to + // |output|. The number of samples removed through time-stretching is + // is provided in the output |length_change_samples|. The method returns + // the outcome of the operation as an enumerator value. If |fast_accelerate| + // is true, the algorithm will relax the requirements on finding strong + // correlations, and may remove multiple pitch periods if possible. + ReturnCodes Process(const int16_t* input, + size_t input_length, + bool fast_accelerate, + AudioMultiVector* output, + size_t* length_change_samples); + + protected: + // Sets the parameters |best_correlation| and |peak_index| to suitable + // values when the signal contains no active speech. + void SetParametersForPassiveSpeech(size_t len, + int16_t* best_correlation, + size_t* peak_index) const override; + + // Checks the criteria for performing the time-stretching operation and, + // if possible, performs the time-stretching. + ReturnCodes CheckCriteriaAndStretch(const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool fast_mode, + AudioMultiVector* output) const override; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(Accelerate); +}; + +struct AccelerateFactory { + AccelerateFactory() {} + virtual ~AccelerateFactory() {} + + virtual Accelerate* Create(int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise) const; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_ACCELERATE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc new file mode 100644 index 0000000000..3181d6fc1c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include +#include +#include + +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "modules/audio_coding/codecs/g722/audio_decoder_g722.h" +#include "modules/audio_coding/codecs/g722/audio_encoder_g722.h" +#include "modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#include "modules/audio_coding/codecs/isac/fix/include/audio_decoder_isacfix.h" +#include "modules/audio_coding/codecs/isac/fix/include/audio_encoder_isacfix.h" +#include "modules/audio_coding/codecs/isac/main/include/audio_decoder_isac.h" +#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h" +#include "modules/audio_coding/codecs/opus/audio_decoder_opus.h" +#include "modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h" +#include "modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h" +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +namespace { +// The absolute difference between the input and output (the first channel) is +// compared vs |tolerance|. The parameter |delay| is used to correct for codec +// delays. +void CompareInputOutput(const std::vector& input, + const std::vector& output, + size_t num_samples, + size_t channels, + int tolerance, + int delay) { + ASSERT_LE(num_samples, input.size()); + ASSERT_LE(num_samples * channels, output.size()); + for (unsigned int n = 0; n < num_samples - delay; ++n) { + ASSERT_NEAR(input[n], output[channels * n + delay], tolerance) + << "Exit test on first diff; n = " << n; + } +} + +// The absolute difference between the first two channels in |output| is +// compared vs |tolerance|. +void CompareTwoChannels(const std::vector& output, + size_t samples_per_channel, + size_t channels, + int tolerance) { + ASSERT_GE(channels, 2u); + ASSERT_LE(samples_per_channel * channels, output.size()); + for (unsigned int n = 0; n < samples_per_channel; ++n) + ASSERT_NEAR(output[channels * n], output[channels * n + 1], tolerance) + << "Stereo samples differ."; +} + +// Calculates mean-squared error between input and output (the first channel). +// The parameter |delay| is used to correct for codec delays. +double MseInputOutput(const std::vector& input, + const std::vector& output, + size_t num_samples, + size_t channels, + int delay) { + assert(delay < static_cast(num_samples)); + assert(num_samples <= input.size()); + assert(num_samples * channels <= output.size()); + if (num_samples == 0) + return 0.0; + double squared_sum = 0.0; + for (unsigned int n = 0; n < num_samples - delay; ++n) { + squared_sum += (input[n] - output[channels * n + delay]) * + (input[n] - output[channels * n + delay]); + } + return squared_sum / (num_samples - delay); +} +} // namespace + +class AudioDecoderTest : public ::testing::Test { + protected: + AudioDecoderTest() + : input_audio_( + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000), + codec_input_rate_hz_(32000), // Legacy default value. + frame_size_(0), + data_length_(0), + channels_(1), + payload_type_(17), + decoder_(NULL) {} + + virtual ~AudioDecoderTest() {} + + virtual void SetUp() { + if (audio_encoder_) + codec_input_rate_hz_ = audio_encoder_->SampleRateHz(); + // Create arrays. + ASSERT_GT(data_length_, 0u) << "The test must set data_length_ > 0"; + } + + virtual void TearDown() { + delete decoder_; + decoder_ = NULL; + } + + virtual void InitEncoder() { } + + // TODO(henrik.lundin) Change return type to size_t once most/all overriding + // implementations are gone. + virtual int EncodeFrame(const int16_t* input, + size_t input_len_samples, + rtc::Buffer* output) { + AudioEncoder::EncodedInfo encoded_info; + const size_t samples_per_10ms = audio_encoder_->SampleRateHz() / 100; + RTC_CHECK_EQ(samples_per_10ms * audio_encoder_->Num10MsFramesInNextPacket(), + input_len_samples); + std::unique_ptr interleaved_input( + new int16_t[channels_ * samples_per_10ms]); + for (size_t i = 0; i < audio_encoder_->Num10MsFramesInNextPacket(); ++i) { + EXPECT_EQ(0u, encoded_info.encoded_bytes); + + // Duplicate the mono input signal to however many channels the test + // wants. + test::InputAudioFile::DuplicateInterleaved(input + i * samples_per_10ms, + samples_per_10ms, channels_, + interleaved_input.get()); + + encoded_info = audio_encoder_->Encode( + 0, rtc::ArrayView(interleaved_input.get(), + audio_encoder_->NumChannels() * + audio_encoder_->SampleRateHz() / + 100), + output); + } + EXPECT_EQ(payload_type_, encoded_info.payload_type); + return static_cast(encoded_info.encoded_bytes); + } + + // Encodes and decodes audio. The absolute difference between the input and + // output is compared vs |tolerance|, and the mean-squared error is compared + // with |mse|. The encoded stream should contain |expected_bytes|. For stereo + // audio, the absolute difference between the two channels is compared vs + // |channel_diff_tolerance|. + void EncodeDecodeTest(size_t expected_bytes, int tolerance, double mse, + int delay = 0, int channel_diff_tolerance = 0) { + ASSERT_GE(tolerance, 0) << "Test must define a tolerance >= 0"; + ASSERT_GE(channel_diff_tolerance, 0) << + "Test must define a channel_diff_tolerance >= 0"; + size_t processed_samples = 0u; + rtc::Buffer encoded; + size_t encoded_bytes = 0u; + InitEncoder(); + std::vector input; + std::vector decoded; + while (processed_samples + frame_size_ <= data_length_) { + // Extend input vector with |frame_size_|. + input.resize(input.size() + frame_size_, 0); + // Read from input file. + ASSERT_GE(input.size() - processed_samples, frame_size_); + ASSERT_TRUE(input_audio_.Read( + frame_size_, codec_input_rate_hz_, &input[processed_samples])); + size_t enc_len = EncodeFrame( + &input[processed_samples], frame_size_, &encoded); + // Make sure that frame_size_ * channels_ samples are allocated and free. + decoded.resize((processed_samples + frame_size_) * channels_, 0); + AudioDecoder::SpeechType speech_type; + size_t dec_len = decoder_->Decode( + &encoded.data()[encoded_bytes], enc_len, codec_input_rate_hz_, + frame_size_ * channels_ * sizeof(int16_t), + &decoded[processed_samples * channels_], &speech_type); + EXPECT_EQ(frame_size_ * channels_, dec_len); + encoded_bytes += enc_len; + processed_samples += frame_size_; + } + // For some codecs it doesn't make sense to check expected number of bytes, + // since the number can vary for different platforms. Opus and iSAC are + // such codecs. In this case expected_bytes is set to 0. + if (expected_bytes) { + EXPECT_EQ(expected_bytes, encoded_bytes); + } + CompareInputOutput( + input, decoded, processed_samples, channels_, tolerance, delay); + if (channels_ == 2) + CompareTwoChannels( + decoded, processed_samples, channels_, channel_diff_tolerance); + EXPECT_LE( + MseInputOutput(input, decoded, processed_samples, channels_, delay), + mse); + } + + // Encodes a payload and decodes it twice with decoder re-init before each + // decode. Verifies that the decoded result is the same. + void ReInitTest() { + InitEncoder(); + std::unique_ptr input(new int16_t[frame_size_]); + ASSERT_TRUE( + input_audio_.Read(frame_size_, codec_input_rate_hz_, input.get())); + rtc::Buffer encoded; + size_t enc_len = EncodeFrame(input.get(), frame_size_, &encoded); + size_t dec_len; + AudioDecoder::SpeechType speech_type1, speech_type2; + decoder_->Reset(); + std::unique_ptr output1(new int16_t[frame_size_ * channels_]); + dec_len = decoder_->Decode(encoded.data(), enc_len, codec_input_rate_hz_, + frame_size_ * channels_ * sizeof(int16_t), + output1.get(), &speech_type1); + ASSERT_LE(dec_len, frame_size_ * channels_); + EXPECT_EQ(frame_size_ * channels_, dec_len); + // Re-init decoder and decode again. + decoder_->Reset(); + std::unique_ptr output2(new int16_t[frame_size_ * channels_]); + dec_len = decoder_->Decode(encoded.data(), enc_len, codec_input_rate_hz_, + frame_size_ * channels_ * sizeof(int16_t), + output2.get(), &speech_type2); + ASSERT_LE(dec_len, frame_size_ * channels_); + EXPECT_EQ(frame_size_ * channels_, dec_len); + for (unsigned int n = 0; n < frame_size_; ++n) { + ASSERT_EQ(output1[n], output2[n]) << "Exit test on first diff; n = " << n; + } + EXPECT_EQ(speech_type1, speech_type2); + } + + // Call DecodePlc and verify that the correct number of samples is produced. + void DecodePlcTest() { + InitEncoder(); + std::unique_ptr input(new int16_t[frame_size_]); + ASSERT_TRUE( + input_audio_.Read(frame_size_, codec_input_rate_hz_, input.get())); + rtc::Buffer encoded; + size_t enc_len = EncodeFrame(input.get(), frame_size_, &encoded); + AudioDecoder::SpeechType speech_type; + decoder_->Reset(); + std::unique_ptr output(new int16_t[frame_size_ * channels_]); + size_t dec_len = decoder_->Decode(encoded.data(), enc_len, + codec_input_rate_hz_, + frame_size_ * channels_ * sizeof(int16_t), + output.get(), &speech_type); + EXPECT_EQ(frame_size_ * channels_, dec_len); + // Call DecodePlc and verify that we get one frame of data. + // (Overwrite the output from the above Decode call, but that does not + // matter.) + dec_len = decoder_->DecodePlc(1, output.get()); + EXPECT_EQ(frame_size_ * channels_, dec_len); + } + + test::ResampleInputAudioFile input_audio_; + int codec_input_rate_hz_; + size_t frame_size_; + size_t data_length_; + size_t channels_; + const int payload_type_; + AudioDecoder* decoder_; + std::unique_ptr audio_encoder_; +}; + +class AudioDecoderPcmUTest : public AudioDecoderTest { + protected: + AudioDecoderPcmUTest() : AudioDecoderTest() { + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderPcmU(1); + AudioEncoderPcmU::Config config; + config.frame_size_ms = static_cast(frame_size_ / 8); + config.payload_type = payload_type_; + audio_encoder_.reset(new AudioEncoderPcmU(config)); + } +}; + +class AudioDecoderPcmATest : public AudioDecoderTest { + protected: + AudioDecoderPcmATest() : AudioDecoderTest() { + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderPcmA(1); + AudioEncoderPcmA::Config config; + config.frame_size_ms = static_cast(frame_size_ / 8); + config.payload_type = payload_type_; + audio_encoder_.reset(new AudioEncoderPcmA(config)); + } +}; + +class AudioDecoderPcm16BTest : public AudioDecoderTest { + protected: + AudioDecoderPcm16BTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 16000; + frame_size_ = 20 * codec_input_rate_hz_ / 1000; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderPcm16B(codec_input_rate_hz_, 1); + assert(decoder_); + AudioEncoderPcm16B::Config config; + config.sample_rate_hz = codec_input_rate_hz_; + config.frame_size_ms = + static_cast(frame_size_ / (config.sample_rate_hz / 1000)); + config.payload_type = payload_type_; + audio_encoder_.reset(new AudioEncoderPcm16B(config)); + } +}; + +class AudioDecoderIlbcTest : public AudioDecoderTest { + protected: + AudioDecoderIlbcTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 8000; + frame_size_ = 240; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderIlbcImpl; + assert(decoder_); + AudioEncoderIlbcConfig config; + config.frame_size_ms = 30; + audio_encoder_.reset(new AudioEncoderIlbcImpl(config, payload_type_)); + } + + // Overload the default test since iLBC's function WebRtcIlbcfix_NetEqPlc does + // not return any data. It simply resets a few states and returns 0. + void DecodePlcTest() { + InitEncoder(); + std::unique_ptr input(new int16_t[frame_size_]); + ASSERT_TRUE( + input_audio_.Read(frame_size_, codec_input_rate_hz_, input.get())); + rtc::Buffer encoded; + size_t enc_len = EncodeFrame(input.get(), frame_size_, &encoded); + AudioDecoder::SpeechType speech_type; + decoder_->Reset(); + std::unique_ptr output(new int16_t[frame_size_ * channels_]); + size_t dec_len = decoder_->Decode(encoded.data(), enc_len, + codec_input_rate_hz_, + frame_size_ * channels_ * sizeof(int16_t), + output.get(), &speech_type); + EXPECT_EQ(frame_size_, dec_len); + // Simply call DecodePlc and verify that we get 0 as return value. + EXPECT_EQ(0U, decoder_->DecodePlc(1, output.get())); + } +}; + +class AudioDecoderIsacFloatTest : public AudioDecoderTest { + protected: + AudioDecoderIsacFloatTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 16000; + frame_size_ = 480; + data_length_ = 10 * frame_size_; + AudioEncoderIsacFloatImpl::Config config; + config.payload_type = payload_type_; + config.sample_rate_hz = codec_input_rate_hz_; + config.adaptive_mode = false; + config.frame_size_ms = + 1000 * static_cast(frame_size_) / codec_input_rate_hz_; + audio_encoder_.reset(new AudioEncoderIsacFloatImpl(config)); + decoder_ = new AudioDecoderIsacFloatImpl(codec_input_rate_hz_); + } +}; + +class AudioDecoderIsacSwbTest : public AudioDecoderTest { + protected: + AudioDecoderIsacSwbTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 32000; + frame_size_ = 960; + data_length_ = 10 * frame_size_; + AudioEncoderIsacFloatImpl::Config config; + config.payload_type = payload_type_; + config.sample_rate_hz = codec_input_rate_hz_; + config.adaptive_mode = false; + config.frame_size_ms = + 1000 * static_cast(frame_size_) / codec_input_rate_hz_; + audio_encoder_.reset(new AudioEncoderIsacFloatImpl(config)); + decoder_ = new AudioDecoderIsacFloatImpl(codec_input_rate_hz_); + } +}; + +class AudioDecoderIsacFixTest : public AudioDecoderTest { + protected: + AudioDecoderIsacFixTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 16000; + frame_size_ = 480; + data_length_ = 10 * frame_size_; + AudioEncoderIsacFixImpl::Config config; + config.payload_type = payload_type_; + config.sample_rate_hz = codec_input_rate_hz_; + config.adaptive_mode = false; + config.frame_size_ms = + 1000 * static_cast(frame_size_) / codec_input_rate_hz_; + audio_encoder_.reset(new AudioEncoderIsacFixImpl(config)); + decoder_ = new AudioDecoderIsacFixImpl(codec_input_rate_hz_); + } +}; + +class AudioDecoderG722Test : public AudioDecoderTest { + protected: + AudioDecoderG722Test() : AudioDecoderTest() { + codec_input_rate_hz_ = 16000; + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderG722Impl; + assert(decoder_); + AudioEncoderG722Config config; + config.frame_size_ms = 10; + config.num_channels = 1; + audio_encoder_.reset(new AudioEncoderG722Impl(config, payload_type_)); + } +}; + +class AudioDecoderG722StereoTest : public AudioDecoderTest { + protected: + AudioDecoderG722StereoTest() : AudioDecoderTest() { + channels_ = 2; + codec_input_rate_hz_ = 16000; + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderG722StereoImpl; + assert(decoder_); + AudioEncoderG722Config config; + config.frame_size_ms = 10; + config.num_channels = 2; + audio_encoder_.reset(new AudioEncoderG722Impl(config, payload_type_)); + } +}; + +class AudioDecoderOpusTest : public AudioDecoderTest { + protected: + AudioDecoderOpusTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 48000; + frame_size_ = 480; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderOpusImpl(1); + AudioEncoderOpusConfig config; + config.frame_size_ms = static_cast(frame_size_) / 48; + config.application = AudioEncoderOpusConfig::ApplicationMode::kVoip; + audio_encoder_ = AudioEncoderOpus::MakeAudioEncoder(config, payload_type_); + } +}; + +class AudioDecoderOpusStereoTest : public AudioDecoderOpusTest { + protected: + AudioDecoderOpusStereoTest() : AudioDecoderOpusTest() { + channels_ = 2; + delete decoder_; + decoder_ = new AudioDecoderOpusImpl(2); + AudioEncoderOpusConfig config; + config.frame_size_ms = static_cast(frame_size_) / 48; + config.num_channels = 2; + config.application = AudioEncoderOpusConfig::ApplicationMode::kAudio; + audio_encoder_ = AudioEncoderOpus::MakeAudioEncoder(config, payload_type_); + } +}; + +TEST_F(AudioDecoderPcmUTest, EncodeDecode) { + int tolerance = 251; + double mse = 1734.0; + EncodeDecodeTest(data_length_, tolerance, mse); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +namespace { +int SetAndGetTargetBitrate(AudioEncoder* audio_encoder, int rate) { + audio_encoder->OnReceivedUplinkBandwidth(rate, rtc::nullopt); + return audio_encoder->GetTargetBitrate(); +} +void TestSetAndGetTargetBitratesWithFixedCodec(AudioEncoder* audio_encoder, + int fixed_rate) { + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, 32000)); + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, fixed_rate - 1)); + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, fixed_rate)); + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, fixed_rate + 1)); +} +} // namespace + +TEST_F(AudioDecoderPcmUTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 64000); +} + +TEST_F(AudioDecoderPcmATest, EncodeDecode) { + int tolerance = 308; + double mse = 1931.0; + EncodeDecodeTest(data_length_, tolerance, mse); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderPcmATest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 64000); +} + +TEST_F(AudioDecoderPcm16BTest, EncodeDecode) { + int tolerance = 0; + double mse = 0.0; + EncodeDecodeTest(2 * data_length_, tolerance, mse); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderPcm16BTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), + codec_input_rate_hz_ * 16); +} + +TEST_F(AudioDecoderIlbcTest, EncodeDecode) { + int tolerance = 6808; + double mse = 2.13e6; + int delay = 80; // Delay from input to output. + EncodeDecodeTest(500, tolerance, mse, delay); + ReInitTest(); + EXPECT_TRUE(decoder_->HasDecodePlc()); + DecodePlcTest(); +} + +TEST_F(AudioDecoderIlbcTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 13333); +} + +TEST_F(AudioDecoderIsacFloatTest, EncodeDecode) { + int tolerance = 3399; + double mse = 434951.0; + int delay = 48; // Delay from input to output. + EncodeDecodeTest(0, tolerance, mse, delay); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderIsacFloatTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 32000); +} + +TEST_F(AudioDecoderIsacSwbTest, EncodeDecode) { + int tolerance = 19757; + double mse = 8.18e6; + int delay = 160; // Delay from input to output. + EncodeDecodeTest(0, tolerance, mse, delay); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderIsacSwbTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 32000); +} + +TEST_F(AudioDecoderIsacFixTest, EncodeDecode) { + int tolerance = 11034; + double mse = 3.46e6; + int delay = 54; // Delay from input to output. +#if defined(WEBRTC_ANDROID) && defined(WEBRTC_ARCH_ARM) + static const int kEncodedBytes = 685; +#elif defined(WEBRTC_ANDROID) && defined(WEBRTC_ARCH_ARM64) + static const int kEncodedBytes = 673; +#else + static const int kEncodedBytes = 671; +#endif + EncodeDecodeTest(kEncodedBytes, tolerance, mse, delay); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderIsacFixTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 32000); +} + +TEST_F(AudioDecoderG722Test, EncodeDecode) { + int tolerance = 6176; + double mse = 238630.0; + int delay = 22; // Delay from input to output. + EncodeDecodeTest(data_length_ / 2, tolerance, mse, delay); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderG722Test, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 64000); +} + +TEST_F(AudioDecoderG722StereoTest, EncodeDecode) { + int tolerance = 6176; + int channel_diff_tolerance = 0; + double mse = 238630.0; + int delay = 22; // Delay from input to output. + EncodeDecodeTest(data_length_, tolerance, mse, delay, channel_diff_tolerance); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderG722StereoTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 128000); +} + +TEST_F(AudioDecoderOpusTest, EncodeDecode) { + int tolerance = 6176; + double mse = 238630.0; + int delay = 22; // Delay from input to output. + EncodeDecodeTest(0, tolerance, mse, delay); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +namespace { +void TestOpusSetTargetBitrates(AudioEncoder* audio_encoder) { + EXPECT_EQ(6000, SetAndGetTargetBitrate(audio_encoder, 5999)); + EXPECT_EQ(6000, SetAndGetTargetBitrate(audio_encoder, 6000)); + EXPECT_EQ(32000, SetAndGetTargetBitrate(audio_encoder, 32000)); + EXPECT_EQ(510000, SetAndGetTargetBitrate(audio_encoder, 510000)); + EXPECT_EQ(510000, SetAndGetTargetBitrate(audio_encoder, 511000)); +} +} // namespace + +TEST_F(AudioDecoderOpusTest, SetTargetBitrate) { + TestOpusSetTargetBitrates(audio_encoder_.get()); +} + +TEST_F(AudioDecoderOpusStereoTest, EncodeDecode) { + int tolerance = 6176; + int channel_diff_tolerance = 0; + double mse = 238630.0; + int delay = 22; // Delay from input to output. + EncodeDecodeTest(0, tolerance, mse, delay, channel_diff_tolerance); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderOpusStereoTest, SetTargetBitrate) { + TestOpusSetTargetBitrates(audio_encoder_.get()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.cc new file mode 100644 index 0000000000..c3e623fdde --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.cc @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" + +#include + +#include + +#include "rtc_base/checks.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +AudioMultiVector::AudioMultiVector(size_t N) { + assert(N > 0); + if (N < 1) N = 1; + for (size_t n = 0; n < N; ++n) { + channels_.push_back(new AudioVector); + } + num_channels_ = N; +} + +AudioMultiVector::AudioMultiVector(size_t N, size_t initial_size) { + assert(N > 0); + if (N < 1) N = 1; + for (size_t n = 0; n < N; ++n) { + channels_.push_back(new AudioVector(initial_size)); + } + num_channels_ = N; +} + +AudioMultiVector::~AudioMultiVector() { + std::vector::iterator it = channels_.begin(); + while (it != channels_.end()) { + delete (*it); + ++it; + } +} + +void AudioMultiVector::Clear() { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->Clear(); + } +} + +void AudioMultiVector::Zeros(size_t length) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->Clear(); + channels_[i]->Extend(length); + } +} + +void AudioMultiVector::CopyTo(AudioMultiVector* copy_to) const { + if (copy_to) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->CopyTo(&(*copy_to)[i]); + } + } +} + +void AudioMultiVector::PushBackInterleaved(const int16_t* append_this, + size_t length) { + assert(length % num_channels_ == 0); + if (num_channels_ == 1) { + // Special case to avoid extra allocation and data shuffling. + channels_[0]->PushBack(append_this, length); + return; + } + size_t length_per_channel = length / num_channels_; + int16_t* temp_array = new int16_t[length_per_channel]; // Temporary storage. + for (size_t channel = 0; channel < num_channels_; ++channel) { + // Copy elements to |temp_array|. + // Set |source_ptr| to first element of this channel. + const int16_t* source_ptr = &append_this[channel]; + for (size_t i = 0; i < length_per_channel; ++i) { + temp_array[i] = *source_ptr; + source_ptr += num_channels_; // Jump to next element of this channel. + } + channels_[channel]->PushBack(temp_array, length_per_channel); + } + delete [] temp_array; +} + +void AudioMultiVector::PushBack(const AudioMultiVector& append_this) { + assert(num_channels_ == append_this.num_channels_); + if (num_channels_ == append_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PushBack(append_this[i]); + } + } +} + +void AudioMultiVector::PushBackFromIndex(const AudioMultiVector& append_this, + size_t index) { + assert(index < append_this.Size()); + index = std::min(index, append_this.Size() - 1); + size_t length = append_this.Size() - index; + assert(num_channels_ == append_this.num_channels_); + if (num_channels_ == append_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PushBack(append_this[i], length, index); + } + } +} + +void AudioMultiVector::PopFront(size_t length) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PopFront(length); + } +} + +void AudioMultiVector::PopBack(size_t length) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PopBack(length); + } +} + +size_t AudioMultiVector::ReadInterleaved(size_t length, + int16_t* destination) const { + return ReadInterleavedFromIndex(0, length, destination); +} + +size_t AudioMultiVector::ReadInterleavedFromIndex(size_t start_index, + size_t length, + int16_t* destination) const { + RTC_DCHECK(destination); + size_t index = 0; // Number of elements written to |destination| so far. + RTC_DCHECK_LE(start_index, Size()); + start_index = std::min(start_index, Size()); + if (length + start_index > Size()) { + length = Size() - start_index; + } + if (num_channels_ == 1) { + // Special case to avoid the nested for loop below. + (*this)[0].CopyTo(length, start_index, destination); + return length; + } + for (size_t i = 0; i < length; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + destination[index] = (*this)[channel][i + start_index]; + ++index; + } + } + return index; +} + +size_t AudioMultiVector::ReadInterleavedFromEnd(size_t length, + int16_t* destination) const { + length = std::min(length, Size()); // Cannot read more than Size() elements. + return ReadInterleavedFromIndex(Size() - length, length, destination); +} + +void AudioMultiVector::OverwriteAt(const AudioMultiVector& insert_this, + size_t length, + size_t position) { + assert(num_channels_ == insert_this.num_channels_); + // Cap |length| at the length of |insert_this|. + assert(length <= insert_this.Size()); + length = std::min(length, insert_this.Size()); + if (num_channels_ == insert_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->OverwriteAt(insert_this[i], length, position); + } + } +} + +void AudioMultiVector::CrossFade(const AudioMultiVector& append_this, + size_t fade_length) { + assert(num_channels_ == append_this.num_channels_); + if (num_channels_ == append_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->CrossFade(append_this[i], fade_length); + } + } +} + +size_t AudioMultiVector::Channels() const { + return num_channels_; +} + +size_t AudioMultiVector::Size() const { + assert(channels_[0]); + return channels_[0]->Size(); +} + +void AudioMultiVector::AssertSize(size_t required_size) { + if (Size() < required_size) { + size_t extend_length = required_size - Size(); + for (size_t channel = 0; channel < num_channels_; ++channel) { + channels_[channel]->Extend(extend_length); + } + } +} + +bool AudioMultiVector::Empty() const { + assert(channels_[0]); + return channels_[0]->Empty(); +} + +void AudioMultiVector::CopyChannel(size_t from_channel, size_t to_channel) { + assert(from_channel < num_channels_); + assert(to_channel < num_channels_); + channels_[from_channel]->CopyTo(channels_[to_channel]); +} + +const AudioVector& AudioMultiVector::operator[](size_t index) const { + return *(channels_[index]); +} + +AudioVector& AudioMultiVector::operator[](size_t index) { + return *(channels_[index]); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.h new file mode 100644 index 0000000000..f54c98b942 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_AUDIO_MULTI_VECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_AUDIO_MULTI_VECTOR_H_ + +#include // Access to size_t. + +#include + +#include "modules/audio_coding/neteq/audio_vector.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioMultiVector { + public: + // Creates an empty AudioMultiVector with |N| audio channels. |N| must be + // larger than 0. + explicit AudioMultiVector(size_t N); + + // Creates an AudioMultiVector with |N| audio channels, each channel having + // an initial size. |N| must be larger than 0. + AudioMultiVector(size_t N, size_t initial_size); + + virtual ~AudioMultiVector(); + + // Deletes all values and make the vector empty. + virtual void Clear(); + + // Clears the vector and inserts |length| zeros into each channel. + virtual void Zeros(size_t length); + + // Copies all values from this vector to |copy_to|. Any contents in |copy_to| + // are deleted. After the operation is done, |copy_to| will be an exact + // replica of this object. The source and the destination must have the same + // number of channels. + virtual void CopyTo(AudioMultiVector* copy_to) const; + + // Appends the contents of array |append_this| to the end of this + // object. The array is assumed to be channel-interleaved. |length| must be + // an even multiple of this object's number of channels. + // The length of this object is increased with the |length| divided by the + // number of channels. + virtual void PushBackInterleaved(const int16_t* append_this, size_t length); + + // Appends the contents of AudioMultiVector |append_this| to this object. The + // length of this object is increased with the length of |append_this|. + virtual void PushBack(const AudioMultiVector& append_this); + + // Appends the contents of AudioMultiVector |append_this| to this object, + // taken from |index| up until the end of |append_this|. The length of this + // object is increased. + virtual void PushBackFromIndex(const AudioMultiVector& append_this, + size_t index); + + // Removes |length| elements from the beginning of this object, from each + // channel. + virtual void PopFront(size_t length); + + // Removes |length| elements from the end of this object, from each + // channel. + virtual void PopBack(size_t length); + + // Reads |length| samples from each channel and writes them interleaved to + // |destination|. The total number of elements written to |destination| is + // returned, i.e., |length| * number of channels. If the AudioMultiVector + // contains less than |length| samples per channel, this is reflected in the + // return value. + virtual size_t ReadInterleaved(size_t length, int16_t* destination) const; + + // Like ReadInterleaved() above, but reads from |start_index| instead of from + // the beginning. + virtual size_t ReadInterleavedFromIndex(size_t start_index, + size_t length, + int16_t* destination) const; + + // Like ReadInterleaved() above, but reads from the end instead of from + // the beginning. + virtual size_t ReadInterleavedFromEnd(size_t length, + int16_t* destination) const; + + // Overwrites each channel in this AudioMultiVector with values taken from + // |insert_this|. The values are taken from the beginning of |insert_this| and + // are inserted starting at |position|. |length| values are written into each + // channel. If |length| and |position| are selected such that the new data + // extends beyond the end of the current AudioVector, the vector is extended + // to accommodate the new data. |length| is limited to the length of + // |insert_this|. + virtual void OverwriteAt(const AudioMultiVector& insert_this, + size_t length, + size_t position); + + // Appends |append_this| to the end of the current vector. Lets the two + // vectors overlap by |fade_length| samples (per channel), and cross-fade + // linearly in this region. + virtual void CrossFade(const AudioMultiVector& append_this, + size_t fade_length); + + // Returns the number of channels. + virtual size_t Channels() const; + + // Returns the number of elements per channel in this AudioMultiVector. + virtual size_t Size() const; + + // Verify that each channel can hold at least |required_size| elements. If + // not, extend accordingly. + virtual void AssertSize(size_t required_size); + + virtual bool Empty() const; + + // Copies the data between two channels in the AudioMultiVector. The method + // does not add any new channel. Thus, |from_channel| and |to_channel| must + // both be valid channel numbers. + virtual void CopyChannel(size_t from_channel, size_t to_channel); + + // Accesses and modifies a channel (i.e., an AudioVector object) of this + // AudioMultiVector. + const AudioVector& operator[](size_t index) const; + AudioVector& operator[](size_t index); + + protected: + std::vector channels_; + size_t num_channels_; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(AudioMultiVector); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_AUDIO_MULTI_VECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc new file mode 100644 index 0000000000..f05aee026c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" + +#include +#include + +#include + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// This is a value-parameterized test. The test cases are instantiated with +// different values for the test parameter, which is used to determine the +// number of channels in the AudioMultiBuffer. Note that it is not possible +// to combine typed testing with value-parameterized testing, and since the +// tests for AudioVector already covers a number of different type parameters, +// this test focuses on testing different number of channels, and keeping the +// value type constant. + +class AudioMultiVectorTest : public ::testing::TestWithParam { + protected: + AudioMultiVectorTest() + : num_channels_(GetParam()), // Get the test parameter. + interleaved_length_(num_channels_ * array_length()) { + array_interleaved_ = new int16_t[num_channels_ * array_length()]; + } + + ~AudioMultiVectorTest() { + delete [] array_interleaved_; + } + + virtual void SetUp() { + // Populate test arrays. + for (size_t i = 0; i < array_length(); ++i) { + array_[i] = static_cast(i); + } + int16_t* ptr = array_interleaved_; + // Write 100, 101, 102, ... for first channel. + // Write 200, 201, 202, ... for second channel. + // And so on. + for (size_t i = 0; i < array_length(); ++i) { + for (size_t j = 1; j <= num_channels_; ++j) { + *ptr = rtc::checked_cast(j * 100 + i); + ++ptr; + } + } + } + + size_t array_length() const { + return sizeof(array_) / sizeof(array_[0]); + } + + const size_t num_channels_; + size_t interleaved_length_; + int16_t array_[10]; + int16_t* array_interleaved_; +}; + +// Create and destroy AudioMultiVector objects, both empty and with a predefined +// length. +TEST_P(AudioMultiVectorTest, CreateAndDestroy) { + AudioMultiVector vec1(num_channels_); + EXPECT_TRUE(vec1.Empty()); + EXPECT_EQ(num_channels_, vec1.Channels()); + EXPECT_EQ(0u, vec1.Size()); + + size_t initial_size = 17; + AudioMultiVector vec2(num_channels_, initial_size); + EXPECT_FALSE(vec2.Empty()); + EXPECT_EQ(num_channels_, vec2.Channels()); + EXPECT_EQ(initial_size, vec2.Size()); +} + +// Test the subscript operator [] for getting and setting. +TEST_P(AudioMultiVectorTest, SubscriptOperator) { + AudioMultiVector vec(num_channels_, array_length()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < array_length(); ++i) { + vec[channel][i] = static_cast(i); + // Make sure to use the const version. + const AudioVector& audio_vec = vec[channel]; + EXPECT_EQ(static_cast(i), audio_vec[i]); + } + } +} + +// Test the PushBackInterleaved method and the CopyFrom method. The Clear +// method is also invoked. +TEST_P(AudioMultiVectorTest, PushBackInterleavedAndCopy) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_, interleaved_length_); + AudioMultiVector vec_copy(num_channels_); + vec.CopyTo(&vec_copy); // Copy from |vec| to |vec_copy|. + ASSERT_EQ(num_channels_, vec.Channels()); + ASSERT_EQ(array_length(), vec.Size()); + ASSERT_EQ(num_channels_, vec_copy.Channels()); + ASSERT_EQ(array_length(), vec_copy.Size()); + for (size_t channel = 0; channel < vec.Channels(); ++channel) { + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(static_cast((channel + 1) * 100 + i), vec[channel][i]); + EXPECT_EQ(vec[channel][i], vec_copy[channel][i]); + } + } + + // Clear |vec| and verify that it is empty. + vec.Clear(); + EXPECT_TRUE(vec.Empty()); + + // Now copy the empty vector and verify that the copy becomes empty too. + vec.CopyTo(&vec_copy); + EXPECT_TRUE(vec_copy.Empty()); +} + +// Try to copy to a NULL pointer. Nothing should happen. +TEST_P(AudioMultiVectorTest, CopyToNull) { + AudioMultiVector vec(num_channels_); + AudioMultiVector* vec_copy = NULL; + vec.PushBackInterleaved(array_interleaved_, interleaved_length_); + vec.CopyTo(vec_copy); +} + +// Test the PushBack method with another AudioMultiVector as input argument. +TEST_P(AudioMultiVectorTest, PushBackVector) { + AudioMultiVector vec1(num_channels_, array_length()); + AudioMultiVector vec2(num_channels_, array_length()); + // Set the first vector to [0, 1, ..., array_length() - 1] + + // 100 * channel_number. + // Set the second vector to [array_length(), array_length() + 1, ..., + // 2 * array_length() - 1] + 100 * channel_number. + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < array_length(); ++i) { + vec1[channel][i] = static_cast(i + 100 * channel); + vec2[channel][i] = + static_cast(i + 100 * channel + array_length()); + } + } + // Append vec2 to the back of vec1. + vec1.PushBack(vec2); + ASSERT_EQ(2u * array_length(), vec1.Size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < 2 * array_length(); ++i) { + EXPECT_EQ(static_cast(i + 100 * channel), vec1[channel][i]); + } + } +} + +// Test the PushBackFromIndex method. +TEST_P(AudioMultiVectorTest, PushBackFromIndex) { + AudioMultiVector vec1(num_channels_); + vec1.PushBackInterleaved(array_interleaved_, interleaved_length_); + AudioMultiVector vec2(num_channels_); + + // Append vec1 to the back of vec2 (which is empty). Read vec1 from the second + // last element. + vec2.PushBackFromIndex(vec1, array_length() - 2); + ASSERT_EQ(2u, vec2.Size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < 2; ++i) { + EXPECT_EQ(array_interleaved_[channel + num_channels_ * + (array_length() - 2 + i)], vec2[channel][i]); + } + } +} + +// Starts with pushing some values to the vector, then test the Zeros method. +TEST_P(AudioMultiVectorTest, Zeros) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_, interleaved_length_); + vec.Zeros(2 * array_length()); + ASSERT_EQ(num_channels_, vec.Channels()); + ASSERT_EQ(2u * array_length(), vec.Size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < 2 * array_length(); ++i) { + EXPECT_EQ(0, vec[channel][i]); + } + } +} + +// Test the ReadInterleaved method +TEST_P(AudioMultiVectorTest, ReadInterleaved) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_, interleaved_length_); + int16_t* output = new int16_t[interleaved_length_]; + // Read 5 samples. + size_t read_samples = 5; + EXPECT_EQ(num_channels_ * read_samples, + vec.ReadInterleaved(read_samples, output)); + EXPECT_EQ(0, + memcmp(array_interleaved_, output, read_samples * sizeof(int16_t))); + + // Read too many samples. Expect to get all samples from the vector. + EXPECT_EQ(interleaved_length_, + vec.ReadInterleaved(array_length() + 1, output)); + EXPECT_EQ(0, + memcmp(array_interleaved_, output, read_samples * sizeof(int16_t))); + + delete [] output; +} + +// Test the PopFront method. +TEST_P(AudioMultiVectorTest, PopFront) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_, interleaved_length_); + vec.PopFront(1); // Remove one element from each channel. + ASSERT_EQ(array_length() - 1u, vec.Size()); + // Let |ptr| point to the second element of the first channel in the + // interleaved array. + int16_t* ptr = &array_interleaved_[num_channels_]; + for (size_t i = 0; i < array_length() - 1; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + EXPECT_EQ(*ptr, vec[channel][i]); + ++ptr; + } + } + vec.PopFront(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the PopBack method. +TEST_P(AudioMultiVectorTest, PopBack) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_, interleaved_length_); + vec.PopBack(1); // Remove one element from each channel. + ASSERT_EQ(array_length() - 1u, vec.Size()); + // Let |ptr| point to the first element of the first channel in the + // interleaved array. + int16_t* ptr = array_interleaved_; + for (size_t i = 0; i < array_length() - 1; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + EXPECT_EQ(*ptr, vec[channel][i]); + ++ptr; + } + } + vec.PopBack(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the AssertSize method. +TEST_P(AudioMultiVectorTest, AssertSize) { + AudioMultiVector vec(num_channels_, array_length()); + EXPECT_EQ(array_length(), vec.Size()); + // Start with asserting with smaller sizes than already allocated. + vec.AssertSize(0); + vec.AssertSize(array_length() - 1); + // Nothing should have changed. + EXPECT_EQ(array_length(), vec.Size()); + // Assert with one element longer than already allocated. + vec.AssertSize(array_length() + 1); + // Expect vector to have grown. + EXPECT_EQ(array_length() + 1, vec.Size()); + // Also check the individual AudioVectors. + for (size_t channel = 0; channel < vec.Channels(); ++channel) { + EXPECT_EQ(array_length() + 1u, vec[channel].Size()); + } +} + +// Test the PushBack method with another AudioMultiVector as input argument. +TEST_P(AudioMultiVectorTest, OverwriteAt) { + AudioMultiVector vec1(num_channels_); + vec1.PushBackInterleaved(array_interleaved_, interleaved_length_); + AudioMultiVector vec2(num_channels_); + vec2.Zeros(3); // 3 zeros in each channel. + // Overwrite vec2 at position 5. + vec1.OverwriteAt(vec2, 3, 5); + // Verify result. + // Length remains the same. + ASSERT_EQ(array_length(), vec1.Size()); + int16_t* ptr = array_interleaved_; + for (size_t i = 0; i < array_length() - 1; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + if (i >= 5 && i <= 7) { + // Elements 5, 6, 7 should have been replaced with zeros. + EXPECT_EQ(0, vec1[channel][i]); + } else { + EXPECT_EQ(*ptr, vec1[channel][i]); + } + ++ptr; + } + } +} + +// Test the CopyChannel method, when the test is instantiated with at least two +// channels. +TEST_P(AudioMultiVectorTest, CopyChannel) { + if (num_channels_ < 2) + return; + + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_, interleaved_length_); + // Create a reference copy. + AudioMultiVector ref(num_channels_); + ref.PushBack(vec); + // Copy from first to last channel. + vec.CopyChannel(0, num_channels_ - 1); + // Verify that the first and last channels are identical; the others should + // be left untouched. + for (size_t i = 0; i < array_length(); ++i) { + // Verify that all but the last channel are untouched. + for (size_t channel = 0; channel < num_channels_ - 1; ++channel) { + EXPECT_EQ(ref[channel][i], vec[channel][i]); + } + // Verify that the last and the first channels are identical. + EXPECT_EQ(vec[0][i], vec[num_channels_ - 1][i]); + } +} + +INSTANTIATE_TEST_CASE_P(TestNumChannels, + AudioMultiVectorTest, + ::testing::Values(static_cast(1), + static_cast(2), + static_cast(5))); +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.cc new file mode 100644 index 0000000000..93cd1fb31c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.cc @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_vector.h" + +#include + +#include +#include + +#include "rtc_base/checks.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +AudioVector::AudioVector() + : AudioVector(kDefaultInitialSize) { + Clear(); +} + +AudioVector::AudioVector(size_t initial_size) + : array_(new int16_t[initial_size + 1]), + capacity_(initial_size + 1), + begin_index_(0), + end_index_(capacity_ - 1) { + memset(array_.get(), 0, capacity_ * sizeof(int16_t)); +} + +AudioVector::~AudioVector() = default; + +void AudioVector::Clear() { + end_index_ = begin_index_ = 0; +} + +void AudioVector::CopyTo(AudioVector* copy_to) const { + RTC_DCHECK(copy_to); + copy_to->Reserve(Size()); + CopyTo(Size(), 0, copy_to->array_.get()); + copy_to->begin_index_ = 0; + copy_to->end_index_ = Size(); +} + +void AudioVector::CopyTo( + size_t length, size_t position, int16_t* copy_to) const { + if (length == 0) + return; + length = std::min(length, Size() - position); + const size_t copy_index = (begin_index_ + position) % capacity_; + const size_t first_chunk_length = + std::min(length, capacity_ - copy_index); + memcpy(copy_to, &array_[copy_index], + first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(©_to[first_chunk_length], array_.get(), + remaining_length * sizeof(int16_t)); + } +} + +void AudioVector::PushFront(const AudioVector& prepend_this) { + const size_t length = prepend_this.Size(); + if (length == 0) + return; + + // Although the subsequent calling to PushFront does Reserve in it, it is + // always more efficient to do a big Reserve first. + Reserve(Size() + length); + + const size_t first_chunk_length = + std::min(length, prepend_this.capacity_ - prepend_this.begin_index_); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) + PushFront(prepend_this.array_.get(), remaining_length); + PushFront(&prepend_this.array_[prepend_this.begin_index_], + first_chunk_length); +} + +void AudioVector::PushFront(const int16_t* prepend_this, size_t length) { + if (length == 0) + return; + Reserve(Size() + length); + const size_t first_chunk_length = std::min(length, begin_index_); + memcpy(&array_[begin_index_ - first_chunk_length], + &prepend_this[length - first_chunk_length], + first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(&array_[capacity_ - remaining_length], prepend_this, + remaining_length * sizeof(int16_t)); + } + begin_index_ = (begin_index_ + capacity_ - length) % capacity_; +} + +void AudioVector::PushBack(const AudioVector& append_this) { + PushBack(append_this, append_this.Size(), 0); +} + +void AudioVector::PushBack( + const AudioVector& append_this, size_t length, size_t position) { + RTC_DCHECK_LE(position, append_this.Size()); + RTC_DCHECK_LE(length, append_this.Size() - position); + + if (length == 0) + return; + + // Although the subsequent calling to PushBack does Reserve in it, it is + // always more efficient to do a big Reserve first. + Reserve(Size() + length); + + const size_t start_index = + (append_this.begin_index_ + position) % append_this.capacity_; + const size_t first_chunk_length = std::min( + length, append_this.capacity_ - start_index); + PushBack(&append_this.array_[start_index], first_chunk_length); + + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) + PushBack(append_this.array_.get(), remaining_length); +} + +void AudioVector::PushBack(const int16_t* append_this, size_t length) { + if (length == 0) + return; + Reserve(Size() + length); + const size_t first_chunk_length = std::min(length, capacity_ - end_index_); + memcpy(&array_[end_index_], append_this, + first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(array_.get(), &append_this[first_chunk_length], + remaining_length * sizeof(int16_t)); + } + end_index_ = (end_index_ + length) % capacity_; +} + +void AudioVector::PopFront(size_t length) { + if (length == 0) + return; + length = std::min(length, Size()); + begin_index_ = (begin_index_ + length) % capacity_; +} + +void AudioVector::PopBack(size_t length) { + if (length == 0) + return; + // Never remove more than what is in the array. + length = std::min(length, Size()); + end_index_ = (end_index_ + capacity_ - length) % capacity_; +} + +void AudioVector::Extend(size_t extra_length) { + if (extra_length == 0) + return; + InsertZerosByPushBack(extra_length, Size()); +} + +void AudioVector::InsertAt(const int16_t* insert_this, + size_t length, + size_t position) { + if (length == 0) + return; + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + // When inserting to a position closer to the beginning, it is more efficient + // to insert by pushing front than to insert by pushing back, since less data + // will be moved, vice versa. + if (position <= Size() - position) { + InsertByPushFront(insert_this, length, position); + } else { + InsertByPushBack(insert_this, length, position); + } +} + +void AudioVector::InsertZerosAt(size_t length, + size_t position) { + if (length == 0) + return; + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + // When inserting to a position closer to the beginning, it is more efficient + // to insert by pushing front than to insert by pushing back, since less data + // will be moved, vice versa. + if (position <= Size() - position) { + InsertZerosByPushFront(length, position); + } else { + InsertZerosByPushBack(length, position); + } +} + +void AudioVector::OverwriteAt(const AudioVector& insert_this, + size_t length, + size_t position) { + RTC_DCHECK_LE(length, insert_this.Size()); + if (length == 0) + return; + + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + // Although the subsequent calling to OverwriteAt does Reserve in it, it is + // always more efficient to do a big Reserve first. + size_t new_size = std::max(Size(), position + length); + Reserve(new_size); + + const size_t first_chunk_length = + std::min(length, insert_this.capacity_ - insert_this.begin_index_); + OverwriteAt(&insert_this.array_[insert_this.begin_index_], first_chunk_length, + position); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + OverwriteAt(insert_this.array_.get(), remaining_length, + position + first_chunk_length); + } +} + +void AudioVector::OverwriteAt(const int16_t* insert_this, + size_t length, + size_t position) { + if (length == 0) + return; + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + size_t new_size = std::max(Size(), position + length); + Reserve(new_size); + + const size_t overwrite_index = (begin_index_ + position) % capacity_; + const size_t first_chunk_length = + std::min(length, capacity_ - overwrite_index); + memcpy(&array_[overwrite_index], insert_this, + first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(array_.get(), &insert_this[first_chunk_length], + remaining_length * sizeof(int16_t)); + } + + end_index_ = (begin_index_ + new_size) % capacity_; +} + +void AudioVector::CrossFade(const AudioVector& append_this, + size_t fade_length) { + // Fade length cannot be longer than the current vector or |append_this|. + assert(fade_length <= Size()); + assert(fade_length <= append_this.Size()); + fade_length = std::min(fade_length, Size()); + fade_length = std::min(fade_length, append_this.Size()); + size_t position = Size() - fade_length + begin_index_; + // Cross fade the overlapping regions. + // |alpha| is the mixing factor in Q14. + // TODO(hlundin): Consider skipping +1 in the denominator to produce a + // smoother cross-fade, in particular at the end of the fade. + int alpha_step = 16384 / (static_cast(fade_length) + 1); + int alpha = 16384; + for (size_t i = 0; i < fade_length; ++i) { + alpha -= alpha_step; + array_[(position + i) % capacity_] = + (alpha * array_[(position + i) % capacity_] + + (16384 - alpha) * append_this[i] + 8192) >> 14; + } + assert(alpha >= 0); // Verify that the slope was correct. + // Append what is left of |append_this|. + size_t samples_to_push_back = append_this.Size() - fade_length; + if (samples_to_push_back > 0) + PushBack(append_this, samples_to_push_back, fade_length); +} + +// Returns the number of elements in this AudioVector. +size_t AudioVector::Size() const { + return (end_index_ + capacity_ - begin_index_) % capacity_; +} + +// Returns true if this AudioVector is empty. +bool AudioVector::Empty() const { + return begin_index_ == end_index_; +} + +void AudioVector::Reserve(size_t n) { + if (capacity_ > n) + return; + const size_t length = Size(); + // Reserve one more sample to remove the ambiguity between empty vector and + // full vector. Therefore |begin_index_| == |end_index_| indicates empty + // vector, and |begin_index_| == (|end_index_| + 1) % capacity indicates + // full vector. + std::unique_ptr temp_array(new int16_t[n + 1]); + CopyTo(length, 0, temp_array.get()); + array_.swap(temp_array); + begin_index_ = 0; + end_index_ = length; + capacity_ = n + 1; +} + +void AudioVector::InsertByPushBack(const int16_t* insert_this, + size_t length, + size_t position) { + const size_t move_chunk_length = Size() - position; + std::unique_ptr temp_array(nullptr); + if (move_chunk_length > 0) { + // TODO(minyue): see if it is possible to avoid copying to a buffer. + temp_array.reset(new int16_t[move_chunk_length]); + CopyTo(move_chunk_length, position, temp_array.get()); + PopBack(move_chunk_length); + } + + Reserve(Size() + length + move_chunk_length); + PushBack(insert_this, length); + if (move_chunk_length > 0) + PushBack(temp_array.get(), move_chunk_length); +} + +void AudioVector::InsertByPushFront(const int16_t* insert_this, + size_t length, + size_t position) { + std::unique_ptr temp_array(nullptr); + if (position > 0) { + // TODO(minyue): see if it is possible to avoid copying to a buffer. + temp_array.reset(new int16_t[position]); + CopyTo(position, 0, temp_array.get()); + PopFront(position); + } + + Reserve(Size() + length + position); + PushFront(insert_this, length); + if (position > 0) + PushFront(temp_array.get(), position); +} + +void AudioVector::InsertZerosByPushBack(size_t length, + size_t position) { + const size_t move_chunk_length = Size() - position; + std::unique_ptr temp_array(nullptr); + if (move_chunk_length > 0) { + temp_array.reset(new int16_t[move_chunk_length]); + CopyTo(move_chunk_length, position, temp_array.get()); + PopBack(move_chunk_length); + } + + Reserve(Size() + length + move_chunk_length); + + const size_t first_zero_chunk_length = + std::min(length, capacity_ - end_index_); + memset(&array_[end_index_], 0, first_zero_chunk_length * sizeof(int16_t)); + const size_t remaining_zero_length = length - first_zero_chunk_length; + if (remaining_zero_length > 0) + memset(array_.get(), 0, remaining_zero_length * sizeof(int16_t)); + end_index_ = (end_index_ + length) % capacity_; + + if (move_chunk_length > 0) + PushBack(temp_array.get(), move_chunk_length); +} + +void AudioVector::InsertZerosByPushFront(size_t length, + size_t position) { + std::unique_ptr temp_array(nullptr); + if (position > 0) { + temp_array.reset(new int16_t[position]); + CopyTo(position, 0, temp_array.get()); + PopFront(position); + } + + Reserve(Size() + length + position); + + const size_t first_zero_chunk_length = std::min(length, begin_index_); + memset(&array_[begin_index_ - first_zero_chunk_length], 0, + first_zero_chunk_length * sizeof(int16_t)); + const size_t remaining_zero_length = length - first_zero_chunk_length; + if (remaining_zero_length > 0) + memset(&array_[capacity_ - remaining_zero_length], 0, + remaining_zero_length * sizeof(int16_t)); + begin_index_ = (begin_index_ + capacity_ - length) % capacity_; + + if (position > 0) + PushFront(temp_array.get(), position); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.h new file mode 100644 index 0000000000..754a9fdbe6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_AUDIO_VECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_AUDIO_VECTOR_H_ + +#include // Access to size_t. +#include + +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioVector { + public: + // Creates an empty AudioVector. + AudioVector(); + + // Creates an AudioVector with an initial size. + explicit AudioVector(size_t initial_size); + + virtual ~AudioVector(); + + // Deletes all values and make the vector empty. + virtual void Clear(); + + // Copies all values from this vector to |copy_to|. Any contents in |copy_to| + // are deleted before the copy operation. After the operation is done, + // |copy_to| will be an exact replica of this object. + virtual void CopyTo(AudioVector* copy_to) const; + + // Copies |length| values from |position| in this vector to |copy_to|. + virtual void CopyTo(size_t length, size_t position, int16_t* copy_to) const; + + // Prepends the contents of AudioVector |prepend_this| to this object. The + // length of this object is increased with the length of |prepend_this|. + virtual void PushFront(const AudioVector& prepend_this); + + // Same as above, but with an array |prepend_this| with |length| elements as + // source. + virtual void PushFront(const int16_t* prepend_this, size_t length); + + // Same as PushFront but will append to the end of this object. + virtual void PushBack(const AudioVector& append_this); + + // Appends a segment of |append_this| to the end of this object. The segment + // starts from |position| and has |length| samples. + virtual void PushBack(const AudioVector& append_this, + size_t length, + size_t position); + + // Same as PushFront but will append to the end of this object. + virtual void PushBack(const int16_t* append_this, size_t length); + + // Removes |length| elements from the beginning of this object. + virtual void PopFront(size_t length); + + // Removes |length| elements from the end of this object. + virtual void PopBack(size_t length); + + // Extends this object with |extra_length| elements at the end. The new + // elements are initialized to zero. + virtual void Extend(size_t extra_length); + + // Inserts |length| elements taken from the array |insert_this| and insert + // them at |position|. The length of the AudioVector is increased by |length|. + // |position| = 0 means that the new values are prepended to the vector. + // |position| = Size() means that the new values are appended to the vector. + virtual void InsertAt(const int16_t* insert_this, size_t length, + size_t position); + + // Like InsertAt, but inserts |length| zero elements at |position|. + virtual void InsertZerosAt(size_t length, size_t position); + + // Overwrites |length| elements of this AudioVector starting from |position| + // with first values in |AudioVector|. The definition of |position| + // is the same as for InsertAt(). If |length| and |position| are selected + // such that the new data extends beyond the end of the current AudioVector, + // the vector is extended to accommodate the new data. + virtual void OverwriteAt(const AudioVector& insert_this, + size_t length, + size_t position); + + // Overwrites |length| elements of this AudioVector with values taken from the + // array |insert_this|, starting at |position|. The definition of |position| + // is the same as for InsertAt(). If |length| and |position| are selected + // such that the new data extends beyond the end of the current AudioVector, + // the vector is extended to accommodate the new data. + virtual void OverwriteAt(const int16_t* insert_this, + size_t length, + size_t position); + + // Appends |append_this| to the end of the current vector. Lets the two + // vectors overlap by |fade_length| samples, and cross-fade linearly in this + // region. + virtual void CrossFade(const AudioVector& append_this, size_t fade_length); + + // Returns the number of elements in this AudioVector. + virtual size_t Size() const; + + // Returns true if this AudioVector is empty. + virtual bool Empty() const; + + // Accesses and modifies an element of AudioVector. + inline const int16_t& operator[](size_t index) const { + return array_[WrapIndex(index, begin_index_, capacity_)]; + } + + inline int16_t& operator[](size_t index) { + return array_[WrapIndex(index, begin_index_, capacity_)]; + } + + private: + static const size_t kDefaultInitialSize = 10; + + // This method is used by the [] operators to calculate an index within the + // capacity of the array, but without using the modulo operation (%). + static inline size_t WrapIndex(size_t index, + size_t begin_index, + size_t capacity) { + RTC_DCHECK_LT(index, capacity); + RTC_DCHECK_LT(begin_index, capacity); + size_t ix = begin_index + index; + RTC_DCHECK_GE(ix, index); // Check for overflow. + if (ix >= capacity) { + ix -= capacity; + } + RTC_DCHECK_LT(ix, capacity); + return ix; + } + + void Reserve(size_t n); + + void InsertByPushBack(const int16_t* insert_this, size_t length, + size_t position); + + void InsertByPushFront(const int16_t* insert_this, size_t length, + size_t position); + + void InsertZerosByPushBack(size_t length, size_t position); + + void InsertZerosByPushFront(size_t length, size_t position); + + std::unique_ptr array_; + + size_t capacity_; // Allocated number of samples in the array. + + // The index of the first sample in |array_|, except when + // |begin_index_ == end_index_|, which indicates an empty buffer. + size_t begin_index_; + + // The index of the sample after the last sample in |array_|. + size_t end_index_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioVector); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_AUDIO_VECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector_unittest.cc new file mode 100644 index 0000000000..1b54abca5c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector_unittest.cc @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_vector.h" + +#include +#include + +#include + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioVectorTest : public ::testing::Test { + protected: + virtual void SetUp() { + // Populate test array. + for (size_t i = 0; i < array_length(); ++i) { + array_[i] = rtc::checked_cast(i); + } + } + + size_t array_length() const { + return sizeof(array_) / sizeof(array_[0]); + } + + int16_t array_[10]; +}; + +// Create and destroy AudioVector objects, both empty and with a predefined +// length. +TEST_F(AudioVectorTest, CreateAndDestroy) { + AudioVector vec1; + EXPECT_TRUE(vec1.Empty()); + EXPECT_EQ(0u, vec1.Size()); + + size_t initial_size = 17; + AudioVector vec2(initial_size); + EXPECT_FALSE(vec2.Empty()); + EXPECT_EQ(initial_size, vec2.Size()); +} + +// Test the subscript operator [] for getting and setting. +TEST_F(AudioVectorTest, SubscriptOperator) { + AudioVector vec(array_length()); + for (size_t i = 0; i < array_length(); ++i) { + vec[i] = static_cast(i); + const int16_t& value = vec[i]; // Make sure to use the const version. + EXPECT_EQ(static_cast(i), value); + } +} + +// Test the PushBack method and the CopyFrom method. The Clear method is also +// invoked. +TEST_F(AudioVectorTest, PushBackAndCopy) { + AudioVector vec; + AudioVector vec_copy; + vec.PushBack(array_, array_length()); + vec.CopyTo(&vec_copy); // Copy from |vec| to |vec_copy|. + ASSERT_EQ(array_length(), vec.Size()); + ASSERT_EQ(array_length(), vec_copy.Size()); + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[i]); + EXPECT_EQ(array_[i], vec_copy[i]); + } + + // Clear |vec| and verify that it is empty. + vec.Clear(); + EXPECT_TRUE(vec.Empty()); + + // Now copy the empty vector and verify that the copy becomes empty too. + vec.CopyTo(&vec_copy); + EXPECT_TRUE(vec_copy.Empty()); +} + +// Test the PushBack method with another AudioVector as input argument. +TEST_F(AudioVectorTest, PushBackVector) { + static const size_t kLength = 10; + AudioVector vec1(kLength); + AudioVector vec2(kLength); + // Set the first vector to [0, 1, ..., kLength - 1]. + // Set the second vector to [kLength, kLength + 1, ..., 2 * kLength - 1]. + for (size_t i = 0; i < kLength; ++i) { + vec1[i] = static_cast(i); + vec2[i] = static_cast(i + kLength); + } + // Append vec2 to the back of vec1. + vec1.PushBack(vec2); + ASSERT_EQ(2 * kLength, vec1.Size()); + for (size_t i = 0; i < 2 * kLength; ++i) { + EXPECT_EQ(static_cast(i), vec1[i]); + } +} + +// Test the PushFront method. +TEST_F(AudioVectorTest, PushFront) { + AudioVector vec; + vec.PushFront(array_, array_length()); + ASSERT_EQ(array_length(), vec.Size()); + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[i]); + } +} + +// Test the PushFront method with another AudioVector as input argument. +TEST_F(AudioVectorTest, PushFrontVector) { + static const size_t kLength = 10; + AudioVector vec1(kLength); + AudioVector vec2(kLength); + // Set the first vector to [0, 1, ..., kLength - 1]. + // Set the second vector to [kLength, kLength + 1, ..., 2 * kLength - 1]. + for (size_t i = 0; i < kLength; ++i) { + vec1[i] = static_cast(i); + vec2[i] = static_cast(i + kLength); + } + // Prepend vec1 to the front of vec2. + vec2.PushFront(vec1); + ASSERT_EQ(2 * kLength, vec2.Size()); + for (size_t i = 0; i < 2 * kLength; ++i) { + EXPECT_EQ(static_cast(i), vec2[i]); + } +} + +// Test the PopFront method. +TEST_F(AudioVectorTest, PopFront) { + AudioVector vec; + vec.PushBack(array_, array_length()); + vec.PopFront(1); // Remove one element. + EXPECT_EQ(array_length() - 1u, vec.Size()); + for (size_t i = 0; i < array_length() - 1; ++i) { + EXPECT_EQ(static_cast(i + 1), vec[i]); + } + vec.PopFront(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the PopBack method. +TEST_F(AudioVectorTest, PopBack) { + AudioVector vec; + vec.PushBack(array_, array_length()); + vec.PopBack(1); // Remove one element. + EXPECT_EQ(array_length() - 1u, vec.Size()); + for (size_t i = 0; i < array_length() - 1; ++i) { + EXPECT_EQ(static_cast(i), vec[i]); + } + vec.PopBack(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the Extend method. +TEST_F(AudioVectorTest, Extend) { + AudioVector vec; + vec.PushBack(array_, array_length()); + vec.Extend(5); // Extend with 5 elements, which should all be zeros. + ASSERT_EQ(array_length() + 5u, vec.Size()); + // Verify that all are zero. + for (size_t i = array_length(); i < array_length() + 5; ++i) { + EXPECT_EQ(0, vec[i]); + } +} + +// Test the InsertAt method with an insert position in the middle of the vector. +TEST_F(AudioVectorTest, InsertAt) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = 5; + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, 1, ..., |insert_position| - 1, 100, 101, ..., 100 + kNewLength - 1, + // |insert_position|, |insert_position| + 1, ..., kLength - 1}. + size_t pos = 0; + for (int i = 0; i < insert_position; ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + for (size_t i = insert_position; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } +} + +// Test the InsertZerosAt method with an insert position in the middle of the +// vector. Use the InsertAt method as reference. +TEST_F(AudioVectorTest, InsertZerosAt) { + AudioVector vec; + AudioVector vec_ref; + vec.PushBack(array_, array_length()); + vec_ref.PushBack(array_, array_length()); + static const int kNewLength = 5; + int insert_position = 5; + vec.InsertZerosAt(kNewLength, insert_position); + int16_t new_array[kNewLength] = {0}; // All zero elements. + vec_ref.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vectors are identical. + ASSERT_EQ(vec_ref.Size(), vec.Size()); + for (size_t i = 0; i < vec.Size(); ++i) { + EXPECT_EQ(vec_ref[i], vec[i]); + } +} + +// Test the InsertAt method with an insert position at the start of the vector. +TEST_F(AudioVectorTest, InsertAtBeginning) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = 0; + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {100, 101, ..., 100 + kNewLength - 1, + // 0, 1, ..., kLength - 1}. + size_t pos = 0; + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + for (size_t i = insert_position; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } +} + +// Test the InsertAt method with an insert position at the end of the vector. +TEST_F(AudioVectorTest, InsertAtEnd) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = rtc::checked_cast(array_length()); + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, 1, ..., kLength - 1, 100, 101, ..., 100 + kNewLength - 1 }. + size_t pos = 0; + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } +} + +// Test the InsertAt method with an insert position beyond the end of the +// vector. Verify that a position beyond the end of the vector does not lead to +// an error. The expected outcome is the same as if the vector end was used as +// input position. That is, the input position should be capped at the maximum +// allowed value. +TEST_F(AudioVectorTest, InsertBeyondEnd) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = rtc::checked_cast( + array_length() + 10); // Too large. + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, 1, ..., kLength - 1, 100, 101, ..., 100 + kNewLength - 1 }. + size_t pos = 0; + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } +} + +// Test the OverwriteAt method with a position such that all of the new values +// fit within the old vector. +TEST_F(AudioVectorTest, OverwriteAt) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + size_t insert_position = 2; + vec.OverwriteAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, ..., |insert_position| - 1, 100, 101, ..., 100 + kNewLength - 1, + // |insert_position|, |insert_position| + 1, ..., kLength - 1}. + size_t pos = 0; + for (pos = 0; pos < insert_position; ++pos) { + EXPECT_EQ(array_[pos], vec[pos]); + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + for (; pos < array_length(); ++pos) { + EXPECT_EQ(array_[pos], vec[pos]); + } +} + +// Test the OverwriteAt method with a position such that some of the new values +// extend beyond the end of the current vector. This is valid, and the vector is +// expected to expand to accommodate the new values. +TEST_F(AudioVectorTest, OverwriteBeyondEnd) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = rtc::checked_cast(array_length() - 2); + vec.OverwriteAt(new_array, kNewLength, insert_position); + ASSERT_EQ(array_length() - 2u + kNewLength, vec.Size()); + // Verify that the vector looks as follows: + // {0, ..., |insert_position| - 1, 100, 101, ..., 100 + kNewLength - 1, + // |insert_position|, |insert_position| + 1, ..., kLength - 1}. + int pos = 0; + for (pos = 0; pos < insert_position; ++pos) { + EXPECT_EQ(array_[pos], vec[pos]); + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + // Verify that we checked to the end of |vec|. + EXPECT_EQ(vec.Size(), static_cast(pos)); +} + +TEST_F(AudioVectorTest, CrossFade) { + static const size_t kLength = 100; + static const size_t kFadeLength = 10; + AudioVector vec1(kLength); + AudioVector vec2(kLength); + // Set all vector elements to 0 in |vec1| and 100 in |vec2|. + for (size_t i = 0; i < kLength; ++i) { + vec1[i] = 0; + vec2[i] = 100; + } + vec1.CrossFade(vec2, kFadeLength); + ASSERT_EQ(2 * kLength - kFadeLength, vec1.Size()); + // First part untouched. + for (size_t i = 0; i < kLength - kFadeLength; ++i) { + EXPECT_EQ(0, vec1[i]); + } + // Check mixing zone. + for (size_t i = 0 ; i < kFadeLength; ++i) { + EXPECT_NEAR((i + 1) * 100 / (kFadeLength + 1), + vec1[kLength - kFadeLength + i], 1); + } + // Second part untouched. + for (size_t i = kLength; i < vec1.Size(); ++i) { + EXPECT_EQ(100, vec1[i]); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.cc new file mode 100644 index 0000000000..eda5c7514d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.cc @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/background_noise.h" + +#include +#include // memcpy + +#include // min, max + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/post_decode_vad.h" + +namespace webrtc { + +// static +const size_t BackgroundNoise::kMaxLpcOrder; + +BackgroundNoise::BackgroundNoise(size_t num_channels) + : num_channels_(num_channels), + channel_parameters_(new ChannelParameters[num_channels_]), + mode_(NetEq::kBgnOn) { + Reset(); +} + +BackgroundNoise::~BackgroundNoise() {} + +void BackgroundNoise::Reset() { + initialized_ = false; + for (size_t channel = 0; channel < num_channels_; ++channel) { + channel_parameters_[channel].Reset(); + } + // Keep _bgnMode as it is. +} + +void BackgroundNoise::Update(const AudioMultiVector& input, + const PostDecodeVad& vad) { + if (vad.running() && vad.active_speech()) { + // Do not update the background noise parameters if we know that the signal + // is active speech. + return; + } + + int32_t auto_correlation[kMaxLpcOrder + 1]; + int16_t fiter_output[kMaxLpcOrder + kResidualLength]; + int16_t reflection_coefficients[kMaxLpcOrder]; + int16_t lpc_coefficients[kMaxLpcOrder + 1]; + + for (size_t channel_ix = 0; channel_ix < num_channels_; ++channel_ix) { + ChannelParameters& parameters = channel_parameters_[channel_ix]; + int16_t temp_signal_array[kVecLen + kMaxLpcOrder] = {0}; + int16_t* temp_signal = &temp_signal_array[kMaxLpcOrder]; + input[channel_ix].CopyTo(kVecLen, input.Size() - kVecLen, temp_signal); + int32_t sample_energy = CalculateAutoCorrelation(temp_signal, kVecLen, + auto_correlation); + + if ((!vad.running() && + sample_energy < parameters.energy_update_threshold) || + (vad.running() && !vad.active_speech())) { + // Generate LPC coefficients. + if (auto_correlation[0] > 0) { + // Regardless of whether the filter is actually updated or not, + // update energy threshold levels, since we have in fact observed + // a low energy signal. + if (sample_energy < parameters.energy_update_threshold) { + // Never go under 1.0 in average sample energy. + parameters.energy_update_threshold = std::max(sample_energy, 1); + parameters.low_energy_update_threshold = 0; + } + + // Only update BGN if filter is stable, i.e., if return value from + // Levinson-Durbin function is 1. + if (WebRtcSpl_LevinsonDurbin(auto_correlation, lpc_coefficients, + reflection_coefficients, + kMaxLpcOrder) != 1) { + return; + } + } else { + // Center value in auto-correlation is not positive. Do not update. + return; + } + + // Generate the CNG gain factor by looking at the energy of the residual. + WebRtcSpl_FilterMAFastQ12(temp_signal + kVecLen - kResidualLength, + fiter_output, lpc_coefficients, + kMaxLpcOrder + 1, kResidualLength); + int32_t residual_energy = WebRtcSpl_DotProductWithScale(fiter_output, + fiter_output, + kResidualLength, + 0); + + // Check spectral flatness. + // Comparing the residual variance with the input signal variance tells + // if the spectrum is flat or not. + // If 5 * residual_energy >= 16 * sample_energy, the spectrum is flat + // enough. Also ensure that the energy is non-zero. + if ((sample_energy > 0) && + (int64_t{5} * residual_energy >= int64_t{16} * sample_energy)) { + // Spectrum is flat enough; save filter parameters. + // |temp_signal| + |kVecLen| - |kMaxLpcOrder| points at the first of the + // |kMaxLpcOrder| samples in the residual signal, which will form the + // filter state for the next noise generation. + SaveParameters(channel_ix, lpc_coefficients, + temp_signal + kVecLen - kMaxLpcOrder, sample_energy, + residual_energy); + } + } else { + // Will only happen if post-decode VAD is disabled and |sample_energy| is + // not low enough. Increase the threshold for update so that it increases + // by a factor 4 in 4 seconds. + IncrementEnergyThreshold(channel_ix, sample_energy); + } + } + return; +} + +int32_t BackgroundNoise::Energy(size_t channel) const { + assert(channel < num_channels_); + return channel_parameters_[channel].energy; +} + +void BackgroundNoise::SetMuteFactor(size_t channel, int16_t value) { + assert(channel < num_channels_); + channel_parameters_[channel].mute_factor = value; +} + +int16_t BackgroundNoise::MuteFactor(size_t channel) const { + assert(channel < num_channels_); + return channel_parameters_[channel].mute_factor; +} + +const int16_t* BackgroundNoise::Filter(size_t channel) const { + assert(channel < num_channels_); + return channel_parameters_[channel].filter; +} + +const int16_t* BackgroundNoise::FilterState(size_t channel) const { + assert(channel < num_channels_); + return channel_parameters_[channel].filter_state; +} + +void BackgroundNoise::SetFilterState(size_t channel, const int16_t* input, + size_t length) { + assert(channel < num_channels_); + length = std::min(length, kMaxLpcOrder); + memcpy(channel_parameters_[channel].filter_state, input, + length * sizeof(int16_t)); +} + +int16_t BackgroundNoise::Scale(size_t channel) const { + assert(channel < num_channels_); + return channel_parameters_[channel].scale; +} +int16_t BackgroundNoise::ScaleShift(size_t channel) const { + assert(channel < num_channels_); + return channel_parameters_[channel].scale_shift; +} + +int32_t BackgroundNoise::CalculateAutoCorrelation( + const int16_t* signal, size_t length, int32_t* auto_correlation) const { + static const int kCorrelationStep = -1; + const int correlation_scale = + CrossCorrelationWithAutoShift(signal, signal, length, kMaxLpcOrder + 1, + kCorrelationStep, auto_correlation); + + // Number of shifts to normalize energy to energy/sample. + int energy_sample_shift = kLogVecLen - correlation_scale; + return auto_correlation[0] >> energy_sample_shift; +} + +void BackgroundNoise::IncrementEnergyThreshold(size_t channel, + int32_t sample_energy) { + // TODO(hlundin): Simplify the below threshold update. What this code + // does is simply "threshold += (increment * threshold) >> 16", but due + // to the limited-width operations, it is not exactly the same. The + // difference should be inaudible, but bit-exactness would not be + // maintained. + assert(channel < num_channels_); + ChannelParameters& parameters = channel_parameters_[channel]; + int32_t temp_energy = + (kThresholdIncrement * parameters.low_energy_update_threshold) >> 16; + temp_energy += kThresholdIncrement * + (parameters.energy_update_threshold & 0xFF); + temp_energy += (kThresholdIncrement * + ((parameters.energy_update_threshold>>8) & 0xFF)) << 8; + parameters.low_energy_update_threshold += temp_energy; + + parameters.energy_update_threshold += kThresholdIncrement * + (parameters.energy_update_threshold>>16); + parameters.energy_update_threshold += + parameters.low_energy_update_threshold >> 16; + parameters.low_energy_update_threshold = + parameters.low_energy_update_threshold & 0x0FFFF; + + // Update maximum energy. + // Decrease by a factor 1/1024 each time. + parameters.max_energy = parameters.max_energy - + (parameters.max_energy >> 10); + if (sample_energy > parameters.max_energy) { + parameters.max_energy = sample_energy; + } + + // Set |energy_update_threshold| to no less than 60 dB lower than + // |max_energy_|. Adding 524288 assures proper rounding. + int32_t energy_update_threshold = (parameters.max_energy + 524288) >> 20; + if (energy_update_threshold > parameters.energy_update_threshold) { + parameters.energy_update_threshold = energy_update_threshold; + } +} + +void BackgroundNoise::SaveParameters(size_t channel, + const int16_t* lpc_coefficients, + const int16_t* filter_state, + int32_t sample_energy, + int32_t residual_energy) { + assert(channel < num_channels_); + ChannelParameters& parameters = channel_parameters_[channel]; + memcpy(parameters.filter, lpc_coefficients, + (kMaxLpcOrder+1) * sizeof(int16_t)); + memcpy(parameters.filter_state, filter_state, + kMaxLpcOrder * sizeof(int16_t)); + // Save energy level and update energy threshold levels. + // Never get under 1.0 in average sample energy. + parameters.energy = std::max(sample_energy, 1); + parameters.energy_update_threshold = parameters.energy; + parameters.low_energy_update_threshold = 0; + + // Normalize residual_energy to 29 or 30 bits before sqrt. + int16_t norm_shift = WebRtcSpl_NormW32(residual_energy) - 1; + if (norm_shift & 0x1) { + norm_shift -= 1; // Even number of shifts required. + } + residual_energy = WEBRTC_SPL_SHIFT_W32(residual_energy, norm_shift); + + // Calculate scale and shift factor. + parameters.scale = static_cast(WebRtcSpl_SqrtFloor(residual_energy)); + // Add 13 to the |scale_shift_|, since the random numbers table is in + // Q13. + // TODO(hlundin): Move the "13" to where the |scale_shift_| is used? + parameters.scale_shift = + static_cast(13 + ((kLogResidualLength + norm_shift) / 2)); + + initialized_ = true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.h new file mode 100644 index 0000000000..718f41db37 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_BACKGROUND_NOISE_H_ +#define MODULES_AUDIO_CODING_NETEQ_BACKGROUND_NOISE_H_ + +#include // size_t +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class PostDecodeVad; + +// This class handles estimation of background noise parameters. +class BackgroundNoise { + public: + // TODO(hlundin): For 48 kHz support, increase kMaxLpcOrder to 10. + // Will work anyway, but probably sound a little worse. + static const size_t kMaxLpcOrder = 8; // 32000 / 8000 + 4. + + explicit BackgroundNoise(size_t num_channels); + virtual ~BackgroundNoise(); + + void Reset(); + + // Updates the parameter estimates based on the signal currently in the + // |sync_buffer|, and on the latest decision in |vad| if it is running. + void Update(const AudioMultiVector& sync_buffer, + const PostDecodeVad& vad); + + // Returns |energy_| for |channel|. + int32_t Energy(size_t channel) const; + + // Sets the value of |mute_factor_| for |channel| to |value|. + void SetMuteFactor(size_t channel, int16_t value); + + // Returns |mute_factor_| for |channel|. + int16_t MuteFactor(size_t channel) const; + + // Returns a pointer to |filter_| for |channel|. + const int16_t* Filter(size_t channel) const; + + // Returns a pointer to |filter_state_| for |channel|. + const int16_t* FilterState(size_t channel) const; + + // Copies |length| elements from |input| to the filter state. Will not copy + // more than |kMaxLpcOrder| elements. + void SetFilterState(size_t channel, const int16_t* input, size_t length); + + // Returns |scale_| for |channel|. + int16_t Scale(size_t channel) const; + + // Returns |scale_shift_| for |channel|. + int16_t ScaleShift(size_t channel) const; + + // Accessors. + bool initialized() const { return initialized_; } + NetEq::BackgroundNoiseMode mode() const { return mode_; } + + // Sets the mode of the background noise playout for cases when there is long + // duration of packet loss. + void set_mode(NetEq::BackgroundNoiseMode mode) { mode_ = mode; } + + private: + static const int kThresholdIncrement = 229; // 0.0035 in Q16. + static const size_t kVecLen = 256; + static const int kLogVecLen = 8; // log2(kVecLen). + static const size_t kResidualLength = 64; + static const int16_t kLogResidualLength = 6; // log2(kResidualLength) + + struct ChannelParameters { + // Constructor. + ChannelParameters() { + Reset(); + } + + void Reset() { + energy = 2500; + max_energy = 0; + energy_update_threshold = 500000; + low_energy_update_threshold = 0; + memset(filter_state, 0, sizeof(filter_state)); + memset(filter, 0, sizeof(filter)); + filter[0] = 4096; + mute_factor = 0; + scale = 20000; + scale_shift = 24; + } + + int32_t energy; + int32_t max_energy; + int32_t energy_update_threshold; + int32_t low_energy_update_threshold; + int16_t filter_state[kMaxLpcOrder]; + int16_t filter[kMaxLpcOrder + 1]; + int16_t mute_factor; + int16_t scale; + int16_t scale_shift; + }; + + int32_t CalculateAutoCorrelation(const int16_t* signal, + size_t length, + int32_t* auto_correlation) const; + + // Increments the energy threshold by a factor 1 + |kThresholdIncrement|. + void IncrementEnergyThreshold(size_t channel, int32_t sample_energy); + + // Updates the filter parameters. + void SaveParameters(size_t channel, + const int16_t* lpc_coefficients, + const int16_t* filter_state, + int32_t sample_energy, + int32_t residual_energy); + + size_t num_channels_; + std::unique_ptr channel_parameters_; + bool initialized_; + NetEq::BackgroundNoiseMode mode_; + + RTC_DISALLOW_COPY_AND_ASSIGN(BackgroundNoise); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_BACKGROUND_NOISE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise_unittest.cc new file mode 100644 index 0000000000..e32492f57e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise_unittest.cc @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for BackgroundNoise class. + +#include "modules/audio_coding/neteq/background_noise.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(BackgroundNoise, CreateAndDestroy) { + size_t channels = 1; + BackgroundNoise bgn(channels); +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.cc new file mode 100644 index 0000000000..6005de6989 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/buffer_level_filter.h" + +#include // Provide access to std::max. + +namespace webrtc { + +BufferLevelFilter::BufferLevelFilter() { + Reset(); +} + +void BufferLevelFilter::Reset() { + filtered_current_level_ = 0; + level_factor_ = 253; +} + +void BufferLevelFilter::Update(size_t buffer_size_packets, + int time_stretched_samples, + size_t packet_len_samples) { + // Filter: + // |filtered_current_level_| = |level_factor_| * |filtered_current_level_| + + // (1 - |level_factor_|) * |buffer_size_packets| + // |level_factor_| and |filtered_current_level_| are in Q8. + // |buffer_size_packets| is in Q0. + filtered_current_level_ = ((level_factor_ * filtered_current_level_) >> 8) + + ((256 - level_factor_) * static_cast(buffer_size_packets)); + + // Account for time-scale operations (accelerate and pre-emptive expand). + if (time_stretched_samples && packet_len_samples > 0) { + // Time-scaling has been performed since last filter update. Subtract the + // value of |time_stretched_samples| from |filtered_current_level_| after + // converting |time_stretched_samples| from samples to packets in Q8. + // Make sure that the filtered value remains non-negative. + filtered_current_level_ = std::max( + 0, filtered_current_level_ - (time_stretched_samples * (1 << 8)) / + static_cast(packet_len_samples)); + } +} + +void BufferLevelFilter::SetTargetBufferLevel(int target_buffer_level) { + if (target_buffer_level <= 1) { + level_factor_ = 251; + } else if (target_buffer_level <= 3) { + level_factor_ = 252; + } else if (target_buffer_level <= 7) { + level_factor_ = 253; + } else { + level_factor_ = 254; + } +} + +int BufferLevelFilter::filtered_current_level() const { + return filtered_current_level_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.h new file mode 100644 index 0000000000..7a48c72b74 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_BUFFER_LEVEL_FILTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_BUFFER_LEVEL_FILTER_H_ + +#include + +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class BufferLevelFilter { + public: + BufferLevelFilter(); + virtual ~BufferLevelFilter() {} + virtual void Reset(); + + // Updates the filter. Current buffer size is |buffer_size_packets| (Q0). + // If |time_stretched_samples| is non-zero, the value is converted to the + // corresponding number of packets, and is subtracted from the filtered + // value (thus bypassing the filter operation). |packet_len_samples| is the + // number of audio samples carried in each incoming packet. + virtual void Update(size_t buffer_size_packets, int time_stretched_samples, + size_t packet_len_samples); + + // Set the current target buffer level (obtained from + // DelayManager::base_target_level()). Used to select the appropriate + // filter coefficient. + virtual void SetTargetBufferLevel(int target_buffer_level); + + virtual int filtered_current_level() const; + + private: + int level_factor_; // Filter factor for the buffer level filter in Q8. + int filtered_current_level_; // Filtered current buffer level in Q8. + + RTC_DISALLOW_COPY_AND_ASSIGN(BufferLevelFilter); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_BUFFER_LEVEL_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc new file mode 100644 index 0000000000..72c8727429 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for BufferLevelFilter class. + +#include "modules/audio_coding/neteq/buffer_level_filter.h" + +#include // Access to pow function. + +#include "test/gtest.h" + +namespace webrtc { + +TEST(BufferLevelFilter, CreateAndDestroy) { + BufferLevelFilter* filter = new BufferLevelFilter(); + EXPECT_EQ(0, filter->filtered_current_level()); + delete filter; +} + +TEST(BufferLevelFilter, ConvergenceTest) { + BufferLevelFilter filter; + for (int times = 10; times <= 50; times += 10) { + for (int value = 100; value <= 200; value += 10) { + filter.Reset(); + filter.SetTargetBufferLevel(1); // Makes filter coefficient 251/256. + std::ostringstream ss; + ss << "times = " << times << ", value = " << value; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + for (int i = 0; i < times; ++i) { + filter.Update(value, 0 /* time_stretched_samples */, + 160 /* packet_len_samples */); + } + // Expect the filtered value to be (theoretically) + // (1 - (251/256) ^ |times|) * |value|. + double expected_value_double = + (1 - pow(251.0 / 256.0, times)) * value; + int expected_value = static_cast(expected_value_double); + // filtered_current_level() returns the value in Q8. + // The actual value may differ slightly from the expected value due to + // intermediate-stage rounding errors in the filter implementation. + // This is why we have to use EXPECT_NEAR with a tolerance of +/-1. + EXPECT_NEAR(expected_value, filter.filtered_current_level() >> 8, 1); + } + } +} + +// Verify that target buffer level impacts on the filter convergence. +TEST(BufferLevelFilter, FilterFactor) { + BufferLevelFilter filter; + // Update 10 times with value 100. + const int kTimes = 10; + const int kValue = 100; + + filter.SetTargetBufferLevel(3); // Makes filter coefficient 252/256. + for (int i = 0; i < kTimes; ++i) { + filter.Update(kValue, 0 /* time_stretched_samples */, + 160 /* packet_len_samples */); + } + // Expect the filtered value to be + // (1 - (252/256) ^ |kTimes|) * |kValue|. + int expected_value = 14; + // filtered_current_level() returns the value in Q8. + EXPECT_EQ(expected_value, filter.filtered_current_level() >> 8); + + filter.Reset(); + filter.SetTargetBufferLevel(7); // Makes filter coefficient 253/256. + for (int i = 0; i < kTimes; ++i) { + filter.Update(kValue, 0 /* time_stretched_samples */, + 160 /* packet_len_samples */); + } + // Expect the filtered value to be + // (1 - (253/256) ^ |kTimes|) * |kValue|. + expected_value = 11; + // filtered_current_level() returns the value in Q8. + EXPECT_EQ(expected_value, filter.filtered_current_level() >> 8); + + filter.Reset(); + filter.SetTargetBufferLevel(8); // Makes filter coefficient 254/256. + for (int i = 0; i < kTimes; ++i) { + filter.Update(kValue, 0 /* time_stretched_samples */, + 160 /* packet_len_samples */); + } + // Expect the filtered value to be + // (1 - (254/256) ^ |kTimes|) * |kValue|. + expected_value = 7; + // filtered_current_level() returns the value in Q8. + EXPECT_EQ(expected_value, filter.filtered_current_level() >> 8); +} + + +TEST(BufferLevelFilter, TimeStretchedSamples) { + BufferLevelFilter filter; + filter.SetTargetBufferLevel(1); // Makes filter coefficient 251/256. + // Update 10 times with value 100. + const int kTimes = 10; + const int kValue = 100; + const int kPacketSizeSamples = 160; + const int kNumPacketsStretched = 2; + const int kTimeStretchedSamples = kNumPacketsStretched * kPacketSizeSamples; + for (int i = 0; i < kTimes; ++i) { + // Packet size set to 0. Do not expect the parameter + // |kTimeStretchedSamples| to have any effect. + filter.Update(kValue, kTimeStretchedSamples, 0 /* packet_len_samples */); + } + // Expect the filtered value to be + // (1 - (251/256) ^ |kTimes|) * |kValue|. + const int kExpectedValue = 17; + // filtered_current_level() returns the value in Q8. + EXPECT_EQ(kExpectedValue, filter.filtered_current_level() >> 8); + + // Update filter again, now with non-zero value for packet length. + // Set the current filtered value to be the input, in order to isolate the + // impact of |kTimeStretchedSamples|. + filter.Update(filter.filtered_current_level() >> 8, kTimeStretchedSamples, + kPacketSizeSamples); + EXPECT_EQ(kExpectedValue - kNumPacketsStretched, + filter.filtered_current_level() >> 8); + // Try negative value and verify that we come back to the previous result. + filter.Update(filter.filtered_current_level() >> 8, -kTimeStretchedSamples, + kPacketSizeSamples); + EXPECT_EQ(kExpectedValue, filter.filtered_current_level() >> 8); +} + +TEST(BufferLevelFilter, TimeStretchedSamplesNegativeUnevenFrames) { + BufferLevelFilter filter; + filter.SetTargetBufferLevel(1); // Makes filter coefficient 251/256. + // Update 10 times with value 100. + const int kTimes = 10; + const int kValue = 100; + const int kPacketSizeSamples = 160; + const int kTimeStretchedSamples = -3.1415 * kPacketSizeSamples; + for (int i = 0; i < kTimes; ++i) { + // Packet size set to 0. Do not expect the parameter + // |kTimeStretchedSamples| to have any effect. + filter.Update(kValue, kTimeStretchedSamples, 0 /* packet_len_samples */); + } + // Expect the filtered value to be + // (1 - (251/256) ^ |kTimes|) * |kValue|. + const int kExpectedValue = 17; + // filtered_current_level() returns the value in Q8. + EXPECT_EQ(kExpectedValue, filter.filtered_current_level() >> 8); + + // Update filter again, now with non-zero value for packet length. + // Set the current filtered value to be the input, in order to isolate the + // impact of |kTimeStretchedSamples|. + filter.Update(filter.filtered_current_level() >> 8, kTimeStretchedSamples, + kPacketSizeSamples); + EXPECT_EQ(21, filter.filtered_current_level() >> 8); + // Try negative value and verify that we come back to the previous result. + filter.Update(filter.filtered_current_level() >> 8, -kTimeStretchedSamples, + kPacketSizeSamples); + EXPECT_EQ(kExpectedValue, filter.filtered_current_level() >> 8); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.cc new file mode 100644 index 0000000000..5e0a87526e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/comfort_noise.h" + +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +void ComfortNoise::Reset() { + first_call_ = true; +} + +int ComfortNoise::UpdateParameters(const Packet& packet) { + // Get comfort noise decoder. + if (decoder_database_->SetActiveCngDecoder(packet.payload_type) != kOK) { + return kUnknownPayloadType; + } + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + RTC_DCHECK(cng_decoder); + cng_decoder->UpdateSid(packet.payload); + return kOK; +} + +int ComfortNoise::Generate(size_t requested_length, + AudioMultiVector* output) { + // TODO(hlundin): Change to an enumerator and skip assert. + assert(fs_hz_ == 8000 || fs_hz_ == 16000 || fs_hz_ == 32000 || + fs_hz_ == 48000); + // Not adapted for multi-channel yet. + if (output->Channels() != 1) { + RTC_LOG(LS_ERROR) << "No multi-channel support"; + return kMultiChannelNotSupported; + } + + size_t number_of_samples = requested_length; + bool new_period = false; + if (first_call_) { + // Generate noise and overlap slightly with old data. + number_of_samples = requested_length + overlap_length_; + new_period = true; + } + output->AssertSize(number_of_samples); + // Get the decoder from the database. + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + if (!cng_decoder) { + RTC_LOG(LS_ERROR) << "Unknwown payload type"; + return kUnknownPayloadType; + } + + std::unique_ptr temp(new int16_t[number_of_samples]); + if (!cng_decoder->Generate( + rtc::ArrayView(temp.get(), number_of_samples), + new_period)) { + // Error returned. + output->Zeros(requested_length); + RTC_LOG(LS_ERROR) + << "ComfortNoiseDecoder::Genererate failed to generate comfort noise"; + return kInternalError; + } + (*output)[0].OverwriteAt(temp.get(), number_of_samples, 0); + + if (first_call_) { + // Set tapering window parameters. Values are in Q15. + int16_t muting_window; // Mixing factor for overlap data. + int16_t muting_window_increment; // Mixing factor increment (negative). + int16_t unmuting_window; // Mixing factor for comfort noise. + int16_t unmuting_window_increment; // Mixing factor increment. + if (fs_hz_ == 8000) { + muting_window = DspHelper::kMuteFactorStart8kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement8kHz; + unmuting_window = DspHelper::kUnmuteFactorStart8kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement8kHz; + } else if (fs_hz_ == 16000) { + muting_window = DspHelper::kMuteFactorStart16kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement16kHz; + unmuting_window = DspHelper::kUnmuteFactorStart16kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement16kHz; + } else if (fs_hz_ == 32000) { + muting_window = DspHelper::kMuteFactorStart32kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement32kHz; + unmuting_window = DspHelper::kUnmuteFactorStart32kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement32kHz; + } else { // fs_hz_ == 48000 + muting_window = DspHelper::kMuteFactorStart48kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement48kHz; + unmuting_window = DspHelper::kUnmuteFactorStart48kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement48kHz; + } + + // Do overlap-add between new vector and overlap. + size_t start_ix = sync_buffer_->Size() - overlap_length_; + for (size_t i = 0; i < overlap_length_; i++) { + /* overlapVec[i] = WinMute * overlapVec[i] + WinUnMute * outData[i] */ + // The expression (*output)[0][i] is the i-th element in the first + // channel. + (*sync_buffer_)[0][start_ix + i] = + (((*sync_buffer_)[0][start_ix + i] * muting_window) + + ((*output)[0][i] * unmuting_window) + 16384) >> 15; + muting_window += muting_window_increment; + unmuting_window += unmuting_window_increment; + } + // Remove |overlap_length_| samples from the front of |output| since they + // were mixed into |sync_buffer_| above. + output->PopFront(overlap_length_); + } + first_call_ = false; + return kOK; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.h new file mode 100644 index 0000000000..18800ad644 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_COMFORT_NOISE_H_ +#define MODULES_AUDIO_CODING_NETEQ_COMFORT_NOISE_H_ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class DecoderDatabase; +class SyncBuffer; +struct Packet; + +// This class acts as an interface to the CNG generator. +class ComfortNoise { + public: + enum ReturnCodes { + kOK = 0, + kUnknownPayloadType, + kInternalError, + kMultiChannelNotSupported + }; + + ComfortNoise(int fs_hz, DecoderDatabase* decoder_database, + SyncBuffer* sync_buffer) + : fs_hz_(fs_hz), + first_call_(true), + overlap_length_(5 * fs_hz_ / 8000), + decoder_database_(decoder_database), + sync_buffer_(sync_buffer) { + } + + // Resets the state. Should be called before each new comfort noise period. + void Reset(); + + // Update the comfort noise generator with the parameters in |packet|. + int UpdateParameters(const Packet& packet); + + // Generates |requested_length| samples of comfort noise and writes to + // |output|. If this is the first in call after Reset (or first after creating + // the object), it will also mix in comfort noise at the end of the + // SyncBuffer object provided in the constructor. + int Generate(size_t requested_length, AudioMultiVector* output); + + // Returns the last error code that was produced by the comfort noise + // decoder. Returns 0 if no error has been encountered since the last reset. + int internal_error_code() { return internal_error_code_; } + + private: + int fs_hz_; + bool first_call_; + size_t overlap_length_; + DecoderDatabase* decoder_database_; + SyncBuffer* sync_buffer_; + int internal_error_code_; + RTC_DISALLOW_COPY_AND_ASSIGN(ComfortNoise); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_COMFORT_NOISE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc new file mode 100644 index 0000000000..b3fbb4eb06 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for ComfortNoise class. + +#include "modules/audio_coding/neteq/comfort_noise.h" + +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(ComfortNoise, CreateAndDestroy) { + int fs = 8000; + MockDecoderDatabase db; + SyncBuffer sync_buffer(1, 1000); + ComfortNoise cn(fs, &db, &sync_buffer); + EXPECT_CALL(db, Die()); // Called when |db| goes out of scope. +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.cc new file mode 100644 index 0000000000..da9c913249 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/cross_correlation.h" + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +namespace webrtc { + +// This function decides the overflow-protecting scaling and calls +// WebRtcSpl_CrossCorrelation. +int CrossCorrelationWithAutoShift(const int16_t* sequence_1, + const int16_t* sequence_2, + size_t sequence_1_length, + size_t cross_correlation_length, + int cross_correlation_step, + int32_t* cross_correlation) { + // Find the maximum absolute value of sequence_1 and 2. + const int16_t max_1 = WebRtcSpl_MaxAbsValueW16(sequence_1, sequence_1_length); + const int sequence_2_shift = + cross_correlation_step * (static_cast(cross_correlation_length) - 1); + const int16_t* sequence_2_start = + sequence_2_shift >= 0 ? sequence_2 : sequence_2 + sequence_2_shift; + const size_t sequence_2_length = + sequence_1_length + std::abs(sequence_2_shift); + const int16_t max_2 = + WebRtcSpl_MaxAbsValueW16(sequence_2_start, sequence_2_length); + + // In order to avoid overflow when computing the sum we should scale the + // samples so that (in_vector_length * max_1 * max_2) will not overflow. + // Expected scaling fulfills + // 1) sufficient: + // sequence_1_length * (max_1 * max_2 >> scaling) <= 0x7fffffff; + // 2) necessary: + // if (scaling > 0) + // sequence_1_length * (max_1 * max_2 >> (scaling - 1)) > 0x7fffffff; + // The following calculation fulfills 1) and almost fulfills 2). + // There are some corner cases that 2) is not satisfied, e.g., + // max_1 = 17, max_2 = 30848, sequence_1_length = 4095, in such case, + // optimal scaling is 0, while the following calculation results in 1. + const int32_t factor = (max_1 * max_2) / (std::numeric_limits::max() + / static_cast(sequence_1_length)); + const int scaling = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); + + WebRtcSpl_CrossCorrelation(cross_correlation, sequence_1, sequence_2, + sequence_1_length, cross_correlation_length, + scaling, cross_correlation_step); + + return scaling; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.h new file mode 100644 index 0000000000..a747772cfc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_CROSS_CORRELATION_H_ +#define MODULES_AUDIO_CODING_NETEQ_CROSS_CORRELATION_H_ + +#include "common_types.h" // NOLINT(build/include) + +namespace webrtc { + +// The function calculates the cross-correlation between two sequences +// |sequence_1| and |sequence_2|. |sequence_1| is taken as reference, with +// |sequence_1_length| as its length. |sequence_2| slides for the calculation of +// cross-correlation. The result will be saved in |cross_correlation|. +// |cross_correlation_length| correlation points are calculated. +// The corresponding lag starts from 0, and increases with a step of +// |cross_correlation_step|. The result is without normalization. To avoid +// overflow, the result will be right shifted. The amount of shifts will be +// returned. +// +// Input: +// - sequence_1 : First sequence (reference). +// - sequence_2 : Second sequence (sliding during calculation). +// - sequence_1_length : Length of |sequence_1|. +// - cross_correlation_length : Number of cross-correlations to calculate. +// - cross_correlation_step : Step in the lag for the cross-correlation. +// +// Output: +// - cross_correlation : The cross-correlation in Q(-right_shifts) +// +// Return: +// Number of right shifts in cross_correlation. + +int CrossCorrelationWithAutoShift(const int16_t* sequence_1, + const int16_t* sequence_2, + size_t sequence_1_length, + size_t cross_correlation_length, + int cross_correlation_step, + int32_t* cross_correlation); + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_CROSS_CORRELATION_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.cc new file mode 100644 index 0000000000..966d5c376d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.cc @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decision_logic.h" + +#include + +#include "modules/audio_coding/neteq/buffer_level_filter.h" +#include "modules/audio_coding/neteq/decision_logic_fax.h" +#include "modules/audio_coding/neteq/decision_logic_normal.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "modules/audio_coding/neteq/sync_buffer.h" + +namespace webrtc { + +DecisionLogic* DecisionLogic::Create(int fs_hz, + size_t output_size_samples, + NetEqPlayoutMode playout_mode, + DecoderDatabase* decoder_database, + const PacketBuffer& packet_buffer, + DelayManager* delay_manager, + BufferLevelFilter* buffer_level_filter, + const TickTimer* tick_timer) { + switch (playout_mode) { + case kPlayoutOn: + case kPlayoutStreaming: + return new DecisionLogicNormal( + fs_hz, output_size_samples, playout_mode, decoder_database, + packet_buffer, delay_manager, buffer_level_filter, tick_timer); + case kPlayoutFax: + case kPlayoutOff: + return new DecisionLogicFax( + fs_hz, output_size_samples, playout_mode, decoder_database, + packet_buffer, delay_manager, buffer_level_filter, tick_timer); + } + // This line cannot be reached, but must be here to avoid compiler errors. + assert(false); + return NULL; +} + +DecisionLogic::DecisionLogic(int fs_hz, + size_t output_size_samples, + NetEqPlayoutMode playout_mode, + DecoderDatabase* decoder_database, + const PacketBuffer& packet_buffer, + DelayManager* delay_manager, + BufferLevelFilter* buffer_level_filter, + const TickTimer* tick_timer) + : decoder_database_(decoder_database), + packet_buffer_(packet_buffer), + delay_manager_(delay_manager), + buffer_level_filter_(buffer_level_filter), + tick_timer_(tick_timer), + cng_state_(kCngOff), + packet_length_samples_(0), + sample_memory_(0), + prev_time_scale_(false), + timescale_countdown_( + tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1)), + num_consecutive_expands_(0), + playout_mode_(playout_mode) { + delay_manager_->set_streaming_mode(playout_mode_ == kPlayoutStreaming); + SetSampleRate(fs_hz, output_size_samples); +} + +DecisionLogic::~DecisionLogic() = default; + +void DecisionLogic::Reset() { + cng_state_ = kCngOff; + noise_fast_forward_ = 0; + packet_length_samples_ = 0; + sample_memory_ = 0; + prev_time_scale_ = false; + timescale_countdown_.reset(); + num_consecutive_expands_ = 0; +} + +void DecisionLogic::SoftReset() { + packet_length_samples_ = 0; + sample_memory_ = 0; + prev_time_scale_ = false; + timescale_countdown_ = + tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1); +} + +void DecisionLogic::SetSampleRate(int fs_hz, size_t output_size_samples) { + // TODO(hlundin): Change to an enumerator and skip assert. + assert(fs_hz == 8000 || fs_hz == 16000 || fs_hz == 32000 || fs_hz == 48000); + fs_mult_ = fs_hz / 8000; + output_size_samples_ = output_size_samples; +} + +Operations DecisionLogic::GetDecision(const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + const Packet* next_packet, + Modes prev_mode, + bool play_dtmf, + size_t generated_noise_samples, + bool* reset_decoder) { + // If last mode was CNG (or Expand, since this could be covering up for + // a lost CNG packet), remember that CNG is on. This is needed if comfort + // noise is interrupted by DTMF. + if (prev_mode == kModeRfc3389Cng) { + cng_state_ = kCngRfc3389On; + } else if (prev_mode == kModeCodecInternalCng) { + cng_state_ = kCngInternalOn; + } + + const size_t samples_left = + sync_buffer.FutureLength() - expand.overlap_length(); + const size_t cur_size_samples = + samples_left + packet_buffer_.NumSamplesInBuffer(decoder_frame_length); + + prev_time_scale_ = prev_time_scale_ && + (prev_mode == kModeAccelerateSuccess || + prev_mode == kModeAccelerateLowEnergy || + prev_mode == kModePreemptiveExpandSuccess || + prev_mode == kModePreemptiveExpandLowEnergy); + + FilterBufferLevel(cur_size_samples, prev_mode); + + return GetDecisionSpecialized(sync_buffer, expand, decoder_frame_length, + next_packet, prev_mode, play_dtmf, + reset_decoder, generated_noise_samples); +} + +void DecisionLogic::ExpandDecision(Operations operation) { + if (operation == kExpand) { + num_consecutive_expands_++; + } else { + num_consecutive_expands_ = 0; + } +} + +void DecisionLogic::FilterBufferLevel(size_t buffer_size_samples, + Modes prev_mode) { + // Do not update buffer history if currently playing CNG since it will bias + // the filtered buffer level. + if ((prev_mode != kModeRfc3389Cng) && (prev_mode != kModeCodecInternalCng)) { + buffer_level_filter_->SetTargetBufferLevel( + delay_manager_->base_target_level()); + + size_t buffer_size_packets = 0; + if (packet_length_samples_ > 0) { + // Calculate size in packets. + buffer_size_packets = buffer_size_samples / packet_length_samples_; + } + int sample_memory_local = 0; + if (prev_time_scale_) { + sample_memory_local = sample_memory_; + timescale_countdown_ = + tick_timer_->GetNewCountdown(kMinTimescaleInterval); + } + buffer_level_filter_->Update(buffer_size_packets, sample_memory_local, + packet_length_samples_); + prev_time_scale_ = false; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.h new file mode 100644 index 0000000000..5b67196c43 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ +#define MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ + +#include "modules/audio_coding/neteq/defines.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/tick_timer.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class BufferLevelFilter; +class DecoderDatabase; +class DelayManager; +class Expand; +class PacketBuffer; +class SyncBuffer; +struct Packet; + +// This is the base class for the decision tree implementations. Derived classes +// must implement the method GetDecisionSpecialized(). +class DecisionLogic { + public: + // Static factory function which creates different types of objects depending + // on the |playout_mode|. + static DecisionLogic* Create(int fs_hz, + size_t output_size_samples, + NetEqPlayoutMode playout_mode, + DecoderDatabase* decoder_database, + const PacketBuffer& packet_buffer, + DelayManager* delay_manager, + BufferLevelFilter* buffer_level_filter, + const TickTimer* tick_timer); + + // Constructor. + DecisionLogic(int fs_hz, + size_t output_size_samples, + NetEqPlayoutMode playout_mode, + DecoderDatabase* decoder_database, + const PacketBuffer& packet_buffer, + DelayManager* delay_manager, + BufferLevelFilter* buffer_level_filter, + const TickTimer* tick_timer); + + virtual ~DecisionLogic(); + + // Resets object to a clean state. + void Reset(); + + // Resets parts of the state. Typically done when switching codecs. + void SoftReset(); + + // Sets the sample rate and the output block size. + void SetSampleRate(int fs_hz, size_t output_size_samples); + + // Returns the operation that should be done next. |sync_buffer| and |expand| + // are provided for reference. |decoder_frame_length| is the number of samples + // obtained from the last decoded frame. If there is a packet available, it + // should be supplied in |next_packet|; otherwise it should be NULL. The mode + // resulting from the last call to NetEqImpl::GetAudio is supplied in + // |prev_mode|. If there is a DTMF event to play, |play_dtmf| should be set to + // true. The output variable |reset_decoder| will be set to true if a reset is + // required; otherwise it is left unchanged (i.e., it can remain true if it + // was true before the call). This method end with calling + // GetDecisionSpecialized to get the actual return value. + Operations GetDecision(const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + const Packet* next_packet, + Modes prev_mode, + bool play_dtmf, + size_t generated_noise_samples, + bool* reset_decoder); + + // These methods test the |cng_state_| for different conditions. + bool CngRfc3389On() const { return cng_state_ == kCngRfc3389On; } + bool CngOff() const { return cng_state_ == kCngOff; } + + // Resets the |cng_state_| to kCngOff. + void SetCngOff() { cng_state_ = kCngOff; } + + // Reports back to DecisionLogic whether the decision to do expand remains or + // not. Note that this is necessary, since an expand decision can be changed + // to kNormal in NetEqImpl::GetDecision if there is still enough data in the + // sync buffer. + virtual void ExpandDecision(Operations operation); + + // Adds |value| to |sample_memory_|. + void AddSampleMemory(int32_t value) { + sample_memory_ += value; + } + + // Accessors and mutators. + void set_sample_memory(int32_t value) { sample_memory_ = value; } + size_t noise_fast_forward() const { return noise_fast_forward_; } + size_t packet_length_samples() const { return packet_length_samples_; } + void set_packet_length_samples(size_t value) { + packet_length_samples_ = value; + } + void set_prev_time_scale(bool value) { prev_time_scale_ = value; } + NetEqPlayoutMode playout_mode() const { return playout_mode_; } + + protected: + // The value 5 sets maximum time-stretch rate to about 100 ms/s. + static const int kMinTimescaleInterval = 5; + + enum CngState { + kCngOff, + kCngRfc3389On, + kCngInternalOn + }; + + // Returns the operation that should be done next. |sync_buffer| and |expand| + // are provided for reference. |decoder_frame_length| is the number of samples + // obtained from the last decoded frame. If there is a packet available, it + // should be supplied in |next_packet|; otherwise it should be NULL. The mode + // resulting from the last call to NetEqImpl::GetAudio is supplied in + // |prev_mode|. If there is a DTMF event to play, |play_dtmf| should be set to + // true. The output variable |reset_decoder| will be set to true if a reset is + // required; otherwise it is left unchanged (i.e., it can remain true if it + // was true before the call). Should be implemented by derived classes. + virtual Operations GetDecisionSpecialized(const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + const Packet* next_packet, + Modes prev_mode, + bool play_dtmf, + bool* reset_decoder, + size_t generated_noise_samples) = 0; + + // Updates the |buffer_level_filter_| with the current buffer level + // |buffer_size_packets|. + void FilterBufferLevel(size_t buffer_size_packets, Modes prev_mode); + + DecoderDatabase* decoder_database_; + const PacketBuffer& packet_buffer_; + DelayManager* delay_manager_; + BufferLevelFilter* buffer_level_filter_; + const TickTimer* tick_timer_; + int fs_mult_; + size_t output_size_samples_; + CngState cng_state_; // Remember if comfort noise is interrupted by other + // event (e.g., DTMF). + size_t noise_fast_forward_ = 0; + size_t packet_length_samples_; + int sample_memory_; + bool prev_time_scale_; + std::unique_ptr timescale_countdown_; + int num_consecutive_expands_; + const NetEqPlayoutMode playout_mode_; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(DecisionLogic); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.cc new file mode 100644 index 0000000000..cc21ee9deb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decision_logic_fax.h" + +#include + +#include + +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/sync_buffer.h" + +namespace webrtc { + +Operations DecisionLogicFax::GetDecisionSpecialized( + const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + const Packet* next_packet, + Modes prev_mode, + bool play_dtmf, + bool* reset_decoder, + size_t generated_noise_samples) { + assert(playout_mode_ == kPlayoutFax || playout_mode_ == kPlayoutOff); + uint32_t target_timestamp = sync_buffer.end_timestamp(); + uint32_t available_timestamp = 0; + int is_cng_packet = 0; + if (next_packet) { + available_timestamp = next_packet->timestamp; + is_cng_packet = + decoder_database_->IsComfortNoise(next_packet->payload_type); + } + if (is_cng_packet) { + if (static_cast((generated_noise_samples + target_timestamp) + - available_timestamp) >= 0) { + // Time to play this packet now. + return kRfc3389Cng; + } else { + // Wait before playing this packet. + return kRfc3389CngNoPacket; + } + } + if (!next_packet) { + // No packet. If in CNG mode, play as usual. Otherwise, use other method to + // generate data. + if (cng_state_ == kCngRfc3389On) { + // Continue playing comfort noise. + return kRfc3389CngNoPacket; + } else if (cng_state_ == kCngInternalOn) { + // Continue playing codec-internal comfort noise. + return kCodecInternalCng; + } else { + // Nothing to play. Generate some data to play out. + switch (playout_mode_) { + case kPlayoutOff: + return kAlternativePlc; + case kPlayoutFax: + return kAudioRepetition; + default: + assert(false); + return kUndefined; + } + } + } else if (target_timestamp == available_timestamp) { + return kNormal; + } else { + if (static_cast((generated_noise_samples + target_timestamp) + - available_timestamp) >= 0) { + return kNormal; + } else { + // If currently playing comfort noise, continue with that. Do not + // increase the timestamp counter since generated_noise_stopwatch_ in + // NetEqImpl will take care of the time-keeping. + if (cng_state_ == kCngRfc3389On) { + return kRfc3389CngNoPacket; + } else if (cng_state_ == kCngInternalOn) { + return kCodecInternalCng; + } else { + // Otherwise, do packet-loss concealment and increase the + // timestamp while waiting for the time to play this packet. + switch (playout_mode_) { + case kPlayoutOff: + return kAlternativePlcIncreaseTimestamp; + case kPlayoutFax: + return kAudioRepetitionIncreaseTimestamp; + default: + assert(0); + return kUndefined; + } + } + } + } +} + + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.h new file mode 100644 index 0000000000..cefd8e4e42 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_FAX_H_ +#define MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_FAX_H_ + +#include "modules/audio_coding/neteq/decision_logic.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Implementation of the DecisionLogic class for playout modes kPlayoutFax and +// kPlayoutOff. +class DecisionLogicFax : public DecisionLogic { + public: + // Constructor. + DecisionLogicFax(int fs_hz, + size_t output_size_samples, + NetEqPlayoutMode playout_mode, + DecoderDatabase* decoder_database, + const PacketBuffer& packet_buffer, + DelayManager* delay_manager, + BufferLevelFilter* buffer_level_filter, + const TickTimer* tick_timer) + : DecisionLogic(fs_hz, + output_size_samples, + playout_mode, + decoder_database, + packet_buffer, + delay_manager, + buffer_level_filter, + tick_timer) {} + + protected: + Operations GetDecisionSpecialized(const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + const Packet* next_packet, + Modes prev_mode, + bool play_dtmf, + bool* reset_decoder, + size_t generated_noise_samples) override; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(DecisionLogicFax); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_FAX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.cc new file mode 100644 index 0000000000..10f501ac9a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.cc @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decision_logic_normal.h" + +#include + +#include + +#include "modules/audio_coding/neteq/buffer_level_filter.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/include/module_common_types.h" + +namespace webrtc { + +Operations DecisionLogicNormal::GetDecisionSpecialized( + const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + const Packet* next_packet, + Modes prev_mode, + bool play_dtmf, + bool* reset_decoder, + size_t generated_noise_samples) { + assert(playout_mode_ == kPlayoutOn || playout_mode_ == kPlayoutStreaming); + // Guard for errors, to avoid getting stuck in error mode. + if (prev_mode == kModeError) { + if (!next_packet) { + return kExpand; + } else { + return kUndefined; // Use kUndefined to flag for a reset. + } + } + + uint32_t target_timestamp = sync_buffer.end_timestamp(); + uint32_t available_timestamp = 0; + bool is_cng_packet = false; + if (next_packet) { + available_timestamp = next_packet->timestamp; + is_cng_packet = + decoder_database_->IsComfortNoise(next_packet->payload_type); + } + + if (is_cng_packet) { + return CngOperation(prev_mode, target_timestamp, available_timestamp, + generated_noise_samples); + } + + // Handle the case with no packet at all available (except maybe DTMF). + if (!next_packet) { + return NoPacket(play_dtmf); + } + + // If the expand period was very long, reset NetEQ since it is likely that the + // sender was restarted. + if (num_consecutive_expands_ > kReinitAfterExpands) { + *reset_decoder = true; + return kNormal; + } + + const uint32_t five_seconds_samples = + static_cast(5 * 8000 * fs_mult_); + // Check if the required packet is available. + if (target_timestamp == available_timestamp) { + return ExpectedPacketAvailable(prev_mode, play_dtmf); + } else if (!PacketBuffer::IsObsoleteTimestamp( + available_timestamp, target_timestamp, five_seconds_samples)) { + return FuturePacketAvailable(sync_buffer, expand, decoder_frame_length, + prev_mode, target_timestamp, + available_timestamp, play_dtmf, + generated_noise_samples); + } else { + // This implies that available_timestamp < target_timestamp, which can + // happen when a new stream or codec is received. Signal for a reset. + return kUndefined; + } +} + +Operations DecisionLogicNormal::CngOperation(Modes prev_mode, + uint32_t target_timestamp, + uint32_t available_timestamp, + size_t generated_noise_samples) { + // Signed difference between target and available timestamp. + int32_t timestamp_diff = static_cast( + static_cast(generated_noise_samples + target_timestamp) - + available_timestamp); + int32_t optimal_level_samp = static_cast( + (delay_manager_->TargetLevel() * packet_length_samples_) >> 8); + const int64_t excess_waiting_time_samp = + -static_cast(timestamp_diff) - optimal_level_samp; + + if (excess_waiting_time_samp > optimal_level_samp / 2) { + // The waiting time for this packet will be longer than 1.5 + // times the wanted buffer delay. Apply fast-forward to cut the + // waiting time down to the optimal. + noise_fast_forward_ = rtc::dchecked_cast(noise_fast_forward_ + + excess_waiting_time_samp); + timestamp_diff = + rtc::saturated_cast(timestamp_diff + excess_waiting_time_samp); + } + + if (timestamp_diff < 0 && prev_mode == kModeRfc3389Cng) { + // Not time to play this packet yet. Wait another round before using this + // packet. Keep on playing CNG from previous CNG parameters. + return kRfc3389CngNoPacket; + } else { + // Otherwise, go for the CNG packet now. + noise_fast_forward_ = 0; + return kRfc3389Cng; + } +} + +Operations DecisionLogicNormal::NoPacket(bool play_dtmf) { + if (cng_state_ == kCngRfc3389On) { + // Keep on playing comfort noise. + return kRfc3389CngNoPacket; + } else if (cng_state_ == kCngInternalOn) { + // Keep on playing codec internal comfort noise. + return kCodecInternalCng; + } else if (play_dtmf) { + return kDtmf; + } else { + // Nothing to play, do expand. + return kExpand; + } +} + +Operations DecisionLogicNormal::ExpectedPacketAvailable(Modes prev_mode, + bool play_dtmf) { + if (prev_mode != kModeExpand && !play_dtmf) { + // Check criterion for time-stretching. + int low_limit, high_limit; + delay_manager_->BufferLimits(&low_limit, &high_limit); + if (buffer_level_filter_->filtered_current_level() >= high_limit << 2) + return kFastAccelerate; + if (TimescaleAllowed()) { + if (buffer_level_filter_->filtered_current_level() >= high_limit) + return kAccelerate; + if (buffer_level_filter_->filtered_current_level() < low_limit) + return kPreemptiveExpand; + } + } + return kNormal; +} + +Operations DecisionLogicNormal::FuturePacketAvailable( + const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + Modes prev_mode, + uint32_t target_timestamp, + uint32_t available_timestamp, + bool play_dtmf, + size_t generated_noise_samples) { + // Required packet is not available, but a future packet is. + // Check if we should continue with an ongoing expand because the new packet + // is too far into the future. + uint32_t timestamp_leap = available_timestamp - target_timestamp; + if ((prev_mode == kModeExpand) && + !ReinitAfterExpands(timestamp_leap) && + !MaxWaitForPacket() && + PacketTooEarly(timestamp_leap) && + UnderTargetLevel()) { + if (play_dtmf) { + // Still have DTMF to play, so do not do expand. + return kDtmf; + } else { + // Nothing to play. + return kExpand; + } + } + + const size_t samples_left = + sync_buffer.FutureLength() - expand.overlap_length(); + const size_t cur_size_samples = samples_left + + packet_buffer_.NumPacketsInBuffer() * decoder_frame_length; + + // If previous was comfort noise, then no merge is needed. + if (prev_mode == kModeRfc3389Cng || + prev_mode == kModeCodecInternalCng) { + // Keep the same delay as before the CNG, but make sure that the number of + // samples in buffer is no higher than 4 times the optimal level. (Note that + // TargetLevel() is in Q8.) + if (static_cast(generated_noise_samples + target_timestamp) >= + available_timestamp || + cur_size_samples > + ((delay_manager_->TargetLevel() * packet_length_samples_) >> 8) * + 4) { + // Time to play this new packet. + return kNormal; + } else { + // Too early to play this new packet; keep on playing comfort noise. + if (prev_mode == kModeRfc3389Cng) { + return kRfc3389CngNoPacket; + } else { // prevPlayMode == kModeCodecInternalCng. + return kCodecInternalCng; + } + } + } + // Do not merge unless we have done an expand before. + if (prev_mode == kModeExpand) { + return kMerge; + } else if (play_dtmf) { + // Play DTMF instead of expand. + return kDtmf; + } else { + return kExpand; + } +} + +bool DecisionLogicNormal::UnderTargetLevel() const { + return buffer_level_filter_->filtered_current_level() <= + delay_manager_->TargetLevel(); +} + +bool DecisionLogicNormal::ReinitAfterExpands(uint32_t timestamp_leap) const { + return timestamp_leap >= + static_cast(output_size_samples_ * kReinitAfterExpands); +} + +bool DecisionLogicNormal::PacketTooEarly(uint32_t timestamp_leap) const { + return timestamp_leap > + static_cast(output_size_samples_ * num_consecutive_expands_); +} + +bool DecisionLogicNormal::MaxWaitForPacket() const { + return num_consecutive_expands_ >= kMaxWaitForPacket; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.h new file mode 100644 index 0000000000..366d10380d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_NORMAL_H_ +#define MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_NORMAL_H_ + +#include "modules/audio_coding/neteq/decision_logic.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Implementation of the DecisionLogic class for playout modes kPlayoutOn and +// kPlayoutStreaming. +class DecisionLogicNormal : public DecisionLogic { + public: + // Constructor. + DecisionLogicNormal(int fs_hz, + size_t output_size_samples, + NetEqPlayoutMode playout_mode, + DecoderDatabase* decoder_database, + const PacketBuffer& packet_buffer, + DelayManager* delay_manager, + BufferLevelFilter* buffer_level_filter, + const TickTimer* tick_timer) + : DecisionLogic(fs_hz, + output_size_samples, + playout_mode, + decoder_database, + packet_buffer, + delay_manager, + buffer_level_filter, + tick_timer) {} + + protected: + static const int kReinitAfterExpands = 100; + static const int kMaxWaitForPacket = 10; + + Operations GetDecisionSpecialized(const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + const Packet* next_packet, + Modes prev_mode, + bool play_dtmf, + bool* reset_decoder, + size_t generated_noise_samples) override; + + // Returns the operation to do given that the expected packet is not + // available, but a packet further into the future is at hand. + virtual Operations FuturePacketAvailable( + const SyncBuffer& sync_buffer, + const Expand& expand, + size_t decoder_frame_length, + Modes prev_mode, + uint32_t target_timestamp, + uint32_t available_timestamp, + bool play_dtmf, + size_t generated_noise_samples); + + // Returns the operation to do given that the expected packet is available. + virtual Operations ExpectedPacketAvailable(Modes prev_mode, bool play_dtmf); + + // Returns the operation given that no packets are available (except maybe + // a DTMF event, flagged by setting |play_dtmf| true). + virtual Operations NoPacket(bool play_dtmf); + + private: + // Returns the operation given that the next available packet is a comfort + // noise payload (RFC 3389 only, not codec-internal). + Operations CngOperation(Modes prev_mode, + uint32_t target_timestamp, + uint32_t available_timestamp, + size_t generated_noise_samples); + + // Checks if enough time has elapsed since the last successful timescale + // operation was done (i.e., accelerate or preemptive expand). + bool TimescaleAllowed() const { + return !timescale_countdown_ || timescale_countdown_->Finished(); + } + + // Checks if the current (filtered) buffer level is under the target level. + bool UnderTargetLevel() const; + + // Checks if |timestamp_leap| is so long into the future that a reset due + // to exceeding kReinitAfterExpands will be done. + bool ReinitAfterExpands(uint32_t timestamp_leap) const; + + // Checks if we still have not done enough expands to cover the distance from + // the last decoded packet to the next available packet, the distance beeing + // conveyed in |timestamp_leap|. + bool PacketTooEarly(uint32_t timestamp_leap) const; + + // Checks if num_consecutive_expands_ >= kMaxWaitForPacket. + bool MaxWaitForPacket() const; + + RTC_DISALLOW_COPY_AND_ASSIGN(DecisionLogicNormal); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_NORMAL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_unittest.cc new file mode 100644 index 0000000000..1a7bab93da --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_unittest.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for DecisionLogic class and derived classes. + +#include "modules/audio_coding/neteq/decision_logic.h" +#include "modules/audio_coding/neteq/buffer_level_filter.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "modules/audio_coding/neteq/delay_peak_detector.h" +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "modules/audio_coding/neteq/tick_timer.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder_factory.h" + +namespace webrtc { + +TEST(DecisionLogic, CreateAndDestroy) { + int fs_hz = 8000; + int output_size_samples = fs_hz / 100; // Samples per 10 ms. + DecoderDatabase decoder_database( + new rtc::RefCountedObject); + TickTimer tick_timer; + PacketBuffer packet_buffer(10, &tick_timer); + DelayPeakDetector delay_peak_detector(&tick_timer); + DelayManager delay_manager(240, &delay_peak_detector, &tick_timer); + BufferLevelFilter buffer_level_filter; + DecisionLogic* logic = DecisionLogic::Create( + fs_hz, output_size_samples, kPlayoutOn, &decoder_database, packet_buffer, + &delay_manager, &buffer_level_filter, &tick_timer); + delete logic; + logic = DecisionLogic::Create( + fs_hz, output_size_samples, kPlayoutStreaming, &decoder_database, + packet_buffer, &delay_manager, &buffer_level_filter, &tick_timer); + delete logic; + logic = DecisionLogic::Create( + fs_hz, output_size_samples, kPlayoutFax, &decoder_database, packet_buffer, + &delay_manager, &buffer_level_filter, &tick_timer); + delete logic; + logic = DecisionLogic::Create( + fs_hz, output_size_samples, kPlayoutOff, &decoder_database, packet_buffer, + &delay_manager, &buffer_level_filter, &tick_timer); + delete logic; +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.cc new file mode 100644 index 0000000000..d289b45cc0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.cc @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decoder_database.h" + +#include // pair + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +DecoderDatabase::DecoderDatabase( + const rtc::scoped_refptr& decoder_factory) + : active_decoder_type_(-1), + active_cng_decoder_type_(-1), + decoder_factory_(decoder_factory) {} + +DecoderDatabase::~DecoderDatabase() = default; + +DecoderDatabase::DecoderInfo::DecoderInfo(const SdpAudioFormat& audio_format, + AudioDecoderFactory* factory, + const std::string& codec_name) + : name_(codec_name), + audio_format_(audio_format), + factory_(factory), + external_decoder_(nullptr), + cng_decoder_(CngDecoder::Create(audio_format)), + subtype_(SubtypeFromFormat(audio_format)) {} + +DecoderDatabase::DecoderInfo::DecoderInfo(const SdpAudioFormat& audio_format, + AudioDecoderFactory* factory) + : DecoderInfo(audio_format, factory, audio_format.name) {} + +DecoderDatabase::DecoderInfo::DecoderInfo(NetEqDecoder ct, + AudioDecoderFactory* factory) + : DecoderInfo(*NetEqDecoderToSdpAudioFormat(ct), factory) {} + +DecoderDatabase::DecoderInfo::DecoderInfo(const SdpAudioFormat& audio_format, + AudioDecoder* ext_dec, + const std::string& codec_name) + : name_(codec_name), + audio_format_(audio_format), + factory_(nullptr), + external_decoder_(ext_dec), + subtype_(Subtype::kNormal) { + RTC_CHECK(ext_dec); +} + +DecoderDatabase::DecoderInfo::DecoderInfo(DecoderInfo&&) = default; +DecoderDatabase::DecoderInfo::~DecoderInfo() = default; + +bool DecoderDatabase::DecoderInfo::CanGetDecoder() const { + if (subtype_ == Subtype::kNormal && !external_decoder_ && !decoder_) { + // TODO(ossu): Keep a check here for now, since a number of tests create + // DecoderInfos without factories. + RTC_DCHECK(factory_); + return factory_->IsSupportedDecoder(audio_format_); + } else { + return true; + } +} + +AudioDecoder* DecoderDatabase::DecoderInfo::GetDecoder() const { + if (subtype_ != Subtype::kNormal) { + // These are handled internally, so they have no AudioDecoder objects. + return nullptr; + } + if (external_decoder_) { + RTC_DCHECK(!decoder_); + RTC_DCHECK(!cng_decoder_); + return external_decoder_; + } + if (!decoder_) { + // TODO(ossu): Keep a check here for now, since a number of tests create + // DecoderInfos without factories. + RTC_DCHECK(factory_); + decoder_ = factory_->MakeAudioDecoder(audio_format_); + } + RTC_DCHECK(decoder_) << "Failed to create: " << audio_format_; + return decoder_.get(); +} + +bool DecoderDatabase::DecoderInfo::IsType(const char* name) const { + return STR_CASE_CMP(audio_format_.name.c_str(), name) == 0; +} + +bool DecoderDatabase::DecoderInfo::IsType(const std::string& name) const { + return IsType(name.c_str()); +} + +rtc::Optional +DecoderDatabase::DecoderInfo::CngDecoder::Create(const SdpAudioFormat& format) { + if (STR_CASE_CMP(format.name.c_str(), "CN") == 0) { + // CN has a 1:1 RTP clock rate to sample rate ratio. + const int sample_rate_hz = format.clockrate_hz; + RTC_DCHECK(sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000); + return DecoderDatabase::DecoderInfo::CngDecoder{sample_rate_hz}; + } else { + return rtc::nullopt; + } +} + +DecoderDatabase::DecoderInfo::Subtype +DecoderDatabase::DecoderInfo::SubtypeFromFormat(const SdpAudioFormat& format) { + if (STR_CASE_CMP(format.name.c_str(), "CN") == 0) { + return Subtype::kComfortNoise; + } else if (STR_CASE_CMP(format.name.c_str(), "telephone-event") == 0) { + return Subtype::kDtmf; + } else if (STR_CASE_CMP(format.name.c_str(), "red") == 0) { + return Subtype::kRed; + } + + return Subtype::kNormal; +} + +bool DecoderDatabase::Empty() const { return decoders_.empty(); } + +int DecoderDatabase::Size() const { return static_cast(decoders_.size()); } + +void DecoderDatabase::Reset() { + decoders_.clear(); + active_decoder_type_ = -1; + active_cng_decoder_type_ = -1; +} + +std::vector DecoderDatabase::SetCodecs( + const std::map& codecs) { + // First collect all payload types that we'll remove or reassign, then remove + // them from the database. + std::vector changed_payload_types; + for (const std::pair kv : decoders_) { + auto i = codecs.find(kv.first); + if (i == codecs.end() || i->second != kv.second.GetFormat()) { + changed_payload_types.push_back(kv.first); + } + } + for (int pl_type : changed_payload_types) { + Remove(pl_type); + } + + // Enter the new and changed payload type mappings into the database. + for (const auto& kv : codecs) { + const int& rtp_payload_type = kv.first; + const SdpAudioFormat& audio_format = kv.second; + RTC_DCHECK_GE(rtp_payload_type, 0); + RTC_DCHECK_LE(rtp_payload_type, 0x7f); + if (decoders_.count(rtp_payload_type) == 0) { + decoders_.insert(std::make_pair( + rtp_payload_type, DecoderInfo(audio_format, decoder_factory_.get()))); + } else { + // The mapping for this payload type hasn't changed. + } + } + + return changed_payload_types; +} + +int DecoderDatabase::RegisterPayload(uint8_t rtp_payload_type, + NetEqDecoder codec_type, + const std::string& name) { + if (rtp_payload_type > 0x7F) { + return kInvalidRtpPayloadType; + } + if (codec_type == NetEqDecoder::kDecoderArbitrary) { + return kCodecNotSupported; // Only supported through InsertExternal. + } + const auto opt_format = NetEqDecoderToSdpAudioFormat(codec_type); + if (!opt_format) { + return kCodecNotSupported; + } + DecoderInfo info(*opt_format, decoder_factory_, name); + if (!info.CanGetDecoder()) { + return kCodecNotSupported; + } + auto ret = + decoders_.insert(std::make_pair(rtp_payload_type, std::move(info))); + if (ret.second == false) { + // Database already contains a decoder with type |rtp_payload_type|. + return kDecoderExists; + } + return kOK; +} + +int DecoderDatabase::RegisterPayload(int rtp_payload_type, + const SdpAudioFormat& audio_format) { + if (rtp_payload_type < 0 || rtp_payload_type > 0x7f) { + return kInvalidRtpPayloadType; + } + const auto ret = decoders_.insert(std::make_pair( + rtp_payload_type, DecoderInfo(audio_format, decoder_factory_.get()))); + if (ret.second == false) { + // Database already contains a decoder with type |rtp_payload_type|. + return kDecoderExists; + } + return kOK; +} + +int DecoderDatabase::InsertExternal(uint8_t rtp_payload_type, + NetEqDecoder codec_type, + const std::string& codec_name, + AudioDecoder* decoder) { + if (rtp_payload_type > 0x7F) { + return kInvalidRtpPayloadType; + } + if (!decoder) { + return kInvalidPointer; + } + + const auto opt_db_format = NetEqDecoderToSdpAudioFormat(codec_type); + const SdpAudioFormat format = opt_db_format.value_or({"arbitrary", 0, 0}); + + std::pair ret; + DecoderInfo info(format, decoder, codec_name); + ret = decoders_.insert(std::make_pair(rtp_payload_type, std::move(info))); + if (ret.second == false) { + // Database already contains a decoder with type |rtp_payload_type|. + return kDecoderExists; + } + return kOK; +} + +int DecoderDatabase::Remove(uint8_t rtp_payload_type) { + if (decoders_.erase(rtp_payload_type) == 0) { + // No decoder with that |rtp_payload_type|. + return kDecoderNotFound; + } + if (active_decoder_type_ == rtp_payload_type) { + active_decoder_type_ = -1; // No active decoder. + } + if (active_cng_decoder_type_ == rtp_payload_type) { + active_cng_decoder_type_ = -1; // No active CNG decoder. + } + return kOK; +} + +void DecoderDatabase::RemoveAll() { + decoders_.clear(); + active_decoder_type_ = -1; // No active decoder. + active_cng_decoder_type_ = -1; // No active CNG decoder. +} + +const DecoderDatabase::DecoderInfo* DecoderDatabase::GetDecoderInfo( + uint8_t rtp_payload_type) const { + DecoderMap::const_iterator it = decoders_.find(rtp_payload_type); + if (it == decoders_.end()) { + // Decoder not found. + return NULL; + } + return &it->second; +} + +int DecoderDatabase::SetActiveDecoder(uint8_t rtp_payload_type, + bool* new_decoder) { + // Check that |rtp_payload_type| exists in the database. + const DecoderInfo *info = GetDecoderInfo(rtp_payload_type); + if (!info) { + // Decoder not found. + return kDecoderNotFound; + } + RTC_CHECK(!info->IsComfortNoise()); + RTC_DCHECK(new_decoder); + *new_decoder = false; + if (active_decoder_type_ < 0) { + // This is the first active decoder. + *new_decoder = true; + } else if (active_decoder_type_ != rtp_payload_type) { + // Moving from one active decoder to another. Delete the first one. + const DecoderInfo *old_info = GetDecoderInfo(active_decoder_type_); + RTC_DCHECK(old_info); + old_info->DropDecoder(); + *new_decoder = true; + } + active_decoder_type_ = rtp_payload_type; + return kOK; +} + +AudioDecoder* DecoderDatabase::GetActiveDecoder() const { + if (active_decoder_type_ < 0) { + // No active decoder. + return NULL; + } + return GetDecoder(active_decoder_type_); +} + +int DecoderDatabase::SetActiveCngDecoder(uint8_t rtp_payload_type) { + // Check that |rtp_payload_type| exists in the database. + const DecoderInfo *info = GetDecoderInfo(rtp_payload_type); + if (!info) { + // Decoder not found. + return kDecoderNotFound; + } + if (active_cng_decoder_type_ >= 0 && + active_cng_decoder_type_ != rtp_payload_type) { + // Moving from one active CNG decoder to another. Delete the first one. + RTC_DCHECK(active_cng_decoder_); + active_cng_decoder_.reset(); + } + active_cng_decoder_type_ = rtp_payload_type; + return kOK; +} + +ComfortNoiseDecoder* DecoderDatabase::GetActiveCngDecoder() const { + if (active_cng_decoder_type_ < 0) { + // No active CNG decoder. + return NULL; + } + if (!active_cng_decoder_) { + active_cng_decoder_.reset(new ComfortNoiseDecoder); + } + return active_cng_decoder_.get(); +} + +AudioDecoder* DecoderDatabase::GetDecoder(uint8_t rtp_payload_type) const { + const DecoderInfo *info = GetDecoderInfo(rtp_payload_type); + return info ? info->GetDecoder() : nullptr; +} + +bool DecoderDatabase::IsType(uint8_t rtp_payload_type, const char* name) const { + const DecoderInfo* info = GetDecoderInfo(rtp_payload_type); + return info && info->IsType(name); +} + +bool DecoderDatabase::IsType(uint8_t rtp_payload_type, + const std::string& name) const { + return IsType(rtp_payload_type, name.c_str()); +} + +bool DecoderDatabase::IsComfortNoise(uint8_t rtp_payload_type) const { + const DecoderInfo *info = GetDecoderInfo(rtp_payload_type); + return info && info->IsComfortNoise(); +} + +bool DecoderDatabase::IsDtmf(uint8_t rtp_payload_type) const { + const DecoderInfo *info = GetDecoderInfo(rtp_payload_type); + return info && info->IsDtmf(); +} + +bool DecoderDatabase::IsRed(uint8_t rtp_payload_type) const { + const DecoderInfo *info = GetDecoderInfo(rtp_payload_type); + return info && info->IsRed(); +} + +int DecoderDatabase::CheckPayloadTypes(const PacketList& packet_list) const { + PacketList::const_iterator it; + for (it = packet_list.begin(); it != packet_list.end(); ++it) { + if (!GetDecoderInfo(it->payload_type)) { + // Payload type is not found. + RTC_LOG(LS_WARNING) << "CheckPayloadTypes: unknown RTP payload type " + << static_cast(it->payload_type); + return kDecoderNotFound; + } + } + return kOK; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.h new file mode 100644 index 0000000000..5f0d173806 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.h @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DECODER_DATABASE_H_ +#define MODULES_AUDIO_CODING_NETEQ_DECODER_DATABASE_H_ + +#include +#include +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_format.h" +#include "common_types.h" // NOLINT(build/include) // NULL +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "modules/audio_coding/neteq/neteq_decoder_enum.h" +#include "modules/audio_coding/neteq/packet.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class DecoderDatabase { + public: + enum DatabaseReturnCodes { + kOK = 0, + kInvalidRtpPayloadType = -1, + kCodecNotSupported = -2, + kInvalidSampleRate = -3, + kDecoderExists = -4, + kDecoderNotFound = -5, + kInvalidPointer = -6 + }; + + // Class that stores decoder info in the database. + class DecoderInfo { + public: + DecoderInfo(const SdpAudioFormat& audio_format, + AudioDecoderFactory* factory, + const std::string& codec_name); + explicit DecoderInfo(const SdpAudioFormat& audio_format, + AudioDecoderFactory* factory = nullptr); + explicit DecoderInfo(NetEqDecoder ct, + AudioDecoderFactory* factory = nullptr); + DecoderInfo(const SdpAudioFormat& audio_format, + AudioDecoder* ext_dec, + const std::string& codec_name); + DecoderInfo(DecoderInfo&&); + ~DecoderInfo(); + + // Was this info object created with a specification that allows us to + // actually produce a decoder? + bool CanGetDecoder() const; + + // Get the AudioDecoder object, creating it first if necessary. + AudioDecoder* GetDecoder() const; + + // Delete the AudioDecoder object, unless it's external. (This means we can + // always recreate it later if we need it.) + void DropDecoder() const { decoder_.reset(); } + + int SampleRateHz() const { + if (IsDtmf()) { + // DTMF has a 1:1 mapping between clock rate and sample rate. + return audio_format_.clockrate_hz; + } + const AudioDecoder* decoder = GetDecoder(); + RTC_DCHECK_EQ(1, !!decoder + !!cng_decoder_); + return decoder ? decoder->SampleRateHz() : cng_decoder_->sample_rate_hz; + } + + const SdpAudioFormat& GetFormat() const { return audio_format_; } + + // Returns true if the decoder's format is comfort noise. + bool IsComfortNoise() const { + RTC_DCHECK_EQ(!!cng_decoder_, subtype_ == Subtype::kComfortNoise); + return subtype_ == Subtype::kComfortNoise; + } + + // Returns true if the decoder's format is DTMF. + bool IsDtmf() const { + return subtype_ == Subtype::kDtmf; + } + + // Returns true if the decoder's format is RED. + bool IsRed() const { + return subtype_ == Subtype::kRed; + } + + // Returns true if the decoder's format is named |name|. + bool IsType(const char* name) const; + // Returns true if the decoder's format is named |name|. + bool IsType(const std::string& name) const; + + const std::string& get_name() const { return name_; } + + private: + // TODO(ossu): |name_| is kept here while we retain the old external + // decoder interface. Remove this once using an + // AudioDecoderFactory has supplanted the old functionality. + const std::string name_; + + const SdpAudioFormat audio_format_; + AudioDecoderFactory* const factory_; + mutable std::unique_ptr decoder_; + + // Set iff this is an external decoder. + AudioDecoder* const external_decoder_; + + // Set iff this is a comfort noise decoder. + struct CngDecoder { + static rtc::Optional Create(const SdpAudioFormat& format); + int sample_rate_hz; + }; + const rtc::Optional cng_decoder_; + + enum class Subtype : int8_t { + kNormal, + kComfortNoise, + kDtmf, + kRed + }; + + static Subtype SubtypeFromFormat(const SdpAudioFormat& format); + + const Subtype subtype_; + }; + + // Maximum value for 8 bits, and an invalid RTP payload type (since it is + // only 7 bits). + static const uint8_t kRtpPayloadTypeError = 0xFF; + + DecoderDatabase( + const rtc::scoped_refptr& decoder_factory); + + virtual ~DecoderDatabase(); + + // Returns true if the database is empty. + virtual bool Empty() const; + + // Returns the number of decoders registered in the database. + virtual int Size() const; + + // Resets the database, erasing all registered payload types, and deleting + // any AudioDecoder objects that were not externally created and inserted + // using InsertExternal(). + virtual void Reset(); + + // Replaces the existing set of decoders with the given set. Returns the + // payload types that were reassigned or removed while doing so. + virtual std::vector SetCodecs( + const std::map& codecs); + + // Registers |rtp_payload_type| as a decoder of type |codec_type|. The |name| + // is only used to populate the name field in the DecoderInfo struct in the + // database, and can be arbitrary (including empty). Returns kOK on success; + // otherwise an error code. + virtual int RegisterPayload(uint8_t rtp_payload_type, + NetEqDecoder codec_type, + const std::string& name); + + // Registers a decoder for the given payload type. Returns kOK on success; + // otherwise an error code. + virtual int RegisterPayload(int rtp_payload_type, + const SdpAudioFormat& audio_format); + + // Registers an externally created AudioDecoder object, and associates it + // as a decoder of type |codec_type| with |rtp_payload_type|. + virtual int InsertExternal(uint8_t rtp_payload_type, + NetEqDecoder codec_type, + const std::string& codec_name, + AudioDecoder* decoder); + + // Removes the entry for |rtp_payload_type| from the database. + // Returns kDecoderNotFound or kOK depending on the outcome of the operation. + virtual int Remove(uint8_t rtp_payload_type); + + // Remove all entries. + virtual void RemoveAll(); + + // Returns a pointer to the DecoderInfo struct for |rtp_payload_type|. If + // no decoder is registered with that |rtp_payload_type|, NULL is returned. + virtual const DecoderInfo* GetDecoderInfo(uint8_t rtp_payload_type) const; + + // Sets the active decoder to be |rtp_payload_type|. If this call results in a + // change of active decoder, |new_decoder| is set to true. The previous active + // decoder's AudioDecoder object is deleted. + virtual int SetActiveDecoder(uint8_t rtp_payload_type, bool* new_decoder); + + // Returns the current active decoder, or NULL if no active decoder exists. + virtual AudioDecoder* GetActiveDecoder() const; + + // Sets the active comfort noise decoder to be |rtp_payload_type|. If this + // call results in a change of active comfort noise decoder, the previous + // active decoder's AudioDecoder object is deleted. + virtual int SetActiveCngDecoder(uint8_t rtp_payload_type); + + // Returns the current active comfort noise decoder, or NULL if no active + // comfort noise decoder exists. + virtual ComfortNoiseDecoder* GetActiveCngDecoder() const; + + // The following are utility methods: they will look up DecoderInfo through + // GetDecoderInfo and call the respective method on that info object, if it + // exists. + + // Returns a pointer to the AudioDecoder object associated with + // |rtp_payload_type|, or NULL if none is registered. If the AudioDecoder + // object does not exist for that decoder, the object is created. + AudioDecoder* GetDecoder(uint8_t rtp_payload_type) const; + + // Returns if |rtp_payload_type| is registered with a format named |name|. + bool IsType(uint8_t rtp_payload_type, const char* name) const; + + // Returns if |rtp_payload_type| is registered with a format named |name|. + bool IsType(uint8_t rtp_payload_type, const std::string& name) const; + + // Returns true if |rtp_payload_type| is registered as comfort noise. + bool IsComfortNoise(uint8_t rtp_payload_type) const; + + // Returns true if |rtp_payload_type| is registered as DTMF. + bool IsDtmf(uint8_t rtp_payload_type) const; + + // Returns true if |rtp_payload_type| is registered as RED. + bool IsRed(uint8_t rtp_payload_type) const; + + // Returns kOK if all packets in |packet_list| carry payload types that are + // registered in the database. Otherwise, returns kDecoderNotFound. + int CheckPayloadTypes(const PacketList& packet_list) const; + + private: + typedef std::map DecoderMap; + + DecoderMap decoders_; + int active_decoder_type_; + int active_cng_decoder_type_; + mutable std::unique_ptr active_cng_decoder_; + rtc::scoped_refptr decoder_factory_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DecoderDatabase); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DECODER_DATABASE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database_unittest.cc new file mode 100644 index 0000000000..53fbadf116 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database_unittest.cc @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decoder_database.h" + +#include +#include + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "rtc_base/refcountedobject.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder.h" +#include "test/mock_audio_decoder_factory.h" + +using testing::_; +using testing::Invoke; + +namespace webrtc { + +TEST(DecoderDatabase, CreateAndDestroy) { + DecoderDatabase db(new rtc::RefCountedObject); + EXPECT_EQ(0, db.Size()); + EXPECT_TRUE(db.Empty()); +} + +TEST(DecoderDatabase, InsertAndRemove) { + rtc::scoped_refptr factory( + new rtc::RefCountedObject); + EXPECT_CALL(*factory, IsSupportedDecoder(_)) + .WillOnce(Invoke([](const SdpAudioFormat& format) { + EXPECT_EQ("pcmu", format.name); + return true; + })); + DecoderDatabase db(factory); + const uint8_t kPayloadType = 0; + const std::string kCodecName = "Robert\'); DROP TABLE Students;"; + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadType, NetEqDecoder::kDecoderPCMu, kCodecName)); + EXPECT_EQ(1, db.Size()); + EXPECT_FALSE(db.Empty()); + EXPECT_EQ(DecoderDatabase::kOK, db.Remove(kPayloadType)); + EXPECT_EQ(0, db.Size()); + EXPECT_TRUE(db.Empty()); +} + +TEST(DecoderDatabase, InsertAndRemoveAll) { + rtc::scoped_refptr factory( + new rtc::RefCountedObject); + EXPECT_CALL(*factory, IsSupportedDecoder(_)) + .WillOnce(Invoke([](const SdpAudioFormat& format) { + EXPECT_EQ("pcmu", format.name); + return true; + })) + .WillOnce(Invoke([](const SdpAudioFormat& format) { + EXPECT_EQ("pcma", format.name); + return true; + })); + DecoderDatabase db(factory); + const std::string kCodecName1 = "Robert\'); DROP TABLE Students;"; + const std::string kCodecName2 = "https://xkcd.com/327/"; + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(0, NetEqDecoder::kDecoderPCMu, kCodecName1)); + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(1, NetEqDecoder::kDecoderPCMa, kCodecName2)); + EXPECT_EQ(2, db.Size()); + EXPECT_FALSE(db.Empty()); + db.RemoveAll(); + EXPECT_EQ(0, db.Size()); + EXPECT_TRUE(db.Empty()); +} + +TEST(DecoderDatabase, GetDecoderInfo) { + rtc::scoped_refptr factory( + new rtc::RefCountedObject); + EXPECT_CALL(*factory, IsSupportedDecoder(_)) + .WillOnce(Invoke([](const SdpAudioFormat& format) { + EXPECT_EQ("pcmu", format.name); + return true; + })); + auto* decoder = new MockAudioDecoder; + EXPECT_CALL(*factory, MakeAudioDecoderMock(_, _)) + .WillOnce(Invoke([decoder](const SdpAudioFormat& format, + std::unique_ptr* dec) { + EXPECT_EQ("pcmu", format.name); + dec->reset(decoder); + })); + DecoderDatabase db(factory); + const uint8_t kPayloadType = 0; + const std::string kCodecName = "Robert\'); DROP TABLE Students;"; + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadType, NetEqDecoder::kDecoderPCMu, kCodecName)); + const DecoderDatabase::DecoderInfo* info; + info = db.GetDecoderInfo(kPayloadType); + ASSERT_TRUE(info != NULL); + EXPECT_TRUE(info->IsType("pcmu")); + EXPECT_EQ(kCodecName, info->get_name()); + EXPECT_EQ(decoder, db.GetDecoder(kPayloadType)); + info = db.GetDecoderInfo(kPayloadType + 1); // Other payload type. + EXPECT_TRUE(info == NULL); // Should not be found. +} + +TEST(DecoderDatabase, GetDecoder) { + DecoderDatabase db(CreateBuiltinAudioDecoderFactory()); + const uint8_t kPayloadType = 0; + const std::string kCodecName = "Robert\'); DROP TABLE Students;"; + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(kPayloadType, NetEqDecoder::kDecoderPCM16B, + kCodecName)); + AudioDecoder* dec = db.GetDecoder(kPayloadType); + ASSERT_TRUE(dec != NULL); +} + +TEST(DecoderDatabase, TypeTests) { + rtc::scoped_refptr factory( + new rtc::RefCountedObject); + EXPECT_CALL(*factory, IsSupportedDecoder(_)) + .WillOnce(Invoke([](const SdpAudioFormat& format) { + EXPECT_EQ("pcmu", format.name); + return true; + })); + DecoderDatabase db(factory); + const uint8_t kPayloadTypePcmU = 0; + const uint8_t kPayloadTypeCng = 13; + const uint8_t kPayloadTypeDtmf = 100; + const uint8_t kPayloadTypeRed = 101; + const uint8_t kPayloadNotUsed = 102; + // Load into database. + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypePcmU, NetEqDecoder::kDecoderPCMu, "pcmu")); + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypeCng, NetEqDecoder::kDecoderCNGnb, + "cng-nb")); + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypeDtmf, NetEqDecoder::kDecoderAVT, "avt")); + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypeRed, NetEqDecoder::kDecoderRED, "red")); + EXPECT_EQ(4, db.Size()); + // Test. + EXPECT_FALSE(db.IsComfortNoise(kPayloadNotUsed)); + EXPECT_FALSE(db.IsDtmf(kPayloadNotUsed)); + EXPECT_FALSE(db.IsRed(kPayloadNotUsed)); + EXPECT_FALSE(db.IsComfortNoise(kPayloadTypePcmU)); + EXPECT_FALSE(db.IsDtmf(kPayloadTypePcmU)); + EXPECT_FALSE(db.IsRed(kPayloadTypePcmU)); + EXPECT_FALSE(db.IsType(kPayloadTypePcmU, "isac")); + EXPECT_TRUE(db.IsType(kPayloadTypePcmU, "pcmu")); + EXPECT_TRUE(db.IsComfortNoise(kPayloadTypeCng)); + EXPECT_TRUE(db.IsDtmf(kPayloadTypeDtmf)); + EXPECT_TRUE(db.IsRed(kPayloadTypeRed)); +} + +TEST(DecoderDatabase, ExternalDecoder) { + DecoderDatabase db(new rtc::RefCountedObject); + const uint8_t kPayloadType = 0; + const std::string kCodecName = "Robert\'); DROP TABLE Students;"; + MockAudioDecoder decoder; + // Load into database. + EXPECT_EQ(DecoderDatabase::kOK, + db.InsertExternal(kPayloadType, NetEqDecoder::kDecoderPCMu, + kCodecName, &decoder)); + EXPECT_EQ(1, db.Size()); + // Get decoder and make sure we get the external one. + EXPECT_EQ(&decoder, db.GetDecoder(kPayloadType)); + // Get the decoder info struct and check it too. + const DecoderDatabase::DecoderInfo* info; + info = db.GetDecoderInfo(kPayloadType); + ASSERT_TRUE(info != NULL); + EXPECT_TRUE(info->IsType("pcmu")); + EXPECT_EQ(info->get_name(), kCodecName); + EXPECT_EQ(kCodecName, info->get_name()); + // Expect not to delete the decoder when removing it from the database, since + // it was declared externally. + EXPECT_CALL(decoder, Die()).Times(0); + EXPECT_EQ(DecoderDatabase::kOK, db.Remove(kPayloadType)); + EXPECT_TRUE(db.Empty()); + + EXPECT_CALL(decoder, Die()).Times(1); // Will be called when |db| is deleted. +} + +TEST(DecoderDatabase, CheckPayloadTypes) { + constexpr int kNumPayloads = 10; + rtc::scoped_refptr factory( + new rtc::RefCountedObject); + EXPECT_CALL(*factory, IsSupportedDecoder(_)) + .Times(kNumPayloads) + .WillRepeatedly(Invoke([](const SdpAudioFormat& format) { + EXPECT_EQ("pcmu", format.name); + return true; + })); + DecoderDatabase db(factory); + // Load a number of payloads into the database. Payload types are 0, 1, ..., + // while the decoder type is the same for all payload types (this does not + // matter for the test). + for (uint8_t payload_type = 0; payload_type < kNumPayloads; ++payload_type) { + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(payload_type, NetEqDecoder::kDecoderPCMu, "")); + } + PacketList packet_list; + for (int i = 0; i < kNumPayloads + 1; ++i) { + // Create packet with payload type |i|. The last packet will have a payload + // type that is not registered in the decoder database. + Packet packet; + packet.payload_type = i; + packet_list.push_back(std::move(packet)); + } + + // Expect to return false, since the last packet is of an unknown type. + EXPECT_EQ(DecoderDatabase::kDecoderNotFound, + db.CheckPayloadTypes(packet_list)); + + packet_list.pop_back(); // Remove the unknown one. + + EXPECT_EQ(DecoderDatabase::kOK, db.CheckPayloadTypes(packet_list)); + + // Delete all packets. + PacketList::iterator it = packet_list.begin(); + while (it != packet_list.end()) { + it = packet_list.erase(it); + } +} + +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +#define IF_ISAC(x) x +#else +#define IF_ISAC(x) DISABLED_##x +#endif + +// Test the methods for setting and getting active speech and CNG decoders. +TEST(DecoderDatabase, IF_ISAC(ActiveDecoders)) { + DecoderDatabase db(CreateBuiltinAudioDecoderFactory()); + // Load payload types. + ASSERT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(0, NetEqDecoder::kDecoderPCMu, "pcmu")); + ASSERT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(103, NetEqDecoder::kDecoderISAC, "isac")); + ASSERT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(13, NetEqDecoder::kDecoderCNGnb, "cng-nb")); + // Verify that no decoders are active from the start. + EXPECT_EQ(NULL, db.GetActiveDecoder()); + EXPECT_EQ(NULL, db.GetActiveCngDecoder()); + + // Set active speech codec. + bool changed; // Should be true when the active decoder changed. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(0, &changed)); + EXPECT_TRUE(changed); + AudioDecoder* decoder = db.GetActiveDecoder(); + ASSERT_FALSE(decoder == NULL); // Should get a decoder here. + + // Set the same again. Expect no change. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(0, &changed)); + EXPECT_FALSE(changed); + decoder = db.GetActiveDecoder(); + ASSERT_FALSE(decoder == NULL); // Should get a decoder here. + + // Change active decoder. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(103, &changed)); + EXPECT_TRUE(changed); + decoder = db.GetActiveDecoder(); + ASSERT_FALSE(decoder == NULL); // Should get a decoder here. + + // Remove the active decoder, and verify that the active becomes NULL. + EXPECT_EQ(DecoderDatabase::kOK, db.Remove(103)); + EXPECT_EQ(NULL, db.GetActiveDecoder()); + + // Set active CNG codec. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveCngDecoder(13)); + ComfortNoiseDecoder* cng = db.GetActiveCngDecoder(); + ASSERT_FALSE(cng == NULL); // Should get a decoder here. + + // Remove the active CNG decoder, and verify that the active becomes NULL. + EXPECT_EQ(DecoderDatabase::kOK, db.Remove(13)); + EXPECT_EQ(NULL, db.GetActiveCngDecoder()); + + // Try to set non-existing codecs as active. + EXPECT_EQ(DecoderDatabase::kDecoderNotFound, + db.SetActiveDecoder(17, &changed)); + EXPECT_EQ(DecoderDatabase::kDecoderNotFound, + db.SetActiveCngDecoder(17)); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/defines.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/defines.h new file mode 100644 index 0000000000..496a36dfa5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/defines.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DEFINES_H_ +#define MODULES_AUDIO_CODING_NETEQ_DEFINES_H_ + +namespace webrtc { + +enum Operations { + kNormal = 0, + kMerge, + kExpand, + kAccelerate, + kFastAccelerate, + kPreemptiveExpand, + kRfc3389Cng, + kRfc3389CngNoPacket, + kCodecInternalCng, + kDtmf, + kAlternativePlc, + kAlternativePlcIncreaseTimestamp, + kAudioRepetition, + kAudioRepetitionIncreaseTimestamp, + kUndefined = -1 +}; + +enum Modes { + kModeNormal = 0, + kModeExpand, + kModeMerge, + kModeAccelerateSuccess, + kModeAccelerateLowEnergy, + kModeAccelerateFail, + kModePreemptiveExpandSuccess, + kModePreemptiveExpandLowEnergy, + kModePreemptiveExpandFail, + kModeRfc3389Cng, + kModeCodecInternalCng, + kModeDtmf, + kModeError, + kModeUndefined = -1 +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DEFINES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.cc new file mode 100644 index 0000000000..dc854c7d57 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.cc @@ -0,0 +1,469 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/delay_manager.h" + +#include +#include + +#include // max, min +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/delay_peak_detector.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +DelayManager::DelayManager(size_t max_packets_in_buffer, + DelayPeakDetector* peak_detector, + const TickTimer* tick_timer) + : first_packet_received_(false), + max_packets_in_buffer_(max_packets_in_buffer), + iat_vector_(kMaxIat + 1, 0), + iat_factor_(0), + tick_timer_(tick_timer), + base_target_level_(4), // In Q0 domain. + target_level_(base_target_level_ << 8), // In Q8 domain. + packet_len_ms_(0), + streaming_mode_(false), + last_seq_no_(0), + last_timestamp_(0), + minimum_delay_ms_(0), + least_required_delay_ms_(target_level_), + maximum_delay_ms_(target_level_), + iat_cumulative_sum_(0), + max_iat_cumulative_sum_(0), + peak_detector_(*peak_detector), + last_pack_cng_or_dtmf_(1), + frame_length_change_experiment_( + field_trial::IsEnabled("WebRTC-Audio-NetEqFramelengthExperiment")) { + assert(peak_detector); // Should never be NULL. + Reset(); +} + +DelayManager::~DelayManager() {} + +const DelayManager::IATVector& DelayManager::iat_vector() const { + return iat_vector_; +} + +// Set the histogram vector to an exponentially decaying distribution +// iat_vector_[i] = 0.5^(i+1), i = 0, 1, 2, ... +// iat_vector_ is in Q30. +void DelayManager::ResetHistogram() { + // Set temp_prob to (slightly more than) 1 in Q14. This ensures that the sum + // of iat_vector_ is 1. + uint16_t temp_prob = 0x4002; // 16384 + 2 = 100000000000010 binary. + IATVector::iterator it = iat_vector_.begin(); + for (; it < iat_vector_.end(); it++) { + temp_prob >>= 1; + (*it) = temp_prob << 16; + } + base_target_level_ = 4; + target_level_ = base_target_level_ << 8; +} + +int DelayManager::Update(uint16_t sequence_number, + uint32_t timestamp, + int sample_rate_hz) { + if (sample_rate_hz <= 0) { + return -1; + } + + if (!first_packet_received_) { + // Prepare for next packet arrival. + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + last_seq_no_ = sequence_number; + last_timestamp_ = timestamp; + first_packet_received_ = true; + return 0; + } + + // Try calculating packet length from current and previous timestamps. + int packet_len_ms; + if (!IsNewerTimestamp(timestamp, last_timestamp_) || + !IsNewerSequenceNumber(sequence_number, last_seq_no_)) { + // Wrong timestamp or sequence order; use stored value. + packet_len_ms = packet_len_ms_; + } else { + // Calculate timestamps per packet and derive packet length in ms. + int64_t packet_len_samp = + static_cast(timestamp - last_timestamp_) / + static_cast(sequence_number - last_seq_no_); + packet_len_ms = + rtc::saturated_cast(1000 * packet_len_samp / sample_rate_hz); + } + + if (packet_len_ms > 0) { + // Cannot update statistics unless |packet_len_ms| is valid. + // Calculate inter-arrival time (IAT) in integer "packet times" + // (rounding down). This is the value used as index to the histogram + // vector |iat_vector_|. + int iat_packets = packet_iat_stopwatch_->ElapsedMs() / packet_len_ms; + + if (streaming_mode_) { + UpdateCumulativeSums(packet_len_ms, sequence_number); + } + + // Check for discontinuous packet sequence and re-ordering. + if (IsNewerSequenceNumber(sequence_number, last_seq_no_ + 1)) { + // Compensate for gap in the sequence numbers. Reduce IAT with the + // expected extra time due to lost packets, but ensure that the IAT is + // not negative. + iat_packets -= static_cast(sequence_number - last_seq_no_ - 1); + iat_packets = std::max(iat_packets, 0); + } else if (!IsNewerSequenceNumber(sequence_number, last_seq_no_)) { + iat_packets += static_cast(last_seq_no_ + 1 - sequence_number); + } + + // Saturate IAT at maximum value. + const int max_iat = kMaxIat; + iat_packets = std::min(iat_packets, max_iat); + UpdateHistogram(iat_packets); + // Calculate new |target_level_| based on updated statistics. + target_level_ = CalculateTargetLevel(iat_packets); + if (streaming_mode_) { + target_level_ = std::max(target_level_, max_iat_cumulative_sum_); + } + + LimitTargetLevel(); + } // End if (packet_len_ms > 0). + + // Prepare for next packet arrival. + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + last_seq_no_ = sequence_number; + last_timestamp_ = timestamp; + return 0; +} + +void DelayManager::UpdateCumulativeSums(int packet_len_ms, + uint16_t sequence_number) { + // Calculate IAT in Q8, including fractions of a packet (i.e., more + // accurate than |iat_packets|. + int iat_packets_q8 = + (packet_iat_stopwatch_->ElapsedMs() << 8) / packet_len_ms; + // Calculate cumulative sum IAT with sequence number compensation. The sum + // is zero if there is no clock-drift. + iat_cumulative_sum_ += (iat_packets_q8 - + (static_cast(sequence_number - last_seq_no_) << 8)); + // Subtract drift term. + iat_cumulative_sum_ -= kCumulativeSumDrift; + // Ensure not negative. + iat_cumulative_sum_ = std::max(iat_cumulative_sum_, 0); + if (iat_cumulative_sum_ > max_iat_cumulative_sum_) { + // Found a new maximum. + max_iat_cumulative_sum_ = iat_cumulative_sum_; + max_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + if (max_iat_stopwatch_->ElapsedMs() > kMaxStreamingPeakPeriodMs) { + // Too long since the last maximum was observed; decrease max value. + max_iat_cumulative_sum_ -= kCumulativeSumDrift; + } +} + +// Each element in the vector is first multiplied by the forgetting factor +// |iat_factor_|. Then the vector element indicated by |iat_packets| is then +// increased (additive) by 1 - |iat_factor_|. This way, the probability of +// |iat_packets| is slightly increased, while the sum of the histogram remains +// constant (=1). +// Due to inaccuracies in the fixed-point arithmetic, the histogram may no +// longer sum up to 1 (in Q30) after the update. To correct this, a correction +// term is added or subtracted from the first element (or elements) of the +// vector. +// The forgetting factor |iat_factor_| is also updated. When the DelayManager +// is reset, the factor is set to 0 to facilitate rapid convergence in the +// beginning. With each update of the histogram, the factor is increased towards +// the steady-state value |kIatFactor_|. +void DelayManager::UpdateHistogram(size_t iat_packets) { + assert(iat_packets < iat_vector_.size()); + int vector_sum = 0; // Sum up the vector elements as they are processed. + // Multiply each element in |iat_vector_| with |iat_factor_|. + for (IATVector::iterator it = iat_vector_.begin(); + it != iat_vector_.end(); ++it) { + *it = (static_cast(*it) * iat_factor_) >> 15; + vector_sum += *it; + } + + // Increase the probability for the currently observed inter-arrival time + // by 1 - |iat_factor_|. The factor is in Q15, |iat_vector_| in Q30. + // Thus, left-shift 15 steps to obtain result in Q30. + iat_vector_[iat_packets] += (32768 - iat_factor_) << 15; + vector_sum += (32768 - iat_factor_) << 15; // Add to vector sum. + + // |iat_vector_| should sum up to 1 (in Q30), but it may not due to + // fixed-point rounding errors. + vector_sum -= 1 << 30; // Should be zero. Compensate if not. + if (vector_sum != 0) { + // Modify a few values early in |iat_vector_|. + int flip_sign = vector_sum > 0 ? -1 : 1; + IATVector::iterator it = iat_vector_.begin(); + while (it != iat_vector_.end() && abs(vector_sum) > 0) { + // Add/subtract 1/16 of the element, but not more than |vector_sum|. + int correction = flip_sign * std::min(abs(vector_sum), (*it) >> 4); + *it += correction; + vector_sum += correction; + ++it; + } + } + assert(vector_sum == 0); // Verify that the above is correct. + + // Update |iat_factor_| (changes only during the first seconds after a reset). + // The factor converges to |kIatFactor_|. + iat_factor_ += (kIatFactor_ - iat_factor_ + 3) >> 2; +} + +// Enforces upper and lower limits for |target_level_|. The upper limit is +// chosen to be minimum of i) 75% of |max_packets_in_buffer_|, to leave some +// headroom for natural fluctuations around the target, and ii) equivalent of +// |maximum_delay_ms_| in packets. Note that in practice, if no +// |maximum_delay_ms_| is specified, this does not have any impact, since the +// target level is far below the buffer capacity in all reasonable cases. +// The lower limit is equivalent of |minimum_delay_ms_| in packets. We update +// |least_required_level_| while the above limits are applied. +// TODO(hlundin): Move this check to the buffer logistics class. +void DelayManager::LimitTargetLevel() { + least_required_delay_ms_ = (target_level_ * packet_len_ms_) >> 8; + + if (packet_len_ms_ > 0 && minimum_delay_ms_ > 0) { + int minimum_delay_packet_q8 = (minimum_delay_ms_ << 8) / packet_len_ms_; + target_level_ = std::max(target_level_, minimum_delay_packet_q8); + } + + if (maximum_delay_ms_ > 0 && packet_len_ms_ > 0) { + int maximum_delay_packet_q8 = (maximum_delay_ms_ << 8) / packet_len_ms_; + target_level_ = std::min(target_level_, maximum_delay_packet_q8); + } + + // Shift to Q8, then 75%.; + int max_buffer_packets_q8 = + static_cast((3 * (max_packets_in_buffer_ << 8)) / 4); + target_level_ = std::min(target_level_, max_buffer_packets_q8); + + // Sanity check, at least 1 packet (in Q8). + target_level_ = std::max(target_level_, 1 << 8); +} + +int DelayManager::CalculateTargetLevel(int iat_packets) { + int limit_probability = kLimitProbability; + if (streaming_mode_) { + limit_probability = kLimitProbabilityStreaming; + } + + // Calculate target buffer level from inter-arrival time histogram. + // Find the |iat_index| for which the probability of observing an + // inter-arrival time larger than or equal to |iat_index| is less than or + // equal to |limit_probability|. The sought probability is estimated using + // the histogram as the reverse cumulant PDF, i.e., the sum of elements from + // the end up until |iat_index|. Now, since the sum of all elements is 1 + // (in Q30) by definition, and since the solution is often a low value for + // |iat_index|, it is more efficient to start with |sum| = 1 and subtract + // elements from the start of the histogram. + size_t index = 0; // Start from the beginning of |iat_vector_|. + int sum = 1 << 30; // Assign to 1 in Q30. + sum -= iat_vector_[index]; // Ensure that target level is >= 1. + + do { + // Subtract the probabilities one by one until the sum is no longer greater + // than limit_probability. + ++index; + sum -= iat_vector_[index]; + } while ((sum > limit_probability) && (index < iat_vector_.size() - 1)); + + // This is the base value for the target buffer level. + int target_level = static_cast(index); + base_target_level_ = static_cast(index); + + // Update detector for delay peaks. + bool delay_peak_found = peak_detector_.Update(iat_packets, target_level); + if (delay_peak_found) { + target_level = std::max(target_level, peak_detector_.MaxPeakHeight()); + } + + // Sanity check. |target_level| must be strictly positive. + target_level = std::max(target_level, 1); + // Scale to Q8 and assign to member variable. + target_level_ = target_level << 8; + return target_level_; +} + +int DelayManager::SetPacketAudioLength(int length_ms) { + if (length_ms <= 0) { + RTC_LOG_F(LS_ERROR) << "length_ms = " << length_ms; + return -1; + } + if (frame_length_change_experiment_ && packet_len_ms_ != length_ms) { + iat_vector_ = ScaleHistogram(iat_vector_, packet_len_ms_, length_ms); + } + + packet_len_ms_ = length_ms; + peak_detector_.SetPacketAudioLength(packet_len_ms_); + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + last_pack_cng_or_dtmf_ = 1; // TODO(hlundin): Legacy. Remove? + return 0; +} + + +void DelayManager::Reset() { + packet_len_ms_ = 0; // Packet size unknown. + streaming_mode_ = false; + peak_detector_.Reset(); + ResetHistogram(); // Resets target levels too. + iat_factor_ = 0; // Adapt the histogram faster for the first few packets. + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + max_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + iat_cumulative_sum_ = 0; + max_iat_cumulative_sum_ = 0; + last_pack_cng_or_dtmf_ = 1; +} + +double DelayManager::EstimatedClockDriftPpm() const { + double sum = 0.0; + // Calculate the expected value based on the probabilities in |iat_vector_|. + for (size_t i = 0; i < iat_vector_.size(); ++i) { + sum += static_cast(iat_vector_[i]) * i; + } + // The probabilities in |iat_vector_| are in Q30. Divide by 1 << 30 to convert + // to Q0; subtract the nominal inter-arrival time (1) to make a zero + // clockdrift represent as 0; mulitply by 1000000 to produce parts-per-million + // (ppm). + return (sum / (1 << 30) - 1) * 1e6; +} + +bool DelayManager::PeakFound() const { + return peak_detector_.peak_found(); +} + +void DelayManager::ResetPacketIatCount() { + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); +} + +// Note that |low_limit| and |higher_limit| are not assigned to +// |minimum_delay_ms_| and |maximum_delay_ms_| defined by the client of this +// class. They are computed from |target_level_| and used for decision making. +void DelayManager::BufferLimits(int* lower_limit, int* higher_limit) const { + if (!lower_limit || !higher_limit) { + RTC_LOG_F(LS_ERROR) << "NULL pointers supplied as input"; + assert(false); + return; + } + + int window_20ms = 0x7FFF; // Default large value for legacy bit-exactness. + if (packet_len_ms_ > 0) { + window_20ms = (20 << 8) / packet_len_ms_; + } + + // |target_level_| is in Q8 already. + *lower_limit = (target_level_ * 3) / 4; + // |higher_limit| is equal to |target_level_|, but should at + // least be 20 ms higher than |lower_limit_|. + *higher_limit = std::max(target_level_, *lower_limit + window_20ms); +} + +int DelayManager::TargetLevel() const { + return target_level_; +} + +void DelayManager::LastDecodedWasCngOrDtmf(bool it_was) { + if (it_was) { + last_pack_cng_or_dtmf_ = 1; + } else if (last_pack_cng_or_dtmf_ != 0) { + last_pack_cng_or_dtmf_ = -1; + } +} + +void DelayManager::RegisterEmptyPacket() { + ++last_seq_no_; +} + +DelayManager::IATVector DelayManager::ScaleHistogram(const IATVector& histogram, + int old_packet_length, + int new_packet_length) { + if (old_packet_length == 0) { + // If we don't know the previous frame length, don't make any changes to the + // histogram. + return histogram; + } + RTC_DCHECK_GT(new_packet_length, 0); + RTC_DCHECK_EQ(old_packet_length % 10, 0); + RTC_DCHECK_EQ(new_packet_length % 10, 0); + IATVector new_histogram(histogram.size(), 0); + int acc = 0; + int time_counter = 0; + size_t new_histogram_idx = 0; + for (size_t i = 0; i < histogram.size(); i++) { + acc += histogram[i]; + time_counter += old_packet_length; + // The bins should be scaled, to ensure the histogram still sums to one. + const int scaled_acc = acc * new_packet_length / time_counter; + int actually_used_acc = 0; + while (time_counter >= new_packet_length) { + actually_used_acc += scaled_acc; + new_histogram[new_histogram_idx] += scaled_acc; + new_histogram_idx = + std::min(new_histogram_idx + 1, new_histogram.size() - 1); + time_counter -= new_packet_length; + } + // Only subtract the part that was succesfully written to the new histogram. + acc -= actually_used_acc; + } + // If there is anything left in acc (due to rounding errors), add it to the + // last bin. + new_histogram[new_histogram_idx] += acc; + RTC_DCHECK_EQ(histogram.size(), new_histogram.size()); + RTC_DCHECK_EQ(accumulate(histogram.begin(), histogram.end(), 0), + accumulate(new_histogram.begin(), new_histogram.end(), 0)); + return new_histogram; +} + +bool DelayManager::SetMinimumDelay(int delay_ms) { + // Minimum delay shouldn't be more than maximum delay, if any maximum is set. + // Also, if possible check |delay| to less than 75% of + // |max_packets_in_buffer_|. + if ((maximum_delay_ms_ > 0 && delay_ms > maximum_delay_ms_) || + (packet_len_ms_ > 0 && + delay_ms > + static_cast(3 * max_packets_in_buffer_ * packet_len_ms_ / 4))) { + return false; + } + minimum_delay_ms_ = delay_ms; + return true; +} + +bool DelayManager::SetMaximumDelay(int delay_ms) { + if (delay_ms == 0) { + // Zero input unsets the maximum delay. + maximum_delay_ms_ = 0; + return true; + } else if (delay_ms < minimum_delay_ms_ || delay_ms < packet_len_ms_) { + // Maximum delay shouldn't be less than minimum delay or less than a packet. + return false; + } + maximum_delay_ms_ = delay_ms; + return true; +} + +int DelayManager::least_required_delay_ms() const { + return least_required_delay_ms_; +} + +int DelayManager::base_target_level() const { return base_target_level_; } +void DelayManager::set_streaming_mode(bool value) { streaming_mode_ = value; } +int DelayManager::last_pack_cng_or_dtmf() const { + return last_pack_cng_or_dtmf_; +} + +void DelayManager::set_last_pack_cng_or_dtmf(int value) { + last_pack_cng_or_dtmf_ = value; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.h new file mode 100644 index 0000000000..0d082c8788 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DELAY_MANAGER_H_ +#define MODULES_AUDIO_CODING_NETEQ_DELAY_MANAGER_H_ + +#include // Provide access to size_t. + +#include +#include + +#include "modules/audio_coding/neteq/tick_timer.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declaration. +class DelayPeakDetector; + +class DelayManager { + public: + typedef std::vector IATVector; + + // Create a DelayManager object. Notify the delay manager that the packet + // buffer can hold no more than |max_packets_in_buffer| packets (i.e., this + // is the number of packet slots in the buffer). Supply a PeakDetector + // object to the DelayManager. + DelayManager(size_t max_packets_in_buffer, + DelayPeakDetector* peak_detector, + const TickTimer* tick_timer); + + virtual ~DelayManager(); + + // Read the inter-arrival time histogram. Mainly for testing purposes. + virtual const IATVector& iat_vector() const; + + // Updates the delay manager with a new incoming packet, with + // |sequence_number| and |timestamp| from the RTP header. This updates the + // inter-arrival time histogram and other statistics, as well as the + // associated DelayPeakDetector. A new target buffer level is calculated. + // Returns 0 on success, -1 on failure (invalid sample rate). + virtual int Update(uint16_t sequence_number, + uint32_t timestamp, + int sample_rate_hz); + + // Calculates a new target buffer level. Called from the Update() method. + // Sets target_level_ (in Q8) and returns the same value. Also calculates + // and updates base_target_level_, which is the target buffer level before + // taking delay peaks into account. + virtual int CalculateTargetLevel(int iat_packets); + + // Notifies the DelayManager of how much audio data is carried in each packet. + // The method updates the DelayPeakDetector too, and resets the inter-arrival + // time counter. Returns 0 on success, -1 on failure. + virtual int SetPacketAudioLength(int length_ms); + + // Resets the DelayManager and the associated DelayPeakDetector. + virtual void Reset(); + + // Calculates the average inter-arrival time deviation from the histogram. + // The result is returned as parts-per-million deviation from the nominal + // inter-arrival time. That is, if the average inter-arrival time is equal to + // the nominal frame time, the return value is zero. A positive value + // corresponds to packet spacing being too large, while a negative value means + // that the packets arrive with less spacing than expected. + virtual double EstimatedClockDriftPpm() const; + + // Returns true if peak-mode is active. That is, delay peaks were observed + // recently. This method simply asks for the same information from the + // DelayPeakDetector object. + virtual bool PeakFound() const; + + // Reset the inter-arrival time counter to 0. + virtual void ResetPacketIatCount(); + + // Writes the lower and higher limits which the buffer level should stay + // within to the corresponding pointers. The values are in (fractions of) + // packets in Q8. + virtual void BufferLimits(int* lower_limit, int* higher_limit) const; + + // Gets the target buffer level, in (fractions of) packets in Q8. This value + // includes any extra delay set through the set_extra_delay_ms() method. + virtual int TargetLevel() const; + + // Informs the delay manager whether or not the last decoded packet contained + // speech. + virtual void LastDecodedWasCngOrDtmf(bool it_was); + + // Notify the delay manager that empty packets have been received. These are + // packets that are part of the sequence number series, so that an empty + // packet will shift the sequence numbers for the following packets. + virtual void RegisterEmptyPacket(); + + // Apply compression or stretching to the IAT histogram, for a change in frame + // size. This returns an updated histogram. This function is public for + // testability. + static IATVector ScaleHistogram(const IATVector& histogram, + int old_packet_length, + int new_packet_length); + + // Accessors and mutators. + // Assuming |delay| is in valid range. + virtual bool SetMinimumDelay(int delay_ms); + virtual bool SetMaximumDelay(int delay_ms); + virtual int least_required_delay_ms() const; + virtual int base_target_level() const; + virtual void set_streaming_mode(bool value); + virtual int last_pack_cng_or_dtmf() const; + virtual void set_last_pack_cng_or_dtmf(int value); + + private: + static const int kLimitProbability = 53687091; // 1/20 in Q30. + static const int kLimitProbabilityStreaming = 536871; // 1/2000 in Q30. + static const int kMaxStreamingPeakPeriodMs = 600000; // 10 minutes in ms. + static const int kCumulativeSumDrift = 2; // Drift term for cumulative sum + // |iat_cumulative_sum_|. + // Steady-state forgetting factor for |iat_vector_|, 0.9993 in Q15. + static const int kIatFactor_ = 32745; + static const int kMaxIat = 64; // Max inter-arrival time to register. + + // Sets |iat_vector_| to the default start distribution and sets the + // |base_target_level_| and |target_level_| to the corresponding values. + void ResetHistogram(); + + // Updates |iat_cumulative_sum_| and |max_iat_cumulative_sum_|. (These are + // used by the streaming mode.) This method is called by Update(). + void UpdateCumulativeSums(int packet_len_ms, uint16_t sequence_number); + + // Updates the histogram |iat_vector_|. The probability for inter-arrival time + // equal to |iat_packets| (in integer packets) is increased slightly, while + // all other entries are decreased. This method is called by Update(). + void UpdateHistogram(size_t iat_packets); + + // Makes sure that |target_level_| is not too large, taking + // |max_packets_in_buffer_| and |extra_delay_ms_| into account. This method is + // called by Update(). + void LimitTargetLevel(); + + bool first_packet_received_; + const size_t max_packets_in_buffer_; // Capacity of the packet buffer. + IATVector iat_vector_; // Histogram of inter-arrival times. + int iat_factor_; // Forgetting factor for updating the IAT histogram (Q15). + const TickTimer* tick_timer_; + // Time elapsed since last packet. + std::unique_ptr packet_iat_stopwatch_; + int base_target_level_; // Currently preferred buffer level before peak + // detection and streaming mode (Q0). + // TODO(turajs) change the comment according to the implementation of + // minimum-delay. + int target_level_; // Currently preferred buffer level in (fractions) + // of packets (Q8), before adding any extra delay. + int packet_len_ms_; // Length of audio in each incoming packet [ms]. + bool streaming_mode_; + uint16_t last_seq_no_; // Sequence number for last received packet. + uint32_t last_timestamp_; // Timestamp for the last received packet. + int minimum_delay_ms_; // Externally set minimum delay. + int least_required_delay_ms_; // Smallest preferred buffer level (same unit + // as |target_level_|), before applying + // |minimum_delay_ms_| and/or |maximum_delay_ms_|. + int maximum_delay_ms_; // Externally set maximum allowed delay. + int iat_cumulative_sum_; // Cumulative sum of delta inter-arrival times. + int max_iat_cumulative_sum_; // Max of |iat_cumulative_sum_|. + // Time elapsed since maximum was observed. + std::unique_ptr max_iat_stopwatch_; + DelayPeakDetector& peak_detector_; + int last_pack_cng_or_dtmf_; + const bool frame_length_change_experiment_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DelayManager); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DELAY_MANAGER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager_unittest.cc new file mode 100644 index 0000000000..6dad4db9f4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager_unittest.cc @@ -0,0 +1,414 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for DelayManager class. + +#include "modules/audio_coding/neteq/delay_manager.h" + +#include + +#include "modules/audio_coding/neteq/mock/mock_delay_peak_detector.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::Return; +using ::testing::_; + +class DelayManagerTest : public ::testing::Test { + protected: + static const int kMaxNumberOfPackets = 240; + static const int kTimeStepMs = 10; + static const int kFs = 8000; + static const int kFrameSizeMs = 20; + static const int kTsIncrement = kFrameSizeMs * kFs / 1000; + + DelayManagerTest(); + virtual void SetUp(); + virtual void TearDown(); + void SetPacketAudioLength(int lengt_ms); + void InsertNextPacket(); + void IncreaseTime(int inc_ms); + + DelayManager* dm_; + TickTimer tick_timer_; + MockDelayPeakDetector detector_; + uint16_t seq_no_; + uint32_t ts_; +}; + +DelayManagerTest::DelayManagerTest() + : dm_(NULL), detector_(&tick_timer_), seq_no_(0x1234), ts_(0x12345678) {} + +void DelayManagerTest::SetUp() { + EXPECT_CALL(detector_, Reset()) + .Times(1); + dm_ = new DelayManager(kMaxNumberOfPackets, &detector_, &tick_timer_); +} + +void DelayManagerTest::SetPacketAudioLength(int lengt_ms) { + EXPECT_CALL(detector_, SetPacketAudioLength(lengt_ms)); + dm_->SetPacketAudioLength(lengt_ms); +} + +void DelayManagerTest::InsertNextPacket() { + EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs)); + seq_no_ += 1; + ts_ += kTsIncrement; +} + +void DelayManagerTest::IncreaseTime(int inc_ms) { + for (int t = 0; t < inc_ms; t += kTimeStepMs) { + tick_timer_.Increment(); + } +} +void DelayManagerTest::TearDown() { + EXPECT_CALL(detector_, Die()); + delete dm_; +} + +TEST_F(DelayManagerTest, CreateAndDestroy) { + // Nothing to do here. The test fixture creates and destroys the DelayManager + // object. +} + +TEST_F(DelayManagerTest, VectorInitialization) { + const DelayManager::IATVector& vec = dm_->iat_vector(); + double sum = 0.0; + for (size_t i = 0; i < vec.size(); i++) { + EXPECT_NEAR(ldexp(pow(0.5, static_cast(i + 1)), 30), vec[i], 65537); + // Tolerance 65537 in Q30 corresponds to a delta of approximately 0.00006. + sum += vec[i]; + } + EXPECT_EQ(1 << 30, static_cast(sum)); // Should be 1 in Q30. +} + +TEST_F(DelayManagerTest, SetPacketAudioLength) { + const int kLengthMs = 30; + // Expect DelayManager to pass on the new length to the detector object. + EXPECT_CALL(detector_, SetPacketAudioLength(kLengthMs)) + .Times(1); + EXPECT_EQ(0, dm_->SetPacketAudioLength(kLengthMs)); + EXPECT_EQ(-1, dm_->SetPacketAudioLength(-1)); // Illegal parameter value. +} + +TEST_F(DelayManagerTest, PeakFound) { + // Expect DelayManager to pass on the question to the detector. + // Call twice, and let the detector return true the first time and false the + // second time. + EXPECT_CALL(detector_, peak_found()) + .WillOnce(Return(true)) + .WillOnce(Return(false)); + EXPECT_TRUE(dm_->PeakFound()); + EXPECT_FALSE(dm_->PeakFound()); +} + +TEST_F(DelayManagerTest, UpdateNormal) { + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Advance time by one frame size. + IncreaseTime(kFrameSizeMs); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to 1 packet, and (base) target level equal to 1 as well. + // Return false to indicate no peaks found. + EXPECT_CALL(detector_, Update(1, 1)) + .WillOnce(Return(false)); + InsertNextPacket(); + EXPECT_EQ(1 << 8, dm_->TargetLevel()); // In Q8. + EXPECT_EQ(1, dm_->base_target_level()); + int lower, higher; + dm_->BufferLimits(&lower, &higher); + // Expect |lower| to be 75% of target level, and |higher| to be target level, + // but also at least 20 ms higher than |lower|, which is the limiting case + // here. + EXPECT_EQ((1 << 8) * 3 / 4, lower); + EXPECT_EQ(lower + (20 << 8) / kFrameSizeMs, higher); +} + +TEST_F(DelayManagerTest, UpdateLongInterArrivalTime) { + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Advance time by two frame size. + IncreaseTime(2 * kFrameSizeMs); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to 1 packet, and (base) target level equal to 1 as well. + // Return false to indicate no peaks found. + EXPECT_CALL(detector_, Update(2, 2)) + .WillOnce(Return(false)); + InsertNextPacket(); + EXPECT_EQ(2 << 8, dm_->TargetLevel()); // In Q8. + EXPECT_EQ(2, dm_->base_target_level()); + int lower, higher; + dm_->BufferLimits(&lower, &higher); + // Expect |lower| to be 75% of target level, and |higher| to be target level, + // but also at least 20 ms higher than |lower|, which is the limiting case + // here. + EXPECT_EQ((2 << 8) * 3 / 4, lower); + EXPECT_EQ(lower + (20 << 8) / kFrameSizeMs, higher); +} + +TEST_F(DelayManagerTest, UpdatePeakFound) { + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Advance time by one frame size. + IncreaseTime(kFrameSizeMs); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to 1 packet, and (base) target level equal to 1 as well. + // Return true to indicate that peaks are found. Let the peak height be 5. + EXPECT_CALL(detector_, Update(1, 1)) + .WillOnce(Return(true)); + EXPECT_CALL(detector_, MaxPeakHeight()) + .WillOnce(Return(5)); + InsertNextPacket(); + EXPECT_EQ(5 << 8, dm_->TargetLevel()); + EXPECT_EQ(1, dm_->base_target_level()); // Base target level is w/o peaks. + int lower, higher; + dm_->BufferLimits(&lower, &higher); + // Expect |lower| to be 75% of target level, and |higher| to be target level. + EXPECT_EQ((5 << 8) * 3 / 4, lower); + EXPECT_EQ(5 << 8, higher); +} + +TEST_F(DelayManagerTest, TargetDelay) { + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Advance time by one frame size. + IncreaseTime(kFrameSizeMs); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to 1 packet, and (base) target level equal to 1 as well. + // Return false to indicate no peaks found. + EXPECT_CALL(detector_, Update(1, 1)) + .WillOnce(Return(false)); + InsertNextPacket(); + const int kExpectedTarget = 1; + EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); // In Q8. + EXPECT_EQ(1, dm_->base_target_level()); + int lower, higher; + dm_->BufferLimits(&lower, &higher); + // Expect |lower| to be 75% of base target level, and |higher| to be + // lower + 20 ms headroom. + EXPECT_EQ((1 << 8) * 3 / 4, lower); + EXPECT_EQ(lower + (20 << 8) / kFrameSizeMs, higher); +} + +TEST_F(DelayManagerTest, MaxAndRequiredDelay) { + const int kExpectedTarget = 5; + const int kTimeIncrement = kExpectedTarget * kFrameSizeMs; + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to |kExpectedTarget| packet. Return true to indicate peaks found. + EXPECT_CALL(detector_, Update(kExpectedTarget, _)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(detector_, MaxPeakHeight()) + .WillRepeatedly(Return(kExpectedTarget)); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + + // No limit is set. + EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); + + int kMaxDelayPackets = kExpectedTarget - 2; + int kMaxDelayMs = kMaxDelayPackets * kFrameSizeMs; + EXPECT_TRUE(dm_->SetMaximumDelay(kMaxDelayMs)); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + EXPECT_EQ(kExpectedTarget * kFrameSizeMs, dm_->least_required_delay_ms()); + EXPECT_EQ(kMaxDelayPackets << 8, dm_->TargetLevel()); + + // Target level at least should be one packet. + EXPECT_FALSE(dm_->SetMaximumDelay(kFrameSizeMs - 1)); +} + +TEST_F(DelayManagerTest, MinAndRequiredDelay) { + const int kExpectedTarget = 5; + const int kTimeIncrement = kExpectedTarget * kFrameSizeMs; + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to |kExpectedTarget| packet. Return true to indicate peaks found. + EXPECT_CALL(detector_, Update(kExpectedTarget, _)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(detector_, MaxPeakHeight()) + .WillRepeatedly(Return(kExpectedTarget)); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + + // No limit is applied. + EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); + + int kMinDelayPackets = kExpectedTarget + 2; + int kMinDelayMs = kMinDelayPackets * kFrameSizeMs; + dm_->SetMinimumDelay(kMinDelayMs); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + EXPECT_EQ(kExpectedTarget * kFrameSizeMs, dm_->least_required_delay_ms()); + EXPECT_EQ(kMinDelayPackets << 8, dm_->TargetLevel()); +} + +// Tests that skipped sequence numbers (simulating empty packets) are handled +// correctly. +TEST_F(DelayManagerTest, EmptyPacketsReported) { + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + + // Advance time by one frame size. + IncreaseTime(kFrameSizeMs); + + // Advance the sequence number by 5, simulating that 5 empty packets were + // received, but never inserted. + seq_no_ += 10; + for (int j = 0; j < 10; ++j) { + dm_->RegisterEmptyPacket(); + } + + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to 1 packet, and (base) target level equal to 1 as well. + // Return false to indicate no peaks found. + EXPECT_CALL(detector_, Update(1, 1)).WillOnce(Return(false)); + InsertNextPacket(); + + EXPECT_EQ(1 << 8, dm_->TargetLevel()); // In Q8. +} + +// Same as above, but do not call RegisterEmptyPacket. Observe the target level +// increase dramatically. +TEST_F(DelayManagerTest, EmptyPacketsNotReported) { + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + + // Advance time by one frame size. + IncreaseTime(kFrameSizeMs); + + // Advance the sequence number by 5, simulating that 5 empty packets were + // received, but never inserted. + seq_no_ += 10; + + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to 1 packet, and (base) target level equal to 1 as well. + // Return false to indicate no peaks found. + EXPECT_CALL(detector_, Update(10, 10)).WillOnce(Return(false)); + InsertNextPacket(); + + // Note 10 times higher target value. + EXPECT_EQ(10 * 1 << 8, dm_->TargetLevel()); // In Q8. +} + +TEST_F(DelayManagerTest, Failures) { + // Wrong sample rate. + EXPECT_EQ(-1, dm_->Update(0, 0, -1)); + // Wrong packet size. + EXPECT_EQ(-1, dm_->SetPacketAudioLength(0)); + EXPECT_EQ(-1, dm_->SetPacketAudioLength(-1)); + + // Minimum delay higher than a maximum delay is not accepted. + EXPECT_TRUE(dm_->SetMaximumDelay(10)); + EXPECT_FALSE(dm_->SetMinimumDelay(20)); + + // Maximum delay less than minimum delay is not accepted. + EXPECT_TRUE(dm_->SetMaximumDelay(100)); + EXPECT_TRUE(dm_->SetMinimumDelay(80)); + EXPECT_FALSE(dm_->SetMaximumDelay(60)); +} + +// Test if the histogram is stretched correctly if the packet size is decreased. +TEST(DelayManagerIATScalingTest, StretchTest) { + using IATVector = DelayManager::IATVector; + // Test a straightforward 60ms to 20ms change. + IATVector iat = {12, 0, 0, 0, 0, 0}; + IATVector expected_result = {4, 4, 4, 0, 0, 0}; + IATVector stretched_iat = DelayManager::ScaleHistogram(iat, 60, 20); + EXPECT_EQ(stretched_iat, expected_result); + + // Test an example where the last bin in the stretched histogram should + // contain the sum of the elements that don't fit into the new histogram. + iat = {18, 15, 12, 9, 6, 3, 0}; + expected_result = {6, 6, 6, 5, 5, 5, 30}; + stretched_iat = DelayManager::ScaleHistogram(iat, 60, 20); + EXPECT_EQ(stretched_iat, expected_result); + + // Test a 120ms to 60ms change. + iat = {18, 16, 14, 4, 0}; + expected_result = {9, 9, 8, 8, 18}; + stretched_iat = DelayManager::ScaleHistogram(iat, 120, 60); + EXPECT_EQ(stretched_iat, expected_result); + + // Test a 120ms to 20ms change. + iat = {19, 12, 0, 0, 0, 0, 0, 0}; + expected_result = {3, 3, 3, 3, 3, 3, 2, 11}; + stretched_iat = DelayManager::ScaleHistogram(iat, 120, 20); + EXPECT_EQ(stretched_iat, expected_result); + + // Test a 70ms to 40ms change. + iat = {13, 7, 5, 3, 1, 5, 12, 11, 3, 0, 0, 0}; + expected_result = {7, 5, 5, 3, 3, 2, 2, 1, 2, 2, 6, 22}; + stretched_iat = DelayManager::ScaleHistogram(iat, 70, 40); + EXPECT_EQ(stretched_iat, expected_result); + + // Test a 30ms to 20ms change. + iat = {13, 7, 5, 3, 1, 5, 12, 11, 3, 0, 0, 0}; + expected_result = {8, 6, 6, 3, 2, 2, 1, 3, 3, 8, 7, 11}; + stretched_iat = DelayManager::ScaleHistogram(iat, 30, 20); + EXPECT_EQ(stretched_iat, expected_result); +} + +// Test if the histogram is compressed correctly if the packet size is +// increased. +TEST(DelayManagerIATScalingTest, CompressionTest) { + using IATVector = DelayManager::IATVector; + // Test a 20 to 60 ms change. + IATVector iat = {12, 11, 10, 3, 2, 1}; + IATVector expected_result = {33, 6, 0, 0, 0, 0}; + IATVector compressed_iat = DelayManager::ScaleHistogram(iat, 20, 60); + EXPECT_EQ(compressed_iat, expected_result); + + // Test a 60ms to 120ms change. + iat = {18, 16, 14, 4, 1}; + expected_result = {34, 18, 1, 0, 0}; + compressed_iat = DelayManager::ScaleHistogram(iat, 60, 120); + EXPECT_EQ(compressed_iat, expected_result); + + // Test a 20ms to 120ms change. + iat = {18, 12, 5, 4, 4, 3, 5, 1}; + expected_result = {46, 6, 0, 0, 0, 0, 0, 0}; + compressed_iat = DelayManager::ScaleHistogram(iat, 20, 120); + EXPECT_EQ(compressed_iat, expected_result); + + // Test a 70ms to 80ms change. + iat = {13, 7, 5, 3, 1, 5, 12, 11, 3}; + expected_result = {11, 8, 6, 2, 5, 12, 13, 3, 0}; + compressed_iat = DelayManager::ScaleHistogram(iat, 70, 80); + EXPECT_EQ(compressed_iat, expected_result); + + // Test a 50ms to 110ms change. + iat = {13, 7, 5, 3, 1, 5, 12, 11, 3}; + expected_result = {18, 8, 16, 16, 2, 0, 0, 0, 0}; + compressed_iat = DelayManager::ScaleHistogram(iat, 50, 110); + EXPECT_EQ(compressed_iat, expected_result); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.cc new file mode 100644 index 0000000000..eb9f6d530d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/delay_peak_detector.h" + +#include // max + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +// The DelayPeakDetector keeps track of severe inter-arrival times, called +// delay peaks. When a peak is observed, the "height" (the time elapsed since +// the previous packet arrival) and the peak "period" (the time since the last +// observed peak) is recorded in a vector. When enough peaks have been observed, +// peak-mode is engaged and the DelayManager asks the DelayPeakDetector for +// the worst peak height. + +DelayPeakDetector::~DelayPeakDetector() = default; + +DelayPeakDetector::DelayPeakDetector(const TickTimer* tick_timer) + : peak_found_(false), + peak_detection_threshold_(0), + tick_timer_(tick_timer), + frame_length_change_experiment_( + field_trial::IsEnabled("WebRTC-Audio-NetEqFramelengthExperiment")) { + RTC_DCHECK(!peak_period_stopwatch_); +} + +void DelayPeakDetector::Reset() { + peak_period_stopwatch_.reset(); + peak_found_ = false; + peak_history_.clear(); +} + +// Calculates the threshold in number of packets. +void DelayPeakDetector::SetPacketAudioLength(int length_ms) { + if (length_ms > 0) { + if (frame_length_change_experiment_) { + peak_detection_threshold_ = std::max(2, kPeakHeightMs / length_ms); + } else { + peak_detection_threshold_ = kPeakHeightMs / length_ms; + } + } + if (frame_length_change_experiment_) { + peak_history_.clear(); + } +} + +bool DelayPeakDetector::peak_found() { + return peak_found_; +} + +int DelayPeakDetector::MaxPeakHeight() const { + int max_height = -1; // Returns -1 for an empty history. + std::list::const_iterator it; + for (it = peak_history_.begin(); it != peak_history_.end(); ++it) { + max_height = std::max(max_height, it->peak_height_packets); + } + return max_height; +} + +uint64_t DelayPeakDetector::MaxPeakPeriod() const { + auto max_period_element = std::max_element( + peak_history_.begin(), peak_history_.end(), + [](Peak a, Peak b) { return a.period_ms < b.period_ms; }); + if (max_period_element == peak_history_.end()) { + return 0; // |peak_history_| is empty. + } + RTC_DCHECK_GT(max_period_element->period_ms, 0); + return max_period_element->period_ms; +} + +bool DelayPeakDetector::Update(int inter_arrival_time, int target_level) { + if (inter_arrival_time > target_level + peak_detection_threshold_ || + inter_arrival_time > 2 * target_level) { + // A delay peak is observed. + if (!peak_period_stopwatch_) { + // This is the first peak. Reset the period counter. + peak_period_stopwatch_ = tick_timer_->GetNewStopwatch(); + } else if (peak_period_stopwatch_->ElapsedMs() > 0) { + if (peak_period_stopwatch_->ElapsedMs() <= kMaxPeakPeriodMs) { + // This is not the first peak, and the period is valid. + // Store peak data in the vector. + Peak peak_data; + peak_data.period_ms = peak_period_stopwatch_->ElapsedMs(); + peak_data.peak_height_packets = inter_arrival_time; + peak_history_.push_back(peak_data); + while (peak_history_.size() > kMaxNumPeaks) { + // Delete the oldest data point. + peak_history_.pop_front(); + } + peak_period_stopwatch_ = tick_timer_->GetNewStopwatch(); + } else if (peak_period_stopwatch_->ElapsedMs() <= 2 * kMaxPeakPeriodMs) { + // Invalid peak due to too long period. Reset period counter and start + // looking for next peak. + peak_period_stopwatch_ = tick_timer_->GetNewStopwatch(); + } else { + // More than 2 times the maximum period has elapsed since the last peak + // was registered. It seams that the network conditions have changed. + // Reset the peak statistics. + Reset(); + } + } + } + return CheckPeakConditions(); +} + +bool DelayPeakDetector::CheckPeakConditions() { + size_t s = peak_history_.size(); + if (s >= kMinPeaksToTrigger && + peak_period_stopwatch_->ElapsedMs() <= 2 * MaxPeakPeriod()) { + peak_found_ = true; + } else { + peak_found_ = false; + } + return peak_found_; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.h new file mode 100644 index 0000000000..9defca5ad0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DELAY_PEAK_DETECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_DELAY_PEAK_DETECTOR_H_ + +#include // size_t + +#include +#include + +#include "modules/audio_coding/neteq/tick_timer.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class DelayPeakDetector { + public: + DelayPeakDetector(const TickTimer* tick_timer); + virtual ~DelayPeakDetector(); + virtual void Reset(); + + // Notifies the DelayPeakDetector of how much audio data is carried in each + // packet. + virtual void SetPacketAudioLength(int length_ms); + + // Returns true if peak-mode is active. That is, delay peaks were observed + // recently. + virtual bool peak_found(); + + // Calculates and returns the maximum delay peak height. Returns -1 if no + // delay peaks have been observed recently. The unit is number of packets. + virtual int MaxPeakHeight() const; + + // Calculates and returns the maximum delay peak distance in ms (strictly + // larger than 0), or 0 if no delay peaks have been observed recently. + virtual uint64_t MaxPeakPeriod() const; + + // Updates the DelayPeakDetector with a new inter-arrival time (in packets) + // and the current target buffer level (needed to decide if a peak is observed + // or not). Returns true if peak-mode is active, false if not. + virtual bool Update(int inter_arrival_time, int target_level); + + private: + static const size_t kMaxNumPeaks = 8; + static const size_t kMinPeaksToTrigger = 2; + static const int kPeakHeightMs = 78; + static const int kMaxPeakPeriodMs = 10000; + + typedef struct { + uint64_t period_ms; + int peak_height_packets; + } Peak; + + bool CheckPeakConditions(); + + std::list peak_history_; + bool peak_found_; + int peak_detection_threshold_; + const TickTimer* tick_timer_; + std::unique_ptr peak_period_stopwatch_; + const bool frame_length_change_experiment_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DelayPeakDetector); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DELAY_PEAK_DETECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector_unittest.cc new file mode 100644 index 0000000000..058ba66362 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector_unittest.cc @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for DelayPeakDetector class. + +#include "modules/audio_coding/neteq/delay_peak_detector.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(DelayPeakDetector, CreateAndDestroy) { + TickTimer tick_timer; + DelayPeakDetector* detector = new DelayPeakDetector(&tick_timer); + EXPECT_FALSE(detector->peak_found()); + delete detector; +} + +TEST(DelayPeakDetector, EmptyHistory) { + TickTimer tick_timer; + DelayPeakDetector detector(&tick_timer); + EXPECT_EQ(-1, detector.MaxPeakHeight()); + EXPECT_EQ(0u, detector.MaxPeakPeriod()); +} + +// Inject a series of packet arrivals into the detector. Three of the packets +// have suffered delays. After the third delay peak, peak-mode is expected to +// start. This should then continue until it is disengaged due to lack of peaks. +TEST(DelayPeakDetector, TriggerPeakMode) { + TickTimer tick_timer; + DelayPeakDetector detector(&tick_timer); + const int kPacketSizeMs = 30; + detector.SetPacketAudioLength(kPacketSizeMs); + + // Load up normal arrival times; 0 ms, 30 ms, 60 ms, 90 ms, ... + const int kNumPackets = 1000; + int arrival_times_ms[kNumPackets]; + for (int i = 0; i < kNumPackets; ++i) { + arrival_times_ms[i] = i * kPacketSizeMs; + } + + // Delay three packets. + const int kPeakDelayMs = 100; + // First delay peak. + arrival_times_ms[100] += kPeakDelayMs; + // Second delay peak. + arrival_times_ms[200] += kPeakDelayMs; + // Third delay peak. Trigger peak-mode after this packet. + arrival_times_ms[400] += kPeakDelayMs; + // The second peak period is the longest, 200 packets. + const uint64_t kWorstPeakPeriod = 200 * kPacketSizeMs; + int peak_mode_start_ms = arrival_times_ms[400]; + // Expect to disengage after no peaks are observed for two period times. + int peak_mode_end_ms = peak_mode_start_ms + 2 * kWorstPeakPeriod; + + // Load into detector. + int time = 0; + int next = 1; // Start with the second packet to get a proper IAT. + while (next < kNumPackets) { + while (next < kNumPackets && arrival_times_ms[next] <= time) { + int iat_packets = (arrival_times_ms[next] - arrival_times_ms[next - 1]) / + kPacketSizeMs; + const int kTargetBufferLevel = 1; // Define peaks to be iat > 2. + if (time < peak_mode_start_ms || time > peak_mode_end_ms) { + EXPECT_FALSE(detector.Update(iat_packets, kTargetBufferLevel)); + } else { + EXPECT_TRUE(detector.Update(iat_packets, kTargetBufferLevel)); + EXPECT_EQ(kWorstPeakPeriod, detector.MaxPeakPeriod()); + EXPECT_EQ(kPeakDelayMs / kPacketSizeMs + 1, detector.MaxPeakHeight()); + } + ++next; + } + tick_timer.Increment(); + time += 10; // Increase time 10 ms. + } +} + +// Same test as TriggerPeakMode, but with base target buffer level increased to +// 2, in order to raise the bar for delay peaks to inter-arrival times > 4. +// The delay pattern has peaks with delay = 3, thus should not trigger. +TEST(DelayPeakDetector, DoNotTriggerPeakMode) { + TickTimer tick_timer; + DelayPeakDetector detector(&tick_timer); + const int kPacketSizeMs = 30; + detector.SetPacketAudioLength(kPacketSizeMs); + + // Load up normal arrival times; 0 ms, 30 ms, 60 ms, 90 ms, ... + const int kNumPackets = 1000; + int arrival_times_ms[kNumPackets]; + for (int i = 0; i < kNumPackets; ++i) { + arrival_times_ms[i] = i * kPacketSizeMs; + } + + // Delay three packets. + const int kPeakDelayMs = 100; + // First delay peak. + arrival_times_ms[100] += kPeakDelayMs; + // Second delay peak. + arrival_times_ms[200] += kPeakDelayMs; + // Third delay peak. + arrival_times_ms[400] += kPeakDelayMs; + + // Load into detector. + int time = 0; + int next = 1; // Start with the second packet to get a proper IAT. + while (next < kNumPackets) { + while (next < kNumPackets && arrival_times_ms[next] <= time) { + int iat_packets = (arrival_times_ms[next] - arrival_times_ms[next - 1]) / + kPacketSizeMs; + const int kTargetBufferLevel = 2; // Define peaks to be iat > 4. + EXPECT_FALSE(detector.Update(iat_packets, kTargetBufferLevel)); + ++next; + } + tick_timer.Increment(); + time += 10; // Increase time 10 ms. + } +} + +// In situations with reordered packets, the DelayPeakDetector may be updated +// back-to-back (i.e., without the tick_timer moving) but still with non-zero +// inter-arrival time. This test is to make sure that this does not cause +// problems. +TEST(DelayPeakDetector, ZeroDistancePeaks) { + TickTimer tick_timer; + DelayPeakDetector detector(&tick_timer); + const int kPacketSizeMs = 30; + detector.SetPacketAudioLength(kPacketSizeMs); + + const int kTargetBufferLevel = 2; // Define peaks to be iat > 4. + const int kInterArrivalTime = 3 * kTargetBufferLevel; // Will trigger a peak. + EXPECT_FALSE(detector.Update(kInterArrivalTime, kTargetBufferLevel)); + EXPECT_FALSE(detector.Update(kInterArrivalTime, kTargetBufferLevel)); + EXPECT_FALSE(detector.Update(kInterArrivalTime, kTargetBufferLevel)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.cc new file mode 100644 index 0000000000..2a1d81ba0d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.cc @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dsp_helper.h" + +#include +#include // Access to memset. + +#include // Access to min, max. + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +namespace webrtc { + +// Table of constants used in method DspHelper::ParabolicFit(). +const int16_t DspHelper::kParabolaCoefficients[17][3] = { + { 120, 32, 64 }, + { 140, 44, 75 }, + { 150, 50, 80 }, + { 160, 57, 85 }, + { 180, 72, 96 }, + { 200, 89, 107 }, + { 210, 98, 112 }, + { 220, 108, 117 }, + { 240, 128, 128 }, + { 260, 150, 139 }, + { 270, 162, 144 }, + { 280, 174, 149 }, + { 300, 200, 160 }, + { 320, 228, 171 }, + { 330, 242, 176 }, + { 340, 257, 181 }, + { 360, 288, 192 } }; + +// Filter coefficients used when downsampling from the indicated sample rates +// (8, 16, 32, 48 kHz) to 4 kHz. Coefficients are in Q12. The corresponding Q0 +// values are provided in the comments before each array. + +// Q0 values: {0.3, 0.4, 0.3}. +const int16_t DspHelper::kDownsample8kHzTbl[3] = { 1229, 1638, 1229 }; + +// Q0 values: {0.15, 0.2, 0.3, 0.2, 0.15}. +const int16_t DspHelper::kDownsample16kHzTbl[5] = { 614, 819, 1229, 819, 614 }; + +// Q0 values: {0.1425, 0.1251, 0.1525, 0.1628, 0.1525, 0.1251, 0.1425}. +const int16_t DspHelper::kDownsample32kHzTbl[7] = { + 584, 512, 625, 667, 625, 512, 584 }; + +// Q0 values: {0.2487, 0.0952, 0.1042, 0.1074, 0.1042, 0.0952, 0.2487}. +const int16_t DspHelper::kDownsample48kHzTbl[7] = { + 1019, 390, 427, 440, 427, 390, 1019 }; + +int DspHelper::RampSignal(const int16_t* input, + size_t length, + int factor, + int increment, + int16_t* output) { + int factor_q20 = (factor << 6) + 32; + // TODO(hlundin): Add 32 to factor_q20 when converting back to Q14? + for (size_t i = 0; i < length; ++i) { + output[i] = (factor * input[i] + 8192) >> 14; + factor_q20 += increment; + factor_q20 = std::max(factor_q20, 0); // Never go negative. + factor = std::min(factor_q20 >> 6, 16384); + } + return factor; +} + +int DspHelper::RampSignal(int16_t* signal, + size_t length, + int factor, + int increment) { + return RampSignal(signal, length, factor, increment, signal); +} + +int DspHelper::RampSignal(AudioVector* signal, + size_t start_index, + size_t length, + int factor, + int increment) { + int factor_q20 = (factor << 6) + 32; + // TODO(hlundin): Add 32 to factor_q20 when converting back to Q14? + for (size_t i = start_index; i < start_index + length; ++i) { + (*signal)[i] = (factor * (*signal)[i] + 8192) >> 14; + factor_q20 += increment; + factor_q20 = std::max(factor_q20, 0); // Never go negative. + factor = std::min(factor_q20 >> 6, 16384); + } + return factor; +} + +int DspHelper::RampSignal(AudioMultiVector* signal, + size_t start_index, + size_t length, + int factor, + int increment) { + assert(start_index + length <= signal->Size()); + if (start_index + length > signal->Size()) { + // Wrong parameters. Do nothing and return the scale factor unaltered. + return factor; + } + int end_factor = 0; + // Loop over the channels, starting at the same |factor| each time. + for (size_t channel = 0; channel < signal->Channels(); ++channel) { + end_factor = + RampSignal(&(*signal)[channel], start_index, length, factor, increment); + } + return end_factor; +} + +void DspHelper::PeakDetection(int16_t* data, size_t data_length, + size_t num_peaks, int fs_mult, + size_t* peak_index, int16_t* peak_value) { + size_t min_index = 0; + size_t max_index = 0; + + for (size_t i = 0; i <= num_peaks - 1; i++) { + if (num_peaks == 1) { + // Single peak. The parabola fit assumes that an extra point is + // available; worst case it gets a zero on the high end of the signal. + // TODO(hlundin): This can potentially get much worse. It breaks the + // API contract, that the length of |data| is |data_length|. + data_length++; + } + + peak_index[i] = WebRtcSpl_MaxIndexW16(data, data_length - 1); + + if (i != num_peaks - 1) { + min_index = (peak_index[i] > 2) ? (peak_index[i] - 2) : 0; + max_index = std::min(data_length - 1, peak_index[i] + 2); + } + + if ((peak_index[i] != 0) && (peak_index[i] != (data_length - 2))) { + ParabolicFit(&data[peak_index[i] - 1], fs_mult, &peak_index[i], + &peak_value[i]); + } else { + if (peak_index[i] == data_length - 2) { + if (data[peak_index[i]] > data[peak_index[i] + 1]) { + ParabolicFit(&data[peak_index[i] - 1], fs_mult, &peak_index[i], + &peak_value[i]); + } else if (data[peak_index[i]] <= data[peak_index[i] + 1]) { + // Linear approximation. + peak_value[i] = (data[peak_index[i]] + data[peak_index[i] + 1]) >> 1; + peak_index[i] = (peak_index[i] * 2 + 1) * fs_mult; + } + } else { + peak_value[i] = data[peak_index[i]]; + peak_index[i] = peak_index[i] * 2 * fs_mult; + } + } + + if (i != num_peaks - 1) { + memset(&data[min_index], 0, + sizeof(data[0]) * (max_index - min_index + 1)); + } + } +} + +void DspHelper::ParabolicFit(int16_t* signal_points, int fs_mult, + size_t* peak_index, int16_t* peak_value) { + uint16_t fit_index[13]; + if (fs_mult == 1) { + fit_index[0] = 0; + fit_index[1] = 8; + fit_index[2] = 16; + } else if (fs_mult == 2) { + fit_index[0] = 0; + fit_index[1] = 4; + fit_index[2] = 8; + fit_index[3] = 12; + fit_index[4] = 16; + } else if (fs_mult == 4) { + fit_index[0] = 0; + fit_index[1] = 2; + fit_index[2] = 4; + fit_index[3] = 6; + fit_index[4] = 8; + fit_index[5] = 10; + fit_index[6] = 12; + fit_index[7] = 14; + fit_index[8] = 16; + } else { + fit_index[0] = 0; + fit_index[1] = 1; + fit_index[2] = 3; + fit_index[3] = 4; + fit_index[4] = 5; + fit_index[5] = 7; + fit_index[6] = 8; + fit_index[7] = 9; + fit_index[8] = 11; + fit_index[9] = 12; + fit_index[10] = 13; + fit_index[11] = 15; + fit_index[12] = 16; + } + + // num = -3 * signal_points[0] + 4 * signal_points[1] - signal_points[2]; + // den = signal_points[0] - 2 * signal_points[1] + signal_points[2]; + int32_t num = (signal_points[0] * -3) + (signal_points[1] * 4) + - signal_points[2]; + int32_t den = signal_points[0] + (signal_points[1] * -2) + signal_points[2]; + int32_t temp = num * 120; + int flag = 1; + int16_t stp = kParabolaCoefficients[fit_index[fs_mult]][0] + - kParabolaCoefficients[fit_index[fs_mult - 1]][0]; + int16_t strt = (kParabolaCoefficients[fit_index[fs_mult]][0] + + kParabolaCoefficients[fit_index[fs_mult - 1]][0]) / 2; + int16_t lmt; + if (temp < -den * strt) { + lmt = strt - stp; + while (flag) { + if ((flag == fs_mult) || (temp > -den * lmt)) { + *peak_value = (den * kParabolaCoefficients[fit_index[fs_mult - flag]][1] + + num * kParabolaCoefficients[fit_index[fs_mult - flag]][2] + + signal_points[0] * 256) / 256; + *peak_index = *peak_index * 2 * fs_mult - flag; + flag = 0; + } else { + flag++; + lmt -= stp; + } + } + } else if (temp > -den * (strt + stp)) { + lmt = strt + 2 * stp; + while (flag) { + if ((flag == fs_mult) || (temp < -den * lmt)) { + int32_t temp_term_1 = + den * kParabolaCoefficients[fit_index[fs_mult+flag]][1]; + int32_t temp_term_2 = + num * kParabolaCoefficients[fit_index[fs_mult+flag]][2]; + int32_t temp_term_3 = signal_points[0] * 256; + *peak_value = (temp_term_1 + temp_term_2 + temp_term_3) / 256; + *peak_index = *peak_index * 2 * fs_mult + flag; + flag = 0; + } else { + flag++; + lmt += stp; + } + } + } else { + *peak_value = signal_points[1]; + *peak_index = *peak_index * 2 * fs_mult; + } +} + +size_t DspHelper::MinDistortion(const int16_t* signal, size_t min_lag, + size_t max_lag, size_t length, + int32_t* distortion_value) { + size_t best_index = 0; + int32_t min_distortion = WEBRTC_SPL_WORD32_MAX; + for (size_t i = min_lag; i <= max_lag; i++) { + int32_t sum_diff = 0; + const int16_t* data1 = signal; + const int16_t* data2 = signal - i; + for (size_t j = 0; j < length; j++) { + sum_diff += WEBRTC_SPL_ABS_W32(data1[j] - data2[j]); + } + // Compare with previous minimum. + if (sum_diff < min_distortion) { + min_distortion = sum_diff; + best_index = i; + } + } + *distortion_value = min_distortion; + return best_index; +} + +void DspHelper::CrossFade(const int16_t* input1, const int16_t* input2, + size_t length, int16_t* mix_factor, + int16_t factor_decrement, int16_t* output) { + int16_t factor = *mix_factor; + int16_t complement_factor = 16384 - factor; + for (size_t i = 0; i < length; i++) { + output[i] = + (factor * input1[i] + complement_factor * input2[i] + 8192) >> 14; + factor -= factor_decrement; + complement_factor += factor_decrement; + } + *mix_factor = factor; +} + +void DspHelper::UnmuteSignal(const int16_t* input, size_t length, + int16_t* factor, int increment, + int16_t* output) { + uint16_t factor_16b = *factor; + int32_t factor_32b = (static_cast(factor_16b) << 6) + 32; + for (size_t i = 0; i < length; i++) { + output[i] = (factor_16b * input[i] + 8192) >> 14; + factor_32b = std::max(factor_32b + increment, 0); + factor_16b = std::min(16384, factor_32b >> 6); + } + *factor = factor_16b; +} + +void DspHelper::MuteSignal(int16_t* signal, int mute_slope, size_t length) { + int32_t factor = (16384 << 6) + 32; + for (size_t i = 0; i < length; i++) { + signal[i] = ((factor >> 6) * signal[i] + 8192) >> 14; + factor -= mute_slope; + } +} + +int DspHelper::DownsampleTo4kHz(const int16_t* input, size_t input_length, + size_t output_length, int input_rate_hz, + bool compensate_delay, int16_t* output) { + // Set filter parameters depending on input frequency. + // NOTE: The phase delay values are wrong compared to the true phase delay + // of the filters. However, the error is preserved (through the +1 term) for + // consistency. + const int16_t* filter_coefficients; // Filter coefficients. + size_t filter_length; // Number of coefficients. + size_t filter_delay; // Phase delay in samples. + int16_t factor; // Conversion rate (inFsHz / 8000). + switch (input_rate_hz) { + case 8000: { + filter_length = 3; + factor = 2; + filter_coefficients = kDownsample8kHzTbl; + filter_delay = 1 + 1; + break; + } + case 16000: { + filter_length = 5; + factor = 4; + filter_coefficients = kDownsample16kHzTbl; + filter_delay = 2 + 1; + break; + } + case 32000: { + filter_length = 7; + factor = 8; + filter_coefficients = kDownsample32kHzTbl; + filter_delay = 3 + 1; + break; + } + case 48000: { + filter_length = 7; + factor = 12; + filter_coefficients = kDownsample48kHzTbl; + filter_delay = 3 + 1; + break; + } + default: { + assert(false); + return -1; + } + } + + if (!compensate_delay) { + // Disregard delay compensation. + filter_delay = 0; + } + + // Returns -1 if input signal is too short; 0 otherwise. + return WebRtcSpl_DownsampleFast( + &input[filter_length - 1], input_length - filter_length + 1, output, + output_length, filter_coefficients, filter_length, factor, filter_delay); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.h new file mode 100644 index 0000000000..7ceb66f57a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DSP_HELPER_H_ +#define MODULES_AUDIO_CODING_NETEQ_DSP_HELPER_H_ + +#include // Access to size_t. + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// This class contains various signal processing functions, all implemented as +// static methods. +class DspHelper { + public: + // Filter coefficients used when downsampling from the indicated sample rates + // (8, 16, 32, 48 kHz) to 4 kHz. Coefficients are in Q12. + static const int16_t kDownsample8kHzTbl[3]; + static const int16_t kDownsample16kHzTbl[5]; + static const int16_t kDownsample32kHzTbl[7]; + static const int16_t kDownsample48kHzTbl[7]; + + // Constants used to mute and unmute over 5 samples. The coefficients are + // in Q15. + static const int kMuteFactorStart8kHz = 27307; + static const int kMuteFactorIncrement8kHz = -5461; + static const int kUnmuteFactorStart8kHz = 5461; + static const int kUnmuteFactorIncrement8kHz = 5461; + static const int kMuteFactorStart16kHz = 29789; + static const int kMuteFactorIncrement16kHz = -2979; + static const int kUnmuteFactorStart16kHz = 2979; + static const int kUnmuteFactorIncrement16kHz = 2979; + static const int kMuteFactorStart32kHz = 31208; + static const int kMuteFactorIncrement32kHz = -1560; + static const int kUnmuteFactorStart32kHz = 1560; + static const int kUnmuteFactorIncrement32kHz = 1560; + static const int kMuteFactorStart48kHz = 31711; + static const int kMuteFactorIncrement48kHz = -1057; + static const int kUnmuteFactorStart48kHz = 1057; + static const int kUnmuteFactorIncrement48kHz = 1057; + + // Multiplies the signal with a gradually changing factor. + // The first sample is multiplied with |factor| (in Q14). For each sample, + // |factor| is increased (additive) by the |increment| (in Q20), which can + // be negative. Returns the scale factor after the last increment. + static int RampSignal(const int16_t* input, + size_t length, + int factor, + int increment, + int16_t* output); + + // Same as above, but with the samples of |signal| being modified in-place. + static int RampSignal(int16_t* signal, + size_t length, + int factor, + int increment); + + // Same as above, but processes |length| samples from |signal|, starting at + // |start_index|. + static int RampSignal(AudioVector* signal, + size_t start_index, + size_t length, + int factor, + int increment); + + // Same as above, but for an AudioMultiVector. + static int RampSignal(AudioMultiVector* signal, + size_t start_index, + size_t length, + int factor, + int increment); + + // Peak detection with parabolic fit. Looks for |num_peaks| maxima in |data|, + // having length |data_length| and sample rate multiplier |fs_mult|. The peak + // locations and values are written to the arrays |peak_index| and + // |peak_value|, respectively. Both arrays must hold at least |num_peaks| + // elements. + static void PeakDetection(int16_t* data, size_t data_length, + size_t num_peaks, int fs_mult, + size_t* peak_index, int16_t* peak_value); + + // Estimates the height and location of a maximum. The three values in the + // array |signal_points| are used as basis for a parabolic fit, which is then + // used to find the maximum in an interpolated signal. The |signal_points| are + // assumed to be from a 4 kHz signal, while the maximum, written to + // |peak_index| and |peak_value| is given in the full sample rate, as + // indicated by the sample rate multiplier |fs_mult|. + static void ParabolicFit(int16_t* signal_points, int fs_mult, + size_t* peak_index, int16_t* peak_value); + + // Calculates the sum-abs-diff for |signal| when compared to a displaced + // version of itself. Returns the displacement lag that results in the minimum + // distortion. The resulting distortion is written to |distortion_value|. + // The values of |min_lag| and |max_lag| are boundaries for the search. + static size_t MinDistortion(const int16_t* signal, size_t min_lag, + size_t max_lag, size_t length, + int32_t* distortion_value); + + // Mixes |length| samples from |input1| and |input2| together and writes the + // result to |output|. The gain for |input1| starts at |mix_factor| (Q14) and + // is decreased by |factor_decrement| (Q14) for each sample. The gain for + // |input2| is the complement 16384 - mix_factor. + static void CrossFade(const int16_t* input1, const int16_t* input2, + size_t length, int16_t* mix_factor, + int16_t factor_decrement, int16_t* output); + + // Scales |input| with an increasing gain. Applies |factor| (Q14) to the first + // sample and increases the gain by |increment| (Q20) for each sample. The + // result is written to |output|. |length| samples are processed. + static void UnmuteSignal(const int16_t* input, size_t length, int16_t* factor, + int increment, int16_t* output); + + // Starts at unity gain and gradually fades out |signal|. For each sample, + // the gain is reduced by |mute_slope| (Q14). |length| samples are processed. + static void MuteSignal(int16_t* signal, int mute_slope, size_t length); + + // Downsamples |input| from |sample_rate_hz| to 4 kHz sample rate. The input + // has |input_length| samples, and the method will write |output_length| + // samples to |output|. Compensates for the phase delay of the downsampling + // filters if |compensate_delay| is true. Returns -1 if the input is too short + // to produce |output_length| samples, otherwise 0. + static int DownsampleTo4kHz(const int16_t* input, size_t input_length, + size_t output_length, int input_rate_hz, + bool compensate_delay, int16_t* output); + + private: + // Table of constants used in method DspHelper::ParabolicFit(). + static const int16_t kParabolaCoefficients[17][3]; + + RTC_DISALLOW_COPY_AND_ASSIGN(DspHelper); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DSP_HELPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc new file mode 100644 index 0000000000..98ae2a2c2d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dsp_helper.h" + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +TEST(DspHelper, RampSignalArray) { + static const int kLen = 100; + int16_t input[kLen]; + int16_t output[kLen]; + // Fill input with 1000. + for (int i = 0; i < kLen; ++i) { + input[i] = 1000; + } + int start_factor = 0; + // Ramp from 0 to 1 (in Q14) over the array. Note that |increment| is in Q20, + // while the factor is in Q14, hence the shift by 6. + int increment = (16384 << 6) / kLen; + + // Test first method. + int stop_factor = DspHelper::RampSignal(input, kLen, start_factor, increment, + output); + EXPECT_EQ(16383, stop_factor); // Almost reach 1 in Q14. + for (int i = 0; i < kLen; ++i) { + EXPECT_EQ(1000 * i / kLen, output[i]); + } + + // Test second method. (Note that this modifies |input|.) + stop_factor = DspHelper::RampSignal(input, kLen, start_factor, increment); + EXPECT_EQ(16383, stop_factor); // Almost reach 1 in Q14. + for (int i = 0; i < kLen; ++i) { + EXPECT_EQ(1000 * i / kLen, input[i]); + } +} + +TEST(DspHelper, RampSignalAudioMultiVector) { + static const int kLen = 100; + static const int kChannels = 5; + AudioMultiVector input(kChannels, kLen * 3); + // Fill input with 1000. + for (int i = 0; i < kLen * 3; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + input[channel][i] = 1000; + } + } + // We want to start ramping at |start_index| and keep ramping for |kLen| + // samples. + int start_index = kLen; + int start_factor = 0; + // Ramp from 0 to 1 (in Q14) in |kLen| samples. Note that |increment| is in + // Q20, while the factor is in Q14, hence the shift by 6. + int increment = (16384 << 6) / kLen; + + int stop_factor = DspHelper::RampSignal(&input, start_index, kLen, + start_factor, increment); + EXPECT_EQ(16383, stop_factor); // Almost reach 1 in Q14. + // Verify that the first |kLen| samples are left untouched. + int i; + for (i = 0; i < kLen; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000, input[channel][i]); + } + } + // Verify that the next block of |kLen| samples are ramped. + for (; i < 2 * kLen; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000 * (i - kLen) / kLen, input[channel][i]); + } + } + // Verify the last |kLen| samples are left untouched. + for (; i < 3 * kLen; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000, input[channel][i]); + } + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.cc new file mode 100644 index 0000000000..dc2588879a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.cc @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dtmf_buffer.h" + +#include +#include // max + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +// Modify the code to obtain backwards bit-exactness. Once bit-exactness is no +// longer required, this #define should be removed (and the code that it +// enables). +#define LEGACY_BITEXACT + +namespace webrtc { + +DtmfBuffer::DtmfBuffer(int fs_hz) { + SetSampleRate(fs_hz); +} + +DtmfBuffer::~DtmfBuffer() = default; + +void DtmfBuffer::Flush() { + buffer_.clear(); +} + +// The ParseEvent method parses 4 bytes from |payload| according to this format +// from RFC 4733: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | event |E|R| volume | duration | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Legend (adapted from RFC 4733) +// - event: The event field is a number between 0 and 255 identifying a +// specific telephony event. The buffer will not accept any event +// numbers larger than 15. +// - E: If set to a value of one, the "end" bit indicates that this +// packet contains the end of the event. For long-lasting events +// that have to be split into segments, only the final packet for +// the final segment will have the E bit set. +// - R: Reserved. +// - volume: For DTMF digits and other events representable as tones, this +// field describes the power level of the tone, expressed in dBm0 +// after dropping the sign. Power levels range from 0 to -63 dBm0. +// Thus, larger values denote lower volume. The buffer discards +// values larger than 36 (i.e., lower than -36 dBm0). +// - duration: The duration field indicates the duration of the event or segment +// being reported, in timestamp units, expressed as an unsigned +// integer in network byte order. For a non-zero value, the event +// or segment began at the instant identified by the RTP timestamp +// and has so far lasted as long as indicated by this parameter. +// The event may or may not have ended. If the event duration +// exceeds the maximum representable by the duration field, the +// event is split into several contiguous segments. The buffer will +// discard zero-duration events. +// +int DtmfBuffer::ParseEvent(uint32_t rtp_timestamp, + const uint8_t* payload, + size_t payload_length_bytes, + DtmfEvent* event) { + RTC_CHECK(payload); + RTC_CHECK(event); + if (payload_length_bytes < 4) { + RTC_LOG(LS_WARNING) << "ParseEvent payload too short"; + return kPayloadTooShort; + } + + event->event_no = payload[0]; + event->end_bit = ((payload[1] & 0x80) != 0); + event->volume = (payload[1] & 0x3F); + event->duration = payload[2] << 8 | payload[3]; + event->timestamp = rtp_timestamp; + return kOK; +} + +// Inserts a DTMF event into the buffer. The event should be parsed from the +// bit stream using the ParseEvent method above before inserting it in the +// buffer. +// DTMF events can be quite long, and in most cases the duration of the event +// is not known when the first packet describing it is sent. To deal with that, +// the RFC 4733 specifies that multiple packets are sent for one and the same +// event as it is being created (typically, as the user is pressing the key). +// These packets will all share the same start timestamp and event number, +// while the duration will be the cumulative duration from the start. When +// inserting a new event, the InsertEvent method tries to find a matching event +// already in the buffer. If so, the new event is simply merged with the +// existing one. +int DtmfBuffer::InsertEvent(const DtmfEvent& event) { + if (event.event_no < 0 || event.event_no > 15 || + event.volume < 0 || event.volume > 63 || + event.duration <= 0 || event.duration > 65535) { + RTC_LOG(LS_WARNING) << "InsertEvent invalid parameters"; + return kInvalidEventParameters; + } + DtmfList::iterator it = buffer_.begin(); + while (it != buffer_.end()) { + if (MergeEvents(it, event)) { + // A matching event was found and the new event was merged. + return kOK; + } + ++it; + } + buffer_.push_back(event); + // Sort the buffer using CompareEvents to rank the events. + buffer_.sort(CompareEvents); + return kOK; +} + +bool DtmfBuffer::GetEvent(uint32_t current_timestamp, DtmfEvent* event) { + DtmfList::iterator it = buffer_.begin(); + while (it != buffer_.end()) { + // |event_end| is an estimate of where the current event ends. If the end + // bit is set, we know that the event ends at |timestamp| + |duration|. + uint32_t event_end = it->timestamp + it->duration; +#ifdef LEGACY_BITEXACT + bool next_available = false; +#endif + if (!it->end_bit) { + // If the end bit is not set, we allow extrapolation of the event for + // some time. + event_end += max_extrapolation_samples_; + DtmfList::iterator next = it; + ++next; + if (next != buffer_.end()) { + // If there is a next event in the buffer, we will not extrapolate over + // the start of that new event. + event_end = std::min(event_end, next->timestamp); +#ifdef LEGACY_BITEXACT + next_available = true; +#endif + } + } + if (current_timestamp >= it->timestamp + && current_timestamp <= event_end) { // TODO(hlundin): Change to <. + // Found a matching event. + if (event) { + event->event_no = it->event_no; + event->end_bit = it->end_bit; + event->volume = it->volume; + event->duration = it->duration; + event->timestamp = it->timestamp; + } +#ifdef LEGACY_BITEXACT + if (it->end_bit && + current_timestamp + frame_len_samples_ >= event_end) { + // We are done playing this. Erase the event. + buffer_.erase(it); + } +#endif + return true; + } else if (current_timestamp > event_end) { // TODO(hlundin): Change to >=. + // Erase old event. Operation returns a valid pointer to the next element + // in the list. +#ifdef LEGACY_BITEXACT + if (!next_available) { + if (event) { + event->event_no = it->event_no; + event->end_bit = it->end_bit; + event->volume = it->volume; + event->duration = it->duration; + event->timestamp = it->timestamp; + } + it = buffer_.erase(it); + return true; + } else { + it = buffer_.erase(it); + } +#else + it = buffer_.erase(it); +#endif + } else { + ++it; + } + } + return false; +} + +size_t DtmfBuffer::Length() const { + return buffer_.size(); +} + +bool DtmfBuffer::Empty() const { + return buffer_.empty(); +} + +int DtmfBuffer::SetSampleRate(int fs_hz) { + if (fs_hz != 8000 && + fs_hz != 16000 && + fs_hz != 32000 && + fs_hz != 44100 && + fs_hz != 48000) { + return kInvalidSampleRate; + } + max_extrapolation_samples_ = 7 * fs_hz / 100; + frame_len_samples_ = fs_hz / 100; + return kOK; +} + +// The method returns true if the two events are considered to be the same. +// The are defined as equal if they share the same timestamp and event number. +// The special case with long-lasting events that have to be split into segments +// is not handled in this method. These will be treated as separate events in +// the buffer. +bool DtmfBuffer::SameEvent(const DtmfEvent& a, const DtmfEvent& b) { + return (a.event_no == b.event_no) && (a.timestamp == b.timestamp); +} + +bool DtmfBuffer::MergeEvents(DtmfList::iterator it, const DtmfEvent& event) { + if (SameEvent(*it, event)) { + if (!it->end_bit) { + // Do not extend the duration of an event for which the end bit was + // already received. + it->duration = std::max(event.duration, it->duration); + } + if (event.end_bit) { + it->end_bit = true; + } + return true; + } else { + return false; + } +} + +// Returns true if |a| goes before |b| in the sorting order ("|a| < |b|"). +// The events are ranked using their start timestamp (taking wrap-around into +// account). In the unlikely situation that two events share the same start +// timestamp, the event number is used to rank the two. Note that packets +// that belong to the same events, and therefore sharing the same start +// timestamp, have already been merged before the sort method is called. +bool DtmfBuffer::CompareEvents(const DtmfEvent& a, const DtmfEvent& b) { + if (a.timestamp == b.timestamp) { + return a.event_no < b.event_no; + } + // Take wrap-around into account. + return (static_cast(b.timestamp - a.timestamp) < 0xFFFFFFFF / 2); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.h new file mode 100644 index 0000000000..87a56554d5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DTMF_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_DTMF_BUFFER_H_ + +#include +#include // size_t + +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +struct DtmfEvent { + uint32_t timestamp; + int event_no; + int volume; + int duration; + bool end_bit; + + // Constructors + DtmfEvent() + : timestamp(0), + event_no(0), + volume(0), + duration(0), + end_bit(false) { + } + DtmfEvent(uint32_t ts, int ev, int vol, int dur, bool end) + : timestamp(ts), + event_no(ev), + volume(vol), + duration(dur), + end_bit(end) { + } +}; + +// This is the buffer holding DTMF events while waiting for them to be played. +class DtmfBuffer { + public: + enum BufferReturnCodes { + kOK = 0, + kInvalidPointer, + kPayloadTooShort, + kInvalidEventParameters, + kInvalidSampleRate + }; + + // Set up the buffer for use at sample rate |fs_hz|. + explicit DtmfBuffer(int fs_hz); + + virtual ~DtmfBuffer(); + + // Flushes the buffer. + virtual void Flush(); + + // Static method to parse 4 bytes from |payload| as a DTMF event (RFC 4733) + // and write the parsed information into the struct |event|. Input variable + // |rtp_timestamp| is simply copied into the struct. + static int ParseEvent(uint32_t rtp_timestamp, + const uint8_t* payload, + size_t payload_length_bytes, + DtmfEvent* event); + + // Inserts |event| into the buffer. The method looks for a matching event and + // merges the two if a match is found. + virtual int InsertEvent(const DtmfEvent& event); + + // Checks if a DTMF event should be played at time |current_timestamp|. If so, + // the method returns true; otherwise false. The parameters of the event to + // play will be written to |event|. + virtual bool GetEvent(uint32_t current_timestamp, DtmfEvent* event); + + // Number of events in the buffer. + virtual size_t Length() const; + + virtual bool Empty() const; + + // Set a new sample rate. + virtual int SetSampleRate(int fs_hz); + + private: + typedef std::list DtmfList; + + int max_extrapolation_samples_; + int frame_len_samples_; // TODO(hlundin): Remove this later. + + // Compares two events and returns true if they are the same. + static bool SameEvent(const DtmfEvent& a, const DtmfEvent& b); + + // Merges |event| to the event pointed out by |it|. The method checks that + // the two events are the same (using the SameEvent method), and merges them + // if that was the case, returning true. If the events are not the same, false + // is returned. + bool MergeEvents(DtmfList::iterator it, const DtmfEvent& event); + + // Method used by the sort algorithm to rank events in the buffer. + static bool CompareEvents(const DtmfEvent& a, const DtmfEvent& b); + + DtmfList buffer_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DtmfBuffer); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DTMF_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc new file mode 100644 index 0000000000..7bcf1e0f4e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dtmf_buffer.h" + +#ifdef WIN32 +#include // ntohl() +#else +#include // ntohl() +#endif + +#include + +#include "test/gtest.h" + +// Modify the tests so that they pass with the modifications done to DtmfBuffer +// for backwards bit-exactness. Once bit-exactness is no longer required, this +// #define should be removed (and the code that it enables). +#define LEGACY_BITEXACT + +namespace webrtc { + +static int sample_rate_hz = 8000; + +static uint32_t MakeDtmfPayload(int event, bool end, int volume, int duration) { + uint32_t payload = 0; +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | event |E|R| volume | duration | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + payload |= (event & 0x00FF) << 24; + payload |= (end ? 0x00800000 : 0x00000000); + payload |= (volume & 0x003F) << 16; + payload |= (duration & 0xFFFF); + payload = ntohl(payload); + return payload; +} + +static bool EqualEvents(const DtmfEvent& a, + const DtmfEvent& b) { + return (a.duration == b.duration + && a.end_bit == b.end_bit + && a.event_no == b.event_no + && a.timestamp == b.timestamp + && a.volume == b.volume); +} + +TEST(DtmfBuffer, CreateAndDestroy) { + DtmfBuffer* buffer = new DtmfBuffer(sample_rate_hz); + delete buffer; +} + +// Test the event parser. +TEST(DtmfBuffer, ParseEvent) { + int event_no = 7; + bool end_bit = true; + int volume = 17; + int duration = 4711; + uint32_t timestamp = 0x12345678; + uint32_t payload = MakeDtmfPayload(event_no, end_bit, volume, duration); + uint8_t* payload_ptr = reinterpret_cast(&payload); + DtmfEvent event; + EXPECT_EQ(DtmfBuffer::kOK, + DtmfBuffer::ParseEvent(timestamp, payload_ptr, sizeof(payload), + &event)); + EXPECT_EQ(duration, event.duration); + EXPECT_EQ(end_bit, event.end_bit); + EXPECT_EQ(event_no, event.event_no); + EXPECT_EQ(timestamp, event.timestamp); + EXPECT_EQ(volume, event.volume); + + EXPECT_EQ(DtmfBuffer::kPayloadTooShort, + DtmfBuffer::ParseEvent(timestamp, payload_ptr, 3, &event)); +} + +TEST(DtmfBuffer, SimpleInsertAndGet) { + int event_no = 7; + bool end_bit = true; + int volume = 17; + int duration = 4711; + uint32_t timestamp = 0x12345678; + DtmfEvent event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); + DtmfEvent out_event; + // Too early to get event. + EXPECT_FALSE(buffer.GetEvent(timestamp - 10, &out_event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); + // Get the event at its starting timestamp. + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(event, out_event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); + // Get the event some time into the event. + EXPECT_TRUE(buffer.GetEvent(timestamp + duration / 2, &out_event)); + EXPECT_TRUE(EqualEvents(event, out_event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); + // Give a "current" timestamp after the event has ended. +#ifdef LEGACY_BITEXACT + EXPECT_TRUE(buffer.GetEvent(timestamp + duration + 10, &out_event)); +#endif + EXPECT_FALSE(buffer.GetEvent(timestamp + duration + 10, &out_event)); + EXPECT_EQ(0u, buffer.Length()); + EXPECT_TRUE(buffer.Empty()); +} + +TEST(DtmfBuffer, MergingPackets) { + int event_no = 0; + bool end_bit = false; + int volume = 17; + int duration = 80; + uint32_t timestamp = 0x12345678; + DtmfEvent event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + + event.duration += 80; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + + event.duration += 80; + event.end_bit = true; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + + EXPECT_EQ(1u, buffer.Length()); + + DtmfEvent out_event; + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(event, out_event)); +} + +// This test case inserts one shorter event completely overlapped by one longer +// event. The expected outcome is that only the longer event is played. +TEST(DtmfBuffer, OverlappingEvents) { + int event_no = 0; + bool end_bit = true; + int volume = 1; + int duration = 80; + uint32_t timestamp = 0x12345678 + 80; + DtmfEvent short_event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(short_event)); + + event_no = 10; + end_bit = false; + timestamp = 0x12345678; + DtmfEvent long_event(timestamp, event_no, volume, duration, end_bit); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(long_event)); + + long_event.duration += 80; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(long_event)); + + long_event.duration += 80; + long_event.end_bit = true; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(long_event)); + + EXPECT_EQ(2u, buffer.Length()); + + DtmfEvent out_event; + // Expect to get the long event. + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(long_event, out_event)); + // Expect no more events. +#ifdef LEGACY_BITEXACT + EXPECT_TRUE(buffer.GetEvent(timestamp + long_event.duration + 10, + &out_event)); + EXPECT_TRUE(EqualEvents(long_event, out_event)); + EXPECT_TRUE(buffer.GetEvent(timestamp + long_event.duration + 10, + &out_event)); + EXPECT_TRUE(EqualEvents(short_event, out_event)); +#else + EXPECT_FALSE(buffer.GetEvent(timestamp + long_event.duration + 10, + &out_event)); +#endif + EXPECT_TRUE(buffer.Empty()); +} + +TEST(DtmfBuffer, ExtrapolationTime) { + int event_no = 0; + bool end_bit = false; + int volume = 1; + int duration = 80; + uint32_t timestamp = 0x12345678; + DtmfEvent event1(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event1)); + EXPECT_EQ(1u, buffer.Length()); + + DtmfEvent out_event; + // Get the event at the start. + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); + // Also get the event 100 samples after the end of the event (since we're + // missing the end bit). + uint32_t timestamp_now = timestamp + duration + 100; + EXPECT_TRUE(buffer.GetEvent(timestamp_now, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); + // Insert another event starting back-to-back with the previous event. + timestamp += duration; + event_no = 1; + DtmfEvent event2(timestamp, event_no, volume, duration, end_bit); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event2)); + EXPECT_EQ(2u, buffer.Length()); + // Now we expect to get the new event when supplying |timestamp_now|. + EXPECT_TRUE(buffer.GetEvent(timestamp_now, &out_event)); + EXPECT_TRUE(EqualEvents(event2, out_event)); + // Expect the the first event to be erased now. + EXPECT_EQ(1u, buffer.Length()); + // Move |timestamp_now| to more than 560 samples after the end of the second + // event. Expect that event to be erased. + timestamp_now = timestamp + duration + 600; +#ifdef LEGACY_BITEXACT + EXPECT_TRUE(buffer.GetEvent(timestamp_now, &out_event)); +#endif + EXPECT_FALSE(buffer.GetEvent(timestamp_now, &out_event)); + EXPECT_TRUE(buffer.Empty()); +} + +TEST(DtmfBuffer, TimestampWraparound) { + int event_no = 0; + bool end_bit = true; + int volume = 1; + int duration = 80; + uint32_t timestamp1 = 0xFFFFFFFF - duration; + DtmfEvent event1(timestamp1, event_no, volume, duration, end_bit); + uint32_t timestamp2 = 0; + DtmfEvent event2(timestamp2, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event1)); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event2)); + EXPECT_EQ(2u, buffer.Length()); + DtmfEvent out_event; + EXPECT_TRUE(buffer.GetEvent(timestamp1, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); +#ifdef LEGACY_BITEXACT + EXPECT_EQ(1u, buffer.Length()); +#else + EXPECT_EQ(2u, buffer.Length()); +#endif + + buffer.Flush(); + // Reverse the insert order. Expect same results. + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event2)); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event1)); + EXPECT_EQ(2u, buffer.Length()); + EXPECT_TRUE(buffer.GetEvent(timestamp1, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); +#ifdef LEGACY_BITEXACT + EXPECT_EQ(1u, buffer.Length()); +#else + EXPECT_EQ(2u, buffer.Length()); +#endif +} + +TEST(DtmfBuffer, InvalidEvents) { + int event_no = 0; + bool end_bit = true; + int volume = 1; + int duration = 80; + uint32_t timestamp = 0x12345678; + DtmfEvent event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + + // Invalid event number. + event.event_no = -1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.event_no = 16; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.event_no = 0; // Valid value; + + // Invalid volume. + event.volume = -1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.volume = 64; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.volume = 0; // Valid value; + + // Invalid duration. + event.duration = -1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.duration = 0; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.duration = 0xFFFF + 1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.duration = 1; // Valid value; + + // Finish with a valid event, just to verify that all is ok. + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc new file mode 100644 index 0000000000..b848c600d0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This class provides a generator for DTMF tones. The tone generation is based +// on a sinusoid recursion. Each sinusoid is generated using a recursion +// formula; x[n] = a * x[n-1] - x[n-2], where the coefficient +// a = 2*cos(2*pi*f/fs). The recursion is started with x[-1] = 0 and +// x[-2] = sin(2*pi*f/fs). (Note that with this initialization, the resulting +// sinusoid gets a "negative" rotation; x[n] = sin(-2*pi*f/fs * n + phi), but +// kept this way due to historical reasons.) +// TODO(hlundin): Change to positive rotation? +// +// Each key on the telephone keypad corresponds to an "event", 0-15. Each event +// is mapped to a tone pair, with a low and a high frequency. There are four +// low and four high frequencies, each corresponding to a row and column, +// respectively, on the keypad as illustrated below. +// +// 1209 Hz 1336 Hz 1477 Hz 1633 Hz +// 697 Hz 1 2 3 12 +// 770 Hz 4 5 6 13 +// 852 Hz 7 8 9 14 +// 941 Hz 10 0 11 15 + +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" + +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// The filter coefficient a = 2*cos(2*pi*f/fs) for the low frequency tone, for +// sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0 through 15. +// Values are in Q14. +const int DtmfToneGenerator::kCoeff1[4][16] = { + { 24219, 27980, 27980, 27980, 26956, 26956, 26956, 25701, 25701, 25701, + 24219, 24219, 27980, 26956, 25701, 24219 }, + { 30556, 31548, 31548, 31548, 31281, 31281, 31281, 30951, 30951, 30951, + 30556, 30556, 31548, 31281, 30951, 30556 }, + { 32210, 32462, 32462, 32462, 32394, 32394, 32394, 32311, 32311, 32311, + 32210, 32210, 32462, 32394, 32311, 32210 }, + { 32520, 32632, 32632, 32632, 32602, 32602, 32602, 32564, 32564, 32564, + 32520, 32520, 32632, 32602, 32564, 32520 } }; + +// The filter coefficient a = 2*cos(2*pi*f/fs) for the high frequency tone, for +// sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0 through 15. +// Values are in Q14. +const int DtmfToneGenerator::kCoeff2[4][16] = { + { 16325, 19073, 16325, 13085, 19073, 16325, 13085, 19073, 16325, 13085, + 19073, 13085, 9315, 9315, 9315, 9315}, + { 28361, 29144, 28361, 27409, 29144, 28361, 27409, 29144, 28361, 27409, + 29144, 27409, 26258, 26258, 26258, 26258}, + { 31647, 31849, 31647, 31400, 31849, 31647, 31400, 31849, 31647, 31400, + 31849, 31400, 31098, 31098, 31098, 31098}, + { 32268, 32359, 32268, 32157, 32359, 32268, 32157, 32359, 32268, 32157, + 32359, 32157, 32022, 32022, 32022, 32022} }; + +// The initialization value x[-2] = sin(2*pi*f/fs) for the low frequency tone, +// for sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0-15. +// Values are in Q14. +const int DtmfToneGenerator::kInitValue1[4][16] = { + { 11036, 8528, 8528, 8528, 9315, 9315, 9315, 10163, 10163, 10163, 11036, + 11036, 8528, 9315, 10163, 11036}, + { 5918, 4429, 4429, 4429, 4879, 4879, 4879, 5380, 5380, 5380, 5918, 5918, + 4429, 4879, 5380, 5918}, + { 3010, 2235, 2235, 2235, 2468, 2468, 2468, 2728, 2728, 2728, 3010, 3010, + 2235, 2468, 2728, 3010}, + { 2013, 1493, 1493, 1493, 1649, 1649, 1649, 1823, 1823, 1823, 2013, 2013, + 1493, 1649, 1823, 2013 } }; + +// The initialization value x[-2] = sin(2*pi*f/fs) for the high frequency tone, +// for sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0-15. +// Values are in Q14. +const int DtmfToneGenerator::kInitValue2[4][16] = { + { 14206, 13323, 14206, 15021, 13323, 14206, 15021, 13323, 14206, 15021, + 13323, 15021, 15708, 15708, 15708, 15708}, + { 8207, 7490, 8207, 8979, 7490, 8207, 8979, 7490, 8207, 8979, 7490, 8979, + 9801, 9801, 9801, 9801}, + { 4249, 3853, 4249, 4685, 3853, 4249, 4685, 3853, 4249, 4685, 3853, 4685, + 5164, 5164, 5164, 5164}, + { 2851, 2582, 2851, 3148, 2582, 2851, 3148, 2582, 2851, 3148, 2582, 3148, + 3476, 3476, 3476, 3476} }; + +// Amplitude multipliers for volume values 0 through 63, corresponding to +// 0 dBm0 through -63 dBm0. Values are in Q14. +// for a in range(0, 64): +// print round(16141.0 * 10**(-float(a)/20)) +const int DtmfToneGenerator::kAmplitude[64] = { + 16141, 14386, 12821, 11427, 10184, 9077, 8090, 7210, 6426, 5727, 5104, 4549, + 4054, 3614, 3221, 2870, 2558, 2280, 2032, 1811, 1614, 1439, 1282, 1143, + 1018, 908, 809, 721, 643, 573, 510, 455, 405, 361, 322, 287, 256, 228, 203, + 181, 161, 144, 128, 114, 102, 91, 81, 72, 64, 57, 51, 45, 41, 36, 32, 29, + 26, 23, 20, 18, 16, 14, 13, 11 }; + +// Constructor. +DtmfToneGenerator::DtmfToneGenerator() + : initialized_(false), + coeff1_(0), + coeff2_(0), + amplitude_(0) { +} + +// Initialize the DTMF generator with sample rate fs Hz (8000, 16000, 32000, +// 48000), event (0-15) and attenuation (0-36 dB). +// Returns 0 on success, otherwise an error code. +int DtmfToneGenerator::Init(int fs, int event, int attenuation) { + initialized_ = false; + size_t fs_index; + if (fs == 8000) { + fs_index = 0; + } else if (fs == 16000) { + fs_index = 1; + } else if (fs == 32000) { + fs_index = 2; + } else if (fs == 48000) { + fs_index = 3; + } else { + RTC_NOTREACHED(); + fs_index = 1; // Default to 8000 Hz. + } + + if (event < 0 || event > 15) { + return kParameterError; // Invalid event number. + } + + if (attenuation < 0 || attenuation > 63) { + return kParameterError; // Invalid attenuation. + } + + // Look up oscillator coefficient for low and high frequencies. + RTC_DCHECK_LE(0, fs_index); + RTC_DCHECK_GT(arraysize(kCoeff1), fs_index); + RTC_DCHECK_GT(arraysize(kCoeff2), fs_index); + RTC_DCHECK_LE(0, event); + RTC_DCHECK_GT(arraysize(kCoeff1[fs_index]), event); + RTC_DCHECK_GT(arraysize(kCoeff2[fs_index]), event); + coeff1_ = kCoeff1[fs_index][event]; + coeff2_ = kCoeff2[fs_index][event]; + + // Look up amplitude multiplier. + RTC_DCHECK_LE(0, attenuation); + RTC_DCHECK_GT(arraysize(kAmplitude), attenuation); + amplitude_ = kAmplitude[attenuation]; + + // Initialize sample history. + RTC_DCHECK_LE(0, fs_index); + RTC_DCHECK_GT(arraysize(kInitValue1), fs_index); + RTC_DCHECK_GT(arraysize(kInitValue2), fs_index); + RTC_DCHECK_LE(0, event); + RTC_DCHECK_GT(arraysize(kInitValue1[fs_index]), event); + RTC_DCHECK_GT(arraysize(kInitValue2[fs_index]), event); + sample_history1_[0] = kInitValue1[fs_index][event]; + sample_history1_[1] = 0; + sample_history2_[0] = kInitValue2[fs_index][event]; + sample_history2_[1] = 0; + + initialized_ = true; + return 0; +} + +// Reset tone generator to uninitialized state. +void DtmfToneGenerator::Reset() { + initialized_ = false; +} + +// Generate num_samples of DTMF signal and write to |output|. +int DtmfToneGenerator::Generate(size_t num_samples, + AudioMultiVector* output) { + if (!initialized_) { + return kNotInitialized; + } + + if (!output) { + return kParameterError; + } + + output->AssertSize(num_samples); + for (size_t i = 0; i < num_samples; ++i) { + // Use recursion formula y[n] = a * y[n - 1] - y[n - 2]. + int16_t temp_val_low = ((coeff1_ * sample_history1_[1] + 8192) >> 14) + - sample_history1_[0]; + int16_t temp_val_high = ((coeff2_ * sample_history2_[1] + 8192) >> 14) + - sample_history2_[0]; + + // Update recursion memory. + sample_history1_[0] = sample_history1_[1]; + sample_history1_[1] = temp_val_low; + sample_history2_[0] = sample_history2_[1]; + sample_history2_[1] = temp_val_high; + + // Attenuate the low frequency tone 3 dB. + int32_t temp_val = + kAmpMultiplier * temp_val_low + temp_val_high * (1 << 15); + // Normalize the signal to Q14 with proper rounding. + temp_val = (temp_val + 16384) >> 15; + // Scale the signal to correct volume. + (*output)[0][i] = + static_cast((temp_val * amplitude_ + 8192) >> 14); + } + // Copy first channel to all other channels. + for (size_t channel = 1; channel < output->Channels(); ++channel) { + output->CopyChannel(0, channel); + } + + return static_cast(num_samples); +} + +bool DtmfToneGenerator::initialized() const { + return initialized_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h new file mode 100644 index 0000000000..faad6a25b1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DTMF_TONE_GENERATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_DTMF_TONE_GENERATOR_H_ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// This class provides a generator for DTMF tones. +class DtmfToneGenerator { + public: + enum ReturnCodes { + kNotInitialized = -1, + kParameterError = -2, + }; + + DtmfToneGenerator(); + virtual ~DtmfToneGenerator() {} + virtual int Init(int fs, int event, int attenuation); + virtual void Reset(); + virtual int Generate(size_t num_samples, AudioMultiVector* output); + virtual bool initialized() const; + + private: + static const int kCoeff1[4][16]; // 1st oscillator model coefficient table. + static const int kCoeff2[4][16]; // 2nd oscillator model coefficient table. + static const int kInitValue1[4][16]; // Initialization for 1st oscillator. + static const int kInitValue2[4][16]; // Initialization for 2nd oscillator. + static const int kAmplitude[64]; // Amplitude for 0 through -63 dBm0. + static const int16_t kAmpMultiplier = 23171; // 3 dB attenuation (in Q15). + + bool initialized_; // True if generator is initialized properly. + int coeff1_; // 1st oscillator coefficient for this event. + int coeff2_; // 2nd oscillator coefficient for this event. + int amplitude_; // Amplitude for this event. + int16_t sample_history1_[2]; // Last 2 samples for the 1st oscillator. + int16_t sample_history2_[2]; // Last 2 samples for the 2nd oscillator. + + RTC_DISALLOW_COPY_AND_ASSIGN(DtmfToneGenerator); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DTMF_TONE_GENERATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc new file mode 100644 index 0000000000..437af0d345 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for DtmfToneGenerator class. + +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" + +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "test/gtest.h" + +namespace webrtc { + +class DtmfToneGeneratorTest : public ::testing::Test { + protected: + static const double kLowFreqHz[16]; + static const double kHighFreqHz[16]; + // This is the attenuation applied to all cases. + const double kBaseAttenuation = 16141.0 / 16384.0; + const double k3dbAttenuation = 23171.0 / 32768; + const int kNumSamples = 10; + + void TestAllTones(int fs_hz, int channels) { + AudioMultiVector signal(channels); + + for (int event = 0; event <= 15; ++event) { + std::ostringstream ss; + ss << "Checking event " << event << " at sample rate " << fs_hz; + SCOPED_TRACE(ss.str()); + const int kAttenuation = 0; + ASSERT_EQ(0, tone_gen_.Init(fs_hz, event, kAttenuation)); + EXPECT_TRUE(tone_gen_.initialized()); + EXPECT_EQ(kNumSamples, tone_gen_.Generate(kNumSamples, &signal)); + + double f1 = kLowFreqHz[event]; + double f2 = kHighFreqHz[event]; + const double pi = 3.14159265358979323846; + + for (int n = 0; n < kNumSamples; ++n) { + double x = k3dbAttenuation * sin(2.0 * pi * f1 / fs_hz * (-n - 1)) + + sin(2.0 * pi * f2 / fs_hz * (-n - 1)); + x *= kBaseAttenuation; + x = ldexp(x, 14); // Scale to Q14. + for (int channel = 0; channel < channels; ++channel) { + EXPECT_NEAR(x, static_cast(signal[channel][n]), 25); + } + } + + tone_gen_.Reset(); + EXPECT_FALSE(tone_gen_.initialized()); + } + } + + void TestAmplitudes(int fs_hz, int channels) { + AudioMultiVector signal(channels); + AudioMultiVector ref_signal(channels); + + const int event_vec[] = {0, 4, 9, 13}; // Test a few events. + for (int e = 0; e < 4; ++e) { + int event = event_vec[e]; + // Create full-scale reference. + ASSERT_EQ(0, tone_gen_.Init(fs_hz, event, 0)); // 0 attenuation. + EXPECT_EQ(kNumSamples, tone_gen_.Generate(kNumSamples, &ref_signal)); + // Test every 5 steps (to save time). + for (int attenuation = 1; attenuation <= 63; attenuation += 5) { + std::ostringstream ss; + ss << "Checking event " << event << " at sample rate " << fs_hz; + ss << "; attenuation " << attenuation; + SCOPED_TRACE(ss.str()); + ASSERT_EQ(0, tone_gen_.Init(fs_hz, event, attenuation)); + EXPECT_EQ(kNumSamples, tone_gen_.Generate(kNumSamples, &signal)); + for (int n = 0; n < kNumSamples; ++n) { + double attenuation_factor = + pow(10, -static_cast(attenuation) / 20); + // Verify that the attenuation is correct. + for (int channel = 0; channel < channels; ++channel) { + EXPECT_NEAR(attenuation_factor * ref_signal[channel][n], + signal[channel][n], + 2); + } + } + + tone_gen_.Reset(); + } + } + } + + DtmfToneGenerator tone_gen_; +}; + +// Low and high frequencies for events 0 through 15. +const double DtmfToneGeneratorTest::kLowFreqHz[16] = { + 941.0, 697.0, 697.0, 697.0, 770.0, 770.0, 770.0, 852.0, + 852.0, 852.0, 941.0, 941.0, 697.0, 770.0, 852.0, 941.0}; +const double DtmfToneGeneratorTest::kHighFreqHz[16] = { + 1336.0, 1209.0, 1336.0, 1477.0, 1209.0, 1336.0, 1477.0, 1209.0, + 1336.0, 1477.0, 1209.0, 1477.0, 1633.0, 1633.0, 1633.0, 1633.0}; + +TEST_F(DtmfToneGeneratorTest, Test8000Mono) { + TestAllTones(8000, 1); + TestAmplitudes(8000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test16000Mono) { + TestAllTones(16000, 1); + TestAmplitudes(16000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test32000Mono) { + TestAllTones(32000, 1); + TestAmplitudes(32000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test48000Mono) { + TestAllTones(48000, 1); + TestAmplitudes(48000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test8000Stereo) { + TestAllTones(8000, 2); + TestAmplitudes(8000, 2); +} + +TEST_F(DtmfToneGeneratorTest, Test16000Stereo) { + TestAllTones(16000, 2); + TestAmplitudes(16000, 2); +} + +TEST_F(DtmfToneGeneratorTest, Test32000Stereo) { + TestAllTones(32000, 2); + TestAmplitudes(32000, 2); +} + +TEST_F(DtmfToneGeneratorTest, Test48000Stereo) { + TestAllTones(48000, 2); + TestAmplitudes(48000, 2); +} + +TEST(DtmfToneGenerator, TestErrors) { + DtmfToneGenerator tone_gen; + const int kNumSamples = 10; + AudioMultiVector signal(1); // One channel. + + // Try to generate tones without initializing. + EXPECT_EQ(DtmfToneGenerator::kNotInitialized, + tone_gen.Generate(kNumSamples, &signal)); + + const int fs = 16000; // Valid sample rate. + const int event = 7; // Valid event. + const int attenuation = 0; // Valid attenuation. + // Initialize with invalid event -1. + EXPECT_EQ(DtmfToneGenerator::kParameterError, + tone_gen.Init(fs, -1, attenuation)); + // Initialize with invalid event 16. + EXPECT_EQ(DtmfToneGenerator::kParameterError, + tone_gen.Init(fs, 16, attenuation)); + // Initialize with invalid attenuation -1. + EXPECT_EQ(DtmfToneGenerator::kParameterError, tone_gen.Init(fs, event, -1)); + // Initialize with invalid attenuation 64. + EXPECT_EQ(DtmfToneGenerator::kParameterError, tone_gen.Init(fs, event, 64)); + EXPECT_FALSE(tone_gen.initialized()); // Should still be uninitialized. + + // Initialize with valid parameters. + ASSERT_EQ(0, tone_gen.Init(fs, event, attenuation)); + EXPECT_TRUE(tone_gen.initialized()); + // NULL pointer to destination. + EXPECT_EQ(DtmfToneGenerator::kParameterError, + tone_gen.Generate(kNumSamples, NULL)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.cc new file mode 100644 index 0000000000..3fb09ead49 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.cc @@ -0,0 +1,978 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/expand.h" + +#include +#include // memset + +#include // min, max +#include // numeric_limits + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +Expand::Expand(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) + : random_vector_(random_vector), + sync_buffer_(sync_buffer), + first_expand_(true), + fs_hz_(fs), + num_channels_(num_channels), + consecutive_expands_(0), + background_noise_(background_noise), + statistics_(statistics), + overlap_length_(5 * fs / 8000), + lag_index_direction_(0), + current_lag_index_(0), + stop_muting_(false), + expand_duration_samples_(0), + channel_parameters_(new ChannelParameters[num_channels_]) { + assert(fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000); + assert(fs <= static_cast(kMaxSampleRate)); // Should not be possible. + assert(num_channels_ > 0); + memset(expand_lags_, 0, sizeof(expand_lags_)); + Reset(); +} + +Expand::~Expand() = default; + +void Expand::Reset() { + first_expand_ = true; + consecutive_expands_ = 0; + max_lag_ = 0; + for (size_t ix = 0; ix < num_channels_; ++ix) { + channel_parameters_[ix].expand_vector0.Clear(); + channel_parameters_[ix].expand_vector1.Clear(); + } +} + +int Expand::Process(AudioMultiVector* output) { + int16_t random_vector[kMaxSampleRate / 8000 * 120 + 30]; + int16_t scaled_random_vector[kMaxSampleRate / 8000 * 125]; + static const int kTempDataSize = 3600; + int16_t temp_data[kTempDataSize]; // TODO(hlundin) Remove this. + int16_t* voiced_vector_storage = temp_data; + int16_t* voiced_vector = &voiced_vector_storage[overlap_length_]; + static const size_t kNoiseLpcOrder = BackgroundNoise::kMaxLpcOrder; + int16_t unvoiced_array_memory[kNoiseLpcOrder + kMaxSampleRate / 8000 * 125]; + int16_t* unvoiced_vector = unvoiced_array_memory + kUnvoicedLpcOrder; + int16_t* noise_vector = unvoiced_array_memory + kNoiseLpcOrder; + + int fs_mult = fs_hz_ / 8000; + + if (first_expand_) { + // Perform initial setup if this is the first expansion since last reset. + AnalyzeSignal(random_vector); + first_expand_ = false; + expand_duration_samples_ = 0; + } else { + // This is not the first expansion, parameters are already estimated. + // Extract a noise segment. + size_t rand_length = max_lag_; + // This only applies to SWB where length could be larger than 256. + assert(rand_length <= kMaxSampleRate / 8000 * 120 + 30); + GenerateRandomVector(2, rand_length, random_vector); + } + + + // Generate signal. + UpdateLagIndex(); + + // Voiced part. + // Generate a weighted vector with the current lag. + size_t expansion_vector_length = max_lag_ + overlap_length_; + size_t current_lag = expand_lags_[current_lag_index_]; + // Copy lag+overlap data. + size_t expansion_vector_position = expansion_vector_length - current_lag - + overlap_length_; + size_t temp_length = current_lag + overlap_length_; + for (size_t channel_ix = 0; channel_ix < num_channels_; ++channel_ix) { + ChannelParameters& parameters = channel_parameters_[channel_ix]; + if (current_lag_index_ == 0) { + // Use only expand_vector0. + assert(expansion_vector_position + temp_length <= + parameters.expand_vector0.Size()); + parameters.expand_vector0.CopyTo(temp_length, expansion_vector_position, + voiced_vector_storage); + } else if (current_lag_index_ == 1) { + std::unique_ptr temp_0(new int16_t[temp_length]); + parameters.expand_vector0.CopyTo(temp_length, expansion_vector_position, + temp_0.get()); + std::unique_ptr temp_1(new int16_t[temp_length]); + parameters.expand_vector1.CopyTo(temp_length, expansion_vector_position, + temp_1.get()); + // Mix 3/4 of expand_vector0 with 1/4 of expand_vector1. + WebRtcSpl_ScaleAndAddVectorsWithRound(temp_0.get(), 3, temp_1.get(), 1, 2, + voiced_vector_storage, temp_length); + } else if (current_lag_index_ == 2) { + // Mix 1/2 of expand_vector0 with 1/2 of expand_vector1. + assert(expansion_vector_position + temp_length <= + parameters.expand_vector0.Size()); + assert(expansion_vector_position + temp_length <= + parameters.expand_vector1.Size()); + + std::unique_ptr temp_0(new int16_t[temp_length]); + parameters.expand_vector0.CopyTo(temp_length, expansion_vector_position, + temp_0.get()); + std::unique_ptr temp_1(new int16_t[temp_length]); + parameters.expand_vector1.CopyTo(temp_length, expansion_vector_position, + temp_1.get()); + WebRtcSpl_ScaleAndAddVectorsWithRound(temp_0.get(), 1, temp_1.get(), 1, 1, + voiced_vector_storage, temp_length); + } + + // Get tapering window parameters. Values are in Q15. + int16_t muting_window, muting_window_increment; + int16_t unmuting_window, unmuting_window_increment; + if (fs_hz_ == 8000) { + muting_window = DspHelper::kMuteFactorStart8kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement8kHz; + unmuting_window = DspHelper::kUnmuteFactorStart8kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement8kHz; + } else if (fs_hz_ == 16000) { + muting_window = DspHelper::kMuteFactorStart16kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement16kHz; + unmuting_window = DspHelper::kUnmuteFactorStart16kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement16kHz; + } else if (fs_hz_ == 32000) { + muting_window = DspHelper::kMuteFactorStart32kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement32kHz; + unmuting_window = DspHelper::kUnmuteFactorStart32kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement32kHz; + } else { // fs_ == 48000 + muting_window = DspHelper::kMuteFactorStart48kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement48kHz; + unmuting_window = DspHelper::kUnmuteFactorStart48kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement48kHz; + } + + // Smooth the expanded if it has not been muted to a low amplitude and + // |current_voice_mix_factor| is larger than 0.5. + if ((parameters.mute_factor > 819) && + (parameters.current_voice_mix_factor > 8192)) { + size_t start_ix = sync_buffer_->Size() - overlap_length_; + for (size_t i = 0; i < overlap_length_; i++) { + // Do overlap add between new vector and overlap. + (*sync_buffer_)[channel_ix][start_ix + i] = + (((*sync_buffer_)[channel_ix][start_ix + i] * muting_window) + + (((parameters.mute_factor * voiced_vector_storage[i]) >> 14) * + unmuting_window) + 16384) >> 15; + muting_window += muting_window_increment; + unmuting_window += unmuting_window_increment; + } + } else if (parameters.mute_factor == 0) { + // The expanded signal will consist of only comfort noise if + // mute_factor = 0. Set the output length to 15 ms for best noise + // production. + // TODO(hlundin): This has been disabled since the length of + // parameters.expand_vector0 and parameters.expand_vector1 no longer + // match with expand_lags_, causing invalid reads and writes. Is it a good + // idea to enable this again, and solve the vector size problem? +// max_lag_ = fs_mult * 120; +// expand_lags_[0] = fs_mult * 120; +// expand_lags_[1] = fs_mult * 120; +// expand_lags_[2] = fs_mult * 120; + } + + // Unvoiced part. + // Filter |scaled_random_vector| through |ar_filter_|. + memcpy(unvoiced_vector - kUnvoicedLpcOrder, parameters.ar_filter_state, + sizeof(int16_t) * kUnvoicedLpcOrder); + int32_t add_constant = 0; + if (parameters.ar_gain_scale > 0) { + add_constant = 1 << (parameters.ar_gain_scale - 1); + } + WebRtcSpl_AffineTransformVector(scaled_random_vector, random_vector, + parameters.ar_gain, add_constant, + parameters.ar_gain_scale, + current_lag); + WebRtcSpl_FilterARFastQ12(scaled_random_vector, unvoiced_vector, + parameters.ar_filter, kUnvoicedLpcOrder + 1, + current_lag); + memcpy(parameters.ar_filter_state, + &(unvoiced_vector[current_lag - kUnvoicedLpcOrder]), + sizeof(int16_t) * kUnvoicedLpcOrder); + + // Combine voiced and unvoiced contributions. + + // Set a suitable cross-fading slope. + // For lag = + // <= 31 * fs_mult => go from 1 to 0 in about 8 ms; + // (>= 31 .. <= 63) * fs_mult => go from 1 to 0 in about 16 ms; + // >= 64 * fs_mult => go from 1 to 0 in about 32 ms. + // temp_shift = getbits(max_lag_) - 5. + int temp_shift = + (31 - WebRtcSpl_NormW32(rtc::dchecked_cast(max_lag_))) - 5; + int16_t mix_factor_increment = 256 >> temp_shift; + if (stop_muting_) { + mix_factor_increment = 0; + } + + // Create combined signal by shifting in more and more of unvoiced part. + temp_shift = 8 - temp_shift; // = getbits(mix_factor_increment). + size_t temp_length = (parameters.current_voice_mix_factor - + parameters.voice_mix_factor) >> temp_shift; + temp_length = std::min(temp_length, current_lag); + DspHelper::CrossFade(voiced_vector, unvoiced_vector, temp_length, + ¶meters.current_voice_mix_factor, + mix_factor_increment, temp_data); + + // End of cross-fading period was reached before end of expanded signal + // path. Mix the rest with a fixed mixing factor. + if (temp_length < current_lag) { + if (mix_factor_increment != 0) { + parameters.current_voice_mix_factor = parameters.voice_mix_factor; + } + int16_t temp_scale = 16384 - parameters.current_voice_mix_factor; + WebRtcSpl_ScaleAndAddVectorsWithRound( + voiced_vector + temp_length, parameters.current_voice_mix_factor, + unvoiced_vector + temp_length, temp_scale, 14, + temp_data + temp_length, current_lag - temp_length); + } + + // Select muting slope depending on how many consecutive expands we have + // done. + if (consecutive_expands_ == 3) { + // Let the mute factor decrease from 1.0 to 0.95 in 6.25 ms. + // mute_slope = 0.0010 / fs_mult in Q20. + parameters.mute_slope = std::max(parameters.mute_slope, 1049 / fs_mult); + } + if (consecutive_expands_ == 7) { + // Let the mute factor decrease from 1.0 to 0.90 in 6.25 ms. + // mute_slope = 0.0020 / fs_mult in Q20. + parameters.mute_slope = std::max(parameters.mute_slope, 2097 / fs_mult); + } + + // Mute segment according to slope value. + if ((consecutive_expands_ != 0) || !parameters.onset) { + // Mute to the previous level, then continue with the muting. + WebRtcSpl_AffineTransformVector(temp_data, temp_data, + parameters.mute_factor, 8192, + 14, current_lag); + + if (!stop_muting_) { + DspHelper::MuteSignal(temp_data, parameters.mute_slope, current_lag); + + // Shift by 6 to go from Q20 to Q14. + // TODO(hlundin): Adding 8192 before shifting 6 steps seems wrong. + // Legacy. + int16_t gain = static_cast(16384 - + (((current_lag * parameters.mute_slope) + 8192) >> 6)); + gain = ((gain * parameters.mute_factor) + 8192) >> 14; + + // Guard against getting stuck with very small (but sometimes audible) + // gain. + if ((consecutive_expands_ > 3) && (gain >= parameters.mute_factor)) { + parameters.mute_factor = 0; + } else { + parameters.mute_factor = gain; + } + } + } + + // Background noise part. + GenerateBackgroundNoise(random_vector, + channel_ix, + channel_parameters_[channel_ix].mute_slope, + TooManyExpands(), + current_lag, + unvoiced_array_memory); + + // Add background noise to the combined voiced-unvoiced signal. + for (size_t i = 0; i < current_lag; i++) { + temp_data[i] = temp_data[i] + noise_vector[i]; + } + if (channel_ix == 0) { + output->AssertSize(current_lag); + } else { + assert(output->Size() == current_lag); + } + (*output)[channel_ix].OverwriteAt(temp_data, current_lag, 0); + } + + // Increase call number and cap it. + consecutive_expands_ = consecutive_expands_ >= kMaxConsecutiveExpands ? + kMaxConsecutiveExpands : consecutive_expands_ + 1; + expand_duration_samples_ += output->Size(); + // Clamp the duration counter at 2 seconds. + expand_duration_samples_ = std::min(expand_duration_samples_, + rtc::dchecked_cast(fs_hz_ * 2)); + return 0; +} + +void Expand::SetParametersForNormalAfterExpand() { + current_lag_index_ = 0; + lag_index_direction_ = 0; + stop_muting_ = true; // Do not mute signal any more. + statistics_->LogDelayedPacketOutageEvent( + rtc::dchecked_cast(expand_duration_samples_) / (fs_hz_ / 1000)); +} + +void Expand::SetParametersForMergeAfterExpand() { + current_lag_index_ = -1; /* out of the 3 possible ones */ + lag_index_direction_ = 1; /* make sure we get the "optimal" lag */ + stop_muting_ = true; +} + +bool Expand::Muted() const { + if (first_expand_ || stop_muting_) + return false; + RTC_DCHECK(channel_parameters_); + for (size_t ch = 0; ch < num_channels_; ++ch) { + if (channel_parameters_[ch].mute_factor != 0) + return false; + } + return true; +} + +size_t Expand::overlap_length() const { + return overlap_length_; +} + +void Expand::InitializeForAnExpandPeriod() { + lag_index_direction_ = 1; + current_lag_index_ = -1; + stop_muting_ = false; + random_vector_->set_seed_increment(1); + consecutive_expands_ = 0; + for (size_t ix = 0; ix < num_channels_; ++ix) { + channel_parameters_[ix].current_voice_mix_factor = 16384; // 1.0 in Q14. + channel_parameters_[ix].mute_factor = 16384; // 1.0 in Q14. + // Start with 0 gain for background noise. + background_noise_->SetMuteFactor(ix, 0); + } +} + +bool Expand::TooManyExpands() { + return consecutive_expands_ >= kMaxConsecutiveExpands; +} + +void Expand::AnalyzeSignal(int16_t* random_vector) { + int32_t auto_correlation[kUnvoicedLpcOrder + 1]; + int16_t reflection_coeff[kUnvoicedLpcOrder]; + int16_t correlation_vector[kMaxSampleRate / 8000 * 102]; + size_t best_correlation_index[kNumCorrelationCandidates]; + int16_t best_correlation[kNumCorrelationCandidates]; + size_t best_distortion_index[kNumCorrelationCandidates]; + int16_t best_distortion[kNumCorrelationCandidates]; + int32_t correlation_vector2[(99 * kMaxSampleRate / 8000) + 1]; + int32_t best_distortion_w32[kNumCorrelationCandidates]; + static const size_t kNoiseLpcOrder = BackgroundNoise::kMaxLpcOrder; + int16_t unvoiced_array_memory[kNoiseLpcOrder + kMaxSampleRate / 8000 * 125]; + int16_t* unvoiced_vector = unvoiced_array_memory + kUnvoicedLpcOrder; + + int fs_mult = fs_hz_ / 8000; + + // Pre-calculate common multiplications with fs_mult. + size_t fs_mult_4 = static_cast(fs_mult * 4); + size_t fs_mult_20 = static_cast(fs_mult * 20); + size_t fs_mult_120 = static_cast(fs_mult * 120); + size_t fs_mult_dist_len = fs_mult * kDistortionLength; + size_t fs_mult_lpc_analysis_len = fs_mult * kLpcAnalysisLength; + + const size_t signal_length = static_cast(256 * fs_mult); + + const size_t audio_history_position = sync_buffer_->Size() - signal_length; + std::unique_ptr audio_history(new int16_t[signal_length]); + (*sync_buffer_)[0].CopyTo(signal_length, audio_history_position, + audio_history.get()); + + // Initialize. + InitializeForAnExpandPeriod(); + + // Calculate correlation in downsampled domain (4 kHz sample rate). + size_t correlation_length = 51; // TODO(hlundin): Legacy bit-exactness. + // If it is decided to break bit-exactness |correlation_length| should be + // initialized to the return value of Correlation(). + Correlation(audio_history.get(), signal_length, correlation_vector); + + // Find peaks in correlation vector. + DspHelper::PeakDetection(correlation_vector, correlation_length, + kNumCorrelationCandidates, fs_mult, + best_correlation_index, best_correlation); + + // Adjust peak locations; cross-correlation lags start at 2.5 ms + // (20 * fs_mult samples). + best_correlation_index[0] += fs_mult_20; + best_correlation_index[1] += fs_mult_20; + best_correlation_index[2] += fs_mult_20; + + // Calculate distortion around the |kNumCorrelationCandidates| best lags. + int distortion_scale = 0; + for (size_t i = 0; i < kNumCorrelationCandidates; i++) { + size_t min_index = std::max(fs_mult_20, + best_correlation_index[i] - fs_mult_4); + size_t max_index = std::min(fs_mult_120 - 1, + best_correlation_index[i] + fs_mult_4); + best_distortion_index[i] = DspHelper::MinDistortion( + &(audio_history[signal_length - fs_mult_dist_len]), min_index, + max_index, fs_mult_dist_len, &best_distortion_w32[i]); + distortion_scale = std::max(16 - WebRtcSpl_NormW32(best_distortion_w32[i]), + distortion_scale); + } + // Shift the distortion values to fit in 16 bits. + WebRtcSpl_VectorBitShiftW32ToW16(best_distortion, kNumCorrelationCandidates, + best_distortion_w32, distortion_scale); + + // Find the maximizing index |i| of the cost function + // f[i] = best_correlation[i] / best_distortion[i]. + int32_t best_ratio = std::numeric_limits::min(); + size_t best_index = std::numeric_limits::max(); + for (size_t i = 0; i < kNumCorrelationCandidates; ++i) { + int32_t ratio; + if (best_distortion[i] > 0) { + ratio = (best_correlation[i] * (1 << 16)) / best_distortion[i]; + } else if (best_correlation[i] == 0) { + ratio = 0; // No correlation set result to zero. + } else { + ratio = std::numeric_limits::max(); // Denominator is zero. + } + if (ratio > best_ratio) { + best_index = i; + best_ratio = ratio; + } + } + + size_t distortion_lag = best_distortion_index[best_index]; + size_t correlation_lag = best_correlation_index[best_index]; + max_lag_ = std::max(distortion_lag, correlation_lag); + + // Calculate the exact best correlation in the range between + // |correlation_lag| and |distortion_lag|. + correlation_length = + std::max(std::min(distortion_lag + 10, fs_mult_120), + static_cast(60 * fs_mult)); + + size_t start_index = std::min(distortion_lag, correlation_lag); + size_t correlation_lags = static_cast( + WEBRTC_SPL_ABS_W16((distortion_lag-correlation_lag)) + 1); + assert(correlation_lags <= static_cast(99 * fs_mult + 1)); + + for (size_t channel_ix = 0; channel_ix < num_channels_; ++channel_ix) { + ChannelParameters& parameters = channel_parameters_[channel_ix]; + // Calculate suitable scaling. + int16_t signal_max = WebRtcSpl_MaxAbsValueW16( + &audio_history[signal_length - correlation_length - start_index + - correlation_lags], + correlation_length + start_index + correlation_lags - 1); + int correlation_scale = (31 - WebRtcSpl_NormW32(signal_max * signal_max)) + + (31 - WebRtcSpl_NormW32(static_cast(correlation_length))) - 31; + correlation_scale = std::max(0, correlation_scale); + + // Calculate the correlation, store in |correlation_vector2|. + WebRtcSpl_CrossCorrelation( + correlation_vector2, + &(audio_history[signal_length - correlation_length]), + &(audio_history[signal_length - correlation_length - start_index]), + correlation_length, correlation_lags, correlation_scale, -1); + + // Find maximizing index. + best_index = WebRtcSpl_MaxIndexW32(correlation_vector2, correlation_lags); + int32_t max_correlation = correlation_vector2[best_index]; + // Compensate index with start offset. + best_index = best_index + start_index; + + // Calculate energies. + int32_t energy1 = WebRtcSpl_DotProductWithScale( + &(audio_history[signal_length - correlation_length]), + &(audio_history[signal_length - correlation_length]), + correlation_length, correlation_scale); + int32_t energy2 = WebRtcSpl_DotProductWithScale( + &(audio_history[signal_length - correlation_length - best_index]), + &(audio_history[signal_length - correlation_length - best_index]), + correlation_length, correlation_scale); + + // Calculate the correlation coefficient between the two portions of the + // signal. + int32_t corr_coefficient; + if ((energy1 > 0) && (energy2 > 0)) { + int energy1_scale = std::max(16 - WebRtcSpl_NormW32(energy1), 0); + int energy2_scale = std::max(16 - WebRtcSpl_NormW32(energy2), 0); + // Make sure total scaling is even (to simplify scale factor after sqrt). + if ((energy1_scale + energy2_scale) & 1) { + // If sum is odd, add 1 to make it even. + energy1_scale += 1; + } + int32_t scaled_energy1 = energy1 >> energy1_scale; + int32_t scaled_energy2 = energy2 >> energy2_scale; + int16_t sqrt_energy_product = static_cast( + WebRtcSpl_SqrtFloor(scaled_energy1 * scaled_energy2)); + // Calculate max_correlation / sqrt(energy1 * energy2) in Q14. + int cc_shift = 14 - (energy1_scale + energy2_scale) / 2; + max_correlation = WEBRTC_SPL_SHIFT_W32(max_correlation, cc_shift); + corr_coefficient = WebRtcSpl_DivW32W16(max_correlation, + sqrt_energy_product); + // Cap at 1.0 in Q14. + corr_coefficient = std::min(16384, corr_coefficient); + } else { + corr_coefficient = 0; + } + + // Extract the two vectors expand_vector0 and expand_vector1 from + // |audio_history|. + size_t expansion_length = max_lag_ + overlap_length_; + const int16_t* vector1 = &(audio_history[signal_length - expansion_length]); + const int16_t* vector2 = vector1 - distortion_lag; + // Normalize the second vector to the same energy as the first. + energy1 = WebRtcSpl_DotProductWithScale(vector1, vector1, expansion_length, + correlation_scale); + energy2 = WebRtcSpl_DotProductWithScale(vector2, vector2, expansion_length, + correlation_scale); + // Confirm that amplitude ratio sqrt(energy1 / energy2) is within 0.5 - 2.0, + // i.e., energy1 / energy2 is within 0.25 - 4. + int16_t amplitude_ratio; + if ((energy1 / 4 < energy2) && (energy1 > energy2 / 4)) { + // Energy constraint fulfilled. Use both vectors and scale them + // accordingly. + int32_t scaled_energy2 = std::max(16 - WebRtcSpl_NormW32(energy2), 0); + int32_t scaled_energy1 = scaled_energy2 - 13; + // Calculate scaled_energy1 / scaled_energy2 in Q13. + int32_t energy_ratio = WebRtcSpl_DivW32W16( + WEBRTC_SPL_SHIFT_W32(energy1, -scaled_energy1), + static_cast(energy2 >> scaled_energy2)); + // Calculate sqrt ratio in Q13 (sqrt of en1/en2 in Q26). + amplitude_ratio = + static_cast(WebRtcSpl_SqrtFloor(energy_ratio << 13)); + // Copy the two vectors and give them the same energy. + parameters.expand_vector0.Clear(); + parameters.expand_vector0.PushBack(vector1, expansion_length); + parameters.expand_vector1.Clear(); + if (parameters.expand_vector1.Size() < expansion_length) { + parameters.expand_vector1.Extend( + expansion_length - parameters.expand_vector1.Size()); + } + std::unique_ptr temp_1(new int16_t[expansion_length]); + WebRtcSpl_AffineTransformVector(temp_1.get(), + const_cast(vector2), + amplitude_ratio, + 4096, + 13, + expansion_length); + parameters.expand_vector1.OverwriteAt(temp_1.get(), expansion_length, 0); + } else { + // Energy change constraint not fulfilled. Only use last vector. + parameters.expand_vector0.Clear(); + parameters.expand_vector0.PushBack(vector1, expansion_length); + // Copy from expand_vector0 to expand_vector1. + parameters.expand_vector0.CopyTo(¶meters.expand_vector1); + // Set the energy_ratio since it is used by muting slope. + if ((energy1 / 4 < energy2) || (energy2 == 0)) { + amplitude_ratio = 4096; // 0.5 in Q13. + } else { + amplitude_ratio = 16384; // 2.0 in Q13. + } + } + + // Set the 3 lag values. + if (distortion_lag == correlation_lag) { + expand_lags_[0] = distortion_lag; + expand_lags_[1] = distortion_lag; + expand_lags_[2] = distortion_lag; + } else { + // |distortion_lag| and |correlation_lag| are not equal; use different + // combinations of the two. + // First lag is |distortion_lag| only. + expand_lags_[0] = distortion_lag; + // Second lag is the average of the two. + expand_lags_[1] = (distortion_lag + correlation_lag) / 2; + // Third lag is the average again, but rounding towards |correlation_lag|. + if (distortion_lag > correlation_lag) { + expand_lags_[2] = (distortion_lag + correlation_lag - 1) / 2; + } else { + expand_lags_[2] = (distortion_lag + correlation_lag + 1) / 2; + } + } + + // Calculate the LPC and the gain of the filters. + + // Calculate kUnvoicedLpcOrder + 1 lags of the auto-correlation function. + size_t temp_index = signal_length - fs_mult_lpc_analysis_len - + kUnvoicedLpcOrder; + // Copy signal to temporary vector to be able to pad with leading zeros. + int16_t* temp_signal = new int16_t[fs_mult_lpc_analysis_len + + kUnvoicedLpcOrder]; + memset(temp_signal, 0, + sizeof(int16_t) * (fs_mult_lpc_analysis_len + kUnvoicedLpcOrder)); + memcpy(&temp_signal[kUnvoicedLpcOrder], + &audio_history[temp_index + kUnvoicedLpcOrder], + sizeof(int16_t) * fs_mult_lpc_analysis_len); + CrossCorrelationWithAutoShift( + &temp_signal[kUnvoicedLpcOrder], &temp_signal[kUnvoicedLpcOrder], + fs_mult_lpc_analysis_len, kUnvoicedLpcOrder + 1, -1, auto_correlation); + delete [] temp_signal; + + // Verify that variance is positive. + if (auto_correlation[0] > 0) { + // Estimate AR filter parameters using Levinson-Durbin algorithm; + // kUnvoicedLpcOrder + 1 filter coefficients. + int16_t stability = WebRtcSpl_LevinsonDurbin(auto_correlation, + parameters.ar_filter, + reflection_coeff, + kUnvoicedLpcOrder); + + // Keep filter parameters only if filter is stable. + if (stability != 1) { + // Set first coefficient to 4096 (1.0 in Q12). + parameters.ar_filter[0] = 4096; + // Set remaining |kUnvoicedLpcOrder| coefficients to zero. + WebRtcSpl_MemSetW16(parameters.ar_filter + 1, 0, kUnvoicedLpcOrder); + } + } + + if (channel_ix == 0) { + // Extract a noise segment. + size_t noise_length; + if (distortion_lag < 40) { + noise_length = 2 * distortion_lag + 30; + } else { + noise_length = distortion_lag + 30; + } + if (noise_length <= RandomVector::kRandomTableSize) { + memcpy(random_vector, RandomVector::kRandomTable, + sizeof(int16_t) * noise_length); + } else { + // Only applies to SWB where length could be larger than + // |kRandomTableSize|. + memcpy(random_vector, RandomVector::kRandomTable, + sizeof(int16_t) * RandomVector::kRandomTableSize); + assert(noise_length <= kMaxSampleRate / 8000 * 120 + 30); + random_vector_->IncreaseSeedIncrement(2); + random_vector_->Generate( + noise_length - RandomVector::kRandomTableSize, + &random_vector[RandomVector::kRandomTableSize]); + } + } + + // Set up state vector and calculate scale factor for unvoiced filtering. + memcpy(parameters.ar_filter_state, + &(audio_history[signal_length - kUnvoicedLpcOrder]), + sizeof(int16_t) * kUnvoicedLpcOrder); + memcpy(unvoiced_vector - kUnvoicedLpcOrder, + &(audio_history[signal_length - 128 - kUnvoicedLpcOrder]), + sizeof(int16_t) * kUnvoicedLpcOrder); + WebRtcSpl_FilterMAFastQ12(&audio_history[signal_length - 128], + unvoiced_vector, + parameters.ar_filter, + kUnvoicedLpcOrder + 1, + 128); + const int unvoiced_max_abs = [&] { + const int16_t max_abs = WebRtcSpl_MaxAbsValueW16(unvoiced_vector, 128); + // Since WebRtcSpl_MaxAbsValueW16 returns 2^15 - 1 when the input contains + // -2^15, we have to conservatively bump the return value by 1 + // if it is 2^15 - 1. + return max_abs == WEBRTC_SPL_WORD16_MAX ? max_abs + 1 : max_abs; + }(); + // Pick the smallest n such that 2^n > unvoiced_max_abs; then the maximum + // value of the dot product is less than 2^7 * 2^(2*n) = 2^(2*n + 7), so to + // prevent overflows we want 2n + 7 <= 31, which means we should shift by + // 2n + 7 - 31 bits, if this value is greater than zero. + int unvoiced_prescale = + std::max(0, 2 * WebRtcSpl_GetSizeInBits(unvoiced_max_abs) - 24); + + int32_t unvoiced_energy = WebRtcSpl_DotProductWithScale(unvoiced_vector, + unvoiced_vector, + 128, + unvoiced_prescale); + + // Normalize |unvoiced_energy| to 28 or 29 bits to preserve sqrt() accuracy. + int16_t unvoiced_scale = WebRtcSpl_NormW32(unvoiced_energy) - 3; + // Make sure we do an odd number of shifts since we already have 7 shifts + // from dividing with 128 earlier. This will make the total scale factor + // even, which is suitable for the sqrt. + unvoiced_scale += ((unvoiced_scale & 0x1) ^ 0x1); + unvoiced_energy = WEBRTC_SPL_SHIFT_W32(unvoiced_energy, unvoiced_scale); + int16_t unvoiced_gain = + static_cast(WebRtcSpl_SqrtFloor(unvoiced_energy)); + parameters.ar_gain_scale = 13 + + (unvoiced_scale + 7 - unvoiced_prescale) / 2; + parameters.ar_gain = unvoiced_gain; + + // Calculate voice_mix_factor from corr_coefficient. + // Let x = corr_coefficient. Then, we compute: + // if (x > 0.48) + // voice_mix_factor = (-5179 + 19931x - 16422x^2 + 5776x^3) / 4096; + // else + // voice_mix_factor = 0; + if (corr_coefficient > 7875) { + int16_t x1, x2, x3; + // |corr_coefficient| is in Q14. + x1 = static_cast(corr_coefficient); + x2 = (x1 * x1) >> 14; // Shift 14 to keep result in Q14. + x3 = (x1 * x2) >> 14; + static const int kCoefficients[4] = { -5179, 19931, -16422, 5776 }; + int32_t temp_sum = kCoefficients[0] * 16384; + temp_sum += kCoefficients[1] * x1; + temp_sum += kCoefficients[2] * x2; + temp_sum += kCoefficients[3] * x3; + parameters.voice_mix_factor = + static_cast(std::min(temp_sum / 4096, 16384)); + parameters.voice_mix_factor = std::max(parameters.voice_mix_factor, + static_cast(0)); + } else { + parameters.voice_mix_factor = 0; + } + + // Calculate muting slope. Reuse value from earlier scaling of + // |expand_vector0| and |expand_vector1|. + int16_t slope = amplitude_ratio; + if (slope > 12288) { + // slope > 1.5. + // Calculate (1 - (1 / slope)) / distortion_lag = + // (slope - 1) / (distortion_lag * slope). + // |slope| is in Q13, so 1 corresponds to 8192. Shift up to Q25 before + // the division. + // Shift the denominator from Q13 to Q5 before the division. The result of + // the division will then be in Q20. + int temp_ratio = WebRtcSpl_DivW32W16( + (slope - 8192) << 12, + static_cast((distortion_lag * slope) >> 8)); + if (slope > 14746) { + // slope > 1.8. + // Divide by 2, with proper rounding. + parameters.mute_slope = (temp_ratio + 1) / 2; + } else { + // Divide by 8, with proper rounding. + parameters.mute_slope = (temp_ratio + 4) / 8; + } + parameters.onset = true; + } else { + // Calculate (1 - slope) / distortion_lag. + // Shift |slope| by 7 to Q20 before the division. The result is in Q20. + parameters.mute_slope = WebRtcSpl_DivW32W16( + (8192 - slope) * 128, static_cast(distortion_lag)); + if (parameters.voice_mix_factor <= 13107) { + // Make sure the mute factor decreases from 1.0 to 0.9 in no more than + // 6.25 ms. + // mute_slope >= 0.005 / fs_mult in Q20. + parameters.mute_slope = std::max(5243 / fs_mult, parameters.mute_slope); + } else if (slope > 8028) { + parameters.mute_slope = 0; + } + parameters.onset = false; + } + } +} + +Expand::ChannelParameters::ChannelParameters() + : mute_factor(16384), + ar_gain(0), + ar_gain_scale(0), + voice_mix_factor(0), + current_voice_mix_factor(0), + onset(false), + mute_slope(0) { + memset(ar_filter, 0, sizeof(ar_filter)); + memset(ar_filter_state, 0, sizeof(ar_filter_state)); +} + +void Expand::Correlation(const int16_t* input, + size_t input_length, + int16_t* output) const { + // Set parameters depending on sample rate. + const int16_t* filter_coefficients; + size_t num_coefficients; + int16_t downsampling_factor; + if (fs_hz_ == 8000) { + num_coefficients = 3; + downsampling_factor = 2; + filter_coefficients = DspHelper::kDownsample8kHzTbl; + } else if (fs_hz_ == 16000) { + num_coefficients = 5; + downsampling_factor = 4; + filter_coefficients = DspHelper::kDownsample16kHzTbl; + } else if (fs_hz_ == 32000) { + num_coefficients = 7; + downsampling_factor = 8; + filter_coefficients = DspHelper::kDownsample32kHzTbl; + } else { // fs_hz_ == 48000. + num_coefficients = 7; + downsampling_factor = 12; + filter_coefficients = DspHelper::kDownsample48kHzTbl; + } + + // Correlate from lag 10 to lag 60 in downsampled domain. + // (Corresponds to 20-120 for narrow-band, 40-240 for wide-band, and so on.) + static const size_t kCorrelationStartLag = 10; + static const size_t kNumCorrelationLags = 54; + static const size_t kCorrelationLength = 60; + // Downsample to 4 kHz sample rate. + static const size_t kDownsampledLength = kCorrelationStartLag + + kNumCorrelationLags + kCorrelationLength; + int16_t downsampled_input[kDownsampledLength]; + static const size_t kFilterDelay = 0; + WebRtcSpl_DownsampleFast( + input + input_length - kDownsampledLength * downsampling_factor, + kDownsampledLength * downsampling_factor, downsampled_input, + kDownsampledLength, filter_coefficients, num_coefficients, + downsampling_factor, kFilterDelay); + + // Normalize |downsampled_input| to using all 16 bits. + int16_t max_value = WebRtcSpl_MaxAbsValueW16(downsampled_input, + kDownsampledLength); + int16_t norm_shift = 16 - WebRtcSpl_NormW32(max_value); + WebRtcSpl_VectorBitShiftW16(downsampled_input, kDownsampledLength, + downsampled_input, norm_shift); + + int32_t correlation[kNumCorrelationLags]; + CrossCorrelationWithAutoShift( + &downsampled_input[kDownsampledLength - kCorrelationLength], + &downsampled_input[kDownsampledLength - kCorrelationLength + - kCorrelationStartLag], + kCorrelationLength, kNumCorrelationLags, -1, correlation); + + // Normalize and move data from 32-bit to 16-bit vector. + int32_t max_correlation = WebRtcSpl_MaxAbsValueW32(correlation, + kNumCorrelationLags); + int16_t norm_shift2 = static_cast( + std::max(18 - WebRtcSpl_NormW32(max_correlation), 0)); + WebRtcSpl_VectorBitShiftW32ToW16(output, kNumCorrelationLags, correlation, + norm_shift2); +} + +void Expand::UpdateLagIndex() { + current_lag_index_ = current_lag_index_ + lag_index_direction_; + // Change direction if needed. + if (current_lag_index_ <= 0) { + lag_index_direction_ = 1; + } + if (current_lag_index_ >= kNumLags - 1) { + lag_index_direction_ = -1; + } +} + +Expand* ExpandFactory::Create(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) const { + return new Expand(background_noise, sync_buffer, random_vector, statistics, + fs, num_channels); +} + +// TODO(turajs): This can be moved to BackgroundNoise class. +void Expand::GenerateBackgroundNoise(int16_t* random_vector, + size_t channel, + int mute_slope, + bool too_many_expands, + size_t num_noise_samples, + int16_t* buffer) { + static const size_t kNoiseLpcOrder = BackgroundNoise::kMaxLpcOrder; + int16_t scaled_random_vector[kMaxSampleRate / 8000 * 125]; + assert(num_noise_samples <= (kMaxSampleRate / 8000 * 125)); + int16_t* noise_samples = &buffer[kNoiseLpcOrder]; + if (background_noise_->initialized()) { + // Use background noise parameters. + memcpy(noise_samples - kNoiseLpcOrder, + background_noise_->FilterState(channel), + sizeof(int16_t) * kNoiseLpcOrder); + + int dc_offset = 0; + if (background_noise_->ScaleShift(channel) > 1) { + dc_offset = 1 << (background_noise_->ScaleShift(channel) - 1); + } + + // Scale random vector to correct energy level. + WebRtcSpl_AffineTransformVector( + scaled_random_vector, random_vector, + background_noise_->Scale(channel), dc_offset, + background_noise_->ScaleShift(channel), + num_noise_samples); + + WebRtcSpl_FilterARFastQ12(scaled_random_vector, noise_samples, + background_noise_->Filter(channel), + kNoiseLpcOrder + 1, + num_noise_samples); + + background_noise_->SetFilterState( + channel, + &(noise_samples[num_noise_samples - kNoiseLpcOrder]), + kNoiseLpcOrder); + + // Unmute the background noise. + int16_t bgn_mute_factor = background_noise_->MuteFactor(channel); + NetEq::BackgroundNoiseMode bgn_mode = background_noise_->mode(); + if (bgn_mode == NetEq::kBgnFade && too_many_expands && + bgn_mute_factor > 0) { + // Fade BGN to zero. + // Calculate muting slope, approximately -2^18 / fs_hz. + int mute_slope; + if (fs_hz_ == 8000) { + mute_slope = -32; + } else if (fs_hz_ == 16000) { + mute_slope = -16; + } else if (fs_hz_ == 32000) { + mute_slope = -8; + } else { + mute_slope = -5; + } + // Use UnmuteSignal function with negative slope. + // |bgn_mute_factor| is in Q14. |mute_slope| is in Q20. + DspHelper::UnmuteSignal(noise_samples, + num_noise_samples, + &bgn_mute_factor, + mute_slope, + noise_samples); + } else if (bgn_mute_factor < 16384) { + // If mode is kBgnOn, or if kBgnFade has started fading, + // use regular |mute_slope|. + if (!stop_muting_ && bgn_mode != NetEq::kBgnOff && + !(bgn_mode == NetEq::kBgnFade && too_many_expands)) { + DspHelper::UnmuteSignal(noise_samples, + static_cast(num_noise_samples), + &bgn_mute_factor, + mute_slope, + noise_samples); + } else { + // kBgnOn and stop muting, or + // kBgnOff (mute factor is always 0), or + // kBgnFade has reached 0. + WebRtcSpl_AffineTransformVector(noise_samples, noise_samples, + bgn_mute_factor, 8192, 14, + num_noise_samples); + } + } + // Update mute_factor in BackgroundNoise class. + background_noise_->SetMuteFactor(channel, bgn_mute_factor); + } else { + // BGN parameters have not been initialized; use zero noise. + memset(noise_samples, 0, sizeof(int16_t) * num_noise_samples); + } +} + +void Expand::GenerateRandomVector(int16_t seed_increment, + size_t length, + int16_t* random_vector) { + // TODO(turajs): According to hlundin The loop should not be needed. Should be + // just as good to generate all of the vector in one call. + size_t samples_generated = 0; + const size_t kMaxRandSamples = RandomVector::kRandomTableSize; + while (samples_generated < length) { + size_t rand_length = std::min(length - samples_generated, kMaxRandSamples); + random_vector_->IncreaseSeedIncrement(seed_increment); + random_vector_->Generate(rand_length, &random_vector[samples_generated]); + samples_generated += rand_length; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.h new file mode 100644 index 0000000000..39249f13c2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ +#define MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ + +#include +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class BackgroundNoise; +class RandomVector; +class StatisticsCalculator; +class SyncBuffer; + +// This class handles extrapolation of audio data from the sync_buffer to +// produce packet-loss concealment. +// TODO(hlundin): Refactor this class to divide the long methods into shorter +// ones. +class Expand { + public: + Expand(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels); + + virtual ~Expand(); + + // Resets the object. + virtual void Reset(); + + // The main method to produce concealment data. The data is appended to the + // end of |output|. + virtual int Process(AudioMultiVector* output); + + // Prepare the object to do extra expansion during normal operation following + // a period of expands. + virtual void SetParametersForNormalAfterExpand(); + + // Prepare the object to do extra expansion during merge operation following + // a period of expands. + virtual void SetParametersForMergeAfterExpand(); + + // Returns the mute factor for |channel|. + int16_t MuteFactor(size_t channel) { + assert(channel < num_channels_); + return channel_parameters_[channel].mute_factor; + } + + // Returns true if expansion has been faded down to zero amplitude (for all + // channels); false otherwise. + bool Muted() const; + + // Accessors and mutators. + virtual size_t overlap_length() const; + size_t max_lag() const { return max_lag_; } + + protected: + static const int kMaxConsecutiveExpands = 200; + void GenerateRandomVector(int16_t seed_increment, + size_t length, + int16_t* random_vector); + + void GenerateBackgroundNoise(int16_t* random_vector, + size_t channel, + int mute_slope, + bool too_many_expands, + size_t num_noise_samples, + int16_t* buffer); + + // Initializes member variables at the beginning of an expand period. + void InitializeForAnExpandPeriod(); + + bool TooManyExpands(); + + // Analyzes the signal history in |sync_buffer_|, and set up all parameters + // necessary to produce concealment data. + void AnalyzeSignal(int16_t* random_vector); + + RandomVector* const random_vector_; + SyncBuffer* const sync_buffer_; + bool first_expand_; + const int fs_hz_; + const size_t num_channels_; + int consecutive_expands_; + + private: + static const size_t kUnvoicedLpcOrder = 6; + static const size_t kNumCorrelationCandidates = 3; + static const size_t kDistortionLength = 20; + static const size_t kLpcAnalysisLength = 160; + static const size_t kMaxSampleRate = 48000; + static const int kNumLags = 3; + + struct ChannelParameters { + ChannelParameters(); + int16_t mute_factor; + int16_t ar_filter[kUnvoicedLpcOrder + 1]; + int16_t ar_filter_state[kUnvoicedLpcOrder]; + int16_t ar_gain; + int16_t ar_gain_scale; + int16_t voice_mix_factor; /* Q14 */ + int16_t current_voice_mix_factor; /* Q14 */ + AudioVector expand_vector0; + AudioVector expand_vector1; + bool onset; + int mute_slope; /* Q20 */ + }; + + // Calculate the auto-correlation of |input|, with length |input_length| + // samples. The correlation is calculated from a downsampled version of + // |input|, and is written to |output|. + void Correlation(const int16_t* input, + size_t input_length, + int16_t* output) const; + + void UpdateLagIndex(); + + BackgroundNoise* const background_noise_; + StatisticsCalculator* const statistics_; + const size_t overlap_length_; + size_t max_lag_; + size_t expand_lags_[kNumLags]; + int lag_index_direction_; + int current_lag_index_; + bool stop_muting_; + size_t expand_duration_samples_; + std::unique_ptr channel_parameters_; + + RTC_DISALLOW_COPY_AND_ASSIGN(Expand); +}; + +struct ExpandFactory { + ExpandFactory() {} + virtual ~ExpandFactory() {} + + virtual Expand* Create(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) const; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand_unittest.cc new file mode 100644 index 0000000000..b4e6466e90 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand_unittest.cc @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Expand class. + +#include "modules/audio_coding/neteq/expand.h" + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +TEST(Expand, CreateAndDestroy) { + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); +} + +TEST(Expand, CreateUsingFactory) { + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + ExpandFactory expand_factory; + Expand* expand = expand_factory.Create(&bgn, &sync_buffer, &random_vector, + &statistics, fs, channels); + EXPECT_TRUE(expand != NULL); + delete expand; +} + +namespace { +class FakeStatisticsCalculator : public StatisticsCalculator { + public: + void LogDelayedPacketOutageEvent(int outage_duration_ms) override { + last_outage_duration_ms_ = outage_duration_ms; + } + + int last_outage_duration_ms() const { return last_outage_duration_ms_; } + + private: + int last_outage_duration_ms_ = 0; +}; + +// This is the same size that is given to the SyncBuffer object in NetEq. +const size_t kNetEqSyncBufferLengthMs = 720; +} // namespace + +class ExpandTest : public ::testing::Test { + protected: + ExpandTest() + : input_file_(test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000), + test_sample_rate_hz_(32000), + num_channels_(1), + background_noise_(num_channels_), + sync_buffer_(num_channels_, + kNetEqSyncBufferLengthMs * test_sample_rate_hz_ / 1000), + expand_(&background_noise_, + &sync_buffer_, + &random_vector_, + &statistics_, + test_sample_rate_hz_, + num_channels_) { + WebRtcSpl_Init(); + input_file_.set_output_rate_hz(test_sample_rate_hz_); + } + + void SetUp() override { + // Fast-forward the input file until there is speech (about 1.1 second into + // the file). + const int speech_start_samples = + static_cast(test_sample_rate_hz_ * 1.1f); + ASSERT_TRUE(input_file_.Seek(speech_start_samples)); + + // Pre-load the sync buffer with speech data. + std::unique_ptr temp(new int16_t[sync_buffer_.Size()]); + ASSERT_TRUE(input_file_.Read(sync_buffer_.Size(), temp.get())); + sync_buffer_.Channel(0).OverwriteAt(temp.get(), sync_buffer_.Size(), 0); + ASSERT_EQ(1u, num_channels_) << "Fix: Must populate all channels."; + } + + test::ResampleInputAudioFile input_file_; + int test_sample_rate_hz_; + size_t num_channels_; + BackgroundNoise background_noise_; + SyncBuffer sync_buffer_; + RandomVector random_vector_; + FakeStatisticsCalculator statistics_; + Expand expand_; +}; + +// This test calls the expand object to produce concealment data a few times, +// and then ends by calling SetParametersForNormalAfterExpand. This simulates +// the situation where the packet next up for decoding was just delayed, not +// lost. +TEST_F(ExpandTest, DelayedPacketOutage) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert |sum_output_len_samples| to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples / + (test_sample_rate_hz_ / 1000)), + statistics_.last_outage_duration_ms()); +} + +// This test is similar to DelayedPacketOutage, but ends by calling +// SetParametersForMergeAfterExpand. This simulates the situation where the +// packet next up for decoding was actually lost (or at least a later packet +// arrived before it). +TEST_F(ExpandTest, LostPacketOutage) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForMergeAfterExpand(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); +} + +// This test is similar to the DelayedPacketOutage test above, but with the +// difference that Expand::Reset() is called after 5 calls to Expand::Process(). +// This should reset the statistics, and will in the end lead to an outage of +// 5 periods instead of 10. +TEST_F(ExpandTest, CheckOutageStatsAfterReset) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + if (i == 5) { + expand_.Reset(); + sum_output_len_samples = 0; + } + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert |sum_output_len_samples| to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples / + (test_sample_rate_hz_ / 1000)), + statistics_.last_outage_duration_ms()); +} + +namespace { +// Runs expand until Muted() returns true. Times out after 1000 calls. +void ExpandUntilMuted(size_t num_channels, Expand* expand) { + EXPECT_FALSE(expand->Muted()) << "Instance is muted from the start"; + AudioMultiVector output(num_channels); + int num_calls = 0; + while (!expand->Muted()) { + ASSERT_LT(num_calls++, 1000) << "Test timed out"; + EXPECT_EQ(0, expand->Process(&output)); + } +} +} // namespace + +// Verifies that Muted() returns true after a long expand period. Also verifies +// that Muted() is reset to false after calling Reset(), +// SetParametersForMergeAfterExpand() and SetParametersForNormalAfterExpand(). +TEST_F(ExpandTest, Muted) { + ExpandUntilMuted(num_channels_, &expand_); + expand_.Reset(); + EXPECT_FALSE(expand_.Muted()); // Should be back to unmuted. + + ExpandUntilMuted(num_channels_, &expand_); + expand_.SetParametersForMergeAfterExpand(); + EXPECT_FALSE(expand_.Muted()); // Should be back to unmuted. + + expand_.Reset(); // Must reset in order to start a new expand period. + ExpandUntilMuted(num_channels_, &expand_); + expand_.SetParametersForNormalAfterExpand(); + EXPECT_FALSE(expand_.Muted()); // Should be back to unmuted. +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/include/neteq.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/include/neteq.h new file mode 100644 index 0000000000..00ed7cdbfa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/include/neteq.h @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_ +#define MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_ + +#include // Provide access to size_t. + +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "api/optional.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/neteq_decoder_enum.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class AudioFrame; +class AudioDecoderFactory; + +struct NetEqNetworkStatistics { + uint16_t current_buffer_size_ms; // Current jitter buffer size in ms. + uint16_t preferred_buffer_size_ms; // Target buffer size in ms. + uint16_t jitter_peaks_found; // 1 if adding extra delay due to peaky + // jitter; 0 otherwise. + uint16_t packet_loss_rate; // Loss rate (network + late) in Q14. + uint16_t expand_rate; // Fraction (of original stream) of synthesized + // audio inserted through expansion (in Q14). + uint16_t speech_expand_rate; // Fraction (of original stream) of synthesized + // speech inserted through expansion (in Q14). + uint16_t preemptive_rate; // Fraction of data inserted through pre-emptive + // expansion (in Q14). + uint16_t accelerate_rate; // Fraction of data removed through acceleration + // (in Q14). + uint16_t secondary_decoded_rate; // Fraction of data coming from FEC/RED + // decoding (in Q14). + uint16_t secondary_discarded_rate; // Fraction of discarded FEC/RED data (in + // Q14). + int32_t clockdrift_ppm; // Average clock-drift in parts-per-million + // (positive or negative). + size_t added_zero_samples; // Number of zero samples added in "off" mode. + // Statistics for packet waiting times, i.e., the time between a packet + // arrives until it is decoded. + int mean_waiting_time_ms; + int median_waiting_time_ms; + int min_waiting_time_ms; + int max_waiting_time_ms; +}; + +// NetEq statistics that persist over the lifetime of the class. +// These metrics are never reset. +struct NetEqLifetimeStatistics { + // Stats below correspond to similarly-named fields in the WebRTC stats spec. + // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats + uint64_t total_samples_received = 0; + uint64_t concealed_samples = 0; + uint64_t concealment_events = 0; + uint64_t jitter_buffer_delay_ms = 0; +}; + +enum NetEqPlayoutMode { + kPlayoutOn, + kPlayoutOff, + kPlayoutFax, + kPlayoutStreaming +}; + +// This is the interface class for NetEq. +class NetEq { + public: + enum BackgroundNoiseMode { + kBgnOn, // Default behavior with eternal noise. + kBgnFade, // Noise fades to zero after some time. + kBgnOff // Background noise is always zero. + }; + + struct Config { + Config() + : sample_rate_hz(16000), + enable_post_decode_vad(false), + max_packets_in_buffer(50), + // |max_delay_ms| has the same effect as calling SetMaximumDelay(). + max_delay_ms(2000), + background_noise_mode(kBgnOff), + playout_mode(kPlayoutOn), + enable_fast_accelerate(false) {} + + std::string ToString() const; + + int sample_rate_hz; // Initial value. Will change with input data. + bool enable_post_decode_vad; + size_t max_packets_in_buffer; + int max_delay_ms; + BackgroundNoiseMode background_noise_mode; + NetEqPlayoutMode playout_mode; + bool enable_fast_accelerate; + bool enable_muted_state = false; + }; + + enum ReturnCodes { + kOK = 0, + kFail = -1, + kNotImplemented = -2 + }; + + // Creates a new NetEq object, with parameters set in |config|. The |config| + // object will only have to be valid for the duration of the call to this + // method. + static NetEq* Create( + const NetEq::Config& config, + const rtc::scoped_refptr& decoder_factory); + + virtual ~NetEq() {} + + // Inserts a new packet into NetEq. The |receive_timestamp| is an indication + // of the time when the packet was received, and should be measured with + // the same tick rate as the RTP timestamp of the current payload. + // Returns 0 on success, -1 on failure. + virtual int InsertPacket(const RTPHeader& rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) = 0; + + // Lets NetEq know that a packet arrived with an empty payload. This typically + // happens when empty packets are used for probing the network channel, and + // these packets use RTP sequence numbers from the same series as the actual + // audio packets. + virtual void InsertEmptyPacket(const RTPHeader& rtp_header) = 0; + + // Instructs NetEq to deliver 10 ms of audio data. The data is written to + // |audio_frame|. All data in |audio_frame| is wiped; |data_|, |speech_type_|, + // |num_channels_|, |sample_rate_hz_|, |samples_per_channel_|, and + // |vad_activity_| are updated upon success. If an error is returned, some + // fields may not have been updated, or may contain inconsistent values. + // If muted state is enabled (through Config::enable_muted_state), |muted| + // may be set to true after a prolonged expand period. When this happens, the + // |data_| in |audio_frame| is not written, but should be interpreted as being + // all zeros. + // Returns kOK on success, or kFail in case of an error. + virtual int GetAudio(AudioFrame* audio_frame, bool* muted) = 0; + + // Replaces the current set of decoders with the given one. + virtual void SetCodecs(const std::map& codecs) = 0; + + // Associates |rtp_payload_type| with |codec| and |codec_name|, and stores the + // information in the codec database. Returns 0 on success, -1 on failure. + // The name is only used to provide information back to the caller about the + // decoders. Hence, the name is arbitrary, and may be empty. + virtual int RegisterPayloadType(NetEqDecoder codec, + const std::string& codec_name, + uint8_t rtp_payload_type) = 0; + + // Provides an externally created decoder object |decoder| to insert in the + // decoder database. The decoder implements a decoder of type |codec| and + // associates it with |rtp_payload_type| and |codec_name|. Returns kOK on + // success, kFail on failure. The name is only used to provide information + // back to the caller about the decoders. Hence, the name is arbitrary, and + // may be empty. + virtual int RegisterExternalDecoder(AudioDecoder* decoder, + NetEqDecoder codec, + const std::string& codec_name, + uint8_t rtp_payload_type) = 0; + + // Associates |rtp_payload_type| with the given codec, which NetEq will + // instantiate when it needs it. Returns true iff successful. + virtual bool RegisterPayloadType(int rtp_payload_type, + const SdpAudioFormat& audio_format) = 0; + + // Removes |rtp_payload_type| from the codec database. Returns 0 on success, + // -1 on failure. Removing a payload type that is not registered is ok and + // will not result in an error. + virtual int RemovePayloadType(uint8_t rtp_payload_type) = 0; + + // Removes all payload types from the codec database. + virtual void RemoveAllPayloadTypes() = 0; + + // Sets a minimum delay in millisecond for packet buffer. The minimum is + // maintained unless a higher latency is dictated by channel condition. + // Returns true if the minimum is successfully applied, otherwise false is + // returned. + virtual bool SetMinimumDelay(int delay_ms) = 0; + + // Sets a maximum delay in milliseconds for packet buffer. The latency will + // not exceed the given value, even required delay (given the channel + // conditions) is higher. Calling this method has the same effect as setting + // the |max_delay_ms| value in the NetEq::Config struct. + virtual bool SetMaximumDelay(int delay_ms) = 0; + + // The smallest latency required. This is computed bases on inter-arrival + // time and internal NetEq logic. Note that in computing this latency none of + // the user defined limits (applied by calling setMinimumDelay() and/or + // SetMaximumDelay()) are applied. + virtual int LeastRequiredDelayMs() const = 0; + + // Not implemented. + virtual int SetTargetDelay() = 0; + + // Returns the current target delay in ms. This includes any extra delay + // requested through SetMinimumDelay. + virtual int TargetDelayMs() const = 0; + + // Returns the current total delay (packet buffer and sync buffer) in ms. + virtual int CurrentDelayMs() const = 0; + + // Returns the current total delay (packet buffer and sync buffer) in ms, + // with smoothing applied to even out short-time fluctuations due to jitter. + // The packet buffer part of the delay is not updated during DTX/CNG periods. + virtual int FilteredCurrentDelayMs() const = 0; + + // Sets the playout mode to |mode|. + // Deprecated. Set the mode in the Config struct passed to the constructor. + // TODO(henrik.lundin) Delete. + virtual void SetPlayoutMode(NetEqPlayoutMode mode) = 0; + + // Returns the current playout mode. + // Deprecated. + // TODO(henrik.lundin) Delete. + virtual NetEqPlayoutMode PlayoutMode() const = 0; + + // Writes the current network statistics to |stats|. The statistics are reset + // after the call. + virtual int NetworkStatistics(NetEqNetworkStatistics* stats) = 0; + + // Returns a copy of this class's lifetime statistics. These statistics are + // never reset. + virtual NetEqLifetimeStatistics GetLifetimeStatistics() const = 0; + + // Writes the current RTCP statistics to |stats|. The statistics are reset + // and a new report period is started with the call. + virtual void GetRtcpStatistics(RtcpStatistics* stats) = 0; + + // Same as RtcpStatistics(), but does not reset anything. + virtual void GetRtcpStatisticsNoReset(RtcpStatistics* stats) = 0; + + // Enables post-decode VAD. When enabled, GetAudio() will return + // kOutputVADPassive when the signal contains no speech. + virtual void EnableVad() = 0; + + // Disables post-decode VAD. + virtual void DisableVad() = 0; + + // Returns the RTP timestamp for the last sample delivered by GetAudio(). + // The return value will be empty if no valid timestamp is available. + virtual rtc::Optional GetPlayoutTimestamp() const = 0; + + // Returns the sample rate in Hz of the audio produced in the last GetAudio + // call. If GetAudio has not been called yet, the configured sample rate + // (Config::sample_rate_hz) is returned. + virtual int last_output_sample_rate_hz() const = 0; + + // Returns info about the decoder for the given payload type, or an empty + // value if we have no decoder for that payload type. + virtual rtc::Optional GetDecoder(int payload_type) const = 0; + + // Returns the decoder format for the given payload type. Returns empty if no + // such payload type was registered. + virtual rtc::Optional GetDecoderFormat( + int payload_type) const = 0; + + // Not implemented. + virtual int SetTargetNumberOfChannels() = 0; + + // Not implemented. + virtual int SetTargetSampleRate() = 0; + + // Flushes both the packet buffer and the sync buffer. + virtual void FlushBuffers() = 0; + + // Current usage of packet-buffer and it's limits. + virtual void PacketBufferStatistics(int* current_num_packets, + int* max_num_packets) const = 0; + + // Enables NACK and sets the maximum size of the NACK list, which should be + // positive and no larger than Nack::kNackListSizeLimit. If NACK is already + // enabled then the maximum NACK list size is modified accordingly. + virtual void EnableNack(size_t max_nack_list_size) = 0; + + virtual void DisableNack() = 0; + + // Returns a list of RTP sequence numbers corresponding to packets to be + // retransmitted, given an estimate of the round-trip time in milliseconds. + virtual std::vector GetNackList( + int64_t round_trip_time_ms) const = 0; + + // Returns a vector containing the timestamps of the packets that were decoded + // in the last GetAudio call. If no packets were decoded in the last call, the + // vector is empty. + // Mainly intended for testing. + virtual std::vector LastDecodedTimestamps() const = 0; + + // Returns the length of the audio yet to play in the sync buffer. + // Mainly intended for testing. + virtual int SyncBufferSizeMs() const = 0; + + protected: + NetEq() {} + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(NetEq); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.cc new file mode 100644 index 0000000000..4a148ad164 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.cc @@ -0,0 +1,387 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/merge.h" + +#include +#include // memmove, memcpy, memset, size_t + +#include // min, max +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +Merge::Merge(int fs_hz, + size_t num_channels, + Expand* expand, + SyncBuffer* sync_buffer) + : fs_hz_(fs_hz), + num_channels_(num_channels), + fs_mult_(fs_hz_ / 8000), + timestamps_per_call_(static_cast(fs_hz_ / 100)), + expand_(expand), + sync_buffer_(sync_buffer), + expanded_(num_channels_) { + assert(num_channels_ > 0); +} + +Merge::~Merge() = default; + +size_t Merge::Process(int16_t* input, size_t input_length, + int16_t* external_mute_factor_array, + AudioMultiVector* output) { + // TODO(hlundin): Change to an enumerator and skip assert. + assert(fs_hz_ == 8000 || fs_hz_ == 16000 || fs_hz_ == 32000 || + fs_hz_ == 48000); + assert(fs_hz_ <= kMaxSampleRate); // Should not be possible. + + size_t old_length; + size_t expand_period; + // Get expansion data to overlap and mix with. + size_t expanded_length = GetExpandedSignal(&old_length, &expand_period); + + // Transfer input signal to an AudioMultiVector. + AudioMultiVector input_vector(num_channels_); + input_vector.PushBackInterleaved(input, input_length); + size_t input_length_per_channel = input_vector.Size(); + assert(input_length_per_channel == input_length / num_channels_); + + size_t best_correlation_index = 0; + size_t output_length = 0; + + std::unique_ptr input_channel( + new int16_t[input_length_per_channel]); + std::unique_ptr expanded_channel(new int16_t[expanded_length]); + for (size_t channel = 0; channel < num_channels_; ++channel) { + input_vector[channel].CopyTo( + input_length_per_channel, 0, input_channel.get()); + expanded_[channel].CopyTo(expanded_length, 0, expanded_channel.get()); + + int16_t new_mute_factor = SignalScaling( + input_channel.get(), input_length_per_channel, expanded_channel.get()); + + // Adjust muting factor (product of "main" muting factor and expand muting + // factor). + int16_t* external_mute_factor = &external_mute_factor_array[channel]; + *external_mute_factor = + (*external_mute_factor * expand_->MuteFactor(channel)) >> 14; + + // Update |external_mute_factor| if it is lower than |new_mute_factor|. + if (new_mute_factor > *external_mute_factor) { + *external_mute_factor = std::min(new_mute_factor, + static_cast(16384)); + } + + if (channel == 0) { + // Downsample, correlate, and find strongest correlation period for the + // master (i.e., first) channel only. + // Downsample to 4kHz sample rate. + Downsample(input_channel.get(), input_length_per_channel, + expanded_channel.get(), expanded_length); + + // Calculate the lag of the strongest correlation period. + best_correlation_index = CorrelateAndPeakSearch( + old_length, input_length_per_channel, expand_period); + } + + temp_data_.resize(input_length_per_channel + best_correlation_index); + int16_t* decoded_output = temp_data_.data() + best_correlation_index; + + // Mute the new decoded data if needed (and unmute it linearly). + // This is the overlapping part of expanded_signal. + size_t interpolation_length = std::min( + kMaxCorrelationLength * fs_mult_, + expanded_length - best_correlation_index); + interpolation_length = std::min(interpolation_length, + input_length_per_channel); + if (*external_mute_factor < 16384) { + // Set a suitable muting slope (Q20). 0.004 for NB, 0.002 for WB, + // and so on. + int increment = 4194 / fs_mult_; + *external_mute_factor = + static_cast(DspHelper::RampSignal(input_channel.get(), + interpolation_length, + *external_mute_factor, + increment)); + DspHelper::UnmuteSignal(&input_channel[interpolation_length], + input_length_per_channel - interpolation_length, + external_mute_factor, increment, + &decoded_output[interpolation_length]); + } else { + // No muting needed. + memmove( + &decoded_output[interpolation_length], + &input_channel[interpolation_length], + sizeof(int16_t) * (input_length_per_channel - interpolation_length)); + } + + // Do overlap and mix linearly. + int16_t increment = + static_cast(16384 / (interpolation_length + 1)); // In Q14. + int16_t mute_factor = 16384 - increment; + memmove(temp_data_.data(), expanded_channel.get(), + sizeof(int16_t) * best_correlation_index); + DspHelper::CrossFade(&expanded_channel[best_correlation_index], + input_channel.get(), interpolation_length, + &mute_factor, increment, decoded_output); + + output_length = best_correlation_index + input_length_per_channel; + if (channel == 0) { + assert(output->Empty()); // Output should be empty at this point. + output->AssertSize(output_length); + } else { + assert(output->Size() == output_length); + } + (*output)[channel].OverwriteAt(temp_data_.data(), output_length, 0); + } + + // Copy back the first part of the data to |sync_buffer_| and remove it from + // |output|. + sync_buffer_->ReplaceAtIndex(*output, old_length, sync_buffer_->next_index()); + output->PopFront(old_length); + + // Return new added length. |old_length| samples were borrowed from + // |sync_buffer_|. + RTC_DCHECK_GE(output_length, old_length); + return output_length - old_length; +} + +size_t Merge::GetExpandedSignal(size_t* old_length, size_t* expand_period) { + // Check how much data that is left since earlier. + *old_length = sync_buffer_->FutureLength(); + // Should never be less than overlap_length. + assert(*old_length >= expand_->overlap_length()); + // Generate data to merge the overlap with using expand. + expand_->SetParametersForMergeAfterExpand(); + + if (*old_length >= 210 * kMaxSampleRate / 8000) { + // TODO(hlundin): Write test case for this. + // The number of samples available in the sync buffer is more than what fits + // in expanded_signal. Keep the first 210 * kMaxSampleRate / 8000 samples, + // but shift them towards the end of the buffer. This is ok, since all of + // the buffer will be expand data anyway, so as long as the beginning is + // left untouched, we're fine. + size_t length_diff = *old_length - 210 * kMaxSampleRate / 8000; + sync_buffer_->InsertZerosAtIndex(length_diff, sync_buffer_->next_index()); + *old_length = 210 * kMaxSampleRate / 8000; + // This is the truncated length. + } + // This assert should always be true thanks to the if statement above. + assert(210 * kMaxSampleRate / 8000 >= *old_length); + + AudioMultiVector expanded_temp(num_channels_); + expand_->Process(&expanded_temp); + *expand_period = expanded_temp.Size(); // Samples per channel. + + expanded_.Clear(); + // Copy what is left since earlier into the expanded vector. + expanded_.PushBackFromIndex(*sync_buffer_, sync_buffer_->next_index()); + assert(expanded_.Size() == *old_length); + assert(expanded_temp.Size() > 0); + // Do "ugly" copy and paste from the expanded in order to generate more data + // to correlate (but not interpolate) with. + const size_t required_length = static_cast((120 + 80 + 2) * fs_mult_); + if (expanded_.Size() < required_length) { + while (expanded_.Size() < required_length) { + // Append one more pitch period each time. + expanded_.PushBack(expanded_temp); + } + // Trim the length to exactly |required_length|. + expanded_.PopBack(expanded_.Size() - required_length); + } + assert(expanded_.Size() >= required_length); + return required_length; +} + +int16_t Merge::SignalScaling(const int16_t* input, size_t input_length, + const int16_t* expanded_signal) const { + // Adjust muting factor if new vector is more or less of the BGN energy. + const auto mod_input_length = rtc::SafeMin( + 64 * rtc::dchecked_cast(fs_mult_), input_length); + + // Missing input, do no muting + if (mod_input_length == 0) { + return 16384; + } + + const int16_t expanded_max = + WebRtcSpl_MaxAbsValueW16(expanded_signal, mod_input_length); + int32_t factor = (expanded_max * expanded_max) / + (std::numeric_limits::max() / + static_cast(mod_input_length)); + const int expanded_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); + int32_t energy_expanded = WebRtcSpl_DotProductWithScale(expanded_signal, + expanded_signal, + mod_input_length, + expanded_shift); + + // Calculate energy of input signal. + const int16_t input_max = WebRtcSpl_MaxAbsValueW16(input, mod_input_length); + factor = (input_max * input_max) / (std::numeric_limits::max() / + static_cast(mod_input_length)); + const int input_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); + int32_t energy_input = WebRtcSpl_DotProductWithScale(input, input, + mod_input_length, + input_shift); + + // Align to the same Q-domain. + if (input_shift > expanded_shift) { + energy_expanded = energy_expanded >> (input_shift - expanded_shift); + } else { + energy_input = energy_input >> (expanded_shift - input_shift); + } + + // Calculate muting factor to use for new frame. + int16_t mute_factor; + if (energy_input > energy_expanded) { + // Normalize |energy_input| to 14 bits. + int16_t temp_shift = WebRtcSpl_NormW32(energy_input) - 17; + energy_input = WEBRTC_SPL_SHIFT_W32(energy_input, temp_shift); + // Put |energy_expanded| in a domain 14 higher, so that + // energy_expanded / energy_input is in Q14. + energy_expanded = WEBRTC_SPL_SHIFT_W32(energy_expanded, temp_shift + 14); + // Calculate sqrt(energy_expanded / energy_input) in Q14. + mute_factor = static_cast( + WebRtcSpl_SqrtFloor((energy_expanded / energy_input) << 14)); + } else { + // Set to 1 (in Q14) when |expanded| has higher energy than |input|. + mute_factor = 16384; + } + + return mute_factor; +} + +// TODO(hlundin): There are some parameter values in this method that seem +// strange. Compare with Expand::Correlation. +void Merge::Downsample(const int16_t* input, size_t input_length, + const int16_t* expanded_signal, size_t expanded_length) { + const int16_t* filter_coefficients; + size_t num_coefficients; + int decimation_factor = fs_hz_ / 4000; + static const size_t kCompensateDelay = 0; + size_t length_limit = static_cast(fs_hz_ / 100); // 10 ms in samples. + if (fs_hz_ == 8000) { + filter_coefficients = DspHelper::kDownsample8kHzTbl; + num_coefficients = 3; + } else if (fs_hz_ == 16000) { + filter_coefficients = DspHelper::kDownsample16kHzTbl; + num_coefficients = 5; + } else if (fs_hz_ == 32000) { + filter_coefficients = DspHelper::kDownsample32kHzTbl; + num_coefficients = 7; + } else { // fs_hz_ == 48000 + filter_coefficients = DspHelper::kDownsample48kHzTbl; + num_coefficients = 7; + } + size_t signal_offset = num_coefficients - 1; + WebRtcSpl_DownsampleFast(&expanded_signal[signal_offset], + expanded_length - signal_offset, + expanded_downsampled_, kExpandDownsampLength, + filter_coefficients, num_coefficients, + decimation_factor, kCompensateDelay); + if (input_length <= length_limit) { + // Not quite long enough, so we have to cheat a bit. + size_t temp_len = input_length - signal_offset; + // TODO(hlundin): Should |downsamp_temp_len| be corrected for round-off + // errors? I.e., (temp_len + decimation_factor - 1) / decimation_factor? + size_t downsamp_temp_len = temp_len / decimation_factor; + WebRtcSpl_DownsampleFast(&input[signal_offset], temp_len, + input_downsampled_, downsamp_temp_len, + filter_coefficients, num_coefficients, + decimation_factor, kCompensateDelay); + memset(&input_downsampled_[downsamp_temp_len], 0, + sizeof(int16_t) * (kInputDownsampLength - downsamp_temp_len)); + } else { + WebRtcSpl_DownsampleFast(&input[signal_offset], + input_length - signal_offset, input_downsampled_, + kInputDownsampLength, filter_coefficients, + num_coefficients, decimation_factor, + kCompensateDelay); + } +} + +size_t Merge::CorrelateAndPeakSearch(size_t start_position, size_t input_length, + size_t expand_period) const { + // Calculate correlation without any normalization. + const size_t max_corr_length = kMaxCorrelationLength; + size_t stop_position_downsamp = + std::min(max_corr_length, expand_->max_lag() / (fs_mult_ * 2) + 1); + + int32_t correlation[kMaxCorrelationLength]; + CrossCorrelationWithAutoShift(input_downsampled_, expanded_downsampled_, + kInputDownsampLength, stop_position_downsamp, 1, + correlation); + + // Normalize correlation to 14 bits and copy to a 16-bit array. + const size_t pad_length = expand_->overlap_length() - 1; + const size_t correlation_buffer_size = 2 * pad_length + kMaxCorrelationLength; + std::unique_ptr correlation16( + new int16_t[correlation_buffer_size]); + memset(correlation16.get(), 0, correlation_buffer_size * sizeof(int16_t)); + int16_t* correlation_ptr = &correlation16[pad_length]; + int32_t max_correlation = WebRtcSpl_MaxAbsValueW32(correlation, + stop_position_downsamp); + int norm_shift = std::max(0, 17 - WebRtcSpl_NormW32(max_correlation)); + WebRtcSpl_VectorBitShiftW32ToW16(correlation_ptr, stop_position_downsamp, + correlation, norm_shift); + + // Calculate allowed starting point for peak finding. + // The peak location bestIndex must fulfill two criteria: + // (1) w16_bestIndex + input_length < + // timestamps_per_call_ + expand_->overlap_length(); + // (2) w16_bestIndex + input_length < start_position. + size_t start_index = timestamps_per_call_ + expand_->overlap_length(); + start_index = std::max(start_position, start_index); + start_index = (input_length > start_index) ? 0 : (start_index - input_length); + // Downscale starting index to 4kHz domain. (fs_mult_ * 2 = fs_hz_ / 4000.) + size_t start_index_downsamp = start_index / (fs_mult_ * 2); + + // Calculate a modified |stop_position_downsamp| to account for the increased + // start index |start_index_downsamp| and the effective array length. + size_t modified_stop_pos = + std::min(stop_position_downsamp, + kMaxCorrelationLength + pad_length - start_index_downsamp); + size_t best_correlation_index; + int16_t best_correlation; + static const size_t kNumCorrelationCandidates = 1; + DspHelper::PeakDetection(&correlation_ptr[start_index_downsamp], + modified_stop_pos, kNumCorrelationCandidates, + fs_mult_, &best_correlation_index, + &best_correlation); + // Compensate for modified start index. + best_correlation_index += start_index; + + // Ensure that underrun does not occur for 10ms case => we have to get at + // least 10ms + overlap . (This should never happen thanks to the above + // modification of peak-finding starting point.) + while (((best_correlation_index + input_length) < + (timestamps_per_call_ + expand_->overlap_length())) || + ((best_correlation_index + input_length) < start_position)) { + assert(false); // Should never happen. + best_correlation_index += expand_period; // Jump one lag ahead. + } + return best_correlation_index; +} + +size_t Merge::RequiredFutureSamples() { + return fs_hz_ / 100 * num_channels_; // 10 ms. +} + + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.h new file mode 100644 index 0000000000..e027cd7963 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MERGE_H_ +#define MODULES_AUDIO_CODING_NETEQ_MERGE_H_ + +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class Expand; +class SyncBuffer; + +// This class handles the transition from expansion to normal operation. +// When a packet is not available for decoding when needed, the expand operation +// is called to generate extrapolation data. If the missing packet arrives, +// i.e., it was just delayed, it can be decoded and appended directly to the +// end of the expanded data (thanks to how the Expand class operates). However, +// if a later packet arrives instead, the loss is a fact, and the new data must +// be stitched together with the end of the expanded data. This stitching is +// what the Merge class does. +class Merge { + public: + Merge(int fs_hz, + size_t num_channels, + Expand* expand, + SyncBuffer* sync_buffer); + virtual ~Merge(); + + // The main method to produce the audio data. The decoded data is supplied in + // |input|, having |input_length| samples in total for all channels + // (interleaved). The result is written to |output|. The number of channels + // allocated in |output| defines the number of channels that will be used when + // de-interleaving |input|. The values in |external_mute_factor_array| (Q14) + // will be used to scale the audio, and is updated in the process. The array + // must have |num_channels_| elements. + virtual size_t Process(int16_t* input, size_t input_length, + int16_t* external_mute_factor_array, + AudioMultiVector* output); + + virtual size_t RequiredFutureSamples(); + + protected: + const int fs_hz_; + const size_t num_channels_; + + private: + static const int kMaxSampleRate = 48000; + static const size_t kExpandDownsampLength = 100; + static const size_t kInputDownsampLength = 40; + static const size_t kMaxCorrelationLength = 60; + + // Calls |expand_| to get more expansion data to merge with. The data is + // written to |expanded_signal_|. Returns the length of the expanded data, + // while |expand_period| will be the number of samples in one expansion period + // (typically one pitch period). The value of |old_length| will be the number + // of samples that were taken from the |sync_buffer_|. + size_t GetExpandedSignal(size_t* old_length, size_t* expand_period); + + // Analyzes |input| and |expanded_signal| and returns muting factor (Q14) to + // be used on the new data. + int16_t SignalScaling(const int16_t* input, size_t input_length, + const int16_t* expanded_signal) const; + + // Downsamples |input| (|input_length| samples) and |expanded_signal| to + // 4 kHz sample rate. The downsampled signals are written to + // |input_downsampled_| and |expanded_downsampled_|, respectively. + void Downsample(const int16_t* input, size_t input_length, + const int16_t* expanded_signal, size_t expanded_length); + + // Calculates cross-correlation between |input_downsampled_| and + // |expanded_downsampled_|, and finds the correlation maximum. The maximizing + // lag is returned. + size_t CorrelateAndPeakSearch(size_t start_position, size_t input_length, + size_t expand_period) const; + + const int fs_mult_; // fs_hz_ / 8000. + const size_t timestamps_per_call_; + Expand* expand_; + SyncBuffer* sync_buffer_; + int16_t expanded_downsampled_[kExpandDownsampLength]; + int16_t input_downsampled_[kInputDownsampLength]; + AudioMultiVector expanded_; + std::vector temp_data_; + + RTC_DISALLOW_COPY_AND_ASSIGN(Merge); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MERGE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge_unittest.cc new file mode 100644 index 0000000000..7ff3b8c7b0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge_unittest.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Merge class. + +#include "modules/audio_coding/neteq/merge.h" + +#include + +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(Merge, CreateAndDestroy) { + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); + Merge merge(fs, channels, &expand, &sync_buffer); +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h new file mode 100644 index 0000000000..f662fb6ec0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ + +#include "modules/audio_coding/neteq/buffer_level_filter.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockBufferLevelFilter : public BufferLevelFilter { + public: + virtual ~MockBufferLevelFilter() { Die(); } + MOCK_METHOD0(Die, + void()); + MOCK_METHOD0(Reset, + void()); + MOCK_METHOD3(Update, + void(size_t buffer_size_packets, int time_stretched_samples, + size_t packet_len_samples)); + MOCK_METHOD1(SetTargetBufferLevel, + void(int target_buffer_level)); + MOCK_CONST_METHOD0(filtered_current_level, + int()); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h new file mode 100644 index 0000000000..049b693210 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DECODER_DATABASE_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DECODER_DATABASE_H_ + +#include + +#include "modules/audio_coding/neteq/decoder_database.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockDecoderDatabase : public DecoderDatabase { + public: + explicit MockDecoderDatabase( + rtc::scoped_refptr factory = nullptr) + : DecoderDatabase(factory) {} + virtual ~MockDecoderDatabase() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_CONST_METHOD0(Empty, + bool()); + MOCK_CONST_METHOD0(Size, + int()); + MOCK_METHOD0(Reset, + void()); + MOCK_METHOD3(RegisterPayload, + int(uint8_t rtp_payload_type, NetEqDecoder codec_type, + const std::string& name)); + MOCK_METHOD2(RegisterPayload, + int(int rtp_payload_type, const SdpAudioFormat& audio_format)); + MOCK_METHOD4(InsertExternal, + int(uint8_t rtp_payload_type, + NetEqDecoder codec_type, + const std::string& codec_name, + AudioDecoder* decoder)); + MOCK_METHOD1(Remove, + int(uint8_t rtp_payload_type)); + MOCK_METHOD0(RemoveAll, void()); + MOCK_CONST_METHOD1(GetDecoderInfo, + const DecoderInfo*(uint8_t rtp_payload_type)); + MOCK_METHOD2(SetActiveDecoder, + int(uint8_t rtp_payload_type, bool* new_decoder)); + MOCK_CONST_METHOD0(GetActiveDecoder, + AudioDecoder*()); + MOCK_METHOD1(SetActiveCngDecoder, + int(uint8_t rtp_payload_type)); + MOCK_CONST_METHOD0(GetActiveCngDecoder, + ComfortNoiseDecoder*()); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DECODER_DATABASE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h new file mode 100644 index 0000000000..61f209d5c4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ + +#include "modules/audio_coding/neteq/delay_manager.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockDelayManager : public DelayManager { + public: + MockDelayManager(size_t max_packets_in_buffer, + DelayPeakDetector* peak_detector, + const TickTimer* tick_timer) + : DelayManager(max_packets_in_buffer, peak_detector, tick_timer) {} + virtual ~MockDelayManager() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_CONST_METHOD0(iat_vector, + const IATVector&()); + MOCK_METHOD3(Update, + int(uint16_t sequence_number, uint32_t timestamp, int sample_rate_hz)); + MOCK_METHOD1(CalculateTargetLevel, + int(int iat_packets)); + MOCK_METHOD1(SetPacketAudioLength, + int(int length_ms)); + MOCK_METHOD0(Reset, + void()); + MOCK_CONST_METHOD0(PeakFound, + bool()); + MOCK_METHOD1(UpdateCounters, + void(int elapsed_time_ms)); + MOCK_METHOD0(ResetPacketIatCount, + void()); + MOCK_CONST_METHOD2(BufferLimits, + void(int* lower_limit, int* higher_limit)); + MOCK_CONST_METHOD0(TargetLevel, + int()); + MOCK_METHOD0(RegisterEmptyPacket, void()); + MOCK_METHOD1(set_extra_delay_ms, + void(int16_t delay)); + MOCK_CONST_METHOD0(base_target_level, + int()); + MOCK_METHOD1(set_streaming_mode, + void(bool value)); + MOCK_CONST_METHOD0(last_pack_cng_or_dtmf, + int()); + MOCK_METHOD1(set_last_pack_cng_or_dtmf, + void(int value)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h new file mode 100644 index 0000000000..f6cdea04f1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_PEAK_DETECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_PEAK_DETECTOR_H_ + +#include "modules/audio_coding/neteq/delay_peak_detector.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockDelayPeakDetector : public DelayPeakDetector { + public: + MockDelayPeakDetector(const TickTimer* tick_timer) + : DelayPeakDetector(tick_timer) {} + virtual ~MockDelayPeakDetector() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD0(Reset, void()); + MOCK_METHOD1(SetPacketAudioLength, void(int length_ms)); + MOCK_METHOD0(peak_found, bool()); + MOCK_CONST_METHOD0(MaxPeakHeight, int()); + MOCK_CONST_METHOD0(MaxPeakPeriod, uint64_t()); + MOCK_METHOD2(Update, bool(int inter_arrival_time, int target_level)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_PEAK_DETECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h new file mode 100644 index 0000000000..153a4d716a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_BUFFER_H_ + +#include "modules/audio_coding/neteq/dtmf_buffer.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockDtmfBuffer : public DtmfBuffer { + public: + MockDtmfBuffer(int fs) : DtmfBuffer(fs) {} + virtual ~MockDtmfBuffer() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD0(Flush, + void()); + MOCK_METHOD1(InsertEvent, + int(const DtmfEvent& event)); + MOCK_METHOD2(GetEvent, + bool(uint32_t current_timestamp, DtmfEvent* event)); + MOCK_CONST_METHOD0(Length, + size_t()); + MOCK_CONST_METHOD0(Empty, + bool()); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h new file mode 100644 index 0000000000..2cb5980891 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_TONE_GENERATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_TONE_GENERATOR_H_ + +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockDtmfToneGenerator : public DtmfToneGenerator { + public: + virtual ~MockDtmfToneGenerator() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD3(Init, + int(int fs, int event, int attenuation)); + MOCK_METHOD0(Reset, + void()); + MOCK_METHOD2(Generate, + int(size_t num_samples, AudioMultiVector* output)); + MOCK_CONST_METHOD0(initialized, + bool()); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_TONE_GENERATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_expand.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_expand.h new file mode 100644 index 0000000000..05fdaec750 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_expand.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXPAND_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXPAND_H_ + +#include "modules/audio_coding/neteq/expand.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockExpand : public Expand { + public: + MockExpand(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) + : Expand(background_noise, + sync_buffer, + random_vector, + statistics, + fs, + num_channels) {} + virtual ~MockExpand() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD0(Reset, + void()); + MOCK_METHOD1(Process, + int(AudioMultiVector* output)); + MOCK_METHOD0(SetParametersForNormalAfterExpand, + void()); + MOCK_METHOD0(SetParametersForMergeAfterExpand, + void()); + MOCK_CONST_METHOD0(overlap_length, + size_t()); +}; + +} // namespace webrtc + +namespace webrtc { + +class MockExpandFactory : public ExpandFactory { + public: + MOCK_CONST_METHOD6(Create, + Expand*(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXPAND_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h new file mode 100644 index 0000000000..b3152405b5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXTERNAL_DECODER_PCM16B_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXTERNAL_DECODER_PCM16B_H_ + +#include "api/audio_codecs/audio_decoder.h" +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "rtc_base/constructormagic.h" +#include "test/gmock.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +using ::testing::_; +using ::testing::Invoke; + +// Implement an external version of the PCM16b decoder. +class ExternalPcm16B : public AudioDecoder { + public: + explicit ExternalPcm16B(int sample_rate_hz) + : sample_rate_hz_(sample_rate_hz) {} + void Reset() override {} + + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override { + EXPECT_EQ(sample_rate_hz_, sample_rate_hz); + size_t ret = WebRtcPcm16b_Decode(encoded, encoded_len, decoded); + *speech_type = ConvertSpeechType(1); + return static_cast(ret); + } + int SampleRateHz() const override { return sample_rate_hz_; } + size_t Channels() const override { return 1; } + + private: + const int sample_rate_hz_; + RTC_DISALLOW_COPY_AND_ASSIGN(ExternalPcm16B); +}; + +// Create a mock of ExternalPcm16B which delegates all calls to the real object. +// The reason is that we can then track that the correct calls are being made. +class MockExternalPcm16B : public AudioDecoder { + public: + explicit MockExternalPcm16B(int sample_rate_hz) : real_(sample_rate_hz) { + // By default, all calls are delegated to the real object. + ON_CALL(*this, DecodeInternal(_, _, _, _, _)) + .WillByDefault(Invoke(&real_, &ExternalPcm16B::DecodeInternal)); + ON_CALL(*this, HasDecodePlc()) + .WillByDefault(Invoke(&real_, &ExternalPcm16B::HasDecodePlc)); + ON_CALL(*this, DecodePlc(_, _)) + .WillByDefault(Invoke(&real_, &ExternalPcm16B::DecodePlc)); + ON_CALL(*this, Reset()) + .WillByDefault(Invoke(&real_, &ExternalPcm16B::Reset)); + ON_CALL(*this, IncomingPacket(_, _, _, _, _)) + .WillByDefault(Invoke(&real_, &ExternalPcm16B::IncomingPacket)); + ON_CALL(*this, ErrorCode()) + .WillByDefault(Invoke(&real_, &ExternalPcm16B::ErrorCode)); + } + virtual ~MockExternalPcm16B() { Die(); } + + MOCK_METHOD0(Die, void()); + MOCK_METHOD5(DecodeInternal, + int(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type)); + MOCK_CONST_METHOD0(HasDecodePlc, + bool()); + MOCK_METHOD2(DecodePlc, + size_t(size_t num_frames, int16_t* decoded)); + MOCK_METHOD0(Reset, void()); + MOCK_METHOD5(IncomingPacket, + int(const uint8_t* payload, size_t payload_len, + uint16_t rtp_sequence_number, uint32_t rtp_timestamp, + uint32_t arrival_timestamp)); + MOCK_METHOD0(ErrorCode, + int()); + + int SampleRateHz() const /* override */ { return real_.SampleRateHz(); } + size_t Channels() const /* override */ { return real_.Channels(); } + + private: + ExternalPcm16B real_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXTERNAL_DECODER_PCM16B_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h new file mode 100644 index 0000000000..ac7d9b7a25 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PACKET_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PACKET_BUFFER_H_ + +#include "modules/audio_coding/neteq/packet_buffer.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockPacketBuffer : public PacketBuffer { + public: + MockPacketBuffer(size_t max_number_of_packets, const TickTimer* tick_timer) + : PacketBuffer(max_number_of_packets, tick_timer) {} + virtual ~MockPacketBuffer() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD0(Flush, + void()); + MOCK_CONST_METHOD0(Empty, + bool()); + int InsertPacket(Packet&& packet, StatisticsCalculator* stats) { + return InsertPacketWrapped(&packet, stats); + } + // Since gtest does not properly support move-only types, InsertPacket is + // implemented as a wrapper. You'll have to implement InsertPacketWrapped + // instead and move from |*packet|. + MOCK_METHOD2(InsertPacketWrapped, + int(Packet* packet, StatisticsCalculator* stats)); + MOCK_METHOD5(InsertPacketList, + int(PacketList* packet_list, + const DecoderDatabase& decoder_database, + rtc::Optional* current_rtp_payload_type, + rtc::Optional* current_cng_rtp_payload_type, + StatisticsCalculator* stats)); + MOCK_CONST_METHOD1(NextTimestamp, + int(uint32_t* next_timestamp)); + MOCK_CONST_METHOD2(NextHigherTimestamp, + int(uint32_t timestamp, uint32_t* next_timestamp)); + MOCK_CONST_METHOD0(PeekNextPacket, + const Packet*()); + MOCK_METHOD0(GetNextPacket, + rtc::Optional()); + MOCK_METHOD1(DiscardNextPacket, int(StatisticsCalculator* stats)); + MOCK_METHOD3(DiscardOldPackets, + void(uint32_t timestamp_limit, + uint32_t horizon_samples, + StatisticsCalculator* stats)); + MOCK_METHOD2(DiscardAllOldPackets, + void(uint32_t timestamp_limit, StatisticsCalculator* stats)); + MOCK_CONST_METHOD0(NumPacketsInBuffer, + size_t()); + MOCK_METHOD1(IncrementWaitingTimes, + void(int)); + MOCK_CONST_METHOD0(current_memory_bytes, + int()); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PACKET_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h new file mode 100644 index 0000000000..27a2276664 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ + +#include "modules/audio_coding/neteq/red_payload_splitter.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockRedPayloadSplitter : public RedPayloadSplitter { + public: + MOCK_METHOD1(SplitRed, bool(PacketList* packet_list)); + MOCK_METHOD2(CheckRedPayloads, + int(PacketList* packet_list, + const DecoderDatabase& decoder_database)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h new file mode 100644 index 0000000000..85f26206c9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_STATISTICS_CALCULATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_STATISTICS_CALCULATOR_H_ + +#include "modules/audio_coding/neteq/statistics_calculator.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockStatisticsCalculator : public StatisticsCalculator { + public: + MOCK_METHOD1(PacketsDiscarded, void(size_t num_packets)); + MOCK_METHOD1(SecondaryPacketsDiscarded, void(size_t num_packets)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_STATISTICS_CALCULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.cc new file mode 100644 index 0000000000..d187883916 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.cc @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/nack_tracker.h" + +#include // For assert. + +#include // For std::max. + +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +const int kDefaultSampleRateKhz = 48; +const int kDefaultPacketSizeMs = 20; + +} // namespace + +NackTracker::NackTracker(int nack_threshold_packets) + : nack_threshold_packets_(nack_threshold_packets), + sequence_num_last_received_rtp_(0), + timestamp_last_received_rtp_(0), + any_rtp_received_(false), + sequence_num_last_decoded_rtp_(0), + timestamp_last_decoded_rtp_(0), + any_rtp_decoded_(false), + sample_rate_khz_(kDefaultSampleRateKhz), + samples_per_packet_(sample_rate_khz_ * kDefaultPacketSizeMs), + max_nack_list_size_(kNackListSizeLimit) {} + +NackTracker::~NackTracker() = default; + +NackTracker* NackTracker::Create(int nack_threshold_packets) { + return new NackTracker(nack_threshold_packets); +} + +void NackTracker::UpdateSampleRate(int sample_rate_hz) { + assert(sample_rate_hz > 0); + sample_rate_khz_ = sample_rate_hz / 1000; +} + +void NackTracker::UpdateLastReceivedPacket(uint16_t sequence_number, + uint32_t timestamp) { + // Just record the value of sequence number and timestamp if this is the + // first packet. + if (!any_rtp_received_) { + sequence_num_last_received_rtp_ = sequence_number; + timestamp_last_received_rtp_ = timestamp; + any_rtp_received_ = true; + // If no packet is decoded, to have a reasonable estimate of time-to-play + // use the given values. + if (!any_rtp_decoded_) { + sequence_num_last_decoded_rtp_ = sequence_number; + timestamp_last_decoded_rtp_ = timestamp; + } + return; + } + + if (sequence_number == sequence_num_last_received_rtp_) + return; + + // Received RTP should not be in the list. + nack_list_.erase(sequence_number); + + // If this is an old sequence number, no more action is required, return. + if (IsNewerSequenceNumber(sequence_num_last_received_rtp_, sequence_number)) + return; + + UpdateSamplesPerPacket(sequence_number, timestamp); + + UpdateList(sequence_number); + + sequence_num_last_received_rtp_ = sequence_number; + timestamp_last_received_rtp_ = timestamp; + LimitNackListSize(); +} + +void NackTracker::UpdateSamplesPerPacket( + uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp) { + uint32_t timestamp_increase = + timestamp_current_received_rtp - timestamp_last_received_rtp_; + uint16_t sequence_num_increase = + sequence_number_current_received_rtp - sequence_num_last_received_rtp_; + + samples_per_packet_ = timestamp_increase / sequence_num_increase; +} + +void NackTracker::UpdateList(uint16_t sequence_number_current_received_rtp) { + // Some of the packets which were considered late, now are considered missing. + ChangeFromLateToMissing(sequence_number_current_received_rtp); + + if (IsNewerSequenceNumber(sequence_number_current_received_rtp, + sequence_num_last_received_rtp_ + 1)) + AddToList(sequence_number_current_received_rtp); +} + +void NackTracker::ChangeFromLateToMissing( + uint16_t sequence_number_current_received_rtp) { + NackList::const_iterator lower_bound = + nack_list_.lower_bound(static_cast( + sequence_number_current_received_rtp - nack_threshold_packets_)); + + for (NackList::iterator it = nack_list_.begin(); it != lower_bound; ++it) + it->second.is_missing = true; +} + +uint32_t NackTracker::EstimateTimestamp(uint16_t sequence_num) { + uint16_t sequence_num_diff = sequence_num - sequence_num_last_received_rtp_; + return sequence_num_diff * samples_per_packet_ + timestamp_last_received_rtp_; +} + +void NackTracker::AddToList(uint16_t sequence_number_current_received_rtp) { + assert(!any_rtp_decoded_ || + IsNewerSequenceNumber(sequence_number_current_received_rtp, + sequence_num_last_decoded_rtp_)); + + // Packets with sequence numbers older than |upper_bound_missing| are + // considered missing, and the rest are considered late. + uint16_t upper_bound_missing = + sequence_number_current_received_rtp - nack_threshold_packets_; + + for (uint16_t n = sequence_num_last_received_rtp_ + 1; + IsNewerSequenceNumber(sequence_number_current_received_rtp, n); ++n) { + bool is_missing = IsNewerSequenceNumber(upper_bound_missing, n); + uint32_t timestamp = EstimateTimestamp(n); + NackElement nack_element(TimeToPlay(timestamp), timestamp, is_missing); + nack_list_.insert(nack_list_.end(), std::make_pair(n, nack_element)); + } +} + +void NackTracker::UpdateEstimatedPlayoutTimeBy10ms() { + while (!nack_list_.empty() && + nack_list_.begin()->second.time_to_play_ms <= 10) + nack_list_.erase(nack_list_.begin()); + + for (NackList::iterator it = nack_list_.begin(); it != nack_list_.end(); ++it) + it->second.time_to_play_ms -= 10; +} + +void NackTracker::UpdateLastDecodedPacket(uint16_t sequence_number, + uint32_t timestamp) { + if (IsNewerSequenceNumber(sequence_number, sequence_num_last_decoded_rtp_) || + !any_rtp_decoded_) { + sequence_num_last_decoded_rtp_ = sequence_number; + timestamp_last_decoded_rtp_ = timestamp; + // Packets in the list with sequence numbers less than the + // sequence number of the decoded RTP should be removed from the lists. + // They will be discarded by the jitter buffer if they arrive. + nack_list_.erase(nack_list_.begin(), + nack_list_.upper_bound(sequence_num_last_decoded_rtp_)); + + // Update estimated time-to-play. + for (NackList::iterator it = nack_list_.begin(); it != nack_list_.end(); + ++it) + it->second.time_to_play_ms = TimeToPlay(it->second.estimated_timestamp); + } else { + assert(sequence_number == sequence_num_last_decoded_rtp_); + + // Same sequence number as before. 10 ms is elapsed, update estimations for + // time-to-play. + UpdateEstimatedPlayoutTimeBy10ms(); + + // Update timestamp for better estimate of time-to-play, for packets which + // are added to NACK list later on. + timestamp_last_decoded_rtp_ += sample_rate_khz_ * 10; + } + any_rtp_decoded_ = true; +} + +NackTracker::NackList NackTracker::GetNackList() const { + return nack_list_; +} + +void NackTracker::Reset() { + nack_list_.clear(); + + sequence_num_last_received_rtp_ = 0; + timestamp_last_received_rtp_ = 0; + any_rtp_received_ = false; + sequence_num_last_decoded_rtp_ = 0; + timestamp_last_decoded_rtp_ = 0; + any_rtp_decoded_ = false; + sample_rate_khz_ = kDefaultSampleRateKhz; + samples_per_packet_ = sample_rate_khz_ * kDefaultPacketSizeMs; +} + +void NackTracker::SetMaxNackListSize(size_t max_nack_list_size) { + RTC_CHECK_GT(max_nack_list_size, 0); + // Ugly hack to get around the problem of passing static consts by reference. + const size_t kNackListSizeLimitLocal = NackTracker::kNackListSizeLimit; + RTC_CHECK_LE(max_nack_list_size, kNackListSizeLimitLocal); + + max_nack_list_size_ = max_nack_list_size; + LimitNackListSize(); +} + +void NackTracker::LimitNackListSize() { + uint16_t limit = sequence_num_last_received_rtp_ - + static_cast(max_nack_list_size_) - 1; + nack_list_.erase(nack_list_.begin(), nack_list_.upper_bound(limit)); +} + +int64_t NackTracker::TimeToPlay(uint32_t timestamp) const { + uint32_t timestamp_increase = timestamp - timestamp_last_decoded_rtp_; + return timestamp_increase / sample_rate_khz_; +} + +// We don't erase elements with time-to-play shorter than round-trip-time. +std::vector NackTracker::GetNackList( + int64_t round_trip_time_ms) const { + RTC_DCHECK_GE(round_trip_time_ms, 0); + std::vector sequence_numbers; + for (NackList::const_iterator it = nack_list_.begin(); it != nack_list_.end(); + ++it) { + if (it->second.is_missing && + it->second.time_to_play_ms > round_trip_time_ms) + sequence_numbers.push_back(it->first); + } + return sequence_numbers; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.h new file mode 100644 index 0000000000..4f88d917c2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.h @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NACK_TRACKER_H_ +#define MODULES_AUDIO_CODING_NETEQ_NACK_TRACKER_H_ + +#include +#include + +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "rtc_base/gtest_prod_util.h" + +// +// The NackTracker class keeps track of the lost packets, an estimate of +// time-to-play for each packet is also given. +// +// Every time a packet is pushed into NetEq, LastReceivedPacket() has to be +// called to update the NACK list. +// +// Every time 10ms audio is pulled from NetEq LastDecodedPacket() should be +// called, and time-to-play is updated at that moment. +// +// If packet N is received, any packet prior to |N - NackThreshold| which is not +// arrived is considered lost, and should be labeled as "missing" (the size of +// the list might be limited and older packet eliminated from the list). Packets +// |N - NackThreshold|, |N - NackThreshold + 1|, ..., |N - 1| are considered +// "late." A "late" packet with sequence number K is changed to "missing" any +// time a packet with sequence number newer than |K + NackList| is arrived. +// +// The NackTracker class has to know about the sample rate of the packets to +// compute time-to-play. So sample rate should be set as soon as the first +// packet is received. If there is a change in the receive codec (sender changes +// codec) then NackTracker should be reset. This is because NetEQ would flush +// its buffer and re-transmission is meaning less for old packet. Therefore, in +// that case, after reset the sampling rate has to be updated. +// +// Thread Safety +// ============= +// Please note that this class in not thread safe. The class must be protected +// if different APIs are called from different threads. +// +namespace webrtc { + +class NackTracker { + public: + // A limit for the size of the NACK list. + static const size_t kNackListSizeLimit = 500; // 10 seconds for 20 ms frame + // packets. + // Factory method. + static NackTracker* Create(int nack_threshold_packets); + + ~NackTracker(); + + // Set a maximum for the size of the NACK list. If the last received packet + // has sequence number of N, then NACK list will not contain any element + // with sequence number earlier than N - |max_nack_list_size|. + // + // The largest maximum size is defined by |kNackListSizeLimit| + void SetMaxNackListSize(size_t max_nack_list_size); + + // Set the sampling rate. + // + // If associated sampling rate of the received packets is changed, call this + // function to update sampling rate. Note that if there is any change in + // received codec then NetEq will flush its buffer and NACK has to be reset. + // After Reset() is called sampling rate has to be set. + void UpdateSampleRate(int sample_rate_hz); + + // Update the sequence number and the timestamp of the last decoded RTP. This + // API should be called every time 10 ms audio is pulled from NetEq. + void UpdateLastDecodedPacket(uint16_t sequence_number, uint32_t timestamp); + + // Update the sequence number and the timestamp of the last received RTP. This + // API should be called every time a packet pushed into ACM. + void UpdateLastReceivedPacket(uint16_t sequence_number, uint32_t timestamp); + + // Get a list of "missing" packets which have expected time-to-play larger + // than the given round-trip-time (in milliseconds). + // Note: Late packets are not included. + std::vector GetNackList(int64_t round_trip_time_ms) const; + + // Reset to default values. The NACK list is cleared. + // |nack_threshold_packets_| & |max_nack_list_size_| preserve their values. + void Reset(); + + private: + // This test need to access the private method GetNackList(). + FRIEND_TEST_ALL_PREFIXES(NackTrackerTest, EstimateTimestampAndTimeToPlay); + + struct NackElement { + NackElement(int64_t initial_time_to_play_ms, + uint32_t initial_timestamp, + bool missing) + : time_to_play_ms(initial_time_to_play_ms), + estimated_timestamp(initial_timestamp), + is_missing(missing) {} + + // Estimated time (ms) left for this packet to be decoded. This estimate is + // updated every time jitter buffer decodes a packet. + int64_t time_to_play_ms; + + // A guess about the timestamp of the missing packet, it is used for + // estimation of |time_to_play_ms|. The estimate might be slightly wrong if + // there has been frame-size change since the last received packet and the + // missing packet. However, the risk of this is low, and in case of such + // errors, there will be a minor misestimation in time-to-play of missing + // packets. This will have a very minor effect on NACK performance. + uint32_t estimated_timestamp; + + // True if the packet is considered missing. Otherwise indicates packet is + // late. + bool is_missing; + }; + + class NackListCompare { + public: + bool operator()(uint16_t sequence_number_old, + uint16_t sequence_number_new) const { + return IsNewerSequenceNumber(sequence_number_new, sequence_number_old); + } + }; + + typedef std::map NackList; + + // Constructor. + explicit NackTracker(int nack_threshold_packets); + + // This API is used only for testing to assess whether time-to-play is + // computed correctly. + NackList GetNackList() const; + + // Given the |sequence_number_current_received_rtp| of currently received RTP, + // recognize packets which are not arrive and add to the list. + void AddToList(uint16_t sequence_number_current_received_rtp); + + // This function subtracts 10 ms of time-to-play for all packets in NACK list. + // This is called when 10 ms elapsed with no new RTP packet decoded. + void UpdateEstimatedPlayoutTimeBy10ms(); + + // Given the |sequence_number_current_received_rtp| and + // |timestamp_current_received_rtp| of currently received RTP update number + // of samples per packet. + void UpdateSamplesPerPacket(uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp); + + // Given the |sequence_number_current_received_rtp| of currently received RTP + // update the list. That is; some packets will change from late to missing, + // some packets are inserted as missing and some inserted as late. + void UpdateList(uint16_t sequence_number_current_received_rtp); + + // Packets which are considered late for too long (according to + // |nack_threshold_packets_|) are flagged as missing. + void ChangeFromLateToMissing(uint16_t sequence_number_current_received_rtp); + + // Packets which have sequence number older that + // |sequence_num_last_received_rtp_| - |max_nack_list_size_| are removed + // from the NACK list. + void LimitNackListSize(); + + // Estimate timestamp of a missing packet given its sequence number. + uint32_t EstimateTimestamp(uint16_t sequence_number); + + // Compute time-to-play given a timestamp. + int64_t TimeToPlay(uint32_t timestamp) const; + + // If packet N is arrived, any packet prior to N - |nack_threshold_packets_| + // which is not arrived is considered missing, and should be in NACK list. + // Also any packet in the range of N-1 and N - |nack_threshold_packets_|, + // exclusive, which is not arrived is considered late, and should should be + // in the list of late packets. + const int nack_threshold_packets_; + + // Valid if a packet is received. + uint16_t sequence_num_last_received_rtp_; + uint32_t timestamp_last_received_rtp_; + bool any_rtp_received_; // If any packet received. + + // Valid if a packet is decoded. + uint16_t sequence_num_last_decoded_rtp_; + uint32_t timestamp_last_decoded_rtp_; + bool any_rtp_decoded_; // If any packet decoded. + + int sample_rate_khz_; // Sample rate in kHz. + + // Number of samples per packet. We update this every time we receive a + // packet, not only for consecutive packets. + int samples_per_packet_; + + // A list of missing packets to be retransmitted. Components of the list + // contain the sequence number of missing packets and the estimated time that + // each pack is going to be played out. + NackList nack_list_; + + // NACK list will not keep track of missing packets prior to + // |sequence_num_last_received_rtp_| - |max_nack_list_size_|. + size_t max_nack_list_size_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_NACK_TRACKER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc new file mode 100644 index 0000000000..19adf30bf6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc @@ -0,0 +1,483 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/nack_tracker.h" + +#include + +#include +#include + +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace { + +const int kNackThreshold = 3; +const int kSampleRateHz = 16000; +const int kPacketSizeMs = 30; +const uint32_t kTimestampIncrement = 480; // 30 ms. +const int64_t kShortRoundTripTimeMs = 1; + +bool IsNackListCorrect(const std::vector& nack_list, + const uint16_t* lost_sequence_numbers, + size_t num_lost_packets) { + if (nack_list.size() != num_lost_packets) + return false; + + if (num_lost_packets == 0) + return true; + + for (size_t k = 0; k < nack_list.size(); ++k) { + int seq_num = nack_list[k]; + bool seq_num_matched = false; + for (size_t n = 0; n < num_lost_packets; ++n) { + if (seq_num == lost_sequence_numbers[n]) { + seq_num_matched = true; + break; + } + } + if (!seq_num_matched) + return false; + } + return true; +} + +} // namespace + +TEST(NackTrackerTest, EmptyListWhenNoPacketLoss) { + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + int seq_num = 1; + uint32_t timestamp = 0; + + std::vector nack_list; + for (int n = 0; n < 100; n++) { + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + seq_num++; + timestamp += kTimestampIncrement; + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } +} + +TEST(NackTrackerTest, NoNackIfReorderWithinNackThreshold) { + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + int seq_num = 1; + uint32_t timestamp = 0; + std::vector nack_list; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + int num_late_packets = kNackThreshold + 1; + + // Push in reverse order + while (num_late_packets > 0) { + nack->UpdateLastReceivedPacket( + seq_num + num_late_packets, + timestamp + num_late_packets * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + num_late_packets--; + } +} + +TEST(NackTrackerTest, LatePacketsMovedToNackThenNackListDoesNotChange) { + const uint16_t kSequenceNumberLostPackets[] = {2, 3, 4, 5, 6, 7, 8, 9}; + static const int kNumAllLostPackets = sizeof(kSequenceNumberLostPackets) / + sizeof(kSequenceNumberLostPackets[0]); + + for (int k = 0; k < 2; k++) { // Two iteration with/without wrap around. + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + uint16_t sequence_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; n++) { + sequence_num_lost_packets[n] = + kSequenceNumberLostPackets[n] + + k * 65531; // Have wrap around in sequence numbers for |k == 1|. + } + uint16_t seq_num = sequence_num_lost_packets[0] - 1; + + uint32_t timestamp = 0; + std::vector nack_list; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + seq_num = sequence_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp += kTimestampIncrement * (kNumAllLostPackets + 1); + int num_lost_packets = std::max(0, kNumAllLostPackets - kNackThreshold); + + for (int n = 0; n < kNackThreshold + 1; ++n) { + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect(nack_list, sequence_num_lost_packets, + num_lost_packets)); + seq_num++; + timestamp += kTimestampIncrement; + num_lost_packets++; + } + + for (int n = 0; n < 100; ++n) { + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect(nack_list, sequence_num_lost_packets, + kNumAllLostPackets)); + seq_num++; + timestamp += kTimestampIncrement; + } + } +} + +TEST(NackTrackerTest, ArrivedPacketsAreRemovedFromNackList) { + const uint16_t kSequenceNumberLostPackets[] = {2, 3, 4, 5, 6, 7, 8, 9}; + static const int kNumAllLostPackets = sizeof(kSequenceNumberLostPackets) / + sizeof(kSequenceNumberLostPackets[0]); + + for (int k = 0; k < 2; ++k) { // Two iteration with/without wrap around. + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + uint16_t sequence_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; ++n) { + sequence_num_lost_packets[n] = kSequenceNumberLostPackets[n] + + k * 65531; // Wrap around for |k == 1|. + } + + uint16_t seq_num = sequence_num_lost_packets[0] - 1; + uint32_t timestamp = 0; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + size_t index_retransmitted_rtp = 0; + uint32_t timestamp_retransmitted_rtp = timestamp + kTimestampIncrement; + + seq_num = sequence_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp += kTimestampIncrement * (kNumAllLostPackets + 1); + size_t num_lost_packets = std::max(0, kNumAllLostPackets - kNackThreshold); + for (int n = 0; n < kNumAllLostPackets; ++n) { + // Number of lost packets does not change for the first + // |kNackThreshold + 1| packets, one is added to the list and one is + // removed. Thereafter, the list shrinks every iteration. + if (n >= kNackThreshold + 1) + num_lost_packets--; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &sequence_num_lost_packets[index_retransmitted_rtp], + num_lost_packets)); + seq_num++; + timestamp += kTimestampIncrement; + + // Retransmission of a lost RTP. + nack->UpdateLastReceivedPacket( + sequence_num_lost_packets[index_retransmitted_rtp], + timestamp_retransmitted_rtp); + index_retransmitted_rtp++; + timestamp_retransmitted_rtp += kTimestampIncrement; + + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &sequence_num_lost_packets[index_retransmitted_rtp], + num_lost_packets - 1)); // One less lost packet in the list. + } + ASSERT_TRUE(nack_list.empty()); + } +} + +// Assess if estimation of timestamps and time-to-play is correct. Introduce all +// combinations that timestamps and sequence numbers might have wrap around. +TEST(NackTrackerTest, EstimateTimestampAndTimeToPlay) { + const uint16_t kLostPackets[] = {2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15}; + static const int kNumAllLostPackets = + sizeof(kLostPackets) / sizeof(kLostPackets[0]); + + for (int k = 0; k < 4; ++k) { + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + // Sequence number wrap around if |k| is 2 or 3; + int seq_num_offset = (k < 2) ? 0 : 65531; + + // Timestamp wrap around if |k| is 1 or 3. + uint32_t timestamp_offset = + (k & 0x1) ? static_cast(0xffffffff) - 6 : 0; + + uint32_t timestamp_lost_packets[kNumAllLostPackets]; + uint16_t seq_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; ++n) { + timestamp_lost_packets[n] = + timestamp_offset + kLostPackets[n] * kTimestampIncrement; + seq_num_lost_packets[n] = seq_num_offset + kLostPackets[n]; + } + + // We and to push two packets before lost burst starts. + uint16_t seq_num = seq_num_lost_packets[0] - 2; + uint32_t timestamp = timestamp_lost_packets[0] - 2 * kTimestampIncrement; + + const uint16_t first_seq_num = seq_num; + const uint32_t first_timestamp = timestamp; + + // Two consecutive packets to have a correct estimate of timestamp increase. + nack->UpdateLastReceivedPacket(seq_num, timestamp); + seq_num++; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // A packet after the last one which is supposed to be lost. + seq_num = seq_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp = + timestamp_lost_packets[kNumAllLostPackets - 1] + kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + NackTracker::NackList nack_list = nack->GetNackList(); + EXPECT_EQ(static_cast(kNumAllLostPackets), nack_list.size()); + + // Pretend the first packet is decoded. + nack->UpdateLastDecodedPacket(first_seq_num, first_timestamp); + nack_list = nack->GetNackList(); + + NackTracker::NackList::iterator it = nack_list.begin(); + while (it != nack_list.end()) { + seq_num = it->first - seq_num_offset; + int index = seq_num - kLostPackets[0]; + EXPECT_EQ(timestamp_lost_packets[index], it->second.estimated_timestamp); + EXPECT_EQ((index + 2) * kPacketSizeMs, it->second.time_to_play_ms); + ++it; + } + + // Pretend 10 ms is passed, and we had pulled audio from NetEq, it still + // reports the same sequence number as decoded, time-to-play should be + // updated by 10 ms. + nack->UpdateLastDecodedPacket(first_seq_num, first_timestamp); + nack_list = nack->GetNackList(); + it = nack_list.begin(); + while (it != nack_list.end()) { + seq_num = it->first - seq_num_offset; + int index = seq_num - kLostPackets[0]; + EXPECT_EQ((index + 2) * kPacketSizeMs - 10, it->second.time_to_play_ms); + ++it; + } + } +} + +TEST(NackTrackerTest, + MissingPacketsPriorToLastDecodedRtpShouldNotBeInNackList) { + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65531; // Wrap around if |m| is 1. + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + // Two consecutive packets to have a correct estimate of timestamp increase. + uint16_t seq_num = 0; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + seq_num++; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + + // Skip 10 packets (larger than NACK threshold). + const int kNumLostPackets = 10; + seq_num += kNumLostPackets + 1; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + + const size_t kExpectedListSize = kNumLostPackets - kNackThreshold; + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + + for (int k = 0; k < 2; ++k) { + // Decoding of the first and the second arrived packets. + for (int n = 0; n < kPacketSizeMs / 10; ++n) { + nack->UpdateLastDecodedPacket(seq_num_offset + k, + k * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + } + } + + // Decoding of the last received packet. + nack->UpdateLastDecodedPacket(seq_num + seq_num_offset, + seq_num * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + // Make sure list of late packets is also empty. To check that, push few + // packets, if the late list is not empty its content will pop up in NACK + // list. + for (int n = 0; n < kNackThreshold + 10; ++n) { + seq_num++; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } + } +} + +TEST(NackTrackerTest, Reset) { + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + // Two consecutive packets to have a correct estimate of timestamp increase. + uint16_t seq_num = 0; + nack->UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + seq_num++; + nack->UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + + // Skip 10 packets (larger than NACK threshold). + const int kNumLostPackets = 10; + seq_num += kNumLostPackets + 1; + nack->UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + + const size_t kExpectedListSize = kNumLostPackets - kNackThreshold; + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + + nack->Reset(); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); +} + +TEST(NackTrackerTest, ListSizeAppliedFromBeginning) { + const size_t kNackListSize = 10; + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65525; // Wrap around if |m| is 1. + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + nack->SetMaxNackListSize(kNackListSize); + + uint16_t seq_num = seq_num_offset; + uint32_t timestamp = 0x12345678; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t num_lost_packets = kNackThreshold + kNackListSize + 5; + + seq_num += num_lost_packets + 1; + timestamp += (num_lost_packets + 1) * kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kNackListSize - kNackThreshold, nack_list.size()); + } +} + +TEST(NackTrackerTest, ChangeOfListSizeAppliedAndOldElementsRemoved) { + const size_t kNackListSize = 10; + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65525; // Wrap around if |m| is 1. + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + uint16_t seq_num = seq_num_offset; + uint32_t timestamp = 0x87654321; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t num_lost_packets = kNackThreshold + kNackListSize + 5; + + std::unique_ptr seq_num_lost(new uint16_t[num_lost_packets]); + for (int n = 0; n < num_lost_packets; ++n) { + seq_num_lost[n] = ++seq_num; + } + + ++seq_num; + timestamp += (num_lost_packets + 1) * kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + size_t expected_size = num_lost_packets - kNackThreshold; + + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(expected_size, nack_list.size()); + + nack->SetMaxNackListSize(kNackListSize); + expected_size = kNackListSize - kNackThreshold; + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize], + expected_size)); + + // NACK list does not change size but the content is changing. The oldest + // element is removed and one from late list is inserted. + size_t n; + for (n = 1; n <= static_cast(kNackThreshold); ++n) { + ++seq_num; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize + n], + expected_size)); + } + + // NACK list should shrink. + for (; n < kNackListSize; ++n) { + ++seq_num; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + --expected_size; + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize + n], + expected_size)); + } + + // After this packet, NACK list should be empty. + ++seq_num; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } +} + +TEST(NackTrackerTest, RoudTripTimeIsApplied) { + const int kNackListSize = 200; + std::unique_ptr nack(NackTracker::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + nack->SetMaxNackListSize(kNackListSize); + + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t kNumLostPackets = kNackThreshold + 5; + + seq_num += (1 + kNumLostPackets); + timestamp += (1 + kNumLostPackets) * kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Expected time-to-play are: + // kPacketSizeMs - 10, 2*kPacketSizeMs - 10, 3*kPacketSizeMs - 10, ... + // + // sequence number: 1, 2, 3, 4, 5 + // time-to-play: 20, 50, 80, 110, 140 + // + std::vector nack_list = nack->GetNackList(100); + ASSERT_EQ(2u, nack_list.size()); + EXPECT_EQ(4, nack_list[0]); + EXPECT_EQ(5, nack_list[1]); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq.cc new file mode 100644 index 0000000000..8b74973aff --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/include/neteq.h" + +#include +#include + +#include "modules/audio_coding/neteq/neteq_impl.h" + +namespace webrtc { + +std::string NetEq::Config::ToString() const { + std::stringstream ss; + ss << "sample_rate_hz=" << sample_rate_hz + << ", enable_post_decode_vad=" + << (enable_post_decode_vad ? "true" : "false") + << ", max_packets_in_buffer=" << max_packets_in_buffer + << ", background_noise_mode=" << background_noise_mode + << ", playout_mode=" << playout_mode + << ", enable_fast_accelerate=" + << (enable_fast_accelerate ? " true": "false") + << ", enable_muted_state=" << (enable_muted_state ? " true": "false"); + return ss.str(); +} + +// Creates all classes needed and inject them into a new NetEqImpl object. +// Return the new object. +NetEq* NetEq::Create( + const NetEq::Config& config, + const rtc::scoped_refptr& decoder_factory) { + return new NetEqImpl(config, + NetEqImpl::Dependencies(config, decoder_factory)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.cc new file mode 100644 index 0000000000..8d66c2a25c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_coding/neteq/neteq_decoder_enum.h" + +namespace webrtc { + +rtc::Optional NetEqDecoderToSdpAudioFormat(NetEqDecoder nd) { + switch (nd) { + case NetEqDecoder::kDecoderPCMu: + return SdpAudioFormat("pcmu", 8000, 1); + case NetEqDecoder::kDecoderPCMa: + return SdpAudioFormat("pcma", 8000, 1); + case NetEqDecoder::kDecoderPCMu_2ch: + return SdpAudioFormat("pcmu", 8000, 2); + case NetEqDecoder::kDecoderPCMa_2ch: + return SdpAudioFormat("pcma", 8000, 2); + case NetEqDecoder::kDecoderILBC: + return SdpAudioFormat("ilbc", 8000, 1); + case NetEqDecoder::kDecoderISAC: + return SdpAudioFormat("isac", 16000, 1); + case NetEqDecoder::kDecoderISACswb: + return SdpAudioFormat("isac", 32000, 1); + case NetEqDecoder::kDecoderPCM16B: + return SdpAudioFormat("l16", 8000, 1); + case NetEqDecoder::kDecoderPCM16Bwb: + return SdpAudioFormat("l16", 16000, 1); + case NetEqDecoder::kDecoderPCM16Bswb32kHz: + return SdpAudioFormat("l16", 32000, 1); + case NetEqDecoder::kDecoderPCM16Bswb48kHz: + return SdpAudioFormat("l16", 48000, 1); + case NetEqDecoder::kDecoderPCM16B_2ch: + return SdpAudioFormat("l16", 8000, 2); + case NetEqDecoder::kDecoderPCM16Bwb_2ch: + return SdpAudioFormat("l16", 16000, 2); + case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: + return SdpAudioFormat("l16", 32000, 2); + case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: + return SdpAudioFormat("l16", 48000, 2); + case NetEqDecoder::kDecoderPCM16B_5ch: + return SdpAudioFormat("l16", 8000, 5); + case NetEqDecoder::kDecoderG722: + return SdpAudioFormat("g722", 8000, 1); + case NetEqDecoder::kDecoderG722_2ch: + return SdpAudioFormat("g722", 8000, 2); + case NetEqDecoder::kDecoderOpus: + return SdpAudioFormat("opus", 48000, 2); + case NetEqDecoder::kDecoderOpus_2ch: + return SdpAudioFormat( + "opus", 48000, 2, + std::map{{"stereo", "1"}}); + case NetEqDecoder::kDecoderRED: + return SdpAudioFormat("red", 8000, 1); + case NetEqDecoder::kDecoderAVT: + return SdpAudioFormat("telephone-event", 8000, 1); + case NetEqDecoder::kDecoderAVT16kHz: + return SdpAudioFormat("telephone-event", 16000, 1); + case NetEqDecoder::kDecoderAVT32kHz: + return SdpAudioFormat("telephone-event", 32000, 1); + case NetEqDecoder::kDecoderAVT48kHz: + return SdpAudioFormat("telephone-event", 48000, 1); + case NetEqDecoder::kDecoderCNGnb: + return SdpAudioFormat("cn", 8000, 1); + case NetEqDecoder::kDecoderCNGwb: + return SdpAudioFormat("cn", 16000, 1); + case NetEqDecoder::kDecoderCNGswb32kHz: + return SdpAudioFormat("cn", 32000, 1); + case NetEqDecoder::kDecoderCNGswb48kHz: + return SdpAudioFormat("cn", 48000, 1); + default: + return rtc::nullopt; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.h new file mode 100644 index 0000000000..024f03cd57 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NETEQ_DECODER_ENUM_H_ +#define MODULES_AUDIO_CODING_NETEQ_NETEQ_DECODER_ENUM_H_ + +#include "api/audio_codecs/audio_format.h" +#include "api/optional.h" + +namespace webrtc { + +enum class NetEqDecoder { + kDecoderPCMu, + kDecoderPCMa, + kDecoderPCMu_2ch, + kDecoderPCMa_2ch, + kDecoderILBC, + kDecoderISAC, + kDecoderISACswb, + kDecoderPCM16B, + kDecoderPCM16Bwb, + kDecoderPCM16Bswb32kHz, + kDecoderPCM16Bswb48kHz, + kDecoderPCM16B_2ch, + kDecoderPCM16Bwb_2ch, + kDecoderPCM16Bswb32kHz_2ch, + kDecoderPCM16Bswb48kHz_2ch, + kDecoderPCM16B_5ch, + kDecoderG722, + kDecoderG722_2ch, + kDecoderRED, + kDecoderAVT, + kDecoderAVT16kHz, + kDecoderAVT32kHz, + kDecoderAVT48kHz, + kDecoderCNGnb, + kDecoderCNGwb, + kDecoderCNGswb32kHz, + kDecoderCNGswb48kHz, + kDecoderArbitrary, + kDecoderOpus, + kDecoderOpus_2ch, +}; + +rtc::Optional NetEqDecoderToSdpAudioFormat(NetEqDecoder nd); + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_NETEQ_DECODER_ENUM_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc new file mode 100644 index 0000000000..ec166277f8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc @@ -0,0 +1,457 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Test to verify correct operation for externally created decoders. + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/neteq_external_decoder_test.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "modules/include/module_common_types.h" +#include "test/gmock.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::Return; + +class NetEqExternalDecoderUnitTest : public test::NetEqExternalDecoderTest { + protected: + static const int kFrameSizeMs = 10; // Frame size of Pcm16B. + + NetEqExternalDecoderUnitTest(NetEqDecoder codec, + int sample_rate_hz, + MockExternalPcm16B* decoder) + : NetEqExternalDecoderTest(codec, sample_rate_hz, decoder), + external_decoder_(decoder), + samples_per_ms_(sample_rate_hz / 1000), + frame_size_samples_(kFrameSizeMs * samples_per_ms_), + rtp_generator_(new test::RtpGenerator(samples_per_ms_)), + input_(new int16_t[frame_size_samples_]), + // Payload should be no larger than input. + encoded_(new uint8_t[2 * frame_size_samples_]), + payload_size_bytes_(0), + last_send_time_(0), + last_arrival_time_(0) { + // NetEq is not allowed to delete the external decoder (hence Times(0)). + EXPECT_CALL(*external_decoder_, Die()).Times(0); + Init(); + + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + input_file_.reset(new test::InputAudioFile(file_name)); + } + + virtual ~NetEqExternalDecoderUnitTest() { + delete [] input_; + delete [] encoded_; + // ~NetEqExternalDecoderTest() will delete |external_decoder_|, so expecting + // Die() to be called. + EXPECT_CALL(*external_decoder_, Die()).Times(1); + } + + // Method to draw kFrameSizeMs audio and verify the output. + // Use gTest methods. e.g. ASSERT_EQ() inside to trigger errors. + virtual void GetAndVerifyOutput() = 0; + + // Method to get the number of calls to the Decode() method of the external + // decoder. + virtual int NumExpectedDecodeCalls(int num_loops) = 0; + + // Method to generate packets and return the send time of the packet. + int GetNewPacket() { + if (!input_file_->Read(frame_size_samples_, input_)) { + return -1; + } + payload_size_bytes_ = WebRtcPcm16b_Encode(input_, frame_size_samples_, + encoded_); + + int next_send_time = rtp_generator_->GetRtpHeader( + kPayloadType, frame_size_samples_, &rtp_header_); + return next_send_time; + } + + // Method to decide packet losses. + virtual bool Lost() { return false; } + + // Method to calculate packet arrival time. + int GetArrivalTime(int send_time) { + int arrival_time = last_arrival_time_ + (send_time - last_send_time_); + last_send_time_ = send_time; + last_arrival_time_ = arrival_time; + return arrival_time; + } + + void RunTest(int num_loops) { + // Get next input packets (mono and multi-channel). + uint32_t next_send_time; + uint32_t next_arrival_time; + do { + next_send_time = GetNewPacket(); + next_arrival_time = GetArrivalTime(next_send_time); + } while (Lost()); // If lost, immediately read the next packet. + + EXPECT_CALL( + *external_decoder_, + DecodeInternal(_, payload_size_bytes_, 1000 * samples_per_ms_, _, _)) + .Times(NumExpectedDecodeCalls(num_loops)); + + uint32_t time_now = 0; + for (int k = 0; k < num_loops; ++k) { + while (time_now >= next_arrival_time) { + InsertPacket(rtp_header_, rtc::ArrayView( + encoded_, payload_size_bytes_), + next_arrival_time); + // Get next input packet. + do { + next_send_time = GetNewPacket(); + next_arrival_time = GetArrivalTime(next_send_time); + } while (Lost()); // If lost, immediately read the next packet. + } + + std::ostringstream ss; + ss << "Lap number " << k << "."; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + // Compare mono and multi-channel. + ASSERT_NO_FATAL_FAILURE(GetAndVerifyOutput()); + + time_now += kOutputLengthMs; + } + } + + void InsertPacket(RTPHeader rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) override { + EXPECT_CALL(*external_decoder_, + IncomingPacket(_, payload.size(), rtp_header.sequenceNumber, + rtp_header.timestamp, receive_timestamp)); + NetEqExternalDecoderTest::InsertPacket(rtp_header, payload, + receive_timestamp); + } + + MockExternalPcm16B* external_decoder() { return external_decoder_.get(); } + + void ResetRtpGenerator(test::RtpGenerator* rtp_generator) { + rtp_generator_.reset(rtp_generator); + } + + int samples_per_ms() const { return samples_per_ms_; } + private: + std::unique_ptr external_decoder_; + int samples_per_ms_; + size_t frame_size_samples_; + std::unique_ptr rtp_generator_; + int16_t* input_; + uint8_t* encoded_; + size_t payload_size_bytes_; + uint32_t last_send_time_; + uint32_t last_arrival_time_; + std::unique_ptr input_file_; + RTPHeader rtp_header_; +}; + +// This test encodes a few packets of PCM16b 32 kHz data and inserts it into two +// different NetEq instances. The first instance uses the internal version of +// the decoder object, while the second one uses an externally created decoder +// object (ExternalPcm16B wrapped in MockExternalPcm16B, both defined above). +// The test verifies that the output from both instances match. +class NetEqExternalVsInternalDecoderTest : public NetEqExternalDecoderUnitTest, + public ::testing::Test { + protected: + static const size_t kMaxBlockSize = 480; // 10 ms @ 48 kHz. + + NetEqExternalVsInternalDecoderTest() + : NetEqExternalDecoderUnitTest(NetEqDecoder::kDecoderPCM16Bswb32kHz, + 32000, + new MockExternalPcm16B(32000)), + sample_rate_hz_(32000) { + NetEq::Config config; + config.sample_rate_hz = sample_rate_hz_; + neteq_internal_.reset( + NetEq::Create(config, CreateBuiltinAudioDecoderFactory())); + } + + void SetUp() override { + ASSERT_EQ(true, neteq_internal_->RegisterPayloadType( + kPayloadType, SdpAudioFormat("L16", 32000, 1))); + } + + void GetAndVerifyOutput() override { + // Get audio from internal decoder instance. + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_internal_->GetAudio(&output_internal_, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(1u, output_internal_.num_channels_); + EXPECT_EQ(static_cast(kOutputLengthMs * sample_rate_hz_ / 1000), + output_internal_.samples_per_channel_); + + // Get audio from external decoder instance. + GetOutputAudio(&output_); + + const int16_t* output_data = output_.data(); + const int16_t* output_internal_data = output_internal_.data(); + for (size_t i = 0; i < output_.samples_per_channel_; ++i) { + ASSERT_EQ(output_data[i], output_internal_data[i]) + << "Diff in sample " << i << "."; + } + } + + void InsertPacket(RTPHeader rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) override { + // Insert packet in internal decoder. + ASSERT_EQ(NetEq::kOK, neteq_internal_->InsertPacket(rtp_header, payload, + receive_timestamp)); + + // Insert packet in external decoder instance. + NetEqExternalDecoderUnitTest::InsertPacket(rtp_header, payload, + receive_timestamp); + } + + int NumExpectedDecodeCalls(int num_loops) override { return num_loops; } + + private: + int sample_rate_hz_; + std::unique_ptr neteq_internal_; + AudioFrame output_internal_; + AudioFrame output_; +}; + +TEST_F(NetEqExternalVsInternalDecoderTest, RunTest) { + RunTest(100); // Run 100 laps @ 10 ms each in the test loop. +} + +class LargeTimestampJumpTest : public NetEqExternalDecoderUnitTest, + public ::testing::Test { + protected: + static const size_t kMaxBlockSize = 480; // 10 ms @ 48 kHz. + + enum TestStates { + kInitialPhase, + kNormalPhase, + kExpandPhase, + kFadedExpandPhase, + kRecovered + }; + + LargeTimestampJumpTest() + : NetEqExternalDecoderUnitTest(NetEqDecoder::kDecoderPCM16B, + 8000, + new MockExternalPcm16B(8000)), + test_state_(kInitialPhase) { + EXPECT_CALL(*external_decoder(), HasDecodePlc()) + .WillRepeatedly(Return(false)); + } + + virtual void UpdateState(AudioFrame::SpeechType output_type) { + switch (test_state_) { + case kInitialPhase: { + if (output_type == AudioFrame::kNormalSpeech) { + test_state_ = kNormalPhase; + } + break; + } + case kNormalPhase: { + if (output_type == AudioFrame::kPLC) { + test_state_ = kExpandPhase; + } + break; + } + case kExpandPhase: { + if (output_type == AudioFrame::kPLCCNG) { + test_state_ = kFadedExpandPhase; + } else if (output_type == AudioFrame::kNormalSpeech) { + test_state_ = kRecovered; + } + break; + } + case kFadedExpandPhase: { + if (output_type == AudioFrame::kNormalSpeech) { + test_state_ = kRecovered; + } + break; + } + case kRecovered: { + break; + } + } + } + + void GetAndVerifyOutput() override { + AudioFrame output; + GetOutputAudio(&output); + UpdateState(output.speech_type_); + + if (test_state_ == kExpandPhase || test_state_ == kFadedExpandPhase) { + // Don't verify the output in this phase of the test. + return; + } + + ASSERT_EQ(1u, output.num_channels_); + const int16_t* output_data = output.data(); + for (size_t i = 0; i < output.samples_per_channel_; ++i) { + if (output_data[i] != 0) + return; + } + EXPECT_TRUE(false) + << "Expected at least one non-zero sample in each output block."; + } + + int NumExpectedDecodeCalls(int num_loops) override { + // Some packets at the end of the stream won't be decoded. When the jump in + // timestamp happens, NetEq will do Expand during one GetAudio call. In the + // next call it will decode the packet after the jump, but the net result is + // that the delay increased by 1 packet. In another call, a Pre-emptive + // Expand operation is performed, leading to delay increase by 1 packet. In + // total, the test will end with a 2-packet delay, which results in the 2 + // last packets not being decoded. + return num_loops - 2; + } + + TestStates test_state_; +}; + +TEST_F(LargeTimestampJumpTest, JumpLongerThanHalfRange) { + // Set the timestamp series to start at 2880, increase to 7200, then jump to + // 2869342376. The sequence numbers start at 42076 and increase by 1 for each + // packet, also when the timestamp jumps. + static const uint16_t kStartSeqeunceNumber = 42076; + static const uint32_t kStartTimestamp = 2880; + static const uint32_t kJumpFromTimestamp = 7200; + static const uint32_t kJumpToTimestamp = 2869342376; + static_assert(kJumpFromTimestamp < kJumpToTimestamp, + "timestamp jump should not result in wrap"); + static_assert( + static_cast(kJumpToTimestamp - kJumpFromTimestamp) > 0x7FFFFFFF, + "jump should be larger than half range"); + // Replace the default RTP generator with one that jumps in timestamp. + ResetRtpGenerator(new test::TimestampJumpRtpGenerator(samples_per_ms(), + kStartSeqeunceNumber, + kStartTimestamp, + kJumpFromTimestamp, + kJumpToTimestamp)); + + RunTest(130); // Run 130 laps @ 10 ms each in the test loop. + EXPECT_EQ(kRecovered, test_state_); +} + +TEST_F(LargeTimestampJumpTest, JumpLongerThanHalfRangeAndWrap) { + // Make a jump larger than half the 32-bit timestamp range. Set the start + // timestamp such that the jump will result in a wrap around. + static const uint16_t kStartSeqeunceNumber = 42076; + // Set the jump length slightly larger than 2^31. + static const uint32_t kStartTimestamp = 3221223116; + static const uint32_t kJumpFromTimestamp = 3221223216; + static const uint32_t kJumpToTimestamp = 1073744278; + static_assert(kJumpToTimestamp < kJumpFromTimestamp, + "timestamp jump should result in wrap"); + static_assert( + static_cast(kJumpToTimestamp - kJumpFromTimestamp) > 0x7FFFFFFF, + "jump should be larger than half range"); + // Replace the default RTP generator with one that jumps in timestamp. + ResetRtpGenerator(new test::TimestampJumpRtpGenerator(samples_per_ms(), + kStartSeqeunceNumber, + kStartTimestamp, + kJumpFromTimestamp, + kJumpToTimestamp)); + + RunTest(130); // Run 130 laps @ 10 ms each in the test loop. + EXPECT_EQ(kRecovered, test_state_); +} + +class ShortTimestampJumpTest : public LargeTimestampJumpTest { + protected: + void UpdateState(AudioFrame::SpeechType output_type) override { + switch (test_state_) { + case kInitialPhase: { + if (output_type == AudioFrame::kNormalSpeech) { + test_state_ = kNormalPhase; + } + break; + } + case kNormalPhase: { + if (output_type == AudioFrame::kPLC) { + test_state_ = kExpandPhase; + } + break; + } + case kExpandPhase: { + if (output_type == AudioFrame::kNormalSpeech) { + test_state_ = kRecovered; + } + break; + } + case kRecovered: { + break; + } + default: { FAIL(); } + } + } + + int NumExpectedDecodeCalls(int num_loops) override { + // Some packets won't be decoded because of the timestamp jump. + return num_loops - 2; + } +}; + +TEST_F(ShortTimestampJumpTest, JumpShorterThanHalfRange) { + // Make a jump shorter than half the 32-bit timestamp range. Set the start + // timestamp such that the jump will not result in a wrap around. + static const uint16_t kStartSeqeunceNumber = 42076; + // Set the jump length slightly smaller than 2^31. + static const uint32_t kStartTimestamp = 4711; + static const uint32_t kJumpFromTimestamp = 4811; + static const uint32_t kJumpToTimestamp = 2147483747; + static_assert(kJumpFromTimestamp < kJumpToTimestamp, + "timestamp jump should not result in wrap"); + static_assert( + static_cast(kJumpToTimestamp - kJumpFromTimestamp) < 0x7FFFFFFF, + "jump should be smaller than half range"); + // Replace the default RTP generator with one that jumps in timestamp. + ResetRtpGenerator(new test::TimestampJumpRtpGenerator(samples_per_ms(), + kStartSeqeunceNumber, + kStartTimestamp, + kJumpFromTimestamp, + kJumpToTimestamp)); + + RunTest(130); // Run 130 laps @ 10 ms each in the test loop. + EXPECT_EQ(kRecovered, test_state_); +} + +TEST_F(ShortTimestampJumpTest, JumpShorterThanHalfRangeAndWrap) { + // Make a jump shorter than half the 32-bit timestamp range. Set the start + // timestamp such that the jump will result in a wrap around. + static const uint16_t kStartSeqeunceNumber = 42076; + // Set the jump length slightly smaller than 2^31. + static const uint32_t kStartTimestamp = 3221227827; + static const uint32_t kJumpFromTimestamp = 3221227927; + static const uint32_t kJumpToTimestamp = 1073739567; + static_assert(kJumpToTimestamp < kJumpFromTimestamp, + "timestamp jump should result in wrap"); + static_assert( + static_cast(kJumpToTimestamp - kJumpFromTimestamp) < 0x7FFFFFFF, + "jump should be smaller than half range"); + // Replace the default RTP generator with one that jumps in timestamp. + ResetRtpGenerator(new test::TimestampJumpRtpGenerator(samples_per_ms(), + kStartSeqeunceNumber, + kStartTimestamp, + kJumpFromTimestamp, + kJumpToTimestamp)); + + RunTest(130); // Run 130 laps @ 10 ms each in the test loop. + EXPECT_EQ(kRecovered, test_state_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.cc new file mode 100644 index 0000000000..b459a7bb3a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -0,0 +1,2139 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/neteq_impl.h" + +#include + +#include +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/accelerate.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/buffer_level_filter.h" +#include "modules/audio_coding/neteq/comfort_noise.h" +#include "modules/audio_coding/neteq/decision_logic.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/defines.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "modules/audio_coding/neteq/delay_peak_detector.h" +#include "modules/audio_coding/neteq/dtmf_buffer.h" +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/merge.h" +#include "modules/audio_coding/neteq/nack_tracker.h" +#include "modules/audio_coding/neteq/normal.h" +#include "modules/audio_coding/neteq/packet.h" +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "modules/audio_coding/neteq/post_decode_vad.h" +#include "modules/audio_coding/neteq/preemptive_expand.h" +#include "modules/audio_coding/neteq/red_payload_splitter.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/audio_coding/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/timestamp_scaler.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/sanitizer.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +NetEqImpl::Dependencies::Dependencies( + const NetEq::Config& config, + const rtc::scoped_refptr& decoder_factory) + : tick_timer(new TickTimer), + buffer_level_filter(new BufferLevelFilter), + decoder_database(new DecoderDatabase(decoder_factory)), + delay_peak_detector(new DelayPeakDetector(tick_timer.get())), + delay_manager(new DelayManager(config.max_packets_in_buffer, + delay_peak_detector.get(), + tick_timer.get())), + dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)), + dtmf_tone_generator(new DtmfToneGenerator), + packet_buffer( + new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())), + red_payload_splitter(new RedPayloadSplitter), + timestamp_scaler(new TimestampScaler(*decoder_database)), + accelerate_factory(new AccelerateFactory), + expand_factory(new ExpandFactory), + preemptive_expand_factory(new PreemptiveExpandFactory) {} + +NetEqImpl::Dependencies::~Dependencies() = default; + +NetEqImpl::NetEqImpl(const NetEq::Config& config, + Dependencies&& deps, + bool create_components) + : tick_timer_(std::move(deps.tick_timer)), + buffer_level_filter_(std::move(deps.buffer_level_filter)), + decoder_database_(std::move(deps.decoder_database)), + delay_manager_(std::move(deps.delay_manager)), + delay_peak_detector_(std::move(deps.delay_peak_detector)), + dtmf_buffer_(std::move(deps.dtmf_buffer)), + dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)), + packet_buffer_(std::move(deps.packet_buffer)), + red_payload_splitter_(std::move(deps.red_payload_splitter)), + timestamp_scaler_(std::move(deps.timestamp_scaler)), + vad_(new PostDecodeVad()), + expand_factory_(std::move(deps.expand_factory)), + accelerate_factory_(std::move(deps.accelerate_factory)), + preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)), + last_mode_(kModeNormal), + decoded_buffer_length_(kMaxFrameSize), + decoded_buffer_(new int16_t[decoded_buffer_length_]), + playout_timestamp_(0), + new_codec_(false), + timestamp_(0), + reset_decoder_(false), + ssrc_(0), + first_packet_(true), + background_noise_mode_(config.background_noise_mode), + playout_mode_(config.playout_mode), + enable_fast_accelerate_(config.enable_fast_accelerate), + nack_enabled_(false), + enable_muted_state_(config.enable_muted_state), + use_dtx_delay_fix_( + field_trial::IsEnabled("WebRTC-NetEqOpusDtxDelayFix")) { + RTC_LOG(LS_INFO) << "NetEq config: " << config.ToString(); + int fs = config.sample_rate_hz; + if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) { + RTC_LOG(LS_ERROR) << "Sample rate " << fs << " Hz not supported. " + << "Changing to 8000 Hz."; + fs = 8000; + } + delay_manager_->SetMaximumDelay(config.max_delay_ms); + fs_hz_ = fs; + fs_mult_ = fs / 8000; + last_output_sample_rate_hz_ = fs; + output_size_samples_ = static_cast(kOutputSizeMs * 8 * fs_mult_); + decoder_frame_length_ = 3 * output_size_samples_; + WebRtcSpl_Init(); + if (create_components) { + SetSampleRateAndChannels(fs, 1); // Default is 1 channel. + } + RTC_DCHECK(!vad_->enabled()); + if (config.enable_post_decode_vad) { + vad_->Enable(); + } +} + +NetEqImpl::~NetEqImpl() = default; + +int NetEqImpl::InsertPacket(const RTPHeader& rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) { + rtc::MsanCheckInitialized(payload); + TRACE_EVENT0("webrtc", "NetEqImpl::InsertPacket"); + rtc::CritScope lock(&crit_sect_); + if (InsertPacketInternal(rtp_header, payload, receive_timestamp) != 0) { + return kFail; + } + return kOK; +} + +void NetEqImpl::InsertEmptyPacket(const RTPHeader& /*rtp_header*/) { + // TODO(henrik.lundin) Handle NACK as well. This will make use of the + // rtp_header parameter. + // https://bugs.chromium.org/p/webrtc/issues/detail?id=7611 + rtc::CritScope lock(&crit_sect_); + delay_manager_->RegisterEmptyPacket(); +} + +namespace { +void SetAudioFrameActivityAndType(bool vad_enabled, + NetEqImpl::OutputType type, + AudioFrame::VADActivity last_vad_activity, + AudioFrame* audio_frame) { + switch (type) { + case NetEqImpl::OutputType::kNormalSpeech: { + audio_frame->speech_type_ = AudioFrame::kNormalSpeech; + audio_frame->vad_activity_ = AudioFrame::kVadActive; + break; + } + case NetEqImpl::OutputType::kVadPassive: { + // This should only be reached if the VAD is enabled. + RTC_DCHECK(vad_enabled); + audio_frame->speech_type_ = AudioFrame::kNormalSpeech; + audio_frame->vad_activity_ = AudioFrame::kVadPassive; + break; + } + case NetEqImpl::OutputType::kCNG: { + audio_frame->speech_type_ = AudioFrame::kCNG; + audio_frame->vad_activity_ = AudioFrame::kVadPassive; + break; + } + case NetEqImpl::OutputType::kPLC: { + audio_frame->speech_type_ = AudioFrame::kPLC; + audio_frame->vad_activity_ = last_vad_activity; + break; + } + case NetEqImpl::OutputType::kPLCCNG: { + audio_frame->speech_type_ = AudioFrame::kPLCCNG; + audio_frame->vad_activity_ = AudioFrame::kVadPassive; + break; + } + default: + RTC_NOTREACHED(); + } + if (!vad_enabled) { + // Always set kVadUnknown when receive VAD is inactive. + audio_frame->vad_activity_ = AudioFrame::kVadUnknown; + } +} +} // namespace + +int NetEqImpl::GetAudio(AudioFrame* audio_frame, bool* muted) { + TRACE_EVENT0("webrtc", "NetEqImpl::GetAudio"); + rtc::CritScope lock(&crit_sect_); + if (GetAudioInternal(audio_frame, muted) != 0) { + return kFail; + } + RTC_DCHECK_EQ( + audio_frame->sample_rate_hz_, + rtc::dchecked_cast(audio_frame->samples_per_channel_ * 100)); + RTC_DCHECK_EQ(*muted, audio_frame->muted()); + SetAudioFrameActivityAndType(vad_->enabled(), LastOutputType(), + last_vad_activity_, audio_frame); + last_vad_activity_ = audio_frame->vad_activity_; + last_output_sample_rate_hz_ = audio_frame->sample_rate_hz_; + RTC_DCHECK(last_output_sample_rate_hz_ == 8000 || + last_output_sample_rate_hz_ == 16000 || + last_output_sample_rate_hz_ == 32000 || + last_output_sample_rate_hz_ == 48000) + << "Unexpected sample rate " << last_output_sample_rate_hz_; + return kOK; +} + +void NetEqImpl::SetCodecs(const std::map& codecs) { + rtc::CritScope lock(&crit_sect_); + const std::vector changed_payload_types = + decoder_database_->SetCodecs(codecs); + for (const int pt : changed_payload_types) { + packet_buffer_->DiscardPacketsWithPayloadType(pt, &stats_); + } +} + +int NetEqImpl::RegisterPayloadType(NetEqDecoder codec, + const std::string& name, + uint8_t rtp_payload_type) { + rtc::CritScope lock(&crit_sect_); + RTC_LOG(LS_VERBOSE) << "RegisterPayloadType " + << static_cast(rtp_payload_type) << " " + << static_cast(codec); + if (decoder_database_->RegisterPayload(rtp_payload_type, codec, name) != + DecoderDatabase::kOK) { + return kFail; + } + return kOK; +} + +int NetEqImpl::RegisterExternalDecoder(AudioDecoder* decoder, + NetEqDecoder codec, + const std::string& codec_name, + uint8_t rtp_payload_type) { + rtc::CritScope lock(&crit_sect_); + RTC_LOG(LS_VERBOSE) << "RegisterExternalDecoder " + << static_cast(rtp_payload_type) << " " + << static_cast(codec); + if (!decoder) { + RTC_LOG(LS_ERROR) << "Cannot register external decoder with NULL pointer"; + assert(false); + return kFail; + } + if (decoder_database_->InsertExternal(rtp_payload_type, codec, codec_name, + decoder) != DecoderDatabase::kOK) { + return kFail; + } + return kOK; +} + +bool NetEqImpl::RegisterPayloadType(int rtp_payload_type, + const SdpAudioFormat& audio_format) { + RTC_LOG(LS_VERBOSE) << "NetEqImpl::RegisterPayloadType: payload type " + << rtp_payload_type << ", codec " << audio_format; + rtc::CritScope lock(&crit_sect_); + return decoder_database_->RegisterPayload(rtp_payload_type, audio_format) == + DecoderDatabase::kOK; +} + +int NetEqImpl::RemovePayloadType(uint8_t rtp_payload_type) { + rtc::CritScope lock(&crit_sect_); + int ret = decoder_database_->Remove(rtp_payload_type); + if (ret == DecoderDatabase::kOK || ret == DecoderDatabase::kDecoderNotFound) { + packet_buffer_->DiscardPacketsWithPayloadType(rtp_payload_type, &stats_); + return kOK; + } + return kFail; +} + +void NetEqImpl::RemoveAllPayloadTypes() { + rtc::CritScope lock(&crit_sect_); + decoder_database_->RemoveAll(); +} + +bool NetEqImpl::SetMinimumDelay(int delay_ms) { + rtc::CritScope lock(&crit_sect_); + if (delay_ms >= 0 && delay_ms <= 10000) { + assert(delay_manager_.get()); + return delay_manager_->SetMinimumDelay(delay_ms); + } + return false; +} + +bool NetEqImpl::SetMaximumDelay(int delay_ms) { + rtc::CritScope lock(&crit_sect_); + if (delay_ms >= 0 && delay_ms <= 10000) { + assert(delay_manager_.get()); + return delay_manager_->SetMaximumDelay(delay_ms); + } + return false; +} + +int NetEqImpl::LeastRequiredDelayMs() const { + rtc::CritScope lock(&crit_sect_); + assert(delay_manager_.get()); + return delay_manager_->least_required_delay_ms(); +} + +int NetEqImpl::SetTargetDelay() { + return kNotImplemented; +} + +int NetEqImpl::TargetDelayMs() const { + rtc::CritScope lock(&crit_sect_); + RTC_DCHECK(delay_manager_.get()); + // The value from TargetLevel() is in number of packets, represented in Q8. + const size_t target_delay_samples = + (delay_manager_->TargetLevel() * decoder_frame_length_) >> 8; + return static_cast(target_delay_samples) / + rtc::CheckedDivExact(fs_hz_, 1000); +} + +int NetEqImpl::CurrentDelayMs() const { + rtc::CritScope lock(&crit_sect_); + if (fs_hz_ == 0) + return 0; + // Sum up the samples in the packet buffer with the future length of the sync + // buffer, and divide the sum by the sample rate. + const size_t delay_samples = + packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) + + sync_buffer_->FutureLength(); + // The division below will truncate. + const int delay_ms = + static_cast(delay_samples) / rtc::CheckedDivExact(fs_hz_, 1000); + return delay_ms; +} + +int NetEqImpl::FilteredCurrentDelayMs() const { + rtc::CritScope lock(&crit_sect_); + // Calculate the filtered packet buffer level in samples. The value from + // |buffer_level_filter_| is in number of packets, represented in Q8. + const size_t packet_buffer_samples = + (buffer_level_filter_->filtered_current_level() * + decoder_frame_length_) >> + 8; + // Sum up the filtered packet buffer level with the future length of the sync + // buffer, and divide the sum by the sample rate. + const size_t delay_samples = + packet_buffer_samples + sync_buffer_->FutureLength(); + // The division below will truncate. The return value is in ms. + return static_cast(delay_samples) / rtc::CheckedDivExact(fs_hz_, 1000); +} + +// Deprecated. +// TODO(henrik.lundin) Delete. +void NetEqImpl::SetPlayoutMode(NetEqPlayoutMode mode) { + rtc::CritScope lock(&crit_sect_); + if (mode != playout_mode_) { + playout_mode_ = mode; + CreateDecisionLogic(); + } +} + +// Deprecated. +// TODO(henrik.lundin) Delete. +NetEqPlayoutMode NetEqImpl::PlayoutMode() const { + rtc::CritScope lock(&crit_sect_); + return playout_mode_; +} + +int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) { + rtc::CritScope lock(&crit_sect_); + assert(decoder_database_.get()); + const size_t total_samples_in_buffers = + packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) + + sync_buffer_->FutureLength(); + assert(delay_manager_.get()); + assert(decision_logic_.get()); + const int ms_per_packet = rtc::dchecked_cast( + decision_logic_->packet_length_samples() / (fs_hz_ / 1000)); + stats_.PopulateDelayManagerStats(ms_per_packet, *delay_manager_.get(), stats); + stats_.GetNetworkStatistics(fs_hz_, total_samples_in_buffers, + decoder_frame_length_, stats); + return 0; +} + +NetEqLifetimeStatistics NetEqImpl::GetLifetimeStatistics() const { + rtc::CritScope lock(&crit_sect_); + return stats_.GetLifetimeStatistics(); +} + +void NetEqImpl::GetRtcpStatistics(RtcpStatistics* stats) { + rtc::CritScope lock(&crit_sect_); + if (stats) { + rtcp_.GetStatistics(false, stats); + } +} + +void NetEqImpl::GetRtcpStatisticsNoReset(RtcpStatistics* stats) { + rtc::CritScope lock(&crit_sect_); + if (stats) { + rtcp_.GetStatistics(true, stats); + } +} + +void NetEqImpl::EnableVad() { + rtc::CritScope lock(&crit_sect_); + assert(vad_.get()); + vad_->Enable(); +} + +void NetEqImpl::DisableVad() { + rtc::CritScope lock(&crit_sect_); + assert(vad_.get()); + vad_->Disable(); +} + +rtc::Optional NetEqImpl::GetPlayoutTimestamp() const { + rtc::CritScope lock(&crit_sect_); + if (first_packet_ || last_mode_ == kModeRfc3389Cng || + last_mode_ == kModeCodecInternalCng) { + // We don't have a valid RTP timestamp until we have decoded our first + // RTP packet. Also, the RTP timestamp is not accurate while playing CNG, + // which is indicated by returning an empty value. + return rtc::nullopt; + } + return timestamp_scaler_->ToExternal(playout_timestamp_); +} + +int NetEqImpl::last_output_sample_rate_hz() const { + return last_output_sample_rate_hz_; +} + +rtc::Optional NetEqImpl::GetDecoder(int payload_type) const { + rtc::CritScope lock(&crit_sect_); + const DecoderDatabase::DecoderInfo* di = + decoder_database_->GetDecoderInfo(payload_type); + if (!di) { + return rtc::nullopt; + } + + // Create a CodecInst with some fields set. The remaining fields are zeroed, + // but we tell MSan to consider them uninitialized. + CodecInst ci = {0}; + rtc::MsanMarkUninitialized(rtc::MakeArrayView(&ci, 1)); + ci.pltype = payload_type; + std::strncpy(ci.plname, di->get_name().c_str(), sizeof(ci.plname)); + ci.plname[sizeof(ci.plname) - 1] = '\0'; + ci.plfreq = di->IsRed() ? 8000 : di->SampleRateHz(); + AudioDecoder* const decoder = di->GetDecoder(); + ci.channels = decoder ? decoder->Channels() : 1; + return ci; +} + +rtc::Optional NetEqImpl::GetDecoderFormat( + int payload_type) const { + rtc::CritScope lock(&crit_sect_); + const DecoderDatabase::DecoderInfo* const di = + decoder_database_->GetDecoderInfo(payload_type); + if (!di) { + return rtc::nullopt; // Payload type not registered. + } + return di->GetFormat(); +} + +int NetEqImpl::SetTargetNumberOfChannels() { + return kNotImplemented; +} + +int NetEqImpl::SetTargetSampleRate() { + return kNotImplemented; +} + +void NetEqImpl::FlushBuffers() { + rtc::CritScope lock(&crit_sect_); + RTC_LOG(LS_VERBOSE) << "FlushBuffers"; + packet_buffer_->Flush(); + assert(sync_buffer_.get()); + assert(expand_.get()); + sync_buffer_->Flush(); + sync_buffer_->set_next_index(sync_buffer_->next_index() - + expand_->overlap_length()); + // Set to wait for new codec. + first_packet_ = true; +} + +void NetEqImpl::PacketBufferStatistics(int* current_num_packets, + int* max_num_packets) const { + rtc::CritScope lock(&crit_sect_); + packet_buffer_->BufferStat(current_num_packets, max_num_packets); +} + +void NetEqImpl::EnableNack(size_t max_nack_list_size) { + rtc::CritScope lock(&crit_sect_); + if (!nack_enabled_) { + const int kNackThresholdPackets = 2; + nack_.reset(NackTracker::Create(kNackThresholdPackets)); + nack_enabled_ = true; + nack_->UpdateSampleRate(fs_hz_); + } + nack_->SetMaxNackListSize(max_nack_list_size); +} + +void NetEqImpl::DisableNack() { + rtc::CritScope lock(&crit_sect_); + nack_.reset(); + nack_enabled_ = false; +} + +std::vector NetEqImpl::GetNackList(int64_t round_trip_time_ms) const { + rtc::CritScope lock(&crit_sect_); + if (!nack_enabled_) { + return std::vector(); + } + RTC_DCHECK(nack_.get()); + return nack_->GetNackList(round_trip_time_ms); +} + +std::vector NetEqImpl::LastDecodedTimestamps() const { + rtc::CritScope lock(&crit_sect_); + return last_decoded_timestamps_; +} + +int NetEqImpl::SyncBufferSizeMs() const { + rtc::CritScope lock(&crit_sect_); + return rtc::dchecked_cast(sync_buffer_->FutureLength() / + rtc::CheckedDivExact(fs_hz_, 1000)); +} + +const SyncBuffer* NetEqImpl::sync_buffer_for_test() const { + rtc::CritScope lock(&crit_sect_); + return sync_buffer_.get(); +} + +Operations NetEqImpl::last_operation_for_test() const { + rtc::CritScope lock(&crit_sect_); + return last_operation_; +} + +// Methods below this line are private. + +int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) { + if (payload.empty()) { + RTC_LOG_F(LS_ERROR) << "payload is empty"; + return kInvalidPointer; + } + + PacketList packet_list; + // Insert packet in a packet list. + packet_list.push_back([&rtp_header, &payload] { + // Convert to Packet. + Packet packet; + packet.payload_type = rtp_header.payloadType; + packet.sequence_number = rtp_header.sequenceNumber; + packet.timestamp = rtp_header.timestamp; + packet.payload.SetData(payload.data(), payload.size()); + // Waiting time will be set upon inserting the packet in the buffer. + RTC_DCHECK(!packet.waiting_time); + return packet; + }()); + + bool update_sample_rate_and_channels = + first_packet_ || (rtp_header.ssrc != ssrc_); + + if (update_sample_rate_and_channels) { + // Reset timestamp scaling. + timestamp_scaler_->Reset(); + } + + if (!decoder_database_->IsRed(rtp_header.payloadType)) { + // Scale timestamp to internal domain (only for some codecs). + timestamp_scaler_->ToInternal(&packet_list); + } + + // Store these for later use, since the first packet may very well disappear + // before we need these values. + uint32_t main_timestamp = packet_list.front().timestamp; + uint8_t main_payload_type = packet_list.front().payload_type; + uint16_t main_sequence_number = packet_list.front().sequence_number; + + // Reinitialize NetEq if it's needed (changed SSRC or first call). + if (update_sample_rate_and_channels) { + // Note: |first_packet_| will be cleared further down in this method, once + // the packet has been successfully inserted into the packet buffer. + + rtcp_.Init(rtp_header.sequenceNumber); + + // Flush the packet buffer and DTMF buffer. + packet_buffer_->Flush(); + dtmf_buffer_->Flush(); + + // Store new SSRC. + ssrc_ = rtp_header.ssrc; + + // Update audio buffer timestamp. + sync_buffer_->IncreaseEndTimestamp(main_timestamp - timestamp_); + + // Update codecs. + timestamp_ = main_timestamp; + } + + // Update RTCP statistics, only for regular packets. + rtcp_.Update(rtp_header, receive_timestamp); + + if (nack_enabled_) { + RTC_DCHECK(nack_); + if (update_sample_rate_and_channels) { + nack_->Reset(); + } + nack_->UpdateLastReceivedPacket(rtp_header.sequenceNumber, + rtp_header.timestamp); + } + + // Check for RED payload type, and separate payloads into several packets. + if (decoder_database_->IsRed(rtp_header.payloadType)) { + if (!red_payload_splitter_->SplitRed(&packet_list)) { + return kRedundancySplitError; + } + // Only accept a few RED payloads of the same type as the main data, + // DTMF events and CNG. + red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); + } + + // Check payload types. + if (decoder_database_->CheckPayloadTypes(packet_list) == + DecoderDatabase::kDecoderNotFound) { + return kUnknownRtpPayloadType; + } + + RTC_DCHECK(!packet_list.empty()); + + // Update main_timestamp, if new packets appear in the list + // after RED splitting. + if (decoder_database_->IsRed(rtp_header.payloadType)) { + timestamp_scaler_->ToInternal(&packet_list); + main_timestamp = packet_list.front().timestamp; + main_payload_type = packet_list.front().payload_type; + main_sequence_number = packet_list.front().sequence_number; + } + + // Process DTMF payloads. Cycle through the list of packets, and pick out any + // DTMF payloads found. + PacketList::iterator it = packet_list.begin(); + while (it != packet_list.end()) { + const Packet& current_packet = (*it); + RTC_DCHECK(!current_packet.payload.empty()); + if (decoder_database_->IsDtmf(current_packet.payload_type)) { + DtmfEvent event; + int ret = DtmfBuffer::ParseEvent(current_packet.timestamp, + current_packet.payload.data(), + current_packet.payload.size(), &event); + if (ret != DtmfBuffer::kOK) { + return kDtmfParsingError; + } + if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) { + return kDtmfInsertError; + } + it = packet_list.erase(it); + } else { + ++it; + } + } + + // Update bandwidth estimate, if the packet is not comfort noise. + if (!packet_list.empty() && + !decoder_database_->IsComfortNoise(main_payload_type)) { + // The list can be empty here if we got nothing but DTMF payloads. + AudioDecoder* decoder = decoder_database_->GetDecoder(main_payload_type); + RTC_DCHECK(decoder); // Should always get a valid object, since we have + // already checked that the payload types are known. + decoder->IncomingPacket(packet_list.front().payload.data(), + packet_list.front().payload.size(), + packet_list.front().sequence_number, + packet_list.front().timestamp, + receive_timestamp); + } + + PacketList parsed_packet_list; + while (!packet_list.empty()) { + Packet& packet = packet_list.front(); + const DecoderDatabase::DecoderInfo* info = + decoder_database_->GetDecoderInfo(packet.payload_type); + if (!info) { + RTC_LOG(LS_WARNING) << "SplitAudio unknown payload type"; + return kUnknownRtpPayloadType; + } + + if (info->IsComfortNoise()) { + // Carry comfort noise packets along. + parsed_packet_list.splice(parsed_packet_list.end(), packet_list, + packet_list.begin()); + } else { + const auto sequence_number = packet.sequence_number; + const auto payload_type = packet.payload_type; + const Packet::Priority original_priority = packet.priority; + auto packet_from_result = [&] (AudioDecoder::ParseResult& result) { + Packet new_packet; + new_packet.sequence_number = sequence_number; + new_packet.payload_type = payload_type; + new_packet.timestamp = result.timestamp; + new_packet.priority.codec_level = result.priority; + new_packet.priority.red_level = original_priority.red_level; + new_packet.frame = std::move(result.frame); + return new_packet; + }; + + std::vector results = + info->GetDecoder()->ParsePayload(std::move(packet.payload), + packet.timestamp); + if (results.empty()) { + packet_list.pop_front(); + } else { + bool first = true; + for (auto& result : results) { + RTC_DCHECK(result.frame); + RTC_DCHECK_GE(result.priority, 0); + if (first) { + // Re-use the node and move it to parsed_packet_list. + packet_list.front() = packet_from_result(result); + parsed_packet_list.splice(parsed_packet_list.end(), packet_list, + packet_list.begin()); + first = false; + } else { + parsed_packet_list.push_back(packet_from_result(result)); + } + } + } + } + } + + // Calculate the number of primary (non-FEC/RED) packets. + const int number_of_primary_packets = std::count_if( + parsed_packet_list.begin(), parsed_packet_list.end(), + [](const Packet& in) { return in.priority.codec_level == 0; }); + + // Insert packets in buffer. + const int ret = packet_buffer_->InsertPacketList( + &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_, + ¤t_cng_rtp_payload_type_, &stats_); + if (ret == PacketBuffer::kFlushed) { + // Reset DSP timestamp etc. if packet buffer flushed. + new_codec_ = true; + update_sample_rate_and_channels = true; + } else if (ret != PacketBuffer::kOK) { + return kOtherError; + } + + if (first_packet_) { + first_packet_ = false; + // Update the codec on the next GetAudio call. + new_codec_ = true; + } + + if (current_rtp_payload_type_) { + RTC_DCHECK(decoder_database_->GetDecoderInfo(*current_rtp_payload_type_)) + << "Payload type " << static_cast(*current_rtp_payload_type_) + << " is unknown where it shouldn't be"; + } + + if (update_sample_rate_and_channels && !packet_buffer_->Empty()) { + // We do not use |current_rtp_payload_type_| to |set payload_type|, but + // get the next RTP header from |packet_buffer_| to obtain the payload type. + // The reason for it is the following corner case. If NetEq receives a + // CNG packet with a sample rate different than the current CNG then it + // flushes its buffer, assuming send codec must have been changed. However, + // payload type of the hypothetically new send codec is not known. + const Packet* next_packet = packet_buffer_->PeekNextPacket(); + RTC_DCHECK(next_packet); + const int payload_type = next_packet->payload_type; + size_t channels = 1; + if (!decoder_database_->IsComfortNoise(payload_type)) { + AudioDecoder* decoder = decoder_database_->GetDecoder(payload_type); + assert(decoder); // Payloads are already checked to be valid. + channels = decoder->Channels(); + } + const DecoderDatabase::DecoderInfo* decoder_info = + decoder_database_->GetDecoderInfo(payload_type); + assert(decoder_info); + if (decoder_info->SampleRateHz() != fs_hz_ || + channels != algorithm_buffer_->Channels()) { + SetSampleRateAndChannels(decoder_info->SampleRateHz(), + channels); + } + if (nack_enabled_) { + RTC_DCHECK(nack_); + // Update the sample rate even if the rate is not new, because of Reset(). + nack_->UpdateSampleRate(fs_hz_); + } + } + + // TODO(hlundin): Move this code to DelayManager class. + const DecoderDatabase::DecoderInfo* dec_info = + decoder_database_->GetDecoderInfo(main_payload_type); + assert(dec_info); // Already checked that the payload type is known. + delay_manager_->LastDecodedWasCngOrDtmf(dec_info->IsComfortNoise() || + dec_info->IsDtmf()); + if (delay_manager_->last_pack_cng_or_dtmf() == 0) { + // Calculate the total speech length carried in each packet. + if (number_of_primary_packets > 0) { + const size_t packet_length_samples = + number_of_primary_packets * decoder_frame_length_; + if (packet_length_samples != decision_logic_->packet_length_samples()) { + decision_logic_->set_packet_length_samples(packet_length_samples); + delay_manager_->SetPacketAudioLength( + rtc::dchecked_cast((1000 * packet_length_samples) / fs_hz_)); + } + } + + // Update statistics. + if ((int32_t)(main_timestamp - timestamp_) >= 0 && !new_codec_) { + // Only update statistics if incoming packet is not older than last played + // out packet, and if new codec flag is not set. + delay_manager_->Update(main_sequence_number, main_timestamp, fs_hz_); + } + } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) { + // This is first "normal" packet after CNG or DTMF. + // Reset packet time counter and measure time until next packet, + // but don't update statistics. + delay_manager_->set_last_pack_cng_or_dtmf(0); + delay_manager_->ResetPacketIatCount(); + } + return 0; +} + +int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame, bool* muted) { + PacketList packet_list; + DtmfEvent dtmf_event; + Operations operation; + bool play_dtmf; + *muted = false; + last_decoded_timestamps_.clear(); + tick_timer_->Increment(); + stats_.IncreaseCounter(output_size_samples_, fs_hz_); + + // Check for muted state. + if (enable_muted_state_ && expand_->Muted() && packet_buffer_->Empty()) { + RTC_DCHECK_EQ(last_mode_, kModeExpand); + audio_frame->Reset(); + RTC_DCHECK(audio_frame->muted()); // Reset() should mute the frame. + playout_timestamp_ += static_cast(output_size_samples_); + audio_frame->sample_rate_hz_ = fs_hz_; + audio_frame->samples_per_channel_ = output_size_samples_; + audio_frame->timestamp_ = + first_packet_ + ? 0 + : timestamp_scaler_->ToExternal(playout_timestamp_) - + static_cast(audio_frame->samples_per_channel_); + audio_frame->num_channels_ = sync_buffer_->Channels(); + stats_.ExpandedNoiseSamples(output_size_samples_, false); + *muted = true; + return 0; + } + + int return_value = GetDecision(&operation, &packet_list, &dtmf_event, + &play_dtmf); + if (return_value != 0) { + last_mode_ = kModeError; + return return_value; + } + + AudioDecoder::SpeechType speech_type; + int length = 0; + const size_t start_num_packets = packet_list.size(); + int decode_return_value = Decode(&packet_list, &operation, + &length, &speech_type); + + assert(vad_.get()); + bool sid_frame_available = + (operation == kRfc3389Cng && !packet_list.empty()); + vad_->Update(decoded_buffer_.get(), static_cast(length), speech_type, + sid_frame_available, fs_hz_); + + // This is the criterion that we did decode some data through the speech + // decoder, and the operation resulted in comfort noise. + const bool codec_internal_sid_frame = + use_dtx_delay_fix_ ? (speech_type == AudioDecoder::kComfortNoise && + start_num_packets > packet_list.size()) + : (speech_type == AudioDecoder::kComfortNoise); + + if (sid_frame_available || codec_internal_sid_frame) { + // Start a new stopwatch since we are decoding a new CNG packet. + generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + + algorithm_buffer_->Clear(); + switch (operation) { + case kNormal: { + DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf); + break; + } + case kMerge: { + DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf); + break; + } + case kExpand: { + return_value = DoExpand(play_dtmf); + break; + } + case kAccelerate: + case kFastAccelerate: { + const bool fast_accelerate = + enable_fast_accelerate_ && (operation == kFastAccelerate); + return_value = DoAccelerate(decoded_buffer_.get(), length, speech_type, + play_dtmf, fast_accelerate); + break; + } + case kPreemptiveExpand: { + return_value = DoPreemptiveExpand(decoded_buffer_.get(), length, + speech_type, play_dtmf); + break; + } + case kRfc3389Cng: + case kRfc3389CngNoPacket: { + return_value = DoRfc3389Cng(&packet_list, play_dtmf); + break; + } + case kCodecInternalCng: { + // This handles the case when there is no transmission and the decoder + // should produce internal comfort noise. + // TODO(hlundin): Write test for codec-internal CNG. + DoCodecInternalCng(decoded_buffer_.get(), length); + break; + } + case kDtmf: { + // TODO(hlundin): Write test for this. + return_value = DoDtmf(dtmf_event, &play_dtmf); + break; + } + case kAlternativePlc: { + // TODO(hlundin): Write test for this. + DoAlternativePlc(false); + break; + } + case kAlternativePlcIncreaseTimestamp: { + // TODO(hlundin): Write test for this. + DoAlternativePlc(true); + break; + } + case kAudioRepetitionIncreaseTimestamp: { + // TODO(hlundin): Write test for this. + sync_buffer_->IncreaseEndTimestamp( + static_cast(output_size_samples_)); + // Skipping break on purpose. Execution should move on into the + // next case. + FALLTHROUGH(); + } + case kAudioRepetition: { + // TODO(hlundin): Write test for this. + // Copy last |output_size_samples_| from |sync_buffer_| to + // |algorithm_buffer|. + algorithm_buffer_->PushBackFromIndex( + *sync_buffer_, sync_buffer_->Size() - output_size_samples_); + expand_->Reset(); + break; + } + case kUndefined: { + RTC_LOG(LS_ERROR) << "Invalid operation kUndefined."; + assert(false); // This should not happen. + last_mode_ = kModeError; + return kInvalidOperation; + } + } // End of switch. + last_operation_ = operation; + if (return_value < 0) { + return return_value; + } + + if (last_mode_ != kModeRfc3389Cng) { + comfort_noise_->Reset(); + } + + // Copy from |algorithm_buffer| to |sync_buffer_|. + sync_buffer_->PushBack(*algorithm_buffer_); + + // Extract data from |sync_buffer_| to |output|. + size_t num_output_samples_per_channel = output_size_samples_; + size_t num_output_samples = output_size_samples_ * sync_buffer_->Channels(); + if (num_output_samples > AudioFrame::kMaxDataSizeSamples) { + RTC_LOG(LS_WARNING) << "Output array is too short. " + << AudioFrame::kMaxDataSizeSamples << " < " + << output_size_samples_ << " * " + << sync_buffer_->Channels(); + num_output_samples = AudioFrame::kMaxDataSizeSamples; + num_output_samples_per_channel = + AudioFrame::kMaxDataSizeSamples / sync_buffer_->Channels(); + } + sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel, + audio_frame); + audio_frame->sample_rate_hz_ = fs_hz_; + if (sync_buffer_->FutureLength() < expand_->overlap_length()) { + // The sync buffer should always contain |overlap_length| samples, but now + // too many samples have been extracted. Reinstall the |overlap_length| + // lookahead by moving the index. + const size_t missing_lookahead_samples = + expand_->overlap_length() - sync_buffer_->FutureLength(); + RTC_DCHECK_GE(sync_buffer_->next_index(), missing_lookahead_samples); + sync_buffer_->set_next_index(sync_buffer_->next_index() - + missing_lookahead_samples); + } + if (audio_frame->samples_per_channel_ != output_size_samples_) { + RTC_LOG(LS_ERROR) << "audio_frame->samples_per_channel_ (" + << audio_frame->samples_per_channel_ + << ") != output_size_samples_ (" << output_size_samples_ + << ")"; + // TODO(minyue): treatment of under-run, filling zeros + audio_frame->Mute(); + return kSampleUnderrun; + } + + // Should always have overlap samples left in the |sync_buffer_|. + RTC_DCHECK_GE(sync_buffer_->FutureLength(), expand_->overlap_length()); + + // TODO(yujo): For muted frames, this can be a copy rather than an addition. + if (play_dtmf) { + return_value = DtmfOverdub(dtmf_event, sync_buffer_->Channels(), + audio_frame->mutable_data()); + } + + // Update the background noise parameters if last operation wrote data + // straight from the decoder to the |sync_buffer_|. That is, none of the + // operations that modify the signal can be followed by a parameter update. + if ((last_mode_ == kModeNormal) || + (last_mode_ == kModeAccelerateFail) || + (last_mode_ == kModePreemptiveExpandFail) || + (last_mode_ == kModeRfc3389Cng) || + (last_mode_ == kModeCodecInternalCng)) { + background_noise_->Update(*sync_buffer_, *vad_.get()); + } + + if (operation == kDtmf) { + // DTMF data was written the end of |sync_buffer_|. + // Update index to end of DTMF data in |sync_buffer_|. + sync_buffer_->set_dtmf_index(sync_buffer_->Size()); + } + + if (last_mode_ != kModeExpand) { + // If last operation was not expand, calculate the |playout_timestamp_| from + // the |sync_buffer_|. However, do not update the |playout_timestamp_| if it + // would be moved "backwards". + uint32_t temp_timestamp = sync_buffer_->end_timestamp() - + static_cast(sync_buffer_->FutureLength()); + if (static_cast(temp_timestamp - playout_timestamp_) > 0) { + playout_timestamp_ = temp_timestamp; + } + } else { + // Use dead reckoning to estimate the |playout_timestamp_|. + playout_timestamp_ += static_cast(output_size_samples_); + } + // Set the timestamp in the audio frame to zero before the first packet has + // been inserted. Otherwise, subtract the frame size in samples to get the + // timestamp of the first sample in the frame (playout_timestamp_ is the + // last + 1). + audio_frame->timestamp_ = + first_packet_ + ? 0 + : timestamp_scaler_->ToExternal(playout_timestamp_) - + static_cast(audio_frame->samples_per_channel_); + + if (!(last_mode_ == kModeRfc3389Cng || + last_mode_ == kModeCodecInternalCng || + last_mode_ == kModeExpand)) { + generated_noise_stopwatch_.reset(); + } + + if (decode_return_value) return decode_return_value; + return return_value; +} + +int NetEqImpl::GetDecision(Operations* operation, + PacketList* packet_list, + DtmfEvent* dtmf_event, + bool* play_dtmf) { + // Initialize output variables. + *play_dtmf = false; + *operation = kUndefined; + + assert(sync_buffer_.get()); + uint32_t end_timestamp = sync_buffer_->end_timestamp(); + if (!new_codec_) { + const uint32_t five_seconds_samples = 5 * fs_hz_; + packet_buffer_->DiscardOldPackets(end_timestamp, five_seconds_samples, + &stats_); + } + const Packet* packet = packet_buffer_->PeekNextPacket(); + + RTC_DCHECK(!generated_noise_stopwatch_ || + generated_noise_stopwatch_->ElapsedTicks() >= 1); + uint64_t generated_noise_samples = + generated_noise_stopwatch_ + ? (generated_noise_stopwatch_->ElapsedTicks() - 1) * + output_size_samples_ + + decision_logic_->noise_fast_forward() + : 0; + + if (decision_logic_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) { + // Because of timestamp peculiarities, we have to "manually" disallow using + // a CNG packet with the same timestamp as the one that was last played. + // This can happen when using redundancy and will cause the timing to shift. + while (packet && decoder_database_->IsComfortNoise(packet->payload_type) && + (end_timestamp >= packet->timestamp || + end_timestamp + generated_noise_samples > packet->timestamp)) { + // Don't use this packet, discard it. + if (packet_buffer_->DiscardNextPacket(&stats_) != PacketBuffer::kOK) { + assert(false); // Must be ok by design. + } + // Check buffer again. + if (!new_codec_) { + packet_buffer_->DiscardOldPackets(end_timestamp, 5 * fs_hz_, &stats_); + } + packet = packet_buffer_->PeekNextPacket(); + } + } + + assert(expand_.get()); + const int samples_left = static_cast(sync_buffer_->FutureLength() - + expand_->overlap_length()); + if (last_mode_ == kModeAccelerateSuccess || + last_mode_ == kModeAccelerateLowEnergy || + last_mode_ == kModePreemptiveExpandSuccess || + last_mode_ == kModePreemptiveExpandLowEnergy) { + // Subtract (samples_left + output_size_samples_) from sampleMemory. + decision_logic_->AddSampleMemory( + -(samples_left + rtc::dchecked_cast(output_size_samples_))); + } + + // Check if it is time to play a DTMF event. + if (dtmf_buffer_->GetEvent( + static_cast( + end_timestamp + generated_noise_samples), + dtmf_event)) { + *play_dtmf = true; + } + + // Get instruction. + assert(sync_buffer_.get()); + assert(expand_.get()); + generated_noise_samples = + generated_noise_stopwatch_ + ? generated_noise_stopwatch_->ElapsedTicks() * output_size_samples_ + + decision_logic_->noise_fast_forward() + : 0; + *operation = decision_logic_->GetDecision( + *sync_buffer_, *expand_, decoder_frame_length_, packet, last_mode_, + *play_dtmf, generated_noise_samples, &reset_decoder_); + + // Check if we already have enough samples in the |sync_buffer_|. If so, + // change decision to normal, unless the decision was merge, accelerate, or + // preemptive expand. + if (samples_left >= rtc::dchecked_cast(output_size_samples_) && + *operation != kMerge && *operation != kAccelerate && + *operation != kFastAccelerate && *operation != kPreemptiveExpand) { + *operation = kNormal; + return 0; + } + + decision_logic_->ExpandDecision(*operation); + + // Check conditions for reset. + if (new_codec_ || *operation == kUndefined) { + // The only valid reason to get kUndefined is that new_codec_ is set. + assert(new_codec_); + if (*play_dtmf && !packet) { + timestamp_ = dtmf_event->timestamp; + } else { + if (!packet) { + RTC_LOG(LS_ERROR) << "Packet missing where it shouldn't."; + return -1; + } + timestamp_ = packet->timestamp; + if (*operation == kRfc3389CngNoPacket && + decoder_database_->IsComfortNoise(packet->payload_type)) { + // Change decision to CNG packet, since we do have a CNG packet, but it + // was considered too early to use. Now, use it anyway. + *operation = kRfc3389Cng; + } else if (*operation != kRfc3389Cng) { + *operation = kNormal; + } + } + // Adjust |sync_buffer_| timestamp before setting |end_timestamp| to the + // new value. + sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp); + end_timestamp = timestamp_; + new_codec_ = false; + decision_logic_->SoftReset(); + buffer_level_filter_->Reset(); + delay_manager_->Reset(); + stats_.ResetMcu(); + } + + size_t required_samples = output_size_samples_; + const size_t samples_10_ms = static_cast(80 * fs_mult_); + const size_t samples_20_ms = 2 * samples_10_ms; + const size_t samples_30_ms = 3 * samples_10_ms; + + switch (*operation) { + case kExpand: { + timestamp_ = end_timestamp; + return 0; + } + case kRfc3389CngNoPacket: + case kCodecInternalCng: { + return 0; + } + case kDtmf: { + // TODO(hlundin): Write test for this. + // Update timestamp. + timestamp_ = end_timestamp; + const uint64_t generated_noise_samples = + generated_noise_stopwatch_ + ? generated_noise_stopwatch_->ElapsedTicks() * + output_size_samples_ + + decision_logic_->noise_fast_forward() + : 0; + if (generated_noise_samples > 0 && last_mode_ != kModeDtmf) { + // Make a jump in timestamp due to the recently played comfort noise. + uint32_t timestamp_jump = + static_cast(generated_noise_samples); + sync_buffer_->IncreaseEndTimestamp(timestamp_jump); + timestamp_ += timestamp_jump; + } + return 0; + } + case kAccelerate: + case kFastAccelerate: { + // In order to do an accelerate we need at least 30 ms of audio data. + if (samples_left >= static_cast(samples_30_ms)) { + // Already have enough data, so we do not need to extract any more. + decision_logic_->set_sample_memory(samples_left); + decision_logic_->set_prev_time_scale(true); + return 0; + } else if (samples_left >= static_cast(samples_10_ms) && + decoder_frame_length_ >= samples_30_ms) { + // Avoid decoding more data as it might overflow the playout buffer. + *operation = kNormal; + return 0; + } else if (samples_left < static_cast(samples_20_ms) && + decoder_frame_length_ < samples_30_ms) { + // Build up decoded data by decoding at least 20 ms of audio data. Do + // not perform accelerate yet, but wait until we only need to do one + // decoding. + required_samples = 2 * output_size_samples_; + *operation = kNormal; + } + // If none of the above is true, we have one of two possible situations: + // (1) 20 ms <= samples_left < 30 ms and decoder_frame_length_ < 30 ms; or + // (2) samples_left < 10 ms and decoder_frame_length_ >= 30 ms. + // In either case, we move on with the accelerate decision, and decode one + // frame now. + break; + } + case kPreemptiveExpand: { + // In order to do a preemptive expand we need at least 30 ms of decoded + // audio data. + if ((samples_left >= static_cast(samples_30_ms)) || + (samples_left >= static_cast(samples_10_ms) && + decoder_frame_length_ >= samples_30_ms)) { + // Already have enough data, so we do not need to extract any more. + // Or, avoid decoding more data as it might overflow the playout buffer. + // Still try preemptive expand, though. + decision_logic_->set_sample_memory(samples_left); + decision_logic_->set_prev_time_scale(true); + return 0; + } + if (samples_left < static_cast(samples_20_ms) && + decoder_frame_length_ < samples_30_ms) { + // Build up decoded data by decoding at least 20 ms of audio data. + // Still try to perform preemptive expand. + required_samples = 2 * output_size_samples_; + } + // Move on with the preemptive expand decision. + break; + } + case kMerge: { + required_samples = + std::max(merge_->RequiredFutureSamples(), required_samples); + break; + } + default: { + // Do nothing. + } + } + + // Get packets from buffer. + int extracted_samples = 0; + if (packet && *operation != kAlternativePlc && + *operation != kAlternativePlcIncreaseTimestamp && + *operation != kAudioRepetition && + *operation != kAudioRepetitionIncreaseTimestamp) { + sync_buffer_->IncreaseEndTimestamp(packet->timestamp - end_timestamp); + if (decision_logic_->CngOff()) { + // Adjustment of timestamp only corresponds to an actual packet loss + // if comfort noise is not played. If comfort noise was just played, + // this adjustment of timestamp is only done to get back in sync with the + // stream timestamp; no loss to report. + stats_.LostSamples(packet->timestamp - end_timestamp); + } + + if (*operation != kRfc3389Cng) { + // We are about to decode and use a non-CNG packet. + decision_logic_->SetCngOff(); + } + + extracted_samples = ExtractPackets(required_samples, packet_list); + if (extracted_samples < 0) { + return kPacketBufferCorruption; + } + } + + if (*operation == kAccelerate || *operation == kFastAccelerate || + *operation == kPreemptiveExpand) { + decision_logic_->set_sample_memory(samples_left + extracted_samples); + decision_logic_->set_prev_time_scale(true); + } + + if (*operation == kAccelerate || *operation == kFastAccelerate) { + // Check that we have enough data (30ms) to do accelerate. + if (extracted_samples + samples_left < static_cast(samples_30_ms)) { + // TODO(hlundin): Write test for this. + // Not enough, do normal operation instead. + *operation = kNormal; + } + } + + timestamp_ = end_timestamp; + return 0; +} + +int NetEqImpl::Decode(PacketList* packet_list, Operations* operation, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) { + *speech_type = AudioDecoder::kSpeech; + + // When packet_list is empty, we may be in kCodecInternalCng mode, and for + // that we use current active decoder. + AudioDecoder* decoder = decoder_database_->GetActiveDecoder(); + + if (!packet_list->empty()) { + const Packet& packet = packet_list->front(); + uint8_t payload_type = packet.payload_type; + if (!decoder_database_->IsComfortNoise(payload_type)) { + decoder = decoder_database_->GetDecoder(payload_type); + assert(decoder); + if (!decoder) { + RTC_LOG(LS_WARNING) + << "Unknown payload type " << static_cast(payload_type); + packet_list->clear(); + return kDecoderNotFound; + } + bool decoder_changed; + decoder_database_->SetActiveDecoder(payload_type, &decoder_changed); + if (decoder_changed) { + // We have a new decoder. Re-init some values. + const DecoderDatabase::DecoderInfo* decoder_info = decoder_database_ + ->GetDecoderInfo(payload_type); + assert(decoder_info); + if (!decoder_info) { + RTC_LOG(LS_WARNING) + << "Unknown payload type " << static_cast(payload_type); + packet_list->clear(); + return kDecoderNotFound; + } + // If sampling rate or number of channels has changed, we need to make + // a reset. + if (decoder_info->SampleRateHz() != fs_hz_ || + decoder->Channels() != algorithm_buffer_->Channels()) { + // TODO(tlegrand): Add unittest to cover this event. + SetSampleRateAndChannels(decoder_info->SampleRateHz(), + decoder->Channels()); + } + sync_buffer_->set_end_timestamp(timestamp_); + playout_timestamp_ = timestamp_; + } + } + } + + if (reset_decoder_) { + // TODO(hlundin): Write test for this. + if (decoder) + decoder->Reset(); + + // Reset comfort noise decoder. + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + if (cng_decoder) + cng_decoder->Reset(); + + reset_decoder_ = false; + } + + *decoded_length = 0; + // Update codec-internal PLC state. + if ((*operation == kMerge) && decoder && decoder->HasDecodePlc()) { + decoder->DecodePlc(1, &decoded_buffer_[*decoded_length]); + } + + int return_value; + if (*operation == kCodecInternalCng) { + RTC_DCHECK(packet_list->empty()); + return_value = DecodeCng(decoder, decoded_length, speech_type); + } else { + return_value = DecodeLoop(packet_list, *operation, decoder, + decoded_length, speech_type); + } + + if (*decoded_length < 0) { + // Error returned from the decoder. + *decoded_length = 0; + sync_buffer_->IncreaseEndTimestamp( + static_cast(decoder_frame_length_)); + int error_code = 0; + if (decoder) + error_code = decoder->ErrorCode(); + if (error_code != 0) { + // Got some error code from the decoder. + return_value = kDecoderErrorCode; + RTC_LOG(LS_WARNING) << "Decoder returned error code: " << error_code; + } else { + // Decoder does not implement error codes. Return generic error. + return_value = kOtherDecoderError; + RTC_LOG(LS_WARNING) << "Decoder error (no error code)"; + } + *operation = kExpand; // Do expansion to get data instead. + } + if (*speech_type != AudioDecoder::kComfortNoise) { + // Don't increment timestamp if codec returned CNG speech type + // since in this case, the we will increment the CNGplayedTS counter. + // Increase with number of samples per channel. + assert(*decoded_length == 0 || + (decoder && decoder->Channels() == sync_buffer_->Channels())); + sync_buffer_->IncreaseEndTimestamp( + *decoded_length / static_cast(sync_buffer_->Channels())); + } + return return_value; +} + +int NetEqImpl::DecodeCng(AudioDecoder* decoder, int* decoded_length, + AudioDecoder::SpeechType* speech_type) { + if (!decoder) { + // This happens when active decoder is not defined. + *decoded_length = -1; + return 0; + } + + while (*decoded_length < rtc::dchecked_cast(output_size_samples_)) { + const int length = decoder->Decode( + nullptr, 0, fs_hz_, + (decoded_buffer_length_ - *decoded_length) * sizeof(int16_t), + &decoded_buffer_[*decoded_length], speech_type); + if (length > 0) { + *decoded_length += length; + } else { + // Error. + RTC_LOG(LS_WARNING) << "Failed to decode CNG"; + *decoded_length = -1; + break; + } + if (*decoded_length > static_cast(decoded_buffer_length_)) { + // Guard against overflow. + RTC_LOG(LS_WARNING) << "Decoded too much CNG."; + return kDecodedTooMuch; + } + } + return 0; +} + +int NetEqImpl::DecodeLoop(PacketList* packet_list, const Operations& operation, + AudioDecoder* decoder, int* decoded_length, + AudioDecoder::SpeechType* speech_type) { + RTC_DCHECK(last_decoded_timestamps_.empty()); + + // Do decoding. + while ( + !packet_list->empty() && + !decoder_database_->IsComfortNoise(packet_list->front().payload_type)) { + assert(decoder); // At this point, we must have a decoder object. + // The number of channels in the |sync_buffer_| should be the same as the + // number decoder channels. + assert(sync_buffer_->Channels() == decoder->Channels()); + assert(decoded_buffer_length_ >= kMaxFrameSize * decoder->Channels()); + assert(operation == kNormal || operation == kAccelerate || + operation == kFastAccelerate || operation == kMerge || + operation == kPreemptiveExpand); + + auto opt_result = packet_list->front().frame->Decode( + rtc::ArrayView(&decoded_buffer_[*decoded_length], + decoded_buffer_length_ - *decoded_length)); + last_decoded_timestamps_.push_back(packet_list->front().timestamp); + packet_list->pop_front(); + if (opt_result) { + const auto& result = *opt_result; + *speech_type = result.speech_type; + if (result.num_decoded_samples > 0) { + *decoded_length += rtc::dchecked_cast(result.num_decoded_samples); + // Update |decoder_frame_length_| with number of samples per channel. + decoder_frame_length_ = + result.num_decoded_samples / decoder->Channels(); + } + } else { + // Error. + // TODO(ossu): What to put here? + RTC_LOG(LS_WARNING) << "Decode error"; + *decoded_length = -1; + packet_list->clear(); + break; + } + if (*decoded_length > rtc::dchecked_cast(decoded_buffer_length_)) { + // Guard against overflow. + RTC_LOG(LS_WARNING) << "Decoded too much."; + packet_list->clear(); + return kDecodedTooMuch; + } + } // End of decode loop. + + // If the list is not empty at this point, either a decoding error terminated + // the while-loop, or list must hold exactly one CNG packet. + assert( + packet_list->empty() || *decoded_length < 0 || + (packet_list->size() == 1 && + decoder_database_->IsComfortNoise(packet_list->front().payload_type))); + return 0; +} + +void NetEqImpl::DoNormal(const int16_t* decoded_buffer, size_t decoded_length, + AudioDecoder::SpeechType speech_type, bool play_dtmf) { + assert(normal_.get()); + assert(mute_factor_array_.get()); + normal_->Process(decoded_buffer, decoded_length, last_mode_, + mute_factor_array_.get(), algorithm_buffer_.get()); + if (decoded_length != 0) { + last_mode_ = kModeNormal; + } + + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if ((speech_type == AudioDecoder::kComfortNoise) + || ((last_mode_ == kModeCodecInternalCng) + && (decoded_length == 0))) { + // TODO(hlundin): Remove second part of || statement above. + last_mode_ = kModeCodecInternalCng; + } + + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } +} + +void NetEqImpl::DoMerge(int16_t* decoded_buffer, size_t decoded_length, + AudioDecoder::SpeechType speech_type, bool play_dtmf) { + assert(mute_factor_array_.get()); + assert(merge_.get()); + size_t new_length = merge_->Process(decoded_buffer, decoded_length, + mute_factor_array_.get(), + algorithm_buffer_.get()); + // Correction can be negative. + int expand_length_correction = + rtc::dchecked_cast(new_length) - + rtc::dchecked_cast(decoded_length / algorithm_buffer_->Channels()); + + // Update in-call and post-call statistics. + if (expand_->MuteFactor(0) == 0) { + // Expand generates only noise. + stats_.ExpandedNoiseSamplesCorrection(expand_length_correction); + } else { + // Expansion generates more than only noise. + stats_.ExpandedVoiceSamplesCorrection(expand_length_correction); + } + + last_mode_ = kModeMerge; + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if (speech_type == AudioDecoder::kComfortNoise) { + last_mode_ = kModeCodecInternalCng; + } + expand_->Reset(); + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } +} + +int NetEqImpl::DoExpand(bool play_dtmf) { + while ((sync_buffer_->FutureLength() - expand_->overlap_length()) < + output_size_samples_) { + algorithm_buffer_->Clear(); + int return_value = expand_->Process(algorithm_buffer_.get()); + size_t length = algorithm_buffer_->Size(); + bool is_new_concealment_event = (last_mode_ != kModeExpand); + + // Update in-call and post-call statistics. + if (expand_->MuteFactor(0) == 0) { + // Expand operation generates only noise. + stats_.ExpandedNoiseSamples(length, is_new_concealment_event); + } else { + // Expand operation generates more than only noise. + stats_.ExpandedVoiceSamples(length, is_new_concealment_event); + } + + last_mode_ = kModeExpand; + + if (return_value < 0) { + return return_value; + } + + sync_buffer_->PushBack(*algorithm_buffer_); + algorithm_buffer_->Clear(); + } + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + + if (!generated_noise_stopwatch_) { + // Start a new stopwatch since we may be covering for a lost CNG packet. + generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + + return 0; +} + +int NetEqImpl::DoAccelerate(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf, + bool fast_accelerate) { + const size_t required_samples = + static_cast(240 * fs_mult_); // Must have 30 ms. + size_t borrowed_samples_per_channel = 0; + size_t num_channels = algorithm_buffer_->Channels(); + size_t decoded_length_per_channel = decoded_length / num_channels; + if (decoded_length_per_channel < required_samples) { + // Must move data from the |sync_buffer_| in order to get 30 ms. + borrowed_samples_per_channel = static_cast(required_samples - + decoded_length_per_channel); + memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels], + decoded_buffer, + sizeof(int16_t) * decoded_length); + sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel, + decoded_buffer); + decoded_length = required_samples * num_channels; + } + + size_t samples_removed; + Accelerate::ReturnCodes return_code = + accelerate_->Process(decoded_buffer, decoded_length, fast_accelerate, + algorithm_buffer_.get(), &samples_removed); + stats_.AcceleratedSamples(samples_removed); + switch (return_code) { + case Accelerate::kSuccess: + last_mode_ = kModeAccelerateSuccess; + break; + case Accelerate::kSuccessLowEnergy: + last_mode_ = kModeAccelerateLowEnergy; + break; + case Accelerate::kNoStretch: + last_mode_ = kModeAccelerateFail; + break; + case Accelerate::kError: + // TODO(hlundin): Map to kModeError instead? + last_mode_ = kModeAccelerateFail; + return kAccelerateError; + } + + if (borrowed_samples_per_channel > 0) { + // Copy borrowed samples back to the |sync_buffer_|. + size_t length = algorithm_buffer_->Size(); + if (length < borrowed_samples_per_channel) { + // This destroys the beginning of the buffer, but will not cause any + // problems. + sync_buffer_->ReplaceAtIndex(*algorithm_buffer_, + sync_buffer_->Size() - + borrowed_samples_per_channel); + sync_buffer_->PushFrontZeros(borrowed_samples_per_channel - length); + algorithm_buffer_->PopFront(length); + assert(algorithm_buffer_->Empty()); + } else { + sync_buffer_->ReplaceAtIndex(*algorithm_buffer_, + borrowed_samples_per_channel, + sync_buffer_->Size() - + borrowed_samples_per_channel); + algorithm_buffer_->PopFront(borrowed_samples_per_channel); + } + } + + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if (speech_type == AudioDecoder::kComfortNoise) { + last_mode_ = kModeCodecInternalCng; + } + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + expand_->Reset(); + return 0; +} + +int NetEqImpl::DoPreemptiveExpand(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) { + const size_t required_samples = + static_cast(240 * fs_mult_); // Must have 30 ms. + size_t num_channels = algorithm_buffer_->Channels(); + size_t borrowed_samples_per_channel = 0; + size_t old_borrowed_samples_per_channel = 0; + size_t decoded_length_per_channel = decoded_length / num_channels; + if (decoded_length_per_channel < required_samples) { + // Must move data from the |sync_buffer_| in order to get 30 ms. + borrowed_samples_per_channel = + required_samples - decoded_length_per_channel; + // Calculate how many of these were already played out. + old_borrowed_samples_per_channel = + (borrowed_samples_per_channel > sync_buffer_->FutureLength()) ? + (borrowed_samples_per_channel - sync_buffer_->FutureLength()) : 0; + memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels], + decoded_buffer, + sizeof(int16_t) * decoded_length); + sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel, + decoded_buffer); + decoded_length = required_samples * num_channels; + } + + size_t samples_added; + PreemptiveExpand::ReturnCodes return_code = preemptive_expand_->Process( + decoded_buffer, decoded_length, + old_borrowed_samples_per_channel, + algorithm_buffer_.get(), &samples_added); + stats_.PreemptiveExpandedSamples(samples_added); + switch (return_code) { + case PreemptiveExpand::kSuccess: + last_mode_ = kModePreemptiveExpandSuccess; + break; + case PreemptiveExpand::kSuccessLowEnergy: + last_mode_ = kModePreemptiveExpandLowEnergy; + break; + case PreemptiveExpand::kNoStretch: + last_mode_ = kModePreemptiveExpandFail; + break; + case PreemptiveExpand::kError: + // TODO(hlundin): Map to kModeError instead? + last_mode_ = kModePreemptiveExpandFail; + return kPreemptiveExpandError; + } + + if (borrowed_samples_per_channel > 0) { + // Copy borrowed samples back to the |sync_buffer_|. + sync_buffer_->ReplaceAtIndex( + *algorithm_buffer_, borrowed_samples_per_channel, + sync_buffer_->Size() - borrowed_samples_per_channel); + algorithm_buffer_->PopFront(borrowed_samples_per_channel); + } + + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if (speech_type == AudioDecoder::kComfortNoise) { + last_mode_ = kModeCodecInternalCng; + } + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + expand_->Reset(); + return 0; +} + +int NetEqImpl::DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) { + if (!packet_list->empty()) { + // Must have exactly one SID frame at this point. + assert(packet_list->size() == 1); + const Packet& packet = packet_list->front(); + if (!decoder_database_->IsComfortNoise(packet.payload_type)) { + RTC_LOG(LS_ERROR) << "Trying to decode non-CNG payload as CNG."; + return kOtherError; + } + if (comfort_noise_->UpdateParameters(packet) == + ComfortNoise::kInternalError) { + algorithm_buffer_->Zeros(output_size_samples_); + return -comfort_noise_->internal_error_code(); + } + } + int cn_return = comfort_noise_->Generate(output_size_samples_, + algorithm_buffer_.get()); + expand_->Reset(); + last_mode_ = kModeRfc3389Cng; + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + if (cn_return == ComfortNoise::kInternalError) { + RTC_LOG(LS_WARNING) << "Comfort noise generator returned error code: " + << comfort_noise_->internal_error_code(); + return kComfortNoiseErrorCode; + } else if (cn_return == ComfortNoise::kUnknownPayloadType) { + return kUnknownRtpPayloadType; + } + return 0; +} + +void NetEqImpl::DoCodecInternalCng(const int16_t* decoded_buffer, + size_t decoded_length) { + RTC_DCHECK(normal_.get()); + RTC_DCHECK(mute_factor_array_.get()); + normal_->Process(decoded_buffer, decoded_length, last_mode_, + mute_factor_array_.get(), algorithm_buffer_.get()); + last_mode_ = kModeCodecInternalCng; + expand_->Reset(); +} + +int NetEqImpl::DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) { + // This block of the code and the block further down, handling |dtmf_switch| + // are commented out. Otherwise playing out-of-band DTMF would fail in VoE + // test, DtmfTest.ManualSuccessfullySendsOutOfBandTelephoneEvents. This is + // equivalent to |dtmf_switch| always be false. + // + // See http://webrtc-codereview.appspot.com/1195004/ for discussion + // On this issue. This change might cause some glitches at the point of + // switch from audio to DTMF. Issue 1545 is filed to track this. + // + // bool dtmf_switch = false; + // if ((last_mode_ != kModeDtmf) && dtmf_tone_generator_->initialized()) { + // // Special case; see below. + // // We must catch this before calling Generate, since |initialized| is + // // modified in that call. + // dtmf_switch = true; + // } + + int dtmf_return_value = 0; + if (!dtmf_tone_generator_->initialized()) { + // Initialize if not already done. + dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no, + dtmf_event.volume); + } + + if (dtmf_return_value == 0) { + // Generate DTMF signal. + dtmf_return_value = dtmf_tone_generator_->Generate(output_size_samples_, + algorithm_buffer_.get()); + } + + if (dtmf_return_value < 0) { + algorithm_buffer_->Zeros(output_size_samples_); + return dtmf_return_value; + } + + // if (dtmf_switch) { + // // This is the special case where the previous operation was DTMF + // // overdub, but the current instruction is "regular" DTMF. We must make + // // sure that the DTMF does not have any discontinuities. The first DTMF + // // sample that we generate now must be played out immediately, therefore + // // it must be copied to the speech buffer. + // // TODO(hlundin): This code seems incorrect. (Legacy.) Write test and + // // verify correct operation. + // assert(false); + // // Must generate enough data to replace all of the |sync_buffer_| + // // "future". + // int required_length = sync_buffer_->FutureLength(); + // assert(dtmf_tone_generator_->initialized()); + // dtmf_return_value = dtmf_tone_generator_->Generate(required_length, + // algorithm_buffer_); + // assert((size_t) required_length == algorithm_buffer_->Size()); + // if (dtmf_return_value < 0) { + // algorithm_buffer_->Zeros(output_size_samples_); + // return dtmf_return_value; + // } + // + // // Overwrite the "future" part of the speech buffer with the new DTMF + // // data. + // // TODO(hlundin): It seems that this overwriting has gone lost. + // // Not adapted for multi-channel yet. + // assert(algorithm_buffer_->Channels() == 1); + // if (algorithm_buffer_->Channels() != 1) { + // RTC_LOG(LS_WARNING) << "DTMF not supported for more than one channel"; + // return kStereoNotSupported; + // } + // // Shuffle the remaining data to the beginning of algorithm buffer. + // algorithm_buffer_->PopFront(sync_buffer_->FutureLength()); + // } + + sync_buffer_->IncreaseEndTimestamp( + static_cast(output_size_samples_)); + expand_->Reset(); + last_mode_ = kModeDtmf; + + // Set to false because the DTMF is already in the algorithm buffer. + *play_dtmf = false; + return 0; +} + +void NetEqImpl::DoAlternativePlc(bool increase_timestamp) { + AudioDecoder* decoder = decoder_database_->GetActiveDecoder(); + size_t length; + if (decoder && decoder->HasDecodePlc()) { + // Use the decoder's packet-loss concealment. + // TODO(hlundin): Will probably need a longer buffer for multi-channel. + int16_t decoded_buffer[kMaxFrameSize]; + length = decoder->DecodePlc(1, decoded_buffer); + if (length > 0) + algorithm_buffer_->PushBackInterleaved(decoded_buffer, length); + } else { + // Do simple zero-stuffing. + length = output_size_samples_; + algorithm_buffer_->Zeros(length); + // By not advancing the timestamp, NetEq inserts samples. + stats_.AddZeros(length); + } + if (increase_timestamp) { + sync_buffer_->IncreaseEndTimestamp(static_cast(length)); + } + expand_->Reset(); +} + +int NetEqImpl::DtmfOverdub(const DtmfEvent& dtmf_event, size_t num_channels, + int16_t* output) const { + size_t out_index = 0; + size_t overdub_length = output_size_samples_; // Default value. + + if (sync_buffer_->dtmf_index() > sync_buffer_->next_index()) { + // Special operation for transition from "DTMF only" to "DTMF overdub". + out_index = std::min( + sync_buffer_->dtmf_index() - sync_buffer_->next_index(), + output_size_samples_); + overdub_length = output_size_samples_ - out_index; + } + + AudioMultiVector dtmf_output(num_channels); + int dtmf_return_value = 0; + if (!dtmf_tone_generator_->initialized()) { + dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no, + dtmf_event.volume); + } + if (dtmf_return_value == 0) { + dtmf_return_value = dtmf_tone_generator_->Generate(overdub_length, + &dtmf_output); + assert(overdub_length == dtmf_output.Size()); + } + dtmf_output.ReadInterleaved(overdub_length, &output[out_index]); + return dtmf_return_value < 0 ? dtmf_return_value : 0; +} + +int NetEqImpl::ExtractPackets(size_t required_samples, + PacketList* packet_list) { + bool first_packet = true; + uint8_t prev_payload_type = 0; + uint32_t prev_timestamp = 0; + uint16_t prev_sequence_number = 0; + bool next_packet_available = false; + + const Packet* next_packet = packet_buffer_->PeekNextPacket(); + RTC_DCHECK(next_packet); + if (!next_packet) { + RTC_LOG(LS_ERROR) << "Packet buffer unexpectedly empty."; + return -1; + } + uint32_t first_timestamp = next_packet->timestamp; + size_t extracted_samples = 0; + + // Packet extraction loop. + do { + timestamp_ = next_packet->timestamp; + rtc::Optional packet = packet_buffer_->GetNextPacket(); + // |next_packet| may be invalid after the |packet_buffer_| operation. + next_packet = nullptr; + if (!packet) { + RTC_LOG(LS_ERROR) << "Should always be able to extract a packet here"; + assert(false); // Should always be able to extract a packet here. + return -1; + } + const uint64_t waiting_time_ms = packet->waiting_time->ElapsedMs(); + stats_.StoreWaitingTime(waiting_time_ms); + RTC_DCHECK(!packet->empty()); + + if (first_packet) { + first_packet = false; + if (nack_enabled_) { + RTC_DCHECK(nack_); + // TODO(henrik.lundin): Should we update this for all decoded packets? + nack_->UpdateLastDecodedPacket(packet->sequence_number, + packet->timestamp); + } + prev_sequence_number = packet->sequence_number; + prev_timestamp = packet->timestamp; + prev_payload_type = packet->payload_type; + } + + const bool has_cng_packet = + decoder_database_->IsComfortNoise(packet->payload_type); + // Store number of extracted samples. + size_t packet_duration = 0; + if (packet->frame) { + packet_duration = packet->frame->Duration(); + // TODO(ossu): Is this the correct way to track Opus FEC packets? + if (packet->priority.codec_level > 0) { + stats_.SecondaryDecodedSamples( + rtc::dchecked_cast(packet_duration)); + } + } else if (!has_cng_packet) { + RTC_LOG(LS_WARNING) << "Unknown payload type " + << static_cast(packet->payload_type); + RTC_NOTREACHED(); + } + + if (packet_duration == 0) { + // Decoder did not return a packet duration. Assume that the packet + // contains the same number of samples as the previous one. + packet_duration = decoder_frame_length_; + } + extracted_samples = packet->timestamp - first_timestamp + packet_duration; + + stats_.JitterBufferDelay(extracted_samples, waiting_time_ms); + + packet_list->push_back(std::move(*packet)); // Store packet in list. + packet = rtc::nullopt; // Ensure it's never used after the move. + + // Check what packet is available next. + next_packet = packet_buffer_->PeekNextPacket(); + next_packet_available = false; + if (next_packet && prev_payload_type == next_packet->payload_type && + !has_cng_packet) { + int16_t seq_no_diff = next_packet->sequence_number - prev_sequence_number; + size_t ts_diff = next_packet->timestamp - prev_timestamp; + if (seq_no_diff == 1 || + (seq_no_diff == 0 && ts_diff == decoder_frame_length_)) { + // The next sequence number is available, or the next part of a packet + // that was split into pieces upon insertion. + next_packet_available = true; + } + prev_sequence_number = next_packet->sequence_number; + } + } while (extracted_samples < required_samples && next_packet_available); + + if (extracted_samples > 0) { + // Delete old packets only when we are going to decode something. Otherwise, + // we could end up in the situation where we never decode anything, since + // all incoming packets are considered too old but the buffer will also + // never be flooded and flushed. + packet_buffer_->DiscardAllOldPackets(timestamp_, &stats_); + } + + return rtc::dchecked_cast(extracted_samples); +} + +void NetEqImpl::UpdatePlcComponents(int fs_hz, size_t channels) { + // Delete objects and create new ones. + expand_.reset(expand_factory_->Create(background_noise_.get(), + sync_buffer_.get(), &random_vector_, + &stats_, fs_hz, channels)); + merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get())); +} + +void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) { + RTC_LOG(LS_VERBOSE) << "SetSampleRateAndChannels " << fs_hz << " " + << channels; + // TODO(hlundin): Change to an enumerator and skip assert. + assert(fs_hz == 8000 || fs_hz == 16000 || fs_hz == 32000 || fs_hz == 48000); + assert(channels > 0); + + fs_hz_ = fs_hz; + fs_mult_ = fs_hz / 8000; + output_size_samples_ = static_cast(kOutputSizeMs * 8 * fs_mult_); + decoder_frame_length_ = 3 * output_size_samples_; // Initialize to 30ms. + + last_mode_ = kModeNormal; + + // Create a new array of mute factors and set all to 1. + mute_factor_array_.reset(new int16_t[channels]); + for (size_t i = 0; i < channels; ++i) { + mute_factor_array_[i] = 16384; // 1.0 in Q14. + } + + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + if (cng_decoder) + cng_decoder->Reset(); + + // Reinit post-decode VAD with new sample rate. + assert(vad_.get()); // Cannot be NULL here. + vad_->Init(); + + // Delete algorithm buffer and create a new one. + algorithm_buffer_.reset(new AudioMultiVector(channels)); + + // Delete sync buffer and create a new one. + sync_buffer_.reset(new SyncBuffer(channels, kSyncBufferSize * fs_mult_)); + + // Delete BackgroundNoise object and create a new one. + background_noise_.reset(new BackgroundNoise(channels)); + background_noise_->set_mode(background_noise_mode_); + + // Reset random vector. + random_vector_.Reset(); + + UpdatePlcComponents(fs_hz, channels); + + // Move index so that we create a small set of future samples (all 0). + sync_buffer_->set_next_index(sync_buffer_->next_index() - + expand_->overlap_length()); + + normal_.reset(new Normal(fs_hz, decoder_database_.get(), *background_noise_, + expand_.get())); + accelerate_.reset( + accelerate_factory_->Create(fs_hz, channels, *background_noise_)); + preemptive_expand_.reset(preemptive_expand_factory_->Create( + fs_hz, channels, *background_noise_, expand_->overlap_length())); + + // Delete ComfortNoise object and create a new one. + comfort_noise_.reset(new ComfortNoise(fs_hz, decoder_database_.get(), + sync_buffer_.get())); + + // Verify that |decoded_buffer_| is long enough. + if (decoded_buffer_length_ < kMaxFrameSize * channels) { + // Reallocate to larger size. + decoded_buffer_length_ = kMaxFrameSize * channels; + decoded_buffer_.reset(new int16_t[decoded_buffer_length_]); + } + + // Create DecisionLogic if it is not created yet, then communicate new sample + // rate and output size to DecisionLogic object. + if (!decision_logic_.get()) { + CreateDecisionLogic(); + } + decision_logic_->SetSampleRate(fs_hz_, output_size_samples_); +} + +NetEqImpl::OutputType NetEqImpl::LastOutputType() { + assert(vad_.get()); + assert(expand_.get()); + if (last_mode_ == kModeCodecInternalCng || last_mode_ == kModeRfc3389Cng) { + return OutputType::kCNG; + } else if (last_mode_ == kModeExpand && expand_->MuteFactor(0) == 0) { + // Expand mode has faded down to background noise only (very long expand). + return OutputType::kPLCCNG; + } else if (last_mode_ == kModeExpand) { + return OutputType::kPLC; + } else if (vad_->running() && !vad_->active_speech()) { + return OutputType::kVadPassive; + } else { + return OutputType::kNormalSpeech; + } +} + +void NetEqImpl::CreateDecisionLogic() { + decision_logic_.reset(DecisionLogic::Create( + fs_hz_, output_size_samples_, playout_mode_, decoder_database_.get(), + *packet_buffer_.get(), delay_manager_.get(), buffer_level_filter_.get(), + tick_timer_.get())); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.h new file mode 100644 index 0000000000..41681b6aa2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.h @@ -0,0 +1,450 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ +#define MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ + +#include +#include + +#include "api/optional.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/defines.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/packet.h" // Declare PacketList. +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/rtcp.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/tick_timer.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/thread_annotations.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class Accelerate; +class BackgroundNoise; +class BufferLevelFilter; +class ComfortNoise; +class DecisionLogic; +class DecoderDatabase; +class DelayManager; +class DelayPeakDetector; +class DtmfBuffer; +class DtmfToneGenerator; +class Expand; +class Merge; +class NackTracker; +class Normal; +class PacketBuffer; +class RedPayloadSplitter; +class PostDecodeVad; +class PreemptiveExpand; +class RandomVector; +class SyncBuffer; +class TimestampScaler; +struct AccelerateFactory; +struct DtmfEvent; +struct ExpandFactory; +struct PreemptiveExpandFactory; + +class NetEqImpl : public webrtc::NetEq { + public: + enum class OutputType { + kNormalSpeech, + kPLC, + kCNG, + kPLCCNG, + kVadPassive + }; + + enum ErrorCodes { + kNoError = 0, + kOtherError, + kUnknownRtpPayloadType, + kDecoderNotFound, + kInvalidPointer, + kAccelerateError, + kPreemptiveExpandError, + kComfortNoiseErrorCode, + kDecoderErrorCode, + kOtherDecoderError, + kInvalidOperation, + kDtmfParsingError, + kDtmfInsertError, + kSampleUnderrun, + kDecodedTooMuch, + kRedundancySplitError, + kPacketBufferCorruption + }; + + struct Dependencies { + // The constructor populates the Dependencies struct with the default + // implementations of the objects. They can all be replaced by the user + // before sending the struct to the NetEqImpl constructor. However, there + // are dependencies between some of the classes inside the struct, so + // swapping out one may make it necessary to re-create another one. + explicit Dependencies( + const NetEq::Config& config, + const rtc::scoped_refptr& decoder_factory); + ~Dependencies(); + + std::unique_ptr tick_timer; + std::unique_ptr buffer_level_filter; + std::unique_ptr decoder_database; + std::unique_ptr delay_peak_detector; + std::unique_ptr delay_manager; + std::unique_ptr dtmf_buffer; + std::unique_ptr dtmf_tone_generator; + std::unique_ptr packet_buffer; + std::unique_ptr red_payload_splitter; + std::unique_ptr timestamp_scaler; + std::unique_ptr accelerate_factory; + std::unique_ptr expand_factory; + std::unique_ptr preemptive_expand_factory; + }; + + // Creates a new NetEqImpl object. + NetEqImpl(const NetEq::Config& config, + Dependencies&& deps, + bool create_components = true); + + ~NetEqImpl() override; + + // Inserts a new packet into NetEq. The |receive_timestamp| is an indication + // of the time when the packet was received, and should be measured with + // the same tick rate as the RTP timestamp of the current payload. + // Returns 0 on success, -1 on failure. + int InsertPacket(const RTPHeader& rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) override; + + void InsertEmptyPacket(const RTPHeader& rtp_header) override; + + int GetAudio(AudioFrame* audio_frame, bool* muted) override; + + void SetCodecs(const std::map& codecs) override; + + int RegisterPayloadType(NetEqDecoder codec, + const std::string& codec_name, + uint8_t rtp_payload_type) override; + + int RegisterExternalDecoder(AudioDecoder* decoder, + NetEqDecoder codec, + const std::string& codec_name, + uint8_t rtp_payload_type) override; + + bool RegisterPayloadType(int rtp_payload_type, + const SdpAudioFormat& audio_format) override; + + // Removes |rtp_payload_type| from the codec database. Returns 0 on success, + // -1 on failure. + int RemovePayloadType(uint8_t rtp_payload_type) override; + + void RemoveAllPayloadTypes() override; + + bool SetMinimumDelay(int delay_ms) override; + + bool SetMaximumDelay(int delay_ms) override; + + int LeastRequiredDelayMs() const override; + + int SetTargetDelay() override; + + int TargetDelayMs() const override; + + int CurrentDelayMs() const override; + + int FilteredCurrentDelayMs() const override; + + // Sets the playout mode to |mode|. + // Deprecated. + // TODO(henrik.lundin) Delete. + void SetPlayoutMode(NetEqPlayoutMode mode) override; + + // Returns the current playout mode. + // Deprecated. + // TODO(henrik.lundin) Delete. + NetEqPlayoutMode PlayoutMode() const override; + + // Writes the current network statistics to |stats|. The statistics are reset + // after the call. + int NetworkStatistics(NetEqNetworkStatistics* stats) override; + + // Writes the current RTCP statistics to |stats|. The statistics are reset + // and a new report period is started with the call. + void GetRtcpStatistics(RtcpStatistics* stats) override; + + NetEqLifetimeStatistics GetLifetimeStatistics() const override; + + // Same as RtcpStatistics(), but does not reset anything. + void GetRtcpStatisticsNoReset(RtcpStatistics* stats) override; + + // Enables post-decode VAD. When enabled, GetAudio() will return + // kOutputVADPassive when the signal contains no speech. + void EnableVad() override; + + // Disables post-decode VAD. + void DisableVad() override; + + rtc::Optional GetPlayoutTimestamp() const override; + + int last_output_sample_rate_hz() const override; + + rtc::Optional GetDecoder(int payload_type) const override; + + rtc::Optional GetDecoderFormat( + int payload_type) const override; + + int SetTargetNumberOfChannels() override; + + int SetTargetSampleRate() override; + + // Flushes both the packet buffer and the sync buffer. + void FlushBuffers() override; + + void PacketBufferStatistics(int* current_num_packets, + int* max_num_packets) const override; + + void EnableNack(size_t max_nack_list_size) override; + + void DisableNack() override; + + std::vector GetNackList(int64_t round_trip_time_ms) const override; + + std::vector LastDecodedTimestamps() const override; + + int SyncBufferSizeMs() const override; + + // This accessor method is only intended for testing purposes. + const SyncBuffer* sync_buffer_for_test() const; + Operations last_operation_for_test() const; + + protected: + static const int kOutputSizeMs = 10; + static const size_t kMaxFrameSize = 5760; // 120 ms @ 48 kHz. + // TODO(hlundin): Provide a better value for kSyncBufferSize. + // Current value is kMaxFrameSize + 60 ms * 48 kHz, which is enough for + // calculating correlations of current frame against history. + static const size_t kSyncBufferSize = kMaxFrameSize + 60 * 48; + + // Inserts a new packet into NetEq. This is used by the InsertPacket method + // above. Returns 0 on success, otherwise an error code. + // TODO(hlundin): Merge this with InsertPacket above? + int InsertPacketInternal(const RTPHeader& rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Delivers 10 ms of audio data. The data is written to |audio_frame|. + // Returns 0 on success, otherwise an error code. + int GetAudioInternal(AudioFrame* audio_frame, bool* muted) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Provides a decision to the GetAudioInternal method. The decision what to + // do is written to |operation|. Packets to decode are written to + // |packet_list|, and a DTMF event to play is written to |dtmf_event|. When + // DTMF should be played, |play_dtmf| is set to true by the method. + // Returns 0 on success, otherwise an error code. + int GetDecision(Operations* operation, + PacketList* packet_list, + DtmfEvent* dtmf_event, + bool* play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Decodes the speech packets in |packet_list|, and writes the results to + // |decoded_buffer|, which is allocated to hold |decoded_buffer_length| + // elements. The length of the decoded data is written to |decoded_length|. + // The speech type -- speech or (codec-internal) comfort noise -- is written + // to |speech_type|. If |packet_list| contains any SID frames for RFC 3389 + // comfort noise, those are not decoded. + int Decode(PacketList* packet_list, + Operations* operation, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method to Decode(). Performs codec internal CNG. + int DecodeCng(AudioDecoder* decoder, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method to Decode(). Performs the actual decoding. + int DecodeLoop(PacketList* packet_list, + const Operations& operation, + AudioDecoder* decoder, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method which calls the Normal class to perform the normal operation. + void DoNormal(const int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method which calls the Merge class to perform the merge operation. + void DoMerge(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method which calls the Expand class to perform the expand operation. + int DoExpand(bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method which calls the Accelerate class to perform the accelerate + // operation. + int DoAccelerate(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf, + bool fast_accelerate) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method which calls the PreemptiveExpand class to perform the + // preemtive expand operation. + int DoPreemptiveExpand(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort + // noise. |packet_list| can either contain one SID frame to update the + // noise parameters, or no payload at all, in which case the previously + // received parameters are used. + int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Calls the audio decoder to generate codec-internal comfort noise when + // no packet was received. + void DoCodecInternalCng(const int16_t* decoded_buffer, size_t decoded_length) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Calls the DtmfToneGenerator class to generate DTMF tones. + int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Produces packet-loss concealment using alternative methods. If the codec + // has an internal PLC, it is called to generate samples. Otherwise, the + // method performs zero-stuffing. + void DoAlternativePlc(bool increase_timestamp) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Overdub DTMF on top of |output|. + int DtmfOverdub(const DtmfEvent& dtmf_event, + size_t num_channels, + int16_t* output) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Extracts packets from |packet_buffer_| to produce at least + // |required_samples| samples. The packets are inserted into |packet_list|. + // Returns the number of samples that the packets in the list will produce, or + // -1 in case of an error. + int ExtractPackets(size_t required_samples, PacketList* packet_list) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Resets various variables and objects to new values based on the sample rate + // |fs_hz| and |channels| number audio channels. + void SetSampleRateAndChannels(int fs_hz, size_t channels) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Returns the output type for the audio produced by the latest call to + // GetAudio(). + OutputType LastOutputType() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Updates Expand and Merge. + virtual void UpdatePlcComponents(int fs_hz, size_t channels) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + // Creates DecisionLogic object with the mode given by |playout_mode_|. + virtual void CreateDecisionLogic() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + + rtc::CriticalSection crit_sect_; + const std::unique_ptr tick_timer_ RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr buffer_level_filter_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr decoder_database_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr delay_manager_ RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr delay_peak_detector_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr dtmf_buffer_ RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr dtmf_tone_generator_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr packet_buffer_ RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr red_payload_splitter_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr timestamp_scaler_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr vad_ RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr expand_factory_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr accelerate_factory_ + RTC_GUARDED_BY(crit_sect_); + const std::unique_ptr preemptive_expand_factory_ + RTC_GUARDED_BY(crit_sect_); + + std::unique_ptr background_noise_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr decision_logic_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr algorithm_buffer_ + RTC_GUARDED_BY(crit_sect_); + std::unique_ptr sync_buffer_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr expand_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr normal_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr merge_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr accelerate_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr preemptive_expand_ + RTC_GUARDED_BY(crit_sect_); + RandomVector random_vector_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr comfort_noise_ RTC_GUARDED_BY(crit_sect_); + Rtcp rtcp_ RTC_GUARDED_BY(crit_sect_); + StatisticsCalculator stats_ RTC_GUARDED_BY(crit_sect_); + int fs_hz_ RTC_GUARDED_BY(crit_sect_); + int fs_mult_ RTC_GUARDED_BY(crit_sect_); + std::atomic last_output_sample_rate_hz_; + size_t output_size_samples_ RTC_GUARDED_BY(crit_sect_); + size_t decoder_frame_length_ RTC_GUARDED_BY(crit_sect_); + Modes last_mode_ RTC_GUARDED_BY(crit_sect_); + Operations last_operation_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr mute_factor_array_ RTC_GUARDED_BY(crit_sect_); + size_t decoded_buffer_length_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr decoded_buffer_ RTC_GUARDED_BY(crit_sect_); + uint32_t playout_timestamp_ RTC_GUARDED_BY(crit_sect_); + bool new_codec_ RTC_GUARDED_BY(crit_sect_); + uint32_t timestamp_ RTC_GUARDED_BY(crit_sect_); + bool reset_decoder_ RTC_GUARDED_BY(crit_sect_); + rtc::Optional current_rtp_payload_type_ RTC_GUARDED_BY(crit_sect_); + rtc::Optional current_cng_rtp_payload_type_ + RTC_GUARDED_BY(crit_sect_); + uint32_t ssrc_ RTC_GUARDED_BY(crit_sect_); + bool first_packet_ RTC_GUARDED_BY(crit_sect_); + const BackgroundNoiseMode background_noise_mode_ RTC_GUARDED_BY(crit_sect_); + NetEqPlayoutMode playout_mode_ RTC_GUARDED_BY(crit_sect_); + bool enable_fast_accelerate_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr nack_ RTC_GUARDED_BY(crit_sect_); + bool nack_enabled_ RTC_GUARDED_BY(crit_sect_); + const bool enable_muted_state_ RTC_GUARDED_BY(crit_sect_); + AudioFrame::VADActivity last_vad_activity_ RTC_GUARDED_BY(crit_sect_) = + AudioFrame::kVadPassive; + std::unique_ptr generated_noise_stopwatch_ + RTC_GUARDED_BY(crit_sect_); + std::vector last_decoded_timestamps_ RTC_GUARDED_BY(crit_sect_); + const bool use_dtx_delay_fix_ RTC_GUARDED_BY(crit_sect_); + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(NetEqImpl); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc new file mode 100644 index 0000000000..3088b37c36 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -0,0 +1,1515 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/accelerate.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/mock/mock_buffer_level_filter.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/mock/mock_delay_manager.h" +#include "modules/audio_coding/neteq/mock/mock_delay_peak_detector.h" +#include "modules/audio_coding/neteq/mock/mock_dtmf_buffer.h" +#include "modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h" +#include "modules/audio_coding/neteq/mock/mock_packet_buffer.h" +#include "modules/audio_coding/neteq/mock/mock_red_payload_splitter.h" +#include "modules/audio_coding/neteq/neteq_impl.h" +#include "modules/audio_coding/neteq/preemptive_expand.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/audio_coding/neteq/timestamp_scaler.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder.h" +#include "test/mock_audio_decoder_factory.h" + +using ::testing::AtLeast; +using ::testing::Return; +using ::testing::ReturnNull; +using ::testing::_; +using ::testing::SetArgPointee; +using ::testing::SetArrayArgument; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::WithArg; +using ::testing::Pointee; +using ::testing::IsNull; + +namespace webrtc { + +// This function is called when inserting a packet list into the mock packet +// buffer. The purpose is to delete all inserted packets properly, to avoid +// memory leaks in the test. +int DeletePacketsAndReturnOk(PacketList* packet_list) { + packet_list->clear(); + return PacketBuffer::kOK; +} + +class NetEqImplTest : public ::testing::Test { + protected: + NetEqImplTest() { config_.sample_rate_hz = 8000; } + + void CreateInstance() { + NetEqImpl::Dependencies deps(config_, CreateBuiltinAudioDecoderFactory()); + + // Get a local pointer to NetEq's TickTimer object. + tick_timer_ = deps.tick_timer.get(); + + if (use_mock_buffer_level_filter_) { + std::unique_ptr mock(new MockBufferLevelFilter); + mock_buffer_level_filter_ = mock.get(); + deps.buffer_level_filter = std::move(mock); + } + buffer_level_filter_ = deps.buffer_level_filter.get(); + + if (use_mock_decoder_database_) { + std::unique_ptr mock(new MockDecoderDatabase); + mock_decoder_database_ = mock.get(); + EXPECT_CALL(*mock_decoder_database_, GetActiveCngDecoder()) + .WillOnce(ReturnNull()); + deps.decoder_database = std::move(mock); + } + decoder_database_ = deps.decoder_database.get(); + + if (use_mock_delay_peak_detector_) { + std::unique_ptr mock( + new MockDelayPeakDetector(tick_timer_)); + mock_delay_peak_detector_ = mock.get(); + EXPECT_CALL(*mock_delay_peak_detector_, Reset()).Times(1); + deps.delay_peak_detector = std::move(mock); + } + delay_peak_detector_ = deps.delay_peak_detector.get(); + + if (use_mock_delay_manager_) { + std::unique_ptr mock(new MockDelayManager( + config_.max_packets_in_buffer, delay_peak_detector_, tick_timer_)); + mock_delay_manager_ = mock.get(); + EXPECT_CALL(*mock_delay_manager_, set_streaming_mode(false)).Times(1); + deps.delay_manager = std::move(mock); + } + delay_manager_ = deps.delay_manager.get(); + + if (use_mock_dtmf_buffer_) { + std::unique_ptr mock( + new MockDtmfBuffer(config_.sample_rate_hz)); + mock_dtmf_buffer_ = mock.get(); + deps.dtmf_buffer = std::move(mock); + } + dtmf_buffer_ = deps.dtmf_buffer.get(); + + if (use_mock_dtmf_tone_generator_) { + std::unique_ptr mock(new MockDtmfToneGenerator); + mock_dtmf_tone_generator_ = mock.get(); + deps.dtmf_tone_generator = std::move(mock); + } + dtmf_tone_generator_ = deps.dtmf_tone_generator.get(); + + if (use_mock_packet_buffer_) { + std::unique_ptr mock( + new MockPacketBuffer(config_.max_packets_in_buffer, tick_timer_)); + mock_packet_buffer_ = mock.get(); + deps.packet_buffer = std::move(mock); + } + packet_buffer_ = deps.packet_buffer.get(); + + if (use_mock_payload_splitter_) { + std::unique_ptr mock(new MockRedPayloadSplitter); + mock_payload_splitter_ = mock.get(); + deps.red_payload_splitter = std::move(mock); + } + red_payload_splitter_ = deps.red_payload_splitter.get(); + + deps.timestamp_scaler = std::unique_ptr( + new TimestampScaler(*deps.decoder_database.get())); + + neteq_.reset(new NetEqImpl(config_, std::move(deps))); + ASSERT_TRUE(neteq_ != NULL); + } + + void UseNoMocks() { + ASSERT_TRUE(neteq_ == NULL) << "Must call UseNoMocks before CreateInstance"; + use_mock_buffer_level_filter_ = false; + use_mock_decoder_database_ = false; + use_mock_delay_peak_detector_ = false; + use_mock_delay_manager_ = false; + use_mock_dtmf_buffer_ = false; + use_mock_dtmf_tone_generator_ = false; + use_mock_packet_buffer_ = false; + use_mock_payload_splitter_ = false; + } + + virtual ~NetEqImplTest() { + if (use_mock_buffer_level_filter_) { + EXPECT_CALL(*mock_buffer_level_filter_, Die()).Times(1); + } + if (use_mock_decoder_database_) { + EXPECT_CALL(*mock_decoder_database_, Die()).Times(1); + } + if (use_mock_delay_manager_) { + EXPECT_CALL(*mock_delay_manager_, Die()).Times(1); + } + if (use_mock_delay_peak_detector_) { + EXPECT_CALL(*mock_delay_peak_detector_, Die()).Times(1); + } + if (use_mock_dtmf_buffer_) { + EXPECT_CALL(*mock_dtmf_buffer_, Die()).Times(1); + } + if (use_mock_dtmf_tone_generator_) { + EXPECT_CALL(*mock_dtmf_tone_generator_, Die()).Times(1); + } + if (use_mock_packet_buffer_) { + EXPECT_CALL(*mock_packet_buffer_, Die()).Times(1); + } + } + + void TestDtmfPacket(NetEqDecoder decoder_type) { + const size_t kPayloadLength = 4; + const uint8_t kPayloadType = 110; + const uint32_t kReceiveTime = 17; + const int kSampleRateHz = 16000; + config_.sample_rate_hz = kSampleRateHz; + UseNoMocks(); + CreateInstance(); + // Event: 2, E bit, Volume: 17, Length: 4336. + uint8_t payload[kPayloadLength] = { 0x02, 0x80 + 0x11, 0x10, 0xF0 }; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterPayloadType( + decoder_type, "telephone-event", kPayloadType)); + + // Insert first packet. + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + // Pull audio once. + const size_t kMaxOutputSize = + static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_FALSE(muted); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Verify first 64 samples of actual output. + const std::vector kOutput({ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1578, -2816, -3460, -3403, -2709, -1594, + -363, 671, 1269, 1328, 908, 202, -513, -964, -955, -431, 504, 1617, + 2602, 3164, 3101, 2364, 1073, -511, -2047, -3198, -3721, -3525, -2688, + -1440, -99, 1015, 1663, 1744, 1319, 588, -171, -680, -747, -315, 515, + 1512, 2378, 2828, 2674, 1877, 568, -986, -2446, -3482, -3864, -3516, + -2534, -1163 }); + ASSERT_GE(kMaxOutputSize, kOutput.size()); + EXPECT_TRUE(std::equal(kOutput.begin(), kOutput.end(), output.data())); + } + + std::unique_ptr neteq_; + NetEq::Config config_; + TickTimer* tick_timer_ = nullptr; + MockBufferLevelFilter* mock_buffer_level_filter_ = nullptr; + BufferLevelFilter* buffer_level_filter_ = nullptr; + bool use_mock_buffer_level_filter_ = true; + MockDecoderDatabase* mock_decoder_database_ = nullptr; + DecoderDatabase* decoder_database_ = nullptr; + bool use_mock_decoder_database_ = true; + MockDelayPeakDetector* mock_delay_peak_detector_ = nullptr; + DelayPeakDetector* delay_peak_detector_ = nullptr; + bool use_mock_delay_peak_detector_ = true; + MockDelayManager* mock_delay_manager_ = nullptr; + DelayManager* delay_manager_ = nullptr; + bool use_mock_delay_manager_ = true; + MockDtmfBuffer* mock_dtmf_buffer_ = nullptr; + DtmfBuffer* dtmf_buffer_ = nullptr; + bool use_mock_dtmf_buffer_ = true; + MockDtmfToneGenerator* mock_dtmf_tone_generator_ = nullptr; + DtmfToneGenerator* dtmf_tone_generator_ = nullptr; + bool use_mock_dtmf_tone_generator_ = true; + MockPacketBuffer* mock_packet_buffer_ = nullptr; + PacketBuffer* packet_buffer_ = nullptr; + bool use_mock_packet_buffer_ = true; + MockRedPayloadSplitter* mock_payload_splitter_ = nullptr; + RedPayloadSplitter* red_payload_splitter_ = nullptr; + bool use_mock_payload_splitter_ = true; +}; + + +// This tests the interface class NetEq. +// TODO(hlundin): Move to separate file? +TEST(NetEq, CreateAndDestroy) { + NetEq::Config config; + NetEq* neteq = NetEq::Create(config, CreateBuiltinAudioDecoderFactory()); + delete neteq; +} + +TEST_F(NetEqImplTest, RegisterPayloadTypeNetEqDecoder) { + CreateInstance(); + uint8_t rtp_payload_type = 0; + NetEqDecoder codec_type = NetEqDecoder::kDecoderPCMu; + const std::string kCodecName = "Robert\'); DROP TABLE Students;"; + EXPECT_CALL(*mock_decoder_database_, + RegisterPayload(rtp_payload_type, codec_type, kCodecName)); + neteq_->RegisterPayloadType(codec_type, kCodecName, rtp_payload_type); +} + +TEST_F(NetEqImplTest, RegisterPayloadType) { + CreateInstance(); + constexpr int rtp_payload_type = 0; + const SdpAudioFormat format("pcmu", 8000, 1); + EXPECT_CALL(*mock_decoder_database_, + RegisterPayload(rtp_payload_type, format)); + neteq_->RegisterPayloadType(rtp_payload_type, format); +} + +TEST_F(NetEqImplTest, RemovePayloadType) { + CreateInstance(); + uint8_t rtp_payload_type = 0; + EXPECT_CALL(*mock_decoder_database_, Remove(rtp_payload_type)) + .WillOnce(Return(DecoderDatabase::kDecoderNotFound)); + // Check that kOK is returned when database returns kDecoderNotFound, because + // removing a payload type that was never registered is not an error. + EXPECT_EQ(NetEq::kOK, neteq_->RemovePayloadType(rtp_payload_type)); +} + +TEST_F(NetEqImplTest, RemoveAllPayloadTypes) { + CreateInstance(); + EXPECT_CALL(*mock_decoder_database_, RemoveAll()).WillOnce(Return()); + neteq_->RemoveAllPayloadTypes(); +} + +TEST_F(NetEqImplTest, InsertPacket) { + CreateInstance(); + const size_t kPayloadLength = 100; + const uint8_t kPayloadType = 0; + const uint16_t kFirstSequenceNumber = 0x1234; + const uint32_t kFirstTimestamp = 0x12345678; + const uint32_t kSsrc = 0x87654321; + const uint32_t kFirstReceiveTime = 17; + uint8_t payload[kPayloadLength] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = kFirstSequenceNumber; + rtp_header.timestamp = kFirstTimestamp; + rtp_header.ssrc = kSsrc; + Packet fake_packet; + fake_packet.payload_type = kPayloadType; + fake_packet.sequence_number = kFirstSequenceNumber; + fake_packet.timestamp = kFirstTimestamp; + + rtc::scoped_refptr mock_decoder_factory( + new rtc::RefCountedObject); + EXPECT_CALL(*mock_decoder_factory, MakeAudioDecoderMock(_, _)) + .WillOnce(Invoke([&](const SdpAudioFormat& format, + std::unique_ptr* dec) { + EXPECT_EQ("pcmu", format.name); + + std::unique_ptr mock_decoder(new MockAudioDecoder); + EXPECT_CALL(*mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(*mock_decoder, SampleRateHz()).WillRepeatedly(Return(8000)); + // BWE update function called with first packet. + EXPECT_CALL(*mock_decoder, + IncomingPacket(_, kPayloadLength, kFirstSequenceNumber, + kFirstTimestamp, kFirstReceiveTime)); + // BWE update function called with second packet. + EXPECT_CALL( + *mock_decoder, + IncomingPacket(_, kPayloadLength, kFirstSequenceNumber + 1, + kFirstTimestamp + 160, kFirstReceiveTime + 155)); + EXPECT_CALL(*mock_decoder, Die()).Times(1); // Called when deleted. + + *dec = std::move(mock_decoder); + })); + DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, + mock_decoder_factory); + + // Expectations for decoder database. + EXPECT_CALL(*mock_decoder_database_, GetDecoderInfo(kPayloadType)) + .WillRepeatedly(Return(&info)); + + // Expectations for packet buffer. + EXPECT_CALL(*mock_packet_buffer_, Empty()) + .WillOnce(Return(false)); // Called once after first packet is inserted. + EXPECT_CALL(*mock_packet_buffer_, Flush()) + .Times(1); + EXPECT_CALL(*mock_packet_buffer_, InsertPacketList(_, _, _, _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<2>(kPayloadType), + WithArg<0>(Invoke(DeletePacketsAndReturnOk)))); + // SetArgPointee<2>(kPayloadType) means that the third argument (zero-based + // index) is a pointer, and the variable pointed to is set to kPayloadType. + // Also invoke the function DeletePacketsAndReturnOk to properly delete all + // packets in the list (to avoid memory leaks in the test). + EXPECT_CALL(*mock_packet_buffer_, PeekNextPacket()) + .Times(1) + .WillOnce(Return(&fake_packet)); + + // Expectations for DTMF buffer. + EXPECT_CALL(*mock_dtmf_buffer_, Flush()) + .Times(1); + + // Expectations for delay manager. + { + // All expectations within this block must be called in this specific order. + InSequence sequence; // Dummy variable. + // Expectations when the first packet is inserted. + EXPECT_CALL(*mock_delay_manager_, last_pack_cng_or_dtmf()) + .Times(2) + .WillRepeatedly(Return(-1)); + EXPECT_CALL(*mock_delay_manager_, set_last_pack_cng_or_dtmf(0)) + .Times(1); + EXPECT_CALL(*mock_delay_manager_, ResetPacketIatCount()).Times(1); + // Expectations when the second packet is inserted. Slightly different. + EXPECT_CALL(*mock_delay_manager_, last_pack_cng_or_dtmf()) + .WillOnce(Return(0)); + EXPECT_CALL(*mock_delay_manager_, SetPacketAudioLength(30)) + .WillOnce(Return(0)); + } + + // Insert first packet. + neteq_->InsertPacket(rtp_header, payload, kFirstReceiveTime); + + // Insert second packet. + rtp_header.timestamp += 160; + rtp_header.sequenceNumber += 1; + neteq_->InsertPacket(rtp_header, payload, kFirstReceiveTime + 155); +} + +TEST_F(NetEqImplTest, InsertPacketsUntilBufferIsFull) { + UseNoMocks(); + CreateInstance(); + + const int kPayloadLengthSamples = 80; + const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterPayloadType( + NetEqDecoder::kDecoderPCM16B, "", kPayloadType)); + + // Insert packets. The buffer should not flush. + for (size_t i = 1; i <= config_.max_packets_in_buffer; ++i) { + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + rtp_header.timestamp += kPayloadLengthSamples; + rtp_header.sequenceNumber += 1; + EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer()); + } + + // Insert one more packet and make sure the buffer got flushed. That is, it + // should only hold one single packet. + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer()); + const Packet* test_packet = packet_buffer_->PeekNextPacket(); + EXPECT_EQ(rtp_header.timestamp, test_packet->timestamp); + EXPECT_EQ(rtp_header.sequenceNumber, test_packet->sequence_number); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT) { + TestDtmfPacket(NetEqDecoder::kDecoderAVT); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT16kHz) { + TestDtmfPacket(NetEqDecoder::kDecoderAVT16kHz); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT32kHz) { + TestDtmfPacket(NetEqDecoder::kDecoderAVT32kHz); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT48kHz) { + TestDtmfPacket(NetEqDecoder::kDecoderAVT48kHz); +} + +// This test verifies that timestamps propagate from the incoming packets +// through to the sync buffer and to the playout timestamp. +TEST_F(NetEqImplTest, VerifyTimestampPropagation) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // This is a dummy decoder that produces as many output samples as the input + // has bytes. The output is an increasing series, starting at 1 for the first + // sample, and then increasing by 1 for each sample. + class CountingSamplesDecoder : public AudioDecoder { + public: + CountingSamplesDecoder() : next_value_(1) {} + + // Produce as many samples as input bytes (|encoded_len|). + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int /* sample_rate_hz */, + int16_t* decoded, + SpeechType* speech_type) override { + for (size_t i = 0; i < encoded_len; ++i) { + decoded[i] = next_value_++; + } + *speech_type = kSpeech; + return rtc::checked_cast(encoded_len); + } + + void Reset() override { next_value_ = 1; } + + int SampleRateHz() const override { return kSampleRateHz; } + + size_t Channels() const override { return 1; } + + uint16_t next_value() const { return next_value_; } + + private: + int16_t next_value_; + } decoder_; + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + &decoder_, NetEqDecoder::kDecoderPCM16B, + "dummy name", kPayloadType)); + + // Insert one packet. + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_FALSE(muted); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Start with a simple check that the fake decoder is behaving as expected. + EXPECT_EQ(kPayloadLengthSamples, + static_cast(decoder_.next_value() - 1)); + + // The value of the last of the output samples is the same as the number of + // samples played from the decoded packet. Thus, this number + the RTP + // timestamp should match the playout timestamp. + // Wrap the expected value in an rtc::Optional to compare them as such. + EXPECT_EQ( + rtc::Optional(rtp_header.timestamp + + output.data()[output.samples_per_channel_ - 1]), + neteq_->GetPlayoutTimestamp()); + + // Check the timestamp for the last value in the sync buffer. This should + // be one full frame length ahead of the RTP timestamp. + const SyncBuffer* sync_buffer = neteq_->sync_buffer_for_test(); + ASSERT_TRUE(sync_buffer != NULL); + EXPECT_EQ(rtp_header.timestamp + kPayloadLengthSamples, + sync_buffer->end_timestamp()); + + // Check that the number of samples still to play from the sync buffer add + // up with what was already played out. + EXPECT_EQ( + kPayloadLengthSamples - output.data()[output.samples_per_channel_ - 1], + sync_buffer->FutureLength()); +} + +TEST_F(NetEqImplTest, ReorderedPacket) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, IncomingPacket(_, kPayloadLengthBytes, _, _, _)) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + int16_t dummy_output[kPayloadLengthSamples] = {0}; + // The below expectation will make the mock decoder write + // |kPayloadLengthSamples| zeros to the output array, and mark it as speech. + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(0), kPayloadLengthBytes, + kSampleRateHz, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + &mock_decoder, NetEqDecoder::kDecoderPCM16B, + "dummy name", kPayloadType)); + + // Insert one packet. + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Insert two more packets. The first one is out of order, and is already too + // old, the second one is the expected next packet. + rtp_header.sequenceNumber -= 1; + rtp_header.timestamp -= kPayloadLengthSamples; + payload[0] = 1; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + rtp_header.sequenceNumber += 2; + rtp_header.timestamp += 2 * kPayloadLengthSamples; + payload[0] = 2; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + // Expect only the second packet to be decoded (the one with "2" as the first + // payload byte). + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(2), kPayloadLengthBytes, + kSampleRateHz, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + + // Pull audio once. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Now check the packet buffer, and make sure it is empty, since the + // out-of-order packet should have been discarded. + EXPECT_TRUE(packet_buffer_->Empty()); + + EXPECT_CALL(mock_decoder, Die()); +} + +// This test verifies that NetEq can handle the situation where the first +// incoming packet is rejected. +TEST_F(NetEqImplTest, FirstPacketUnknown) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Insert one packet. Note that we have not registered any payload type, so + // this packet will be rejected. + EXPECT_EQ(NetEq::kFail, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_LE(output.samples_per_channel_, kMaxOutputSize); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kPLC, output.speech_type_); + + // Register the payload type. + EXPECT_EQ(NetEq::kOK, neteq_->RegisterPayloadType( + NetEqDecoder::kDecoderPCM16B, "", kPayloadType)); + + // Insert 10 packets. + for (size_t i = 0; i < 10; ++i) { + rtp_header.sequenceNumber++; + rtp_header.timestamp += kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer()); + } + + // Pull audio repeatedly and make sure we get normal output, that is not PLC. + for (size_t i = 0; i < 3; ++i) { + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_LE(output.samples_per_channel_, kMaxOutputSize); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_) + << "NetEq did not decode the packets as expected."; + } +} + +// This test verifies that NetEq can handle comfort noise and enters/quits codec +// internal CNG mode properly. +TEST_F(NetEqImplTest, CodecInternalCng) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateKhz = 48; + const size_t kPayloadLengthSamples = + static_cast(20 * kSampleRateKhz); // 20 ms. + const size_t kPayloadLengthBytes = 10; + uint8_t payload[kPayloadLengthBytes] = {0}; + int16_t dummy_output[kPayloadLengthSamples] = {0}; + + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateKhz * 1000)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, IncomingPacket(_, kPayloadLengthBytes, _, _, _)) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + // Packed duration when asking the decoder for more CNG data (without a new + // packet). + EXPECT_CALL(mock_decoder, PacketDuration(nullptr, 0)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + + // Pointee(x) verifies that first byte of the payload equals x, this makes it + // possible to verify that the correct payload is fed to Decode(). + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(0), kPayloadLengthBytes, + kSampleRateKhz * 1000, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(1), kPayloadLengthBytes, + kSampleRateKhz * 1000, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kComfortNoise), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + + EXPECT_CALL(mock_decoder, + DecodeInternal(IsNull(), 0, kSampleRateKhz * 1000, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kComfortNoise), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(2), kPayloadLengthBytes, + kSampleRateKhz * 1000, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + &mock_decoder, NetEqDecoder::kDecoderOpus, + "dummy name", kPayloadType)); + + // Insert one packet (decoder will return speech). + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + // Insert second packet (decoder will return CNG). + payload[0] = 1; + rtp_header.sequenceNumber++; + rtp_header.timestamp += kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + const size_t kMaxOutputSize = static_cast(10 * kSampleRateKhz); + AudioFrame output; + AudioFrame::SpeechType expected_type[8] = { + AudioFrame::kNormalSpeech, AudioFrame::kNormalSpeech, + AudioFrame::kCNG, AudioFrame::kCNG, + AudioFrame::kCNG, AudioFrame::kCNG, + AudioFrame::kNormalSpeech, AudioFrame::kNormalSpeech + }; + int expected_timestamp_increment[8] = { + -1, // will not be used. + 10 * kSampleRateKhz, + -1, -1, // timestamp will be empty during CNG mode; indicated by -1 here. + -1, -1, + 50 * kSampleRateKhz, 10 * kSampleRateKhz + }; + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + rtc::Optional last_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(last_timestamp); + + // Lambda for verifying the timestamps. + auto verify_timestamp = [&last_timestamp, &expected_timestamp_increment]( + rtc::Optional ts, size_t i) { + if (expected_timestamp_increment[i] == -1) { + // Expect to get an empty timestamp value during CNG and PLC. + EXPECT_FALSE(ts) << "i = " << i; + } else { + ASSERT_TRUE(ts) << "i = " << i; + EXPECT_EQ(*ts, *last_timestamp + expected_timestamp_increment[i]) + << "i = " << i; + last_timestamp = ts; + } + }; + + for (size_t i = 1; i < 6; ++i) { + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(expected_type[i - 1], output.speech_type_); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + SCOPED_TRACE(""); + verify_timestamp(neteq_->GetPlayoutTimestamp(), i); + } + + // Insert third packet, which leaves a gap from last packet. + payload[0] = 2; + rtp_header.sequenceNumber += 2; + rtp_header.timestamp += 2 * kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + for (size_t i = 6; i < 8; ++i) { + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(expected_type[i - 1], output.speech_type_); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + SCOPED_TRACE(""); + verify_timestamp(neteq_->GetPlayoutTimestamp(), i); + } + + // Now check the packet buffer, and make sure it is empty. + EXPECT_TRUE(packet_buffer_->Empty()); + + EXPECT_CALL(mock_decoder, Die()); +} + +TEST_F(NetEqImplTest, UnsupportedDecoder) { + UseNoMocks(); + CreateInstance(); + static const size_t kNetEqMaxFrameSize = 5760; // 120 ms @ 48 kHz. + static const size_t kChannels = 2; + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateHz = 8000; + + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = 1; + uint8_t payload[kPayloadLengthBytes] = {0}; + int16_t dummy_output[kPayloadLengthSamples * kChannels] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + ::testing::NiceMock decoder; + + const uint8_t kFirstPayloadValue = 1; + const uint8_t kSecondPayloadValue = 2; + + EXPECT_CALL(decoder, + PacketDuration(Pointee(kFirstPayloadValue), kPayloadLengthBytes)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(rtc::checked_cast(kNetEqMaxFrameSize + 1))); + + EXPECT_CALL(decoder, DecodeInternal(Pointee(kFirstPayloadValue), _, _, _, _)) + .Times(0); + + EXPECT_CALL(decoder, DecodeInternal(Pointee(kSecondPayloadValue), + kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples * kChannels), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(static_cast(kPayloadLengthSamples * kChannels)))); + + EXPECT_CALL(decoder, + PacketDuration(Pointee(kSecondPayloadValue), kPayloadLengthBytes)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(rtc::checked_cast(kNetEqMaxFrameSize))); + + EXPECT_CALL(decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + + EXPECT_CALL(decoder, Channels()) + .WillRepeatedly(Return(kChannels)); + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + &decoder, NetEqDecoder::kDecoderPCM16B, + "dummy name", kPayloadType)); + + // Insert one packet. + payload[0] = kFirstPayloadValue; // This will make Decode() fail. + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + // Insert another packet. + payload[0] = kSecondPayloadValue; // This will make Decode() successful. + rtp_header.sequenceNumber++; + // The second timestamp needs to be at least 30 ms after the first to make + // the second packet get decoded. + rtp_header.timestamp += 3 * kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + AudioFrame output; + bool muted; + // First call to GetAudio will try to decode the "faulty" packet. + // Expect kFail return value. + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); + // Output size and number of channels should be correct. + const size_t kExpectedOutputSize = 10 * (kSampleRateHz / 1000) * kChannels; + EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels); + EXPECT_EQ(kChannels, output.num_channels_); + + // Second call to GetAudio will decode the packet that is ok. No errors are + // expected. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels); + EXPECT_EQ(kChannels, output.num_channels_); + + // Die isn't called through NiceMock (since it's called by the + // MockAudioDecoder constructor), so it needs to be mocked explicitly. + EXPECT_CALL(decoder, Die()); +} + +// This test inserts packets until the buffer is flushed. After that, it asks +// NetEq for the network statistics. The purpose of the test is to make sure +// that even though the buffer size increment is negative (which it becomes when +// the packet causing a flush is inserted), the packet length stored in the +// decision logic remains valid. +TEST_F(NetEqImplTest, FloodBufferAndGetNetworkStats) { + UseNoMocks(); + CreateInstance(); + + const size_t kPayloadLengthSamples = 80; + const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterPayloadType( + NetEqDecoder::kDecoderPCM16B, "", kPayloadType)); + + // Insert packets until the buffer flushes. + for (size_t i = 0; i <= config_.max_packets_in_buffer; ++i) { + EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer()); + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + rtp_header.timestamp += rtc::checked_cast(kPayloadLengthSamples); + ++rtp_header.sequenceNumber; + } + EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer()); + + // Ask for network statistics. This should not crash. + NetEqNetworkStatistics stats; + EXPECT_EQ(NetEq::kOK, neteq_->NetworkStatistics(&stats)); +} + +TEST_F(NetEqImplTest, DecodedPayloadTooShort) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, IncomingPacket(_, kPayloadLengthBytes, _, _, _)) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mock_decoder, PacketDuration(_, _)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + int16_t dummy_output[kPayloadLengthSamples] = {0}; + // The below expectation will make the mock decoder write + // |kPayloadLengthSamples| - 5 zeros to the output array, and mark it as + // speech. That is, the decoded length is 5 samples shorter than the expected. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .WillOnce( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples - 5), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples - 5)))); + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + &mock_decoder, NetEqDecoder::kDecoderPCM16B, + "dummy name", kPayloadType)); + + // Insert one packet. + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + + EXPECT_EQ(5u, neteq_->sync_buffer_for_test()->FutureLength()); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + EXPECT_CALL(mock_decoder, Die()); +} + +// This test checks the behavior of NetEq when audio decoder fails. +TEST_F(NetEqImplTest, DecodingError) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateHz = 8000; + const int kDecoderErrorCode = -97; // Any negative number. + + // We let decoder return 5 ms each time, and therefore, 2 packets make 10 ms. + const size_t kFrameLengthSamples = + static_cast(5 * kSampleRateHz / 1000); + + const size_t kPayloadLengthBytes = 1; // This can be arbitrary. + + uint8_t payload[kPayloadLengthBytes] = {0}; + + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, IncomingPacket(_, kPayloadLengthBytes, _, _, _)) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mock_decoder, PacketDuration(_, _)) + .WillRepeatedly(Return(rtc::checked_cast(kFrameLengthSamples))); + EXPECT_CALL(mock_decoder, ErrorCode()) + .WillOnce(Return(kDecoderErrorCode)); + EXPECT_CALL(mock_decoder, HasDecodePlc()) + .WillOnce(Return(false)); + int16_t dummy_output[kFrameLengthSamples] = {0}; + + { + InSequence sequence; // Dummy variable. + // Mock decoder works normally the first time. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kFrameLengthSamples)))) + .RetiresOnSaturation(); + + // Then mock decoder fails. A common reason for failure can be buffer being + // too short + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .WillOnce(Return(-1)) + .RetiresOnSaturation(); + + // Mock decoder finally returns to normal. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kFrameLengthSamples)))); + } + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + &mock_decoder, NetEqDecoder::kDecoderPCM16B, + "dummy name", kPayloadType)); + + // Insert packets. + for (int i = 0; i < 6; ++i) { + rtp_header.sequenceNumber += 1; + rtp_header.timestamp += kFrameLengthSamples; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + } + + // Pull audio. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Pull audio again. Decoder fails. + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + // We are not expecting anything for output.speech_type_, since an error was + // returned. + + // Pull audio again, should continue an expansion. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kPLC, output.speech_type_); + + // Pull audio again, should behave normal. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + EXPECT_CALL(mock_decoder, Die()); +} + +// This test checks the behavior of NetEq when audio decoder fails during CNG. +TEST_F(NetEqImplTest, DecodingErrorDuringInternalCng) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const uint32_t kReceiveTime = 17; // Value doesn't matter for this test. + const int kSampleRateHz = 8000; + const int kDecoderErrorCode = -97; // Any negative number. + + // We let decoder return 5 ms each time, and therefore, 2 packets make 10 ms. + const size_t kFrameLengthSamples = + static_cast(5 * kSampleRateHz / 1000); + + const size_t kPayloadLengthBytes = 1; // This can be arbitrary. + + uint8_t payload[kPayloadLengthBytes] = {0}; + + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, IncomingPacket(_, kPayloadLengthBytes, _, _, _)) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mock_decoder, PacketDuration(_, _)) + .WillRepeatedly(Return(rtc::checked_cast(kFrameLengthSamples))); + EXPECT_CALL(mock_decoder, ErrorCode()) + .WillOnce(Return(kDecoderErrorCode)); + int16_t dummy_output[kFrameLengthSamples] = {0}; + + { + InSequence sequence; // Dummy variable. + // Mock decoder works normally the first 2 times. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kComfortNoise), + Return(rtc::checked_cast(kFrameLengthSamples)))) + .RetiresOnSaturation(); + + // Then mock decoder fails. A common reason for failure can be buffer being + // too short + EXPECT_CALL(mock_decoder, DecodeInternal(nullptr, 0, kSampleRateHz, _, _)) + .WillOnce(Return(-1)) + .RetiresOnSaturation(); + + // Mock decoder finally returns to normal. + EXPECT_CALL(mock_decoder, DecodeInternal(nullptr, 0, kSampleRateHz, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kComfortNoise), + Return(rtc::checked_cast(kFrameLengthSamples)))); + } + + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + &mock_decoder, NetEqDecoder::kDecoderPCM16B, + "dummy name", kPayloadType)); + + // Insert 2 packets. This will make netEq into codec internal CNG mode. + for (int i = 0; i < 2; ++i) { + rtp_header.sequenceNumber += 1; + rtp_header.timestamp += kFrameLengthSamples; + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, kReceiveTime)); + } + + // Pull audio. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kCNG, output.speech_type_); + + // Pull audio again. Decoder fails. + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + // We are not expecting anything for output.speech_type_, since an error was + // returned. + + // Pull audio again, should resume codec CNG. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kCNG, output.speech_type_); + + EXPECT_CALL(mock_decoder, Die()); +} + +// Tests that the return value from last_output_sample_rate_hz() is equal to the +// configured inital sample rate. +TEST_F(NetEqImplTest, InitialLastOutputSampleRate) { + UseNoMocks(); + config_.sample_rate_hz = 48000; + CreateInstance(); + EXPECT_EQ(48000, neteq_->last_output_sample_rate_hz()); +} + +TEST_F(NetEqImplTest, TickTimerIncrement) { + UseNoMocks(); + CreateInstance(); + ASSERT_TRUE(tick_timer_); + EXPECT_EQ(0u, tick_timer_->ticks()); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(1u, tick_timer_->ticks()); +} + +TEST_F(NetEqImplTest, TargetDelayMs) { + UseNoMocks(); + use_mock_delay_manager_ = true; + CreateInstance(); + // Let the dummy target delay be 17 packets. + constexpr int kTargetLevelPacketsQ8 = 17 << 8; + EXPECT_CALL(*mock_delay_manager_, TargetLevel()) + .WillOnce(Return(kTargetLevelPacketsQ8)); + // Default packet size before any packet has been decoded is 30 ms, so we are + // expecting 17 * 30 = 510 ms target delay. + EXPECT_EQ(17 * 30, neteq_->TargetDelayMs()); +} + +TEST_F(NetEqImplTest, InsertEmptyPacket) { + UseNoMocks(); + use_mock_delay_manager_ = true; + CreateInstance(); + + RTPHeader rtp_header; + rtp_header.payloadType = 17; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_CALL(*mock_delay_manager_, RegisterEmptyPacket()); + neteq_->InsertEmptyPacket(rtp_header); +} + +class Decoder120ms : public AudioDecoder { + public: + Decoder120ms(int sample_rate_hz, SpeechType speech_type) + : sample_rate_hz_(sample_rate_hz), + next_value_(1), + speech_type_(speech_type) {} + + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override { + EXPECT_EQ(sample_rate_hz_, sample_rate_hz); + size_t decoded_len = + rtc::CheckedDivExact(sample_rate_hz, 1000) * 120 * Channels(); + for (size_t i = 0; i < decoded_len; ++i) { + decoded[i] = next_value_++; + } + *speech_type = speech_type_; + return rtc::checked_cast(decoded_len); + } + + void Reset() override { next_value_ = 1; } + int SampleRateHz() const override { return sample_rate_hz_; } + size_t Channels() const override { return 2; } + + private: + int sample_rate_hz_; + int16_t next_value_; + SpeechType speech_type_; +}; + +class NetEqImplTest120ms : public NetEqImplTest { + protected: + NetEqImplTest120ms() : NetEqImplTest() {} + virtual ~NetEqImplTest120ms() {} + + void CreateInstanceNoMocks() { + UseNoMocks(); + CreateInstance(); + } + + void CreateInstanceWithDelayManagerMock() { + UseNoMocks(); + use_mock_delay_manager_ = true; + CreateInstance(); + } + + uint32_t timestamp_diff_between_packets() const { + return rtc::CheckedDivExact(kSamplingFreq_, 1000u) * 120; + } + + uint32_t first_timestamp() const { return 10u; } + + void GetFirstPacket() { + bool muted; + for (int i = 0; i < 12; i++) { + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_FALSE(muted); + } + } + + void InsertPacket(uint32_t timestamp) { + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = sequence_number_; + rtp_header.timestamp = timestamp; + rtp_header.ssrc = 15; + const size_t kPayloadLengthBytes = 1; // This can be arbitrary. + uint8_t payload[kPayloadLengthBytes] = {0}; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload, 10)); + sequence_number_++; + } + + void Register120msCodec(AudioDecoder::SpeechType speech_type) { + decoder_.reset(new Decoder120ms(kSamplingFreq_, speech_type)); + ASSERT_EQ(2u, decoder_->Channels()); + EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder( + decoder_.get(), NetEqDecoder::kDecoderOpus_2ch, + "120ms codec", kPayloadType)); + } + + std::unique_ptr decoder_; + AudioFrame output_; + const uint32_t kPayloadType = 17; + const uint32_t kSamplingFreq_ = 48000; + uint16_t sequence_number_ = 1; +}; + +TEST_F(NetEqImplTest120ms, AudioRepetition) { + config_.playout_mode = kPlayoutFax; + CreateInstanceNoMocks(); + Register120msCodec(AudioDecoder::kSpeech); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kAudioRepetition, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, AlternativePlc) { + config_.playout_mode = kPlayoutOff; + CreateInstanceNoMocks(); + Register120msCodec(AudioDecoder::kSpeech); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kAlternativePlc, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, CodecInternalCng) { + CreateInstanceNoMocks(); + Register120msCodec(AudioDecoder::kComfortNoise); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kCodecInternalCng, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Normal) { + CreateInstanceNoMocks(); + Register120msCodec(AudioDecoder::kSpeech); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + EXPECT_EQ(kNormal, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Merge) { + CreateInstanceWithDelayManagerMock(); + + Register120msCodec(AudioDecoder::kSpeech); + InsertPacket(first_timestamp()); + + GetFirstPacket(); + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + + InsertPacket(first_timestamp() + 2 * timestamp_diff_between_packets()); + + // Delay manager reports a target level which should cause a Merge. + EXPECT_CALL(*mock_delay_manager_, TargetLevel()).WillOnce(Return(-10)); + + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kMerge, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Expand) { + CreateInstanceNoMocks(); + Register120msCodec(AudioDecoder::kSpeech); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kExpand, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, FastAccelerate) { + CreateInstanceWithDelayManagerMock(); + Register120msCodec(AudioDecoder::kSpeech); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + InsertPacket(first_timestamp() + timestamp_diff_between_packets()); + + // Delay manager report buffer limit which should cause a FastAccelerate. + EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<0>(0), SetArgPointee<1>(0))); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kFastAccelerate, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, PreemptiveExpand) { + CreateInstanceWithDelayManagerMock(); + Register120msCodec(AudioDecoder::kSpeech); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + InsertPacket(first_timestamp() + timestamp_diff_between_packets()); + + // Delay manager report buffer limit which should cause a PreemptiveExpand. + EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<0>(100), SetArgPointee<1>(100))); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kPreemptiveExpand, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Accelerate) { + CreateInstanceWithDelayManagerMock(); + Register120msCodec(AudioDecoder::kSpeech); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + InsertPacket(first_timestamp() + timestamp_diff_between_packets()); + + // Delay manager report buffer limit which should cause a Accelerate. + EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<0>(1), SetArgPointee<1>(2))); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(kAccelerate, neteq_->last_operation_for_test()); +} + +}// namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc new file mode 100644 index 0000000000..334715f8d1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/tools/neteq_external_decoder_test.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "modules/include/module_common_types.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +using ::testing::_; +using ::testing::SetArgPointee; +using ::testing::Return; + +class MockAudioDecoder final : public AudioDecoder { + public: + // TODO(nisse): Valid overrides commented out, because the gmock + // methods don't use any override declarations, and we want to avoid + // warnings from -Winconsistent-missing-override. See + // http://crbug.com/428099. + static const int kPacketDuration = 960; // 48 kHz * 20 ms + + MockAudioDecoder(int sample_rate_hz, size_t num_channels) + : sample_rate_hz_(sample_rate_hz), + num_channels_(num_channels), + fec_enabled_(false) {} + ~MockAudioDecoder() /* override */ { Die(); } + MOCK_METHOD0(Die, void()); + + MOCK_METHOD0(Reset, void()); + + class MockFrame : public AudioDecoder::EncodedAudioFrame { + public: + MockFrame(size_t num_channels) : num_channels_(num_channels) {} + + size_t Duration() const override { return kPacketDuration; } + + rtc::Optional Decode( + rtc::ArrayView decoded) const override { + const size_t output_size = + sizeof(int16_t) * kPacketDuration * num_channels_; + if (decoded.size() >= output_size) { + memset(decoded.data(), 0, + sizeof(int16_t) * kPacketDuration * num_channels_); + return DecodeResult{kPacketDuration * num_channels_, kSpeech}; + } else { + ADD_FAILURE() << "Expected decoded.size() to be >= output_size (" + << decoded.size() << " vs. " << output_size << ")"; + return rtc::nullopt; + } + } + + private: + const size_t num_channels_; + }; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) /* override */ { + std::vector results; + if (fec_enabled_) { + std::unique_ptr fec_frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp - kPacketDuration, 1, + std::move(fec_frame)); + } + + std::unique_ptr frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; + } + + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const + /* override */ { + ADD_FAILURE() << "Since going through ParsePayload, PacketDuration should " + "never get called."; + return kPacketDuration; + } + + bool PacketHasFec( + const uint8_t* encoded, size_t encoded_len) const /* override */ { + ADD_FAILURE() << "Since going through ParsePayload, PacketHasFec should " + "never get called."; + return fec_enabled_; + } + + int SampleRateHz() const /* override */ { return sample_rate_hz_; } + + size_t Channels() const /* override */ { return num_channels_; } + + void set_fec_enabled(bool enable_fec) { fec_enabled_ = enable_fec; } + + bool fec_enabled() const { return fec_enabled_; } + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) /* override */ { + ADD_FAILURE() << "Since going through ParsePayload, DecodeInternal should " + "never get called."; + return -1; + } + + private: + const int sample_rate_hz_; + const size_t num_channels_; + bool fec_enabled_; +}; + +class NetEqNetworkStatsTest : public NetEqExternalDecoderTest { + public: + static const int kPayloadSizeByte = 30; + static const int kFrameSizeMs = 20; + +enum logic { + kIgnore, + kEqual, + kSmallerThan, + kLargerThan, +}; + +struct NetEqNetworkStatsCheck { + logic current_buffer_size_ms; + logic preferred_buffer_size_ms; + logic jitter_peaks_found; + logic packet_loss_rate; + logic expand_rate; + logic speech_expand_rate; + logic preemptive_rate; + logic accelerate_rate; + logic secondary_decoded_rate; + logic secondary_discarded_rate; + logic clockdrift_ppm; + logic added_zero_samples; + NetEqNetworkStatistics stats_ref; +}; + +NetEqNetworkStatsTest(NetEqDecoder codec, + int sample_rate_hz, + MockAudioDecoder* decoder) + : NetEqExternalDecoderTest(codec, sample_rate_hz, decoder), + external_decoder_(decoder), + samples_per_ms_(sample_rate_hz / 1000), + frame_size_samples_(kFrameSizeMs * samples_per_ms_), + rtp_generator_(new test::RtpGenerator(samples_per_ms_)), + last_lost_time_(0), + packet_loss_interval_(0xffffffff) { + Init(); + } + + bool Lost(uint32_t send_time) { + if (send_time - last_lost_time_ >= packet_loss_interval_) { + last_lost_time_ = send_time; + return true; + } + return false; + } + + void SetPacketLossRate(double loss_rate) { + packet_loss_interval_ = (loss_rate >= 1e-3 ? + static_cast(kFrameSizeMs) / loss_rate : 0xffffffff); + } + + // |stats_ref| + // expects.x = -1, do not care + // expects.x = 0, 'x' in current stats should equal 'x' in |stats_ref| + // expects.x = 1, 'x' in current stats should < 'x' in |stats_ref| + // expects.x = 2, 'x' in current stats should > 'x' in |stats_ref| + void CheckNetworkStatistics(NetEqNetworkStatsCheck expects) { + NetEqNetworkStatistics stats; + neteq()->NetworkStatistics(&stats); + +#define CHECK_NETEQ_NETWORK_STATS(x)\ + switch (expects.x) {\ + case kEqual:\ + EXPECT_EQ(stats.x, expects.stats_ref.x);\ + break;\ + case kSmallerThan:\ + EXPECT_LT(stats.x, expects.stats_ref.x);\ + break;\ + case kLargerThan:\ + EXPECT_GT(stats.x, expects.stats_ref.x);\ + break;\ + default:\ + break;\ + } + + CHECK_NETEQ_NETWORK_STATS(current_buffer_size_ms); + CHECK_NETEQ_NETWORK_STATS(preferred_buffer_size_ms); + CHECK_NETEQ_NETWORK_STATS(jitter_peaks_found); + CHECK_NETEQ_NETWORK_STATS(packet_loss_rate); + CHECK_NETEQ_NETWORK_STATS(expand_rate); + CHECK_NETEQ_NETWORK_STATS(speech_expand_rate); + CHECK_NETEQ_NETWORK_STATS(preemptive_rate); + CHECK_NETEQ_NETWORK_STATS(accelerate_rate); + CHECK_NETEQ_NETWORK_STATS(secondary_decoded_rate); + CHECK_NETEQ_NETWORK_STATS(secondary_discarded_rate); + CHECK_NETEQ_NETWORK_STATS(clockdrift_ppm); + CHECK_NETEQ_NETWORK_STATS(added_zero_samples); + +#undef CHECK_NETEQ_NETWORK_STATS + + // Compare with CurrentDelay, which should be identical. + EXPECT_EQ(stats.current_buffer_size_ms, neteq()->CurrentDelayMs()); + } + + void RunTest(int num_loops, NetEqNetworkStatsCheck expects) { + uint32_t time_now; + uint32_t next_send_time; + + // Initiate |last_lost_time_|. + time_now = next_send_time = last_lost_time_ = + rtp_generator_->GetRtpHeader(kPayloadType, frame_size_samples_, + &rtp_header_); + for (int k = 0; k < num_loops; ++k) { + // Delay by one frame such that the FEC can come in. + while (time_now + kFrameSizeMs >= next_send_time) { + next_send_time = rtp_generator_->GetRtpHeader(kPayloadType, + frame_size_samples_, + &rtp_header_); + if (!Lost(next_send_time)) { + static const uint8_t payload[kPayloadSizeByte] = {0}; + InsertPacket(rtp_header_, payload, next_send_time); + } + } + GetOutputAudio(&output_frame_); + time_now += kOutputLengthMs; + } + CheckNetworkStatistics(expects); + neteq()->FlushBuffers(); + } + + void DecodeFecTest() { + external_decoder_->set_fec_enabled(false); + NetEqNetworkStatsCheck expects = { + kIgnore, // current_buffer_size_ms + kIgnore, // preferred_buffer_size_ms + kIgnore, // jitter_peaks_found + kEqual, // packet_loss_rate + kEqual, // expand_rate + kEqual, // voice_expand_rate + kIgnore, // preemptive_rate + kEqual, // accelerate_rate + kEqual, // decoded_fec_rate + kEqual, // discarded_fec_rate + kIgnore, // clockdrift_ppm + kEqual, // added_zero_samples + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + }; + RunTest(50, expects); + + // Next we introduce packet losses. + SetPacketLossRate(0.1); + expects.stats_ref.packet_loss_rate = 1337; + expects.stats_ref.expand_rate = expects.stats_ref.speech_expand_rate = 1065; + RunTest(50, expects); + + // Next we enable FEC. + external_decoder_->set_fec_enabled(true); + // If FEC fills in the lost packets, no packet loss will be counted. + expects.stats_ref.packet_loss_rate = 0; + expects.stats_ref.expand_rate = expects.stats_ref.speech_expand_rate = 0; + expects.stats_ref.secondary_decoded_rate = 2006; + expects.stats_ref.secondary_discarded_rate = 14336; + RunTest(50, expects); + } + + void NoiseExpansionTest() { + NetEqNetworkStatsCheck expects = { + kIgnore, // current_buffer_size_ms + kIgnore, // preferred_buffer_size_ms + kIgnore, // jitter_peaks_found + kEqual, // packet_loss_rate + kEqual, // expand_rate + kEqual, // speech_expand_rate + kIgnore, // preemptive_rate + kEqual, // accelerate_rate + kEqual, // decoded_fec_rate + kEqual, // discard_fec_rate + kIgnore, // clockdrift_ppm + kEqual, // added_zero_samples + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + }; + RunTest(50, expects); + + SetPacketLossRate(1); + expects.stats_ref.expand_rate = 16384; + expects.stats_ref.speech_expand_rate = 5324; + RunTest(10, expects); // Lost 10 * 20ms in a row. + } + + private: + MockAudioDecoder* external_decoder_; + const int samples_per_ms_; + const size_t frame_size_samples_; + std::unique_ptr rtp_generator_; + RTPHeader rtp_header_; + uint32_t last_lost_time_; + uint32_t packet_loss_interval_; + AudioFrame output_frame_; +}; + +TEST(NetEqNetworkStatsTest, DecodeFec) { + MockAudioDecoder decoder(48000, 1); + NetEqNetworkStatsTest test(NetEqDecoder::kDecoderOpus, 48000, &decoder); + test.DecodeFecTest(); + EXPECT_CALL(decoder, Die()).Times(1); +} + +TEST(NetEqNetworkStatsTest, StereoDecodeFec) { + MockAudioDecoder decoder(48000, 2); + NetEqNetworkStatsTest test(NetEqDecoder::kDecoderOpus, 48000, &decoder); + test.DecodeFecTest(); + EXPECT_CALL(decoder, Die()).Times(1); +} + +TEST(NetEqNetworkStatsTest, NoiseExpansionTest) { + MockAudioDecoder decoder(48000, 1); + NetEqNetworkStatsTest test(NetEqDecoder::kDecoderOpus, 48000, &decoder); + test.NoiseExpansionTest(); + EXPECT_CALL(decoder, Die()).Times(1); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc new file mode 100644 index 0000000000..1bef9c83a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc @@ -0,0 +1,442 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Test to verify correct stereo and multi-channel operation. + +#include +#include +#include +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "modules/include/module_common_types.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +struct TestParameters { + int frame_size; + int sample_rate; + size_t num_channels; +}; + +// This is a parameterized test. The test parameters are supplied through a +// TestParameters struct, which is obtained through the GetParam() method. +// +// The objective of the test is to create a mono input signal and a +// multi-channel input signal, where each channel is identical to the mono +// input channel. The two input signals are processed through their respective +// NetEq instances. After that, the output signals are compared. The expected +// result is that each channel in the multi-channel output is identical to the +// mono output. +class NetEqStereoTest : public ::testing::TestWithParam { + protected: + static const int kTimeStepMs = 10; + static const size_t kMaxBlockSize = 480; // 10 ms @ 48 kHz. + static const uint8_t kPayloadTypeMono = 95; + static const uint8_t kPayloadTypeMulti = 96; + + NetEqStereoTest() + : num_channels_(GetParam().num_channels), + sample_rate_hz_(GetParam().sample_rate), + samples_per_ms_(sample_rate_hz_ / 1000), + frame_size_ms_(GetParam().frame_size), + frame_size_samples_( + static_cast(frame_size_ms_ * samples_per_ms_)), + output_size_samples_(10 * samples_per_ms_), + rtp_generator_mono_(samples_per_ms_), + rtp_generator_(samples_per_ms_), + payload_size_bytes_(0), + multi_payload_size_bytes_(0), + last_send_time_(0), + last_arrival_time_(0) { + NetEq::Config config; + config.sample_rate_hz = sample_rate_hz_; + rtc::scoped_refptr factory = + CreateBuiltinAudioDecoderFactory(); + neteq_mono_ = NetEq::Create(config, factory); + neteq_ = NetEq::Create(config, factory); + input_ = new int16_t[frame_size_samples_]; + encoded_ = new uint8_t[2 * frame_size_samples_]; + input_multi_channel_ = new int16_t[frame_size_samples_ * num_channels_]; + encoded_multi_channel_ = new uint8_t[frame_size_samples_ * 2 * + num_channels_]; + } + + ~NetEqStereoTest() { + delete neteq_mono_; + delete neteq_; + delete [] input_; + delete [] encoded_; + delete [] input_multi_channel_; + delete [] encoded_multi_channel_; + } + + virtual void SetUp() { + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + input_file_.reset(new test::InputAudioFile(file_name)); + NetEqDecoder mono_decoder; + NetEqDecoder multi_decoder; + switch (sample_rate_hz_) { + case 8000: + mono_decoder = NetEqDecoder::kDecoderPCM16B; + if (num_channels_ == 2) { + multi_decoder = NetEqDecoder::kDecoderPCM16B_2ch; + } else if (num_channels_ == 5) { + multi_decoder = NetEqDecoder::kDecoderPCM16B_5ch; + } else { + FAIL() << "Only 2 and 5 channels supported for 8000 Hz."; + } + break; + case 16000: + mono_decoder = NetEqDecoder::kDecoderPCM16Bwb; + if (num_channels_ == 2) { + multi_decoder = NetEqDecoder::kDecoderPCM16Bwb_2ch; + } else { + FAIL() << "More than 2 channels is not supported for 16000 Hz."; + } + break; + case 32000: + mono_decoder = NetEqDecoder::kDecoderPCM16Bswb32kHz; + if (num_channels_ == 2) { + multi_decoder = NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch; + } else { + FAIL() << "More than 2 channels is not supported for 32000 Hz."; + } + break; + case 48000: + mono_decoder = NetEqDecoder::kDecoderPCM16Bswb48kHz; + if (num_channels_ == 2) { + multi_decoder = NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch; + } else { + FAIL() << "More than 2 channels is not supported for 48000 Hz."; + } + break; + default: + FAIL() << "We shouldn't get here."; + } + ASSERT_EQ(NetEq::kOK, neteq_mono_->RegisterPayloadType(mono_decoder, "mono", + kPayloadTypeMono)); + ASSERT_EQ(NetEq::kOK, + neteq_->RegisterPayloadType(multi_decoder, "multi-channel", + kPayloadTypeMulti)); + } + + virtual void TearDown() {} + + int GetNewPackets() { + if (!input_file_->Read(frame_size_samples_, input_)) { + return -1; + } + payload_size_bytes_ = WebRtcPcm16b_Encode(input_, frame_size_samples_, + encoded_); + if (frame_size_samples_ * 2 != payload_size_bytes_) { + return -1; + } + int next_send_time = rtp_generator_mono_.GetRtpHeader(kPayloadTypeMono, + frame_size_samples_, + &rtp_header_mono_); + test::InputAudioFile::DuplicateInterleaved(input_, frame_size_samples_, + num_channels_, + input_multi_channel_); + multi_payload_size_bytes_ = WebRtcPcm16b_Encode( + input_multi_channel_, frame_size_samples_ * num_channels_, + encoded_multi_channel_); + if (frame_size_samples_ * 2 * num_channels_ != multi_payload_size_bytes_) { + return -1; + } + rtp_generator_.GetRtpHeader(kPayloadTypeMulti, frame_size_samples_, + &rtp_header_); + return next_send_time; + } + + virtual void VerifyOutput(size_t num_samples) { + const int16_t* output_data = output_.data(); + const int16_t* output_multi_channel_data = output_multi_channel_.data(); + for (size_t i = 0; i < num_samples; ++i) { + for (size_t j = 0; j < num_channels_; ++j) { + ASSERT_EQ(output_data[i], + output_multi_channel_data[i * num_channels_ + j]) + << "Diff in sample " << i << ", channel " << j << "."; + } + } + } + + virtual int GetArrivalTime(int send_time) { + int arrival_time = last_arrival_time_ + (send_time - last_send_time_); + last_send_time_ = send_time; + last_arrival_time_ = arrival_time; + return arrival_time; + } + + virtual bool Lost() { return false; } + + void RunTest(int num_loops) { + // Get next input packets (mono and multi-channel). + int next_send_time; + int next_arrival_time; + do { + next_send_time = GetNewPackets(); + ASSERT_NE(-1, next_send_time); + next_arrival_time = GetArrivalTime(next_send_time); + } while (Lost()); // If lost, immediately read the next packet. + + int time_now = 0; + for (int k = 0; k < num_loops; ++k) { + while (time_now >= next_arrival_time) { + // Insert packet in mono instance. + ASSERT_EQ(NetEq::kOK, + neteq_mono_->InsertPacket(rtp_header_mono_, + rtc::ArrayView( + encoded_, payload_size_bytes_), + next_arrival_time)); + // Insert packet in multi-channel instance. + ASSERT_EQ(NetEq::kOK, + neteq_->InsertPacket( + rtp_header_, + rtc::ArrayView(encoded_multi_channel_, + multi_payload_size_bytes_), + next_arrival_time)); + // Get next input packets (mono and multi-channel). + do { + next_send_time = GetNewPackets(); + ASSERT_NE(-1, next_send_time); + next_arrival_time = GetArrivalTime(next_send_time); + } while (Lost()); // If lost, immediately read the next packet. + } + // Get audio from mono instance. + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_mono_->GetAudio(&output_, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(1u, output_.num_channels_); + EXPECT_EQ(output_size_samples_, output_.samples_per_channel_); + // Get audio from multi-channel instance. + ASSERT_EQ(NetEq::kOK, neteq_->GetAudio(&output_multi_channel_, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(num_channels_, output_multi_channel_.num_channels_); + EXPECT_EQ(output_size_samples_, + output_multi_channel_.samples_per_channel_); + std::ostringstream ss; + ss << "Lap number " << k << "."; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + // Compare mono and multi-channel. + ASSERT_NO_FATAL_FAILURE(VerifyOutput(output_size_samples_)); + + time_now += kTimeStepMs; + } + } + + const size_t num_channels_; + const int sample_rate_hz_; + const int samples_per_ms_; + const int frame_size_ms_; + const size_t frame_size_samples_; + const size_t output_size_samples_; + NetEq* neteq_mono_; + NetEq* neteq_; + test::RtpGenerator rtp_generator_mono_; + test::RtpGenerator rtp_generator_; + int16_t* input_; + int16_t* input_multi_channel_; + uint8_t* encoded_; + uint8_t* encoded_multi_channel_; + AudioFrame output_; + AudioFrame output_multi_channel_; + RTPHeader rtp_header_mono_; + RTPHeader rtp_header_; + size_t payload_size_bytes_; + size_t multi_payload_size_bytes_; + int last_send_time_; + int last_arrival_time_; + std::unique_ptr input_file_; +}; + +class NetEqStereoTestNoJitter : public NetEqStereoTest { + protected: + NetEqStereoTestNoJitter() + : NetEqStereoTest() { + // Start the sender 100 ms before the receiver to pre-fill the buffer. + // This is to avoid doing preemptive expand early in the test. + // TODO(hlundin): Mock the decision making instead to control the modes. + last_arrival_time_ = -100; + } +}; + +TEST_P(NetEqStereoTestNoJitter, RunTest) { + RunTest(8); +} + +class NetEqStereoTestPositiveDrift : public NetEqStereoTest { + protected: + NetEqStereoTestPositiveDrift() + : NetEqStereoTest(), + drift_factor(0.9) { + // Start the sender 100 ms before the receiver to pre-fill the buffer. + // This is to avoid doing preemptive expand early in the test. + // TODO(hlundin): Mock the decision making instead to control the modes. + last_arrival_time_ = -100; + } + virtual int GetArrivalTime(int send_time) { + int arrival_time = last_arrival_time_ + + drift_factor * (send_time - last_send_time_); + last_send_time_ = send_time; + last_arrival_time_ = arrival_time; + return arrival_time; + } + + double drift_factor; +}; + +TEST_P(NetEqStereoTestPositiveDrift, RunTest) { + RunTest(100); +} + +class NetEqStereoTestNegativeDrift : public NetEqStereoTestPositiveDrift { + protected: + NetEqStereoTestNegativeDrift() + : NetEqStereoTestPositiveDrift() { + drift_factor = 1.1; + last_arrival_time_ = 0; + } +}; + +TEST_P(NetEqStereoTestNegativeDrift, RunTest) { + RunTest(100); +} + +class NetEqStereoTestDelays : public NetEqStereoTest { + protected: + static const int kDelayInterval = 10; + static const int kDelay = 1000; + NetEqStereoTestDelays() + : NetEqStereoTest(), + frame_index_(0) { + } + + virtual int GetArrivalTime(int send_time) { + // Deliver immediately, unless we have a back-log. + int arrival_time = std::min(last_arrival_time_, send_time); + if (++frame_index_ % kDelayInterval == 0) { + // Delay this packet. + arrival_time += kDelay; + } + last_send_time_ = send_time; + last_arrival_time_ = arrival_time; + return arrival_time; + } + + int frame_index_; +}; + +TEST_P(NetEqStereoTestDelays, RunTest) { + RunTest(1000); +} + +class NetEqStereoTestLosses : public NetEqStereoTest { + protected: + static const int kLossInterval = 10; + NetEqStereoTestLosses() + : NetEqStereoTest(), + frame_index_(0) { + } + + virtual bool Lost() { + return (++frame_index_) % kLossInterval == 0; + } + + // TODO(hlundin): NetEq is not giving bitexact results for these cases. + virtual void VerifyOutput(size_t num_samples) { + for (size_t i = 0; i < num_samples; ++i) { + const int16_t* output_data = output_.data(); + const int16_t* output_multi_channel_data = output_multi_channel_.data(); + auto first_channel_sample = + output_multi_channel_data[i * num_channels_]; + for (size_t j = 0; j < num_channels_; ++j) { + const int kErrorMargin = 200; + EXPECT_NEAR(output_data[i], + output_multi_channel_data[i * num_channels_ + j], + kErrorMargin) + << "Diff in sample " << i << ", channel " << j << "."; + EXPECT_EQ(first_channel_sample, + output_multi_channel_data[i * num_channels_ + j]); + } + } + } + + int frame_index_; +}; + +TEST_P(NetEqStereoTestLosses, RunTest) { + RunTest(100); +} + + +// Creates a list of parameter sets. +std::list GetTestParameters() { + std::list l; + const int sample_rates[] = {8000, 16000, 32000}; + const int num_rates = sizeof(sample_rates) / sizeof(sample_rates[0]); + // Loop through sample rates. + for (int rate_index = 0; rate_index < num_rates; ++rate_index) { + int sample_rate = sample_rates[rate_index]; + // Loop through all frame sizes between 10 and 60 ms. + for (int frame_size = 10; frame_size <= 60; frame_size += 10) { + TestParameters p; + p.frame_size = frame_size; + p.sample_rate = sample_rate; + p.num_channels = 2; + l.push_back(p); + if (sample_rate == 8000) { + // Add a five-channel test for 8000 Hz. + p.num_channels = 5; + l.push_back(p); + } + } + } + return l; +} + +// Pretty-printing the test parameters in case of an error. +void PrintTo(const TestParameters& p, ::std::ostream* os) { + *os << "{frame_size = " << p.frame_size << + ", num_channels = " << p.num_channels << + ", sample_rate = " << p.sample_rate << "}"; +} + +// Instantiate the tests. Each test is instantiated using the function above, +// so that all different parameter combinations are tested. +INSTANTIATE_TEST_CASE_P(MultiChannel, + NetEqStereoTestNoJitter, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_CASE_P(MultiChannel, + NetEqStereoTestPositiveDrift, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_CASE_P(MultiChannel, + NetEqStereoTestNegativeDrift, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_CASE_P(MultiChannel, + NetEqStereoTestDelays, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_CASE_P(MultiChannel, + NetEqStereoTestLosses, + ::testing::ValuesIn(GetTestParameters())); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.cc new file mode 100644 index 0000000000..e97d7c2d54 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.cc @@ -0,0 +1,1804 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/include/neteq.h" + +#include +#include +#include // memset + +#include +#include +#include +#include +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/flags.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/protobuf_utils.h" +#include "rtc_base/sha1digest.h" +#include "rtc_base/stringencode.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.pb.h" +#else +#include "modules/audio_coding/neteq/neteq_unittest.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +DEFINE_bool(gen_ref, false, "Generate reference files."); + +namespace webrtc { + +namespace { + +const std::string& PlatformChecksum(const std::string& checksum_general, + const std::string& checksum_android_32, + const std::string& checksum_android_64, + const std::string& checksum_win_32, + const std::string& checksum_win_64) { +#if defined(WEBRTC_ANDROID) + #ifdef WEBRTC_ARCH_64_BITS + return checksum_android_64; + #else + return checksum_android_32; + #endif // WEBRTC_ARCH_64_BITS +#elif defined(WEBRTC_WIN) + #ifdef WEBRTC_ARCH_64_BITS + return checksum_win_64; + #else + return checksum_win_32; + #endif // WEBRTC_ARCH_64_BITS +#else + return checksum_general; +#endif // WEBRTC_WIN +} + +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT +void Convert(const webrtc::NetEqNetworkStatistics& stats_raw, + webrtc::neteq_unittest::NetEqNetworkStatistics* stats) { + stats->set_current_buffer_size_ms(stats_raw.current_buffer_size_ms); + stats->set_preferred_buffer_size_ms(stats_raw.preferred_buffer_size_ms); + stats->set_jitter_peaks_found(stats_raw.jitter_peaks_found); + stats->set_packet_loss_rate(stats_raw.packet_loss_rate); + stats->set_expand_rate(stats_raw.expand_rate); + stats->set_speech_expand_rate(stats_raw.speech_expand_rate); + stats->set_preemptive_rate(stats_raw.preemptive_rate); + stats->set_accelerate_rate(stats_raw.accelerate_rate); + stats->set_secondary_decoded_rate(stats_raw.secondary_decoded_rate); + stats->set_secondary_discarded_rate(stats_raw.secondary_discarded_rate); + stats->set_clockdrift_ppm(stats_raw.clockdrift_ppm); + stats->set_added_zero_samples(stats_raw.added_zero_samples); + stats->set_mean_waiting_time_ms(stats_raw.mean_waiting_time_ms); + stats->set_median_waiting_time_ms(stats_raw.median_waiting_time_ms); + stats->set_min_waiting_time_ms(stats_raw.min_waiting_time_ms); + stats->set_max_waiting_time_ms(stats_raw.max_waiting_time_ms); +} + +void Convert(const webrtc::RtcpStatistics& stats_raw, + webrtc::neteq_unittest::RtcpStatistics* stats) { + stats->set_fraction_lost(stats_raw.fraction_lost); + stats->set_cumulative_lost(stats_raw.packets_lost); + stats->set_extended_max_sequence_number( + stats_raw.extended_highest_sequence_number); + stats->set_jitter(stats_raw.jitter); +} + +void AddMessage(FILE* file, rtc::MessageDigest* digest, + const std::string& message) { + int32_t size = message.length(); + if (file) + ASSERT_EQ(1u, fwrite(&size, sizeof(size), 1, file)); + digest->Update(&size, sizeof(size)); + + if (file) + ASSERT_EQ(static_cast(size), + fwrite(message.data(), sizeof(char), size, file)); + digest->Update(message.data(), sizeof(char) * size); +} + +#endif // WEBRTC_NETEQ_UNITTEST_BITEXACT + +void LoadDecoders(webrtc::NetEq* neteq) { + ASSERT_EQ(true, + neteq->RegisterPayloadType(0, SdpAudioFormat("pcmu", 8000, 1))); + // Use non-SdpAudioFormat argument when registering PCMa, so that we get test + // coverage for that as well. + ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderPCMa, + "pcma", 8)); +#ifdef WEBRTC_CODEC_ILBC + ASSERT_EQ(true, + neteq->RegisterPayloadType(102, SdpAudioFormat("ilbc", 8000, 1))); +#endif +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) + ASSERT_EQ(true, + neteq->RegisterPayloadType(103, SdpAudioFormat("isac", 16000, 1))); +#endif +#ifdef WEBRTC_CODEC_ISAC + ASSERT_EQ(true, + neteq->RegisterPayloadType(104, SdpAudioFormat("isac", 32000, 1))); +#endif +#ifdef WEBRTC_CODEC_OPUS + ASSERT_EQ(true, + neteq->RegisterPayloadType( + 111, SdpAudioFormat("opus", 48000, 2, {{"stereo", "0"}}))); +#endif + ASSERT_EQ(true, + neteq->RegisterPayloadType(93, SdpAudioFormat("L16", 8000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(94, SdpAudioFormat("L16", 16000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(95, SdpAudioFormat("L16", 32000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(13, SdpAudioFormat("cn", 8000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(98, SdpAudioFormat("cn", 16000, 1))); +} +} // namespace + +class ResultSink { + public: + explicit ResultSink(const std::string& output_file); + ~ResultSink(); + + template void AddResult(const T* test_results, size_t length); + + void AddResult(const NetEqNetworkStatistics& stats); + void AddResult(const RtcpStatistics& stats); + + void VerifyChecksum(const std::string& ref_check_sum); + + private: + FILE* output_fp_; + std::unique_ptr digest_; +}; + +ResultSink::ResultSink(const std::string &output_file) + : output_fp_(nullptr), + digest_(new rtc::Sha1Digest()) { + if (!output_file.empty()) { + output_fp_ = fopen(output_file.c_str(), "wb"); + EXPECT_TRUE(output_fp_ != NULL); + } +} + +ResultSink::~ResultSink() { + if (output_fp_) + fclose(output_fp_); +} + +template +void ResultSink::AddResult(const T* test_results, size_t length) { + if (output_fp_) { + ASSERT_EQ(length, fwrite(test_results, sizeof(T), length, output_fp_)); + } + digest_->Update(test_results, sizeof(T) * length); +} + +void ResultSink::AddResult(const NetEqNetworkStatistics& stats_raw) { +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT + neteq_unittest::NetEqNetworkStatistics stats; + Convert(stats_raw, &stats); + + ProtoString stats_string; + ASSERT_TRUE(stats.SerializeToString(&stats_string)); + AddMessage(output_fp_, digest_.get(), stats_string); +#else + FAIL() << "Writing to reference file requires Proto Buffer."; +#endif // WEBRTC_NETEQ_UNITTEST_BITEXACT +} + +void ResultSink::AddResult(const RtcpStatistics& stats_raw) { +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT + neteq_unittest::RtcpStatistics stats; + Convert(stats_raw, &stats); + + ProtoString stats_string; + ASSERT_TRUE(stats.SerializeToString(&stats_string)); + AddMessage(output_fp_, digest_.get(), stats_string); +#else + FAIL() << "Writing to reference file requires Proto Buffer."; +#endif // WEBRTC_NETEQ_UNITTEST_BITEXACT +} + +void ResultSink::VerifyChecksum(const std::string& checksum) { + std::vector buffer; + buffer.resize(digest_->Size()); + digest_->Finish(&buffer[0], buffer.size()); + const std::string result = rtc::hex_encode(&buffer[0], digest_->Size()); + EXPECT_EQ(checksum, result); +} + +class NetEqDecodingTest : public ::testing::Test { + protected: + // NetEQ must be polled for data once every 10 ms. Thus, neither of the + // constants below can be changed. + static const int kTimeStepMs = 10; + static const size_t kBlockSize8kHz = kTimeStepMs * 8; + static const size_t kBlockSize16kHz = kTimeStepMs * 16; + static const size_t kBlockSize32kHz = kTimeStepMs * 32; + static const size_t kBlockSize48kHz = kTimeStepMs * 48; + static const int kInitSampleRateHz = 8000; + + NetEqDecodingTest(); + virtual void SetUp(); + virtual void TearDown(); + void SelectDecoders(NetEqDecoder* used_codec); + void OpenInputFile(const std::string &rtp_file); + void Process(); + + void DecodeAndCompare(const std::string& rtp_file, + const std::string& output_checksum, + const std::string& network_stats_checksum, + const std::string& rtcp_stats_checksum, + bool gen_ref); + + static void PopulateRtpInfo(int frame_index, + int timestamp, + RTPHeader* rtp_info); + static void PopulateCng(int frame_index, + int timestamp, + RTPHeader* rtp_info, + uint8_t* payload, + size_t* payload_len); + + void WrapTest(uint16_t start_seq_no, uint32_t start_timestamp, + const std::set& drop_seq_numbers, + bool expect_seq_no_wrap, bool expect_timestamp_wrap); + + void LongCngWithClockDrift(double drift_factor, + double network_freeze_ms, + bool pull_audio_during_freeze, + int delay_tolerance_ms, + int max_time_to_speech_ms); + + void DuplicateCng(); + + NetEq* neteq_; + NetEq::Config config_; + std::unique_ptr rtp_source_; + std::unique_ptr packet_; + unsigned int sim_clock_; + AudioFrame out_frame_; + int output_sample_rate_; + int algorithmic_delay_ms_; +}; + +// Allocating the static const so that it can be passed by reference. +const int NetEqDecodingTest::kTimeStepMs; +const size_t NetEqDecodingTest::kBlockSize8kHz; +const size_t NetEqDecodingTest::kBlockSize16kHz; +const size_t NetEqDecodingTest::kBlockSize32kHz; +const int NetEqDecodingTest::kInitSampleRateHz; + +NetEqDecodingTest::NetEqDecodingTest() + : neteq_(NULL), + config_(), + sim_clock_(0), + output_sample_rate_(kInitSampleRateHz), + algorithmic_delay_ms_(0) { + config_.sample_rate_hz = kInitSampleRateHz; +} + +void NetEqDecodingTest::SetUp() { + neteq_ = NetEq::Create(config_, CreateBuiltinAudioDecoderFactory()); + NetEqNetworkStatistics stat; + ASSERT_EQ(0, neteq_->NetworkStatistics(&stat)); + algorithmic_delay_ms_ = stat.current_buffer_size_ms; + ASSERT_TRUE(neteq_); + LoadDecoders(neteq_); +} + +void NetEqDecodingTest::TearDown() { + delete neteq_; +} + +void NetEqDecodingTest::OpenInputFile(const std::string &rtp_file) { + rtp_source_.reset(test::RtpFileSource::Create(rtp_file)); +} + +void NetEqDecodingTest::Process() { + // Check if time to receive. + while (packet_ && sim_clock_ >= packet_->time_ms()) { + if (packet_->payload_length_bytes() > 0) { +#ifndef WEBRTC_CODEC_ISAC + // Ignore payload type 104 (iSAC-swb) if ISAC is not supported. + if (packet_->header().payloadType != 104) +#endif + ASSERT_EQ(0, + neteq_->InsertPacket( + packet_->header(), + rtc::ArrayView( + packet_->payload(), packet_->payload_length_bytes()), + static_cast(packet_->time_ms() * + (output_sample_rate_ / 1000)))); + } + // Get next packet. + packet_ = rtp_source_->NextPacket(); + } + + // Get audio from NetEq. + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + ASSERT_TRUE((out_frame_.samples_per_channel_ == kBlockSize8kHz) || + (out_frame_.samples_per_channel_ == kBlockSize16kHz) || + (out_frame_.samples_per_channel_ == kBlockSize32kHz) || + (out_frame_.samples_per_channel_ == kBlockSize48kHz)); + output_sample_rate_ = out_frame_.sample_rate_hz_; + EXPECT_EQ(output_sample_rate_, neteq_->last_output_sample_rate_hz()); + + // Increase time. + sim_clock_ += kTimeStepMs; +} + +void NetEqDecodingTest::DecodeAndCompare( + const std::string& rtp_file, + const std::string& output_checksum, + const std::string& network_stats_checksum, + const std::string& rtcp_stats_checksum, + bool gen_ref) { + OpenInputFile(rtp_file); + + std::string ref_out_file = + gen_ref ? webrtc::test::OutputPath() + "neteq_universal_ref.pcm" : ""; + ResultSink output(ref_out_file); + + std::string stat_out_file = + gen_ref ? webrtc::test::OutputPath() + "neteq_network_stats.dat" : ""; + ResultSink network_stats(stat_out_file); + + std::string rtcp_out_file = + gen_ref ? webrtc::test::OutputPath() + "neteq_rtcp_stats.dat" : ""; + ResultSink rtcp_stats(rtcp_out_file); + + packet_ = rtp_source_->NextPacket(); + int i = 0; + uint64_t last_concealed_samples = 0; + uint64_t last_total_samples_received = 0; + while (packet_) { + std::ostringstream ss; + ss << "Lap number " << i++ << " in DecodeAndCompare while loop"; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + ASSERT_NO_FATAL_FAILURE(Process()); + ASSERT_NO_FATAL_FAILURE(output.AddResult( + out_frame_.data(), out_frame_.samples_per_channel_)); + + // Query the network statistics API once per second + if (sim_clock_ % 1000 == 0) { + // Process NetworkStatistics. + NetEqNetworkStatistics current_network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(¤t_network_stats)); + ASSERT_NO_FATAL_FAILURE(network_stats.AddResult(current_network_stats)); + + // Compare with CurrentDelay, which should be identical. + EXPECT_EQ(current_network_stats.current_buffer_size_ms, + neteq_->CurrentDelayMs()); + + // Verify that liftime stats and network stats report similar loss + // concealment rates. + auto lifetime_stats = neteq_->GetLifetimeStatistics(); + const uint64_t delta_concealed_samples = + lifetime_stats.concealed_samples - last_concealed_samples; + last_concealed_samples = lifetime_stats.concealed_samples; + const uint64_t delta_total_samples_received = + lifetime_stats.total_samples_received - last_total_samples_received; + last_total_samples_received = lifetime_stats.total_samples_received; + // The tolerance is 1% but expressed in Q14. + EXPECT_NEAR( + (delta_concealed_samples << 14) / delta_total_samples_received, + current_network_stats.expand_rate, (2 << 14) / 100.0); + + // Process RTCPstat. + RtcpStatistics current_rtcp_stats; + neteq_->GetRtcpStatistics(¤t_rtcp_stats); + ASSERT_NO_FATAL_FAILURE(rtcp_stats.AddResult(current_rtcp_stats)); + } + } + + SCOPED_TRACE("Check output audio."); + output.VerifyChecksum(output_checksum); + SCOPED_TRACE("Check network stats."); + network_stats.VerifyChecksum(network_stats_checksum); + SCOPED_TRACE("Check rtcp stats."); + rtcp_stats.VerifyChecksum(rtcp_stats_checksum); +} + +void NetEqDecodingTest::PopulateRtpInfo(int frame_index, + int timestamp, + RTPHeader* rtp_info) { + rtp_info->sequenceNumber = frame_index; + rtp_info->timestamp = timestamp; + rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info->payloadType = 94; // PCM16b WB codec. + rtp_info->markerBit = 0; +} + +void NetEqDecodingTest::PopulateCng(int frame_index, + int timestamp, + RTPHeader* rtp_info, + uint8_t* payload, + size_t* payload_len) { + rtp_info->sequenceNumber = frame_index; + rtp_info->timestamp = timestamp; + rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info->payloadType = 98; // WB CNG. + rtp_info->markerBit = 0; + payload[0] = 64; // Noise level -64 dBov, quite arbitrarily chosen. + *payload_len = 1; // Only noise level, no spectral parameters. +} + +#if !defined(WEBRTC_IOS) && defined(WEBRTC_NETEQ_UNITTEST_BITEXACT) && \ + (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) && \ + defined(WEBRTC_CODEC_ILBC) && !defined(WEBRTC_ARCH_ARM64) +#define MAYBE_TestBitExactness TestBitExactness +#else +#define MAYBE_TestBitExactness DISABLED_TestBitExactness +#endif +TEST_F(NetEqDecodingTest, MAYBE_TestBitExactness) { + const std::string input_rtp_file = + webrtc::test::ResourcePath("audio_coding/neteq_universal_new", "rtp"); + + const std::string output_checksum = PlatformChecksum( + "09fa7646e2ad032a0b156177b95f09012430f81f", + "1c64eb8b55ce8878676c6a1e6ddd78f48de0668b", + "not used", + "09fa7646e2ad032a0b156177b95f09012430f81f", + "759fef89a5de52bd17e733dc255c671ce86be909"); + + const std::string network_stats_checksum = + PlatformChecksum("5b4262ca328e5f066af5d34f3380521583dd20de", + "80235b6d727281203acb63b98f9a9e85d95f7ec0", + "not used", + "5b4262ca328e5f066af5d34f3380521583dd20de", + "5b4262ca328e5f066af5d34f3380521583dd20de"); + + const std::string rtcp_stats_checksum = PlatformChecksum( + "b8880bf9fed2487efbddcb8d94b9937a29ae521d", + "f3f7b3d3e71d7e635240b5373b57df6a7e4ce9d4", + "not used", + "b8880bf9fed2487efbddcb8d94b9937a29ae521d", + "b8880bf9fed2487efbddcb8d94b9937a29ae521d"); + + DecodeAndCompare(input_rtp_file, + output_checksum, + network_stats_checksum, + rtcp_stats_checksum, + FLAG_gen_ref); +} + +#if !defined(WEBRTC_IOS) && \ + defined(WEBRTC_NETEQ_UNITTEST_BITEXACT) && \ + defined(WEBRTC_CODEC_OPUS) +#define MAYBE_TestOpusBitExactness TestOpusBitExactness +#else +#define MAYBE_TestOpusBitExactness DISABLED_TestOpusBitExactness +#endif +TEST_F(NetEqDecodingTest, MAYBE_TestOpusBitExactness) { + const std::string input_rtp_file = + webrtc::test::ResourcePath("audio_coding/neteq_opus", "rtp"); + + const std::string output_checksum = PlatformChecksum( + "7ea28d7edf9395f4ac8e8d8dd3a9e5c620b1bf48", + "5b1e691ab1c4465c742d6d944bc71e3b1c0e4c0e", + "b096114dd8c233eaf2b0ce9802ac95af13933772", + "7ea28d7edf9395f4ac8e8d8dd3a9e5c620b1bf48", + "7ea28d7edf9395f4ac8e8d8dd3a9e5c620b1bf48"); + + const std::string network_stats_checksum = + PlatformChecksum("9e72233c78baf685e500dd6c94212b30a4c5f27d", + "9a37270e4242fbd31e80bb47dc5e7ab82cf2d557", + "4f1e9734bc80a290faaf9d611efcb8d7802dbc4f", + "9e72233c78baf685e500dd6c94212b30a4c5f27d", + "9e72233c78baf685e500dd6c94212b30a4c5f27d"); + + const std::string rtcp_stats_checksum = PlatformChecksum( + "e37c797e3de6a64dda88c9ade7a013d022a2e1e0", + "e37c797e3de6a64dda88c9ade7a013d022a2e1e0", + "e37c797e3de6a64dda88c9ade7a013d022a2e1e0", + "e37c797e3de6a64dda88c9ade7a013d022a2e1e0", + "e37c797e3de6a64dda88c9ade7a013d022a2e1e0"); + + DecodeAndCompare(input_rtp_file, + output_checksum, + network_stats_checksum, + rtcp_stats_checksum, + FLAG_gen_ref); +} + +// This test fixture is identical to NetEqDecodingTest, except that it enables +// the WebRTC-NetEqOpusDtxDelayFix field trial. +// TODO(bugs.webrtc.org/8488): When the field trial is over and the feature is +// default enabled, remove this fixture class and let the +// TestOpusDtxBitExactness test build directly on NetEqDecodingTest. +class NetEqDecodingTestWithOpusDtxFieldTrial : public NetEqDecodingTest { + public: + NetEqDecodingTestWithOpusDtxFieldTrial() + : override_field_trials_("WebRTC-NetEqOpusDtxDelayFix/Enabled/") {} + + private: + test::ScopedFieldTrials override_field_trials_; +}; + +#if !defined(WEBRTC_IOS) && \ + defined(WEBRTC_NETEQ_UNITTEST_BITEXACT) && \ + defined(WEBRTC_CODEC_OPUS) +#define MAYBE_TestOpusDtxBitExactness TestOpusDtxBitExactness +#else +#define MAYBE_TestOpusDtxBitExactness DISABLED_TestOpusDtxBitExactness +#endif +TEST_F(NetEqDecodingTestWithOpusDtxFieldTrial, MAYBE_TestOpusDtxBitExactness) { + const std::string input_rtp_file = + webrtc::test::ResourcePath("audio_coding/neteq_opus_dtx", "rtp"); + + const std::string output_checksum = + PlatformChecksum("713af6c92881f5aab1285765ee6680da9d1c06ce", + "3ec991b96872123f1554c03c543ca5d518431e46", + "da9f9a2d94e0c2d67342fad4965d7b91cda50b25", + "713af6c92881f5aab1285765ee6680da9d1c06ce", + "713af6c92881f5aab1285765ee6680da9d1c06ce"); + + const std::string network_stats_checksum = + "bab58dc587d956f326056d7340c96eb9d2d3cc21"; + + const std::string rtcp_stats_checksum = + "ac27a7f305efb58b39bf123dccee25dee5758e63"; + + DecodeAndCompare(input_rtp_file, output_checksum, network_stats_checksum, + rtcp_stats_checksum, FLAG_gen_ref); +} + +// Use fax mode to avoid time-scaling. This is to simplify the testing of +// packet waiting times in the packet buffer. +class NetEqDecodingTestFaxMode : public NetEqDecodingTest { + protected: + NetEqDecodingTestFaxMode() : NetEqDecodingTest() { + config_.playout_mode = kPlayoutFax; + } + void TestJitterBufferDelay(bool apply_packet_loss); +}; + +TEST_F(NetEqDecodingTestFaxMode, TestFrameWaitingTimeStatistics) { + // Insert 30 dummy packets at once. Each packet contains 10 ms 16 kHz audio. + size_t num_frames = 30; + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + for (size_t i = 0; i < num_frames; ++i) { + const uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + rtp_info.sequenceNumber = rtc::checked_cast(i); + rtp_info.timestamp = rtc::checked_cast(i * kSamples); + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + } + // Pull out all data. + for (size_t i = 0; i < num_frames; ++i) { + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + NetEqNetworkStatistics stats; + EXPECT_EQ(0, neteq_->NetworkStatistics(&stats)); + // Since all frames are dumped into NetEQ at once, but pulled out with 10 ms + // spacing (per definition), we expect the delay to increase with 10 ms for + // each packet. Thus, we are calculating the statistics for a series from 10 + // to 300, in steps of 10 ms. + EXPECT_EQ(155, stats.mean_waiting_time_ms); + EXPECT_EQ(155, stats.median_waiting_time_ms); + EXPECT_EQ(10, stats.min_waiting_time_ms); + EXPECT_EQ(300, stats.max_waiting_time_ms); + + // Check statistics again and make sure it's been reset. + EXPECT_EQ(0, neteq_->NetworkStatistics(&stats)); + EXPECT_EQ(-1, stats.mean_waiting_time_ms); + EXPECT_EQ(-1, stats.median_waiting_time_ms); + EXPECT_EQ(-1, stats.min_waiting_time_ms); + EXPECT_EQ(-1, stats.max_waiting_time_ms); +} + +TEST_F(NetEqDecodingTest, TestAverageInterArrivalTimeNegative) { + const int kNumFrames = 3000; // Needed for convergence. + int frame_index = 0; + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + while (frame_index < kNumFrames) { + // Insert one packet each time, except every 10th time where we insert two + // packets at once. This will create a negative clock-drift of approx. 10%. + int num_packets = (frame_index % 10 == 0 ? 2 : 1); + for (int n = 0; n < num_packets; ++n) { + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(frame_index, frame_index * kSamples, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + ++frame_index; + } + + // Pull out data once. + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + NetEqNetworkStatistics network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats)); + EXPECT_EQ(-103192, network_stats.clockdrift_ppm); +} + +TEST_F(NetEqDecodingTest, TestAverageInterArrivalTimePositive) { + const int kNumFrames = 5000; // Needed for convergence. + int frame_index = 0; + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + for (int i = 0; i < kNumFrames; ++i) { + // Insert one packet each time, except every 10th time where we don't insert + // any packet. This will create a positive clock-drift of approx. 11%. + int num_packets = (i % 10 == 9 ? 0 : 1); + for (int n = 0; n < num_packets; ++n) { + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(frame_index, frame_index * kSamples, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + ++frame_index; + } + + // Pull out data once. + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + NetEqNetworkStatistics network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats)); + EXPECT_EQ(110953, network_stats.clockdrift_ppm); +} + +void NetEqDecodingTest::LongCngWithClockDrift(double drift_factor, + double network_freeze_ms, + bool pull_audio_during_freeze, + int delay_tolerance_ms, + int max_time_to_speech_ms) { + uint16_t seq_no = 0; + uint32_t timestamp = 0; + const int kFrameSizeMs = 30; + const size_t kSamples = kFrameSizeMs * 16; + const size_t kPayloadBytes = kSamples * 2; + double next_input_time_ms = 0.0; + double t_ms; + bool muted; + + // Insert speech for 5 seconds. + const int kSpeechDurationMs = 5000; + for (t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one 30 ms speech frame. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + ++seq_no; + timestamp += kSamples; + next_input_time_ms += static_cast(kFrameSizeMs) * drift_factor; + } + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + rtc::Optional playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + int32_t delay_before = timestamp - *playout_timestamp; + + // Insert CNG for 1 minute (= 60000 ms). + const int kCngPeriodMs = 100; + const int kCngPeriodSamples = kCngPeriodMs * 16; // Period in 16 kHz samples. + const int kCngDurationMs = 60000; + for (; t_ms < kSpeechDurationMs + kCngDurationMs; t_ms += 10) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one CNG frame each 100 ms. + uint8_t payload[kPayloadBytes]; + size_t payload_len; + RTPHeader rtp_info; + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + ASSERT_EQ(0, neteq_->InsertPacket( + rtp_info, + rtc::ArrayView(payload, payload_len), 0)); + ++seq_no; + timestamp += kCngPeriodSamples; + next_input_time_ms += static_cast(kCngPeriodMs) * drift_factor; + } + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + + if (network_freeze_ms > 0) { + // First keep pulling audio for |network_freeze_ms| without inserting + // any data, then insert CNG data corresponding to |network_freeze_ms| + // without pulling any output audio. + const double loop_end_time = t_ms + network_freeze_ms; + for (; t_ms < loop_end_time; t_ms += 10) { + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + } + bool pull_once = pull_audio_during_freeze; + // If |pull_once| is true, GetAudio will be called once half-way through + // the network recovery period. + double pull_time_ms = (t_ms + next_input_time_ms) / 2; + while (next_input_time_ms <= t_ms) { + if (pull_once && next_input_time_ms >= pull_time_ms) { + pull_once = false; + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + t_ms += 10; + } + // Insert one CNG frame each 100 ms. + uint8_t payload[kPayloadBytes]; + size_t payload_len; + RTPHeader rtp_info; + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + ASSERT_EQ(0, neteq_->InsertPacket( + rtp_info, + rtc::ArrayView(payload, payload_len), 0)); + ++seq_no; + timestamp += kCngPeriodSamples; + next_input_time_ms += kCngPeriodMs * drift_factor; + } + } + + // Insert speech again until output type is speech. + double speech_restart_time_ms = t_ms; + while (out_frame_.speech_type_ != AudioFrame::kNormalSpeech) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one 30 ms speech frame. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + ++seq_no; + timestamp += kSamples; + next_input_time_ms += kFrameSizeMs * drift_factor; + } + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + // Increase clock. + t_ms += 10; + } + + // Check that the speech starts again within reasonable time. + double time_until_speech_returns_ms = t_ms - speech_restart_time_ms; + EXPECT_LT(time_until_speech_returns_ms, max_time_to_speech_ms); + playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + int32_t delay_after = timestamp - *playout_timestamp; + // Compare delay before and after, and make sure it differs less than 20 ms. + EXPECT_LE(delay_after, delay_before + delay_tolerance_ms * 16); + EXPECT_GE(delay_after, delay_before - delay_tolerance_ms * 16); +} + +TEST_F(NetEqDecodingTest, LongCngWithNegativeClockDrift) { + // Apply a clock drift of -25 ms / s (sender faster than receiver). + const double kDriftFactor = 1000.0 / (1000.0 + 25.0); + const double kNetworkFreezeTimeMs = 0.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 20; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, + kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, + kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithPositiveClockDrift) { + // Apply a clock drift of +25 ms / s (sender slower than receiver). + const double kDriftFactor = 1000.0 / (1000.0 - 25.0); + const double kNetworkFreezeTimeMs = 0.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 20; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, + kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, + kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithNegativeClockDriftNetworkFreeze) { + // Apply a clock drift of -25 ms / s (sender faster than receiver). + const double kDriftFactor = 1000.0 / (1000.0 + 25.0); + const double kNetworkFreezeTimeMs = 5000.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 50; + const int kMaxTimeToSpeechMs = 200; + LongCngWithClockDrift(kDriftFactor, + kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, + kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithPositiveClockDriftNetworkFreeze) { + // Apply a clock drift of +25 ms / s (sender slower than receiver). + const double kDriftFactor = 1000.0 / (1000.0 - 25.0); + const double kNetworkFreezeTimeMs = 5000.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 20; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, + kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, + kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithPositiveClockDriftNetworkFreezeExtraPull) { + // Apply a clock drift of +25 ms / s (sender slower than receiver). + const double kDriftFactor = 1000.0 / (1000.0 - 25.0); + const double kNetworkFreezeTimeMs = 5000.0; + const bool kGetAudioDuringFreezeRecovery = true; + const int kDelayToleranceMs = 20; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, + kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, + kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithoutClockDrift) { + const double kDriftFactor = 1.0; // No drift. + const double kNetworkFreezeTimeMs = 0.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 10; + const int kMaxTimeToSpeechMs = 50; + LongCngWithClockDrift(kDriftFactor, + kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, + kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, UnknownPayloadType) { + const size_t kPayloadBytes = 100; + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + rtp_info.payloadType = 1; // Not registered as a decoder. + EXPECT_EQ(NetEq::kFail, neteq_->InsertPacket(rtp_info, payload, 0)); +} + +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +#define MAYBE_DecoderError DecoderError +#else +#define MAYBE_DecoderError DISABLED_DecoderError +#endif + +TEST_F(NetEqDecodingTest, MAYBE_DecoderError) { + const size_t kPayloadBytes = 100; + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + rtp_info.payloadType = 103; // iSAC, but the payload is invalid. + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + // Set all of |out_data_| to 1, and verify that it was set to 0 by the call + // to GetAudio. + int16_t* out_frame_data = out_frame_.mutable_data(); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; ++i) { + out_frame_data[i] = 1; + } + bool muted; + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + + // Verify that the first 160 samples are set to 0. + static const int kExpectedOutputLength = 160; // 10 ms at 16 kHz sample rate. + const int16_t* const_out_frame_data = out_frame_.data(); + for (int i = 0; i < kExpectedOutputLength; ++i) { + std::ostringstream ss; + ss << "i = " << i; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + EXPECT_EQ(0, const_out_frame_data[i]); + } +} + +TEST_F(NetEqDecodingTest, GetAudioBeforeInsertPacket) { + // Set all of |out_data_| to 1, and verify that it was set to 0 by the call + // to GetAudio. + int16_t* out_frame_data = out_frame_.mutable_data(); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; ++i) { + out_frame_data[i] = 1; + } + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + // Verify that the first block of samples is set to 0. + static const int kExpectedOutputLength = + kInitSampleRateHz / 100; // 10 ms at initial sample rate. + const int16_t* const_out_frame_data = out_frame_.data(); + for (int i = 0; i < kExpectedOutputLength; ++i) { + std::ostringstream ss; + ss << "i = " << i; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + EXPECT_EQ(0, const_out_frame_data[i]); + } + // Verify that the sample rate did not change from the initial configuration. + EXPECT_EQ(config_.sample_rate_hz, neteq_->last_output_sample_rate_hz()); +} + +class NetEqBgnTest : public NetEqDecodingTest { + protected: + virtual void TestCondition(double sum_squared_noise, + bool should_be_faded) = 0; + + void CheckBgn(int sampling_rate_hz) { + size_t expected_samples_per_channel = 0; + uint8_t payload_type = 0xFF; // Invalid. + if (sampling_rate_hz == 8000) { + expected_samples_per_channel = kBlockSize8kHz; + payload_type = 93; // PCM 16, 8 kHz. + } else if (sampling_rate_hz == 16000) { + expected_samples_per_channel = kBlockSize16kHz; + payload_type = 94; // PCM 16, 16 kHZ. + } else if (sampling_rate_hz == 32000) { + expected_samples_per_channel = kBlockSize32kHz; + payload_type = 95; // PCM 16, 32 kHz. + } else { + ASSERT_TRUE(false); // Unsupported test case. + } + + AudioFrame output; + test::AudioLoop input; + // We are using the same 32 kHz input file for all tests, regardless of + // |sampling_rate_hz|. The output may sound weird, but the test is still + // valid. + ASSERT_TRUE(input.Init( + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 10 * sampling_rate_hz, // Max 10 seconds loop length. + expected_samples_per_channel)); + + // Payload of 10 ms of PCM16 32 kHz. + uint8_t payload[kBlockSize32kHz * sizeof(int16_t)]; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + rtp_info.payloadType = payload_type; + + uint32_t receive_timestamp = 0; + bool muted; + for (int n = 0; n < 10; ++n) { // Insert few packets and get audio. + auto block = input.GetNextBlock(); + ASSERT_EQ(expected_samples_per_channel, block.size()); + size_t enc_len_bytes = + WebRtcPcm16b_Encode(block.data(), block.size(), payload); + ASSERT_EQ(enc_len_bytes, expected_samples_per_channel * 2); + + ASSERT_EQ(0, neteq_->InsertPacket( + rtp_info, + rtc::ArrayView(payload, enc_len_bytes), + receive_timestamp)); + output.Reset(); + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(1u, output.num_channels_); + ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_); + ASSERT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Next packet. + rtp_info.timestamp += rtc::checked_cast( + expected_samples_per_channel); + rtp_info.sequenceNumber++; + receive_timestamp += rtc::checked_cast( + expected_samples_per_channel); + } + + output.Reset(); + + // Get audio without inserting packets, expecting PLC and PLC-to-CNG. Pull + // one frame without checking speech-type. This is the first frame pulled + // without inserting any packet, and might not be labeled as PLC. + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(1u, output.num_channels_); + ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_); + + // To be able to test the fading of background noise we need at lease to + // pull 611 frames. + const int kFadingThreshold = 611; + + // Test several CNG-to-PLC packet for the expected behavior. The number 20 + // is arbitrary, but sufficiently large to test enough number of frames. + const int kNumPlcToCngTestFrames = 20; + bool plc_to_cng = false; + for (int n = 0; n < kFadingThreshold + kNumPlcToCngTestFrames; ++n) { + output.Reset(); + // Set to non-zero. + memset(output.mutable_data(), 1, AudioFrame::kMaxDataSizeBytes); + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_FALSE(muted); + ASSERT_EQ(1u, output.num_channels_); + ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_); + if (output.speech_type_ == AudioFrame::kPLCCNG) { + plc_to_cng = true; + double sum_squared = 0; + const int16_t* output_data = output.data(); + for (size_t k = 0; + k < output.num_channels_ * output.samples_per_channel_; ++k) + sum_squared += output_data[k] * output_data[k]; + TestCondition(sum_squared, n > kFadingThreshold); + } else { + EXPECT_EQ(AudioFrame::kPLC, output.speech_type_); + } + } + EXPECT_TRUE(plc_to_cng); // Just to be sure that PLC-to-CNG has occurred. + } +}; + +class NetEqBgnTestOn : public NetEqBgnTest { + protected: + NetEqBgnTestOn() : NetEqBgnTest() { + config_.background_noise_mode = NetEq::kBgnOn; + } + + void TestCondition(double sum_squared_noise, bool /*should_be_faded*/) { + EXPECT_NE(0, sum_squared_noise); + } +}; + +class NetEqBgnTestOff : public NetEqBgnTest { + protected: + NetEqBgnTestOff() : NetEqBgnTest() { + config_.background_noise_mode = NetEq::kBgnOff; + } + + void TestCondition(double sum_squared_noise, bool /*should_be_faded*/) { + EXPECT_EQ(0, sum_squared_noise); + } +}; + +class NetEqBgnTestFade : public NetEqBgnTest { + protected: + NetEqBgnTestFade() : NetEqBgnTest() { + config_.background_noise_mode = NetEq::kBgnFade; + } + + void TestCondition(double sum_squared_noise, bool should_be_faded) { + if (should_be_faded) + EXPECT_EQ(0, sum_squared_noise); + } +}; + +TEST_F(NetEqBgnTestOn, RunTest) { + CheckBgn(8000); + CheckBgn(16000); + CheckBgn(32000); +} + +TEST_F(NetEqBgnTestOff, RunTest) { + CheckBgn(8000); + CheckBgn(16000); + CheckBgn(32000); +} + +TEST_F(NetEqBgnTestFade, RunTest) { + CheckBgn(8000); + CheckBgn(16000); + CheckBgn(32000); +} + +void NetEqDecodingTest::WrapTest(uint16_t start_seq_no, + uint32_t start_timestamp, + const std::set& drop_seq_numbers, + bool expect_seq_no_wrap, + bool expect_timestamp_wrap) { + uint16_t seq_no = start_seq_no; + uint32_t timestamp = start_timestamp; + const int kBlocksPerFrame = 3; // Number of 10 ms blocks per frame. + const int kFrameSizeMs = kBlocksPerFrame * kTimeStepMs; + const int kSamples = kBlockSize16kHz * kBlocksPerFrame; + const size_t kPayloadBytes = kSamples * sizeof(int16_t); + double next_input_time_ms = 0.0; + uint32_t receive_timestamp = 0; + + // Insert speech for 2 seconds. + const int kSpeechDurationMs = 2000; + int packets_inserted = 0; + uint16_t last_seq_no; + uint32_t last_timestamp; + bool timestamp_wrapped = false; + bool seq_no_wrapped = false; + for (double t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one 30 ms speech frame. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + if (drop_seq_numbers.find(seq_no) == drop_seq_numbers.end()) { + // This sequence number was not in the set to drop. Insert it. + ASSERT_EQ(0, + neteq_->InsertPacket(rtp_info, payload, receive_timestamp)); + ++packets_inserted; + } + NetEqNetworkStatistics network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats)); + + // Due to internal NetEq logic, preferred buffer-size is about 4 times the + // packet size for first few packets. Therefore we refrain from checking + // the criteria. + if (packets_inserted > 4) { + // Expect preferred and actual buffer size to be no more than 2 frames. + EXPECT_LE(network_stats.preferred_buffer_size_ms, kFrameSizeMs * 2); + EXPECT_LE(network_stats.current_buffer_size_ms, kFrameSizeMs * 2 + + algorithmic_delay_ms_); + } + last_seq_no = seq_no; + last_timestamp = timestamp; + + ++seq_no; + timestamp += kSamples; + receive_timestamp += kSamples; + next_input_time_ms += static_cast(kFrameSizeMs); + + seq_no_wrapped |= seq_no < last_seq_no; + timestamp_wrapped |= timestamp < last_timestamp; + } + // Pull out data once. + AudioFrame output; + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_); + ASSERT_EQ(1u, output.num_channels_); + + // Expect delay (in samples) to be less than 2 packets. + rtc::Optional playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + EXPECT_LE(timestamp - *playout_timestamp, + static_cast(kSamples * 2)); + } + // Make sure we have actually tested wrap-around. + ASSERT_EQ(expect_seq_no_wrap, seq_no_wrapped); + ASSERT_EQ(expect_timestamp_wrap, timestamp_wrapped); +} + +TEST_F(NetEqDecodingTest, SequenceNumberWrap) { + // Start with a sequence number that will soon wrap. + std::set drop_seq_numbers; // Don't drop any packets. + WrapTest(0xFFFF - 10, 0, drop_seq_numbers, true, false); +} + +TEST_F(NetEqDecodingTest, SequenceNumberWrapAndDrop) { + // Start with a sequence number that will soon wrap. + std::set drop_seq_numbers; + drop_seq_numbers.insert(0xFFFF); + drop_seq_numbers.insert(0x0); + WrapTest(0xFFFF - 10, 0, drop_seq_numbers, true, false); +} + +TEST_F(NetEqDecodingTest, TimestampWrap) { + // Start with a timestamp that will soon wrap. + std::set drop_seq_numbers; + WrapTest(0, 0xFFFFFFFF - 3000, drop_seq_numbers, false, true); +} + +TEST_F(NetEqDecodingTest, TimestampAndSequenceNumberWrap) { + // Start with a timestamp and a sequence number that will wrap at the same + // time. + std::set drop_seq_numbers; + WrapTest(0xFFFF - 10, 0xFFFFFFFF - 5000, drop_seq_numbers, true, true); +} + +void NetEqDecodingTest::DuplicateCng() { + uint16_t seq_no = 0; + uint32_t timestamp = 0; + const int kFrameSizeMs = 10; + const int kSampleRateKhz = 16; + const int kSamples = kFrameSizeMs * kSampleRateKhz; + const size_t kPayloadBytes = kSamples * 2; + + const int algorithmic_delay_samples = std::max( + algorithmic_delay_ms_ * kSampleRateKhz, 5 * kSampleRateKhz / 8); + // Insert three speech packets. Three are needed to get the frame length + // correct. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + bool muted; + for (int i = 0; i < 3; ++i) { + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + ++seq_no; + timestamp += kSamples; + + // Pull audio once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + // Verify speech output. + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + + // Insert same CNG packet twice. + const int kCngPeriodMs = 100; + const int kCngPeriodSamples = kCngPeriodMs * kSampleRateKhz; + size_t payload_len; + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + // This is the first time this CNG packet is inserted. + ASSERT_EQ( + 0, neteq_->InsertPacket( + rtp_info, rtc::ArrayView(payload, payload_len), 0)); + + // Pull audio once and make sure CNG is played. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + EXPECT_FALSE( + neteq_->GetPlayoutTimestamp()); // Returns empty value during CNG. + EXPECT_EQ(timestamp - algorithmic_delay_samples, + out_frame_.timestamp_ + out_frame_.samples_per_channel_); + + // Insert the same CNG packet again. Note that at this point it is old, since + // we have already decoded the first copy of it. + ASSERT_EQ( + 0, neteq_->InsertPacket( + rtp_info, rtc::ArrayView(payload, payload_len), 0)); + + // Pull audio until we have played |kCngPeriodMs| of CNG. Start at 10 ms since + // we have already pulled out CNG once. + for (int cng_time_ms = 10; cng_time_ms < kCngPeriodMs; cng_time_ms += 10) { + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + EXPECT_FALSE( + neteq_->GetPlayoutTimestamp()); // Returns empty value during CNG. + EXPECT_EQ(timestamp - algorithmic_delay_samples, + out_frame_.timestamp_ + out_frame_.samples_per_channel_); + } + + // Insert speech again. + ++seq_no; + timestamp += kCngPeriodSamples; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + + // Pull audio once and verify that the output is speech again. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + rtc::Optional playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + EXPECT_EQ(timestamp + kSamples - algorithmic_delay_samples, + *playout_timestamp); +} + +TEST_F(NetEqDecodingTest, DiscardDuplicateCng) { DuplicateCng(); } + +TEST_F(NetEqDecodingTest, CngFirst) { + uint16_t seq_no = 0; + uint32_t timestamp = 0; + const int kFrameSizeMs = 10; + const int kSampleRateKhz = 16; + const int kSamples = kFrameSizeMs * kSampleRateKhz; + const int kPayloadBytes = kSamples * 2; + const int kCngPeriodMs = 100; + const int kCngPeriodSamples = kCngPeriodMs * kSampleRateKhz; + size_t payload_len; + + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + ASSERT_EQ( + NetEq::kOK, + neteq_->InsertPacket( + rtp_info, rtc::ArrayView(payload, payload_len), 0)); + ++seq_no; + timestamp += kCngPeriodSamples; + + // Pull audio once and make sure CNG is played. + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + + // Insert some speech packets. + const uint32_t first_speech_timestamp = timestamp; + int timeout_counter = 0; + do { + ASSERT_LT(timeout_counter++, 20) << "Test timed out"; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + ++seq_no; + timestamp += kSamples; + + // Pull audio once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } while (!IsNewerTimestamp(out_frame_.timestamp_, first_speech_timestamp)); + // Verify speech output. + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); +} + +class NetEqDecodingTestWithMutedState : public NetEqDecodingTest { + public: + NetEqDecodingTestWithMutedState() : NetEqDecodingTest() { + config_.enable_muted_state = true; + } + + protected: + static constexpr size_t kSamples = 10 * 16; + static constexpr size_t kPayloadBytes = kSamples * 2; + + void InsertPacket(uint32_t rtp_timestamp) { + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, rtp_timestamp, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + } + + void InsertCngPacket(uint32_t rtp_timestamp) { + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + size_t payload_len; + PopulateCng(0, rtp_timestamp, &rtp_info, payload, &payload_len); + EXPECT_EQ( + NetEq::kOK, + neteq_->InsertPacket( + rtp_info, rtc::ArrayView(payload, payload_len), 0)); + } + + bool GetAudioReturnMuted() { + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + return muted; + } + + void GetAudioUntilMuted() { + while (!GetAudioReturnMuted()) { + ASSERT_LT(counter_++, 1000) << "Test timed out"; + } + } + + void GetAudioUntilNormal() { + bool muted = false; + while (out_frame_.speech_type_ != AudioFrame::kNormalSpeech) { + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_LT(counter_++, 1000) << "Test timed out"; + } + EXPECT_FALSE(muted); + } + + int counter_ = 0; +}; + +// Verifies that NetEq goes in and out of muted state as expected. +TEST_F(NetEqDecodingTestWithMutedState, MutedState) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + EXPECT_TRUE(out_frame_.muted()); + + // Verify that output audio is not written during muted mode. Other parameters + // should be correct, though. + AudioFrame new_frame; + int16_t* frame_data = new_frame.mutable_data(); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; i++) { + frame_data[i] = 17; + } + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&new_frame, &muted)); + EXPECT_TRUE(muted); + EXPECT_TRUE(out_frame_.muted()); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; i++) { + EXPECT_EQ(17, frame_data[i]); + } + EXPECT_EQ(out_frame_.timestamp_ + out_frame_.samples_per_channel_, + new_frame.timestamp_); + EXPECT_EQ(out_frame_.samples_per_channel_, new_frame.samples_per_channel_); + EXPECT_EQ(out_frame_.sample_rate_hz_, new_frame.sample_rate_hz_); + EXPECT_EQ(out_frame_.num_channels_, new_frame.num_channels_); + EXPECT_EQ(out_frame_.speech_type_, new_frame.speech_type_); + EXPECT_EQ(out_frame_.vad_activity_, new_frame.vad_activity_); + + // Insert new data. Timestamp is corrected for the time elapsed since the last + // packet. Verify that normal operation resumes. + InsertPacket(kSamples * counter_); + GetAudioUntilNormal(); + EXPECT_FALSE(out_frame_.muted()); + + NetEqNetworkStatistics stats; + EXPECT_EQ(0, neteq_->NetworkStatistics(&stats)); + // NetEqNetworkStatistics::expand_rate tells the fraction of samples that were + // concealment samples, in Q14 (16384 = 100%) .The vast majority should be + // concealment samples in this test. + EXPECT_GT(stats.expand_rate, 14000); + // And, it should be greater than the speech_expand_rate. + EXPECT_GT(stats.expand_rate, stats.speech_expand_rate); +} + +// Verifies that NetEq goes out of muted state when given a delayed packet. +TEST_F(NetEqDecodingTestWithMutedState, MutedStateDelayedPacket) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + // Insert new data. Timestamp is only corrected for the half of the time + // elapsed since the last packet. That is, the new packet is delayed. Verify + // that normal operation resumes. + InsertPacket(kSamples * counter_ / 2); + GetAudioUntilNormal(); +} + +// Verifies that NetEq goes out of muted state when given a future packet. +TEST_F(NetEqDecodingTestWithMutedState, MutedStateFuturePacket) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + // Insert new data. Timestamp is over-corrected for the time elapsed since the + // last packet. That is, the new packet is too early. Verify that normal + // operation resumes. + InsertPacket(kSamples * counter_ * 2); + GetAudioUntilNormal(); +} + +// Verifies that NetEq goes out of muted state when given an old packet. +TEST_F(NetEqDecodingTestWithMutedState, MutedStateOldPacket) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + + EXPECT_NE(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + // Insert packet which is older than the first packet. + InsertPacket(kSamples * (counter_ - 1000)); + EXPECT_FALSE(GetAudioReturnMuted()); + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); +} + +// Verifies that NetEq doesn't enter muted state when CNG mode is active and the +// packet stream is suspended for a long time. +TEST_F(NetEqDecodingTestWithMutedState, DoNotMuteExtendedCngWithoutPackets) { + // Insert one CNG packet. + InsertCngPacket(0); + + // Pull 10 seconds of audio (10 ms audio generated per lap). + for (int i = 0; i < 1000; ++i) { + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + } + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); +} + +// Verifies that NetEq goes back to normal after a long CNG period with the +// packet stream suspended. +TEST_F(NetEqDecodingTestWithMutedState, RecoverAfterExtendedCngWithoutPackets) { + // Insert one CNG packet. + InsertCngPacket(0); + + // Pull 10 seconds of audio (10 ms audio generated per lap). + for (int i = 0; i < 1000; ++i) { + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + } + + // Insert new data. Timestamp is corrected for the time elapsed since the last + // packet. Verify that normal operation resumes. + InsertPacket(kSamples * counter_); + GetAudioUntilNormal(); +} + +class NetEqDecodingTestTwoInstances : public NetEqDecodingTest { + public: + NetEqDecodingTestTwoInstances() : NetEqDecodingTest() {} + + void SetUp() override { + NetEqDecodingTest::SetUp(); + config2_ = config_; + } + + void CreateSecondInstance() { + neteq2_.reset(NetEq::Create(config2_, CreateBuiltinAudioDecoderFactory())); + ASSERT_TRUE(neteq2_); + LoadDecoders(neteq2_.get()); + } + + protected: + std::unique_ptr neteq2_; + NetEq::Config config2_; +}; + +namespace { +::testing::AssertionResult AudioFramesEqualExceptData(const AudioFrame& a, + const AudioFrame& b) { + if (a.timestamp_ != b.timestamp_) + return ::testing::AssertionFailure() << "timestamp_ diff (" << a.timestamp_ + << " != " << b.timestamp_ << ")"; + if (a.sample_rate_hz_ != b.sample_rate_hz_) + return ::testing::AssertionFailure() << "sample_rate_hz_ diff (" + << a.sample_rate_hz_ + << " != " << b.sample_rate_hz_ << ")"; + if (a.samples_per_channel_ != b.samples_per_channel_) + return ::testing::AssertionFailure() + << "samples_per_channel_ diff (" << a.samples_per_channel_ + << " != " << b.samples_per_channel_ << ")"; + if (a.num_channels_ != b.num_channels_) + return ::testing::AssertionFailure() << "num_channels_ diff (" + << a.num_channels_ + << " != " << b.num_channels_ << ")"; + if (a.speech_type_ != b.speech_type_) + return ::testing::AssertionFailure() << "speech_type_ diff (" + << a.speech_type_ + << " != " << b.speech_type_ << ")"; + if (a.vad_activity_ != b.vad_activity_) + return ::testing::AssertionFailure() << "vad_activity_ diff (" + << a.vad_activity_ + << " != " << b.vad_activity_ << ")"; + return ::testing::AssertionSuccess(); +} + +::testing::AssertionResult AudioFramesEqual(const AudioFrame& a, + const AudioFrame& b) { + ::testing::AssertionResult res = AudioFramesEqualExceptData(a, b); + if (!res) + return res; + if (memcmp( + a.data(), b.data(), + a.samples_per_channel_ * a.num_channels_ * sizeof(*a.data())) != 0) { + return ::testing::AssertionFailure() << "data_ diff"; + } + return ::testing::AssertionSuccess(); +} + +} // namespace + +TEST_F(NetEqDecodingTestTwoInstances, CompareMutedStateOnOff) { + ASSERT_FALSE(config_.enable_muted_state); + config2_.enable_muted_state = true; + CreateSecondInstance(); + + // Insert one speech packet into both NetEqs. + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + EXPECT_EQ(0, neteq2_->InsertPacket(rtp_info, payload, 0)); + + AudioFrame out_frame1, out_frame2; + bool muted; + for (int i = 0; i < 1000; ++i) { + std::ostringstream ss; + ss << "i = " << i; + SCOPED_TRACE(ss.str()); // Print out the loop iterator on failure. + EXPECT_EQ(0, neteq_->GetAudio(&out_frame1, &muted)); + EXPECT_FALSE(muted); + EXPECT_EQ(0, neteq2_->GetAudio(&out_frame2, &muted)); + if (muted) { + EXPECT_TRUE(AudioFramesEqualExceptData(out_frame1, out_frame2)); + } else { + EXPECT_TRUE(AudioFramesEqual(out_frame1, out_frame2)); + } + } + EXPECT_TRUE(muted); + + // Insert new data. Timestamp is corrected for the time elapsed since the last + // packet. + PopulateRtpInfo(0, kSamples * 1000, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + EXPECT_EQ(0, neteq2_->InsertPacket(rtp_info, payload, 0)); + + int counter = 0; + while (out_frame1.speech_type_ != AudioFrame::kNormalSpeech) { + ASSERT_LT(counter++, 1000) << "Test timed out"; + std::ostringstream ss; + ss << "counter = " << counter; + SCOPED_TRACE(ss.str()); // Print out the loop iterator on failure. + EXPECT_EQ(0, neteq_->GetAudio(&out_frame1, &muted)); + EXPECT_FALSE(muted); + EXPECT_EQ(0, neteq2_->GetAudio(&out_frame2, &muted)); + if (muted) { + EXPECT_TRUE(AudioFramesEqualExceptData(out_frame1, out_frame2)); + } else { + EXPECT_TRUE(AudioFramesEqual(out_frame1, out_frame2)); + } + } + EXPECT_FALSE(muted); +} + +TEST_F(NetEqDecodingTest, LastDecodedTimestampsEmpty) { + EXPECT_TRUE(neteq_->LastDecodedTimestamps().empty()); + + // Pull out data once. + AudioFrame output; + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + + EXPECT_TRUE(neteq_->LastDecodedTimestamps().empty()); +} + +TEST_F(NetEqDecodingTest, LastDecodedTimestampsOneDecoded) { + // Insert one packet with PCM16b WB data (this is what PopulateRtpInfo does by + // default). Make the length 10 ms. + constexpr size_t kPayloadSamples = 16 * 10; + constexpr size_t kPayloadBytes = 2 * kPayloadSamples; + uint8_t payload[kPayloadBytes] = {0}; + + RTPHeader rtp_info; + constexpr uint32_t kRtpTimestamp = 0x1234; + PopulateRtpInfo(0, kRtpTimestamp, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + + // Pull out data once. + AudioFrame output; + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + + EXPECT_EQ(std::vector({kRtpTimestamp}), + neteq_->LastDecodedTimestamps()); + + // Nothing decoded on the second call. + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + EXPECT_TRUE(neteq_->LastDecodedTimestamps().empty()); +} + +TEST_F(NetEqDecodingTest, LastDecodedTimestampsTwoDecoded) { + // Insert two packets with PCM16b WB data (this is what PopulateRtpInfo does + // by default). Make the length 5 ms so that NetEq must decode them both in + // the same GetAudio call. + constexpr size_t kPayloadSamples = 16 * 5; + constexpr size_t kPayloadBytes = 2 * kPayloadSamples; + uint8_t payload[kPayloadBytes] = {0}; + + RTPHeader rtp_info; + constexpr uint32_t kRtpTimestamp1 = 0x1234; + PopulateRtpInfo(0, kRtpTimestamp1, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + constexpr uint32_t kRtpTimestamp2 = kRtpTimestamp1 + kPayloadSamples; + PopulateRtpInfo(1, kRtpTimestamp2, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0)); + + // Pull out data once. + AudioFrame output; + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + + EXPECT_EQ(std::vector({kRtpTimestamp1, kRtpTimestamp2}), + neteq_->LastDecodedTimestamps()); +} + +TEST_F(NetEqDecodingTest, TestConcealmentEvents) { + const int kNumConcealmentEvents = 19; + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + int seq_no = 0; + RTPHeader rtp_info; + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + const uint8_t payload[kPayloadBytes] = {0}; + bool muted; + + for (int i = 0; i < kNumConcealmentEvents; i++) { + // Insert some packets of 10 ms size. + for (int j = 0; j < 10; j++) { + rtp_info.sequenceNumber = seq_no++; + rtp_info.timestamp = rtp_info.sequenceNumber * kSamples; + neteq_->InsertPacket(rtp_info, payload, 0); + neteq_->GetAudio(&out_frame_, &muted); + } + + // Lose a number of packets. + int num_lost = 1 + i; + for (int j = 0; j < num_lost; j++) { + seq_no++; + neteq_->GetAudio(&out_frame_, &muted); + } + } + + // Check number of concealment events. + NetEqLifetimeStatistics stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(kNumConcealmentEvents, static_cast(stats.concealment_events)); +} + +// Test that the jitter buffer delay stat is computed correctly. +void NetEqDecodingTestFaxMode::TestJitterBufferDelay(bool apply_packet_loss) { + const int kNumPackets = 10; + const int kDelayInNumPackets = 2; + const int kPacketLenMs = 10; // All packets are of 10 ms size. + const size_t kSamples = kPacketLenMs * 16; + const size_t kPayloadBytes = kSamples * 2; + RTPHeader rtp_info; + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + const uint8_t payload[kPayloadBytes] = {0}; + bool muted; + int packets_sent = 0; + int packets_received = 0; + int expected_delay = 0; + while (packets_received < kNumPackets) { + // Insert packet. + if (packets_sent < kNumPackets) { + rtp_info.sequenceNumber = packets_sent++; + rtp_info.timestamp = rtp_info.sequenceNumber * kSamples; + neteq_->InsertPacket(rtp_info, payload, 0); + } + + // Get packet. + if (packets_sent > kDelayInNumPackets) { + neteq_->GetAudio(&out_frame_, &muted); + packets_received++; + + // The delay reported by the jitter buffer never exceeds + // the number of samples previously fetched with GetAudio + // (hence the min()). + int packets_delay = std::min(packets_received, kDelayInNumPackets + 1); + + // The increase of the expected delay is the product of + // the current delay of the jitter buffer in ms * the + // number of samples that are sent for play out. + int current_delay_ms = packets_delay * kPacketLenMs; + expected_delay += current_delay_ms * kSamples; + } + } + + if (apply_packet_loss) { + // Extra call to GetAudio to cause concealment. + neteq_->GetAudio(&out_frame_, &muted); + } + + // Check jitter buffer delay. + NetEqLifetimeStatistics stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(expected_delay, static_cast(stats.jitter_buffer_delay_ms)); +} + +TEST_F(NetEqDecodingTestFaxMode, TestJitterBufferDelayWithoutLoss) { + TestJitterBufferDelay(false); +} + +TEST_F(NetEqDecodingTestFaxMode, TestJitterBufferDelayWithLoss) { + TestJitterBufferDelay(true); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.proto b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.proto new file mode 100644 index 0000000000..344dcf6083 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.proto @@ -0,0 +1,32 @@ +syntax = "proto2"; +option optimize_for = LITE_RUNTIME; +package webrtc.neteq_unittest; + +message NetEqNetworkStatistics { + // Next field number 18. + optional uint32 current_buffer_size_ms = 1; + optional uint32 preferred_buffer_size_ms = 2; + optional uint32 jitter_peaks_found = 3; + optional uint32 packet_loss_rate = 4; + optional uint32 packet_discard_rate = 5 [deprecated = true]; + optional uint32 expand_rate = 6; + optional uint32 speech_expand_rate = 7; + optional uint32 preemptive_rate = 8; + optional uint32 accelerate_rate = 9; + optional uint32 secondary_decoded_rate = 10; + optional uint32 secondary_discarded_rate = 17; + optional int32 clockdrift_ppm = 11; + optional uint64 added_zero_samples = 12; + optional int32 mean_waiting_time_ms = 13; + optional int32 median_waiting_time_ms = 14; + optional int32 min_waiting_time_ms = 15; + optional int32 max_waiting_time_ms = 16; +} + +message RtcpStatistics { + optional uint32 fraction_lost = 1; + optional uint32 cumulative_lost = 2; + optional uint32 extended_max_sequence_number = 3; + optional uint32 jitter = 4; +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.cc new file mode 100644 index 0000000000..48d723a81b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.cc @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/normal.h" + +#include // memset, memcpy + +#include // min + +#include "api/audio_codecs/audio_decoder.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/expand.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +int Normal::Process(const int16_t* input, + size_t length, + Modes last_mode, + int16_t* external_mute_factor_array, + AudioMultiVector* output) { + if (length == 0) { + // Nothing to process. + output->Clear(); + return static_cast(length); + } + + RTC_DCHECK(output->Empty()); + // Output should be empty at this point. + if (length % output->Channels() != 0) { + // The length does not match the number of channels. + output->Clear(); + return 0; + } + output->PushBackInterleaved(input, length); + + const int fs_mult = fs_hz_ / 8000; + RTC_DCHECK_GT(fs_mult, 0); + // fs_shift = log2(fs_mult), rounded down. + // Note that |fs_shift| is not "exact" for 48 kHz. + // TODO(hlundin): Investigate this further. + const int fs_shift = 30 - WebRtcSpl_NormW32(fs_mult); + + // Check if last RecOut call resulted in an Expand. If so, we have to take + // care of some cross-fading and unmuting. + if (last_mode == kModeExpand) { + // Generate interpolation data using Expand. + // First, set Expand parameters to appropriate values. + expand_->SetParametersForNormalAfterExpand(); + + // Call Expand. + AudioMultiVector expanded(output->Channels()); + expand_->Process(&expanded); + expand_->Reset(); + + size_t length_per_channel = length / output->Channels(); + std::unique_ptr signal(new int16_t[length_per_channel]); + for (size_t channel_ix = 0; channel_ix < output->Channels(); ++channel_ix) { + // Adjust muting factor (main muting factor times expand muting factor). + external_mute_factor_array[channel_ix] = static_cast( + (external_mute_factor_array[channel_ix] * + expand_->MuteFactor(channel_ix)) >> 14); + + (*output)[channel_ix].CopyTo(length_per_channel, 0, signal.get()); + + // Find largest absolute value in new data. + int16_t decoded_max = + WebRtcSpl_MaxAbsValueW16(signal.get(), length_per_channel); + // Adjust muting factor if needed (to BGN level). + size_t energy_length = + std::min(static_cast(fs_mult * 64), length_per_channel); + int scaling = 6 + fs_shift + - WebRtcSpl_NormW32(decoded_max * decoded_max); + scaling = std::max(scaling, 0); // |scaling| should always be >= 0. + int32_t energy = WebRtcSpl_DotProductWithScale(signal.get(), signal.get(), + energy_length, scaling); + int32_t scaled_energy_length = + static_cast(energy_length >> scaling); + if (scaled_energy_length > 0) { + energy = energy / scaled_energy_length; + } else { + energy = 0; + } + + int mute_factor; + if ((energy != 0) && + (energy > background_noise_.Energy(channel_ix))) { + // Normalize new frame energy to 15 bits. + scaling = WebRtcSpl_NormW32(energy) - 16; + // We want background_noise_.energy() / energy in Q14. + int32_t bgn_energy = WEBRTC_SPL_SHIFT_W32( + background_noise_.Energy(channel_ix), scaling + 14); + int16_t energy_scaled = + static_cast(WEBRTC_SPL_SHIFT_W32(energy, scaling)); + int32_t ratio = WebRtcSpl_DivW32W16(bgn_energy, energy_scaled); + mute_factor = WebRtcSpl_SqrtFloor(ratio << 14); + } else { + mute_factor = 16384; // 1.0 in Q14. + } + if (mute_factor > external_mute_factor_array[channel_ix]) { + external_mute_factor_array[channel_ix] = + static_cast(std::min(mute_factor, 16384)); + } + + // If muted increase by 0.64 for every 20 ms (NB/WB 0.0040/0.0020 in Q14). + int increment = 64 / fs_mult; + for (size_t i = 0; i < length_per_channel; i++) { + // Scale with mute factor. + RTC_DCHECK_LT(channel_ix, output->Channels()); + RTC_DCHECK_LT(i, output->Size()); + int32_t scaled_signal = (*output)[channel_ix][i] * + external_mute_factor_array[channel_ix]; + // Shift 14 with proper rounding. + (*output)[channel_ix][i] = + static_cast((scaled_signal + 8192) >> 14); + // Increase mute_factor towards 16384. + external_mute_factor_array[channel_ix] = static_cast(std::min( + external_mute_factor_array[channel_ix] + increment, 16384)); + } + + // Interpolate the expanded data into the new vector. + // (NB/WB/SWB32/SWB48 8/16/32/48 samples.) + size_t win_length = samples_per_ms_; + int16_t win_slope_Q14 = default_win_slope_Q14_; + RTC_DCHECK_LT(channel_ix, output->Channels()); + if (win_length > output->Size()) { + win_length = output->Size(); + win_slope_Q14 = (1 << 14) / static_cast(win_length); + } + int16_t win_up_Q14 = 0; + for (size_t i = 0; i < win_length; i++) { + win_up_Q14 += win_slope_Q14; + (*output)[channel_ix][i] = + (win_up_Q14 * (*output)[channel_ix][i] + + ((1 << 14) - win_up_Q14) * expanded[channel_ix][i] + (1 << 13)) >> + 14; + } + RTC_DCHECK_GT(win_up_Q14, + (1 << 14) - 32); // Worst case rouding is a length of 34 + } + } else if (last_mode == kModeRfc3389Cng) { + RTC_DCHECK_EQ(output->Channels(), 1); // Not adapted for multi-channel yet. + static const size_t kCngLength = 48; + RTC_DCHECK_LE(8 * fs_mult, kCngLength); + int16_t cng_output[kCngLength]; + // Reset mute factor and start up fresh. + external_mute_factor_array[0] = 16384; + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + + if (cng_decoder) { + // Generate long enough for 48kHz. + if (!cng_decoder->Generate(cng_output, 0)) { + // Error returned; set return vector to all zeros. + memset(cng_output, 0, sizeof(cng_output)); + } + } else { + // If no CNG instance is defined, just copy from the decoded data. + // (This will result in interpolating the decoded with itself.) + (*output)[0].CopyTo(fs_mult * 8, 0, cng_output); + } + // Interpolate the CNG into the new vector. + // (NB/WB/SWB32/SWB48 8/16/32/48 samples.) + size_t win_length = samples_per_ms_; + int16_t win_slope_Q14 = default_win_slope_Q14_; + if (win_length > kCngLength) { + win_length = kCngLength; + win_slope_Q14 = (1 << 14) / static_cast(win_length); + } + int16_t win_up_Q14 = 0; + for (size_t i = 0; i < win_length; i++) { + win_up_Q14 += win_slope_Q14; + (*output)[0][i] = + (win_up_Q14 * (*output)[0][i] + + ((1 << 14) - win_up_Q14) * cng_output[i] + (1 << 13)) >> + 14; + } + RTC_DCHECK_GT(win_up_Q14, + (1 << 14) - 32); // Worst case rouding is a length of 34 + } else if (external_mute_factor_array[0] < 16384) { + // Previous was neither of Expand, FadeToBGN or RFC3389_CNG, but we are + // still ramping up from previous muting. + // If muted increase by 0.64 for every 20 ms (NB/WB 0.0040/0.0020 in Q14). + int increment = 64 / fs_mult; + size_t length_per_channel = length / output->Channels(); + for (size_t i = 0; i < length_per_channel; i++) { + for (size_t channel_ix = 0; channel_ix < output->Channels(); + ++channel_ix) { + // Scale with mute factor. + RTC_DCHECK_LT(channel_ix, output->Channels()); + RTC_DCHECK_LT(i, output->Size()); + int32_t scaled_signal = (*output)[channel_ix][i] * + external_mute_factor_array[channel_ix]; + // Shift 14 with proper rounding. + (*output)[channel_ix][i] = + static_cast((scaled_signal + 8192) >> 14); + // Increase mute_factor towards 16384. + external_mute_factor_array[channel_ix] = static_cast(std::min( + 16384, external_mute_factor_array[channel_ix] + increment)); + } + } + } + + return static_cast(length); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.h new file mode 100644 index 0000000000..ab02217a37 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NORMAL_H_ +#define MODULES_AUDIO_CODING_NETEQ_NORMAL_H_ + +#include // Access to size_t. + +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/defines.h" +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class BackgroundNoise; +class DecoderDatabase; +class Expand; + +// This class provides the "Normal" DSP operation, that is performed when +// there is no data loss, no need to stretch the timing of the signal, and +// no other "special circumstances" are at hand. +class Normal { + public: + Normal(int fs_hz, + DecoderDatabase* decoder_database, + const BackgroundNoise& background_noise, + Expand* expand) + : fs_hz_(fs_hz), + decoder_database_(decoder_database), + background_noise_(background_noise), + expand_(expand), + samples_per_ms_(rtc::CheckedDivExact(fs_hz_, 1000)), + default_win_slope_Q14_( + rtc::dchecked_cast((1 << 14) / samples_per_ms_)) {} + + virtual ~Normal() {} + + // Performs the "Normal" operation. The decoder data is supplied in |input|, + // having |length| samples in total for all channels (interleaved). The + // result is written to |output|. The number of channels allocated in + // |output| defines the number of channels that will be used when + // de-interleaving |input|. |last_mode| contains the mode used in the previous + // GetAudio call (i.e., not the current one), and |external_mute_factor| is + // a pointer to the mute factor in the NetEqImpl class. + int Process(const int16_t* input, size_t length, + Modes last_mode, + int16_t* external_mute_factor_array, + AudioMultiVector* output); + + private: + int fs_hz_; + DecoderDatabase* decoder_database_; + const BackgroundNoise& background_noise_; + Expand* expand_; + const size_t samples_per_ms_; + const int16_t default_win_slope_Q14_; + + RTC_DISALLOW_COPY_AND_ASSIGN(Normal); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_NORMAL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal_unittest.cc new file mode 100644 index 0000000000..b0655d9a81 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal_unittest.cc @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Normal class. + +#include "modules/audio_coding/neteq/normal.h" + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/mock/mock_expand.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::Invoke; + +namespace webrtc { + +namespace { + +int ExpandProcess120ms(AudioMultiVector* output) { + AudioMultiVector dummy_audio(1, 11520u); + dummy_audio.CopyTo(output); + return 0; +} + +} // namespace + +TEST(Normal, CreateAndDestroy) { + MockDecoderDatabase db; + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); + Normal normal(fs, &db, bgn, &expand); + EXPECT_CALL(db, Die()); // Called when |db| goes out of scope. +} + +TEST(Normal, AvoidDivideByZero) { + WebRtcSpl_Init(); + MockDecoderDatabase db; + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); + Normal normal(fs, &db, bgn, &expand); + + int16_t input[1000] = {0}; + std::unique_ptr mute_factor_array(new int16_t[channels]); + for (size_t i = 0; i < channels; ++i) { + mute_factor_array[i] = 16384; + } + AudioMultiVector output(channels); + + // Zero input length. + EXPECT_EQ( + 0, + normal.Process(input, 0, kModeExpand, mute_factor_array.get(), &output)); + EXPECT_EQ(0u, output.Size()); + + // Try to make energy_length >> scaling = 0; + EXPECT_CALL(expand, SetParametersForNormalAfterExpand()); + EXPECT_CALL(expand, Process(_)); + EXPECT_CALL(expand, Reset()); + // If input_size_samples < 64, then energy_length in Normal::Process() will + // be equal to input_size_samples. Since the input is all zeros, decoded_max + // will be zero, and scaling will be >= 6. Thus, energy_length >> scaling = 0, + // and using this as a denominator would lead to problems. + int input_size_samples = 63; + EXPECT_EQ(input_size_samples, + normal.Process(input, + input_size_samples, + kModeExpand, + mute_factor_array.get(), + &output)); + + EXPECT_CALL(db, Die()); // Called when |db| goes out of scope. + EXPECT_CALL(expand, Die()); // Called when |expand| goes out of scope. +} + +TEST(Normal, InputLengthAndChannelsDoNotMatch) { + WebRtcSpl_Init(); + MockDecoderDatabase db; + int fs = 8000; + size_t channels = 2; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(channels, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); + Normal normal(fs, &db, bgn, &expand); + + int16_t input[1000] = {0}; + std::unique_ptr mute_factor_array(new int16_t[channels]); + for (size_t i = 0; i < channels; ++i) { + mute_factor_array[i] = 16384; + } + AudioMultiVector output(channels); + + // Let the number of samples be one sample less than 80 samples per channel. + size_t input_len = 80 * channels - 1; + EXPECT_EQ( + 0, + normal.Process( + input, input_len, kModeExpand, mute_factor_array.get(), &output)); + EXPECT_EQ(0u, output.Size()); + + EXPECT_CALL(db, Die()); // Called when |db| goes out of scope. + EXPECT_CALL(expand, Die()); // Called when |expand| goes out of scope. +} + +TEST(Normal, LastModeExpand120msPacket) { + WebRtcSpl_Init(); + MockDecoderDatabase db; + const int kFs = 48000; + const size_t kPacketsizeBytes = 11520u; + const size_t kChannels = 1; + BackgroundNoise bgn(kChannels); + SyncBuffer sync_buffer(kChannels, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, kFs, + kChannels); + Normal normal(kFs, &db, bgn, &expand); + + int16_t input[kPacketsizeBytes] = {0}; + + std::unique_ptr mute_factor_array(new int16_t[kChannels]); + for (size_t i = 0; i < kChannels; ++i) { + mute_factor_array[i] = 16384; + } + + AudioMultiVector output(kChannels); + + EXPECT_CALL(expand, SetParametersForNormalAfterExpand()); + EXPECT_CALL(expand, Process(_)).WillOnce(Invoke(ExpandProcess120ms)); + EXPECT_CALL(expand, Reset()); + EXPECT_EQ(static_cast(kPacketsizeBytes), + normal.Process(input, + kPacketsizeBytes, + kModeExpand, + mute_factor_array.get(), + &output)); + + EXPECT_EQ(kPacketsizeBytes, output.Size()); + + EXPECT_CALL(db, Die()); // Called when |db| goes out of scope. + EXPECT_CALL(expand, Die()); // Called when |expand| goes out of scope. +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.cc new file mode 100644 index 0000000000..3cec310902 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.cc @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/packet.h" + +namespace webrtc { + +Packet::Packet() = default; +Packet::Packet(Packet&& b) = default; + +Packet::~Packet() = default; + +Packet& Packet::operator=(Packet&& b) = default; + +Packet Packet::Clone() const { + RTC_CHECK(!frame); + + Packet clone; + clone.timestamp = timestamp; + clone.sequence_number = sequence_number; + clone.payload_type = payload_type; + clone.payload.SetData(payload.data(), payload.size()); + clone.priority = priority; + + return clone; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.h new file mode 100644 index 0000000000..61b014429a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_PACKET_H_ +#define MODULES_AUDIO_CODING_NETEQ_PACKET_H_ + +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "modules/audio_coding/neteq/tick_timer.h" +#include "rtc_base/buffer.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Struct for holding RTP packets. +struct Packet { + struct Priority { + Priority() : codec_level(0), red_level(0) {} + Priority(int codec_level, int red_level) + : codec_level(codec_level), red_level(red_level) { + CheckInvariant(); + } + + int codec_level; + int red_level; + + // Priorities are sorted low-to-high, first on the level the codec + // prioritizes it, then on the level of RED packet it is; i.e. if it is a + // primary or secondary payload of a RED packet. For example: with Opus, an + // Fec packet (which the decoder prioritizes lower than a regular packet) + // will not be used if there is _any_ RED payload for the same + // timeframe. The highest priority packet will have levels {0, 0}. Negative + // priorities are not allowed. + bool operator<(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + if (codec_level == b.codec_level) + return red_level < b.red_level; + + return codec_level < b.codec_level; + } + bool operator==(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + return codec_level == b.codec_level && red_level == b.red_level; + } + bool operator!=(const Priority& b) const { return !(*this == b); } + bool operator>(const Priority& b) const { return b < *this; } + bool operator<=(const Priority& b) const { return !(b > *this); } + bool operator>=(const Priority& b) const { return !(b < *this); } + + private: + void CheckInvariant() const { + RTC_DCHECK_GE(codec_level, 0); + RTC_DCHECK_GE(red_level, 0); + } + }; + + uint32_t timestamp; + uint16_t sequence_number; + uint8_t payload_type; + // Datagram excluding RTP header and header extension. + rtc::Buffer payload; + Priority priority; + std::unique_ptr waiting_time; + std::unique_ptr frame; + + Packet(); + Packet(Packet&& b); + ~Packet(); + + // Packets should generally be moved around but sometimes it's useful to make + // a copy, for example for testing purposes. NOTE: Will only work for + // un-parsed packets, i.e. |frame| must be unset. The payload will, however, + // be copied. |waiting_time| will also not be copied. + Packet Clone() const; + + Packet& operator=(Packet&& b); + + // Comparison operators. Establish a packet ordering based on (1) timestamp, + // (2) sequence number and (3) redundancy. + // Timestamp and sequence numbers are compared taking wrap-around into + // account. For two packets with the same sequence number and timestamp a + // primary payload is considered "smaller" than a secondary. + bool operator==(const Packet& rhs) const { + return (this->timestamp == rhs.timestamp && + this->sequence_number == rhs.sequence_number && + this->priority == rhs.priority); + } + bool operator!=(const Packet& rhs) const { return !operator==(rhs); } + bool operator<(const Packet& rhs) const { + if (this->timestamp == rhs.timestamp) { + if (this->sequence_number == rhs.sequence_number) { + // Timestamp and sequence numbers are identical - deem the left hand + // side to be "smaller" (i.e., "earlier") if it has higher priority. + return this->priority < rhs.priority; + } + return (static_cast(rhs.sequence_number - + this->sequence_number) < 0xFFFF / 2); + } + return (static_cast(rhs.timestamp - this->timestamp) < + 0xFFFFFFFF / 2); + } + bool operator>(const Packet& rhs) const { return rhs.operator<(*this); } + bool operator<=(const Packet& rhs) const { return !operator>(rhs); } + bool operator>=(const Packet& rhs) const { return !operator<(rhs); } + + bool empty() const { return !frame && payload.empty(); } +}; + +// A list of packets. +typedef std::list PacketList; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_PACKET_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.cc new file mode 100644 index 0000000000..dfffebdae6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.cc @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This is the implementation of the PacketBuffer class. It is mostly based on +// an STL list. The list is kept sorted at all times so that the next packet to +// decode is at the beginning of the list. + +#include "modules/audio_coding/neteq/packet_buffer.h" + +#include // find_if() + +#include "api/audio_codecs/audio_decoder.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/tick_timer.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { +// Predicate used when inserting packets in the buffer list. +// Operator() returns true when |packet| goes before |new_packet|. +class NewTimestampIsLarger { + public: + explicit NewTimestampIsLarger(const Packet& new_packet) + : new_packet_(new_packet) { + } + bool operator()(const Packet& packet) { + return (new_packet_ >= packet); + } + + private: + const Packet& new_packet_; +}; + +// Returns true if both payload types are known to the decoder database, and +// have the same sample rate. +bool EqualSampleRates(uint8_t pt1, + uint8_t pt2, + const DecoderDatabase& decoder_database) { + auto* di1 = decoder_database.GetDecoderInfo(pt1); + auto* di2 = decoder_database.GetDecoderInfo(pt2); + return di1 && di2 && di1->SampleRateHz() == di2->SampleRateHz(); +} + +void LogPacketDiscarded(int codec_level, StatisticsCalculator* stats) { + RTC_CHECK(stats); + if (codec_level > 0) { + stats->SecondaryPacketsDiscarded(1); + } else { + stats->PacketsDiscarded(1); + } +} + +} // namespace + +PacketBuffer::PacketBuffer(size_t max_number_of_packets, + const TickTimer* tick_timer) + : max_number_of_packets_(max_number_of_packets), tick_timer_(tick_timer) {} + +// Destructor. All packets in the buffer will be destroyed. +PacketBuffer::~PacketBuffer() { + Flush(); +} + +// Flush the buffer. All packets in the buffer will be destroyed. +void PacketBuffer::Flush() { + buffer_.clear(); +} + +bool PacketBuffer::Empty() const { + return buffer_.empty(); +} + +int PacketBuffer::InsertPacket(Packet&& packet, StatisticsCalculator* stats) { + if (packet.empty()) { + RTC_LOG(LS_WARNING) << "InsertPacket invalid packet"; + return kInvalidPacket; + } + + RTC_DCHECK_GE(packet.priority.codec_level, 0); + RTC_DCHECK_GE(packet.priority.red_level, 0); + + int return_val = kOK; + + packet.waiting_time = tick_timer_->GetNewStopwatch(); + + if (buffer_.size() >= max_number_of_packets_) { + // Buffer is full. Flush it. + Flush(); + RTC_LOG(LS_WARNING) << "Packet buffer flushed"; + return_val = kFlushed; + } + + // Get an iterator pointing to the place in the buffer where the new packet + // should be inserted. The list is searched from the back, since the most + // likely case is that the new packet should be near the end of the list. + PacketList::reverse_iterator rit = std::find_if( + buffer_.rbegin(), buffer_.rend(), + NewTimestampIsLarger(packet)); + + // The new packet is to be inserted to the right of |rit|. If it has the same + // timestamp as |rit|, which has a higher priority, do not insert the new + // packet to list. + if (rit != buffer_.rend() && packet.timestamp == rit->timestamp) { + LogPacketDiscarded(packet.priority.codec_level, stats); + return return_val; + } + + // The new packet is to be inserted to the left of |it|. If it has the same + // timestamp as |it|, which has a lower priority, replace |it| with the new + // packet. + PacketList::iterator it = rit.base(); + if (it != buffer_.end() && packet.timestamp == it->timestamp) { + LogPacketDiscarded(packet.priority.codec_level, stats); + it = buffer_.erase(it); + } + buffer_.insert(it, std::move(packet)); // Insert the packet at that position. + + return return_val; +} + +int PacketBuffer::InsertPacketList( + PacketList* packet_list, + const DecoderDatabase& decoder_database, + rtc::Optional* current_rtp_payload_type, + rtc::Optional* current_cng_rtp_payload_type, + StatisticsCalculator* stats) { + RTC_DCHECK(stats); + bool flushed = false; + for (auto& packet : *packet_list) { + if (decoder_database.IsComfortNoise(packet.payload_type)) { + if (*current_cng_rtp_payload_type && + **current_cng_rtp_payload_type != packet.payload_type) { + // New CNG payload type implies new codec type. + *current_rtp_payload_type = rtc::nullopt; + Flush(); + flushed = true; + } + *current_cng_rtp_payload_type = packet.payload_type; + } else if (!decoder_database.IsDtmf(packet.payload_type)) { + // This must be speech. + if ((*current_rtp_payload_type && + **current_rtp_payload_type != packet.payload_type) || + (*current_cng_rtp_payload_type && + !EqualSampleRates(packet.payload_type, + **current_cng_rtp_payload_type, + decoder_database))) { + *current_cng_rtp_payload_type = rtc::nullopt; + Flush(); + flushed = true; + } + *current_rtp_payload_type = packet.payload_type; + } + int return_val = InsertPacket(std::move(packet), stats); + if (return_val == kFlushed) { + // The buffer flushed, but this is not an error. We can still continue. + flushed = true; + } else if (return_val != kOK) { + // An error occurred. Delete remaining packets in list and return. + packet_list->clear(); + return return_val; + } + } + packet_list->clear(); + return flushed ? kFlushed : kOK; +} + +int PacketBuffer::NextTimestamp(uint32_t* next_timestamp) const { + if (Empty()) { + return kBufferEmpty; + } + if (!next_timestamp) { + return kInvalidPointer; + } + *next_timestamp = buffer_.front().timestamp; + return kOK; +} + +int PacketBuffer::NextHigherTimestamp(uint32_t timestamp, + uint32_t* next_timestamp) const { + if (Empty()) { + return kBufferEmpty; + } + if (!next_timestamp) { + return kInvalidPointer; + } + PacketList::const_iterator it; + for (it = buffer_.begin(); it != buffer_.end(); ++it) { + if (it->timestamp >= timestamp) { + // Found a packet matching the search. + *next_timestamp = it->timestamp; + return kOK; + } + } + return kNotFound; +} + +const Packet* PacketBuffer::PeekNextPacket() const { + return buffer_.empty() ? nullptr : &buffer_.front(); +} + +rtc::Optional PacketBuffer::GetNextPacket() { + if (Empty()) { + // Buffer is empty. + return rtc::nullopt; + } + + rtc::Optional packet(std::move(buffer_.front())); + // Assert that the packet sanity checks in InsertPacket method works. + RTC_DCHECK(!packet->empty()); + buffer_.pop_front(); + + return packet; +} + +int PacketBuffer::DiscardNextPacket(StatisticsCalculator* stats) { + if (Empty()) { + return kBufferEmpty; + } + // Assert that the packet sanity checks in InsertPacket method works. + const Packet& packet = buffer_.front(); + RTC_DCHECK(!packet.empty()); + LogPacketDiscarded(packet.priority.codec_level, stats); + buffer_.pop_front(); + return kOK; +} + +void PacketBuffer::DiscardOldPackets(uint32_t timestamp_limit, + uint32_t horizon_samples, + StatisticsCalculator* stats) { + buffer_.remove_if([timestamp_limit, horizon_samples, stats](const Packet& p) { + if (timestamp_limit == p.timestamp || + !IsObsoleteTimestamp(p.timestamp, timestamp_limit, horizon_samples)) { + return false; + } + LogPacketDiscarded(p.priority.codec_level, stats); + return true; + }); +} + +void PacketBuffer::DiscardAllOldPackets(uint32_t timestamp_limit, + StatisticsCalculator* stats) { + DiscardOldPackets(timestamp_limit, 0, stats); +} + +void PacketBuffer::DiscardPacketsWithPayloadType(uint8_t payload_type, + StatisticsCalculator* stats) { + buffer_.remove_if([payload_type, stats](const Packet& p) { + if (p.payload_type != payload_type) { + return false; + } + LogPacketDiscarded(p.priority.codec_level, stats); + return true; + }); +} + +size_t PacketBuffer::NumPacketsInBuffer() const { + return buffer_.size(); +} + +size_t PacketBuffer::NumSamplesInBuffer(size_t last_decoded_length) const { + size_t num_samples = 0; + size_t last_duration = last_decoded_length; + for (const Packet& packet : buffer_) { + if (packet.frame) { + // TODO(hlundin): Verify that it's fine to count all packets and remove + // this check. + if (packet.priority != Packet::Priority(0, 0)) { + continue; + } + size_t duration = packet.frame->Duration(); + if (duration > 0) { + last_duration = duration; // Save the most up-to-date (valid) duration. + } + } + num_samples += last_duration; + } + return num_samples; +} + +void PacketBuffer::BufferStat(int* num_packets, int* max_num_packets) const { + *num_packets = static_cast(buffer_.size()); + *max_num_packets = static_cast(max_number_of_packets_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.h new file mode 100644 index 0000000000..c83bf89c7d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_PACKET_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_PACKET_BUFFER_H_ + +#include "api/optional.h" +#include "modules/audio_coding/neteq/packet.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class DecoderDatabase; +class StatisticsCalculator; +class TickTimer; + +// This is the actual buffer holding the packets before decoding. +class PacketBuffer { + public: + enum BufferReturnCodes { + kOK = 0, + kFlushed, + kNotFound, + kBufferEmpty, + kInvalidPacket, + kInvalidPointer + }; + + // Constructor creates a buffer which can hold a maximum of + // |max_number_of_packets| packets. + PacketBuffer(size_t max_number_of_packets, const TickTimer* tick_timer); + + // Deletes all packets in the buffer before destroying the buffer. + virtual ~PacketBuffer(); + + // Flushes the buffer and deletes all packets in it. + virtual void Flush(); + + // Returns true for an empty buffer. + virtual bool Empty() const; + + // Inserts |packet| into the buffer. The buffer will take over ownership of + // the packet object. + // Returns PacketBuffer::kOK on success, PacketBuffer::kFlushed if the buffer + // was flushed due to overfilling. + virtual int InsertPacket(Packet&& packet, StatisticsCalculator* stats); + + // Inserts a list of packets into the buffer. The buffer will take over + // ownership of the packet objects. + // Returns PacketBuffer::kOK if all packets were inserted successfully. + // If the buffer was flushed due to overfilling, only a subset of the list is + // inserted, and PacketBuffer::kFlushed is returned. + // The last three parameters are included for legacy compatibility. + // TODO(hlundin): Redesign to not use current_*_payload_type and + // decoder_database. + virtual int InsertPacketList( + PacketList* packet_list, + const DecoderDatabase& decoder_database, + rtc::Optional* current_rtp_payload_type, + rtc::Optional* current_cng_rtp_payload_type, + StatisticsCalculator* stats); + + // Gets the timestamp for the first packet in the buffer and writes it to the + // output variable |next_timestamp|. + // Returns PacketBuffer::kBufferEmpty if the buffer is empty, + // PacketBuffer::kOK otherwise. + virtual int NextTimestamp(uint32_t* next_timestamp) const; + + // Gets the timestamp for the first packet in the buffer with a timestamp no + // lower than the input limit |timestamp|. The result is written to the output + // variable |next_timestamp|. + // Returns PacketBuffer::kBufferEmpty if the buffer is empty, + // PacketBuffer::kOK otherwise. + virtual int NextHigherTimestamp(uint32_t timestamp, + uint32_t* next_timestamp) const; + + // Returns a (constant) pointer to the first packet in the buffer. Returns + // NULL if the buffer is empty. + virtual const Packet* PeekNextPacket() const; + + // Extracts the first packet in the buffer and returns it. + // Returns an empty optional if the buffer is empty. + virtual rtc::Optional GetNextPacket(); + + // Discards the first packet in the buffer. The packet is deleted. + // Returns PacketBuffer::kBufferEmpty if the buffer is empty, + // PacketBuffer::kOK otherwise. + virtual int DiscardNextPacket(StatisticsCalculator* stats); + + // Discards all packets that are (strictly) older than timestamp_limit, + // but newer than timestamp_limit - horizon_samples. Setting horizon_samples + // to zero implies that the horizon is set to half the timestamp range. That + // is, if a packet is more than 2^31 timestamps into the future compared with + // timestamp_limit (including wrap-around), it is considered old. + virtual void DiscardOldPackets(uint32_t timestamp_limit, + uint32_t horizon_samples, + StatisticsCalculator* stats); + + // Discards all packets that are (strictly) older than timestamp_limit. + virtual void DiscardAllOldPackets(uint32_t timestamp_limit, + StatisticsCalculator* stats); + + // Removes all packets with a specific payload type from the buffer. + virtual void DiscardPacketsWithPayloadType(uint8_t payload_type, + StatisticsCalculator* stats); + + // Returns the number of packets in the buffer, including duplicates and + // redundant packets. + virtual size_t NumPacketsInBuffer() const; + + // Returns the number of samples in the buffer, including samples carried in + // duplicate and redundant packets. + virtual size_t NumSamplesInBuffer(size_t last_decoded_length) const; + + virtual void BufferStat(int* num_packets, int* max_num_packets) const; + + // Static method returning true if |timestamp| is older than |timestamp_limit| + // but less than |horizon_samples| behind |timestamp_limit|. For instance, + // with timestamp_limit = 100 and horizon_samples = 10, a timestamp in the + // range (90, 100) is considered obsolete, and will yield true. + // Setting |horizon_samples| to 0 is the same as setting it to 2^31, i.e., + // half the 32-bit timestamp range. + static bool IsObsoleteTimestamp(uint32_t timestamp, + uint32_t timestamp_limit, + uint32_t horizon_samples) { + return IsNewerTimestamp(timestamp_limit, timestamp) && + (horizon_samples == 0 || + IsNewerTimestamp(timestamp, timestamp_limit - horizon_samples)); + } + + private: + size_t max_number_of_packets_; + PacketList buffer_; + const TickTimer* tick_timer_; + RTC_DISALLOW_COPY_AND_ASSIGN(PacketBuffer); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_PACKET_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc new file mode 100644 index 0000000000..0ddeb8a585 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc @@ -0,0 +1,733 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for PacketBuffer class. + +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/mock/mock_statistics_calculator.h" +#include "modules/audio_coding/neteq/packet.h" +#include "modules/audio_coding/neteq/tick_timer.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::Return; +using ::testing::StrictMock; +using ::testing::_; +using ::testing::InSequence; +using ::testing::MockFunction; + +namespace webrtc { + +// Helper class to generate packets. Packets must be deleted by the user. +class PacketGenerator { + public: + PacketGenerator(uint16_t seq_no, uint32_t ts, uint8_t pt, int frame_size); + virtual ~PacketGenerator() {} + void Reset(uint16_t seq_no, uint32_t ts, uint8_t pt, int frame_size); + Packet NextPacket(int payload_size_bytes); + + uint16_t seq_no_; + uint32_t ts_; + uint8_t pt_; + int frame_size_; +}; + +PacketGenerator::PacketGenerator(uint16_t seq_no, uint32_t ts, uint8_t pt, + int frame_size) { + Reset(seq_no, ts, pt, frame_size); +} + +void PacketGenerator::Reset(uint16_t seq_no, uint32_t ts, uint8_t pt, + int frame_size) { + seq_no_ = seq_no; + ts_ = ts; + pt_ = pt; + frame_size_ = frame_size; +} + +Packet PacketGenerator::NextPacket(int payload_size_bytes) { + Packet packet; + packet.sequence_number = seq_no_; + packet.timestamp = ts_; + packet.payload_type = pt_; + packet.payload.SetSize(payload_size_bytes); + ++seq_no_; + ts_ += frame_size_; + return packet; +} + +struct PacketsToInsert { + uint16_t sequence_number; + uint32_t timestamp; + uint8_t payload_type; + bool primary; + // Order of this packet to appear upon extraction, after inserting a series + // of packets. A negative number means that it should have been discarded + // before extraction. + int extract_order; +}; + +// Start of test definitions. + +TEST(PacketBuffer, CreateAndDestroy) { + TickTimer tick_timer; + PacketBuffer* buffer = new PacketBuffer(10, &tick_timer); // 10 packets. + EXPECT_TRUE(buffer->Empty()); + delete buffer; +} + +TEST(PacketBuffer, InsertPacket) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(17u, 4711u, 0, 10); + StrictMock mock_stats; + + const int payload_len = 100; + const Packet packet = gen.NextPacket(payload_len); + EXPECT_EQ(0, buffer.InsertPacket(packet.Clone(), &mock_stats)); + uint32_t next_ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&next_ts)); + EXPECT_EQ(4711u, next_ts); + EXPECT_FALSE(buffer.Empty()); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + const Packet* next_packet = buffer.PeekNextPacket(); + EXPECT_EQ(packet, *next_packet); // Compare contents. + + // Do not explicitly flush buffer or delete packet to test that it is deleted + // with the buffer. (Tested with Valgrind or similar tool.) +} + +// Test to flush buffer. +TEST(PacketBuffer, FlushBuffer) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + const int payload_len = 10; + StrictMock mock_stats; + + // Insert 10 small packets; should be ok. + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacket(gen.NextPacket(payload_len), &mock_stats)); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + EXPECT_FALSE(buffer.Empty()); + + buffer.Flush(); + // Buffer should delete the payloads itself. + EXPECT_EQ(0u, buffer.NumPacketsInBuffer()); + EXPECT_TRUE(buffer.Empty()); +} + +// Test to fill the buffer over the limits, and verify that it flushes. +TEST(PacketBuffer, OverfillBuffer) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + StrictMock mock_stats; + + // Insert 10 small packets; should be ok. + const int payload_len = 10; + int i; + for (i = 0; i < 10; ++i) { + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacket(gen.NextPacket(payload_len), &mock_stats)); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + uint32_t next_ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&next_ts)); + EXPECT_EQ(0u, next_ts); // Expect first inserted packet to be first in line. + + const Packet packet = gen.NextPacket(payload_len); + // Insert 11th packet; should flush the buffer and insert it after flushing. + EXPECT_EQ(PacketBuffer::kFlushed, + buffer.InsertPacket(packet.Clone(), &mock_stats)); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&next_ts)); + // Expect last inserted packet to be first in line. + EXPECT_EQ(packet.timestamp, next_ts); + + // Flush buffer to delete all packets. + buffer.Flush(); +} + +// Test inserting a list of packets. +TEST(PacketBuffer, InsertPacketList) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + PacketList list; + const int payload_len = 10; + + // Insert 10 small packets. + for (int i = 0; i < 10; ++i) { + list.push_back(gen.NextPacket(payload_len)); + } + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info)); + + StrictMock mock_stats; + + rtc::Optional current_pt; + rtc::Optional current_cng_pt; + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacketList(&list, decoder_database, ¤t_pt, + ¤t_cng_pt, &mock_stats)); + EXPECT_TRUE(list.empty()); // The PacketBuffer should have depleted the list. + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + EXPECT_EQ(0, current_pt); // Current payload type changed to 0. + EXPECT_EQ(rtc::nullopt, current_cng_pt); // CNG payload type not changed. + + buffer.Flush(); // Clean up. + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test inserting a list of packets. Last packet is of a different payload type. +// Expecting the buffer to flush. +// TODO(hlundin): Remove this test when legacy operation is no longer needed. +TEST(PacketBuffer, InsertPacketListChangePayloadType) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + PacketList list; + const int payload_len = 10; + + // Insert 10 small packets. + for (int i = 0; i < 10; ++i) { + list.push_back(gen.NextPacket(payload_len)); + } + // Insert 11th packet of another payload type (not CNG). + { + Packet packet = gen.NextPacket(payload_len); + packet.payload_type = 1; + list.push_back(std::move(packet)); + } + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info0(NetEqDecoder::kDecoderPCMu, factory); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info0)); + const DecoderDatabase::DecoderInfo info1(NetEqDecoder::kDecoderPCMa, factory); + EXPECT_CALL(decoder_database, GetDecoderInfo(1)) + .WillRepeatedly(Return(&info1)); + + StrictMock mock_stats; + + rtc::Optional current_pt; + rtc::Optional current_cng_pt; + EXPECT_EQ(PacketBuffer::kFlushed, + buffer.InsertPacketList(&list, decoder_database, ¤t_pt, + ¤t_cng_pt, &mock_stats)); + EXPECT_TRUE(list.empty()); // The PacketBuffer should have depleted the list. + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); // Only the last packet. + EXPECT_EQ(1, current_pt); // Current payload type changed to 1. + EXPECT_EQ(rtc::nullopt, current_cng_pt); // CNG payload type not changed. + + buffer.Flush(); // Clean up. + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +TEST(PacketBuffer, ExtractOrderRedundancy) { + TickTimer tick_timer; + PacketBuffer buffer(100, &tick_timer); // 100 packets. + const int kPackets = 18; + const int kFrameSize = 10; + const int kPayloadLength = 10; + + PacketsToInsert packet_facts[kPackets] = { + {0xFFFD, 0xFFFFFFD7, 0, true, 0}, + {0xFFFE, 0xFFFFFFE1, 0, true, 1}, + {0xFFFE, 0xFFFFFFD7, 1, false, -1}, + {0xFFFF, 0xFFFFFFEB, 0, true, 2}, + {0xFFFF, 0xFFFFFFE1, 1, false, -1}, + {0x0000, 0xFFFFFFF5, 0, true, 3}, + {0x0000, 0xFFFFFFEB, 1, false, -1}, + {0x0001, 0xFFFFFFFF, 0, true, 4}, + {0x0001, 0xFFFFFFF5, 1, false, -1}, + {0x0002, 0x0000000A, 0, true, 5}, + {0x0002, 0xFFFFFFFF, 1, false, -1}, + {0x0003, 0x0000000A, 1, false, -1}, + {0x0004, 0x0000001E, 0, true, 7}, + {0x0004, 0x00000014, 1, false, 6}, + {0x0005, 0x0000001E, 0, true, -1}, + {0x0005, 0x00000014, 1, false, -1}, + {0x0006, 0x00000028, 0, true, 8}, + {0x0006, 0x0000001E, 1, false, -1}, + }; + + const size_t kExpectPacketsInBuffer = 9; + + std::vector expect_order(kExpectPacketsInBuffer); + + PacketGenerator gen(0, 0, 0, kFrameSize); + + StrictMock mock_stats; + + // Interleaving the EXPECT_CALL sequence with expectations on the MockFunction + // check ensures that exactly one call to PacketsDiscarded happens in each + // DiscardNextPacket call. + InSequence s; + MockFunction check; + for (int i = 0; i < kPackets; ++i) { + gen.Reset(packet_facts[i].sequence_number, + packet_facts[i].timestamp, + packet_facts[i].payload_type, + kFrameSize); + Packet packet = gen.NextPacket(kPayloadLength); + packet.priority.codec_level = packet_facts[i].primary ? 0 : 1; + if (packet_facts[i].extract_order < 0) { + if (packet.priority.codec_level > 0) { + EXPECT_CALL(mock_stats, SecondaryPacketsDiscarded(1)); + } else { + EXPECT_CALL(mock_stats, PacketsDiscarded(1)); + } + } + EXPECT_CALL(check, Call(i)); + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacket(packet.Clone(), &mock_stats)); + if (packet_facts[i].extract_order >= 0) { + expect_order[packet_facts[i].extract_order] = std::move(packet); + } + check.Call(i); + } + + EXPECT_EQ(kExpectPacketsInBuffer, buffer.NumPacketsInBuffer()); + + for (size_t i = 0; i < kExpectPacketsInBuffer; ++i) { + const rtc::Optional packet = buffer.GetNextPacket(); + EXPECT_EQ(packet, expect_order[i]); // Compare contents. + } + EXPECT_TRUE(buffer.Empty()); +} + +TEST(PacketBuffer, DiscardPackets) { + TickTimer tick_timer; + PacketBuffer buffer(100, &tick_timer); // 100 packets. + const uint16_t start_seq_no = 17; + const uint32_t start_ts = 4711; + const uint32_t ts_increment = 10; + PacketGenerator gen(start_seq_no, start_ts, 0, ts_increment); + PacketList list; + const int payload_len = 10; + StrictMock mock_stats; + + constexpr int kTotalPackets = 10; + // Insert 10 small packets. + for (int i = 0; i < kTotalPackets; ++i) { + buffer.InsertPacket(gen.NextPacket(payload_len), &mock_stats); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + + uint32_t current_ts = start_ts; + + // Discard them one by one and make sure that the right packets are at the + // front of the buffer. + constexpr int kDiscardPackets = 5; + + // Interleaving the EXPECT_CALL sequence with expectations on the MockFunction + // check ensures that exactly one call to PacketsDiscarded happens in each + // DiscardNextPacket call. + InSequence s; + MockFunction check; + for (int i = 0; i < kDiscardPackets; ++i) { + uint32_t ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&ts)); + EXPECT_EQ(current_ts, ts); + EXPECT_CALL(mock_stats, PacketsDiscarded(1)); + EXPECT_CALL(check, Call(i)); + EXPECT_EQ(PacketBuffer::kOK, buffer.DiscardNextPacket(&mock_stats)); + current_ts += ts_increment; + check.Call(i); + } + + constexpr int kRemainingPackets = kTotalPackets - kDiscardPackets; + // This will discard all remaining packets but one. The oldest packet is older + // than the indicated horizon_samples, and will thus be left in the buffer. + constexpr size_t kSkipPackets = 1; + EXPECT_CALL(mock_stats, PacketsDiscarded(1)) + .Times(kRemainingPackets - kSkipPackets); + EXPECT_CALL(check, Call(17)); // Arbitrary id number. + buffer.DiscardOldPackets(start_ts + kTotalPackets * ts_increment, + kRemainingPackets * ts_increment, &mock_stats); + check.Call(17); // Same arbitrary id number. + + EXPECT_EQ(kSkipPackets, buffer.NumPacketsInBuffer()); + uint32_t ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&ts)); + EXPECT_EQ(current_ts, ts); + + // Discard all remaining packets. + EXPECT_CALL(mock_stats, PacketsDiscarded(kSkipPackets)); + buffer.DiscardAllOldPackets(start_ts + kTotalPackets * ts_increment, + &mock_stats); + + EXPECT_TRUE(buffer.Empty()); +} + +TEST(PacketBuffer, Reordering) { + TickTimer tick_timer; + PacketBuffer buffer(100, &tick_timer); // 100 packets. + const uint16_t start_seq_no = 17; + const uint32_t start_ts = 4711; + const uint32_t ts_increment = 10; + PacketGenerator gen(start_seq_no, start_ts, 0, ts_increment); + const int payload_len = 10; + + // Generate 10 small packets and insert them into a PacketList. Insert every + // odd packet to the front, and every even packet to the back, thus creating + // a (rather strange) reordering. + PacketList list; + for (int i = 0; i < 10; ++i) { + Packet packet = gen.NextPacket(payload_len); + if (i % 2) { + list.push_front(std::move(packet)); + } else { + list.push_back(std::move(packet)); + } + } + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info)); + rtc::Optional current_pt; + rtc::Optional current_cng_pt; + + StrictMock mock_stats; + + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacketList(&list, decoder_database, ¤t_pt, + ¤t_cng_pt, &mock_stats)); + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + + // Extract them and make sure that come out in the right order. + uint32_t current_ts = start_ts; + for (int i = 0; i < 10; ++i) { + const rtc::Optional packet = buffer.GetNextPacket(); + ASSERT_TRUE(packet); + EXPECT_EQ(current_ts, packet->timestamp); + current_ts += ts_increment; + } + EXPECT_TRUE(buffer.Empty()); + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// The test first inserts a packet with narrow-band CNG, then a packet with +// wide-band speech. The expected behavior of the packet buffer is to detect a +// change in sample rate, even though no speech packet has been inserted before, +// and flush out the CNG packet. +TEST(PacketBuffer, CngFirstThenSpeechWithNewSampleRate) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + const uint8_t kCngPt = 13; + const int kPayloadLen = 10; + const uint8_t kSpeechPt = 100; + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info_cng(NetEqDecoder::kDecoderCNGnb, + factory); + EXPECT_CALL(decoder_database, GetDecoderInfo(kCngPt)) + .WillRepeatedly(Return(&info_cng)); + const DecoderDatabase::DecoderInfo info_speech(NetEqDecoder::kDecoderPCM16Bwb, + factory); + EXPECT_CALL(decoder_database, GetDecoderInfo(kSpeechPt)) + .WillRepeatedly(Return(&info_speech)); + + // Insert first packet, which is narrow-band CNG. + PacketGenerator gen(0, 0, kCngPt, 10); + PacketList list; + list.push_back(gen.NextPacket(kPayloadLen)); + rtc::Optional current_pt; + rtc::Optional current_cng_pt; + + StrictMock mock_stats; + + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacketList(&list, decoder_database, ¤t_pt, + ¤t_cng_pt, &mock_stats)); + EXPECT_TRUE(list.empty()); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + ASSERT_TRUE(buffer.PeekNextPacket()); + EXPECT_EQ(kCngPt, buffer.PeekNextPacket()->payload_type); + EXPECT_EQ(current_pt, rtc::nullopt); // Current payload type not set. + EXPECT_EQ(kCngPt, current_cng_pt); // CNG payload type set. + + // Insert second packet, which is wide-band speech. + { + Packet packet = gen.NextPacket(kPayloadLen); + packet.payload_type = kSpeechPt; + list.push_back(std::move(packet)); + } + // Expect the buffer to flush out the CNG packet, since it does not match the + // new speech sample rate. + EXPECT_EQ(PacketBuffer::kFlushed, + buffer.InsertPacketList(&list, decoder_database, ¤t_pt, + ¤t_cng_pt, &mock_stats)); + EXPECT_TRUE(list.empty()); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + ASSERT_TRUE(buffer.PeekNextPacket()); + EXPECT_EQ(kSpeechPt, buffer.PeekNextPacket()->payload_type); + + EXPECT_EQ(kSpeechPt, current_pt); // Current payload type set. + EXPECT_EQ(rtc::nullopt, current_cng_pt); // CNG payload type reset. + + buffer.Flush(); // Clean up. + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +TEST(PacketBuffer, Failures) { + const uint16_t start_seq_no = 17; + const uint32_t start_ts = 4711; + const uint32_t ts_increment = 10; + int payload_len = 100; + PacketGenerator gen(start_seq_no, start_ts, 0, ts_increment); + TickTimer tick_timer; + StrictMock mock_stats; + + PacketBuffer* buffer = new PacketBuffer(100, &tick_timer); // 100 packets. + { + Packet packet = gen.NextPacket(payload_len); + packet.payload.Clear(); + EXPECT_EQ(PacketBuffer::kInvalidPacket, + buffer->InsertPacket(std::move(packet), &mock_stats)); + } + // Buffer should still be empty. Test all empty-checks. + uint32_t temp_ts; + EXPECT_EQ(PacketBuffer::kBufferEmpty, buffer->NextTimestamp(&temp_ts)); + EXPECT_EQ(PacketBuffer::kBufferEmpty, + buffer->NextHigherTimestamp(0, &temp_ts)); + EXPECT_EQ(NULL, buffer->PeekNextPacket()); + EXPECT_FALSE(buffer->GetNextPacket()); + + // Discarding packets will not invoke mock_stats.PacketDiscarded() because the + // packet buffer is empty. + EXPECT_EQ(PacketBuffer::kBufferEmpty, buffer->DiscardNextPacket(&mock_stats)); + buffer->DiscardAllOldPackets(0, &mock_stats); + + // Insert one packet to make the buffer non-empty. + EXPECT_EQ(PacketBuffer::kOK, + buffer->InsertPacket(gen.NextPacket(payload_len), &mock_stats)); + EXPECT_EQ(PacketBuffer::kInvalidPointer, buffer->NextTimestamp(NULL)); + EXPECT_EQ(PacketBuffer::kInvalidPointer, + buffer->NextHigherTimestamp(0, NULL)); + delete buffer; + + // Insert packet list of three packets, where the second packet has an invalid + // payload. Expect first packet to be inserted, and the remaining two to be + // discarded. + buffer = new PacketBuffer(100, &tick_timer); // 100 packets. + PacketList list; + list.push_back(gen.NextPacket(payload_len)); // Valid packet. + { + Packet packet = gen.NextPacket(payload_len); + packet.payload.Clear(); // Invalid. + list.push_back(std::move(packet)); + } + list.push_back(gen.NextPacket(payload_len)); // Valid packet. + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info)); + rtc::Optional current_pt; + rtc::Optional current_cng_pt; + EXPECT_EQ(PacketBuffer::kInvalidPacket, + buffer->InsertPacketList(&list, decoder_database, ¤t_pt, + ¤t_cng_pt, &mock_stats)); + EXPECT_TRUE(list.empty()); // The PacketBuffer should have depleted the list. + EXPECT_EQ(1u, buffer->NumPacketsInBuffer()); + delete buffer; + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test packet comparison function. +// The function should return true if the first packet "goes before" the second. +TEST(PacketBuffer, ComparePackets) { + PacketGenerator gen(0, 0, 0, 10); + Packet a(gen.NextPacket(10)); // SN = 0, TS = 0. + Packet b(gen.NextPacket(10)); // SN = 1, TS = 10. + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Testing wrap-around case; 'a' is earlier but has a larger timestamp value. + a.timestamp = 0xFFFFFFFF - 10; + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Test equal packets. + EXPECT_TRUE(a == a); + EXPECT_FALSE(a != a); + EXPECT_FALSE(a < a); + EXPECT_FALSE(a > a); + EXPECT_TRUE(a <= a); + EXPECT_TRUE(a >= a); + + // Test equal timestamps but different sequence numbers (0 and 1). + a.timestamp = b.timestamp; + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Test equal timestamps but different sequence numbers (32767 and 1). + a.sequence_number = 0xFFFF; + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Test equal timestamps and sequence numbers, but differing priorities. + a.sequence_number = b.sequence_number; + a.priority = {1, 0}; + b.priority = {0, 0}; + // a after b + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_FALSE(a < b); + EXPECT_TRUE(a > b); + EXPECT_FALSE(a <= b); + EXPECT_TRUE(a >= b); + + Packet c(gen.NextPacket(0)); // SN = 2, TS = 20. + Packet d(gen.NextPacket(0)); // SN = 3, TS = 20. + c.timestamp = b.timestamp; + d.timestamp = b.timestamp; + c.sequence_number = b.sequence_number; + d.sequence_number = b.sequence_number; + c.priority = {1, 1}; + d.priority = {0, 1}; + // c after d + EXPECT_FALSE(c == d); + EXPECT_TRUE(c != d); + EXPECT_FALSE(c < d); + EXPECT_TRUE(c > d); + EXPECT_FALSE(c <= d); + EXPECT_TRUE(c >= d); + + // c after a + EXPECT_FALSE(c == a); + EXPECT_TRUE(c != a); + EXPECT_FALSE(c < a); + EXPECT_TRUE(c > a); + EXPECT_FALSE(c <= a); + EXPECT_TRUE(c >= a); + + // c after b + EXPECT_FALSE(c == b); + EXPECT_TRUE(c != b); + EXPECT_FALSE(c < b); + EXPECT_TRUE(c > b); + EXPECT_FALSE(c <= b); + EXPECT_TRUE(c >= b); + + // a after d + EXPECT_FALSE(a == d); + EXPECT_TRUE(a != d); + EXPECT_FALSE(a < d); + EXPECT_TRUE(a > d); + EXPECT_FALSE(a <= d); + EXPECT_TRUE(a >= d); + + // d after b + EXPECT_FALSE(d == b); + EXPECT_TRUE(d != b); + EXPECT_FALSE(d < b); + EXPECT_TRUE(d > b); + EXPECT_FALSE(d <= b); + EXPECT_TRUE(d >= b); +} + +namespace { +void TestIsObsoleteTimestamp(uint32_t limit_timestamp) { + // Check with zero horizon, which implies that the horizon is at 2^31, i.e., + // half the timestamp range. + static const uint32_t kZeroHorizon = 0; + static const uint32_t k2Pow31Minus1 = 0x7FFFFFFF; + // Timestamp on the limit is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp, limit_timestamp, kZeroHorizon)); + // 1 sample behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp - 1, limit_timestamp, kZeroHorizon)); + // 2^31 - 1 samples behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp - k2Pow31Minus1, limit_timestamp, kZeroHorizon)); + // 1 sample ahead is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp + 1, limit_timestamp, kZeroHorizon)); + // If |t1-t2|=2^31 and t1>t2, t2 is older than t1 but not the opposite. + uint32_t other_timestamp = limit_timestamp + (1 << 31); + uint32_t lowest_timestamp = std::min(limit_timestamp, other_timestamp); + uint32_t highest_timestamp = std::max(limit_timestamp, other_timestamp); + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp( + lowest_timestamp, highest_timestamp, kZeroHorizon)); + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + highest_timestamp, lowest_timestamp, kZeroHorizon)); + + // Fixed horizon at 10 samples. + static const uint32_t kHorizon = 10; + // Timestamp on the limit is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp, limit_timestamp, kHorizon)); + // 1 sample behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp - 1, limit_timestamp, kHorizon)); + // 9 samples behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp - 9, limit_timestamp, kHorizon)); + // 10 samples behind is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp - 10, limit_timestamp, kHorizon)); + // 2^31 - 1 samples behind is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp - k2Pow31Minus1, limit_timestamp, kHorizon)); + // 1 sample ahead is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp + 1, limit_timestamp, kHorizon)); + // 2^31 samples ahead is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp + (1 << 31), limit_timestamp, kHorizon)); +} +} // namespace + +// Test the IsObsoleteTimestamp method with different limit timestamps. +TEST(PacketBuffer, IsObsoleteTimestamp) { + TestIsObsoleteTimestamp(0); + TestIsObsoleteTimestamp(1); + TestIsObsoleteTimestamp(0xFFFFFFFF); // -1 in uint32_t. + TestIsObsoleteTimestamp(0x80000000); // 2^31. + TestIsObsoleteTimestamp(0x80000001); // 2^31 + 1. + TestIsObsoleteTimestamp(0x7FFFFFFF); // 2^31 - 1. +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.cc new file mode 100644 index 0000000000..a09d18ffd8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.cc @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/post_decode_vad.h" + +namespace webrtc { + +PostDecodeVad::~PostDecodeVad() { + if (vad_instance_) + WebRtcVad_Free(vad_instance_); +} + +void PostDecodeVad::Enable() { + if (!vad_instance_) { + // Create the instance. + vad_instance_ = WebRtcVad_Create(); + if (vad_instance_ == nullptr) { + // Failed to create instance. + Disable(); + return; + } + } + Init(); + enabled_ = true; +} + +void PostDecodeVad::Disable() { + enabled_ = false; + running_ = false; +} + +void PostDecodeVad::Init() { + running_ = false; + if (vad_instance_) { + WebRtcVad_Init(vad_instance_); + WebRtcVad_set_mode(vad_instance_, kVadMode); + running_ = true; + } +} + +void PostDecodeVad::Update(int16_t* signal, size_t length, + AudioDecoder::SpeechType speech_type, + bool sid_frame, + int fs_hz) { + if (!vad_instance_ || !enabled_) { + return; + } + + if (speech_type == AudioDecoder::kComfortNoise || sid_frame || + fs_hz > 16000) { + // TODO(hlundin): Remove restriction on fs_hz. + running_ = false; + active_speech_ = true; + sid_interval_counter_ = 0; + } else if (!running_) { + ++sid_interval_counter_; + } + + if (sid_interval_counter_ >= kVadAutoEnable) { + Init(); + } + + if (length > 0 && running_) { + size_t vad_sample_index = 0; + active_speech_ = false; + // Loop through frame sizes 30, 20, and 10 ms. + for (int vad_frame_size_ms = 30; vad_frame_size_ms >= 10; + vad_frame_size_ms -= 10) { + size_t vad_frame_size_samples = + static_cast(vad_frame_size_ms * fs_hz / 1000); + while (length - vad_sample_index >= vad_frame_size_samples) { + int vad_return = WebRtcVad_Process( + vad_instance_, fs_hz, &signal[vad_sample_index], + vad_frame_size_samples); + active_speech_ |= (vad_return == 1); + vad_sample_index += vad_frame_size_samples; + } + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.h new file mode 100644 index 0000000000..7b67bbe9b8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_POST_DECODE_VAD_H_ +#define MODULES_AUDIO_CODING_NETEQ_POST_DECODE_VAD_H_ + +#include // size_t + +#include "api/audio_codecs/audio_decoder.h" +#include "common_audio/vad/include/webrtc_vad.h" +#include "common_types.h" // NOLINT(build/include) // NULL +#include "modules/audio_coding/neteq/defines.h" +#include "modules/audio_coding/neteq/packet.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class PostDecodeVad { + public: + PostDecodeVad() + : enabled_(false), + running_(false), + active_speech_(true), + sid_interval_counter_(0), + vad_instance_(NULL) { + } + + virtual ~PostDecodeVad(); + + // Enables post-decode VAD. + void Enable(); + + // Disables post-decode VAD. + void Disable(); + + // Initializes post-decode VAD. + void Init(); + + // Updates post-decode VAD with the audio data in |signal| having |length| + // samples. The data is of type |speech_type|, at the sample rate |fs_hz|. + void Update(int16_t* signal, size_t length, + AudioDecoder::SpeechType speech_type, bool sid_frame, int fs_hz); + + // Accessors. + bool enabled() const { return enabled_; } + bool running() const { return running_; } + bool active_speech() const { return active_speech_; } + + private: + static const int kVadMode = 0; // Sets aggressiveness to "Normal". + // Number of Update() calls without CNG/SID before re-enabling VAD. + static const int kVadAutoEnable = 3000; + + bool enabled_; + bool running_; + bool active_speech_; + int sid_interval_counter_; + ::VadInst* vad_instance_; + + RTC_DISALLOW_COPY_AND_ASSIGN(PostDecodeVad); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_POST_DECODE_VAD_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc new file mode 100644 index 0000000000..da3e4e864e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for PostDecodeVad class. + +#include "modules/audio_coding/neteq/post_decode_vad.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(PostDecodeVad, CreateAndDestroy) { + PostDecodeVad vad; +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.cc new file mode 100644 index 0000000000..bc753895b1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.cc @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/preemptive_expand.h" + +#include // min, max + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +namespace webrtc { + +PreemptiveExpand::ReturnCodes PreemptiveExpand::Process( + const int16_t* input, + size_t input_length, + size_t old_data_length, + AudioMultiVector* output, + size_t* length_change_samples) { + old_data_length_per_channel_ = old_data_length; + // Input length must be (almost) 30 ms. + // Also, the new part must be at least |overlap_samples_| elements. + static const size_t k15ms = 120; // 15 ms = 120 samples at 8 kHz sample rate. + if (num_channels_ == 0 || + input_length / num_channels_ < (2 * k15ms - 1) * fs_mult_ || + old_data_length >= input_length / num_channels_ - overlap_samples_) { + // Length of input data too short to do preemptive expand. Simply move all + // data from input to output. + output->PushBackInterleaved(input, input_length); + return kError; + } + const bool kFastMode = false; // Fast mode is not available for PE Expand. + return TimeStretch::Process(input, input_length, kFastMode, output, + length_change_samples); +} + +void PreemptiveExpand::SetParametersForPassiveSpeech(size_t len, + int16_t* best_correlation, + size_t* peak_index) const { + // When the signal does not contain any active speech, the correlation does + // not matter. Simply set it to zero. + *best_correlation = 0; + + // For low energy expansion, the new data can be less than 15 ms, + // but we must ensure that best_correlation is not larger than the length of + // the new data. + // but we must ensure that best_correlation is not larger than the new data. + *peak_index = std::min(*peak_index, + len - old_data_length_per_channel_); +} + +PreemptiveExpand::ReturnCodes PreemptiveExpand::CheckCriteriaAndStretch( + const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool /*fast_mode*/, + AudioMultiVector* output) const { + // Pre-calculate common multiplication with |fs_mult_|. + // 120 corresponds to 15 ms. + size_t fs_mult_120 = static_cast(fs_mult_ * 120); + // Check for strong correlation (>0.9 in Q14) and at least 15 ms new data, + // or passive speech. + if (((best_correlation > kCorrelationThreshold) && + (old_data_length_per_channel_ <= fs_mult_120)) || + !active_speech) { + // Do accelerate operation by overlap add. + + // Set length of the first part, not to be modified. + size_t unmodified_length = std::max(old_data_length_per_channel_, + fs_mult_120); + // Copy first part, including cross-fade region. + output->PushBackInterleaved( + input, (unmodified_length + peak_index) * num_channels_); + // Copy the last |peak_index| samples up to 15 ms to |temp_vector|. + AudioMultiVector temp_vector(num_channels_); + temp_vector.PushBackInterleaved( + &input[(unmodified_length - peak_index) * num_channels_], + peak_index * num_channels_); + // Cross-fade |temp_vector| onto the end of |output|. + output->CrossFade(temp_vector, peak_index); + // Copy the last unmodified part, 15 ms + pitch period until the end. + output->PushBackInterleaved( + &input[unmodified_length * num_channels_], + input_length - unmodified_length * num_channels_); + + if (active_speech) { + return kSuccess; + } else { + return kSuccessLowEnergy; + } + } else { + // Accelerate not allowed. Simply move all data from decoded to outData. + output->PushBackInterleaved(input, input_length); + return kNoStretch; + } +} + +PreemptiveExpand* PreemptiveExpandFactory::Create( + int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise, + size_t overlap_samples) const { + return new PreemptiveExpand( + sample_rate_hz, num_channels, background_noise, overlap_samples); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.h new file mode 100644 index 0000000000..303501db12 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_PREEMPTIVE_EXPAND_H_ +#define MODULES_AUDIO_CODING_NETEQ_PREEMPTIVE_EXPAND_H_ + +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/time_stretch.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class BackgroundNoise; + +// This class implements the PreemptiveExpand operation. Most of the work is +// done in the base class TimeStretch, which is shared with the Accelerate +// operation. In the PreemptiveExpand class, the operations that are specific to +// PreemptiveExpand are implemented. +class PreemptiveExpand : public TimeStretch { + public: + PreemptiveExpand(int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise, + size_t overlap_samples) + : TimeStretch(sample_rate_hz, num_channels, background_noise), + old_data_length_per_channel_(0), + overlap_samples_(overlap_samples) { + } + + // This method performs the actual PreemptiveExpand operation. The samples are + // read from |input|, of length |input_length| elements, and are written to + // |output|. The number of samples added through time-stretching is + // is provided in the output |length_change_samples|. The method returns + // the outcome of the operation as an enumerator value. + ReturnCodes Process(const int16_t *pw16_decoded, + size_t len, + size_t old_data_len, + AudioMultiVector* output, + size_t* length_change_samples); + + protected: + // Sets the parameters |best_correlation| and |peak_index| to suitable + // values when the signal contains no active speech. + void SetParametersForPassiveSpeech(size_t input_length, + int16_t* best_correlation, + size_t* peak_index) const override; + + // Checks the criteria for performing the time-stretching operation and, + // if possible, performs the time-stretching. + ReturnCodes CheckCriteriaAndStretch(const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool /*fast_mode*/, + AudioMultiVector* output) const override; + + private: + size_t old_data_length_per_channel_; + size_t overlap_samples_; + + RTC_DISALLOW_COPY_AND_ASSIGN(PreemptiveExpand); +}; + +struct PreemptiveExpandFactory { + PreemptiveExpandFactory() {} + virtual ~PreemptiveExpandFactory() {} + + virtual PreemptiveExpand* Create( + int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise, + size_t overlap_samples) const; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_PREEMPTIVE_EXPAND_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.cc new file mode 100644 index 0000000000..c2df8cf80a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/random_vector.h" + +namespace webrtc { + +const int16_t RandomVector::kRandomTable[RandomVector::kRandomTableSize] = { + 2680, 5532, 441, 5520, 16170, -5146, -1024, -8733, 3115, 9598, -10380, + -4959, -1280, -21716, 7133, -1522, 13458, -3902, 2789, -675, 3441, 5016, + -13599, -4003, -2739, 3922, -7209, 13352, -11617, -7241, 12905, -2314, 5426, + 10121, -9702, 11207, -13542, 1373, 816, -5934, -12504, 4798, 1811, 4112, + -613, 201, -10367, -2960, -2419, 3442, 4299, -6116, -6092, 1552, -1650, + -480, -1237, 18720, -11858, -8303, -8212, 865, -2890, -16968, 12052, -5845, + -5912, 9777, -5665, -6294, 5426, -4737, -6335, 1652, 761, 3832, 641, -8552, + -9084, -5753, 8146, 12156, -4915, 15086, -1231, -1869, 11749, -9319, -6403, + 11407, 6232, -1683, 24340, -11166, 4017, -10448, 3153, -2936, 6212, 2891, + -866, -404, -4807, -2324, -1917, -2388, -6470, -3895, -10300, 5323, -5403, + 2205, 4640, 7022, -21186, -6244, -882, -10031, -3395, -12885, 7155, -5339, + 5079, -2645, -9515, 6622, 14651, 15852, 359, 122, 8246, -3502, -6696, -3679, + -13535, -1409, -704, -7403, -4007, 1798, 279, -420, -12796, -14219, 1141, + 3359, 11434, 7049, -6684, -7473, 14283, -4115, -9123, -8969, 4152, 4117, + 13792, 5742, 16168, 8661, -1609, -6095, 1881, 14380, -5588, 6758, -6425, + -22969, -7269, 7031, 1119, -1611, -5850, -11281, 3559, -8952, -10146, -4667, + -16251, -1538, 2062, -1012, -13073, 227, -3142, -5265, 20, 5770, -7559, + 4740, -4819, 992, -8208, -7130, -4652, 6725, 7369, -1036, 13144, -1588, + -5304, -2344, -449, -5705, -8894, 5205, -17904, -11188, -1022, 4852, 10101, + -5255, -4200, -752, 7941, -1543, 5959, 14719, 13346, 17045, -15605, -1678, + -1600, -9230, 68, 23348, 1172, 7750, 11212, -18227, 9956, 4161, 883, 3947, + 4341, 1014, -4889, -2603, 1246, -5630, -3596, -870, -1298, 2784, -3317, + -6612, -20541, 4166, 4181, -8625, 3562, 12890, 4761, 3205, -12259, -8579 }; + +void RandomVector::Reset() { + seed_ = 777; + seed_increment_ = 1; +} + +void RandomVector::Generate(size_t length, int16_t* output) { + for (size_t i = 0; i < length; i++) { + seed_ += seed_increment_; + size_t position = seed_ & (kRandomTableSize - 1); + output[i] = kRandomTable[position]; + } +} + +void RandomVector::IncreaseSeedIncrement(int16_t increase_by) { + seed_increment_+= increase_by; + seed_increment_ &= kRandomTableSize - 1; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.h new file mode 100644 index 0000000000..18adbe095c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_RANDOM_VECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_RANDOM_VECTOR_H_ + +#include // size_t + +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// This class generates pseudo-random samples. +class RandomVector { + public: + static const size_t kRandomTableSize = 256; + static const int16_t kRandomTable[kRandomTableSize]; + + RandomVector() + : seed_(777), + seed_increment_(1) { + } + + void Reset(); + + void Generate(size_t length, int16_t* output); + + void IncreaseSeedIncrement(int16_t increase_by); + + // Accessors and mutators. + int16_t seed_increment() { return seed_increment_; } + void set_seed_increment(int16_t value) { seed_increment_ = value; } + + private: + uint32_t seed_; + int16_t seed_increment_; + + RTC_DISALLOW_COPY_AND_ASSIGN(RandomVector); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_RANDOM_VECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector_unittest.cc new file mode 100644 index 0000000000..44479a6dd6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector_unittest.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for RandomVector class. + +#include "modules/audio_coding/neteq/random_vector.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(RandomVector, CreateAndDestroy) { + RandomVector random_vector; +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc new file mode 100644 index 0000000000..85e399c863 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/red_payload_splitter.h" + +#include +#include + +#include "modules/audio_coding/neteq/decoder_database.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +// The method loops through a list of packets {A, B, C, ...}. Each packet is +// split into its corresponding RED payloads, {A1, A2, ...}, which is +// temporarily held in the list |new_packets|. +// When the first packet in |packet_list| has been processed, the orignal packet +// is replaced by the new ones in |new_packets|, so that |packet_list| becomes: +// {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all +// the original packets have been replaced by their split payloads. +bool RedPayloadSplitter::SplitRed(PacketList* packet_list) { + // Too many RED blocks indicates that something is wrong. Clamp it at some + // reasonable value. + const size_t kMaxRedBlocks = 32; + bool ret = true; + PacketList::iterator it = packet_list->begin(); + while (it != packet_list->end()) { + const Packet& red_packet = *it; + assert(!red_packet.payload.empty()); + const uint8_t* payload_ptr = red_packet.payload.data(); + + // Read RED headers (according to RFC 2198): + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |F| block PT | timestamp offset | block length | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Last RED header: + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |0| Block PT | + // +-+-+-+-+-+-+-+-+ + + struct RedHeader { + uint8_t payload_type; + uint32_t timestamp; + size_t payload_length; + }; + + std::vector new_headers; + bool last_block = false; + size_t sum_length = 0; + while (!last_block) { + RedHeader new_header; + // Check the F bit. If F == 0, this was the last block. + last_block = ((*payload_ptr & 0x80) == 0); + // Bits 1 through 7 are payload type. + new_header.payload_type = payload_ptr[0] & 0x7F; + if (last_block) { + // No more header data to read. + ++sum_length; // Account for RED header size of 1 byte. + new_header.timestamp = red_packet.timestamp; + new_header.payload_length = red_packet.payload.size() - sum_length; + payload_ptr += 1; // Advance to first payload byte. + } else { + // Bits 8 through 21 are timestamp offset. + int timestamp_offset = + (payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2); + new_header.timestamp = red_packet.timestamp - timestamp_offset; + // Bits 22 through 31 are payload length. + new_header.payload_length = + ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3]; + payload_ptr += 4; // Advance to next RED header. + } + sum_length += new_header.payload_length; + sum_length += 4; // Account for RED header size of 4 bytes. + // Store in new list of packets. + new_headers.push_back(new_header); + } + + if (new_headers.size() <= kMaxRedBlocks) { + // Populate the new packets with payload data. + // |payload_ptr| now points at the first payload byte. + PacketList new_packets; // An empty list to store the split packets in. + for (size_t i = 0; i != new_headers.size(); ++i) { + const auto& new_header = new_headers[i]; + size_t payload_length = new_header.payload_length; + if (payload_ptr + payload_length > + red_packet.payload.data() + red_packet.payload.size()) { + // The block lengths in the RED headers do not match the overall + // packet length. Something is corrupt. Discard this and the remaining + // payloads from this packet. + RTC_LOG(LS_WARNING) << "SplitRed length mismatch"; + ret = false; + break; + } + + Packet new_packet; + new_packet.timestamp = new_header.timestamp; + new_packet.payload_type = new_header.payload_type; + new_packet.sequence_number = red_packet.sequence_number; + new_packet.priority.red_level = + rtc::dchecked_cast((new_headers.size() - 1) - i); + new_packet.payload.SetData(payload_ptr, payload_length); + new_packets.push_front(std::move(new_packet)); + payload_ptr += payload_length; + } + // Insert new packets into original list, before the element pointed to by + // iterator |it|. + packet_list->splice(it, std::move(new_packets)); + } else { + RTC_LOG(LS_WARNING) << "SplitRed too many blocks: " << new_headers.size(); + ret = false; + } + // Remove |it| from the packet list. This operation effectively moves the + // iterator |it| to the next packet in the list. Thus, we do not have to + // increment it manually. + it = packet_list->erase(it); + } + return ret; +} + +int RedPayloadSplitter::CheckRedPayloads( + PacketList* packet_list, + const DecoderDatabase& decoder_database) { + int main_payload_type = -1; + int num_deleted_packets = 0; + for (auto it = packet_list->begin(); it != packet_list->end(); /* */) { + uint8_t this_payload_type = it->payload_type; + if (!decoder_database.IsDtmf(this_payload_type) && + !decoder_database.IsComfortNoise(this_payload_type)) { + if (main_payload_type == -1) { + // This is the first packet in the list which is non-DTMF non-CNG. + main_payload_type = this_payload_type; + } else { + if (this_payload_type != main_payload_type) { + // We do not allow redundant payloads of a different type. + // Remove |it| from the packet list. This operation effectively + // moves the iterator |it| to the next packet in the list. Thus, we + // do not have to increment it manually. + it = packet_list->erase(it); + ++num_deleted_packets; + continue; + } + } + } + ++it; + } + return num_deleted_packets; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.h new file mode 100644 index 0000000000..1475b1b4a1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ + +#include "modules/audio_coding/neteq/packet.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Forward declarations. +class DecoderDatabase; + +// This class handles splitting of RED payloads into smaller parts. +// Codec-specific packet splitting can be performed by +// AudioDecoder::ParsePayload. +class RedPayloadSplitter { + public: + RedPayloadSplitter() {} + + virtual ~RedPayloadSplitter() {} + + // Splits each packet in |packet_list| into its separate RED payloads. Each + // RED payload is packetized into a Packet. The original elements in + // |packet_list| are properly deleted, and replaced by the new packets. + // Note that all packets in |packet_list| must be RED payloads, i.e., have + // RED headers according to RFC 2198 at the very beginning of the payload. + // Returns kOK or an error. + virtual bool SplitRed(PacketList* packet_list); + + // Checks all packets in |packet_list|. Packets that are DTMF events or + // comfort noise payloads are kept. Except that, only one single payload type + // is accepted. Any packet with another payload type is discarded. Returns + // the number of discarded packets. + virtual int CheckRedPayloads(PacketList* packet_list, + const DecoderDatabase& decoder_database); + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(RedPayloadSplitter); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc new file mode 100644 index 0000000000..4f511ad643 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for RedPayloadSplitter class. + +#include "modules/audio_coding/neteq/red_payload_splitter.h" + +#include + +#include +#include // pair + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/packet.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder_factory.h" + +using ::testing::Return; +using ::testing::ReturnNull; + +namespace webrtc { + +static const int kRedPayloadType = 100; +static const size_t kPayloadLength = 10; +static const size_t kRedHeaderLength = 4; // 4 bytes RED header. +static const uint16_t kSequenceNumber = 0; +static const uint32_t kBaseTimestamp = 0x12345678; + +// A possible Opus packet that contains FEC is the following. +// The frame is 20 ms in duration. +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |0|0|0|0|1|0|0|0|x|1|x|x|x|x|x|x|x| | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | +// | Compressed frame 1 (N-2 bytes)... : +// : | +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +void CreateOpusFecPayload(uint8_t* payload, + size_t payload_length, + uint8_t payload_value) { + if (payload_length < 2) { + return; + } + payload[0] = 0x08; + payload[1] = 0x40; + memset(&payload[2], payload_value, payload_length - 2); +} + +// RED headers (according to RFC 2198): +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |F| block PT | timestamp offset | block length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Last RED header: +// 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +// |0| Block PT | +// +-+-+-+-+-+-+-+-+ + +// Creates a RED packet, with |num_payloads| payloads, with payload types given +// by the values in array |payload_types| (which must be of length +// |num_payloads|). Each redundant payload is |timestamp_offset| samples +// "behind" the the previous payload. +Packet CreateRedPayload(size_t num_payloads, + uint8_t* payload_types, + int timestamp_offset, + bool embed_opus_fec = false) { + Packet packet; + packet.payload_type = kRedPayloadType; + packet.timestamp = kBaseTimestamp; + packet.sequence_number = kSequenceNumber; + packet.payload.SetSize((kPayloadLength + 1) + + (num_payloads - 1) * + (kPayloadLength + kRedHeaderLength)); + uint8_t* payload_ptr = packet.payload.data(); + for (size_t i = 0; i < num_payloads; ++i) { + // Write the RED headers. + if (i == num_payloads - 1) { + // Special case for last payload. + *payload_ptr = payload_types[i] & 0x7F; // F = 0; + ++payload_ptr; + break; + } + *payload_ptr = payload_types[i] & 0x7F; + // Not the last block; set F = 1. + *payload_ptr |= 0x80; + ++payload_ptr; + int this_offset = rtc::checked_cast( + (num_payloads - i - 1) * timestamp_offset); + *payload_ptr = this_offset >> 6; + ++payload_ptr; + assert(kPayloadLength <= 1023); // Max length described by 10 bits. + *payload_ptr = ((this_offset & 0x3F) << 2) | (kPayloadLength >> 8); + ++payload_ptr; + *payload_ptr = kPayloadLength & 0xFF; + ++payload_ptr; + } + for (size_t i = 0; i < num_payloads; ++i) { + // Write |i| to all bytes in each payload. + if (embed_opus_fec) { + CreateOpusFecPayload(payload_ptr, kPayloadLength, + static_cast(i)); + } else { + memset(payload_ptr, static_cast(i), kPayloadLength); + } + payload_ptr += kPayloadLength; + } + return packet; +} + +// Create a packet with all payload bytes set to |payload_value|. +Packet CreatePacket(uint8_t payload_type, + size_t payload_length, + uint8_t payload_value, + bool opus_fec = false) { + Packet packet; + packet.payload_type = payload_type; + packet.timestamp = kBaseTimestamp; + packet.sequence_number = kSequenceNumber; + packet.payload.SetSize(payload_length); + if (opus_fec) { + CreateOpusFecPayload(packet.payload.data(), packet.payload.size(), + payload_value); + } else { + memset(packet.payload.data(), payload_value, packet.payload.size()); + } + return packet; +} + +// Checks that |packet| has the attributes given in the remaining parameters. +void VerifyPacket(const Packet& packet, + size_t payload_length, + uint8_t payload_type, + uint16_t sequence_number, + uint32_t timestamp, + uint8_t payload_value, + Packet::Priority priority) { + EXPECT_EQ(payload_length, packet.payload.size()); + EXPECT_EQ(payload_type, packet.payload_type); + EXPECT_EQ(sequence_number, packet.sequence_number); + EXPECT_EQ(timestamp, packet.timestamp); + EXPECT_EQ(priority, packet.priority); + ASSERT_FALSE(packet.payload.empty()); + for (size_t i = 0; i < packet.payload.size(); ++i) { + ASSERT_EQ(payload_value, packet.payload.data()[i]); + } +} + +void VerifyPacket(const Packet& packet, + size_t payload_length, + uint8_t payload_type, + uint16_t sequence_number, + uint32_t timestamp, + uint8_t payload_value, + bool primary) { + return VerifyPacket(packet, payload_length, payload_type, sequence_number, + timestamp, payload_value, + Packet::Priority{0, primary ? 0 : 1}); +} + +// Start of test definitions. + +TEST(RedPayloadSplitter, CreateAndDestroy) { + RedPayloadSplitter* splitter = new RedPayloadSplitter; + delete splitter; +} + +// Packet A is split into A1 and A2. +TEST(RedPayloadSplitter, OnePacketTwoPayloads) { + uint8_t payload_types[] = {0, 0}; + const int kTimestampOffset = 160; + PacketList packet_list; + packet_list.push_back(CreateRedPayload(2, payload_types, kTimestampOffset)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(2u, packet_list.size()); + // Check first packet. The first in list should always be the primary payload. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[1], + kSequenceNumber, kBaseTimestamp, 1, true); + packet_list.pop_front(); + // Check second packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp - kTimestampOffset, 0, false); +} + +// Packets A and B are not split at all. Only the RED header in each packet is +// removed. +TEST(RedPayloadSplitter, TwoPacketsOnePayload) { + uint8_t payload_types[] = {0}; + const int kTimestampOffset = 160; + // Create first packet, with a single RED payload. + PacketList packet_list; + packet_list.push_back(CreateRedPayload(1, payload_types, kTimestampOffset)); + // Create second packet, with a single RED payload. + { + Packet packet = CreateRedPayload(1, payload_types, kTimestampOffset); + // Manually change timestamp and sequence number of second packet. + packet.timestamp += kTimestampOffset; + packet.sequence_number++; + packet_list.push_back(std::move(packet)); + } + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(2u, packet_list.size()); + // Check first packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp, 0, true); + packet_list.pop_front(); + // Check second packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber + 1, kBaseTimestamp + kTimestampOffset, 0, true); +} + +// Packets A and B are split into packets A1, A2, A3, B1, B2, B3, with +// attributes as follows: +// +// A1* A2 A3 B1* B2 B3 +// Payload type 0 1 2 0 1 2 +// Timestamp b b-o b-2o b+o b b-o +// Sequence number 0 0 0 1 1 1 +// +// b = kBaseTimestamp, o = kTimestampOffset, * = primary. +TEST(RedPayloadSplitter, TwoPacketsThreePayloads) { + uint8_t payload_types[] = {2, 1, 0}; // Primary is the last one. + const int kTimestampOffset = 160; + // Create first packet, with 3 RED payloads. + PacketList packet_list; + packet_list.push_back(CreateRedPayload(3, payload_types, kTimestampOffset)); + // Create first packet, with 3 RED payloads. + { + Packet packet = CreateRedPayload(3, payload_types, kTimestampOffset); + // Manually change timestamp and sequence number of second packet. + packet.timestamp += kTimestampOffset; + packet.sequence_number++; + packet_list.push_back(std::move(packet)); + } + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(6u, packet_list.size()); + // Check first packet, A1. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[2], + kSequenceNumber, kBaseTimestamp, 2, {0, 0}); + packet_list.pop_front(); + // Check second packet, A2. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[1], + kSequenceNumber, kBaseTimestamp - kTimestampOffset, 1, {0, 1}); + packet_list.pop_front(); + // Check third packet, A3. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp - 2 * kTimestampOffset, 0, + {0, 2}); + packet_list.pop_front(); + // Check fourth packet, B1. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[2], + kSequenceNumber + 1, kBaseTimestamp + kTimestampOffset, 2, + {0, 0}); + packet_list.pop_front(); + // Check fifth packet, B2. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[1], + kSequenceNumber + 1, kBaseTimestamp, 1, {0, 1}); + packet_list.pop_front(); + // Check sixth packet, B3. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber + 1, kBaseTimestamp - kTimestampOffset, 0, + {0, 2}); +} + +// Creates a list with 4 packets with these payload types: +// 0 = CNGnb +// 1 = PCMu +// 2 = DTMF (AVT) +// 3 = iLBC +// We expect the method CheckRedPayloads to discard the iLBC packet, since it +// is a non-CNG, non-DTMF payload of another type than the first speech payload +// found in the list (which is PCMu). +TEST(RedPayloadSplitter, CheckRedPayloads) { + PacketList packet_list; + for (uint8_t i = 0; i <= 3; ++i) { + // Create packet with payload type |i|, payload length 10 bytes, all 0. + packet_list.push_back(CreatePacket(i, 10, 0)); + } + + // Use a real DecoderDatabase object here instead of a mock, since it is + // easier to just register the payload types and let the actual implementation + // do its job. + DecoderDatabase decoder_database( + new rtc::RefCountedObject); + decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderCNGnb, "cng-nb"); + decoder_database.RegisterPayload(1, NetEqDecoder::kDecoderPCMu, "pcmu"); + decoder_database.RegisterPayload(2, NetEqDecoder::kDecoderAVT, "avt"); + decoder_database.RegisterPayload(3, NetEqDecoder::kDecoderILBC, "ilbc"); + + RedPayloadSplitter splitter; + splitter.CheckRedPayloads(&packet_list, decoder_database); + + ASSERT_EQ(3u, packet_list.size()); // Should have dropped the last packet. + // Verify packets. The loop verifies that payload types 0, 1, and 2 are in the + // list. + for (int i = 0; i <= 2; ++i) { + VerifyPacket(packet_list.front(), 10, i, kSequenceNumber, kBaseTimestamp, 0, + true); + packet_list.pop_front(); + } + EXPECT_TRUE(packet_list.empty()); +} + +// Packet A is split into A1, A2 and A3. But the length parameter is off, so +// the last payloads should be discarded. +TEST(RedPayloadSplitter, WrongPayloadLength) { + uint8_t payload_types[] = {0, 0, 0}; + const int kTimestampOffset = 160; + PacketList packet_list; + { + Packet packet = CreateRedPayload(3, payload_types, kTimestampOffset); + // Manually tamper with the payload length of the packet. + // This is one byte too short for the second payload (out of three). + // We expect only the first payload to be returned. + packet.payload.SetSize(packet.payload.size() - (kPayloadLength + 1)); + packet_list.push_back(std::move(packet)); + } + RedPayloadSplitter splitter; + EXPECT_FALSE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(1u, packet_list.size()); + // Check first packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp - 2 * kTimestampOffset, 0, + {0, 2}); + packet_list.pop_front(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.cc new file mode 100644 index 0000000000..2885398584 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/rtcp.h" + +#include +#include + +#include + +#include "modules/include/module_common_types.h" + +namespace webrtc { + +void Rtcp::Init(uint16_t start_sequence_number) { + cycles_ = 0; + max_seq_no_ = start_sequence_number; + base_seq_no_ = start_sequence_number; + received_packets_ = 0; + received_packets_prior_ = 0; + expected_prior_ = 0; + jitter_ = 0; + transit_ = 0; +} + +void Rtcp::Update(const RTPHeader& rtp_header, uint32_t receive_timestamp) { + // Update number of received packets, and largest packet number received. + received_packets_++; + int16_t sn_diff = rtp_header.sequenceNumber - max_seq_no_; + if (sn_diff >= 0) { + if (rtp_header.sequenceNumber < max_seq_no_) { + // Wrap-around detected. + cycles_++; + } + max_seq_no_ = rtp_header.sequenceNumber; + } + + // Calculate jitter according to RFC 3550, and update previous timestamps. + // Note that the value in |jitter_| is in Q4. + if (received_packets_ > 1) { + int32_t ts_diff = receive_timestamp - (rtp_header.timestamp - transit_); + int64_t jitter_diff = (std::abs(int64_t{ts_diff}) << 4) - jitter_; + // Calculate 15 * jitter_ / 16 + jitter_diff / 16 (with proper rounding). + jitter_ = jitter_ + ((jitter_diff + 8) >> 4); + RTC_DCHECK_GE(jitter_, 0); + } + transit_ = rtp_header.timestamp - receive_timestamp; +} + +void Rtcp::GetStatistics(bool no_reset, RtcpStatistics* stats) { + // Extended highest sequence number received. + stats->extended_highest_sequence_number = + (static_cast(cycles_) << 16) + max_seq_no_; + + // Calculate expected number of packets and compare it with the number of + // packets that were actually received. The cumulative number of lost packets + // can be extracted. + uint32_t expected_packets = + stats->extended_highest_sequence_number - base_seq_no_ + 1; + if (received_packets_ == 0) { + // No packets received, assume none lost. + stats->packets_lost = 0; + } else if (expected_packets > received_packets_) { + stats->packets_lost = expected_packets - received_packets_; + if (stats->packets_lost > 0xFFFFFF) { + stats->packets_lost = 0xFFFFFF; + } + } else { + stats->packets_lost = 0; + } + + // Fraction lost since last report. + uint32_t expected_since_last = expected_packets - expected_prior_; + uint32_t received_since_last = received_packets_ - received_packets_prior_; + if (!no_reset) { + expected_prior_ = expected_packets; + received_packets_prior_ = received_packets_; + } + int32_t lost = expected_since_last - received_since_last; + if (expected_since_last == 0 || lost <= 0 || received_packets_ == 0) { + stats->fraction_lost = 0; + } else { + stats->fraction_lost = std::min(0xFFU, (lost << 8) / expected_since_last); + } + + stats->jitter = jitter_ >> 4; // Scaling from Q4. +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.h new file mode 100644 index 0000000000..ce2035b219 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_RTCP_H_ +#define MODULES_AUDIO_CODING_NETEQ_RTCP_H_ + +#include "modules/audio_coding/neteq/include/neteq.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declaration. +struct RTPHeader; + +class Rtcp { + public: + Rtcp() { + Init(0); + } + + ~Rtcp() {} + + // Resets the RTCP statistics, and sets the first received sequence number. + void Init(uint16_t start_sequence_number); + + // Updates the RTCP statistics with a new received packet. + void Update(const RTPHeader& rtp_header, uint32_t receive_timestamp); + + // Returns the current RTCP statistics. If |no_reset| is true, the statistics + // are not reset, otherwise they are. + void GetStatistics(bool no_reset, RtcpStatistics* stats); + + private: + uint16_t cycles_; // The number of wrap-arounds for the sequence number. + uint16_t max_seq_no_; // The maximum sequence number received. Starts over + // from 0 after wrap-around. + uint16_t base_seq_no_; // The sequence number of the first received packet. + uint32_t received_packets_; // The number of packets that have been received. + uint32_t received_packets_prior_; // Number of packets received when last + // report was generated. + uint32_t expected_prior_; // Expected number of packets, at the time of the + // last report. + int64_t jitter_; // Current jitter value in Q4. + int32_t transit_; // Clock difference for previous packet. + + RTC_DISALLOW_COPY_AND_ASSIGN(Rtcp); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_RTCP_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.cc new file mode 100644 index 0000000000..7228a5778e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.cc @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/statistics_calculator.h" + +#include +#include // memset +#include + +#include "modules/audio_coding/neteq/delay_manager.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { +size_t AddIntToSizeTWithLowerCap(int a, size_t b) { + const size_t ret = b + a; + // If a + b is negative, resulting in a negative wrap, cap it to zero instead. + static_assert(sizeof(size_t) >= sizeof(int), + "int must not be wider than size_t for this to work"); + return (a < 0 && ret > b) ? 0 : ret; +} +} // namespace + +// Allocating the static const so that it can be passed by reference to +// RTC_DCHECK. +const size_t StatisticsCalculator::kLenWaitingTimes; + +StatisticsCalculator::PeriodicUmaLogger::PeriodicUmaLogger( + const std::string& uma_name, + int report_interval_ms, + int max_value) + : uma_name_(uma_name), + report_interval_ms_(report_interval_ms), + max_value_(max_value), + timer_(0) { +} + +StatisticsCalculator::PeriodicUmaLogger::~PeriodicUmaLogger() = default; + +void StatisticsCalculator::PeriodicUmaLogger::AdvanceClock(int step_ms) { + timer_ += step_ms; + if (timer_ < report_interval_ms_) { + return; + } + LogToUma(Metric()); + Reset(); + timer_ -= report_interval_ms_; + RTC_DCHECK_GE(timer_, 0); +} + +void StatisticsCalculator::PeriodicUmaLogger::LogToUma(int value) const { + RTC_HISTOGRAM_COUNTS_SPARSE(uma_name_, value, 1, max_value_, 50); +} + +StatisticsCalculator::PeriodicUmaCount::PeriodicUmaCount( + const std::string& uma_name, + int report_interval_ms, + int max_value) + : PeriodicUmaLogger(uma_name, report_interval_ms, max_value) { +} + +StatisticsCalculator::PeriodicUmaCount::~PeriodicUmaCount() { + // Log the count for the current (incomplete) interval. + LogToUma(Metric()); +} + +void StatisticsCalculator::PeriodicUmaCount::RegisterSample() { + ++counter_; +} + +int StatisticsCalculator::PeriodicUmaCount::Metric() const { + return counter_; +} + +void StatisticsCalculator::PeriodicUmaCount::Reset() { + counter_ = 0; +} + +StatisticsCalculator::PeriodicUmaAverage::PeriodicUmaAverage( + const std::string& uma_name, + int report_interval_ms, + int max_value) + : PeriodicUmaLogger(uma_name, report_interval_ms, max_value) { +} + +StatisticsCalculator::PeriodicUmaAverage::~PeriodicUmaAverage() { + // Log the average for the current (incomplete) interval. + LogToUma(Metric()); +} + +void StatisticsCalculator::PeriodicUmaAverage::RegisterSample(int value) { + sum_ += value; + ++counter_; +} + +int StatisticsCalculator::PeriodicUmaAverage::Metric() const { + return counter_ == 0 ? 0 : static_cast(sum_ / counter_); +} + +void StatisticsCalculator::PeriodicUmaAverage::Reset() { + sum_ = 0.0; + counter_ = 0; +} + +StatisticsCalculator::StatisticsCalculator() + : preemptive_samples_(0), + accelerate_samples_(0), + added_zero_samples_(0), + expanded_speech_samples_(0), + expanded_noise_samples_(0), + discarded_packets_(0), + lost_timestamps_(0), + timestamps_since_last_report_(0), + secondary_decoded_samples_(0), + discarded_secondary_packets_(0), + delayed_packet_outage_counter_( + "WebRTC.Audio.DelayedPacketOutageEventsPerMinute", + 60000, // 60 seconds report interval. + 100), + excess_buffer_delay_("WebRTC.Audio.AverageExcessBufferDelayMs", + 60000, // 60 seconds report interval. + 1000) {} + +StatisticsCalculator::~StatisticsCalculator() = default; + +void StatisticsCalculator::Reset() { + preemptive_samples_ = 0; + accelerate_samples_ = 0; + added_zero_samples_ = 0; + expanded_speech_samples_ = 0; + expanded_noise_samples_ = 0; + secondary_decoded_samples_ = 0; + discarded_secondary_packets_ = 0; + waiting_times_.clear(); +} + +void StatisticsCalculator::ResetMcu() { + discarded_packets_ = 0; + lost_timestamps_ = 0; + timestamps_since_last_report_ = 0; +} + +void StatisticsCalculator::ExpandedVoiceSamples(size_t num_samples, + bool is_new_concealment_event) { + expanded_speech_samples_ += num_samples; + ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples)); + lifetime_stats_.concealment_events += is_new_concealment_event; +} + +void StatisticsCalculator::ExpandedNoiseSamples(size_t num_samples, + bool is_new_concealment_event) { + expanded_noise_samples_ += num_samples; + ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples)); + lifetime_stats_.concealment_events += is_new_concealment_event; +} + +void StatisticsCalculator::ExpandedVoiceSamplesCorrection(int num_samples) { + expanded_speech_samples_ = + AddIntToSizeTWithLowerCap(num_samples, expanded_speech_samples_); + ConcealedSamplesCorrection(num_samples); +} + +void StatisticsCalculator::ExpandedNoiseSamplesCorrection(int num_samples) { + expanded_noise_samples_ = + AddIntToSizeTWithLowerCap(num_samples, expanded_noise_samples_); + ConcealedSamplesCorrection(num_samples); +} + +void StatisticsCalculator::ConcealedSamplesCorrection(int num_samples) { + if (num_samples < 0) { + // Store negative correction to subtract from future positive additions. + // See also the function comment in the header file. + concealed_samples_correction_ -= num_samples; + return; + } + + const size_t canceled_out = + std::min(static_cast(num_samples), concealed_samples_correction_); + concealed_samples_correction_ -= canceled_out; + lifetime_stats_.concealed_samples += num_samples - canceled_out; +} + +void StatisticsCalculator::PreemptiveExpandedSamples(size_t num_samples) { + preemptive_samples_ += num_samples; +} + +void StatisticsCalculator::AcceleratedSamples(size_t num_samples) { + accelerate_samples_ += num_samples; +} + +void StatisticsCalculator::AddZeros(size_t num_samples) { + added_zero_samples_ += num_samples; +} + +void StatisticsCalculator::PacketsDiscarded(size_t num_packets) { + discarded_packets_ += num_packets; +} + +void StatisticsCalculator::SecondaryPacketsDiscarded(size_t num_packets) { + discarded_secondary_packets_ += num_packets; +} + +void StatisticsCalculator::LostSamples(size_t num_samples) { + lost_timestamps_ += num_samples; +} + +void StatisticsCalculator::IncreaseCounter(size_t num_samples, int fs_hz) { + const int time_step_ms = + rtc::CheckedDivExact(static_cast(1000 * num_samples), fs_hz); + delayed_packet_outage_counter_.AdvanceClock(time_step_ms); + excess_buffer_delay_.AdvanceClock(time_step_ms); + timestamps_since_last_report_ += static_cast(num_samples); + if (timestamps_since_last_report_ > + static_cast(fs_hz * kMaxReportPeriod)) { + lost_timestamps_ = 0; + timestamps_since_last_report_ = 0; + discarded_packets_ = 0; + } + lifetime_stats_.total_samples_received += num_samples; +} + +void StatisticsCalculator::JitterBufferDelay(size_t num_samples, + uint64_t waiting_time_ms) { + lifetime_stats_.jitter_buffer_delay_ms += waiting_time_ms * num_samples; +} + +void StatisticsCalculator::SecondaryDecodedSamples(int num_samples) { + secondary_decoded_samples_ += num_samples; +} + +void StatisticsCalculator::LogDelayedPacketOutageEvent(int outage_duration_ms) { + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.DelayedPacketOutageEventMs", + outage_duration_ms, 1 /* min */, 2000 /* max */, + 100 /* bucket count */); + delayed_packet_outage_counter_.RegisterSample(); +} + +void StatisticsCalculator::StoreWaitingTime(int waiting_time_ms) { + excess_buffer_delay_.RegisterSample(waiting_time_ms); + RTC_DCHECK_LE(waiting_times_.size(), kLenWaitingTimes); + if (waiting_times_.size() == kLenWaitingTimes) { + // Erase first value. + waiting_times_.pop_front(); + } + waiting_times_.push_back(waiting_time_ms); +} + +void StatisticsCalculator::GetNetworkStatistics( + int fs_hz, + size_t num_samples_in_buffers, + size_t samples_per_packet, + NetEqNetworkStatistics *stats) { + RTC_DCHECK_GT(fs_hz, 0); + RTC_DCHECK(stats); + + stats->added_zero_samples = added_zero_samples_; + stats->current_buffer_size_ms = + static_cast(num_samples_in_buffers * 1000 / fs_hz); + + stats->packet_loss_rate = + CalculateQ14Ratio(lost_timestamps_, timestamps_since_last_report_); + + stats->accelerate_rate = + CalculateQ14Ratio(accelerate_samples_, timestamps_since_last_report_); + + stats->preemptive_rate = + CalculateQ14Ratio(preemptive_samples_, timestamps_since_last_report_); + + stats->expand_rate = + CalculateQ14Ratio(expanded_speech_samples_ + expanded_noise_samples_, + timestamps_since_last_report_); + + stats->speech_expand_rate = + CalculateQ14Ratio(expanded_speech_samples_, + timestamps_since_last_report_); + + stats->secondary_decoded_rate = + CalculateQ14Ratio(secondary_decoded_samples_, + timestamps_since_last_report_); + + const size_t discarded_secondary_samples = + discarded_secondary_packets_ * samples_per_packet; + stats->secondary_discarded_rate = CalculateQ14Ratio( + discarded_secondary_samples, + static_cast(discarded_secondary_samples + + secondary_decoded_samples_)); + + if (waiting_times_.size() == 0) { + stats->mean_waiting_time_ms = -1; + stats->median_waiting_time_ms = -1; + stats->min_waiting_time_ms = -1; + stats->max_waiting_time_ms = -1; + } else { + std::sort(waiting_times_.begin(), waiting_times_.end()); + // Find mid-point elements. If the size is odd, the two values + // |middle_left| and |middle_right| will both be the one middle element; if + // the size is even, they will be the the two neighboring elements at the + // middle of the list. + const int middle_left = waiting_times_[(waiting_times_.size() - 1) / 2]; + const int middle_right = waiting_times_[waiting_times_.size() / 2]; + // Calculate the average of the two. (Works also for odd sizes.) + stats->median_waiting_time_ms = (middle_left + middle_right) / 2; + stats->min_waiting_time_ms = waiting_times_.front(); + stats->max_waiting_time_ms = waiting_times_.back(); + double sum = 0; + for (auto time : waiting_times_) { + sum += time; + } + stats->mean_waiting_time_ms = static_cast(sum / waiting_times_.size()); + } + + // Reset counters. + ResetMcu(); + Reset(); +} + +void StatisticsCalculator::PopulateDelayManagerStats( + int ms_per_packet, + const DelayManager& delay_manager, + NetEqNetworkStatistics* stats) { + RTC_DCHECK(stats); + stats->preferred_buffer_size_ms = + (delay_manager.TargetLevel() >> 8) * ms_per_packet; + stats->jitter_peaks_found = delay_manager.PeakFound(); + stats->clockdrift_ppm = + rtc::saturated_cast(delay_manager.EstimatedClockDriftPpm()); +} + +NetEqLifetimeStatistics StatisticsCalculator::GetLifetimeStatistics() const { + return lifetime_stats_; +} + +uint16_t StatisticsCalculator::CalculateQ14Ratio(size_t numerator, + uint32_t denominator) { + if (numerator == 0) { + return 0; + } else if (numerator < denominator) { + // Ratio must be smaller than 1 in Q14. + assert((numerator << 14) / denominator < (1 << 14)); + return static_cast((numerator << 14) / denominator); + } else { + // Will not produce a ratio larger than 1, since this is probably an error. + return 1 << 14; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.h new file mode 100644 index 0000000000..c3d5c86cff --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_STATISTICS_CALCULATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_STATISTICS_CALCULATOR_H_ + +#include +#include + +#include "modules/audio_coding/neteq/include/neteq.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class DelayManager; + +// This class handles various network statistics in NetEq. +class StatisticsCalculator { + public: + StatisticsCalculator(); + + virtual ~StatisticsCalculator(); + + // Resets most of the counters. + void Reset(); + + // Resets the counters that are not handled by Reset(). + void ResetMcu(); + + // Reports that |num_samples| samples were produced through expansion, and + // that the expansion produced other than just noise samples. + void ExpandedVoiceSamples(size_t num_samples, bool is_new_concealment_event); + + // Reports that |num_samples| samples were produced through expansion, and + // that the expansion produced only noise samples. + void ExpandedNoiseSamples(size_t num_samples, bool is_new_concealment_event); + + // Corrects the statistics for number of samples produced through non-noise + // expansion by adding |num_samples| (negative or positive) to the current + // value. The result is capped to zero to avoid negative values. + void ExpandedVoiceSamplesCorrection(int num_samples); + + // Same as ExpandedVoiceSamplesCorrection but for noise samples. + void ExpandedNoiseSamplesCorrection(int num_samples); + + // Reports that |num_samples| samples were produced through preemptive + // expansion. + void PreemptiveExpandedSamples(size_t num_samples); + + // Reports that |num_samples| samples were removed through accelerate. + void AcceleratedSamples(size_t num_samples); + + // Reports that |num_samples| zeros were inserted into the output. + void AddZeros(size_t num_samples); + + // Reports that |num_packets| packets were discarded. + virtual void PacketsDiscarded(size_t num_packets); + + // Reports that |num_packets| packets samples were discarded. + virtual void SecondaryPacketsDiscarded(size_t num_samples); + + // Reports that |num_samples| were lost. + void LostSamples(size_t num_samples); + + // Increases the report interval counter with |num_samples| at a sample rate + // of |fs_hz|. This is how the StatisticsCalculator gets notified that current + // time is increasing. + void IncreaseCounter(size_t num_samples, int fs_hz); + + // Update jitter buffer delay counter. + void JitterBufferDelay(size_t num_samples, uint64_t waiting_time_ms); + + // Stores new packet waiting time in waiting time statistics. + void StoreWaitingTime(int waiting_time_ms); + + // Reports that |num_samples| samples were decoded from secondary packets. + void SecondaryDecodedSamples(int num_samples); + + // Logs a delayed packet outage event of |outage_duration_ms|. A delayed + // packet outage event is defined as an expand period caused not by an actual + // packet loss, but by a delayed packet. + virtual void LogDelayedPacketOutageEvent(int outage_duration_ms); + + // Returns the current network statistics in |stats|. The current sample rate + // is |fs_hz|, the total number of samples in packet buffer and sync buffer + // yet to play out is |num_samples_in_buffers|, and the number of samples per + // packet is |samples_per_packet|. The method does not populate + // |preferred_buffer_size_ms|, |jitter_peaks_found| or |clockdrift_ppm|; use + // the PopulateDelayManagerStats method for those. + void GetNetworkStatistics(int fs_hz, + size_t num_samples_in_buffers, + size_t samples_per_packet, + NetEqNetworkStatistics *stats); + + // Populates |preferred_buffer_size_ms|, |jitter_peaks_found| and + // |clockdrift_ppm| in |stats|. This is a convenience method, and does not + // strictly have to be in the StatisticsCalculator class, but it makes sense + // since all other stats fields are populated by that class. + static void PopulateDelayManagerStats(int ms_per_packet, + const DelayManager& delay_manager, + NetEqNetworkStatistics* stats); + + // Returns a copy of this class's lifetime statistics. These statistics are + // never reset. + NetEqLifetimeStatistics GetLifetimeStatistics() const; + + private: + static const int kMaxReportPeriod = 60; // Seconds before auto-reset. + static const size_t kLenWaitingTimes = 100; + + class PeriodicUmaLogger { + public: + PeriodicUmaLogger(const std::string& uma_name, + int report_interval_ms, + int max_value); + virtual ~PeriodicUmaLogger(); + void AdvanceClock(int step_ms); + + protected: + void LogToUma(int value) const; + virtual int Metric() const = 0; + virtual void Reset() = 0; + + const std::string uma_name_; + const int report_interval_ms_; + const int max_value_; + int timer_ = 0; + }; + + class PeriodicUmaCount final : public PeriodicUmaLogger { + public: + PeriodicUmaCount(const std::string& uma_name, + int report_interval_ms, + int max_value); + ~PeriodicUmaCount() override; + void RegisterSample(); + + protected: + int Metric() const override; + void Reset() override; + + private: + int counter_ = 0; + }; + + class PeriodicUmaAverage final : public PeriodicUmaLogger { + public: + PeriodicUmaAverage(const std::string& uma_name, + int report_interval_ms, + int max_value); + ~PeriodicUmaAverage() override; + void RegisterSample(int value); + + protected: + int Metric() const override; + void Reset() override; + + private: + double sum_ = 0.0; + int counter_ = 0; + }; + + // Corrects the concealed samples counter in lifetime_stats_. The value of + // num_samples_ is added directly to the stat if the correction is positive. + // If the correction is negative, it is cached and will be subtracted against + // future additions to the counter. This is meant to be called from + // Expanded{Voice,Noise}Samples{Correction}. + void ConcealedSamplesCorrection(int num_samples); + + // Calculates numerator / denominator, and returns the value in Q14. + static uint16_t CalculateQ14Ratio(size_t numerator, uint32_t denominator); + + NetEqLifetimeStatistics lifetime_stats_; + size_t concealed_samples_correction_ = 0; + size_t preemptive_samples_; + size_t accelerate_samples_; + size_t added_zero_samples_; + size_t expanded_speech_samples_; + size_t expanded_noise_samples_; + size_t discarded_packets_; + size_t lost_timestamps_; + uint32_t timestamps_since_last_report_; + std::deque waiting_times_; + uint32_t secondary_decoded_samples_; + size_t discarded_secondary_packets_; + PeriodicUmaCount delayed_packet_outage_counter_; + PeriodicUmaAverage excess_buffer_delay_; + + RTC_DISALLOW_COPY_AND_ASSIGN(StatisticsCalculator); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_STATISTICS_CALCULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc new file mode 100644 index 0000000000..0a4901d51b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/statistics_calculator.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(LifetimeStatistics, TotalSamplesReceived) { + StatisticsCalculator stats; + for (int i = 0; i < 10; ++i) { + stats.IncreaseCounter(480, 48000); // 10 ms at 48 kHz. + } + EXPECT_EQ(10 * 480u, stats.GetLifetimeStatistics().total_samples_received); +} + +TEST(LifetimeStatistics, SamplesConcealed) { + StatisticsCalculator stats; + stats.ExpandedVoiceSamples(100, false); + stats.ExpandedNoiseSamples(17, false); + EXPECT_EQ(100u + 17u, stats.GetLifetimeStatistics().concealed_samples); +} + +// This test verifies that a negative correction of concealed_samples does not +// result in a decrease in the stats value (because stats-consuming applications +// would not expect the value to decrease). Instead, the correction should be +// made to future increments to the stat. +TEST(LifetimeStatistics, SamplesConcealedCorrection) { + StatisticsCalculator stats; + stats.ExpandedVoiceSamples(100, false); + EXPECT_EQ(100u, stats.GetLifetimeStatistics().concealed_samples); + stats.ExpandedVoiceSamplesCorrection(-10); + // Do not subtract directly, but keep the correction for later. + EXPECT_EQ(100u, stats.GetLifetimeStatistics().concealed_samples); + stats.ExpandedVoiceSamplesCorrection(20); + // The total correction is 20 - 10. + EXPECT_EQ(110u, stats.GetLifetimeStatistics().concealed_samples); + + // Also test correction done to the next ExpandedVoiceSamples call. + stats.ExpandedVoiceSamplesCorrection(-17); + EXPECT_EQ(110u, stats.GetLifetimeStatistics().concealed_samples); + stats.ExpandedVoiceSamples(100, false); + EXPECT_EQ(110u + 100u - 17u, stats.GetLifetimeStatistics().concealed_samples); +} + +// This test verifies that neither "accelerate" nor "pre-emptive expand" reults +// in a modification to concealed_samples stats. Only PLC operations (i.e., +// "expand" and "merge") should affect the stat. +TEST(LifetimeStatistics, NoUpdateOnTimeStretch) { + StatisticsCalculator stats; + stats.ExpandedVoiceSamples(100, false); + stats.AcceleratedSamples(4711); + stats.PreemptiveExpandedSamples(17); + stats.ExpandedVoiceSamples(100, false); + EXPECT_EQ(200u, stats.GetLifetimeStatistics().concealed_samples); +} + +TEST(StatisticsCalculator, ExpandedSamplesCorrection) { + StatisticsCalculator stats; + NetEqNetworkStatistics stats_output; + constexpr int kSampleRateHz = 48000; + constexpr int k10MsSamples = kSampleRateHz / 100; + constexpr int kPacketSizeMs = 20; + constexpr size_t kSamplesPerPacket = kPacketSizeMs * kSampleRateHz / 1000; + // Assume 2 packets in the buffer. + constexpr size_t kNumSamplesInBuffer = 2 * kSamplesPerPacket; + + // Advance time by 10 ms. + stats.IncreaseCounter(k10MsSamples, kSampleRateHz); + + stats.GetNetworkStatistics(kSampleRateHz, kNumSamplesInBuffer, + kSamplesPerPacket, &stats_output); + + EXPECT_EQ(0u, stats_output.expand_rate); + EXPECT_EQ(0u, stats_output.speech_expand_rate); + + // Correct with a negative value. + stats.ExpandedVoiceSamplesCorrection(-100); + stats.ExpandedNoiseSamplesCorrection(-100); + stats.IncreaseCounter(k10MsSamples, kSampleRateHz); + stats.GetNetworkStatistics(kSampleRateHz, kNumSamplesInBuffer, + kSamplesPerPacket, &stats_output); + // Expect no change, since negative values are disallowed. + EXPECT_EQ(0u, stats_output.expand_rate); + EXPECT_EQ(0u, stats_output.speech_expand_rate); + + // Correct with a positive value. + stats.ExpandedVoiceSamplesCorrection(50); + stats.ExpandedNoiseSamplesCorrection(200); + stats.IncreaseCounter(k10MsSamples, kSampleRateHz); + stats.GetNetworkStatistics(kSampleRateHz, kNumSamplesInBuffer, + kSamplesPerPacket, &stats_output); + // Calculate expected rates in Q14. Expand rate is noise + voice, while + // speech expand rate is only voice. + EXPECT_EQ(((50u + 200u) << 14) / k10MsSamples, stats_output.expand_rate); + EXPECT_EQ((50u << 14) / k10MsSamples, stats_output.speech_expand_rate); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.cc new file mode 100644 index 0000000000..28d7649fdb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include // Access to min. + +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +size_t SyncBuffer::FutureLength() const { + return Size() - next_index_; +} + +void SyncBuffer::PushBack(const AudioMultiVector& append_this) { + size_t samples_added = append_this.Size(); + AudioMultiVector::PushBack(append_this); + AudioMultiVector::PopFront(samples_added); + if (samples_added <= next_index_) { + next_index_ -= samples_added; + } else { + // This means that we are pushing out future data that was never used. +// assert(false); + // TODO(hlundin): This assert must be disabled to support 60 ms frames. + // This should not happen even for 60 ms frames, but it does. Investigate + // why. + next_index_ = 0; + } + dtmf_index_ -= std::min(dtmf_index_, samples_added); +} + +void SyncBuffer::PushFrontZeros(size_t length) { + InsertZerosAtIndex(length, 0); +} + +void SyncBuffer::InsertZerosAtIndex(size_t length, size_t position) { + position = std::min(position, Size()); + length = std::min(length, Size() - position); + AudioMultiVector::PopBack(length); + for (size_t channel = 0; channel < Channels(); ++channel) { + channels_[channel]->InsertZerosAt(length, position); + } + if (next_index_ >= position) { + // We are moving the |next_index_| sample. + set_next_index(next_index_ + length); // Overflow handled by subfunction. + } + if (dtmf_index_ > 0 && dtmf_index_ >= position) { + // We are moving the |dtmf_index_| sample. + set_dtmf_index(dtmf_index_ + length); // Overflow handled by subfunction. + } +} + +void SyncBuffer::ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t length, + size_t position) { + position = std::min(position, Size()); // Cap |position| in the valid range. + length = std::min(length, Size() - position); + AudioMultiVector::OverwriteAt(insert_this, length, position); +} + +void SyncBuffer::ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t position) { + ReplaceAtIndex(insert_this, insert_this.Size(), position); +} + +void SyncBuffer::GetNextAudioInterleaved(size_t requested_len, + AudioFrame* output) { + RTC_DCHECK(output); + const size_t samples_to_read = std::min(FutureLength(), requested_len); + output->ResetWithoutMuting(); + const size_t tot_samples_read = + ReadInterleavedFromIndex(next_index_, samples_to_read, + output->mutable_data()); + const size_t samples_read_per_channel = tot_samples_read / Channels(); + next_index_ += samples_read_per_channel; + output->num_channels_ = Channels(); + output->samples_per_channel_ = samples_read_per_channel; +} + +void SyncBuffer::IncreaseEndTimestamp(uint32_t increment) { + end_timestamp_ += increment; +} + +void SyncBuffer::Flush() { + Zeros(Size()); + next_index_ = Size(); + end_timestamp_ = 0; + dtmf_index_ = 0; +} + +void SyncBuffer::set_next_index(size_t value) { + // Cannot set |next_index_| larger than the size of the buffer. + next_index_ = std::min(value, Size()); +} + +void SyncBuffer::set_dtmf_index(size_t value) { + // Cannot set |dtmf_index_| larger than the size of the buffer. + dtmf_index_ = std::min(value, Size()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.h new file mode 100644 index 0000000000..ab9ff525ef --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class SyncBuffer : public AudioMultiVector { + public: + SyncBuffer(size_t channels, size_t length) + : AudioMultiVector(channels, length), + next_index_(length), + end_timestamp_(0), + dtmf_index_(0) {} + + // Returns the number of samples yet to play out from the buffer. + size_t FutureLength() const; + + // Adds the contents of |append_this| to the back of the SyncBuffer. Removes + // the same number of samples from the beginning of the SyncBuffer, to + // maintain a constant buffer size. The |next_index_| is updated to reflect + // the move of the beginning of "future" data. + void PushBack(const AudioMultiVector& append_this) override; + + // Adds |length| zeros to the beginning of each channel. Removes + // the same number of samples from the end of the SyncBuffer, to + // maintain a constant buffer size. The |next_index_| is updated to reflect + // the move of the beginning of "future" data. + // Note that this operation may delete future samples that are waiting to + // be played. + void PushFrontZeros(size_t length); + + // Inserts |length| zeros into each channel at index |position|. The size of + // the SyncBuffer is kept constant, which means that the last |length| + // elements in each channel will be purged. + virtual void InsertZerosAtIndex(size_t length, size_t position); + + // Overwrites each channel in this SyncBuffer with values taken from + // |insert_this|. The values are taken from the beginning of |insert_this| and + // are inserted starting at |position|. |length| values are written into each + // channel. The size of the SyncBuffer is kept constant. That is, if |length| + // and |position| are selected such that the new data would extend beyond the + // end of the current SyncBuffer, the buffer is not extended. + // The |next_index_| is not updated. + virtual void ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t length, + size_t position); + + // Same as the above method, but where all of |insert_this| is written (with + // the same constraints as above, that the SyncBuffer is not extended). + virtual void ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t position); + + // Reads |requested_len| samples from each channel and writes them interleaved + // into |output|. The |next_index_| is updated to point to the sample to read + // next time. The AudioFrame |output| is first reset, and the |data_|, + // |num_channels_|, and |samples_per_channel_| fields are updated. + void GetNextAudioInterleaved(size_t requested_len, AudioFrame* output); + + // Adds |increment| to |end_timestamp_|. + void IncreaseEndTimestamp(uint32_t increment); + + // Flushes the buffer. The buffer will contain only zeros after the flush, and + // |next_index_| will point to the end, like when the buffer was first + // created. + void Flush(); + + const AudioVector& Channel(size_t n) const { return *channels_[n]; } + AudioVector& Channel(size_t n) { return *channels_[n]; } + + // Accessors and mutators. + size_t next_index() const { return next_index_; } + void set_next_index(size_t value); + uint32_t end_timestamp() const { return end_timestamp_; } + void set_end_timestamp(uint32_t value) { end_timestamp_ = value; } + size_t dtmf_index() const { return dtmf_index_; } + void set_dtmf_index(size_t value); + + private: + size_t next_index_; + uint32_t end_timestamp_; // The timestamp of the last sample in the buffer. + size_t dtmf_index_; // Index to the first non-DTMF sample in the buffer. + + RTC_DISALLOW_COPY_AND_ASSIGN(SyncBuffer); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc new file mode 100644 index 0000000000..29c3bcaffe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/numerics/safe_conversions.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(SyncBuffer, CreateAndDestroy) { + // Create a SyncBuffer with two channels and 10 samples each. + static const size_t kLen = 10; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + EXPECT_EQ(kChannels, sync_buffer.Channels()); + EXPECT_EQ(kLen, sync_buffer.Size()); + // When the buffer is empty, the next index to play out is at the end. + EXPECT_EQ(kLen, sync_buffer.next_index()); + // Verify that all elements are zero. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kLen; ++i) { + EXPECT_EQ(0, sync_buffer[channel][i]); + } + } +} + +TEST(SyncBuffer, SetNextIndex) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + sync_buffer.set_next_index(0); + EXPECT_EQ(0u, sync_buffer.next_index()); + sync_buffer.set_next_index(kLen / 2); + EXPECT_EQ(kLen / 2, sync_buffer.next_index()); + sync_buffer.set_next_index(kLen); + EXPECT_EQ(kLen, sync_buffer.next_index()); + // Try to set larger than the buffer size; should cap at buffer size. + sync_buffer.set_next_index(kLen + 1); + EXPECT_EQ(kLen, sync_buffer.next_index()); +} + +TEST(SyncBuffer, PushBackAndFlush) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + static const size_t kNewLen = 10; + AudioMultiVector new_data(kChannels, kNewLen); + // Populate |new_data|. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + new_data[channel][i] = rtc::checked_cast(i); + } + } + // Push back |new_data| into |sync_buffer|. This operation should pop out + // data from the front of |sync_buffer|, so that the size of the buffer + // remains the same. The |next_index_| should also move with the same length. + sync_buffer.PushBack(new_data); + ASSERT_EQ(kLen, sync_buffer.Size()); + // Verify that |next_index_| moved accordingly. + EXPECT_EQ(kLen - kNewLen, sync_buffer.next_index()); + // Verify the new contents. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + EXPECT_EQ(new_data[channel][i], + sync_buffer[channel][sync_buffer.next_index() + i]); + } + } + + // Now flush the buffer, and verify that it is all zeros, and that next_index + // points to the end. + sync_buffer.Flush(); + ASSERT_EQ(kLen, sync_buffer.Size()); + EXPECT_EQ(kLen, sync_buffer.next_index()); + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kLen; ++i) { + EXPECT_EQ(0, sync_buffer[channel][i]); + } + } +} + +TEST(SyncBuffer, PushFrontZeros) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + static const size_t kNewLen = 10; + AudioMultiVector new_data(kChannels, kNewLen); + // Populate |new_data|. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + new_data[channel][i] = rtc::checked_cast(1000 + i); + } + } + sync_buffer.PushBack(new_data); + EXPECT_EQ(kLen, sync_buffer.Size()); + + // Push |kNewLen| - 1 zeros into each channel in the front of the SyncBuffer. + sync_buffer.PushFrontZeros(kNewLen - 1); + EXPECT_EQ(kLen, sync_buffer.Size()); // Size should remain the same. + // Verify that |next_index_| moved accordingly. Should be at the end - 1. + EXPECT_EQ(kLen - 1, sync_buffer.next_index()); + // Verify the zeros. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen - 1; ++i) { + EXPECT_EQ(0, sync_buffer[channel][i]); + } + } + // Verify that the correct data is at the end of the SyncBuffer. + for (size_t channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000, sync_buffer[channel][sync_buffer.next_index()]); + } +} + +TEST(SyncBuffer, GetNextAudioInterleaved) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + static const size_t kNewLen = 10; + AudioMultiVector new_data(kChannels, kNewLen); + // Populate |new_data|. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + new_data[channel][i] = rtc::checked_cast(i); + } + } + // Push back |new_data| into |sync_buffer|. This operation should pop out + // data from the front of |sync_buffer|, so that the size of the buffer + // remains the same. The |next_index_| should also move with the same length. + sync_buffer.PushBack(new_data); + + // Read to interleaved output. Read in two batches, where each read operation + // should automatically update the |net_index_| in the SyncBuffer. + // Note that |samples_read| is the number of samples read from each channel. + // That is, the number of samples written to |output| is + // |samples_read| * |kChannels|. + AudioFrame output1; + sync_buffer.GetNextAudioInterleaved(kNewLen / 2, &output1); + EXPECT_EQ(kChannels, output1.num_channels_); + EXPECT_EQ(kNewLen / 2, output1.samples_per_channel_); + + AudioFrame output2; + sync_buffer.GetNextAudioInterleaved(kNewLen / 2, &output2); + EXPECT_EQ(kChannels, output2.num_channels_); + EXPECT_EQ(kNewLen / 2, output2.samples_per_channel_); + + // Verify the data. + const int16_t* output_ptr = output1.data(); + for (size_t i = 0; i < kNewLen / 2; ++i) { + for (size_t channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(new_data[channel][i], *output_ptr); + ++output_ptr; + } + } + output_ptr = output2.data(); + for (size_t i = kNewLen / 2; i < kNewLen; ++i) { + for (size_t channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(new_data[channel][i], *output_ptr); + ++output_ptr; + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m new file mode 100644 index 0000000000..031d8a39ee --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m @@ -0,0 +1,201 @@ +% +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function outStruct = parse_delay_file(file) + +fid = fopen(file, 'rb'); +if fid == -1 + error('Cannot open file %s', file); +end + +textline = fgetl(fid); +if ~strncmp(textline, '#!NetEQ_Delay_Logging', 21) + error('Wrong file format'); +end + +ver = sscanf(textline, '#!NetEQ_Delay_Logging%d.%d'); +if ~all(ver == [2; 0]) + error('Wrong version of delay logging function') +end + + +start_pos = ftell(fid); +fseek(fid, -12, 'eof'); +textline = fgetl(fid); +if ~strncmp(textline, 'End of file', 21) + error('File ending is not correct. Seems like the simulation ended abnormally.'); +end + +fseek(fid,-12-4, 'eof'); +Npackets = fread(fid, 1, 'int32'); +fseek(fid, start_pos, 'bof'); + +rtpts = zeros(Npackets, 1); +seqno = zeros(Npackets, 1); +pt = zeros(Npackets, 1); +plen = zeros(Npackets, 1); +recin_t = nan*ones(Npackets, 1); +decode_t = nan*ones(Npackets, 1); +playout_delay = zeros(Npackets, 1); +optbuf = zeros(Npackets, 1); + +fs_ix = 1; +clock = 0; +ts_ix = 1; +ended = 0; +late_packets = 0; +fs_now = 8000; +last_decode_k = 0; +tot_expand = 0; +tot_accelerate = 0; +tot_preemptive = 0; + +while not(ended) + signal = fread(fid, 1, '*int32'); + + switch signal + case 3 % NETEQ_DELAY_LOGGING_SIGNAL_CLOCK + clock = fread(fid, 1, '*float32'); + + % keep on reading batches of M until the signal is no longer "3" + % read int32 + float32 in one go + % this is to save execution time + temp = [3; 0]; + M = 120; + while all(temp(1,:) == 3) + fp = ftell(fid); + temp = fread(fid, [2 M], '*int32'); + end + + % back up to last clock event + fseek(fid, fp - ftell(fid) + ... + (find(temp(1,:) ~= 3, 1 ) - 2) * 2 * 4 + 4, 'cof'); + % read the last clock value + clock = fread(fid, 1, '*float32'); + + case 1 % NETEQ_DELAY_LOGGING_SIGNAL_RECIN + temp_ts = fread(fid, 1, 'uint32'); + + if late_packets > 0 + temp_ix = ts_ix - 1; + while (temp_ix >= 1) && (rtpts(temp_ix) ~= temp_ts) + % TODO(hlundin): use matlab vector search instead? + temp_ix = temp_ix - 1; + end + + if temp_ix >= 1 + % the ts was found in the vector + late_packets = late_packets - 1; + else + temp_ix = ts_ix; + ts_ix = ts_ix + 1; + end + else + temp_ix = ts_ix; + ts_ix = ts_ix + 1; + end + + rtpts(temp_ix) = temp_ts; + seqno(temp_ix) = fread(fid, 1, 'uint16'); + pt(temp_ix) = fread(fid, 1, 'int32'); + plen(temp_ix) = fread(fid, 1, 'int16'); + recin_t(temp_ix) = clock; + + case 2 % NETEQ_DELAY_LOGGING_SIGNAL_FLUSH + % do nothing + + case 4 % NETEQ_DELAY_LOGGING_SIGNAL_EOF + ended = 1; + + case 5 % NETEQ_DELAY_LOGGING_SIGNAL_DECODE + last_decode_ts = fread(fid, 1, 'uint32'); + temp_delay = fread(fid, 1, 'uint16'); + + k = find(rtpts(1:(ts_ix - 1))==last_decode_ts,1,'last'); + if ~isempty(k) + decode_t(k) = clock; + playout_delay(k) = temp_delay + ... + 5 * fs_now / 8000; % add overlap length + last_decode_k = k; + end + + case 6 % NETEQ_DELAY_LOGGING_SIGNAL_CHANGE_FS + fsvec(fs_ix) = fread(fid, 1, 'uint16'); + fschange_ts(fs_ix) = last_decode_ts; + fs_now = fsvec(fs_ix); + fs_ix = fs_ix + 1; + + case 7 % NETEQ_DELAY_LOGGING_SIGNAL_MERGE_INFO + playout_delay(last_decode_k) = playout_delay(last_decode_k) ... + + fread(fid, 1, 'int32'); + + case 8 % NETEQ_DELAY_LOGGING_SIGNAL_EXPAND_INFO + temp = fread(fid, 1, 'int32'); + if last_decode_k ~= 0 + tot_expand = tot_expand + temp / (fs_now / 1000); + end + + case 9 % NETEQ_DELAY_LOGGING_SIGNAL_ACCELERATE_INFO + temp = fread(fid, 1, 'int32'); + if last_decode_k ~= 0 + tot_accelerate = tot_accelerate + temp / (fs_now / 1000); + end + + case 10 % NETEQ_DELAY_LOGGING_SIGNAL_PREEMPTIVE_INFO + temp = fread(fid, 1, 'int32'); + if last_decode_k ~= 0 + tot_preemptive = tot_preemptive + temp / (fs_now / 1000); + end + + case 11 % NETEQ_DELAY_LOGGING_SIGNAL_OPTBUF + optbuf(last_decode_k) = fread(fid, 1, 'int32'); + + case 12 % NETEQ_DELAY_LOGGING_SIGNAL_DECODE_ONE_DESC + last_decode_ts = fread(fid, 1, 'uint32'); + k = ts_ix - 1; + + while (k >= 1) && (rtpts(k) ~= last_decode_ts) + % TODO(hlundin): use matlab vector search instead? + k = k - 1; + end + + if k < 1 + % packet not received yet + k = ts_ix; + rtpts(ts_ix) = last_decode_ts; + late_packets = late_packets + 1; + end + + decode_t(k) = clock; + playout_delay(k) = fread(fid, 1, 'uint16') + ... + 5 * fs_now / 8000; % add overlap length + last_decode_k = k; + + end + +end + + +fclose(fid); + +outStruct = struct(... + 'ts', rtpts, ... + 'sn', seqno, ... + 'pt', pt,... + 'plen', plen,... + 'arrival', recin_t,... + 'decode', decode_t,... + 'fs', fsvec(:),... + 'fschange_ts', fschange_ts(:),... + 'playout_delay', playout_delay,... + 'tot_expand', tot_expand,... + 'tot_accelerate', tot_accelerate,... + 'tot_preemptive', tot_preemptive,... + 'optbuf', optbuf); diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m new file mode 100644 index 0000000000..86d533fbeb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m @@ -0,0 +1,197 @@ +% +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function [delay_struct, delayvalues] = plot_neteq_delay(delayfile, varargin) + +% InfoStruct = plot_neteq_delay(delayfile) +% InfoStruct = plot_neteq_delay(delayfile, 'skipdelay', skip_seconds) +% +% Henrik Lundin, 2006-11-17 +% Henrik Lundin, 2011-05-17 +% + +try + s = parse_delay_file(delayfile); +catch + error(lasterr); +end + +delayskip=0; +noplot=0; +arg_ptr=1; +delaypoints=[]; + +s.sn=unwrap_seqno(s.sn); + +while arg_ptr+1 <= nargin + switch lower(varargin{arg_ptr}) + case {'skipdelay', 'delayskip'} + % skip a number of seconds in the beginning when calculating delays + delayskip = varargin{arg_ptr+1}; + arg_ptr = arg_ptr + 2; + case 'noplot' + noplot=1; + arg_ptr = arg_ptr + 1; + case {'get_delay', 'getdelay'} + % return a vector of delay values for the points in the given vector + delaypoints = varargin{arg_ptr+1}; + arg_ptr = arg_ptr + 2; + otherwise + warning('Unknown switch %s\n', varargin{arg_ptr}); + arg_ptr = arg_ptr + 1; + end +end + +% find lost frames that were covered by one-descriptor decoding +one_desc_ix=find(isnan(s.arrival)); +for k=1:length(one_desc_ix) + ix=find(s.ts==max(s.ts(s.ts(one_desc_ix(k))>s.ts))); + s.sn(one_desc_ix(k))=s.sn(ix)+1; + s.pt(one_desc_ix(k))=s.pt(ix); + s.arrival(one_desc_ix(k))=s.arrival(ix)+s.decode(one_desc_ix(k))-s.decode(ix); +end + +% remove duplicate received frames that were never decoded (RED codec) +if length(unique(s.ts(isfinite(s.ts)))) < length(s.ts(isfinite(s.ts))) + ix=find(isfinite(s.decode)); + s.sn=s.sn(ix); + s.ts=s.ts(ix); + s.arrival=s.arrival(ix); + s.playout_delay=s.playout_delay(ix); + s.pt=s.pt(ix); + s.optbuf=s.optbuf(ix); + plen=plen(ix); + s.decode=s.decode(ix); +end + +% find non-unique sequence numbers +[~,un_ix]=unique(s.sn); +nonun_ix=setdiff(1:length(s.sn),un_ix); +if ~isempty(nonun_ix) + warning('RTP sequence numbers are in error'); +end + +% sort vectors +[s.sn,sort_ix]=sort(s.sn); +s.ts=s.ts(sort_ix); +s.arrival=s.arrival(sort_ix); +s.decode=s.decode(sort_ix); +s.playout_delay=s.playout_delay(sort_ix); +s.pt=s.pt(sort_ix); + +send_t=s.ts-s.ts(1); +if length(s.fs)<1 + warning('No info about sample rate found in file. Using default 8000.'); + s.fs(1)=8000; + s.fschange_ts(1)=min(s.ts); +elseif s.fschange_ts(1)>min(s.ts) + s.fschange_ts(1)=min(s.ts); +end + +end_ix=length(send_t); +for k=length(s.fs):-1:1 + start_ix=find(s.ts==s.fschange_ts(k)); + send_t(start_ix:end_ix)=send_t(start_ix:end_ix)/s.fs(k)*1000; + s.playout_delay(start_ix:end_ix)=s.playout_delay(start_ix:end_ix)/s.fs(k)*1000; + s.optbuf(start_ix:end_ix)=s.optbuf(start_ix:end_ix)/s.fs(k)*1000; + end_ix=start_ix-1; +end + +tot_time=max(send_t)-min(send_t); + +seq_ix=s.sn-min(s.sn)+1; +send_t=send_t+max(min(s.arrival-send_t),0); + +plot_send_t=nan*ones(max(seq_ix),1); +plot_send_t(seq_ix)=send_t; +plot_nw_delay=nan*ones(max(seq_ix),1); +plot_nw_delay(seq_ix)=s.arrival-send_t; + +cng_ix=find(s.pt~=13); % find those packets that are not CNG/SID + +if noplot==0 + h=plot(plot_send_t/1000,plot_nw_delay); + set(h,'color',0.75*[1 1 1]); + hold on + if any(s.optbuf~=0) + peak_ix=find(s.optbuf(cng_ix)<0); % peak mode is labeled with negative values + no_peak_ix=find(s.optbuf(cng_ix)>0); %setdiff(1:length(cng_ix),peak_ix); + h1=plot(send_t(cng_ix(peak_ix))/1000,... + s.arrival(cng_ix(peak_ix))+abs(s.optbuf(cng_ix(peak_ix)))-send_t(cng_ix(peak_ix)),... + 'r.'); + h2=plot(send_t(cng_ix(no_peak_ix))/1000,... + s.arrival(cng_ix(no_peak_ix))+abs(s.optbuf(cng_ix(no_peak_ix)))-send_t(cng_ix(no_peak_ix)),... + 'g.'); + set([h1, h2],'markersize',1) + end + %h=plot(send_t(seq_ix)/1000,s.decode+s.playout_delay-send_t(seq_ix)); + h=plot(send_t(cng_ix)/1000,s.decode(cng_ix)+s.playout_delay(cng_ix)-send_t(cng_ix)); + set(h,'linew',1.5); + hold off + ax1=axis; + axis tight + ax2=axis; + axis([ax2(1:3) ax1(4)]) +end + + +% calculate delays and other parameters + +delayskip_ix = find(send_t-send_t(1)>=delayskip*1000, 1 ); + +use_ix = intersect(cng_ix,... % use those that are not CNG/SID frames... + intersect(find(isfinite(s.decode)),... % ... that did arrive ... + (delayskip_ix:length(s.decode))')); % ... and are sent after delayskip seconds + +mean_delay = mean(s.decode(use_ix)+s.playout_delay(use_ix)-send_t(use_ix)); +neteq_delay = mean(s.decode(use_ix)+s.playout_delay(use_ix)-s.arrival(use_ix)); + +Npack=max(s.sn(delayskip_ix:end))-min(s.sn(delayskip_ix:end))+1; +nw_lossrate=(Npack-length(s.sn(delayskip_ix:end)))/Npack; +neteq_lossrate=(length(s.sn(delayskip_ix:end))-length(use_ix))/Npack; + +delay_struct=struct('mean_delay',mean_delay,'neteq_delay',neteq_delay,... + 'nw_lossrate',nw_lossrate,'neteq_lossrate',neteq_lossrate,... + 'tot_expand',round(s.tot_expand),'tot_accelerate',round(s.tot_accelerate),... + 'tot_preemptive',round(s.tot_preemptive),'tot_time',tot_time,... + 'filename',delayfile,'units','ms','fs',unique(s.fs)); + +if not(isempty(delaypoints)) + delayvalues=interp1(send_t(cng_ix),... + s.decode(cng_ix)+s.playout_delay(cng_ix)-send_t(cng_ix),... + delaypoints,'nearest',NaN); +else + delayvalues=[]; +end + + + +% SUBFUNCTIONS % + +function y=unwrap_seqno(x) + +jumps=find(abs((diff(x)-1))>65000); + +while ~isempty(jumps) + n=jumps(1); + if x(n+1)-x(n) < 0 + % negative jump + x(n+1:end)=x(n+1:end)+65536; + else + % positive jump + x(n+1:end)=x(n+1:end)-65536; + end + + jumps=find(abs((diff(x(n+1:end))-1))>65000); +end + +y=x; + +return; diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc new file mode 100644 index 0000000000..bca401a5d2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "rtc_base/checks.h" +#include "rtc_base/flags.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/testsupport/fileutils.h" + +using testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { +static const int kInputSampleRateKhz = 8; +static const int kOutputSampleRateKhz = 8; + +DEFINE_int(frame_size_ms, 20, "Codec frame size (milliseconds)."); + +} // namespace + +class NetEqIlbcQualityTest : public NetEqQualityTest { + protected: + NetEqIlbcQualityTest() + : NetEqQualityTest(FLAG_frame_size_ms, + kInputSampleRateKhz, + kOutputSampleRateKhz, + NetEqDecoder::kDecoderILBC) { + // Flag validation + RTC_CHECK(FLAG_frame_size_ms == 20 || FLAG_frame_size_ms == 30 || + FLAG_frame_size_ms == 40 || FLAG_frame_size_ms == 60) + << "Invalid frame size, should be 20, 30, 40, or 60 ms."; + } + + void SetUp() override { + ASSERT_EQ(1u, channels_) << "iLBC supports only mono audio."; + AudioEncoderIlbcConfig config; + config.frame_size_ms = FLAG_frame_size_ms; + encoder_.reset(new AudioEncoderIlbcImpl(config, 102)); + NetEqQualityTest::SetUp(); + } + + int EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, size_t max_bytes) override { + const size_t kFrameSizeSamples = 80; // Samples per 10 ms. + size_t encoded_samples = 0; + uint32_t dummy_timestamp = 0; + AudioEncoder::EncodedInfo info; + do { + info = encoder_->Encode(dummy_timestamp, + rtc::ArrayView( + in_data + encoded_samples, kFrameSizeSamples), + payload); + encoded_samples += kFrameSizeSamples; + } while (info.encoded_bytes == 0); + return rtc::checked_cast(info.encoded_bytes); + } + + private: + std::unique_ptr encoder_; +}; + +TEST_F(NetEqIlbcQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_isac_quality_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_isac_quality_test.cc new file mode 100644 index 0000000000..d88f789f02 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_isac_quality_test.cc @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/fix/include/isacfix.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "rtc_base/flags.h" + +using testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { +static const int kIsacBlockDurationMs = 30; +static const int kIsacInputSamplingKhz = 16; +static const int kIsacOutputSamplingKhz = 16; + +DEFINE_int(bit_rate_kbps, 32, "Target bit rate (kbps)."); + +} // namespace + +class NetEqIsacQualityTest : public NetEqQualityTest { + protected: + NetEqIsacQualityTest(); + void SetUp() override; + void TearDown() override; + int EncodeBlock(int16_t* in_data, size_t block_size_samples, + rtc::Buffer* payload, size_t max_bytes) override; + private: + ISACFIX_MainStruct* isac_encoder_; + int bit_rate_kbps_; +}; + +NetEqIsacQualityTest::NetEqIsacQualityTest() + : NetEqQualityTest(kIsacBlockDurationMs, + kIsacInputSamplingKhz, + kIsacOutputSamplingKhz, + NetEqDecoder::kDecoderISAC), + isac_encoder_(NULL), + bit_rate_kbps_(FLAG_bit_rate_kbps) { + // Flag validation + RTC_CHECK(FLAG_bit_rate_kbps >= 10 && FLAG_bit_rate_kbps <= 32) + << "Invalid bit rate, should be between 10 and 32 kbps."; + } + +void NetEqIsacQualityTest::SetUp() { + ASSERT_EQ(1u, channels_) << "iSAC supports only mono audio."; + // Create encoder memory. + WebRtcIsacfix_Create(&isac_encoder_); + ASSERT_TRUE(isac_encoder_ != NULL); + EXPECT_EQ(0, WebRtcIsacfix_EncoderInit(isac_encoder_, 1)); + // Set bitrate and block length. + EXPECT_EQ(0, WebRtcIsacfix_Control(isac_encoder_, bit_rate_kbps_ * 1000, + kIsacBlockDurationMs)); + NetEqQualityTest::SetUp(); +} + +void NetEqIsacQualityTest::TearDown() { + // Free memory. + EXPECT_EQ(0, WebRtcIsacfix_Free(isac_encoder_)); + NetEqQualityTest::TearDown(); +} + +int NetEqIsacQualityTest::EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, size_t max_bytes) { + // ISAC takes 10 ms for every call. + const int subblocks = kIsacBlockDurationMs / 10; + const int subblock_length = 10 * kIsacInputSamplingKhz; + int value = 0; + + int pointer = 0; + for (int idx = 0; idx < subblocks; idx++, pointer += subblock_length) { + // The Isac encoder does not perform encoding (and returns 0) until it + // receives a sequence of sub-blocks that amount to the frame duration. + EXPECT_EQ(0, value); + payload->AppendData(max_bytes, [&] (rtc::ArrayView payload) { + value = WebRtcIsacfix_Encode(isac_encoder_, &in_data[pointer], + payload.data()); + return (value >= 0) ? static_cast(value) : 0; + }); + } + EXPECT_GT(value, 0); + return value; +} + +TEST_F(NetEqIsacQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc new file mode 100644 index 0000000000..c2542b6937 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/codecs/opus/opus_inst.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "rtc_base/flags.h" + +using testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { + +static const int kOpusBlockDurationMs = 20; +static const int kOpusSamplingKhz = 48; + +DEFINE_int(bit_rate_kbps, 32, "Target bit rate (kbps)."); + +DEFINE_int(complexity, 10, "Complexity: 0 ~ 10 -- defined as in Opus" + "specification."); + +DEFINE_int(maxplaybackrate, 48000, "Maximum playback rate (Hz)."); + +DEFINE_int(application, 0, "Application mode: 0 -- VOIP, 1 -- Audio."); + +DEFINE_int(reported_loss_rate, 10, "Reported percentile of packet loss."); + +DEFINE_bool(fec, false, "Enable FEC for encoding (-nofec to disable)."); + +DEFINE_bool(dtx, false, "Enable DTX for encoding (-nodtx to disable)."); + +DEFINE_int(sub_packets, 1, "Number of sub packets to repacketize."); + +} // namespace + +class NetEqOpusQualityTest : public NetEqQualityTest { + protected: + NetEqOpusQualityTest(); + void SetUp() override; + void TearDown() override; + int EncodeBlock(int16_t* in_data, size_t block_size_samples, + rtc::Buffer* payload, size_t max_bytes) override; + private: + WebRtcOpusEncInst* opus_encoder_; + OpusRepacketizer* repacketizer_; + size_t sub_block_size_samples_; + int bit_rate_kbps_; + bool fec_; + bool dtx_; + int complexity_; + int maxplaybackrate_; + int target_loss_rate_; + int sub_packets_; + int application_; +}; + +NetEqOpusQualityTest::NetEqOpusQualityTest() + : NetEqQualityTest(kOpusBlockDurationMs * FLAG_sub_packets, + kOpusSamplingKhz, + kOpusSamplingKhz, + NetEqDecoder::kDecoderOpus), + opus_encoder_(NULL), + repacketizer_(NULL), + sub_block_size_samples_( + static_cast(kOpusBlockDurationMs * kOpusSamplingKhz)), + bit_rate_kbps_(FLAG_bit_rate_kbps), + fec_(FLAG_fec), + dtx_(FLAG_dtx), + complexity_(FLAG_complexity), + maxplaybackrate_(FLAG_maxplaybackrate), + target_loss_rate_(FLAG_reported_loss_rate), + sub_packets_(FLAG_sub_packets) { + // Flag validation + RTC_CHECK(FLAG_bit_rate_kbps >= 6 && FLAG_bit_rate_kbps <= 510) + << "Invalid bit rate, should be between 6 and 510 kbps."; + + RTC_CHECK(FLAG_complexity >= -1 && FLAG_complexity <= 10) + << "Invalid complexity setting, should be between 0 and 10."; + + RTC_CHECK(FLAG_application == 0 || FLAG_application == 1) + << "Invalid application mode, should be 0 or 1."; + + RTC_CHECK(FLAG_reported_loss_rate >= 0 && FLAG_reported_loss_rate <= 100) + << "Invalid packet loss percentile, should be between 0 and 100."; + + RTC_CHECK(FLAG_sub_packets >= 1 && FLAG_sub_packets <= 3) + << "Invalid number of sub packets, should be between 1 and 3."; + + // Redefine decoder type if input is stereo. + if (channels_ > 1) { + decoder_type_ = NetEqDecoder::kDecoderOpus_2ch; + } + application_ = FLAG_application; +} + +void NetEqOpusQualityTest::SetUp() { + // Create encoder memory. + WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, application_); + ASSERT_TRUE(opus_encoder_); + + // Create repacketizer. + repacketizer_ = opus_repacketizer_create(); + ASSERT_TRUE(repacketizer_); + + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, bit_rate_kbps_ * 1000)); + if (fec_) { + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_encoder_)); + } + if (dtx_) { + EXPECT_EQ(0, WebRtcOpus_EnableDtx(opus_encoder_)); + } + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, complexity_)); + EXPECT_EQ(0, WebRtcOpus_SetMaxPlaybackRate(opus_encoder_, maxplaybackrate_)); + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_encoder_, + target_loss_rate_)); + NetEqQualityTest::SetUp(); +} + +void NetEqOpusQualityTest::TearDown() { + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + opus_repacketizer_destroy(repacketizer_); + NetEqQualityTest::TearDown(); +} + +int NetEqOpusQualityTest::EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, size_t max_bytes) { + EXPECT_EQ(block_size_samples, sub_block_size_samples_ * sub_packets_); + int16_t* pointer = in_data; + int value; + opus_repacketizer_init(repacketizer_); + for (int idx = 0; idx < sub_packets_; idx++) { + payload->AppendData(max_bytes, [&] (rtc::ArrayView payload) { + value = WebRtcOpus_Encode(opus_encoder_, + pointer, sub_block_size_samples_, + max_bytes, payload.data()); + + Log() << "Encoded a frame with Opus mode " + << (value == 0 ? 0 : payload[0] >> 3) + << std::endl; + + return (value >= 0) ? static_cast(value) : 0; + }); + + if (OPUS_OK != opus_repacketizer_cat(repacketizer_, + payload->data(), value)) { + opus_repacketizer_init(repacketizer_); + // If the repacketization fails, we discard this frame. + return 0; + } + pointer += sub_block_size_samples_ * channels_; + } + value = opus_repacketizer_out(repacketizer_, payload->data(), + static_cast(max_bytes)); + EXPECT_GE(value, 0); + return value; +} + +TEST_F(NetEqOpusQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc new file mode 100644 index 0000000000..bc3c1682e4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "rtc_base/checks.h" +#include "rtc_base/flags.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/testsupport/fileutils.h" + +using testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { +static const int kInputSampleRateKhz = 8; +static const int kOutputSampleRateKhz = 8; + +DEFINE_int(frame_size_ms, 20, "Codec frame size (milliseconds)."); + +} // namespace + +class NetEqPcmuQualityTest : public NetEqQualityTest { + protected: + NetEqPcmuQualityTest() + : NetEqQualityTest(FLAG_frame_size_ms, + kInputSampleRateKhz, + kOutputSampleRateKhz, + NetEqDecoder::kDecoderPCMu) { + // Flag validation + RTC_CHECK(FLAG_frame_size_ms >= 10 && FLAG_frame_size_ms <= 60 && + (FLAG_frame_size_ms % 10) == 0) + << "Invalid frame size, should be 10, 20, ..., 60 ms."; + } + + void SetUp() override { + ASSERT_EQ(1u, channels_) << "PCMu supports only mono audio."; + AudioEncoderPcmU::Config config; + config.frame_size_ms = FLAG_frame_size_ms; + encoder_.reset(new AudioEncoderPcmU(config)); + NetEqQualityTest::SetUp(); + } + + int EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, size_t max_bytes) override { + const size_t kFrameSizeSamples = 80; // Samples per 10 ms. + size_t encoded_samples = 0; + uint32_t dummy_timestamp = 0; + AudioEncoder::EncodedInfo info; + do { + info = encoder_->Encode(dummy_timestamp, + rtc::ArrayView( + in_data + encoded_samples, kFrameSizeSamples), + payload); + encoded_samples += kFrameSizeSamples; + } while (info.encoded_bytes == 0); + return rtc::checked_cast(info.encoded_bytes); + } + + private: + std::unique_ptr encoder_; +}; + +TEST_F(NetEqPcmuQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc new file mode 100644 index 0000000000..0510af8004 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_performance_test.h" +#include "test/gtest.h" +#include "test/testsupport/perf_test.h" +#include "typedefs.h" // NOLINT(build/include) +#include "system_wrappers/include/field_trial.h" + +// Runs a test with 10% packet losses and 10% clock drift, to exercise +// both loss concealment and time-stretching code. +TEST(NetEqPerformanceTest, Run) { + const int kSimulationTimeMs = 10000000; + const int kQuickSimulationTimeMs = 100000; + const int kLossPeriod = 10; // Drop every 10th packet. + const double kDriftFactor = 0.1; + int64_t runtime = webrtc::test::NetEqPerformanceTest::Run( + webrtc::field_trial::IsEnabled("WebRTC-QuickPerfTest") + ? kQuickSimulationTimeMs + : kSimulationTimeMs, + kLossPeriod, kDriftFactor); + ASSERT_GT(runtime, 0); + webrtc::test::PrintResult( + "neteq_performance", "", "10_pl_10_drift", runtime, "ms", true); +} + +// Runs a test with neither packet losses nor clock drift, to put +// emphasis on the "good-weather" code path, which is presumably much +// more lightweight. +TEST(NetEqPerformanceTest, RunClean) { + const int kSimulationTimeMs = 10000000; + const int kQuickSimulationTimeMs = 100000; + const int kLossPeriod = 0; // No losses. + const double kDriftFactor = 0.0; // No clock drift. + int64_t runtime = webrtc::test::NetEqPerformanceTest::Run( + webrtc::field_trial::IsEnabled("WebRTC-QuickPerfTest") + ? kQuickSimulationTimeMs + : kSimulationTimeMs, + kLossPeriod, kDriftFactor); + ASSERT_GT(runtime, 0); + webrtc::test::PrintResult( + "neteq_performance", "", "0_pl_0_drift", runtime, "ms", true); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc new file mode 100644 index 0000000000..ad123fe926 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include + +#include "modules/audio_coding/neteq/tools/neteq_performance_test.h" +#include "rtc_base/flags.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +// Define command line flags. +DEFINE_int(runtime_ms, 10000, "Simulated runtime in ms."); +DEFINE_int(lossrate, 10, + "Packet lossrate; drop every N packets."); +DEFINE_float(drift, 0.1f, + "Clockdrift factor."); +DEFINE_bool(help, false, "Print this message."); + +int main(int argc, char* argv[]) { + std::string program_name = argv[0]; + std::string usage = "Tool for measuring the speed of NetEq.\n" + "Usage: " + program_name + " [options]\n\n" + " --runtime_ms=N runtime in ms; default is 10000 ms\n" + " --lossrate=N drop every N packets; default is 10\n" + " --drift=F clockdrift factor between 0.0 and 1.0; " + "default is 0.1\n"; + webrtc::test::SetExecutablePath(argv[0]); + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || + FLAG_help || argc != 1) { + printf("%s", usage.c_str()); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + RTC_CHECK_GT(FLAG_runtime_ms, 0); + RTC_CHECK_GE(FLAG_lossrate, 0); + RTC_CHECK(FLAG_drift >= 0.0 && FLAG_drift < 1.0); + + int64_t result = + webrtc::test::NetEqPerformanceTest::Run(FLAG_runtime_ms, FLAG_lossrate, + FLAG_drift); + if (result <= 0) { + std::cout << "There was an error" << std::endl; + return -1; + } + + std::cout << "Simulation done" << std::endl; + std::cout << "Runtime = " << result << " ms" << std::endl; + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.cc new file mode 100644 index 0000000000..17f83b14ef --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tick_timer.h" + +namespace webrtc { + +TickTimer::Stopwatch::Stopwatch(const TickTimer& ticktimer) + : ticktimer_(ticktimer), starttick_(ticktimer.ticks()) {} + +TickTimer::Countdown::Countdown(const TickTimer& ticktimer, + uint64_t ticks_to_count) + : stopwatch_(ticktimer.GetNewStopwatch()), + ticks_to_count_(ticks_to_count) {} + +TickTimer::Countdown::~Countdown() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.h new file mode 100644 index 0000000000..4a9ade5cb3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TICK_TIMER_H_ +#define MODULES_AUDIO_CODING_NETEQ_TICK_TIMER_H_ + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Implements a time counter. The counter is advanced with the Increment() +// methods, and is queried with the ticks() accessor. It is assumed that one +// "tick" och the counter corresponds to 10 ms. +// A TickTimer object can provide two types of associated time-measuring +// objects: Stopwatch and Countdown. +class TickTimer { + public: + // Stopwatch measures time elapsed since it was started, by querying the + // associated TickTimer for the current time. The intended use is to request a + // new Stopwatch object from a TickTimer object with the GetNewStopwatch() + // method. Note: since the Stopwatch object contains a reference to the + // TickTimer it is associated with, it cannot outlive the TickTimer. + class Stopwatch { + public: + explicit Stopwatch(const TickTimer& ticktimer); + + uint64_t ElapsedTicks() const { return ticktimer_.ticks() - starttick_; } + + uint64_t ElapsedMs() const { + const uint64_t elapsed_ticks = ticktimer_.ticks() - starttick_; + const int ms_per_tick = ticktimer_.ms_per_tick(); + return elapsed_ticks < UINT64_MAX / ms_per_tick + ? elapsed_ticks * ms_per_tick + : UINT64_MAX; + } + + private: + const TickTimer& ticktimer_; + const uint64_t starttick_; + }; + + // Countdown counts down from a given start value with each tick of the + // associated TickTimer, until zero is reached. The Finished() method will + // return true if zero has been reached, false otherwise. The intended use is + // to request a new Countdown object from a TickTimer object with the + // GetNewCountdown() method. Note: since the Countdown object contains a + // reference to the TickTimer it is associated with, it cannot outlive the + // TickTimer. + class Countdown { + public: + Countdown(const TickTimer& ticktimer, uint64_t ticks_to_count); + + ~Countdown(); + + bool Finished() const { + return stopwatch_->ElapsedTicks() >= ticks_to_count_; + } + + private: + const std::unique_ptr stopwatch_; + const uint64_t ticks_to_count_; + }; + + TickTimer() : TickTimer(10) {} + explicit TickTimer(int ms_per_tick) : ms_per_tick_(ms_per_tick) { + RTC_DCHECK_GT(ms_per_tick_, 0); + } + + void Increment() { ++ticks_; } + + // Mainly intended for testing. + void Increment(uint64_t x) { ticks_ += x; } + + uint64_t ticks() const { return ticks_; } + + int ms_per_tick() const { return ms_per_tick_; } + + // Returns a new Stopwatch object, based on the current TickTimer. Note that + // the new Stopwatch object contains a reference to the current TickTimer, + // and must therefore not outlive the TickTimer. + std::unique_ptr GetNewStopwatch() const { + return std::unique_ptr(new Stopwatch(*this)); + } + + // Returns a new Countdown object, based on the current TickTimer. Note that + // the new Countdown object contains a reference to the current TickTimer, + // and must therefore not outlive the TickTimer. + std::unique_ptr GetNewCountdown(uint64_t ticks_to_count) const { + return std::unique_ptr(new Countdown(*this, ticks_to_count)); + } + + private: + uint64_t ticks_ = 0; + const int ms_per_tick_; + RTC_DISALLOW_COPY_AND_ASSIGN(TickTimer); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TICK_TIMER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer_unittest.cc new file mode 100644 index 0000000000..875f04db0a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer_unittest.cc @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/neteq/tick_timer.h" + +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +// Verify that the default value for ms_per_tick is 10. +TEST(TickTimer, DefaultMsPerTick) { + TickTimer tt; + EXPECT_EQ(10, tt.ms_per_tick()); +} + +TEST(TickTimer, CustomMsPerTick) { + TickTimer tt(17); + EXPECT_EQ(17, tt.ms_per_tick()); +} + +TEST(TickTimer, Increment) { + TickTimer tt; + EXPECT_EQ(0u, tt.ticks()); + tt.Increment(); + EXPECT_EQ(1u, tt.ticks()); + + for (int i = 0; i < 17; ++i) { + tt.Increment(); + } + EXPECT_EQ(18u, tt.ticks()); + + tt.Increment(17); + EXPECT_EQ(35u, tt.ticks()); +} + +TEST(TickTimer, WrapAround) { + TickTimer tt; + tt.Increment(UINT64_MAX); + EXPECT_EQ(UINT64_MAX, tt.ticks()); + tt.Increment(); + EXPECT_EQ(0u, tt.ticks()); +} + +TEST(TickTimer, Stopwatch) { + TickTimer tt; + // Increment it a "random" number of steps. + tt.Increment(17); + + std::unique_ptr sw = tt.GetNewStopwatch(); + ASSERT_TRUE(sw); + + EXPECT_EQ(0u, sw->ElapsedTicks()); // Starts at zero. + EXPECT_EQ(0u, sw->ElapsedMs()); + tt.Increment(); + EXPECT_EQ(1u, sw->ElapsedTicks()); // Increases with the TickTimer. + EXPECT_EQ(10u, sw->ElapsedMs()); +} + +TEST(TickTimer, StopwatchWrapAround) { + TickTimer tt; + tt.Increment(UINT64_MAX); + + std::unique_ptr sw = tt.GetNewStopwatch(); + ASSERT_TRUE(sw); + + tt.Increment(); + EXPECT_EQ(0u, tt.ticks()); + EXPECT_EQ(1u, sw->ElapsedTicks()); + EXPECT_EQ(10u, sw->ElapsedMs()); + + tt.Increment(); + EXPECT_EQ(1u, tt.ticks()); + EXPECT_EQ(2u, sw->ElapsedTicks()); + EXPECT_EQ(20u, sw->ElapsedMs()); +} + +TEST(TickTimer, StopwatchMsOverflow) { + TickTimer tt; + std::unique_ptr sw = tt.GetNewStopwatch(); + ASSERT_TRUE(sw); + + tt.Increment(UINT64_MAX / 10); + EXPECT_EQ(UINT64_MAX, sw->ElapsedMs()); + + tt.Increment(); + EXPECT_EQ(UINT64_MAX, sw->ElapsedMs()); + + tt.Increment(UINT64_MAX - tt.ticks()); + EXPECT_EQ(UINT64_MAX, tt.ticks()); + EXPECT_EQ(UINT64_MAX, sw->ElapsedMs()); +} + +TEST(TickTimer, StopwatchWithCustomTicktime) { + const int kMsPerTick = 17; + TickTimer tt(kMsPerTick); + std::unique_ptr sw = tt.GetNewStopwatch(); + ASSERT_TRUE(sw); + + EXPECT_EQ(0u, sw->ElapsedMs()); + tt.Increment(); + EXPECT_EQ(static_cast(kMsPerTick), sw->ElapsedMs()); +} + +TEST(TickTimer, Countdown) { + TickTimer tt; + // Increment it a "random" number of steps. + tt.Increment(4711); + + std::unique_ptr cd = tt.GetNewCountdown(17); + ASSERT_TRUE(cd); + + EXPECT_FALSE(cd->Finished()); + tt.Increment(); + EXPECT_FALSE(cd->Finished()); + + tt.Increment(16); // Total increment is now 17. + EXPECT_TRUE(cd->Finished()); + + // Further increments do not change the state. + tt.Increment(); + EXPECT_TRUE(cd->Finished()); + tt.Increment(1234); + EXPECT_TRUE(cd->Finished()); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.cc new file mode 100644 index 0000000000..8a1bfa2835 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.cc @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/time_stretch.h" + +#include // min, max +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +TimeStretch::ReturnCodes TimeStretch::Process(const int16_t* input, + size_t input_len, + bool fast_mode, + AudioMultiVector* output, + size_t* length_change_samples) { + // Pre-calculate common multiplication with |fs_mult_|. + size_t fs_mult_120 = + static_cast(fs_mult_ * 120); // Corresponds to 15 ms. + + const int16_t* signal; + std::unique_ptr signal_array; + size_t signal_len; + if (num_channels_ == 1) { + signal = input; + signal_len = input_len; + } else { + // We want |signal| to be only the first channel of |input|, which is + // interleaved. Thus, we take the first sample, skip forward |num_channels| + // samples, and continue like that. + signal_len = input_len / num_channels_; + signal_array.reset(new int16_t[signal_len]); + signal = signal_array.get(); + size_t j = master_channel_; + for (size_t i = 0; i < signal_len; ++i) { + signal_array[i] = input[j]; + j += num_channels_; + } + } + + // Find maximum absolute value of input signal. + max_input_value_ = WebRtcSpl_MaxAbsValueW16(signal, signal_len); + + // Downsample to 4 kHz sample rate and calculate auto-correlation. + DspHelper::DownsampleTo4kHz(signal, signal_len, kDownsampledLen, + sample_rate_hz_, true /* compensate delay*/, + downsampled_input_); + AutoCorrelation(); + + // Find the strongest correlation peak. + static const size_t kNumPeaks = 1; + size_t peak_index; + int16_t peak_value; + DspHelper::PeakDetection(auto_correlation_, kCorrelationLen, kNumPeaks, + fs_mult_, &peak_index, &peak_value); + // Assert that |peak_index| stays within boundaries. + assert(peak_index <= (2 * kCorrelationLen - 1) * fs_mult_); + + // Compensate peak_index for displaced starting position. The displacement + // happens in AutoCorrelation(). Here, |kMinLag| is in the down-sampled 4 kHz + // domain, while the |peak_index| is in the original sample rate; hence, the + // multiplication by fs_mult_ * 2. + peak_index += kMinLag * fs_mult_ * 2; + // Assert that |peak_index| stays within boundaries. + assert(peak_index >= static_cast(20 * fs_mult_)); + assert(peak_index <= 20 * fs_mult_ + (2 * kCorrelationLen - 1) * fs_mult_); + + // Calculate scaling to ensure that |peak_index| samples can be square-summed + // without overflowing. + int scaling = 31 - WebRtcSpl_NormW32(max_input_value_ * max_input_value_) - + WebRtcSpl_NormW32(static_cast(peak_index)); + scaling = std::max(0, scaling); + + // |vec1| starts at 15 ms minus one pitch period. + const int16_t* vec1 = &signal[fs_mult_120 - peak_index]; + // |vec2| start at 15 ms. + const int16_t* vec2 = &signal[fs_mult_120]; + // Calculate energies for |vec1| and |vec2|, assuming they both contain + // |peak_index| samples. + int32_t vec1_energy = + WebRtcSpl_DotProductWithScale(vec1, vec1, peak_index, scaling); + int32_t vec2_energy = + WebRtcSpl_DotProductWithScale(vec2, vec2, peak_index, scaling); + + // Calculate cross-correlation between |vec1| and |vec2|. + int32_t cross_corr = + WebRtcSpl_DotProductWithScale(vec1, vec2, peak_index, scaling); + + // Check if the signal seems to be active speech or not (simple VAD). + bool active_speech = SpeechDetection(vec1_energy, vec2_energy, peak_index, + scaling); + + int16_t best_correlation; + if (!active_speech) { + SetParametersForPassiveSpeech(signal_len, &best_correlation, &peak_index); + } else { + // Calculate correlation: + // cross_corr / sqrt(vec1_energy * vec2_energy). + + // Start with calculating scale values. + int energy1_scale = std::max(0, 16 - WebRtcSpl_NormW32(vec1_energy)); + int energy2_scale = std::max(0, 16 - WebRtcSpl_NormW32(vec2_energy)); + + // Make sure total scaling is even (to simplify scale factor after sqrt). + if ((energy1_scale + energy2_scale) & 1) { + // The sum is odd. + energy1_scale += 1; + } + + // Scale energies to int16_t. + int16_t vec1_energy_int16 = + static_cast(vec1_energy >> energy1_scale); + int16_t vec2_energy_int16 = + static_cast(vec2_energy >> energy2_scale); + + // Calculate square-root of energy product. + int16_t sqrt_energy_prod = WebRtcSpl_SqrtFloor(vec1_energy_int16 * + vec2_energy_int16); + + // Calculate cross_corr / sqrt(en1*en2) in Q14. + int temp_scale = 14 - (energy1_scale + energy2_scale) / 2; + cross_corr = WEBRTC_SPL_SHIFT_W32(cross_corr, temp_scale); + cross_corr = std::max(0, cross_corr); // Don't use if negative. + best_correlation = WebRtcSpl_DivW32W16(cross_corr, sqrt_energy_prod); + // Make sure |best_correlation| is no larger than 1 in Q14. + best_correlation = std::min(static_cast(16384), best_correlation); + } + + + // Check accelerate criteria and stretch the signal. + ReturnCodes return_value = + CheckCriteriaAndStretch(input, input_len, peak_index, best_correlation, + active_speech, fast_mode, output); + switch (return_value) { + case kSuccess: + *length_change_samples = peak_index; + break; + case kSuccessLowEnergy: + *length_change_samples = peak_index; + break; + case kNoStretch: + case kError: + *length_change_samples = 0; + break; + } + return return_value; +} + +void TimeStretch::AutoCorrelation() { + // Calculate correlation from lag kMinLag to lag kMaxLag in 4 kHz domain. + int32_t auto_corr[kCorrelationLen]; + CrossCorrelationWithAutoShift( + &downsampled_input_[kMaxLag], &downsampled_input_[kMaxLag - kMinLag], + kCorrelationLen, kMaxLag - kMinLag, -1, auto_corr); + + // Normalize correlation to 14 bits and write to |auto_correlation_|. + int32_t max_corr = WebRtcSpl_MaxAbsValueW32(auto_corr, kCorrelationLen); + int scaling = std::max(0, 17 - WebRtcSpl_NormW32(max_corr)); + WebRtcSpl_VectorBitShiftW32ToW16(auto_correlation_, kCorrelationLen, + auto_corr, scaling); +} + +bool TimeStretch::SpeechDetection(int32_t vec1_energy, int32_t vec2_energy, + size_t peak_index, int scaling) const { + // Check if the signal seems to be active speech or not (simple VAD). + // If (vec1_energy + vec2_energy) / (2 * peak_index) <= + // 8 * background_noise_energy, then we say that the signal contains no + // active speech. + // Rewrite the inequality as: + // (vec1_energy + vec2_energy) / 16 <= peak_index * background_noise_energy. + // The two sides of the inequality will be denoted |left_side| and + // |right_side|. + int32_t left_side = rtc::saturated_cast( + (static_cast(vec1_energy) + vec2_energy) / 16); + int32_t right_side; + if (background_noise_.initialized()) { + right_side = background_noise_.Energy(master_channel_); + } else { + // If noise parameters have not been estimated, use a fixed threshold. + right_side = 75000; + } + int right_scale = 16 - WebRtcSpl_NormW32(right_side); + right_scale = std::max(0, right_scale); + left_side = left_side >> right_scale; + right_side = + rtc::dchecked_cast(peak_index) * (right_side >> right_scale); + + // Scale |left_side| properly before comparing with |right_side|. + // (|scaling| is the scale factor before energy calculation, thus the scale + // factor for the energy is 2 * scaling.) + if (WebRtcSpl_NormW32(left_side) < 2 * scaling) { + // Cannot scale only |left_side|, must scale |right_side| too. + int temp_scale = WebRtcSpl_NormW32(left_side); + left_side = left_side << temp_scale; + right_side = right_side >> (2 * scaling - temp_scale); + } else { + left_side = left_side << 2 * scaling; + } + return left_side > right_side; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.h new file mode 100644 index 0000000000..ace10cd055 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TIME_STRETCH_H_ +#define MODULES_AUDIO_CODING_NETEQ_TIME_STRETCH_H_ + +#include +#include // memset, size_t + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declarations. +class BackgroundNoise; + +// This is the base class for Accelerate and PreemptiveExpand. This class +// cannot be instantiated, but must be used through either of the derived +// classes. +class TimeStretch { + public: + enum ReturnCodes { + kSuccess = 0, + kSuccessLowEnergy = 1, + kNoStretch = 2, + kError = -1 + }; + + TimeStretch(int sample_rate_hz, size_t num_channels, + const BackgroundNoise& background_noise) + : sample_rate_hz_(sample_rate_hz), + fs_mult_(sample_rate_hz / 8000), + num_channels_(num_channels), + master_channel_(0), // First channel is master. + background_noise_(background_noise), + max_input_value_(0) { + assert(sample_rate_hz_ == 8000 || + sample_rate_hz_ == 16000 || + sample_rate_hz_ == 32000 || + sample_rate_hz_ == 48000); + assert(num_channels_ > 0); + assert(master_channel_ < num_channels_); + memset(auto_correlation_, 0, sizeof(auto_correlation_)); + } + + virtual ~TimeStretch() {} + + // This method performs the processing common to both Accelerate and + // PreemptiveExpand. + ReturnCodes Process(const int16_t* input, + size_t input_len, + bool fast_mode, + AudioMultiVector* output, + size_t* length_change_samples); + + protected: + // Sets the parameters |best_correlation| and |peak_index| to suitable + // values when the signal contains no active speech. This method must be + // implemented by the sub-classes. + virtual void SetParametersForPassiveSpeech(size_t input_length, + int16_t* best_correlation, + size_t* peak_index) const = 0; + + // Checks the criteria for performing the time-stretching operation and, + // if possible, performs the time-stretching. This method must be implemented + // by the sub-classes. + virtual ReturnCodes CheckCriteriaAndStretch( + const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool fast_mode, + AudioMultiVector* output) const = 0; + + static const size_t kCorrelationLen = 50; + static const size_t kLogCorrelationLen = 6; // >= log2(kCorrelationLen). + static const size_t kMinLag = 10; + static const size_t kMaxLag = 60; + static const size_t kDownsampledLen = kCorrelationLen + kMaxLag; + static const int kCorrelationThreshold = 14746; // 0.9 in Q14. + + const int sample_rate_hz_; + const int fs_mult_; // Sample rate multiplier = sample_rate_hz_ / 8000. + const size_t num_channels_; + const size_t master_channel_; + const BackgroundNoise& background_noise_; + int16_t max_input_value_; + int16_t downsampled_input_[kDownsampledLen]; + // Adding 1 to the size of |auto_correlation_| because of how it is used + // by the peak-detection algorithm. + int16_t auto_correlation_[kCorrelationLen + 1]; + + private: + // Calculates the auto-correlation of |downsampled_input_| and writes the + // result to |auto_correlation_|. + void AutoCorrelation(); + + // Performs a simple voice-activity detection based on the input parameters. + bool SpeechDetection(int32_t vec1_energy, int32_t vec2_energy, + size_t peak_index, int scaling) const; + + RTC_DISALLOW_COPY_AND_ASSIGN(TimeStretch); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TIME_STRETCH_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch_unittest.cc new file mode 100644 index 0000000000..8d0f4d4a64 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch_unittest.cc @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Accelerate and PreemptiveExpand classes. + +#include "modules/audio_coding/neteq/accelerate.h" +#include "modules/audio_coding/neteq/preemptive_expand.h" + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +namespace { +const size_t kNumChannels = 1; +} + +TEST(TimeStretch, CreateAndDestroy) { + const int kSampleRate = 8000; + const int kOverlapSamples = 5 * kSampleRate / 8000; + BackgroundNoise bgn(kNumChannels); + Accelerate accelerate(kSampleRate, kNumChannels, bgn); + PreemptiveExpand preemptive_expand( + kSampleRate, kNumChannels, bgn, kOverlapSamples); +} + +TEST(TimeStretch, CreateUsingFactory) { + const int kSampleRate = 8000; + const int kOverlapSamples = 5 * kSampleRate / 8000; + BackgroundNoise bgn(kNumChannels); + + AccelerateFactory accelerate_factory; + Accelerate* accelerate = + accelerate_factory.Create(kSampleRate, kNumChannels, bgn); + EXPECT_TRUE(accelerate != NULL); + delete accelerate; + + PreemptiveExpandFactory preemptive_expand_factory; + PreemptiveExpand* preemptive_expand = preemptive_expand_factory.Create( + kSampleRate, kNumChannels, bgn, kOverlapSamples); + EXPECT_TRUE(preemptive_expand != NULL); + delete preemptive_expand; +} + +class TimeStretchTest : public ::testing::Test { + protected: + TimeStretchTest() + : input_file_(new test::InputAudioFile( + test::ResourcePath("audio_coding/testfile32kHz", "pcm"))), + sample_rate_hz_(32000), + block_size_(30 * sample_rate_hz_ / 1000), // 30 ms + audio_(new int16_t[block_size_]), + background_noise_(kNumChannels) { + WebRtcSpl_Init(); + } + + const int16_t* Next30Ms() { + RTC_CHECK(input_file_->Read(block_size_, audio_.get())); + return audio_.get(); + } + + // Returns the total length change (in samples) that the accelerate operation + // resulted in during the run. + size_t TestAccelerate(size_t loops, bool fast_mode) { + Accelerate accelerate(sample_rate_hz_, kNumChannels, background_noise_); + size_t total_length_change = 0; + for (size_t i = 0; i < loops; ++i) { + AudioMultiVector output(kNumChannels); + size_t length_change; + UpdateReturnStats(accelerate.Process(Next30Ms(), block_size_, fast_mode, + &output, &length_change)); + total_length_change += length_change; + } + return total_length_change; + } + + void UpdateReturnStats(TimeStretch::ReturnCodes ret) { + switch (ret) { + case TimeStretch::kSuccess: + case TimeStretch::kSuccessLowEnergy: + case TimeStretch::kNoStretch: + ++return_stats_[ret]; + break; + case TimeStretch::kError: + FAIL() << "Process returned an error"; + } + } + + std::unique_ptr input_file_; + const int sample_rate_hz_; + const size_t block_size_; + std::unique_ptr audio_; + std::map return_stats_; + BackgroundNoise background_noise_; +}; + +TEST_F(TimeStretchTest, Accelerate) { + // TestAccelerate returns the total length change in samples. + EXPECT_EQ(15268U, TestAccelerate(100, false)); + EXPECT_EQ(9, return_stats_[TimeStretch::kSuccess]); + EXPECT_EQ(58, return_stats_[TimeStretch::kSuccessLowEnergy]); + EXPECT_EQ(33, return_stats_[TimeStretch::kNoStretch]); +} + +TEST_F(TimeStretchTest, AccelerateFastMode) { + // TestAccelerate returns the total length change in samples. + EXPECT_EQ(21400U, TestAccelerate(100, true)); + EXPECT_EQ(31, return_stats_[TimeStretch::kSuccess]); + EXPECT_EQ(58, return_stats_[TimeStretch::kSuccessLowEnergy]); + EXPECT_EQ(11, return_stats_[TimeStretch::kNoStretch]); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.cc new file mode 100644 index 0000000000..d7aa9fe868 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/timestamp_scaler.h" + +#include "modules/audio_coding/neteq/decoder_database.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +void TimestampScaler::Reset() { + first_packet_received_ = false; +} + +void TimestampScaler::ToInternal(Packet* packet) { + if (!packet) { + return; + } + packet->timestamp = ToInternal(packet->timestamp, packet->payload_type); +} + +void TimestampScaler::ToInternal(PacketList* packet_list) { + PacketList::iterator it; + for (it = packet_list->begin(); it != packet_list->end(); ++it) { + ToInternal(&(*it)); + } +} + +uint32_t TimestampScaler::ToInternal(uint32_t external_timestamp, + uint8_t rtp_payload_type) { + const DecoderDatabase::DecoderInfo* info = + decoder_database_.GetDecoderInfo(rtp_payload_type); + if (!info) { + // Payload type is unknown. Do not scale. + return external_timestamp; + } + if (!(info->IsComfortNoise() || info->IsDtmf())) { + // Do not change the timestamp scaling settings for DTMF or CNG. + numerator_ = info->SampleRateHz(); + if (info->GetFormat().clockrate_hz == 0) { + // If the clockrate is invalid (i.e. with an old-style external codec) + // we cannot do any timestamp scaling. + denominator_ = numerator_; + } else { + denominator_ = info->GetFormat().clockrate_hz; + } + } + if (numerator_ != denominator_) { + // We have a scale factor != 1. + if (!first_packet_received_) { + external_ref_ = external_timestamp; + internal_ref_ = external_timestamp; + first_packet_received_ = true; + } + const int64_t external_diff = int64_t{external_timestamp} - external_ref_; + RTC_DCHECK_GT(denominator_, 0); + external_ref_ = external_timestamp; + internal_ref_ += (external_diff * numerator_) / denominator_; + return internal_ref_; + } else { + // No scaling. + return external_timestamp; + } +} + + +uint32_t TimestampScaler::ToExternal(uint32_t internal_timestamp) const { + if (!first_packet_received_ || (numerator_ == denominator_)) { + // Not initialized, or scale factor is 1. + return internal_timestamp; + } else { + const int64_t internal_diff = int64_t{internal_timestamp} - internal_ref_; + RTC_DCHECK_GT(numerator_, 0); + // Do not update references in this method. + // Switch |denominator_| and |numerator_| to convert the other way. + return external_ref_ + (internal_diff * denominator_) / numerator_; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.h new file mode 100644 index 0000000000..f0b05d634c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TIMESTAMP_SCALER_H_ +#define MODULES_AUDIO_CODING_NETEQ_TIMESTAMP_SCALER_H_ + +#include "modules/audio_coding/neteq/packet.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Forward declaration. +class DecoderDatabase; + +// This class scales timestamps for codecs that need timestamp scaling. +// This is done for codecs where one RTP timestamp does not correspond to +// one sample. +class TimestampScaler { + public: + explicit TimestampScaler(const DecoderDatabase& decoder_database) + : first_packet_received_(false), + numerator_(1), + denominator_(1), + external_ref_(0), + internal_ref_(0), + decoder_database_(decoder_database) {} + + virtual ~TimestampScaler() {} + + // Start over. + virtual void Reset(); + + // Scale the timestamp in |packet| from external to internal. + virtual void ToInternal(Packet* packet); + + // Scale the timestamp for all packets in |packet_list| from external to + // internal. + virtual void ToInternal(PacketList* packet_list); + + // Returns the internal equivalent of |external_timestamp|, given the + // RTP payload type |rtp_payload_type|. + virtual uint32_t ToInternal(uint32_t external_timestamp, + uint8_t rtp_payload_type); + + // Scales back to external timestamp. This is the inverse of ToInternal(). + virtual uint32_t ToExternal(uint32_t internal_timestamp) const; + + private: + bool first_packet_received_; + int numerator_; + int denominator_; + uint32_t external_ref_; + uint32_t internal_ref_; + const DecoderDatabase& decoder_database_; + + RTC_DISALLOW_COPY_AND_ASSIGN(TimestampScaler); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TIMESTAMP_SCALER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc new file mode 100644 index 0000000000..b3c1bb0ee0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/packet.h" +#include "modules/audio_coding/neteq/timestamp_scaler.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::Return; +using ::testing::ReturnNull; +using ::testing::_; + +namespace webrtc { + +TEST(TimestampScaler, TestNoScaling) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use PCMu, because it doesn't use scaled timestamps. + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory); + static const uint8_t kRtpPayloadType = 0; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + for (uint32_t timestamp = 0xFFFFFFFF - 5; timestamp != 5; ++timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(timestamp, scaler.ToInternal(timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(timestamp, scaler.ToExternal(timestamp)); + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestNoScalingLargeStep) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use PCMu, because it doesn't use scaled timestamps. + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory); + static const uint8_t kRtpPayloadType = 0; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + static const uint32_t kStep = 160; + uint32_t start_timestamp = 0; + // |external_timestamp| will be a large positive value. + start_timestamp = start_timestamp - 5 * kStep; + for (uint32_t timestamp = start_timestamp; timestamp != 5 * kStep; + timestamp += kStep) { + // Scale to internal timestamp. + EXPECT_EQ(timestamp, scaler.ToInternal(timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(timestamp, scaler.ToExternal(timestamp)); + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5; ++external_timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722LargeStep) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + static const uint32_t kStep = 320; + uint32_t external_timestamp = 0; + // |external_timestamp| will be a large positive value. + external_timestamp = external_timestamp - 5 * kStep; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5 * kStep; external_timestamp += kStep) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + // Internal timestamp should be incremented with twice the step. + internal_timestamp += 2 * kStep; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722WithCng) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info_g722(NetEqDecoder::kDecoderG722, + factory); + const DecoderDatabase::DecoderInfo info_cng(NetEqDecoder::kDecoderCNGwb, + factory); + static const uint8_t kRtpPayloadTypeG722 = 17; + static const uint8_t kRtpPayloadTypeCng = 13; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadTypeG722)) + .WillRepeatedly(Return(&info_g722)); + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadTypeCng)) + .WillRepeatedly(Return(&info_cng)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + bool next_is_cng = false; + for (; external_timestamp != 5; ++external_timestamp) { + // Alternate between G.722 and CNG every other packet. + if (next_is_cng) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadTypeCng)); + next_is_cng = false; + } else { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadTypeG722)); + next_is_cng = true; + } + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +// Make sure that the method ToInternal(Packet* packet) is wired up correctly. +// Since it is simply calling the other ToInternal method, we are not doing +// as many tests here. +TEST(TimestampScaler, TestG722Packet) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + Packet packet; + packet.payload_type = kRtpPayloadType; + for (; external_timestamp != 5; ++external_timestamp) { + packet.timestamp = external_timestamp; + // Scale to internal timestamp. + scaler.ToInternal(&packet); + EXPECT_EQ(internal_timestamp, packet.timestamp); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +// Make sure that the method ToInternal(PacketList* packet_list) is wired up +// correctly. Since it is simply calling the ToInternal(Packet* packet) method, +// we are not doing as many tests here. +TEST(TimestampScaler, TestG722PacketList) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + PacketList packet_list; + { + Packet packet1; + packet1.payload_type = kRtpPayloadType; + packet1.timestamp = external_timestamp; + Packet packet2; + packet2.payload_type = kRtpPayloadType; + packet2.timestamp = external_timestamp + 10; + packet_list.push_back(std::move(packet1)); + packet_list.push_back(std::move(packet2)); + } + + scaler.ToInternal(&packet_list); + EXPECT_EQ(internal_timestamp, packet_list.front().timestamp); + packet_list.pop_front(); + EXPECT_EQ(internal_timestamp + 20, packet_list.front().timestamp); + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722Reset) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5; ++external_timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + // Reset the scaler. After this, we expect the internal and external to start + // over at the same value again. + scaler.Reset(); + internal_timestamp = external_timestamp; + for (; external_timestamp != 15; ++external_timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +// TODO(minyue): This test becomes trivial since Opus does not need a timestamp +// scaler. Therefore, this test may be removed in future. There is no harm to +// keep it, since it can be taken as a test case for the situation of a trivial +// timestamp scaler. +TEST(TimestampScaler, TestOpusLargeStep) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderOpus, factory); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + static const uint32_t kStep = 960; + uint32_t external_timestamp = 0; + // |external_timestamp| will be a large positive value. + external_timestamp = external_timestamp - 5 * kStep; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5 * kStep; external_timestamp += kStep) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += kStep; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, Failures) { + static const uint8_t kRtpPayloadType = 17; + MockDecoderDatabase db; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillOnce(ReturnNull()); // Return NULL to indicate unknown payload type. + + TimestampScaler scaler(db); + uint32_t timestamp = 4711; // Some number. + EXPECT_EQ(timestamp, scaler.ToInternal(timestamp, kRtpPayloadType)); + + Packet* packet = NULL; + scaler.ToInternal(packet); // Should not crash. That's all we can test. + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/DEPS b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/DEPS new file mode 100644 index 0000000000..4db1e1d6e5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+logging/rtc_event_log", +] diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_checksum.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_checksum.h new file mode 100644 index 0000000000..0445141096 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_checksum.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_CHECKSUM_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_CHECKSUM_H_ + +#include + +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/md5digest.h" +#include "rtc_base/stringencode.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +class AudioChecksum : public AudioSink { + public: + AudioChecksum() : finished_(false) {} + + bool WriteArray(const int16_t* audio, size_t num_samples) override { + if (finished_) + return false; + +#ifndef WEBRTC_ARCH_LITTLE_ENDIAN +#error "Big-endian gives a different checksum" +#endif + checksum_.Update(audio, num_samples * sizeof(*audio)); + return true; + } + + // Finalizes the computations, and returns the checksum. + std::string Finish() { + if (!finished_) { + finished_ = true; + checksum_.Finish(checksum_result_, rtc::Md5Digest::kSize); + } + return rtc::hex_encode(checksum_result_, rtc::Md5Digest::kSize); + } + + private: + rtc::Md5Digest checksum_; + char checksum_result_[rtc::Md5Digest::kSize]; + bool finished_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioChecksum); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_CHECKSUM_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.cc new file mode 100644 index 0000000000..b5ad8817fe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/audio_loop.h" + +#include +#include +#include + +namespace webrtc { +namespace test { + +bool AudioLoop::Init(const std::string file_name, + size_t max_loop_length_samples, + size_t block_length_samples) { + FILE* fp = fopen(file_name.c_str(), "rb"); + if (!fp) return false; + + audio_array_.reset(new int16_t[max_loop_length_samples + + block_length_samples]); + size_t samples_read = fread(audio_array_.get(), sizeof(int16_t), + max_loop_length_samples, fp); + fclose(fp); + + // Block length must be shorter than the loop length. + if (block_length_samples > samples_read) return false; + + // Add an extra block length of samples to the end of the array, starting + // over again from the beginning of the array. This is done to simplify + // the reading process when reading over the end of the loop. + memcpy(&audio_array_[samples_read], audio_array_.get(), + block_length_samples * sizeof(int16_t)); + + loop_length_samples_ = samples_read; + block_length_samples_ = block_length_samples; + next_index_ = 0; + return true; +} + +rtc::ArrayView AudioLoop::GetNextBlock() { + // Check that the AudioLoop is initialized. + if (block_length_samples_ == 0) + return rtc::ArrayView(); + + const int16_t* output_ptr = &audio_array_[next_index_]; + next_index_ = (next_index_ + block_length_samples_) % loop_length_samples_; + return rtc::ArrayView(output_ptr, block_length_samples_); +} + + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.h new file mode 100644 index 0000000000..abb1a36378 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_loop.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_LOOP_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_LOOP_H_ + +#include +#include + +#include "api/array_view.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +// Class serving as an infinite source of audio, realized by looping an audio +// clip. +class AudioLoop { + public: + AudioLoop() + : next_index_(0), + loop_length_samples_(0), + block_length_samples_(0) { + } + + virtual ~AudioLoop() {} + + // Initializes the AudioLoop by reading from |file_name|. The loop will be no + // longer than |max_loop_length_samples|, if the length of the file is + // greater. Otherwise, the loop length is the same as the file length. + // The audio will be delivered in blocks of |block_length_samples|. + // Returns false if the initialization failed, otherwise true. + bool Init(const std::string file_name, size_t max_loop_length_samples, + size_t block_length_samples); + + // Returns a (pointer,size) pair for the next block of audio. The size is + // equal to the |block_length_samples| Init() argument. + rtc::ArrayView GetNextBlock(); + + private: + size_t next_index_; + size_t loop_length_samples_; + size_t block_length_samples_; + std::unique_ptr audio_array_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioLoop); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_LOOP_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.cc new file mode 100644 index 0000000000..7d7af7ef9f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.cc @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/audio_sink.h" + +namespace webrtc { +namespace test { + +bool AudioSinkFork::WriteArray(const int16_t* audio, size_t num_samples) { + return left_sink_->WriteArray(audio, num_samples) && + right_sink_->WriteArray(audio, num_samples); +} + +bool VoidAudioSink::WriteArray(const int16_t* audio, size_t num_samples) { + return true; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.h new file mode 100644 index 0000000000..ecec51b895 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/audio_sink.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_ + +#include "modules/include/module_common_types.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +// Interface class for an object receiving raw output audio from test +// applications. +class AudioSink { + public: + AudioSink() {} + virtual ~AudioSink() {} + + // Writes |num_samples| from |audio| to the AudioSink. Returns true if + // successful, otherwise false. + virtual bool WriteArray(const int16_t* audio, size_t num_samples) = 0; + + // Writes |audio_frame| to the AudioSink. Returns true if successful, + // otherwise false. + bool WriteAudioFrame(const AudioFrame& audio_frame) { + return WriteArray( + audio_frame.data(), + audio_frame.samples_per_channel_ * audio_frame.num_channels_); + } + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(AudioSink); +}; + +// Forks the output audio to two AudioSink objects. +class AudioSinkFork : public AudioSink { + public: + AudioSinkFork(AudioSink* left, AudioSink* right) + : left_sink_(left), right_sink_(right) {} + + bool WriteArray(const int16_t* audio, size_t num_samples) override; + + private: + AudioSink* left_sink_; + AudioSink* right_sink_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioSinkFork); +}; + +// An AudioSink implementation that does nothing. +class VoidAudioSink : public AudioSink { + public: + VoidAudioSink() = default; + bool WriteArray(const int16_t* audio, size_t num_samples) override; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(VoidAudioSink); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc new file mode 100644 index 0000000000..6b325b6c5c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/constant_pcm_packet_source.h" + +#include + +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +ConstantPcmPacketSource::ConstantPcmPacketSource(size_t payload_len_samples, + int16_t sample_value, + int sample_rate_hz, + int payload_type) + : payload_len_samples_(payload_len_samples), + packet_len_bytes_(2 * payload_len_samples_ + kHeaderLenBytes), + samples_per_ms_(sample_rate_hz / 1000), + next_arrival_time_ms_(0.0), + payload_type_(payload_type), + seq_number_(0), + timestamp_(0), + payload_ssrc_(0xABCD1234) { + size_t encoded_len = WebRtcPcm16b_Encode(&sample_value, 1, encoded_sample_); + RTC_CHECK_EQ(2U, encoded_len); +} + +std::unique_ptr ConstantPcmPacketSource::NextPacket() { + RTC_CHECK_GT(packet_len_bytes_, kHeaderLenBytes); + uint8_t* packet_memory = new uint8_t[packet_len_bytes_]; + // Fill the payload part of the packet memory with the pre-encoded value. + for (unsigned i = 0; i < 2 * payload_len_samples_; ++i) + packet_memory[kHeaderLenBytes + i] = encoded_sample_[i % 2]; + WriteHeader(packet_memory); + // |packet| assumes ownership of |packet_memory|. + std::unique_ptr packet( + new Packet(packet_memory, packet_len_bytes_, next_arrival_time_ms_)); + next_arrival_time_ms_ += payload_len_samples_ / samples_per_ms_; + return packet; +} + +void ConstantPcmPacketSource::WriteHeader(uint8_t* packet_memory) { + packet_memory[0] = 0x80; + packet_memory[1] = static_cast(payload_type_); + packet_memory[2] = seq_number_ >> 8; + packet_memory[3] = seq_number_ & 0xFF; + packet_memory[4] = timestamp_ >> 24; + packet_memory[5] = (timestamp_ >> 16) & 0xFF; + packet_memory[6] = (timestamp_ >> 8) & 0xFF; + packet_memory[7] = timestamp_ & 0xFF; + packet_memory[8] = payload_ssrc_ >> 24; + packet_memory[9] = (payload_ssrc_ >> 16) & 0xFF; + packet_memory[10] = (payload_ssrc_ >> 8) & 0xFF; + packet_memory[11] = payload_ssrc_ & 0xFF; + ++seq_number_; + timestamp_ += static_cast(payload_len_samples_); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h new file mode 100644 index 0000000000..4e216e42af --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_CONSTANT_PCM_PACKET_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_CONSTANT_PCM_PACKET_SOURCE_H_ + +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace test { + +// This class implements a packet source that delivers PCM16b encoded packets +// with a constant sample value. The payload length, constant sample value, +// sample rate, and payload type are all set in the constructor. +class ConstantPcmPacketSource : public PacketSource { + public: + ConstantPcmPacketSource(size_t payload_len_samples, + int16_t sample_value, + int sample_rate_hz, + int payload_type); + + std::unique_ptr NextPacket() override; + + private: + void WriteHeader(uint8_t* packet_memory); + + const size_t kHeaderLenBytes = 12; + const size_t payload_len_samples_; + const size_t packet_len_bytes_; + uint8_t encoded_sample_[2]; + const int samples_per_ms_; + double next_arrival_time_ms_; + const int payload_type_; + uint16_t seq_number_; + uint32_t timestamp_; + const uint32_t payload_ssrc_; + + RTC_DISALLOW_COPY_AND_ASSIGN(ConstantPcmPacketSource); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_CONSTANT_PCM_PACKET_SOURCE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc new file mode 100644 index 0000000000..a8d1bdfe0a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/encode_neteq_input.h" + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { +namespace test { + +EncodeNetEqInput::EncodeNetEqInput(std::unique_ptr generator, + std::unique_ptr encoder, + int64_t input_duration_ms) + : generator_(std::move(generator)), + encoder_(std::move(encoder)), + input_duration_ms_(input_duration_ms) { + CreatePacket(); +} + +rtc::Optional EncodeNetEqInput::NextPacketTime() const { + RTC_DCHECK(packet_data_); + return static_cast(packet_data_->time_ms); +} + +rtc::Optional EncodeNetEqInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +std::unique_ptr EncodeNetEqInput::PopPacket() { + RTC_DCHECK(packet_data_); + // Grab the packet to return... + std::unique_ptr packet_to_return = std::move(packet_data_); + // ... and line up the next packet for future use. + CreatePacket(); + + return packet_to_return; +} + +void EncodeNetEqInput::AdvanceOutputEvent() { + next_output_event_ms_ += kOutputPeriodMs; +} + +rtc::Optional EncodeNetEqInput::NextHeader() const { + RTC_DCHECK(packet_data_); + return packet_data_->header; +} + +void EncodeNetEqInput::CreatePacket() { + // Create a new PacketData object. + RTC_DCHECK(!packet_data_); + packet_data_.reset(new NetEqInput::PacketData); + RTC_DCHECK_EQ(packet_data_->payload.size(), 0); + + // Loop until we get a packet. + AudioEncoder::EncodedInfo info; + RTC_DCHECK(!info.send_even_if_empty); + int num_blocks = 0; + while (packet_data_->payload.size() == 0 && !info.send_even_if_empty) { + const size_t num_samples = rtc::CheckedDivExact( + static_cast(encoder_->SampleRateHz() * kOutputPeriodMs), 1000); + + info = encoder_->Encode(rtp_timestamp_, generator_->Generate(num_samples), + &packet_data_->payload); + + rtp_timestamp_ += rtc::dchecked_cast( + num_samples * encoder_->RtpTimestampRateHz() / + encoder_->SampleRateHz()); + ++num_blocks; + } + packet_data_->header.timestamp = info.encoded_timestamp; + packet_data_->header.payloadType = info.payload_type; + packet_data_->header.sequenceNumber = sequence_number_++; + packet_data_->time_ms = next_packet_time_ms_; + next_packet_time_ms_ += num_blocks * kOutputPeriodMs; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h new file mode 100644 index 0000000000..b44d4ac08c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_ENCODE_NETEQ_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_ENCODE_NETEQ_INPUT_H_ + +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" +#include "modules/include/module_common_types.h" + +namespace webrtc { +namespace test { + +// This class provides a NetEqInput that takes audio from a generator object and +// encodes it using a given audio encoder. +class EncodeNetEqInput : public NetEqInput { + public: + // Generator class, to be provided to the EncodeNetEqInput constructor. + class Generator { + public: + virtual ~Generator() = default; + // Returns the next num_samples values from the signal generator. + virtual rtc::ArrayView Generate(size_t num_samples) = 0; + }; + + // The source will end after the given input duration. + EncodeNetEqInput(std::unique_ptr generator, + std::unique_ptr encoder, + int64_t input_duration_ms); + + rtc::Optional NextPacketTime() const override; + + rtc::Optional NextOutputEventTime() const override; + + std::unique_ptr PopPacket() override; + + void AdvanceOutputEvent() override; + + bool ended() const override { + return next_output_event_ms_ <= input_duration_ms_; + } + + rtc::Optional NextHeader() const override; + + private: + static constexpr int64_t kOutputPeriodMs = 10; + + void CreatePacket(); + + std::unique_ptr generator_; + std::unique_ptr encoder_; + std::unique_ptr packet_data_; + uint32_t rtp_timestamp_ = 0; + int16_t sequence_number_ = 0; + int64_t next_packet_time_ms_ = 0; + int64_t next_output_event_ms_ = 0; + const int64_t input_duration_ms_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_ENCODE_NETEQ_INPUT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc new file mode 100644 index 0000000000..f4376088b8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h" + +#include "modules/rtp_rtcp/source/byte_io.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { +namespace test { + +int FakeDecodeFromFile::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + if (encoded_len == 0) { + // Decoder is asked to produce codec-internal comfort noise. + RTC_DCHECK(!encoded); // NetEq always sends nullptr in this case. + RTC_DCHECK(cng_mode_); + RTC_DCHECK_GT(last_decoded_length_, 0); + std::fill_n(decoded, last_decoded_length_, 0); + *speech_type = kComfortNoise; + return rtc::dchecked_cast(last_decoded_length_); + } + + RTC_CHECK_GE(encoded_len, 12); + uint32_t timestamp_to_decode = + ByteReader::ReadLittleEndian(encoded); + uint32_t samples_to_decode = + ByteReader::ReadLittleEndian(&encoded[4]); + if (samples_to_decode == 0) { + // Number of samples in packet is unknown. + if (last_decoded_length_ > 0) { + // Use length of last decoded packet, but since this is the total for all + // channels, we have to divide by 2 in the stereo case. + samples_to_decode = rtc::dchecked_cast(rtc::CheckedDivExact( + last_decoded_length_, static_cast(stereo_ ? 2uL : 1uL))); + } else { + // This is the first packet to decode, and we do not know the length of + // it. Set it to 10 ms. + samples_to_decode = rtc::CheckedDivExact(sample_rate_hz, 100); + } + } + + if (next_timestamp_from_input_ && + timestamp_to_decode != *next_timestamp_from_input_) { + // A gap in the timestamp sequence is detected. Skip the same number of + // samples from the file. + uint32_t jump = timestamp_to_decode - *next_timestamp_from_input_; + RTC_CHECK(input_->Seek(jump)); + } + + next_timestamp_from_input_ = timestamp_to_decode + samples_to_decode; + + uint32_t original_payload_size_bytes = + ByteReader::ReadLittleEndian(&encoded[8]); + if (original_payload_size_bytes == 1) { + // This is a comfort noise payload. + RTC_DCHECK_GT(last_decoded_length_, 0); + std::fill_n(decoded, last_decoded_length_, 0); + *speech_type = kComfortNoise; + cng_mode_ = true; + return rtc::dchecked_cast(last_decoded_length_); + } + + cng_mode_ = false; + RTC_CHECK(input_->Read(static_cast(samples_to_decode), decoded)); + + if (stereo_) { + InputAudioFile::DuplicateInterleaved(decoded, samples_to_decode, 2, + decoded); + samples_to_decode *= 2; + } + + *speech_type = kSpeech; + last_decoded_length_ = samples_to_decode; + return rtc::dchecked_cast(last_decoded_length_); +} + +void FakeDecodeFromFile::PrepareEncoded(uint32_t timestamp, + size_t samples, + size_t original_payload_size_bytes, + rtc::ArrayView encoded) { + RTC_CHECK_GE(encoded.size(), 12); + ByteWriter::WriteLittleEndian(&encoded[0], timestamp); + ByteWriter::WriteLittleEndian(&encoded[4], + rtc::checked_cast(samples)); + ByteWriter::WriteLittleEndian( + &encoded[8], rtc::checked_cast(original_payload_size_bytes)); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h new file mode 100644 index 0000000000..7aa8e6e40f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_FAKE_DECODE_FROM_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_FAKE_DECODE_FROM_FILE_H_ + +#include + +#include "api/array_view.h" +#include "api/audio_codecs/audio_decoder.h" +#include "api/optional.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" + +namespace webrtc { +namespace test { + +// Provides an AudioDecoder implementation that delivers audio data from a file. +// The "encoded" input should contain information about what RTP timestamp the +// encoding represents, and how many samples the decoder should produce for that +// encoding. A helper method PrepareEncoded is provided to prepare such +// encodings. If packets are missing, as determined from the timestamps, the +// file reading will skip forward to match the loss. +class FakeDecodeFromFile : public AudioDecoder { + public: + FakeDecodeFromFile(std::unique_ptr input, + int sample_rate_hz, + bool stereo) + : input_(std::move(input)), + sample_rate_hz_(sample_rate_hz), + stereo_(stereo) {} + + ~FakeDecodeFromFile() = default; + + void Reset() override {} + + int SampleRateHz() const override { return sample_rate_hz_; } + + size_t Channels() const override { return stereo_ ? 2 : 1; } + + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + // Helper method. Writes |timestamp|, |samples| and + // |original_payload_size_bytes| to |encoded| in a format that the + // FakeDecodeFromFile decoder will understand. |encoded| must be at least 12 + // bytes long. + static void PrepareEncoded(uint32_t timestamp, + size_t samples, + size_t original_payload_size_bytes, + rtc::ArrayView encoded); + + private: + std::unique_ptr input_; + rtc::Optional next_timestamp_from_input_; + const int sample_rate_hz_; + const bool stereo_; + size_t last_decoded_length_ = 0; + bool cng_mode_ = false; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_FAKE_DECODE_FROM_FILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc new file mode 100644 index 0000000000..31ebf98342 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/input_audio_file.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +InputAudioFile::InputAudioFile(const std::string file_name, bool loop_at_end) + : loop_at_end_(loop_at_end) { + fp_ = fopen(file_name.c_str(), "rb"); +} + +InputAudioFile::~InputAudioFile() { fclose(fp_); } + +bool InputAudioFile::Read(size_t samples, int16_t* destination) { + if (!fp_) { + return false; + } + size_t samples_read = fread(destination, sizeof(int16_t), samples, fp_); + if (samples_read < samples) { + if (!loop_at_end_) { + return false; + } + // Rewind and read the missing samples. + rewind(fp_); + size_t missing_samples = samples - samples_read; + if (fread(destination + samples_read, sizeof(int16_t), missing_samples, + fp_) < missing_samples) { + // Could not read enough even after rewinding the file. + return false; + } + } + return true; +} + +bool InputAudioFile::Seek(int samples) { + if (!fp_) { + return false; + } + // Find file boundaries. + const long current_pos = ftell(fp_); + RTC_CHECK_NE(EOF, current_pos) + << "Error returned when getting file position."; + RTC_CHECK_EQ(0, fseek(fp_, 0, SEEK_END)); // Move to end of file. + const long file_size = ftell(fp_); + RTC_CHECK_NE(EOF, file_size) << "Error returned when getting file position."; + // Find new position. + long new_pos = current_pos + sizeof(int16_t) * samples; // Samples to bytes. + RTC_CHECK_GE(new_pos, 0) + << "Trying to move to before the beginning of the file"; + if (loop_at_end_) { + new_pos = new_pos % file_size; // Wrap around the end of the file. + } else { + new_pos = new_pos > file_size ? file_size : new_pos; // Don't loop. + } + // Move to new position relative to the beginning of the file. + RTC_CHECK_EQ(0, fseek(fp_, new_pos, SEEK_SET)); + return true; +} + +void InputAudioFile::DuplicateInterleaved(const int16_t* source, size_t samples, + size_t channels, + int16_t* destination) { + // Start from the end of |source| and |destination|, and work towards the + // beginning. This is to allow in-place interleaving of the same array (i.e., + // |source| and |destination| are the same array). + for (int i = static_cast(samples - 1); i >= 0; --i) { + for (int j = static_cast(channels - 1); j >= 0; --j) { + destination[i * channels + j] = source[i]; + } + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h new file mode 100644 index 0000000000..6bfa369b38 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_INPUT_AUDIO_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_INPUT_AUDIO_FILE_H_ + +#include + +#include + +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +// Class for handling a looping input audio file. +class InputAudioFile { + public: + explicit InputAudioFile(const std::string file_name, bool loop_at_end = true); + + virtual ~InputAudioFile(); + + // Reads |samples| elements from source file to |destination|. Returns true + // if the read was successful, otherwise false. If the file end is reached, + // the file is rewound and reading continues from the beginning. + // The output |destination| must have the capacity to hold |samples| elements. + virtual bool Read(size_t samples, int16_t* destination); + + // Fast-forwards (|samples| > 0) or -backwards (|samples| < 0) the file by the + // indicated number of samples. Just like Read(), Seek() starts over at the + // beginning of the file if the end is reached. However, seeking backwards + // past the beginning of the file is not possible. + virtual bool Seek(int samples); + + // Creates a multi-channel signal from a mono signal. Each sample is repeated + // |channels| times to create an interleaved multi-channel signal where all + // channels are identical. The output |destination| must have the capacity to + // hold samples * channels elements. Note that |source| and |destination| can + // be the same array (i.e., point to the same address). + static void DuplicateInterleaved(const int16_t* source, size_t samples, + size_t channels, int16_t* destination); + + private: + FILE* fp_; + const bool loop_at_end_; + RTC_DISALLOW_COPY_AND_ASSIGN(InputAudioFile); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_INPUT_AUDIO_FILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc new file mode 100644 index 0000000000..bf016a1246 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for test InputAudioFile class. + +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "rtc_base/numerics/safe_conversions.h" + +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +TEST(TestInputAudioFile, DuplicateInterleaveSeparateSrcDst) { + static const size_t kSamples = 10; + static const size_t kChannels = 2; + int16_t input[kSamples]; + for (size_t i = 0; i < kSamples; ++i) { + input[i] = rtc::checked_cast(i); + } + int16_t output[kSamples * kChannels]; + InputAudioFile::DuplicateInterleaved(input, kSamples, kChannels, output); + + // Verify output + int16_t* output_ptr = output; + for (size_t i = 0; i < kSamples; ++i) { + for (size_t j = 0; j < kChannels; ++j) { + EXPECT_EQ(static_cast(i), *output_ptr++); + } + } +} + +TEST(TestInputAudioFile, DuplicateInterleaveSameSrcDst) { + static const size_t kSamples = 10; + static const size_t kChannels = 5; + int16_t input[kSamples * kChannels]; + for (size_t i = 0; i < kSamples; ++i) { + input[i] = rtc::checked_cast(i); + } + InputAudioFile::DuplicateInterleaved(input, kSamples, kChannels, input); + + // Verify output + int16_t* output_ptr = input; + for (size_t i = 0; i < kSamples; ++i) { + for (size_t j = 0; j < kChannels; ++j) { + EXPECT_EQ(static_cast(i), *output_ptr++); + } + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc new file mode 100644 index 0000000000..34e7a84200 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" + +#include +#include +#include +#include +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { +namespace { +// Helper function for NetEqDelayAnalyzer::CreateGraphs. Returns the +// interpolated value of a function at the point x. Vector x_vec contains the +// sample points, and y_vec contains the function values at these points. The +// return value is a linear interpolation between y_vec values. +double LinearInterpolate(double x, + const std::vector& x_vec, + const std::vector& y_vec) { + // Find first element which is larger than x. + auto it = std::upper_bound(x_vec.begin(), x_vec.end(), x); + if (it == x_vec.end()) { + --it; + } + const size_t upper_ix = it - x_vec.begin(); + + size_t lower_ix; + if (upper_ix == 0 || x_vec[upper_ix] <= x) { + lower_ix = upper_ix; + } else { + lower_ix = upper_ix - 1; + } + double y; + if (lower_ix == upper_ix) { + y = y_vec[lower_ix]; + } else { + RTC_DCHECK_NE(x_vec[lower_ix], x_vec[upper_ix]); + y = (x - x_vec[lower_ix]) * (y_vec[upper_ix] - y_vec[lower_ix]) / + (x_vec[upper_ix] - x_vec[lower_ix]) + + y_vec[lower_ix]; + } + return y; +} +} // namespace + +void NetEqDelayAnalyzer::AfterInsertPacket( + const test::NetEqInput::PacketData& packet, + NetEq* neteq) { + data_.insert( + std::make_pair(packet.header.timestamp, TimingData(packet.time_ms))); + ssrcs_.insert(packet.header.ssrc); + payload_types_.insert(packet.header.payloadType); +} + +void NetEqDelayAnalyzer::BeforeGetAudio(NetEq* neteq) { + last_sync_buffer_ms_ = neteq->SyncBufferSizeMs(); +} + +void NetEqDelayAnalyzer::AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool /*muted*/, + NetEq* neteq) { + get_audio_time_ms_.push_back(time_now_ms); + // Check what timestamps were decoded in the last GetAudio call. + std::vector dec_ts = neteq->LastDecodedTimestamps(); + // Find those timestamps in data_, insert their decoding time and sync + // delay. + for (uint32_t ts : dec_ts) { + auto it = data_.find(ts); + if (it == data_.end()) { + // This is a packet that was split out from another packet. Skip it. + continue; + } + auto& it_timing = it->second; + RTC_CHECK(!it_timing.decode_get_audio_count) + << "Decode time already written"; + it_timing.decode_get_audio_count = get_audio_count_; + RTC_CHECK(!it_timing.sync_delay_ms) << "Decode time already written"; + it_timing.sync_delay_ms = last_sync_buffer_ms_; + it_timing.target_delay_ms = neteq->TargetDelayMs(); + it_timing.current_delay_ms = neteq->FilteredCurrentDelayMs(); + } + last_sample_rate_hz_ = audio_frame.sample_rate_hz_; + ++get_audio_count_; +} + +void NetEqDelayAnalyzer::CreateGraphs( + std::vector* send_time_s, + std::vector* arrival_delay_ms, + std::vector* corrected_arrival_delay_ms, + std::vector>* playout_delay_ms, + std::vector>* target_delay_ms) const { + if (get_audio_time_ms_.empty()) { + return; + } + // Create nominal_get_audio_time_ms, a vector starting at + // get_audio_time_ms_[0] and increasing by 10 for each element. + std::vector nominal_get_audio_time_ms(get_audio_time_ms_.size()); + nominal_get_audio_time_ms[0] = get_audio_time_ms_[0]; + std::transform( + nominal_get_audio_time_ms.begin(), nominal_get_audio_time_ms.end() - 1, + nominal_get_audio_time_ms.begin() + 1, [](int64_t& x) { return x + 10; }); + RTC_DCHECK( + std::is_sorted(get_audio_time_ms_.begin(), get_audio_time_ms_.end())); + + std::vector rtp_timestamps_ms; + double offset = std::numeric_limits::max(); + TimestampUnwrapper unwrapper; + // This loop traverses data_ and populates rtp_timestamps_ms as well as + // calculates the base offset. + for (auto& d : data_) { + rtp_timestamps_ms.push_back( + unwrapper.Unwrap(d.first) / + rtc::CheckedDivExact(last_sample_rate_hz_, 1000)); + offset = + std::min(offset, d.second.arrival_time_ms - rtp_timestamps_ms.back()); + } + + // Calculate send times in seconds for each packet. This is the (unwrapped) + // RTP timestamp in ms divided by 1000. + send_time_s->resize(rtp_timestamps_ms.size()); + std::transform(rtp_timestamps_ms.begin(), rtp_timestamps_ms.end(), + send_time_s->begin(), [rtp_timestamps_ms](double x) { + return (x - rtp_timestamps_ms[0]) / 1000.f; + }); + RTC_DCHECK_EQ(send_time_s->size(), rtp_timestamps_ms.size()); + + // This loop traverses the data again and populates the graph vectors. The + // reason to have two loops and traverse twice is that the offset cannot be + // known until the first traversal is done. Meanwhile, the final offset must + // be known already at the start of this second loop. + auto data_it = data_.cbegin(); + for (size_t i = 0; i < send_time_s->size(); ++i, ++data_it) { + RTC_DCHECK(data_it != data_.end()); + const double offset_send_time_ms = rtp_timestamps_ms[i] + offset; + const auto& timing = data_it->second; + corrected_arrival_delay_ms->push_back( + LinearInterpolate(timing.arrival_time_ms, get_audio_time_ms_, + nominal_get_audio_time_ms) - + offset_send_time_ms); + arrival_delay_ms->push_back(timing.arrival_time_ms - offset_send_time_ms); + + if (timing.decode_get_audio_count) { + // This packet was decoded. + RTC_DCHECK(timing.sync_delay_ms); + const float playout_ms = *timing.decode_get_audio_count * 10 + + get_audio_time_ms_[0] + *timing.sync_delay_ms - + offset_send_time_ms; + playout_delay_ms->push_back(playout_ms); + RTC_DCHECK(timing.target_delay_ms); + RTC_DCHECK(timing.current_delay_ms); + const float target = + playout_ms - *timing.current_delay_ms + *timing.target_delay_ms; + target_delay_ms->push_back(target); + } else { + // This packet was never decoded. Mark target and playout delays as empty. + playout_delay_ms->push_back(rtc::nullopt); + target_delay_ms->push_back(rtc::nullopt); + } + } + RTC_DCHECK(data_it == data_.end()); + RTC_DCHECK_EQ(send_time_s->size(), corrected_arrival_delay_ms->size()); + RTC_DCHECK_EQ(send_time_s->size(), playout_delay_ms->size()); + RTC_DCHECK_EQ(send_time_s->size(), target_delay_ms->size()); +} + +void NetEqDelayAnalyzer::CreateMatlabScript( + const std::string& script_name) const { + std::vector send_time_s; + std::vector arrival_delay_ms; + std::vector corrected_arrival_delay_ms; + std::vector> playout_delay_ms; + std::vector> target_delay_ms; + CreateGraphs(&send_time_s, &arrival_delay_ms, &corrected_arrival_delay_ms, + &playout_delay_ms, &target_delay_ms); + + // Create an output file stream to Matlab script file. + std::ofstream output(script_name); + // The iterator is used to batch-output comma-separated values from vectors. + std::ostream_iterator output_iterator(output, ","); + + output << "send_time_s = [ "; + std::copy(send_time_s.begin(), send_time_s.end(), output_iterator); + output << "];" << std::endl; + + output << "arrival_delay_ms = [ "; + std::copy(arrival_delay_ms.begin(), arrival_delay_ms.end(), output_iterator); + output << "];" << std::endl; + + output << "corrected_arrival_delay_ms = [ "; + std::copy(corrected_arrival_delay_ms.begin(), + corrected_arrival_delay_ms.end(), output_iterator); + output << "];" << std::endl; + + output << "playout_delay_ms = [ "; + for (const auto& v : playout_delay_ms) { + if (!v) { + output << "nan, "; + } else { + output << *v << ", "; + } + } + output << "];" << std::endl; + + output << "target_delay_ms = [ "; + for (const auto& v : target_delay_ms) { + if (!v) { + output << "nan, "; + } else { + output << *v << ", "; + } + } + output << "];" << std::endl; + + output << "h=plot(send_time_s, arrival_delay_ms, " + << "send_time_s, target_delay_ms, 'g.', " + << "send_time_s, playout_delay_ms);" << std::endl; + output << "set(h(1),'color',0.75*[1 1 1]);" << std::endl; + output << "set(h(2),'markersize',6);" << std::endl; + output << "set(h(3),'linew',1.5);" << std::endl; + output << "ax1=axis;" << std::endl; + output << "axis tight" << std::endl; + output << "ax2=axis;" << std::endl; + output << "axis([ax2(1:3) ax1(4)])" << std::endl; + output << "xlabel('send time [s]');" << std::endl; + output << "ylabel('relative delay [ms]');" << std::endl; + if (!ssrcs_.empty()) { + auto ssrc_it = ssrcs_.cbegin(); + output << "title('SSRC: 0x" << std::hex << static_cast(*ssrc_it++); + while (ssrc_it != ssrcs_.end()) { + output << ", 0x" << std::hex << static_cast(*ssrc_it++); + } + output << std::dec; + auto pt_it = payload_types_.cbegin(); + output << "; Payload Types: " << *pt_it++; + while (pt_it != payload_types_.end()) { + output << ", " << *pt_it++; + } + output << "');" << std::endl; + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h new file mode 100644 index 0000000000..62bea4ea8d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_DELAY_ANALYZER_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_DELAY_ANALYZER_H_ + +#include +#include +#include +#include + +#include "api/optional.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +class NetEqDelayAnalyzer : public test::NetEqPostInsertPacket, + public test::NetEqGetAudioCallback { + public: + void AfterInsertPacket(const test::NetEqInput::PacketData& packet, + NetEq* neteq) override; + + void BeforeGetAudio(NetEq* neteq) override; + + void AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool muted, + NetEq* neteq) override; + + void CreateGraphs(std::vector* send_times_s, + std::vector* arrival_delay_ms, + std::vector* corrected_arrival_delay_ms, + std::vector>* playout_delay_ms, + std::vector>* target_delay_ms) const; + + // Creates a matlab script with file name script_name. When executed in + // Matlab, the script will generate graphs with the same timing information + // as provided by CreateGraphs. + void CreateMatlabScript(const std::string& script_name) const; + + private: + struct TimingData { + explicit TimingData(double at) : arrival_time_ms(at) {} + double arrival_time_ms; + rtc::Optional decode_get_audio_count; + rtc::Optional sync_delay_ms; + rtc::Optional target_delay_ms; + rtc::Optional current_delay_ms; + }; + std::map data_; + std::vector get_audio_time_ms_; + size_t get_audio_count_ = 0; + size_t last_sync_buffer_ms_ = 0; + int last_sample_rate_hz_ = 0; + std::set ssrcs_; + std::set payload_types_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_DELAY_ANALYZER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc new file mode 100644 index 0000000000..68dde5296f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#include "modules/audio_coding/neteq/tools/neteq_external_decoder_test.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "rtc_base/format_macros.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +NetEqExternalDecoderTest::NetEqExternalDecoderTest(NetEqDecoder codec, + int sample_rate_hz, + AudioDecoder* decoder) + : codec_(codec), + decoder_(decoder), + sample_rate_hz_(sample_rate_hz), + channels_(decoder_->Channels()) { + NetEq::Config config; + config.sample_rate_hz = sample_rate_hz_; + neteq_.reset(NetEq::Create(config, CreateBuiltinAudioDecoderFactory())); +} + +void NetEqExternalDecoderTest::Init() { + ASSERT_EQ(NetEq::kOK, + neteq_->RegisterExternalDecoder(decoder_, codec_, name_, + kPayloadType)); +} + +void NetEqExternalDecoderTest::InsertPacket( + RTPHeader rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp) { + ASSERT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_header, payload, receive_timestamp)); +} + +void NetEqExternalDecoderTest::GetOutputAudio(AudioFrame* output) { + // Get audio from regular instance. + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(output, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(channels_, output->num_channels_); + EXPECT_EQ(static_cast(kOutputLengthMs * sample_rate_hz_ / 1000), + output->samples_per_channel_); + EXPECT_EQ(sample_rate_hz_, neteq_->last_output_sample_rate_hz()); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h new file mode 100644 index 0000000000..aefa62e4d1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_EXTERNAL_DECODER_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_EXTERNAL_DECODER_TEST_H_ + +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/include/module_common_types.h" + +namespace webrtc { +namespace test { +// This test class provides a way run NetEQ with an external decoder. +class NetEqExternalDecoderTest { + protected: + static const uint8_t kPayloadType = 95; + static const int kOutputLengthMs = 10; + + // The external decoder |decoder| is suppose to be of type |codec|. + NetEqExternalDecoderTest(NetEqDecoder codec, + int sample_rate_hz, + AudioDecoder* decoder); + + virtual ~NetEqExternalDecoderTest() { } + + // In Init(), we register the external decoder. + void Init(); + + // Inserts a new packet with |rtp_header| and |payload| of + // |payload_size_bytes| bytes. The |receive_timestamp| is an indication + // of the time when the packet was received, and should be measured with + // the same tick rate as the RTP timestamp of the current payload. + virtual void InsertPacket(RTPHeader rtp_header, + rtc::ArrayView payload, + uint32_t receive_timestamp); + + // Get 10 ms of audio data. + void GetOutputAudio(AudioFrame* output); + + NetEq* neteq() { return neteq_.get(); } + + private: + NetEqDecoder codec_; + std::string name_ = "dummy name"; + AudioDecoder* decoder_; + int sample_rate_hz_; + size_t channels_; + std::unique_ptr neteq_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_EXTERNAL_DECODER_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.cc new file mode 100644 index 0000000000..44513ab15b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.cc @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_input.h" + +#include + +namespace webrtc { +namespace test { + +std::string NetEqInput::PacketData::ToString() const { + std::stringstream ss; + ss << "{" + << "time_ms: " << static_cast(time_ms) << ", " + << "header: {" + << "pt: " << static_cast(header.payloadType) << ", " + << "sn: " << header.sequenceNumber << ", " + << "ts: " << header.timestamp << ", " + << "ssrc: " << header.ssrc << "}, " + << "payload bytes: " << payload.size() << "}"; + return ss.str(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.h new file mode 100644 index 0000000000..88d9eb9413 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_input.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_INPUT_H_ + +#include +#include +#include + +#include "api/optional.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "rtc_base/buffer.h" + +namespace webrtc { +namespace test { + +// Interface class for input to the NetEqTest class. +class NetEqInput { + public: + struct PacketData { + std::string ToString() const; + + RTPHeader header; + rtc::Buffer payload; + double time_ms; + }; + + virtual ~NetEqInput() = default; + + // Returns at what time (in ms) NetEq::InsertPacket should be called next, or + // empty if the source is out of packets. + virtual rtc::Optional NextPacketTime() const = 0; + + // Returns at what time (in ms) NetEq::GetAudio should be called next, or + // empty if no more output events are available. + virtual rtc::Optional NextOutputEventTime() const = 0; + + // Returns the time (in ms) for the next event from either NextPacketTime() + // or NextOutputEventTime(), or empty if both are out of events. + rtc::Optional NextEventTime() const { + const auto a = NextPacketTime(); + const auto b = NextOutputEventTime(); + // Return the minimum of non-empty |a| and |b|, or empty if both are empty. + if (a) { + return b ? std::min(*a, *b) : a; + } + return b ? b : rtc::nullopt; + } + + // Returns the next packet to be inserted into NetEq. The packet following the + // returned one is pre-fetched in the NetEqInput object, such that future + // calls to NextPacketTime() or NextHeader() will return information from that + // packet. + virtual std::unique_ptr PopPacket() = 0; + + // Move to the next output event. This will make NextOutputEventTime() return + // a new value (potentially the same if several output events share the same + // time). + virtual void AdvanceOutputEvent() = 0; + + // Returns true if the source has come to an end. An implementation must + // eventually return true from this method, or the test will end up in an + // infinite loop. + virtual bool ended() const = 0; + + // Returns the RTP header for the next packet, i.e., the packet that will be + // delivered next by PopPacket(). + virtual rtc::Optional NextHeader() const = 0; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_INPUT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc new file mode 100644 index 0000000000..0741d7c88d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_packet_source_input.h" + +#include +#include + +#include "modules/audio_coding/neteq/tools/rtc_event_log_source.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +NetEqPacketSourceInput::NetEqPacketSourceInput() : next_output_event_ms_(0) {} + +rtc::Optional NetEqPacketSourceInput::NextPacketTime() const { + return packet_ + ? rtc::Optional(static_cast(packet_->time_ms())) + : rtc::nullopt; +} + +rtc::Optional NetEqPacketSourceInput::NextHeader() const { + return packet_ ? rtc::Optional(packet_->header()) + : rtc::nullopt; +} + +void NetEqPacketSourceInput::LoadNextPacket() { + packet_ = source()->NextPacket(); +} + +std::unique_ptr NetEqPacketSourceInput::PopPacket() { + if (!packet_) { + return std::unique_ptr(); + } + std::unique_ptr packet_data(new PacketData); + packet_data->header = packet_->header(); + if (packet_->payload_length_bytes() == 0 && + packet_->virtual_payload_length_bytes() > 0) { + // This is a header-only "dummy" packet. Set the payload to all zeros, with + // length according to the virtual length. + packet_data->payload.SetSize(packet_->virtual_payload_length_bytes()); + std::fill_n(packet_data->payload.data(), packet_data->payload.size(), 0); + } else { + packet_data->payload.SetData(packet_->payload(), + packet_->payload_length_bytes()); + } + packet_data->time_ms = packet_->time_ms(); + + LoadNextPacket(); + + return packet_data; +} + +NetEqRtpDumpInput::NetEqRtpDumpInput(const std::string& file_name, + const RtpHeaderExtensionMap& hdr_ext_map) + : source_(RtpFileSource::Create(file_name)) { + for (const auto& ext_pair : hdr_ext_map) { + source_->RegisterRtpHeaderExtension(ext_pair.second, ext_pair.first); + } + LoadNextPacket(); +} + +rtc::Optional NetEqRtpDumpInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +void NetEqRtpDumpInput::AdvanceOutputEvent() { + if (next_output_event_ms_) { + *next_output_event_ms_ += kOutputPeriodMs; + } + if (!NextPacketTime()) { + next_output_event_ms_ = rtc::nullopt; + } +} + +PacketSource* NetEqRtpDumpInput::source() { + return source_.get(); +} + +NetEqEventLogInput::NetEqEventLogInput(const std::string& file_name, + const RtpHeaderExtensionMap& hdr_ext_map) + : source_(RtcEventLogSource::Create(file_name)) { + for (const auto& ext_pair : hdr_ext_map) { + source_->RegisterRtpHeaderExtension(ext_pair.second, ext_pair.first); + } + LoadNextPacket(); + AdvanceOutputEvent(); +} + +rtc::Optional NetEqEventLogInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +void NetEqEventLogInput::AdvanceOutputEvent() { + next_output_event_ms_ = source_->NextAudioOutputEventMs(); + if (*next_output_event_ms_ == std::numeric_limits::max()) { + next_output_event_ms_ = rtc::nullopt; + } +} + +PacketSource* NetEqEventLogInput::source() { + return source_.get(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h new file mode 100644 index 0000000000..b482556c5a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PACKET_SOURCE_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PACKET_SOURCE_INPUT_H_ + +#include +#include + +#include "modules/audio_coding/neteq/tools/neteq_input.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { +namespace test { + +class RtpFileSource; +class RtcEventLogSource; + +// An adapter class to dress up a PacketSource object as a NetEqInput. +class NetEqPacketSourceInput : public NetEqInput { + public: + using RtpHeaderExtensionMap = std::map; + + NetEqPacketSourceInput(); + rtc::Optional NextPacketTime() const override; + std::unique_ptr PopPacket() override; + rtc::Optional NextHeader() const override; + bool ended() const override { return !next_output_event_ms_; } + + protected: + virtual PacketSource* source() = 0; + void LoadNextPacket(); + + rtc::Optional next_output_event_ms_; + + private: + std::unique_ptr packet_; +}; + +// Implementation of NetEqPacketSourceInput to be used with an RtpFileSource. +class NetEqRtpDumpInput final : public NetEqPacketSourceInput { + public: + NetEqRtpDumpInput(const std::string& file_name, + const RtpHeaderExtensionMap& hdr_ext_map); + + rtc::Optional NextOutputEventTime() const override; + void AdvanceOutputEvent() override; + + protected: + PacketSource* source() override; + + private: + static constexpr int64_t kOutputPeriodMs = 10; + + std::unique_ptr source_; +}; + +// Implementation of NetEqPacketSourceInput to be used with an +// RtcEventLogSource. +class NetEqEventLogInput final : public NetEqPacketSourceInput { + public: + NetEqEventLogInput(const std::string& file_name, + const RtpHeaderExtensionMap& hdr_ext_map); + + rtc::Optional NextOutputEventTime() const override; + void AdvanceOutputEvent() override; + + protected: + PacketSource* source() override; + + private: + std::unique_ptr source_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PACKET_SOURCE_INPUT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc new file mode 100644 index 0000000000..27ecdf4679 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_performance_test.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/clock.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +using webrtc::NetEq; +using webrtc::test::AudioLoop; +using webrtc::test::RtpGenerator; + +namespace webrtc { +namespace test { + +int64_t NetEqPerformanceTest::Run(int runtime_ms, + int lossrate, + double drift_factor) { + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + const int kSampRateHz = 32000; + const webrtc::NetEqDecoder kDecoderType = + webrtc::NetEqDecoder::kDecoderPCM16Bswb32kHz; + const std::string kDecoderName = "pcm16-swb32"; + const int kPayloadType = 95; + + // Initialize NetEq instance. + NetEq::Config config; + config.sample_rate_hz = kSampRateHz; + NetEq* neteq = NetEq::Create(config, CreateBuiltinAudioDecoderFactory()); + // Register decoder in |neteq|. + if (neteq->RegisterPayloadType(kDecoderType, kDecoderName, kPayloadType) != 0) + return -1; + + // Set up AudioLoop object. + AudioLoop audio_loop; + const size_t kMaxLoopLengthSamples = kSampRateHz * 10; // 10 second loop. + const size_t kInputBlockSizeSamples = 60 * kSampRateHz / 1000; // 60 ms. + if (!audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)) + return -1; + + int32_t time_now_ms = 0; + + // Get first input packet. + RTPHeader rtp_header; + RtpGenerator rtp_gen(kSampRateHz / 1000); + // Start with positive drift first half of simulation. + rtp_gen.set_drift_factor(drift_factor); + bool drift_flipped = false; + int32_t packet_input_time_ms = + rtp_gen.GetRtpHeader(kPayloadType, kInputBlockSizeSamples, &rtp_header); + auto input_samples = audio_loop.GetNextBlock(); + if (input_samples.empty()) + exit(1); + uint8_t input_payload[kInputBlockSizeSamples * sizeof(int16_t)]; + size_t payload_len = WebRtcPcm16b_Encode(input_samples.data(), + input_samples.size(), input_payload); + RTC_CHECK_EQ(sizeof(input_payload), payload_len); + + // Main loop. + webrtc::Clock* clock = webrtc::Clock::GetRealTimeClock(); + int64_t start_time_ms = clock->TimeInMilliseconds(); + AudioFrame out_frame; + while (time_now_ms < runtime_ms) { + while (packet_input_time_ms <= time_now_ms) { + // Drop every N packets, where N = FLAG_lossrate. + bool lost = false; + if (lossrate > 0) { + lost = ((rtp_header.sequenceNumber - 1) % lossrate) == 0; + } + if (!lost) { + // Insert packet. + int error = + neteq->InsertPacket(rtp_header, input_payload, + packet_input_time_ms * kSampRateHz / 1000); + if (error != NetEq::kOK) + return -1; + } + + // Get next packet. + packet_input_time_ms = rtp_gen.GetRtpHeader(kPayloadType, + kInputBlockSizeSamples, + &rtp_header); + input_samples = audio_loop.GetNextBlock(); + if (input_samples.empty()) + return -1; + payload_len = WebRtcPcm16b_Encode(input_samples.data(), + input_samples.size(), input_payload); + assert(payload_len == kInputBlockSizeSamples * sizeof(int16_t)); + } + + // Get output audio, but don't do anything with it. + bool muted; + int error = neteq->GetAudio(&out_frame, &muted); + RTC_CHECK(!muted); + if (error != NetEq::kOK) + return -1; + + assert(out_frame.samples_per_channel_ == + static_cast(kSampRateHz * 10 / 1000)); + + static const int kOutputBlockSizeMs = 10; + time_now_ms += kOutputBlockSizeMs; + if (time_now_ms >= runtime_ms / 2 && !drift_flipped) { + // Apply negative drift second half of simulation. + rtp_gen.set_drift_factor(-drift_factor); + drift_flipped = true; + } + } + int64_t end_time_ms = clock->TimeInMilliseconds(); + delete neteq; + return end_time_ms - start_time_ms; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h new file mode 100644 index 0000000000..dcf0314b4b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PERFORMANCE_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PERFORMANCE_TEST_H_ + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +class NetEqPerformanceTest { + public: + // Runs a performance test with parameters as follows: + // |runtime_ms|: the simulation time, i.e., the duration of the audio data. + // |lossrate|: drop one out of |lossrate| packets, e.g., one out of 10. + // |drift_factor|: clock drift in [0, 1]. + // Returns the runtime in ms. + static int64_t Run(int runtime_ms, int lossrate, double drift_factor); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PERFORMANCE_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc new file mode 100644 index 0000000000..7df46ff4f3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "modules/audio_coding/neteq/tools/output_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_wav_file.h" +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "rtc_base/checks.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +const uint8_t kPayloadType = 95; +const int kOutputSizeMs = 10; +const int kInitSeed = 0x12345678; +const int kPacketLossTimeUnitMs = 10; + +const std::string& DefaultInFilename() { + static const std::string path = + ResourcePath("audio_coding/speech_mono_16kHz", "pcm"); + return path; +} + +const std::string& DefaultOutFilename() { + static const std::string path = OutputPath() + "neteq_quality_test_out.pcm"; + return path; +} + +// Common validator for file names. +static bool ValidateFilename(const std::string& value, bool write) { + FILE* fid = write ? fopen(value.c_str(), "wb") : fopen(value.c_str(), "rb"); + if (fid == nullptr) + return false; + fclose(fid); + return true; +} + +DEFINE_string(in_filename, DefaultInFilename().c_str(), + "Filename for input audio (specify sample rate with --input_sample_rate, " + "and channels with --channels)."); + +DEFINE_int(input_sample_rate, 16000, "Sample rate of input file in Hz."); + +DEFINE_int(channels, 1, "Number of channels in input audio."); + +DEFINE_string(out_filename, DefaultOutFilename().c_str(), + "Name of output audio file."); + +DEFINE_int(runtime_ms, 10000, "Simulated runtime (milliseconds)."); + +DEFINE_int(packet_loss_rate, 10, "Percentile of packet loss."); + +DEFINE_int(random_loss_mode, 1, + "Random loss mode: 0--no loss, 1--uniform loss, 2--Gilbert Elliot loss."); + +DEFINE_int(burst_length, 30, + "Burst length in milliseconds, only valid for Gilbert Elliot loss."); + +DEFINE_float(drift_factor, 0.0, "Time drift factor."); + +// ProbTrans00Solver() is to calculate the transition probability from no-loss +// state to itself in a modified Gilbert Elliot packet loss model. The result is +// to achieve the target packet loss rate |loss_rate|, when a packet is not +// lost only if all |units| drawings within the duration of the packet result in +// no-loss. +static double ProbTrans00Solver(int units, double loss_rate, + double prob_trans_10) { + if (units == 1) + return prob_trans_10 / (1.0f - loss_rate) - prob_trans_10; +// 0 == prob_trans_00 ^ (units - 1) + (1 - loss_rate) / prob_trans_10 * +// prob_trans_00 - (1 - loss_rate) * (1 + 1 / prob_trans_10). +// There is a unique solution between 0.0 and 1.0, due to the monotonicity and +// an opposite sign at 0.0 and 1.0. +// For simplicity, we reformulate the equation as +// f(x) = x ^ (units - 1) + a x + b. +// Its derivative is +// f'(x) = (units - 1) x ^ (units - 2) + a. +// The derivative is strictly greater than 0 when x is between 0 and 1. +// We use Newton's method to solve the equation, iteration is +// x(k+1) = x(k) - f(x) / f'(x); + const double kPrecision = 0.001f; + const int kIterations = 100; + const double a = (1.0f - loss_rate) / prob_trans_10; + const double b = (loss_rate - 1.0f) * (1.0f + 1.0f / prob_trans_10); + double x = 0.0f; // Starting point; + double f = b; + double f_p; + int iter = 0; + while ((f >= kPrecision || f <= -kPrecision) && iter < kIterations) { + f_p = (units - 1.0f) * pow(x, units - 2) + a; + x -= f / f_p; + if (x > 1.0f) { + x = 1.0f; + } else if (x < 0.0f) { + x = 0.0f; + } + f = pow(x, units - 1) + a * x + b; + iter ++; + } + return x; +} + +NetEqQualityTest::NetEqQualityTest(int block_duration_ms, + int in_sampling_khz, + int out_sampling_khz, + NetEqDecoder decoder_type) + : decoder_type_(decoder_type), + channels_(static_cast(FLAG_channels)), + decoded_time_ms_(0), + decodable_time_ms_(0), + drift_factor_(FLAG_drift_factor), + packet_loss_rate_(FLAG_packet_loss_rate), + block_duration_ms_(block_duration_ms), + in_sampling_khz_(in_sampling_khz), + out_sampling_khz_(out_sampling_khz), + in_size_samples_( + static_cast(in_sampling_khz_ * block_duration_ms_)), + payload_size_bytes_(0), + max_payload_bytes_(0), + in_file_(new ResampleInputAudioFile(FLAG_in_filename, + FLAG_input_sample_rate, + in_sampling_khz * 1000)), + rtp_generator_( + new RtpGenerator(in_sampling_khz_, 0, 0, decodable_time_ms_)), + total_payload_size_bytes_(0) { + // Flag validation + RTC_CHECK(ValidateFilename(FLAG_in_filename, false)) + << "Invalid input filename."; + + RTC_CHECK(FLAG_input_sample_rate == 8000 || FLAG_input_sample_rate == 16000 || + FLAG_input_sample_rate == 32000 || FLAG_input_sample_rate == 48000) + << "Invalid sample rate should be 8000, 16000, 32000 or 48000 Hz."; + + RTC_CHECK_EQ(FLAG_channels, 1) + << "Invalid number of channels, current support only 1."; + + RTC_CHECK(ValidateFilename(FLAG_out_filename, true)) + << "Invalid output filename."; + + RTC_CHECK_GT(FLAG_runtime_ms, 0) + << "Invalid runtime, should be greater than 0."; + + RTC_CHECK(FLAG_packet_loss_rate >= 0 && FLAG_packet_loss_rate <= 100) + << "Invalid packet loss percentile, should be between 0 and 100."; + + RTC_CHECK(FLAG_random_loss_mode >= 0 && FLAG_random_loss_mode <= 2) + << "Invalid random packet loss mode, should be between 0 and 2."; + + RTC_CHECK_GE(FLAG_burst_length, kPacketLossTimeUnitMs) + << "Invalid burst length, should be greater than or equal to " + << kPacketLossTimeUnitMs << " ms."; + + RTC_CHECK_GT(FLAG_drift_factor, -0.1) + << "Invalid drift factor, should be greater than -0.1."; + + const std::string out_filename = FLAG_out_filename; + const std::string log_filename = out_filename + ".log"; + log_file_.open(log_filename.c_str(), std::ofstream::out); + RTC_CHECK(log_file_.is_open()); + + if (out_filename.size() >= 4 && + out_filename.substr(out_filename.size() - 4) == ".wav") { + // Open a wav file. + output_.reset( + new webrtc::test::OutputWavFile(out_filename, 1000 * out_sampling_khz)); + } else { + // Open a pcm file. + output_.reset(new webrtc::test::OutputAudioFile(out_filename)); + } + + NetEq::Config config; + config.sample_rate_hz = out_sampling_khz_ * 1000; + neteq_.reset( + NetEq::Create(config, webrtc::CreateBuiltinAudioDecoderFactory())); + max_payload_bytes_ = in_size_samples_ * channels_ * sizeof(int16_t); + in_data_.reset(new int16_t[in_size_samples_ * channels_]); +} + +NetEqQualityTest::~NetEqQualityTest() { + log_file_.close(); +} + +bool NoLoss::Lost() { + return false; +} + +UniformLoss::UniformLoss(double loss_rate) + : loss_rate_(loss_rate) { +} + +bool UniformLoss::Lost() { + int drop_this = rand(); + return (drop_this < loss_rate_ * RAND_MAX); +} + +GilbertElliotLoss::GilbertElliotLoss(double prob_trans_11, double prob_trans_01) + : prob_trans_11_(prob_trans_11), + prob_trans_01_(prob_trans_01), + lost_last_(false), + uniform_loss_model_(new UniformLoss(0)) { +} + +GilbertElliotLoss::~GilbertElliotLoss() {} + +bool GilbertElliotLoss::Lost() { + // Simulate bursty channel (Gilbert model). + // (1st order) Markov chain model with memory of the previous/last + // packet state (lost or received). + if (lost_last_) { + // Previous packet was not received. + uniform_loss_model_->set_loss_rate(prob_trans_11_); + return lost_last_ = uniform_loss_model_->Lost(); + } else { + uniform_loss_model_->set_loss_rate(prob_trans_01_); + return lost_last_ = uniform_loss_model_->Lost(); + } +} + +void NetEqQualityTest::SetUp() { + ASSERT_EQ(0, + neteq_->RegisterPayloadType(decoder_type_, "noname", kPayloadType)); + rtp_generator_->set_drift_factor(drift_factor_); + + int units = block_duration_ms_ / kPacketLossTimeUnitMs; + switch (FLAG_random_loss_mode) { + case 1: { + // |unit_loss_rate| is the packet loss rate for each unit time interval + // (kPacketLossTimeUnitMs). Since a packet loss event is generated if any + // of |block_duration_ms_ / kPacketLossTimeUnitMs| unit time intervals of + // a full packet duration is drawn with a loss, |unit_loss_rate| fulfills + // (1 - unit_loss_rate) ^ (block_duration_ms_ / kPacketLossTimeUnitMs) == + // 1 - packet_loss_rate. + double unit_loss_rate = (1.0f - pow(1.0f - 0.01f * packet_loss_rate_, + 1.0f / units)); + loss_model_.reset(new UniformLoss(unit_loss_rate)); + break; + } + case 2: { + // |FLAG_burst_length| should be integer times of kPacketLossTimeUnitMs. + ASSERT_EQ(0, FLAG_burst_length % kPacketLossTimeUnitMs); + + // We do not allow 100 percent packet loss in Gilbert Elliot model, which + // makes no sense. + ASSERT_GT(100, packet_loss_rate_); + + // To guarantee the overall packet loss rate, transition probabilities + // need to satisfy: + // pi_0 * (1 - prob_trans_01_) ^ units + + // pi_1 * prob_trans_10_ ^ (units - 1) == 1 - loss_rate + // pi_0 = prob_trans_10 / (prob_trans_10 + prob_trans_01_) + // is the stationary state probability of no-loss + // pi_1 = prob_trans_01_ / (prob_trans_10 + prob_trans_01_) + // is the stationary state probability of loss + // After a derivation prob_trans_00 should satisfy: + // prob_trans_00 ^ (units - 1) = (loss_rate - 1) / prob_trans_10 * + // prob_trans_00 + (1 - loss_rate) * (1 + 1 / prob_trans_10). + double loss_rate = 0.01f * packet_loss_rate_; + double prob_trans_10 = 1.0f * kPacketLossTimeUnitMs / FLAG_burst_length; + double prob_trans_00 = ProbTrans00Solver(units, loss_rate, prob_trans_10); + loss_model_.reset(new GilbertElliotLoss(1.0f - prob_trans_10, + 1.0f - prob_trans_00)); + break; + } + default: { + loss_model_.reset(new NoLoss); + break; + } + } + + // Make sure that the packet loss profile is same for all derived tests. + srand(kInitSeed); +} + +std::ofstream& NetEqQualityTest::Log() { + return log_file_; +} + +bool NetEqQualityTest::PacketLost() { + int cycles = block_duration_ms_ / kPacketLossTimeUnitMs; + + // The loop is to make sure that codecs with different block lengths share the + // same packet loss profile. + bool lost = false; + for (int idx = 0; idx < cycles; idx ++) { + if (loss_model_->Lost()) { + // The packet will be lost if any of the drawings indicates a loss, but + // the loop has to go on to make sure that codecs with different block + // lengths keep the same pace. + lost = true; + } + } + return lost; +} + +int NetEqQualityTest::Transmit() { + int packet_input_time_ms = + rtp_generator_->GetRtpHeader(kPayloadType, in_size_samples_, + &rtp_header_); + Log() << "Packet of size " + << payload_size_bytes_ + << " bytes, for frame at " + << packet_input_time_ms + << " ms "; + if (payload_size_bytes_ > 0) { + if (!PacketLost()) { + int ret = neteq_->InsertPacket( + rtp_header_, + rtc::ArrayView(payload_.data(), payload_size_bytes_), + packet_input_time_ms * in_sampling_khz_); + if (ret != NetEq::kOK) + return -1; + Log() << "was sent."; + } else { + Log() << "was lost."; + } + } + Log() << std::endl; + return packet_input_time_ms; +} + +int NetEqQualityTest::DecodeBlock() { + bool muted; + int ret = neteq_->GetAudio(&out_frame_, &muted); + RTC_CHECK(!muted); + + if (ret != NetEq::kOK) { + return -1; + } else { + RTC_DCHECK_EQ(out_frame_.num_channels_, channels_); + RTC_DCHECK_EQ(out_frame_.samples_per_channel_, + static_cast(kOutputSizeMs * out_sampling_khz_)); + RTC_CHECK(output_->WriteArray( + out_frame_.data(), + out_frame_.samples_per_channel_ * out_frame_.num_channels_)); + return static_cast(out_frame_.samples_per_channel_); + } +} + +void NetEqQualityTest::Simulate() { + int audio_size_samples; + + while (decoded_time_ms_ < FLAG_runtime_ms) { + // Assume 10 packets in packets buffer. + while (decodable_time_ms_ - 10 * block_duration_ms_ < decoded_time_ms_) { + ASSERT_TRUE(in_file_->Read(in_size_samples_ * channels_, &in_data_[0])); + payload_.Clear(); + payload_size_bytes_ = EncodeBlock(&in_data_[0], + in_size_samples_, &payload_, + max_payload_bytes_); + total_payload_size_bytes_ += payload_size_bytes_; + decodable_time_ms_ = Transmit() + block_duration_ms_; + } + audio_size_samples = DecodeBlock(); + if (audio_size_samples > 0) { + decoded_time_ms_ += audio_size_samples / out_sampling_khz_; + } + } + Log() << "Average bit rate was " + << 8.0f * total_payload_size_bytes_ / FLAG_runtime_ms + << " kbps" + << std::endl; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h new file mode 100644 index 0000000000..8532f25a0d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_QUALITY_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_QUALITY_TEST_H_ + +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/flags.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +class LossModel { + public: + virtual ~LossModel() {}; + virtual bool Lost() = 0; +}; + +class NoLoss : public LossModel { + public: + bool Lost() override; +}; + +class UniformLoss : public LossModel { + public: + UniformLoss(double loss_rate); + bool Lost() override; + void set_loss_rate(double loss_rate) { loss_rate_ = loss_rate; } + + private: + double loss_rate_; +}; + +class GilbertElliotLoss : public LossModel { + public: + GilbertElliotLoss(double prob_trans_11, double prob_trans_01); + ~GilbertElliotLoss() override; + bool Lost() override; + + private: + // Prob. of losing current packet, when previous packet is lost. + double prob_trans_11_; + // Prob. of losing current packet, when previous packet is not lost. + double prob_trans_01_; + bool lost_last_; + std::unique_ptr uniform_loss_model_; +}; + +class NetEqQualityTest : public ::testing::Test { + protected: + NetEqQualityTest(int block_duration_ms, + int in_sampling_khz, + int out_sampling_khz, + NetEqDecoder decoder_type); + ~NetEqQualityTest() override; + + void SetUp() override; + + // EncodeBlock(...) does the following: + // 1. encodes a block of audio, saved in |in_data| and has a length of + // |block_size_samples| (samples per channel), + // 2. save the bit stream to |payload| of |max_bytes| bytes in size, + // 3. returns the length of the payload (in bytes), + virtual int EncodeBlock(int16_t* in_data, size_t block_size_samples, + rtc::Buffer* payload, size_t max_bytes) = 0; + + // PacketLost(...) determines weather a packet sent at an indicated time gets + // lost or not. + bool PacketLost(); + + // DecodeBlock() decodes a block of audio using the payload stored in + // |payload_| with the length of |payload_size_bytes_| (bytes). The decoded + // audio is to be stored in |out_data_|. + int DecodeBlock(); + + // Transmit() uses |rtp_generator_| to generate a packet and passes it to + // |neteq_|. + int Transmit(); + + // Runs encoding / transmitting / decoding. + void Simulate(); + + // Write to log file. Usage Log() << ... + std::ofstream& Log(); + + NetEqDecoder decoder_type_; + const size_t channels_; + + private: + int decoded_time_ms_; + int decodable_time_ms_; + double drift_factor_; + int packet_loss_rate_; + const int block_duration_ms_; + const int in_sampling_khz_; + const int out_sampling_khz_; + + // Number of samples per channel in a frame. + const size_t in_size_samples_; + + size_t payload_size_bytes_; + size_t max_payload_bytes_; + + std::unique_ptr in_file_; + std::unique_ptr output_; + std::ofstream log_file_; + + std::unique_ptr rtp_generator_; + std::unique_ptr neteq_; + std::unique_ptr loss_model_; + + std::unique_ptr in_data_; + rtc::Buffer payload_; + AudioFrame out_frame_; + RTPHeader rtp_header_; + + size_t total_payload_size_bytes_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_QUALITY_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc new file mode 100644 index 0000000000..6c846c0d0d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_replacement_input.h" + +#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +NetEqReplacementInput::NetEqReplacementInput( + std::unique_ptr source, + uint8_t replacement_payload_type, + const std::set& comfort_noise_types, + const std::set& forbidden_types) + : source_(std::move(source)), + replacement_payload_type_(replacement_payload_type), + comfort_noise_types_(comfort_noise_types), + forbidden_types_(forbidden_types) { + RTC_CHECK(source_); + packet_ = source_->PopPacket(); + ReplacePacket(); + RTC_CHECK(packet_); +} + +rtc::Optional NetEqReplacementInput::NextPacketTime() const { + return packet_ + ? rtc::Optional(static_cast(packet_->time_ms)) + : rtc::nullopt; +} + +rtc::Optional NetEqReplacementInput::NextOutputEventTime() const { + return source_->NextOutputEventTime(); +} + +std::unique_ptr NetEqReplacementInput::PopPacket() { + std::unique_ptr to_return = std::move(packet_); + packet_ = source_->PopPacket(); + ReplacePacket(); + return to_return; +} + +void NetEqReplacementInput::AdvanceOutputEvent() { + source_->AdvanceOutputEvent(); +} + +bool NetEqReplacementInput::ended() const { + return source_->ended(); +} + +rtc::Optional NetEqReplacementInput::NextHeader() const { + return source_->NextHeader(); +} + +void NetEqReplacementInput::ReplacePacket() { + if (!source_->NextPacketTime()) { + // End of input. Cannot do proper replacement on the very last packet, so we + // delete it instead. + packet_.reset(); + return; + } + + RTC_DCHECK(packet_); + + RTC_CHECK_EQ(forbidden_types_.count(packet_->header.payloadType), 0) + << "Payload type " << static_cast(packet_->header.payloadType) + << " is forbidden."; + + // Check if this packet is comfort noise. + if (comfort_noise_types_.count(packet_->header.payloadType) != 0) { + // If CNG, simply insert a zero-energy one-byte payload. + uint8_t cng_payload[1] = {127}; // Max attenuation of CNG. + packet_->payload.SetData(cng_payload); + return; + } + + rtc::Optional next_hdr = source_->NextHeader(); + RTC_DCHECK(next_hdr); + uint8_t payload[12]; + RTC_DCHECK_LE(last_frame_size_timestamps_, 120 * 48); + uint32_t input_frame_size_timestamps = last_frame_size_timestamps_; + const uint32_t timestamp_diff = + next_hdr->timestamp - packet_->header.timestamp; + if (next_hdr->sequenceNumber == packet_->header.sequenceNumber + 1 && + timestamp_diff <= 120 * 48) { + // Packets are in order and the timestamp diff is less than 5760 samples. + // Accept the timestamp diff as a valid frame size. + input_frame_size_timestamps = timestamp_diff; + last_frame_size_timestamps_ = input_frame_size_timestamps; + } + RTC_DCHECK_LE(input_frame_size_timestamps, 120 * 48); + FakeDecodeFromFile::PrepareEncoded(packet_->header.timestamp, + input_frame_size_timestamps, + packet_->payload.size(), payload); + packet_->payload.SetData(payload); + packet_->header.payloadType = replacement_payload_type_; + return; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h new file mode 100644 index 0000000000..3a89399771 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_REPLACEMENT_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_REPLACEMENT_INPUT_H_ + +#include +#include + +#include "modules/audio_coding/neteq/tools/neteq_input.h" + +namespace webrtc { +namespace test { + +// This class converts the packets from a NetEqInput to fake encodings to be +// decoded by a FakeDecodeFromFile decoder. +class NetEqReplacementInput : public NetEqInput { + public: + NetEqReplacementInput(std::unique_ptr source, + uint8_t replacement_payload_type, + const std::set& comfort_noise_types, + const std::set& forbidden_types); + + rtc::Optional NextPacketTime() const override; + rtc::Optional NextOutputEventTime() const override; + std::unique_ptr PopPacket() override; + void AdvanceOutputEvent() override; + bool ended() const override; + rtc::Optional NextHeader() const override; + + private: + void ReplacePacket(); + + std::unique_ptr source_; + const uint8_t replacement_payload_type_; + const std::set comfort_noise_types_; + const std::set forbidden_types_; + std::unique_ptr packet_; // The next packet to deliver. + uint32_t last_frame_size_timestamps_ = 960; // Initial guess: 20 ms @ 48 kHz. +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_REPLACEMENT_INPUT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc new file mode 100644 index 0000000000..2657bfaeb5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc @@ -0,0 +1,670 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include // For ULONG_MAX returned by strtoul. +#include +#include // For strtoul. +#include + +#include +#include +#include +#include +#include +#include + +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" +#include "modules/audio_coding/neteq/tools/neteq_packet_source_input.h" +#include "modules/audio_coding/neteq/tools/neteq_replacement_input.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" +#include "modules/audio_coding/neteq/tools/output_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_wav_file.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" +#include "rtc_base/flags.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { +namespace { + +// Parses the input string for a valid SSRC (at the start of the string). If a +// valid SSRC is found, it is written to the output variable |ssrc|, and true is +// returned. Otherwise, false is returned. +bool ParseSsrc(const std::string& str, uint32_t* ssrc) { + if (str.empty()) + return true; + int base = 10; + // Look for "0x" or "0X" at the start and change base to 16 if found. + if ((str.compare(0, 2, "0x") == 0) || (str.compare(0, 2, "0X") == 0)) + base = 16; + errno = 0; + char* end_ptr; + unsigned long value = strtoul(str.c_str(), &end_ptr, base); + if (value == ULONG_MAX && errno == ERANGE) + return false; // Value out of range for unsigned long. + if (sizeof(unsigned long) > sizeof(uint32_t) && value > 0xFFFFFFFF) + return false; // Value out of range for uint32_t. + if (end_ptr - str.c_str() < static_cast(str.length())) + return false; // Part of the string was not parsed. + *ssrc = static_cast(value); + return true; +} + +// Flag validators. +bool ValidatePayloadType(int value) { + if (value >= 0 && value <= 127) // Value is ok. + return true; + printf("Payload type must be between 0 and 127, not %d\n", + static_cast(value)); + return false; +} + +bool ValidateSsrcValue(const std::string& str) { + uint32_t dummy_ssrc; + if (ParseSsrc(str, &dummy_ssrc)) // Value is ok. + return true; + printf("Invalid SSRC: %s\n", str.c_str()); + return false; +} + +static bool ValidateExtensionId(int value) { + if (value > 0 && value <= 255) // Value is ok. + return true; + printf("Extension ID must be between 1 and 255, not %d\n", + static_cast(value)); + return false; +} + +// Define command line flags. +DEFINE_int(pcmu, 0, "RTP payload type for PCM-u"); +DEFINE_int(pcma, 8, "RTP payload type for PCM-a"); +DEFINE_int(ilbc, 102, "RTP payload type for iLBC"); +DEFINE_int(isac, 103, "RTP payload type for iSAC"); +DEFINE_int(isac_swb, 104, "RTP payload type for iSAC-swb (32 kHz)"); +DEFINE_int(opus, 111, "RTP payload type for Opus"); +DEFINE_int(pcm16b, 93, "RTP payload type for PCM16b-nb (8 kHz)"); +DEFINE_int(pcm16b_wb, 94, "RTP payload type for PCM16b-wb (16 kHz)"); +DEFINE_int(pcm16b_swb32, 95, "RTP payload type for PCM16b-swb32 (32 kHz)"); +DEFINE_int(pcm16b_swb48, 96, "RTP payload type for PCM16b-swb48 (48 kHz)"); +DEFINE_int(g722, 9, "RTP payload type for G.722"); +DEFINE_int(avt, 106, "RTP payload type for AVT/DTMF (8 kHz)"); +DEFINE_int(avt_16, 114, "RTP payload type for AVT/DTMF (16 kHz)"); +DEFINE_int(avt_32, 115, "RTP payload type for AVT/DTMF (32 kHz)"); +DEFINE_int(avt_48, 116, "RTP payload type for AVT/DTMF (48 kHz)"); +DEFINE_int(red, 117, "RTP payload type for redundant audio (RED)"); +DEFINE_int(cn_nb, 13, "RTP payload type for comfort noise (8 kHz)"); +DEFINE_int(cn_wb, 98, "RTP payload type for comfort noise (16 kHz)"); +DEFINE_int(cn_swb32, 99, "RTP payload type for comfort noise (32 kHz)"); +DEFINE_int(cn_swb48, 100, "RTP payload type for comfort noise (48 kHz)"); +DEFINE_bool(codec_map, false, "Prints the mapping between RTP payload type and " + "codec"); +DEFINE_string(replacement_audio_file, "", + "A PCM file that will be used to populate ""dummy"" RTP packets"); +DEFINE_string(ssrc, + "", + "Only use packets with this SSRC (decimal or hex, the latter " + "starting with 0x)"); +DEFINE_int(audio_level, 1, "Extension ID for audio level (RFC 6464)"); +DEFINE_int(abs_send_time, 3, "Extension ID for absolute sender time"); +DEFINE_int(transport_seq_no, 5, "Extension ID for transport sequence number"); +DEFINE_bool(matlabplot, + false, + "Generates a matlab script for plotting the delay profile"); +DEFINE_bool(help, false, "Prints this message"); + +// Maps a codec type to a printable name string. +std::string CodecName(NetEqDecoder codec) { + switch (codec) { + case NetEqDecoder::kDecoderPCMu: + return "PCM-u"; + case NetEqDecoder::kDecoderPCMa: + return "PCM-a"; + case NetEqDecoder::kDecoderILBC: + return "iLBC"; + case NetEqDecoder::kDecoderISAC: + return "iSAC"; + case NetEqDecoder::kDecoderISACswb: + return "iSAC-swb (32 kHz)"; + case NetEqDecoder::kDecoderOpus: + return "Opus"; + case NetEqDecoder::kDecoderPCM16B: + return "PCM16b-nb (8 kHz)"; + case NetEqDecoder::kDecoderPCM16Bwb: + return "PCM16b-wb (16 kHz)"; + case NetEqDecoder::kDecoderPCM16Bswb32kHz: + return "PCM16b-swb32 (32 kHz)"; + case NetEqDecoder::kDecoderPCM16Bswb48kHz: + return "PCM16b-swb48 (48 kHz)"; + case NetEqDecoder::kDecoderG722: + return "G.722"; + case NetEqDecoder::kDecoderRED: + return "redundant audio (RED)"; + case NetEqDecoder::kDecoderAVT: + return "AVT/DTMF (8 kHz)"; + case NetEqDecoder::kDecoderAVT16kHz: + return "AVT/DTMF (16 kHz)"; + case NetEqDecoder::kDecoderAVT32kHz: + return "AVT/DTMF (32 kHz)"; + case NetEqDecoder::kDecoderAVT48kHz: + return "AVT/DTMF (48 kHz)"; + case NetEqDecoder::kDecoderCNGnb: + return "comfort noise (8 kHz)"; + case NetEqDecoder::kDecoderCNGwb: + return "comfort noise (16 kHz)"; + case NetEqDecoder::kDecoderCNGswb32kHz: + return "comfort noise (32 kHz)"; + case NetEqDecoder::kDecoderCNGswb48kHz: + return "comfort noise (48 kHz)"; + default: + FATAL(); + return "undefined"; + } +} + +void PrintCodecMappingEntry(NetEqDecoder codec, int flag) { + std::cout << CodecName(codec) << ": " << flag << std::endl; +} + +void PrintCodecMapping() { + PrintCodecMappingEntry(NetEqDecoder::kDecoderPCMu, FLAG_pcmu); + PrintCodecMappingEntry(NetEqDecoder::kDecoderPCMa, FLAG_pcma); + PrintCodecMappingEntry(NetEqDecoder::kDecoderILBC, FLAG_ilbc); + PrintCodecMappingEntry(NetEqDecoder::kDecoderISAC, FLAG_isac); + PrintCodecMappingEntry(NetEqDecoder::kDecoderISACswb, FLAG_isac_swb); + PrintCodecMappingEntry(NetEqDecoder::kDecoderOpus, FLAG_opus); + PrintCodecMappingEntry(NetEqDecoder::kDecoderPCM16B, FLAG_pcm16b); + PrintCodecMappingEntry(NetEqDecoder::kDecoderPCM16Bwb, FLAG_pcm16b_wb); + PrintCodecMappingEntry(NetEqDecoder::kDecoderPCM16Bswb32kHz, + FLAG_pcm16b_swb32); + PrintCodecMappingEntry(NetEqDecoder::kDecoderPCM16Bswb48kHz, + FLAG_pcm16b_swb48); + PrintCodecMappingEntry(NetEqDecoder::kDecoderG722, FLAG_g722); + PrintCodecMappingEntry(NetEqDecoder::kDecoderAVT, FLAG_avt); + PrintCodecMappingEntry(NetEqDecoder::kDecoderAVT16kHz, FLAG_avt_16); + PrintCodecMappingEntry(NetEqDecoder::kDecoderAVT32kHz, FLAG_avt_32); + PrintCodecMappingEntry(NetEqDecoder::kDecoderAVT48kHz, FLAG_avt_48); + PrintCodecMappingEntry(NetEqDecoder::kDecoderRED, FLAG_red); + PrintCodecMappingEntry(NetEqDecoder::kDecoderCNGnb, FLAG_cn_nb); + PrintCodecMappingEntry(NetEqDecoder::kDecoderCNGwb, FLAG_cn_wb); + PrintCodecMappingEntry(NetEqDecoder::kDecoderCNGswb32kHz, FLAG_cn_swb32); + PrintCodecMappingEntry(NetEqDecoder::kDecoderCNGswb48kHz, FLAG_cn_swb48); +} + +rtc::Optional CodecSampleRate(uint8_t payload_type) { + if (payload_type == FLAG_pcmu || payload_type == FLAG_pcma || + payload_type == FLAG_ilbc || payload_type == FLAG_pcm16b || + payload_type == FLAG_cn_nb || payload_type == FLAG_avt) + return 8000; + if (payload_type == FLAG_isac || payload_type == FLAG_pcm16b_wb || + payload_type == FLAG_g722 || payload_type == FLAG_cn_wb || + payload_type == FLAG_avt_16) + return 16000; + if (payload_type == FLAG_isac_swb || payload_type == FLAG_pcm16b_swb32 || + payload_type == FLAG_cn_swb32 || payload_type == FLAG_avt_32) + return 32000; + if (payload_type == FLAG_opus || payload_type == FLAG_pcm16b_swb48 || + payload_type == FLAG_cn_swb48 || payload_type == FLAG_avt_48) + return 48000; + if (payload_type == FLAG_red) + return 0; + return rtc::nullopt; +} + +// Class to let through only the packets with a given SSRC. Should be used as an +// outer layer on another NetEqInput object. +class FilterSsrcInput : public NetEqInput { + public: + FilterSsrcInput(std::unique_ptr source, uint32_t ssrc) + : source_(std::move(source)), ssrc_(ssrc) { + FindNextWithCorrectSsrc(); + RTC_CHECK(source_->NextHeader()) << "Found no packet with SSRC = 0x" + << std::hex << ssrc_; + } + + // All methods but PopPacket() simply relay to the |source_| object. + rtc::Optional NextPacketTime() const override { + return source_->NextPacketTime(); + } + rtc::Optional NextOutputEventTime() const override { + return source_->NextOutputEventTime(); + } + + // Returns the next packet, and throws away upcoming packets that do not match + // the desired SSRC. + std::unique_ptr PopPacket() override { + std::unique_ptr packet_to_return = source_->PopPacket(); + RTC_DCHECK(!packet_to_return || packet_to_return->header.ssrc == ssrc_); + // Pre-fetch the next packet with correct SSRC. Hence, |source_| will always + // be have a valid packet (or empty if no more packets are available) when + // this method returns. + FindNextWithCorrectSsrc(); + return packet_to_return; + } + + void AdvanceOutputEvent() override { source_->AdvanceOutputEvent(); } + + bool ended() const override { return source_->ended(); } + + rtc::Optional NextHeader() const override { + return source_->NextHeader(); + } + + private: + void FindNextWithCorrectSsrc() { + while (source_->NextHeader() && source_->NextHeader()->ssrc != ssrc_) { + source_->PopPacket(); + } + } + + std::unique_ptr source_; + uint32_t ssrc_; +}; + +// A callback class which prints whenver the inserted packet stream changes +// the SSRC. +class SsrcSwitchDetector : public NetEqPostInsertPacket { + public: + // Takes a pointer to another callback object, which will be invoked after + // this object finishes. This does not transfer ownership, and null is a + // valid value. + explicit SsrcSwitchDetector(NetEqPostInsertPacket* other_callback) + : other_callback_(other_callback) {} + + void AfterInsertPacket(const NetEqInput::PacketData& packet, + NetEq* neteq) override { + if (last_ssrc_ && packet.header.ssrc != *last_ssrc_) { + std::cout << "Changing streams from 0x" << std::hex << *last_ssrc_ + << " to 0x" << std::hex << packet.header.ssrc + << std::dec << " (payload type " + << static_cast(packet.header.payloadType) << ")" + << std::endl; + } + last_ssrc_ = packet.header.ssrc; + if (other_callback_) { + other_callback_->AfterInsertPacket(packet, neteq); + } + } + + private: + NetEqPostInsertPacket* other_callback_; + rtc::Optional last_ssrc_; +}; + +class StatsGetter : public NetEqGetAudioCallback { + public: + // This struct is a replica of webrtc::NetEqNetworkStatistics, but with all + // values stored in double precision. + struct Stats { + double current_buffer_size_ms = 0.0; + double preferred_buffer_size_ms = 0.0; + double jitter_peaks_found = 0.0; + double packet_loss_rate = 0.0; + double expand_rate = 0.0; + double speech_expand_rate = 0.0; + double preemptive_rate = 0.0; + double accelerate_rate = 0.0; + double secondary_decoded_rate = 0.0; + double secondary_discarded_rate = 0.0; + double clockdrift_ppm = 0.0; + double added_zero_samples = 0.0; + double mean_waiting_time_ms = 0.0; + double median_waiting_time_ms = 0.0; + double min_waiting_time_ms = 0.0; + double max_waiting_time_ms = 0.0; + }; + + // Takes a pointer to another callback object, which will be invoked after + // this object finishes. This does not transfer ownership, and null is a + // valid value. + explicit StatsGetter(NetEqGetAudioCallback* other_callback) + : other_callback_(other_callback) {} + + void BeforeGetAudio(NetEq* neteq) override { + if (other_callback_) { + other_callback_->BeforeGetAudio(neteq); + } + } + + void AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool muted, + NetEq* neteq) override { + if (++counter_ >= 100) { + counter_ = 0; + NetEqNetworkStatistics stats; + RTC_CHECK_EQ(neteq->NetworkStatistics(&stats), 0); + stats_.push_back(stats); + } + if (other_callback_) { + other_callback_->AfterGetAudio(time_now_ms, audio_frame, muted, neteq); + } + } + + double AverageSpeechExpandRate() const { + double sum_speech_expand = + std::accumulate(stats_.begin(), stats_.end(), double{0.0}, + [](double a, NetEqNetworkStatistics b) { + return a + static_cast(b.speech_expand_rate); + }); + return sum_speech_expand / 16384.0 / stats_.size(); + } + + Stats AverageStats() const { + Stats sum_stats = std::accumulate( + stats_.begin(), stats_.end(), Stats(), + [](Stats a, NetEqNetworkStatistics b) { + a.current_buffer_size_ms += b.current_buffer_size_ms; + a.preferred_buffer_size_ms += b.preferred_buffer_size_ms; + a.jitter_peaks_found += b.jitter_peaks_found; + a.packet_loss_rate += b.packet_loss_rate / 16384.0; + a.expand_rate += b.expand_rate / 16384.0; + a.speech_expand_rate += b.speech_expand_rate / 16384.0; + a.preemptive_rate += b.preemptive_rate / 16384.0; + a.accelerate_rate += b.accelerate_rate / 16384.0; + a.secondary_decoded_rate += b.secondary_decoded_rate / 16384.0; + a.secondary_discarded_rate += b.secondary_discarded_rate / 16384.0; + a.clockdrift_ppm += b.clockdrift_ppm; + a.added_zero_samples += b.added_zero_samples; + a.mean_waiting_time_ms += b.mean_waiting_time_ms; + a.median_waiting_time_ms += b.median_waiting_time_ms; + a.min_waiting_time_ms = + std::min(a.min_waiting_time_ms, + static_cast(b.min_waiting_time_ms)); + a.max_waiting_time_ms = + std::max(a.max_waiting_time_ms, + static_cast(b.max_waiting_time_ms)); + return a; + }); + + sum_stats.current_buffer_size_ms /= stats_.size(); + sum_stats.preferred_buffer_size_ms /= stats_.size(); + sum_stats.jitter_peaks_found /= stats_.size(); + sum_stats.packet_loss_rate /= stats_.size(); + sum_stats.expand_rate /= stats_.size(); + sum_stats.speech_expand_rate /= stats_.size(); + sum_stats.preemptive_rate /= stats_.size(); + sum_stats.accelerate_rate /= stats_.size(); + sum_stats.secondary_decoded_rate /= stats_.size(); + sum_stats.secondary_discarded_rate /= stats_.size(); + sum_stats.clockdrift_ppm /= stats_.size(); + sum_stats.added_zero_samples /= stats_.size(); + sum_stats.mean_waiting_time_ms /= stats_.size(); + sum_stats.median_waiting_time_ms /= stats_.size(); + + return sum_stats; + } + + private: + NetEqGetAudioCallback* other_callback_; + size_t counter_ = 0; + std::vector stats_; +}; + +int RunTest(int argc, char* argv[]) { + std::string program_name = argv[0]; + std::string usage = "Tool for decoding an RTP dump file using NetEq.\n" + "Run " + program_name + " --help for usage.\n" + "Example usage:\n" + program_name + + " input.rtp output.{pcm, wav}\n"; + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true)) { + return 1; + } + if (FLAG_help) { + std::cout << usage; + rtc::FlagList::Print(nullptr, false); + return 0; + } + + if (FLAG_codec_map) { + PrintCodecMapping(); + } + + if (argc != 3) { + if (FLAG_codec_map) { + // We have already printed the codec map. Just end the program. + return 0; + } + // Print usage information. + std::cout << usage; + return 0; + } + RTC_CHECK(ValidatePayloadType(FLAG_pcmu)); + RTC_CHECK(ValidatePayloadType(FLAG_pcma)); + RTC_CHECK(ValidatePayloadType(FLAG_ilbc)); + RTC_CHECK(ValidatePayloadType(FLAG_isac)); + RTC_CHECK(ValidatePayloadType(FLAG_isac_swb)); + RTC_CHECK(ValidatePayloadType(FLAG_opus)); + RTC_CHECK(ValidatePayloadType(FLAG_pcm16b)); + RTC_CHECK(ValidatePayloadType(FLAG_pcm16b_wb)); + RTC_CHECK(ValidatePayloadType(FLAG_pcm16b_swb32)); + RTC_CHECK(ValidatePayloadType(FLAG_pcm16b_swb48)); + RTC_CHECK(ValidatePayloadType(FLAG_g722)); + RTC_CHECK(ValidatePayloadType(FLAG_avt)); + RTC_CHECK(ValidatePayloadType(FLAG_avt_16)); + RTC_CHECK(ValidatePayloadType(FLAG_avt_32)); + RTC_CHECK(ValidatePayloadType(FLAG_avt_48)); + RTC_CHECK(ValidatePayloadType(FLAG_red)); + RTC_CHECK(ValidatePayloadType(FLAG_cn_nb)); + RTC_CHECK(ValidatePayloadType(FLAG_cn_wb)); + RTC_CHECK(ValidatePayloadType(FLAG_cn_swb32)); + RTC_CHECK(ValidatePayloadType(FLAG_cn_swb48)); + RTC_CHECK(ValidateSsrcValue(FLAG_ssrc)); + RTC_CHECK(ValidateExtensionId(FLAG_audio_level)); + RTC_CHECK(ValidateExtensionId(FLAG_abs_send_time)); + RTC_CHECK(ValidateExtensionId(FLAG_transport_seq_no)); + + // Gather RTP header extensions in a map. + NetEqPacketSourceInput::RtpHeaderExtensionMap rtp_ext_map = { + {FLAG_audio_level, kRtpExtensionAudioLevel}, + {FLAG_abs_send_time, kRtpExtensionAbsoluteSendTime}, + {FLAG_transport_seq_no, kRtpExtensionTransportSequenceNumber}}; + + const std::string input_file_name = argv[1]; + std::unique_ptr input; + if (RtpFileSource::ValidRtpDump(input_file_name) || + RtpFileSource::ValidPcap(input_file_name)) { + input.reset(new NetEqRtpDumpInput(input_file_name, rtp_ext_map)); + } else { + input.reset(new NetEqEventLogInput(input_file_name, rtp_ext_map)); + } + + std::cout << "Input file: " << input_file_name << std::endl; + RTC_CHECK(input) << "Cannot open input file"; + RTC_CHECK(!input->ended()) << "Input file is empty"; + + // Check if an SSRC value was provided. + if (strlen(FLAG_ssrc) > 0) { + uint32_t ssrc; + RTC_CHECK(ParseSsrc(FLAG_ssrc, &ssrc)) << "Flag verification has failed."; + input.reset(new FilterSsrcInput(std::move(input), ssrc)); + } + + // Check the sample rate. + rtc::Optional sample_rate_hz; + std::set> discarded_pt_and_ssrc; + while (input->NextHeader()) { + rtc::Optional first_rtp_header = input->NextHeader(); + RTC_DCHECK(first_rtp_header); + sample_rate_hz = CodecSampleRate(first_rtp_header->payloadType); + if (sample_rate_hz) { + std::cout << "Found valid packet with payload type " + << static_cast(first_rtp_header->payloadType) + << " and SSRC 0x" << std::hex << first_rtp_header->ssrc + << std::dec << std::endl; + break; + } + // Discard this packet and move to the next. Keep track of discarded payload + // types and SSRCs. + discarded_pt_and_ssrc.emplace(first_rtp_header->payloadType, + first_rtp_header->ssrc); + input->PopPacket(); + } + if (!discarded_pt_and_ssrc.empty()) { + std::cout << "Discarded initial packets with the following payload types " + "and SSRCs:" + << std::endl; + for (const auto& d : discarded_pt_and_ssrc) { + std::cout << "PT " << d.first << "; SSRC 0x" << std::hex + << static_cast(d.second) << std::dec << std::endl; + } + } + if (!sample_rate_hz) { + std::cout << "Cannot find any packets with known payload types" + << std::endl; + RTC_NOTREACHED(); + } + + // Open the output file now that we know the sample rate. (Rate is only needed + // for wav files.) + const std::string output_file_name = argv[2]; + std::unique_ptr output; + if (output_file_name.size() >= 4 && + output_file_name.substr(output_file_name.size() - 4) == ".wav") { + // Open a wav file. + output.reset(new OutputWavFile(output_file_name, *sample_rate_hz)); + } else { + // Open a pcm file. + output.reset(new OutputAudioFile(output_file_name)); + } + + std::cout << "Output file: " << output_file_name << std::endl; + + NetEqTest::DecoderMap codecs = { + {FLAG_pcmu, std::make_pair(NetEqDecoder::kDecoderPCMu, "pcmu")}, + {FLAG_pcma, std::make_pair(NetEqDecoder::kDecoderPCMa, "pcma")}, + {FLAG_ilbc, std::make_pair(NetEqDecoder::kDecoderILBC, "ilbc")}, + {FLAG_isac, std::make_pair(NetEqDecoder::kDecoderISAC, "isac")}, + {FLAG_isac_swb, + std::make_pair(NetEqDecoder::kDecoderISACswb, "isac-swb")}, + {FLAG_opus, std::make_pair(NetEqDecoder::kDecoderOpus, "opus")}, + {FLAG_pcm16b, std::make_pair(NetEqDecoder::kDecoderPCM16B, "pcm16-nb")}, + {FLAG_pcm16b_wb, + std::make_pair(NetEqDecoder::kDecoderPCM16Bwb, "pcm16-wb")}, + {FLAG_pcm16b_swb32, + std::make_pair(NetEqDecoder::kDecoderPCM16Bswb32kHz, "pcm16-swb32")}, + {FLAG_pcm16b_swb48, + std::make_pair(NetEqDecoder::kDecoderPCM16Bswb48kHz, "pcm16-swb48")}, + {FLAG_g722, std::make_pair(NetEqDecoder::kDecoderG722, "g722")}, + {FLAG_avt, std::make_pair(NetEqDecoder::kDecoderAVT, "avt")}, + {FLAG_avt_16, std::make_pair(NetEqDecoder::kDecoderAVT16kHz, "avt-16")}, + {FLAG_avt_32, + std::make_pair(NetEqDecoder::kDecoderAVT32kHz, "avt-32")}, + {FLAG_avt_48, + std::make_pair(NetEqDecoder::kDecoderAVT48kHz, "avt-48")}, + {FLAG_red, std::make_pair(NetEqDecoder::kDecoderRED, "red")}, + {FLAG_cn_nb, std::make_pair(NetEqDecoder::kDecoderCNGnb, "cng-nb")}, + {FLAG_cn_wb, std::make_pair(NetEqDecoder::kDecoderCNGwb, "cng-wb")}, + {FLAG_cn_swb32, + std::make_pair(NetEqDecoder::kDecoderCNGswb32kHz, "cng-swb32")}, + {FLAG_cn_swb48, + std::make_pair(NetEqDecoder::kDecoderCNGswb48kHz, "cng-swb48")}}; + + // Check if a replacement audio file was provided. + std::unique_ptr replacement_decoder; + NetEqTest::ExtDecoderMap ext_codecs; + if (strlen(FLAG_replacement_audio_file) > 0) { + // Find largest unused payload type. + int replacement_pt = 127; + while (!(codecs.find(replacement_pt) == codecs.end() && + ext_codecs.find(replacement_pt) == ext_codecs.end())) { + --replacement_pt; + RTC_CHECK_GE(replacement_pt, 0); + } + + auto std_set_int32_to_uint8 = [](const std::set& a) { + std::set b; + for (auto& x : a) { + b.insert(static_cast(x)); + } + return b; + }; + + std::set cn_types = std_set_int32_to_uint8( + {FLAG_cn_nb, FLAG_cn_wb, FLAG_cn_swb32, FLAG_cn_swb48}); + std::set forbidden_types = + std_set_int32_to_uint8({FLAG_g722, FLAG_red, FLAG_avt, + FLAG_avt_16, FLAG_avt_32, FLAG_avt_48}); + input.reset(new NetEqReplacementInput(std::move(input), replacement_pt, + cn_types, forbidden_types)); + + replacement_decoder.reset(new FakeDecodeFromFile( + std::unique_ptr( + new InputAudioFile(FLAG_replacement_audio_file)), + 48000, false)); + NetEqTest::ExternalDecoderInfo ext_dec_info = { + replacement_decoder.get(), NetEqDecoder::kDecoderArbitrary, + "replacement codec"}; + ext_codecs[replacement_pt] = ext_dec_info; + } + + NetEqTest::Callbacks callbacks; + std::unique_ptr delay_analyzer; + if (FLAG_matlabplot) { + delay_analyzer.reset(new NetEqDelayAnalyzer); + } + + SsrcSwitchDetector ssrc_switch_detector(delay_analyzer.get()); + callbacks.post_insert_packet = &ssrc_switch_detector; + StatsGetter stats_getter(delay_analyzer.get()); + callbacks.get_audio_callback = &stats_getter; + NetEq::Config config; + config.sample_rate_hz = *sample_rate_hz; + NetEqTest test(config, codecs, ext_codecs, std::move(input), + std::move(output), callbacks); + + int64_t test_duration_ms = test.Run(); + + if (FLAG_matlabplot) { + auto matlab_script_name = output_file_name; + std::replace(matlab_script_name.begin(), matlab_script_name.end(), '.', + '_'); + std::cout << "Creating Matlab plot script " << matlab_script_name + ".m" + << std::endl; + delay_analyzer->CreateMatlabScript(matlab_script_name + ".m"); + } + + printf("Simulation statistics:\n"); + printf(" output duration: %" PRId64 " ms\n", test_duration_ms); + auto stats = stats_getter.AverageStats(); + printf(" packet_loss_rate: %f %%\n", 100.0 * stats.packet_loss_rate); + printf(" expand_rate: %f %%\n", 100.0 * stats.expand_rate); + printf(" speech_expand_rate: %f %%\n", 100.0 * stats.speech_expand_rate); + printf(" preemptive_rate: %f %%\n", 100.0 * stats.preemptive_rate); + printf(" accelerate_rate: %f %%\n", 100.0 * stats.accelerate_rate); + printf(" secondary_decoded_rate: %f %%\n", + 100.0 * stats.secondary_decoded_rate); + printf(" secondary_discarded_rate: %f %%\n", + 100.0 * stats.secondary_discarded_rate); + printf(" clockdrift_ppm: %f ppm\n", stats.clockdrift_ppm); + printf(" mean_waiting_time_ms: %f ms\n", stats.mean_waiting_time_ms); + printf(" median_waiting_time_ms: %f ms\n", stats.median_waiting_time_ms); + printf(" min_waiting_time_ms: %f ms\n", stats.min_waiting_time_ms); + printf(" max_waiting_time_ms: %f ms\n", stats.max_waiting_time_ms); + printf(" current_buffer_size_ms: %f ms\n", stats.current_buffer_size_ms); + printf(" preferred_buffer_size_ms: %f ms\n", stats.preferred_buffer_size_ms); + + return 0; +} + +} // namespace +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + webrtc::test::RunTest(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc new file mode 100644 index 0000000000..39d9549577 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_test.h" + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" + +namespace webrtc { +namespace test { + +void DefaultNetEqTestErrorCallback::OnInsertPacketError( + const NetEqInput::PacketData& packet) { + std::cerr << "InsertPacket returned an error." << std::endl; + std::cerr << "Packet data: " << packet.ToString() << std::endl; + FATAL(); +} + +void DefaultNetEqTestErrorCallback::OnGetAudioError() { + std::cerr << "GetAudio returned an error." << std::endl; + FATAL(); +} + +NetEqTest::NetEqTest(const NetEq::Config& config, + const DecoderMap& codecs, + const ExtDecoderMap& ext_codecs, + std::unique_ptr input, + std::unique_ptr output, + Callbacks callbacks) + : neteq_(NetEq::Create(config, CreateBuiltinAudioDecoderFactory())), + input_(std::move(input)), + output_(std::move(output)), + callbacks_(callbacks), + sample_rate_hz_(config.sample_rate_hz) { + RTC_CHECK(!config.enable_muted_state) + << "The code does not handle enable_muted_state"; + RegisterDecoders(codecs); + RegisterExternalDecoders(ext_codecs); +} + +int64_t NetEqTest::Run() { + const int64_t start_time_ms = *input_->NextEventTime(); + int64_t time_now_ms = start_time_ms; + + while (!input_->ended()) { + // Advance time to next event. + RTC_DCHECK(input_->NextEventTime()); + time_now_ms = *input_->NextEventTime(); + // Check if it is time to insert packet. + if (input_->NextPacketTime() && time_now_ms >= *input_->NextPacketTime()) { + std::unique_ptr packet_data = input_->PopPacket(); + RTC_CHECK(packet_data); + int error = neteq_->InsertPacket( + packet_data->header, + rtc::ArrayView(packet_data->payload), + static_cast(packet_data->time_ms * sample_rate_hz_ / 1000)); + if (error != NetEq::kOK && callbacks_.error_callback) { + callbacks_.error_callback->OnInsertPacketError(*packet_data); + } + if (callbacks_.post_insert_packet) { + callbacks_.post_insert_packet->AfterInsertPacket(*packet_data, + neteq_.get()); + } + } + + // Check if it is time to get output audio. + if (input_->NextOutputEventTime() && + time_now_ms >= *input_->NextOutputEventTime()) { + if (callbacks_.get_audio_callback) { + callbacks_.get_audio_callback->BeforeGetAudio(neteq_.get()); + } + AudioFrame out_frame; + bool muted; + int error = neteq_->GetAudio(&out_frame, &muted); + RTC_CHECK(!muted) << "The code does not handle enable_muted_state"; + if (error != NetEq::kOK) { + if (callbacks_.error_callback) { + callbacks_.error_callback->OnGetAudioError(); + } + } else { + sample_rate_hz_ = out_frame.sample_rate_hz_; + } + if (callbacks_.get_audio_callback) { + callbacks_.get_audio_callback->AfterGetAudio(time_now_ms, out_frame, + muted, neteq_.get()); + } + + if (output_) { + RTC_CHECK(output_->WriteArray( + out_frame.data(), + out_frame.samples_per_channel_ * out_frame.num_channels_)); + } + + input_->AdvanceOutputEvent(); + } + } + return time_now_ms - start_time_ms; +} + +NetEqNetworkStatistics NetEqTest::SimulationStats() { + NetEqNetworkStatistics stats; + RTC_CHECK_EQ(neteq_->NetworkStatistics(&stats), 0); + return stats; +} + +void NetEqTest::RegisterDecoders(const DecoderMap& codecs) { + for (const auto& c : codecs) { + RTC_CHECK_EQ( + neteq_->RegisterPayloadType(c.second.first, c.second.second, c.first), + NetEq::kOK) + << "Cannot register " << c.second.second << " to payload type " + << c.first; + } +} + +void NetEqTest::RegisterExternalDecoders(const ExtDecoderMap& codecs) { + for (const auto& c : codecs) { + RTC_CHECK_EQ( + neteq_->RegisterExternalDecoder(c.second.decoder, c.second.codec, + c.second.codec_name, c.first), + NetEq::kOK) + << "Cannot register " << c.second.codec_name << " to payload type " + << c.first; + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.h new file mode 100644 index 0000000000..2c0a07caad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/neteq_test.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_H_ + +#include +#include +#include +#include + +#include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" + +namespace webrtc { +namespace test { + +class NetEqTestErrorCallback { + public: + virtual ~NetEqTestErrorCallback() = default; + virtual void OnInsertPacketError(const NetEqInput::PacketData& packet) {} + virtual void OnGetAudioError() {} +}; + +class DefaultNetEqTestErrorCallback : public NetEqTestErrorCallback { + void OnInsertPacketError(const NetEqInput::PacketData& packet) override; + void OnGetAudioError() override; +}; + +class NetEqPostInsertPacket { + public: + virtual ~NetEqPostInsertPacket() = default; + virtual void AfterInsertPacket(const NetEqInput::PacketData& packet, + NetEq* neteq) = 0; +}; + +class NetEqGetAudioCallback { + public: + virtual ~NetEqGetAudioCallback() = default; + virtual void BeforeGetAudio(NetEq* neteq) = 0; + virtual void AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool muted, + NetEq* neteq) = 0; +}; + +// Class that provides an input--output test for NetEq. The input (both packets +// and output events) is provided by a NetEqInput object, while the output is +// directed to an AudioSink object. +class NetEqTest { + public: + using DecoderMap = std::map >; + + struct ExternalDecoderInfo { + AudioDecoder* decoder; + NetEqDecoder codec; + std::string codec_name; + }; + + using ExtDecoderMap = std::map; + + struct Callbacks { + NetEqTestErrorCallback* error_callback = nullptr; + NetEqPostInsertPacket* post_insert_packet = nullptr; + NetEqGetAudioCallback* get_audio_callback = nullptr; + }; + + // Sets up the test with given configuration, codec mappings, input, ouput, + // and callback objects for error reporting. + NetEqTest(const NetEq::Config& config, + const DecoderMap& codecs, + const ExtDecoderMap& ext_codecs, + std::unique_ptr input, + std::unique_ptr output, + Callbacks callbacks); + + ~NetEqTest() = default; + + // Runs the test. Returns the duration of the produced audio in ms. + int64_t Run(); + + // Returns the statistics from NetEq. + NetEqNetworkStatistics SimulationStats(); + + private: + void RegisterDecoders(const DecoderMap& codecs); + void RegisterExternalDecoders(const ExtDecoderMap& codecs); + + std::unique_ptr neteq_; + std::unique_ptr input_; + std::unique_ptr output_; + Callbacks callbacks_; + int sample_rate_hz_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_audio_file.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_audio_file.h new file mode 100644 index 0000000000..7e65bc25db --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_audio_file.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_AUDIO_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_AUDIO_FILE_H_ + +#include +#include +#include + +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace test { + +class OutputAudioFile : public AudioSink { + public: + // Creates an OutputAudioFile, opening a file named |file_name| for writing. + // The file format is 16-bit signed host-endian PCM. + explicit OutputAudioFile(const std::string& file_name) { + out_file_ = fopen(file_name.c_str(), "wb"); + } + + virtual ~OutputAudioFile() { + if (out_file_) + fclose(out_file_); + } + + bool WriteArray(const int16_t* audio, size_t num_samples) override { + assert(out_file_); + return fwrite(audio, sizeof(*audio), num_samples, out_file_) == num_samples; + } + + private: + FILE* out_file_; + + RTC_DISALLOW_COPY_AND_ASSIGN(OutputAudioFile); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_AUDIO_FILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_wav_file.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_wav_file.h new file mode 100644 index 0000000000..031a8cb929 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/output_wav_file.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_WAV_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_WAV_FILE_H_ + +#include + +#include "common_audio/wav_file.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace test { + +class OutputWavFile : public AudioSink { + public: + // Creates an OutputWavFile, opening a file named |file_name| for writing. + // The output file is a PCM encoded wav file. + OutputWavFile(const std::string& file_name, int sample_rate_hz) + : wav_writer_(file_name, sample_rate_hz, 1) {} + + bool WriteArray(const int16_t* audio, size_t num_samples) override { + wav_writer_.WriteSamples(audio, num_samples); + return true; + } + + private: + WavWriter wav_writer_; + + RTC_DISALLOW_COPY_AND_ASSIGN(OutputWavFile); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_WAV_FILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.cc new file mode 100644 index 0000000000..71337b6e68 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.cc @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/packet.h" + +#include + +#include + +#include "modules/include/module_common_types.h" +#include "modules/rtp_rtcp/include/rtp_header_parser.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +Packet::Packet(uint8_t* packet_memory, + size_t allocated_bytes, + double time_ms, + const RtpHeaderParser& parser) + : payload_memory_(packet_memory), + payload_(NULL), + packet_length_bytes_(allocated_bytes), + payload_length_bytes_(0), + virtual_packet_length_bytes_(allocated_bytes), + virtual_payload_length_bytes_(0), + time_ms_(time_ms) { + valid_header_ = ParseHeader(parser); +} + +Packet::Packet(uint8_t* packet_memory, + size_t allocated_bytes, + size_t virtual_packet_length_bytes, + double time_ms, + const RtpHeaderParser& parser) + : payload_memory_(packet_memory), + payload_(NULL), + packet_length_bytes_(allocated_bytes), + payload_length_bytes_(0), + virtual_packet_length_bytes_(virtual_packet_length_bytes), + virtual_payload_length_bytes_(0), + time_ms_(time_ms) { + valid_header_ = ParseHeader(parser); +} + +Packet::Packet(uint8_t* packet_memory, size_t allocated_bytes, double time_ms) + : payload_memory_(packet_memory), + payload_(NULL), + packet_length_bytes_(allocated_bytes), + payload_length_bytes_(0), + virtual_packet_length_bytes_(allocated_bytes), + virtual_payload_length_bytes_(0), + time_ms_(time_ms) { + std::unique_ptr parser(RtpHeaderParser::Create()); + valid_header_ = ParseHeader(*parser); +} + +Packet::Packet(uint8_t* packet_memory, + size_t allocated_bytes, + size_t virtual_packet_length_bytes, + double time_ms) + : payload_memory_(packet_memory), + payload_(NULL), + packet_length_bytes_(allocated_bytes), + payload_length_bytes_(0), + virtual_packet_length_bytes_(virtual_packet_length_bytes), + virtual_payload_length_bytes_(0), + time_ms_(time_ms) { + std::unique_ptr parser(RtpHeaderParser::Create()); + valid_header_ = ParseHeader(*parser); +} + +Packet::~Packet() = default; + +bool Packet::ExtractRedHeaders(std::list* headers) const { + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |1| block PT | timestamp offset | block length | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |1| ... | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |0| block PT | + // +-+-+-+-+-+-+-+-+ + // + + assert(payload_); + const uint8_t* payload_ptr = payload_; + const uint8_t* payload_end_ptr = payload_ptr + payload_length_bytes_; + + // Find all RED headers with the extension bit set to 1. That is, all headers + // but the last one. + while ((payload_ptr < payload_end_ptr) && (*payload_ptr & 0x80)) { + RTPHeader* header = new RTPHeader; + CopyToHeader(header); + header->payloadType = payload_ptr[0] & 0x7F; + uint32_t offset = (payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2); + header->timestamp -= offset; + headers->push_front(header); + payload_ptr += 4; + } + // Last header. + assert(payload_ptr < payload_end_ptr); + if (payload_ptr >= payload_end_ptr) { + return false; // Payload too short. + } + RTPHeader* header = new RTPHeader; + CopyToHeader(header); + header->payloadType = payload_ptr[0] & 0x7F; + headers->push_front(header); + return true; +} + +void Packet::DeleteRedHeaders(std::list* headers) { + while (!headers->empty()) { + delete headers->front(); + headers->pop_front(); + } +} + +bool Packet::ParseHeader(const RtpHeaderParser& parser) { + bool valid_header = parser.Parse( + payload_memory_.get(), static_cast(packet_length_bytes_), &header_); + // Special case for dummy packets that have padding marked in the RTP header. + // This causes the RTP header parser to report failure, but is fine in this + // context. + const bool header_only_with_padding = + (header_.headerLength == packet_length_bytes_ && + header_.paddingLength > 0); + if (!valid_header && !header_only_with_padding) { + return false; + } + assert(header_.headerLength <= packet_length_bytes_); + payload_ = &payload_memory_[header_.headerLength]; + assert(packet_length_bytes_ >= header_.headerLength); + payload_length_bytes_ = packet_length_bytes_ - header_.headerLength; + RTC_CHECK_GE(virtual_packet_length_bytes_, packet_length_bytes_); + assert(virtual_packet_length_bytes_ >= header_.headerLength); + virtual_payload_length_bytes_ = + virtual_packet_length_bytes_ - header_.headerLength; + return true; +} + +void Packet::CopyToHeader(RTPHeader* destination) const { + destination->markerBit = header_.markerBit; + destination->payloadType = header_.payloadType; + destination->sequenceNumber = header_.sequenceNumber; + destination->timestamp = header_.timestamp; + destination->ssrc = header_.ssrc; + destination->numCSRCs = header_.numCSRCs; + destination->paddingLength = header_.paddingLength; + destination->headerLength = header_.headerLength; + destination->payload_type_frequency = header_.payload_type_frequency; + memcpy(&destination->arrOfCSRCs, + &header_.arrOfCSRCs, + sizeof(header_.arrOfCSRCs)); + memcpy( + &destination->extension, &header_.extension, sizeof(header_.extension)); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.h new file mode 100644 index 0000000000..a72f7d1899 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_H_ + +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class RtpHeaderParser; + +namespace test { + +// Class for handling RTP packets in test applications. +class Packet { + public: + // Creates a packet, with the packet payload (including header bytes) in + // |packet_memory|. The length of |packet_memory| is |allocated_bytes|. + // The new object assumes ownership of |packet_memory| and will delete it + // when the Packet object is deleted. The |time_ms| is an extra time + // associated with this packet, typically used to denote arrival time. + // The first bytes in |packet_memory| will be parsed using |parser|. + Packet(uint8_t* packet_memory, + size_t allocated_bytes, + double time_ms, + const RtpHeaderParser& parser); + + // Same as above, but with the extra argument |virtual_packet_length_bytes|. + // This is typically used when reading RTP dump files that only contain the + // RTP headers, and no payload (a.k.a RTP dummy files or RTP light). The + // |virtual_packet_length_bytes| tells what size the packet had on wire, + // including the now discarded payload, whereas |allocated_bytes| is the + // length of the remaining payload (typically only the RTP header). + Packet(uint8_t* packet_memory, + size_t allocated_bytes, + size_t virtual_packet_length_bytes, + double time_ms, + const RtpHeaderParser& parser); + + // The following two constructors are the same as above, but without a + // parser. Note that when the object is constructed using any of these + // methods, the header will be parsed using a default RtpHeaderParser object. + // In particular, RTP header extensions won't be parsed. + Packet(uint8_t* packet_memory, size_t allocated_bytes, double time_ms); + + Packet(uint8_t* packet_memory, + size_t allocated_bytes, + size_t virtual_packet_length_bytes, + double time_ms); + + virtual ~Packet(); + + // Parses the first bytes of the RTP payload, interpreting them as RED headers + // according to RFC 2198. The headers will be inserted into |headers|. The + // caller of the method assumes ownership of the objects in the list, and + // must delete them properly. + bool ExtractRedHeaders(std::list* headers) const; + + // Deletes all RTPHeader objects in |headers|, but does not delete |headers| + // itself. + static void DeleteRedHeaders(std::list* headers); + + const uint8_t* payload() const { return payload_; } + + size_t packet_length_bytes() const { return packet_length_bytes_; } + + size_t payload_length_bytes() const { return payload_length_bytes_; } + + size_t virtual_packet_length_bytes() const { + return virtual_packet_length_bytes_; + } + + size_t virtual_payload_length_bytes() const { + return virtual_payload_length_bytes_; + } + + const RTPHeader& header() const { return header_; } + + void set_time_ms(double time) { time_ms_ = time; } + double time_ms() const { return time_ms_; } + bool valid_header() const { return valid_header_; } + + private: + bool ParseHeader(const RtpHeaderParser& parser); + void CopyToHeader(RTPHeader* destination) const; + + RTPHeader header_; + std::unique_ptr payload_memory_; + const uint8_t* payload_; // First byte after header. + const size_t packet_length_bytes_; // Total length of packet. + size_t payload_length_bytes_; // Length of the payload, after RTP header. + // Zero for dummy RTP packets. + // Virtual lengths are used when parsing RTP header files (dummy RTP files). + const size_t virtual_packet_length_bytes_; + size_t virtual_payload_length_bytes_; + double time_ms_; // Used to denote a packet's arrival time. + bool valid_header_; // Set by the RtpHeaderParser. + + RTC_DISALLOW_COPY_AND_ASSIGN(Packet); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.cc new file mode 100644 index 0000000000..30bf431835 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.cc @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/packet_source.h" + +namespace webrtc { +namespace test { + +PacketSource::PacketSource() : use_ssrc_filter_(false), ssrc_(0) {} + +PacketSource::~PacketSource() = default; + +void PacketSource::FilterOutPayloadType(uint8_t payload_type) { + filter_.set(payload_type, true); +} + +void PacketSource::SelectSsrc(uint32_t ssrc) { + use_ssrc_filter_ = true; + ssrc_ = ssrc; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.h new file mode 100644 index 0000000000..3f98ba16a9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_source.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_SOURCE_H_ + +#include +#include + +#include "modules/audio_coding/neteq/tools/packet.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +// Interface class for an object delivering RTP packets to test applications. +class PacketSource { + public: + PacketSource(); + virtual ~PacketSource(); + + // Returns next packet. Returns nullptr if the source is depleted, or if an + // error occurred. + virtual std::unique_ptr NextPacket() = 0; + + virtual void FilterOutPayloadType(uint8_t payload_type); + + virtual void SelectSsrc(uint32_t ssrc); + + protected: + std::bitset<128> filter_; // Payload type is 7 bits in the RFC. + // If SSRC filtering discards all packet that do not match the SSRC. + bool use_ssrc_filter_; // True when SSRC filtering is active. + uint32_t ssrc_; // The selected SSRC. All other SSRCs will be discarded. + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(PacketSource); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_SOURCE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_unittest.cc new file mode 100644 index 0000000000..ce6a3b9697 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/packet_unittest.cc @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for test Packet class. + +#include "modules/audio_coding/neteq/tools/packet.h" + +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +const int kHeaderLengthBytes = 12; + +void MakeRtpHeader(int payload_type, + int seq_number, + uint32_t timestamp, + uint32_t ssrc, + uint8_t* rtp_data) { + rtp_data[0] = 0x80; + rtp_data[1] = static_cast(payload_type); + rtp_data[2] = (seq_number >> 8) & 0xFF; + rtp_data[3] = (seq_number) & 0xFF; + rtp_data[4] = timestamp >> 24; + rtp_data[5] = (timestamp >> 16) & 0xFF; + rtp_data[6] = (timestamp >> 8) & 0xFF; + rtp_data[7] = timestamp & 0xFF; + rtp_data[8] = ssrc >> 24; + rtp_data[9] = (ssrc >> 16) & 0xFF; + rtp_data[10] = (ssrc >> 8) & 0xFF; + rtp_data[11] = ssrc & 0xFF; +} +} // namespace + +TEST(TestPacket, RegularPacket) { + const size_t kPacketLengthBytes = 100; + uint8_t* packet_memory = new uint8_t[kPacketLengthBytes]; + const uint8_t kPayloadType = 17; + const uint16_t kSequenceNumber = 4711; + const uint32_t kTimestamp = 47114711; + const uint32_t kSsrc = 0x12345678; + MakeRtpHeader( + kPayloadType, kSequenceNumber, kTimestamp, kSsrc, packet_memory); + const double kPacketTime = 1.0; + // Hand over ownership of |packet_memory| to |packet|. + Packet packet(packet_memory, kPacketLengthBytes, kPacketTime); + ASSERT_TRUE(packet.valid_header()); + EXPECT_EQ(kPayloadType, packet.header().payloadType); + EXPECT_EQ(kSequenceNumber, packet.header().sequenceNumber); + EXPECT_EQ(kTimestamp, packet.header().timestamp); + EXPECT_EQ(kSsrc, packet.header().ssrc); + EXPECT_EQ(0, packet.header().numCSRCs); + EXPECT_EQ(kPacketLengthBytes, packet.packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.payload_length_bytes()); + EXPECT_EQ(kPacketLengthBytes, packet.virtual_packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.virtual_payload_length_bytes()); + EXPECT_EQ(kPacketTime, packet.time_ms()); +} + +TEST(TestPacket, DummyPacket) { + const size_t kPacketLengthBytes = kHeaderLengthBytes; // Only RTP header. + const size_t kVirtualPacketLengthBytes = 100; + uint8_t* packet_memory = new uint8_t[kPacketLengthBytes]; + const uint8_t kPayloadType = 17; + const uint16_t kSequenceNumber = 4711; + const uint32_t kTimestamp = 47114711; + const uint32_t kSsrc = 0x12345678; + MakeRtpHeader( + kPayloadType, kSequenceNumber, kTimestamp, kSsrc, packet_memory); + const double kPacketTime = 1.0; + // Hand over ownership of |packet_memory| to |packet|. + Packet packet(packet_memory, + kPacketLengthBytes, + kVirtualPacketLengthBytes, + kPacketTime); + ASSERT_TRUE(packet.valid_header()); + EXPECT_EQ(kPayloadType, packet.header().payloadType); + EXPECT_EQ(kSequenceNumber, packet.header().sequenceNumber); + EXPECT_EQ(kTimestamp, packet.header().timestamp); + EXPECT_EQ(kSsrc, packet.header().ssrc); + EXPECT_EQ(0, packet.header().numCSRCs); + EXPECT_EQ(kPacketLengthBytes, packet.packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.payload_length_bytes()); + EXPECT_EQ(kVirtualPacketLengthBytes, packet.virtual_packet_length_bytes()); + EXPECT_EQ(kVirtualPacketLengthBytes - kHeaderLengthBytes, + packet.virtual_payload_length_bytes()); + EXPECT_EQ(kPacketTime, packet.time_ms()); +} + +namespace { +// Writes one RED block header starting at |rtp_data|, according to RFC 2198. +// returns the number of bytes written (1 or 4). +// +// Format if |last_payoad| is false: +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |1| block PT | timestamp offset | block length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Format if |last_payoad| is true: +// 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +// |0| Block PT | +// +-+-+-+-+-+-+-+-+ + +int MakeRedHeader(int payload_type, + uint32_t timestamp_offset, + int block_length, + bool last_payload, + uint8_t* rtp_data) { + rtp_data[0] = 0x80 | (payload_type & 0x7F); // Set the first bit to 1. + if (last_payload) { + rtp_data[0] &= 0x7F; // Reset the first but to 0 to indicate last block. + return 1; + } + rtp_data[1] = timestamp_offset >> 6; + rtp_data[2] = (timestamp_offset & 0x3F) << 2; + rtp_data[2] |= block_length >> 8; + rtp_data[3] = block_length & 0xFF; + return 4; +} +} // namespace + +TEST(TestPacket, RED) { + const size_t kPacketLengthBytes = 100; + uint8_t* packet_memory = new uint8_t[kPacketLengthBytes]; + const uint8_t kRedPayloadType = 17; + const uint16_t kSequenceNumber = 4711; + const uint32_t kTimestamp = 47114711; + const uint32_t kSsrc = 0x12345678; + MakeRtpHeader( + kRedPayloadType, kSequenceNumber, kTimestamp, kSsrc, packet_memory); + // Create four RED headers. + // Payload types are just the same as the block index the offset is 100 times + // the block index. + const int kRedBlocks = 4; + uint8_t* payload_ptr = + &packet_memory[kHeaderLengthBytes]; // First byte after header. + for (int i = 0; i < kRedBlocks; ++i) { + int payload_type = i; + // Offset value is not used for the last block. + uint32_t timestamp_offset = 100 * i; + int block_length = 10 * i; + bool last_block = (i == kRedBlocks - 1) ? true : false; + payload_ptr += MakeRedHeader( + payload_type, timestamp_offset, block_length, last_block, payload_ptr); + } + const double kPacketTime = 1.0; + // Hand over ownership of |packet_memory| to |packet|. + Packet packet(packet_memory, kPacketLengthBytes, kPacketTime); + ASSERT_TRUE(packet.valid_header()); + EXPECT_EQ(kRedPayloadType, packet.header().payloadType); + EXPECT_EQ(kSequenceNumber, packet.header().sequenceNumber); + EXPECT_EQ(kTimestamp, packet.header().timestamp); + EXPECT_EQ(kSsrc, packet.header().ssrc); + EXPECT_EQ(0, packet.header().numCSRCs); + EXPECT_EQ(kPacketLengthBytes, packet.packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.payload_length_bytes()); + EXPECT_EQ(kPacketLengthBytes, packet.virtual_packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.virtual_payload_length_bytes()); + EXPECT_EQ(kPacketTime, packet.time_ms()); + std::list red_headers; + EXPECT_TRUE(packet.ExtractRedHeaders(&red_headers)); + EXPECT_EQ(kRedBlocks, static_cast(red_headers.size())); + int block_index = 0; + for (std::list::reverse_iterator it = red_headers.rbegin(); + it != red_headers.rend(); + ++it) { + // Reading list from the back, since the extraction puts the main payload + // (which is the last one on wire) first. + RTPHeader* red_block = *it; + EXPECT_EQ(block_index, red_block->payloadType); + EXPECT_EQ(kSequenceNumber, red_block->sequenceNumber); + if (block_index == kRedBlocks - 1) { + // Last block has zero offset per definition. + EXPECT_EQ(kTimestamp, red_block->timestamp); + } else { + EXPECT_EQ(kTimestamp - 100 * block_index, red_block->timestamp); + } + EXPECT_EQ(kSsrc, red_block->ssrc); + EXPECT_EQ(0, red_block->numCSRCs); + ++block_index; + } + Packet::DeleteRedHeaders(&red_headers); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc new file mode 100644 index 0000000000..5050e1fb17 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +bool ResampleInputAudioFile::Read(size_t samples, + int output_rate_hz, + int16_t* destination) { + const size_t samples_to_read = samples * file_rate_hz_ / output_rate_hz; + RTC_CHECK_EQ(samples_to_read * output_rate_hz, samples * file_rate_hz_) + << "Frame size and sample rates don't add up to an integer."; + std::unique_ptr temp_destination(new int16_t[samples_to_read]); + if (!InputAudioFile::Read(samples_to_read, temp_destination.get())) + return false; + resampler_.ResetIfNeeded(file_rate_hz_, output_rate_hz, 1); + size_t output_length = 0; + RTC_CHECK_EQ(resampler_.Push(temp_destination.get(), samples_to_read, + destination, samples, output_length), + 0); + RTC_CHECK_EQ(samples, output_length); + return true; +} + +bool ResampleInputAudioFile::Read(size_t samples, int16_t* destination) { + RTC_CHECK_GT(output_rate_hz_, 0) << "Output rate not set."; + return Read(samples, output_rate_hz_, destination); +} + +void ResampleInputAudioFile::set_output_rate_hz(int rate_hz) { + output_rate_hz_ = rate_hz; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h new file mode 100644 index 0000000000..13c419d969 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RESAMPLE_INPUT_AUDIO_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RESAMPLE_INPUT_AUDIO_FILE_H_ + +#include + +#include "common_audio/resampler/include/resampler.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +// Class for handling a looping input audio file with resampling. +class ResampleInputAudioFile : public InputAudioFile { + public: + ResampleInputAudioFile(const std::string file_name, int file_rate_hz) + : InputAudioFile(file_name), + file_rate_hz_(file_rate_hz), + output_rate_hz_(-1) {} + ResampleInputAudioFile(const std::string file_name, + int file_rate_hz, + int output_rate_hz) + : InputAudioFile(file_name), + file_rate_hz_(file_rate_hz), + output_rate_hz_(output_rate_hz) {} + + bool Read(size_t samples, int output_rate_hz, int16_t* destination); + bool Read(size_t samples, int16_t* destination) override; + void set_output_rate_hz(int rate_hz); + + private: + const int file_rate_hz_; + int output_rate_hz_; + Resampler resampler_; + RTC_DISALLOW_COPY_AND_ASSIGN(ResampleInputAudioFile); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RESAMPLE_INPUT_AUDIO_FILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc new file mode 100644 index 0000000000..e2c9ac9371 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/rtc_event_log_source.h" + +#include +#include +#include +#include + +#include "call/call.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/rtp_rtcp/include/rtp_header_parser.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +RtcEventLogSource* RtcEventLogSource::Create(const std::string& file_name) { + RtcEventLogSource* source = new RtcEventLogSource(); + RTC_CHECK(source->OpenFile(file_name)); + return source; +} + +RtcEventLogSource::~RtcEventLogSource() {} + +bool RtcEventLogSource::RegisterRtpHeaderExtension(RTPExtensionType type, + uint8_t id) { + RTC_CHECK(parser_.get()); + return parser_->RegisterRtpHeaderExtension(type, id); +} + +std::unique_ptr RtcEventLogSource::NextPacket() { + for (; rtp_packet_index_ < parsed_stream_.GetNumberOfEvents(); + rtp_packet_index_++) { + if (parsed_stream_.GetEventType(rtp_packet_index_) == + ParsedRtcEventLog::RTP_EVENT) { + PacketDirection direction; + size_t header_length; + size_t packet_length; + uint64_t timestamp_us = parsed_stream_.GetTimestamp(rtp_packet_index_); + parsed_stream_.GetRtpHeader(rtp_packet_index_, &direction, nullptr, + &header_length, &packet_length, nullptr); + + if (direction != kIncomingPacket) { + continue; + } + + uint8_t* packet_header = new uint8_t[header_length]; + parsed_stream_.GetRtpHeader(rtp_packet_index_, nullptr, packet_header, + nullptr, nullptr, nullptr); + std::unique_ptr packet( + new Packet(packet_header, header_length, packet_length, + static_cast(timestamp_us) / 1000, *parser_.get())); + + if (!packet->valid_header()) { + std::cout << "Warning: Packet with index " << rtp_packet_index_ + << " has an invalid header and will be ignored." << std::endl; + continue; + } + + if (parsed_stream_.GetMediaType(packet->header().ssrc, direction) != + webrtc::ParsedRtcEventLog::MediaType::AUDIO) { + continue; + } + + // Check if the packet should not be filtered out. + if (!filter_.test(packet->header().payloadType) && + !(use_ssrc_filter_ && packet->header().ssrc != ssrc_)) { + ++rtp_packet_index_; + return packet; + } + } + } + return nullptr; +} + +int64_t RtcEventLogSource::NextAudioOutputEventMs() { + while (audio_output_index_ < parsed_stream_.GetNumberOfEvents()) { + if (parsed_stream_.GetEventType(audio_output_index_) == + ParsedRtcEventLog::AUDIO_PLAYOUT_EVENT) { + uint64_t timestamp_us = parsed_stream_.GetTimestamp(audio_output_index_); + // We call GetAudioPlayout only to check that the protobuf event is + // well-formed. + parsed_stream_.GetAudioPlayout(audio_output_index_, nullptr); + audio_output_index_++; + return timestamp_us / 1000; + } + audio_output_index_++; + } + return std::numeric_limits::max(); +} + +RtcEventLogSource::RtcEventLogSource() + : PacketSource(), parser_(RtpHeaderParser::Create()) {} + +bool RtcEventLogSource::OpenFile(const std::string& file_name) { + return parsed_stream_.ParseFile(file_name); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h new file mode 100644 index 0000000000..df01e06535 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RTC_EVENT_LOG_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RTC_EVENT_LOG_SOURCE_H_ + +#include +#include + +#include "logging/rtc_event_log/rtc_event_log_parser.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class RtpHeaderParser; + +namespace test { + +class Packet; + +class RtcEventLogSource : public PacketSource { + public: + // Creates an RtcEventLogSource reading from |file_name|. If the file cannot + // be opened, or has the wrong format, NULL will be returned. + static RtcEventLogSource* Create(const std::string& file_name); + + virtual ~RtcEventLogSource(); + + // Registers an RTP header extension and binds it to |id|. + virtual bool RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id); + + std::unique_ptr NextPacket() override; + + // Returns the timestamp of the next audio output event, in milliseconds. The + // maximum value of int64_t is returned if there are no more audio output + // events available. + int64_t NextAudioOutputEventMs(); + + private: + RtcEventLogSource(); + + bool OpenFile(const std::string& file_name); + + size_t rtp_packet_index_ = 0; + size_t audio_output_index_ = 0; + + ParsedRtcEventLog parsed_stream_; + std::unique_ptr parser_; + + RTC_DISALLOW_COPY_AND_ASSIGN(RtcEventLogSource); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RTC_EVENT_LOG_SOURCE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc new file mode 100644 index 0000000000..12721ccc96 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include +#include + +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "rtc_base/flags.h" + +// Define command line flags. +DEFINE_int(red, 117, "RTP payload type for RED"); +DEFINE_int(audio_level, -1, "Extension ID for audio level (RFC 6464); " + "-1 not to print audio level"); +DEFINE_int(abs_send_time, -1, "Extension ID for absolute sender time; " + "-1 not to print absolute send time"); +DEFINE_bool(help, false, "Print this message"); + +int main(int argc, char* argv[]) { + std::string program_name = argv[0]; + std::string usage = + "Tool for parsing an RTP dump file to text output.\n" + "Run " + + program_name + + " --help for usage.\n" + "Example usage:\n" + + program_name + " input.rtp output.txt\n\n" + + "Output is sent to stdout if no output file is given. " + + "Note that this tool can read files with or without payloads.\n"; + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || + FLAG_help || (argc != 2 && argc != 3)) { + printf("%s", usage.c_str()); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + + RTC_CHECK(FLAG_red >= 0 && FLAG_red <= 127); // Payload type + RTC_CHECK(FLAG_audio_level == -1 || // Default + (FLAG_audio_level > 0 && FLAG_audio_level <= 255)); // Extension ID + RTC_CHECK(FLAG_abs_send_time == -1 || // Default + (FLAG_abs_send_time > 0 && FLAG_abs_send_time <= 255)); // Extension ID + + printf("Input file: %s\n", argv[1]); + std::unique_ptr file_source( + webrtc::test::RtpFileSource::Create(argv[1])); + assert(file_source.get()); + // Set RTP extension IDs. + bool print_audio_level = false; + if (FLAG_audio_level != -1) { + print_audio_level = true; + file_source->RegisterRtpHeaderExtension(webrtc::kRtpExtensionAudioLevel, + FLAG_audio_level); + } + bool print_abs_send_time = false; + if (FLAG_abs_send_time != -1) { + print_abs_send_time = true; + file_source->RegisterRtpHeaderExtension( + webrtc::kRtpExtensionAbsoluteSendTime, FLAG_abs_send_time); + } + + FILE* out_file; + if (argc == 3) { + out_file = fopen(argv[2], "wt"); + if (!out_file) { + printf("Cannot open output file %s\n", argv[2]); + return -1; + } + printf("Output file: %s\n\n", argv[2]); + } else { + out_file = stdout; + } + + // Print file header. + fprintf(out_file, "SeqNo TimeStamp SendTime Size PT M SSRC"); + if (print_audio_level) { + fprintf(out_file, " AuLvl (V)"); + } + if (print_abs_send_time) { + fprintf(out_file, " AbsSendTime"); + } + fprintf(out_file, "\n"); + + uint32_t max_abs_send_time = 0; + int cycles = -1; + std::unique_ptr packet; + while (true) { + packet = file_source->NextPacket(); + if (!packet.get()) { + // End of file reached. + break; + } + // Write packet data to file. Use virtual_packet_length_bytes so that the + // correct packet sizes are printed also for RTP header-only dumps. + fprintf(out_file, + "%5u %10u %10u %5i %5i %2i %#08X", + packet->header().sequenceNumber, + packet->header().timestamp, + static_cast(packet->time_ms()), + static_cast(packet->virtual_packet_length_bytes()), + packet->header().payloadType, + packet->header().markerBit, + packet->header().ssrc); + if (print_audio_level && packet->header().extension.hasAudioLevel) { + fprintf(out_file, + " %5u (%1i)", + packet->header().extension.audioLevel, + packet->header().extension.voiceActivity); + } + if (print_abs_send_time && packet->header().extension.hasAbsoluteSendTime) { + if (cycles == -1) { + // Initialize. + max_abs_send_time = packet->header().extension.absoluteSendTime; + cycles = 0; + } + // Abs sender time is 24 bit 6.18 fixed point. Shift by 8 to normalize to + // 32 bits (unsigned). Calculate the difference between this packet's + // send time and the maximum observed. Cast to signed 32-bit to get the + // desired wrap-around behavior. + if (static_cast( + (packet->header().extension.absoluteSendTime << 8) - + (max_abs_send_time << 8)) >= 0) { + // The difference is non-negative, meaning that this packet is newer + // than the previously observed maximum absolute send time. + if (packet->header().extension.absoluteSendTime < max_abs_send_time) { + // Wrap detected. + cycles++; + } + max_abs_send_time = packet->header().extension.absoluteSendTime; + } + // Abs sender time is 24 bit 6.18 fixed point. Divide by 2^18 to convert + // to floating point representation. + double send_time_seconds = + static_cast(packet->header().extension.absoluteSendTime) / + 262144 + + 64.0 * cycles; + fprintf(out_file, " %11f", send_time_seconds); + } + fprintf(out_file, "\n"); + + if (packet->header().payloadType == FLAG_red) { + std::list red_headers; + packet->ExtractRedHeaders(&red_headers); + while (!red_headers.empty()) { + webrtc::RTPHeader* red = red_headers.front(); + assert(red); + fprintf(out_file, + "* %5u %10u %10u %5i\n", + red->sequenceNumber, + red->timestamp, + static_cast(packet->time_ms()), + red->payloadType); + red_headers.pop_front(); + delete red; + } + } + } + + fclose(out_file); + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_encode.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_encode.cc new file mode 100644 index 0000000000..ddf6ed89c3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_encode.cc @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#ifdef WIN32 +#include +#endif +#ifdef WEBRTC_LINUX +#include +#endif + +#include +#include +#include + +#include "api/audio_codecs/L16/audio_encoder_L16.h" +#include "api/audio_codecs/g711/audio_encoder_g711.h" +#include "api/audio_codecs/g722/audio_encoder_g722.h" +#include "api/audio_codecs/ilbc/audio_encoder_ilbc.h" +#include "api/audio_codecs/isac/audio_encoder_isac.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "rtc_base/flags.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/ptr_util.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { +namespace { + +// Define command line flags. +DEFINE_bool(list_codecs, false, "Enumerate all codecs"); +DEFINE_string(codec, "opus", "Codec to use"); +DEFINE_int(frame_len, 0, "Frame length in ms; 0 indicates codec default value"); +DEFINE_int(bitrate, 0, "Bitrate in kbps; 0 indicates codec default value"); +DEFINE_int(payload_type, + -1, + "RTP payload type; -1 indicates codec default value"); +DEFINE_int(cng_payload_type, + -1, + "RTP payload type for CNG; -1 indicates default value"); +DEFINE_int(ssrc, 0, "SSRC to write to the RTP header"); +DEFINE_bool(dtx, false, "Use DTX/CNG"); +DEFINE_int(sample_rate, 48000, "Sample rate of the input file"); +DEFINE_bool(help, false, "Print this message"); + +// Add new codecs here, and to the map below. +enum class CodecType { + kOpus, + kPcmU, + kPcmA, + kG722, + kPcm16b8, + kPcm16b16, + kPcm16b32, + kPcm16b48, + kIlbc, + kIsac +}; + +struct CodecTypeAndInfo { + CodecType type; + int default_payload_type; + bool internal_dtx; +}; + +// List all supported codecs here. This map defines the command-line parameter +// value (the key string) for selecting each codec, together with information +// whether it is using internal or external DTX/CNG. +const std::map& CodecList() { + static const auto* const codec_list = + new std::map{ + {"opus", {CodecType::kOpus, 111, true}}, + {"pcmu", {CodecType::kPcmU, 0, false}}, + {"pcma", {CodecType::kPcmA, 8, false}}, + {"g722", {CodecType::kG722, 9, false}}, + {"pcm16b_8", {CodecType::kPcm16b8, 93, false}}, + {"pcm16b_16", {CodecType::kPcm16b16, 94, false}}, + {"pcm16b_32", {CodecType::kPcm16b32, 95, false}}, + {"pcm16b_48", {CodecType::kPcm16b48, 96, false}}, + {"ilbc", {CodecType::kIlbc, 102, false}}, + {"isac", {CodecType::kIsac, 103, false}}}; + return *codec_list; +} + +// This class will receive callbacks from ACM when a packet is ready, and write +// it to the output file. +class Packetizer : public AudioPacketizationCallback { + public: + Packetizer(FILE* out_file, uint32_t ssrc, int timestamp_rate_hz) + : out_file_(out_file), + ssrc_(ssrc), + timestamp_rate_hz_(timestamp_rate_hz) {} + + int32_t SendData(FrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + const RTPFragmentationHeader* fragmentation) override { + RTC_CHECK(!fragmentation); + if (payload_len_bytes == 0) { + return 0; + } + + constexpr size_t kRtpHeaderLength = 12; + constexpr size_t kRtpDumpHeaderLength = 8; + const uint16_t length = htons(rtc::checked_cast( + kRtpHeaderLength + kRtpDumpHeaderLength + payload_len_bytes)); + const uint16_t plen = htons( + rtc::checked_cast(kRtpHeaderLength + payload_len_bytes)); + const uint32_t offset = htonl(timestamp / (timestamp_rate_hz_ / 1000)); + RTC_CHECK_EQ(fwrite(&length, sizeof(uint16_t), 1, out_file_), 1); + RTC_CHECK_EQ(fwrite(&plen, sizeof(uint16_t), 1, out_file_), 1); + RTC_CHECK_EQ(fwrite(&offset, sizeof(uint32_t), 1, out_file_), 1); + + const uint8_t rtp_header[] = {0x80, + static_cast(payload_type & 0x7F), + static_cast(sequence_number_ >> 8), + static_cast(sequence_number_), + static_cast(timestamp >> 24), + static_cast(timestamp >> 16), + static_cast(timestamp >> 8), + static_cast(timestamp), + static_cast(ssrc_ >> 24), + static_cast(ssrc_ >> 16), + static_cast(ssrc_ >> 8), + static_cast(ssrc_)}; + static_assert(sizeof(rtp_header) == kRtpHeaderLength, ""); + RTC_CHECK_EQ( + fwrite(rtp_header, sizeof(uint8_t), kRtpHeaderLength, out_file_), + kRtpHeaderLength); + ++sequence_number_; // Intended to wrap on overflow. + + RTC_CHECK_EQ( + fwrite(payload_data, sizeof(uint8_t), payload_len_bytes, out_file_), + payload_len_bytes); + + return 0; + } + + private: + FILE* const out_file_; + const uint32_t ssrc_; + const int timestamp_rate_hz_; + uint16_t sequence_number_ = 0; +}; + +void SetFrameLenIfFlagIsPositive(int* config_frame_len) { + if (FLAG_frame_len > 0) { + *config_frame_len = FLAG_frame_len; + } +} + +AudioEncoderL16::Config Pcm16bConfig(CodecType codec_type) { + AudioEncoderL16::Config config; + SetFrameLenIfFlagIsPositive(&config.frame_size_ms); + switch (codec_type) { + case CodecType::kPcm16b8: + config.sample_rate_hz = 8000; + return config; + case CodecType::kPcm16b16: + config.sample_rate_hz = 16000; + return config; + case CodecType::kPcm16b32: + config.sample_rate_hz = 32000; + return config; + case CodecType::kPcm16b48: + config.sample_rate_hz = 48000; + return config; + default: + RTC_NOTREACHED(); + return config; + } +} + +std::unique_ptr CreateEncoder(CodecType codec_type, + int payload_type) { + switch (codec_type) { + case CodecType::kOpus: { + AudioEncoderOpusConfig config; + if (FLAG_bitrate > 0) { + config.bitrate_bps = FLAG_bitrate; + } + config.dtx_enabled = FLAG_dtx; + SetFrameLenIfFlagIsPositive(&config.frame_size_ms); + RTC_CHECK(config.IsOk()); + return AudioEncoderOpus::MakeAudioEncoder(config, payload_type); + } + + case CodecType::kPcmU: + case CodecType::kPcmA: { + AudioEncoderG711::Config config; + SetFrameLenIfFlagIsPositive(&config.frame_size_ms); + config.type = codec_type == CodecType::kPcmU + ? AudioEncoderG711::Config::Type::kPcmU + : AudioEncoderG711::Config::Type::kPcmA; + RTC_CHECK(config.IsOk()); + return AudioEncoderG711::MakeAudioEncoder(config, payload_type); + } + + case CodecType::kG722: { + AudioEncoderG722Config config; + SetFrameLenIfFlagIsPositive(&config.frame_size_ms); + RTC_CHECK(config.IsOk()); + return AudioEncoderG722::MakeAudioEncoder(config, payload_type); + } + + case CodecType::kPcm16b8: + case CodecType::kPcm16b16: + case CodecType::kPcm16b32: + case CodecType::kPcm16b48: { + return AudioEncoderL16::MakeAudioEncoder(Pcm16bConfig(codec_type), + payload_type); + } + + case CodecType::kIlbc: { + AudioEncoderIlbcConfig config; + SetFrameLenIfFlagIsPositive(&config.frame_size_ms); + RTC_CHECK(config.IsOk()); + return AudioEncoderIlbc::MakeAudioEncoder(config, payload_type); + } + + case CodecType::kIsac: { + AudioEncoderIsac::Config config; + SetFrameLenIfFlagIsPositive(&config.frame_size_ms); + RTC_CHECK(config.IsOk()); + return AudioEncoderIsac::MakeAudioEncoder(config, payload_type); + } + } + RTC_NOTREACHED(); + return nullptr; +} + +AudioEncoderCng::Config GetCngConfig(int sample_rate_hz) { + AudioEncoderCng::Config cng_config; + const auto default_payload_type = [&] { + switch (sample_rate_hz) { + case 8000: return 13; + case 16000: return 98; + case 32000: return 99; + case 48000: return 100; + default: RTC_NOTREACHED(); + } + return 0; + }; + cng_config.payload_type = FLAG_cng_payload_type != -1 + ? FLAG_cng_payload_type + : default_payload_type(); + return cng_config; +} + +int RunRtpEncode(int argc, char* argv[]) { + const std::string program_name = argv[0]; + const std::string usage = + "Tool for generating an RTP dump file from audio input.\n" + "Run " + + program_name + + " --help for usage.\n" + "Example usage:\n" + + program_name + " input.pcm output.rtp --codec=[codec] " + + "--frame_len=[frame_len] --bitrate=[bitrate]\n\n"; + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || FLAG_help || + (!FLAG_list_codecs && argc != 3)) { + printf("%s", usage.c_str()); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + + if (FLAG_list_codecs) { + printf("The following arguments are valid --codec parameters:\n"); + for (const auto& c : CodecList()) { + printf(" %s\n", c.first.c_str()); + } + return 0; + } + + const auto codec_it = CodecList().find(FLAG_codec); + if (codec_it == CodecList().end()) { + printf("%s is not a valid codec name.\n", FLAG_codec); + printf("Use argument --list_codecs to see all valid codec names.\n"); + return 1; + } + + // Create the codec. + const int payload_type = FLAG_payload_type == -1 + ? codec_it->second.default_payload_type + : FLAG_payload_type; + std::unique_ptr codec = + CreateEncoder(codec_it->second.type, payload_type); + + // Create an external VAD/CNG encoder if needed. + if (FLAG_dtx && !codec_it->second.internal_dtx) { + AudioEncoderCng::Config cng_config = GetCngConfig(codec->SampleRateHz()); + RTC_DCHECK(codec); + cng_config.speech_encoder = std::move(codec); + codec = rtc::MakeUnique(std::move(cng_config)); + } + RTC_DCHECK(codec); + + // Set up ACM. + const int timestamp_rate_hz = codec->RtpTimestampRateHz(); + AudioCodingModule::Config config; + std::unique_ptr acm(AudioCodingModule::Create(config)); + acm->SetEncoder(std::move(codec)); + + // Open files. + printf("Input file: %s\n", argv[1]); + InputAudioFile input_file(argv[1], false); // Open input in non-looping mode. + FILE* out_file = fopen(argv[2], "wb"); + RTC_CHECK(out_file) << "Could not open file " << argv[2] << " for writing"; + printf("Output file: %s\n", argv[2]); + fprintf(out_file, "#!rtpplay1.0 \n"); //, + // Write 3 32-bit values followed by 2 16-bit values, all set to 0. This means + // a total of 16 bytes. + const uint8_t file_header[16] = {0}; + RTC_CHECK_EQ(fwrite(file_header, sizeof(file_header), 1, out_file), 1); + + // Create and register the packetizer, which will write the packets to file. + Packetizer packetizer(out_file, FLAG_ssrc, timestamp_rate_hz); + RTC_DCHECK_EQ(acm->RegisterTransportCallback(&packetizer), 0); + + AudioFrame audio_frame; + audio_frame.samples_per_channel_ = FLAG_sample_rate / 100; // 10 ms + audio_frame.sample_rate_hz_ = FLAG_sample_rate; + audio_frame.num_channels_ = 1; + + while (input_file.Read(audio_frame.samples_per_channel_, + audio_frame.mutable_data())) { + RTC_CHECK_GE(acm->Add10MsData(audio_frame), 0); + audio_frame.timestamp_ += audio_frame.samples_per_channel_; + } + + return 0; +} + +} // namespace +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::RunRtpEncode(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc new file mode 100644 index 0000000000..c9ae5f2e71 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" + +#include +#include +#ifdef WIN32 +#include +#else +#include +#endif + +#include + +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/rtp_rtcp/include/rtp_header_parser.h" +#include "rtc_base/checks.h" +#include "test/rtp_file_reader.h" + +namespace webrtc { +namespace test { + +RtpFileSource* RtpFileSource::Create(const std::string& file_name) { + RtpFileSource* source = new RtpFileSource(); + RTC_CHECK(source->OpenFile(file_name)); + return source; +} + +bool RtpFileSource::ValidRtpDump(const std::string& file_name) { + std::unique_ptr temp_file( + RtpFileReader::Create(RtpFileReader::kRtpDump, file_name)); + return !!temp_file; +} + +bool RtpFileSource::ValidPcap(const std::string& file_name) { + std::unique_ptr temp_file( + RtpFileReader::Create(RtpFileReader::kPcap, file_name)); + return !!temp_file; +} + +RtpFileSource::~RtpFileSource() { +} + +bool RtpFileSource::RegisterRtpHeaderExtension(RTPExtensionType type, + uint8_t id) { + assert(parser_.get()); + return parser_->RegisterRtpHeaderExtension(type, id); +} + +std::unique_ptr RtpFileSource::NextPacket() { + while (true) { + RtpPacket temp_packet; + if (!rtp_reader_->NextPacket(&temp_packet)) { + return NULL; + } + if (temp_packet.original_length == 0) { + // May be an RTCP packet. + // Read the next one. + continue; + } + std::unique_ptr packet_memory(new uint8_t[temp_packet.length]); + memcpy(packet_memory.get(), temp_packet.data, temp_packet.length); + std::unique_ptr packet(new Packet( + packet_memory.release(), temp_packet.length, + temp_packet.original_length, temp_packet.time_ms, *parser_.get())); + if (!packet->valid_header()) { + continue; + } + if (filter_.test(packet->header().payloadType) || + (use_ssrc_filter_ && packet->header().ssrc != ssrc_)) { + // This payload type should be filtered out. Continue to the next packet. + continue; + } + return packet; + } +} + +RtpFileSource::RtpFileSource() + : PacketSource(), + parser_(RtpHeaderParser::Create()) {} + +bool RtpFileSource::OpenFile(const std::string& file_name) { + rtp_reader_.reset(RtpFileReader::Create(RtpFileReader::kRtpDump, file_name)); + if (rtp_reader_) + return true; + rtp_reader_.reset(RtpFileReader::Create(RtpFileReader::kPcap, file_name)); + if (!rtp_reader_) { + FATAL() << "Couldn't open input file as either a rtpdump or .pcap. Note " + "that .pcapng is not supported."; + } + return true; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.h new file mode 100644 index 0000000000..b44bc64d39 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_file_source.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_FILE_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_FILE_SOURCE_H_ + +#include + +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class RtpHeaderParser; + +namespace test { + +class RtpFileReader; + +class RtpFileSource : public PacketSource { + public: + // Creates an RtpFileSource reading from |file_name|. If the file cannot be + // opened, or has the wrong format, NULL will be returned. + static RtpFileSource* Create(const std::string& file_name); + + // Checks whether a files is a valid RTP dump or PCAP (Wireshark) file. + static bool ValidRtpDump(const std::string& file_name); + static bool ValidPcap(const std::string& file_name); + + virtual ~RtpFileSource(); + + // Registers an RTP header extension and binds it to |id|. + virtual bool RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id); + + std::unique_ptr NextPacket() override; + + private: + static const int kFirstLineLength = 40; + static const int kRtpFileHeaderSize = 4 + 4 + 4 + 2 + 2; + static const size_t kPacketHeaderSize = 8; + + RtpFileSource(); + + bool OpenFile(const std::string& file_name); + + std::unique_ptr rtp_reader_; + std::unique_ptr parser_; + + RTC_DISALLOW_COPY_AND_ASSIGN(RtpFileSource); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_FILE_SOURCE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.cc new file mode 100644 index 0000000000..cedd7aeb9e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/neteq/tools/rtp_generator.h" + +namespace webrtc { +namespace test { + +uint32_t RtpGenerator::GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header) { + assert(rtp_header); + if (!rtp_header) { + return 0; + } + rtp_header->sequenceNumber = seq_number_++; + rtp_header->timestamp = timestamp_; + timestamp_ += static_cast(payload_length_samples); + rtp_header->payloadType = payload_type; + rtp_header->markerBit = false; + rtp_header->ssrc = ssrc_; + rtp_header->numCSRCs = 0; + + uint32_t this_send_time = next_send_time_ms_; + assert(samples_per_ms_ > 0); + next_send_time_ms_ += ((1.0 + drift_factor_) * payload_length_samples) / + samples_per_ms_; + return this_send_time; +} + +void RtpGenerator::set_drift_factor(double factor) { + if (factor > -1.0) { + drift_factor_ = factor; + } +} + +uint32_t TimestampJumpRtpGenerator::GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header) { + uint32_t ret = RtpGenerator::GetRtpHeader( + payload_type, payload_length_samples, rtp_header); + if (timestamp_ - static_cast(payload_length_samples) <= + jump_from_timestamp_ && + timestamp_ > jump_from_timestamp_) { + // We just moved across the |jump_from_timestamp_| timestamp. Do the jump. + timestamp_ = jump_to_timestamp_; + } + return ret; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.h b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.h new file mode 100644 index 0000000000..b705f8346d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_generator.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_GENERATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_GENERATOR_H_ + +#include "common_types.h" // NOLINT(build/include) +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { + +// Class for generating RTP headers. +class RtpGenerator { + public: + RtpGenerator(int samples_per_ms, + uint16_t start_seq_number = 0, + uint32_t start_timestamp = 0, + uint32_t start_send_time_ms = 0, + uint32_t ssrc = 0x12345678) + : seq_number_(start_seq_number), + timestamp_(start_timestamp), + next_send_time_ms_(start_send_time_ms), + ssrc_(ssrc), + samples_per_ms_(samples_per_ms), + drift_factor_(0.0) { + } + + virtual ~RtpGenerator() {} + + // Writes the next RTP header to |rtp_header|, which will be of type + // |payload_type|. Returns the send time for this packet (in ms). The value of + // |payload_length_samples| determines the send time for the next packet. + virtual uint32_t GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header); + + void set_drift_factor(double factor); + + protected: + uint16_t seq_number_; + uint32_t timestamp_; + uint32_t next_send_time_ms_; + const uint32_t ssrc_; + const int samples_per_ms_; + double drift_factor_; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(RtpGenerator); +}; + +class TimestampJumpRtpGenerator : public RtpGenerator { + public: + TimestampJumpRtpGenerator(int samples_per_ms, + uint16_t start_seq_number, + uint32_t start_timestamp, + uint32_t jump_from_timestamp, + uint32_t jump_to_timestamp) + : RtpGenerator(samples_per_ms, start_seq_number, start_timestamp), + jump_from_timestamp_(jump_from_timestamp), + jump_to_timestamp_(jump_to_timestamp) {} + + uint32_t GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header) override; + + private: + uint32_t jump_from_timestamp_; + uint32_t jump_to_timestamp_; + RTC_DISALLOW_COPY_AND_ASSIGN(TimestampJumpRtpGenerator); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_GENERATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc new file mode 100644 index 0000000000..d92fed06cf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include +#include + +#include "api/array_view.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "rtc_base/buffer.h" +#include "rtc_base/flags.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { +namespace { + +DEFINE_bool(help, false, "Print help message"); + +constexpr size_t kRtpDumpHeaderLength = 8; + +// Returns the next packet or an empty buffer if end of file was encountered. +rtc::Buffer ReadNextPacket(FILE* file) { + // Read the rtpdump header for the next packet. + rtc::Buffer buffer; + buffer.SetData(kRtpDumpHeaderLength, [&](rtc::ArrayView x) { + return fread(x.data(), 1, x.size(), file); + }); + if (buffer.size() != kRtpDumpHeaderLength) { + return rtc::Buffer(); + } + + // Get length field. This is the total length for this packet written to file, + // including the kRtpDumpHeaderLength bytes already read. + const uint16_t len = ByteReader::ReadBigEndian(buffer.data()); + RTC_CHECK_GE(len, kRtpDumpHeaderLength); + + // Read remaining data from file directly into buffer. + buffer.AppendData(len - kRtpDumpHeaderLength, [&](rtc::ArrayView x) { + return fread(x.data(), 1, x.size(), file); + }); + if (buffer.size() != len) { + buffer.Clear(); + } + return buffer; +} + +struct PacketAndTime { + rtc::Buffer packet; + int time; +}; + +void WritePacket(const PacketAndTime& packet, FILE* file) { + // Write the first 4 bytes from the original packet. + const auto* payload_ptr = packet.packet.data(); + RTC_CHECK_EQ(fwrite(payload_ptr, 4, 1, file), 1); + payload_ptr += 4; + + // Convert the new time offset to network endian, and write to file. + uint8_t time[sizeof(uint32_t)]; + ByteWriter::WriteBigEndian(time, packet.time); + RTC_CHECK_EQ(fwrite(time, sizeof(uint32_t), 1, file), 1); + payload_ptr += 4; // Skip the old time in the original payload. + + // Write the remaining part of the payload. + RTC_DCHECK_EQ(payload_ptr - packet.packet.data(), kRtpDumpHeaderLength); + RTC_CHECK_EQ( + fwrite(payload_ptr, packet.packet.size() - kRtpDumpHeaderLength, 1, file), + 1); +} + +int RunRtpJitter(int argc, char* argv[]) { + const std::string program_name = argv[0]; + const std::string usage = + "Tool for alternating the arrival times in an RTP dump file.\n" + "Example usage:\n" + + program_name + " input.rtp arrival_times_ms.txt output.rtp\n\n"; + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || FLAG_help || + argc != 4) { + printf("%s", usage.c_str()); + return FLAG_help ? 0 : 1; + } + + printf("Input RTP file: %s\n", argv[1]); + FILE* in_file = fopen(argv[1], "rb"); + RTC_CHECK(in_file) << "Could not open file " << argv[1] << " for reading"; + printf("Timing file: %s\n", argv[2]); + std::ifstream timing_file(argv[2]); + printf("Output file: %s\n", argv[3]); + FILE* out_file = fopen(argv[3], "wb"); + RTC_CHECK(out_file) << "Could not open file " << argv[2] << " for writing"; + + // Copy the RTP file header to the output file. + char header_string[30]; + RTC_CHECK(fgets(header_string, 30, in_file)); + fprintf(out_file, "%s", header_string); + uint8_t file_header[16]; + RTC_CHECK_EQ(fread(file_header, sizeof(file_header), 1, in_file), 1); + RTC_CHECK_EQ(fwrite(file_header, sizeof(file_header), 1, out_file), 1); + + // Read all time values from the timing file. Store in a vector. + std::vector new_arrival_times; + int new_time; + while (timing_file >> new_time) { + new_arrival_times.push_back(new_time); + } + + // Read all packets from the input RTP file, but no more than the number of + // new time values. Store RTP packets together with new time values. + auto time_it = new_arrival_times.begin(); + std::vector packets; + while (1) { + auto packet = ReadNextPacket(in_file); + if (packet.empty() || time_it == new_arrival_times.end()) { + break; + } + packets.push_back({std::move(packet), *time_it}); + ++time_it; + } + + // Sort on new time values. + std::sort(packets.begin(), packets.end(), + [](const PacketAndTime& a, const PacketAndTime& b) { + return a.time < b.time; + }); + + // Write packets to output file. + for (const auto& p : packets) { + WritePacket(p, out_file); + } + + fclose(in_file); + fclose(out_file); + return 0; +} + +} // namespace +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::RunRtpJitter(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtpcat.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtpcat.cc new file mode 100644 index 0000000000..431de553ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tools/rtpcat.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include + +#include "rtc_base/checks.h" +#include "test/rtp_file_reader.h" +#include "test/rtp_file_writer.h" + +using webrtc::test::RtpFileReader; +using webrtc::test::RtpFileWriter; + +int main(int argc, char* argv[]) { + if (argc < 3) { + printf("Concatenates multiple rtpdump files into one.\n"); + printf("Usage: rtpcat in1.rtp int2.rtp [...] out.rtp\n"); + exit(1); + } + + std::unique_ptr output( + RtpFileWriter::Create(RtpFileWriter::kRtpDump, argv[argc - 1])); + RTC_CHECK(output.get() != NULL) << "Cannot open output file."; + printf("Output RTP file: %s\n", argv[argc - 1]); + + for (int i = 1; i < argc - 1; i++) { + std::unique_ptr input( + RtpFileReader::Create(RtpFileReader::kRtpDump, argv[i])); + RTC_CHECK(input.get() != NULL) << "Cannot open input file " << argv[i]; + printf("Input RTP file: %s\n", argv[i]); + + webrtc::test::RtpPacket packet; + while (input->NextPacket(&packet)) + RTC_CHECK(output->WritePacket(&packet)); + } + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq_decoder_enum_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq_decoder_enum_gn/moz.build new file mode 100644 index 0000000000..d23dbff13f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq_decoder_enum_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_decoder_enum.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("neteq_decoder_enum_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/neteq_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq_gn/moz.build new file mode 100644 index 0000000000..f24c85e6c6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/neteq_gn/moz.build @@ -0,0 +1,264 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_vector.cc" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/accelerate.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/audio_multi_vector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/background_noise.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/buffer_level_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/comfort_noise.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/cross_correlation.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_fax.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decision_logic_normal.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/decoder_database.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_manager.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/delay_peak_detector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dsp_helper.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/expand.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/merge.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/nack_tracker.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/neteq_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/normal.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/packet_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/post_decode_vad.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/preemptive_expand.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/random_vector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/rtcp.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/statistics_calculator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/sync_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/tick_timer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/time_stretch.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/neteq/timestamp_scaler.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("neteq_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_c_gn/moz.build new file mode 100644 index 0000000000..e18fc99100 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_c_gn/moz.build @@ -0,0 +1,209 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + +Library("pcm16b_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_gn/moz.build new file mode 100644 index 0000000000..80bc2f3805 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/pcm16b_gn/moz.build @@ -0,0 +1,219 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("pcm16b_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/rent_a_codec_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/rent_a_codec_gn/moz.build new file mode 100644 index 0000000000..69deb5f6a1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/rent_a_codec_gn/moz.build @@ -0,0 +1,390 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_CODEC_ILBC"] = True +DEFINES["WEBRTC_CODEC_OPUS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_OPUS_SUPPORT_120MS_PTIME"] = "1" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/media/libopus/include/", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/fix/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/acm_codec_database.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/acm2/rent_a_codec.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_CODEC_ISACFX"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "ppc64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + + OS_LIBS += [ + "m" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" + ] + +Library("rent_a_codec_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/ACMTest.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/ACMTest.h new file mode 100644 index 0000000000..3fc97ca31b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/ACMTest.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_ACMTEST_H_ +#define MODULES_AUDIO_CODING_TEST_ACMTEST_H_ + +class ACMTest { + public: + ACMTest() {} + virtual ~ACMTest() {} + virtual void Perform() = 0; +}; + +#endif // MODULES_AUDIO_CODING_TEST_ACMTEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.cc new file mode 100644 index 0000000000..dbf5ad4534 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.cc @@ -0,0 +1,1106 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/APITest.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/test/utility.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/event_wrapper.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#define TEST_DURATION_SEC 600 +#define NUMBER_OF_SENDER_TESTS 6 +#define MAX_FILE_NAME_LENGTH_BYTE 500 + +void APITest::Wait(uint32_t waitLengthMs) { + if (_randomTest) { + return; + } else { + EventWrapper* myEvent = EventWrapper::Create(); + myEvent->Wait(waitLengthMs); + delete myEvent; + return; + } +} + +APITest::APITest() + : _acmA(AudioCodingModule::Create()), + _acmB(AudioCodingModule::Create()), + _channel_A2B(NULL), + _channel_B2A(NULL), + _writeToFile(true), + _pullEventA(NULL), + _pushEventA(NULL), + _processEventA(NULL), + _apiEventA(NULL), + _pullEventB(NULL), + _pushEventB(NULL), + _processEventB(NULL), + _apiEventB(NULL), + _codecCntrA(0), + _codecCntrB(0), + _thereIsEncoderA(false), + _thereIsEncoderB(false), + _thereIsDecoderA(false), + _thereIsDecoderB(false), + _sendVADA(false), + _sendDTXA(false), + _sendVADModeA(VADNormal), + _sendVADB(false), + _sendDTXB(false), + _sendVADModeB(VADNormal), + _minDelayA(0), + _minDelayB(0), + _dotPositionA(0), + _dotMoveDirectionA(1), + _dotPositionB(39), + _dotMoveDirectionB(-1), + _vadCallbackA(NULL), + _vadCallbackB(NULL), + _apiTestRWLock(*RWLockWrapper::CreateRWLock()), + _randomTest(false), + _testNumA(0), + _testNumB(1) { + int n; + for (n = 0; n < 32; n++) { + _payloadUsed[n] = false; + } + + _movingDot[40] = '\0'; + + for (int n = 0; n < 40; n++) { + _movingDot[n] = ' '; + } +} + +APITest::~APITest() { + DELETE_POINTER(_channel_A2B); + DELETE_POINTER(_channel_B2A); + + DELETE_POINTER(_pushEventA); + DELETE_POINTER(_pullEventA); + DELETE_POINTER(_processEventA); + DELETE_POINTER(_apiEventA); + + DELETE_POINTER(_pushEventB); + DELETE_POINTER(_pullEventB); + DELETE_POINTER(_processEventB); + DELETE_POINTER(_apiEventB); + + _inFileA.Close(); + _outFileA.Close(); + + _inFileB.Close(); + _outFileB.Close(); + + DELETE_POINTER(_vadCallbackA); + DELETE_POINTER(_vadCallbackB); + + delete &_apiTestRWLock; +} + +int16_t APITest::SetUp() { + CodecInst dummyCodec; + int lastPayloadType = 0; + + int16_t numCodecs = _acmA->NumberOfCodecs(); + for (uint8_t n = 0; n < numCodecs; n++) { + AudioCodingModule::Codec(n, &dummyCodec); + if ((STR_CASE_CMP(dummyCodec.plname, "CN") == 0) + && (dummyCodec.plfreq == 32000)) { + continue; + } + + printf("Register Receive Codec %s ", dummyCodec.plname); + + if ((n != 0) && !FixedPayloadTypeCodec(dummyCodec.plname)) { + // Check registration with an already occupied payload type + int currentPayloadType = dummyCodec.pltype; + dummyCodec.pltype = 97; //lastPayloadType; + EXPECT_EQ(true, _acmB->RegisterReceiveCodec(dummyCodec.pltype, + CodecInstToSdp(dummyCodec))); + dummyCodec.pltype = currentPayloadType; + } + + if ((n < numCodecs - 1) && !FixedPayloadTypeCodec(dummyCodec.plname)) { + // test if re-registration works; + CodecInst nextCodec; + int currentPayloadType = dummyCodec.pltype; + AudioCodingModule::Codec(n + 1, &nextCodec); + dummyCodec.pltype = nextCodec.pltype; + if (!FixedPayloadTypeCodec(nextCodec.plname)) { + _acmB->RegisterReceiveCodec(dummyCodec.pltype, + CodecInstToSdp(dummyCodec)); + } + dummyCodec.pltype = currentPayloadType; + } + + if ((n < numCodecs - 1) && !FixedPayloadTypeCodec(dummyCodec.plname)) { + // test if un-registration works; + CodecInst nextCodec; + AudioCodingModule::Codec(n + 1, &nextCodec); + nextCodec.pltype = dummyCodec.pltype; + if (!FixedPayloadTypeCodec(nextCodec.plname)) { + EXPECT_EQ(true, _acmA->RegisterReceiveCodec(nextCodec.pltype, + CodecInstToSdp(nextCodec))); + CHECK_ERROR_MT(_acmA->UnregisterReceiveCodec(nextCodec.pltype)); + } + } + + EXPECT_EQ(true, _acmA->RegisterReceiveCodec(dummyCodec.pltype, + CodecInstToSdp(dummyCodec))); + printf(" side A done!"); + EXPECT_EQ(true, _acmB->RegisterReceiveCodec(dummyCodec.pltype, + CodecInstToSdp(dummyCodec))); + printf(" side B done!\n"); + + if (!strcmp(dummyCodec.plname, "CN")) { + CHECK_ERROR_MT(_acmA->RegisterSendCodec(dummyCodec)); + CHECK_ERROR_MT(_acmB->RegisterSendCodec(dummyCodec)); + } + lastPayloadType = dummyCodec.pltype; + if ((lastPayloadType >= 96) && (lastPayloadType <= 127)) { + _payloadUsed[lastPayloadType - 96] = true; + } + } + _thereIsDecoderA = true; + _thereIsDecoderB = true; + + // Register Send Codec + AudioCodingModule::Codec((uint8_t) _codecCntrA, &dummyCodec); + CHECK_ERROR_MT(_acmA->RegisterSendCodec(dummyCodec)); + _thereIsEncoderA = true; + // + AudioCodingModule::Codec((uint8_t) _codecCntrB, &dummyCodec); + CHECK_ERROR_MT(_acmB->RegisterSendCodec(dummyCodec)); + _thereIsEncoderB = true; + + uint16_t frequencyHz; + + printf("\n\nAPI Test\n"); + printf("========\n"); + printf("Hit enter to accept the default values indicated in []\n\n"); + + //--- Input A + std::string file_name = webrtc::test::ResourcePath( + "audio_coding/testfile32kHz", "pcm"); + frequencyHz = 32000; + printf("Enter input file at side A [%s]: ", file_name.c_str()); + PCMFile::ChooseFile(&file_name, 499, &frequencyHz); + _inFileA.Open(file_name, frequencyHz, "rb", true); + + //--- Output A + std::string out_file_a = webrtc::test::OutputPath() + "outA.pcm"; + printf("Enter output file at side A [%s]: ", out_file_a.c_str()); + PCMFile::ChooseFile(&out_file_a, 499, &frequencyHz); + _outFileA.Open(out_file_a, frequencyHz, "wb"); + + //--- Input B + file_name = webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + printf("\n\nEnter input file at side B [%s]: ", file_name.c_str()); + PCMFile::ChooseFile(&file_name, 499, &frequencyHz); + _inFileB.Open(file_name, frequencyHz, "rb", true); + + //--- Output B + std::string out_file_b = webrtc::test::OutputPath() + "outB.pcm"; + printf("Enter output file at side B [%s]: ", out_file_b.c_str()); + PCMFile::ChooseFile(&out_file_b, 499, &frequencyHz); + _outFileB.Open(out_file_b, frequencyHz, "wb"); + + //--- Set A-to-B channel + _channel_A2B = new Channel(2); + CHECK_ERROR_MT(_acmA->RegisterTransportCallback(_channel_A2B)); + _channel_A2B->RegisterReceiverACM(_acmB.get()); + + //--- Set B-to-A channel + _channel_B2A = new Channel(1); + CHECK_ERROR_MT(_acmB->RegisterTransportCallback(_channel_B2A)); + _channel_B2A->RegisterReceiverACM(_acmA.get()); + + //--- EVENT TIMERS + // A + _pullEventA = EventTimerWrapper::Create(); + _pushEventA = EventTimerWrapper::Create(); + _processEventA = EventTimerWrapper::Create(); + _apiEventA = EventWrapper::Create(); + // B + _pullEventB = EventTimerWrapper::Create(); + _pushEventB = EventTimerWrapper::Create(); + _processEventB = EventTimerWrapper::Create(); + _apiEventB = EventWrapper::Create(); + + //--- I/O params + // A + _outFreqHzA = _outFileA.SamplingFrequency(); + // B + _outFreqHzB = _outFileB.SamplingFrequency(); + + char print[11]; + + printf("\nRandom Test (y/n)?"); + EXPECT_TRUE(fgets(print, 10, stdin) != NULL); + print[10] = '\0'; + if (strstr(print, "y") != NULL) { + _randomTest = true; + _verbose = false; + _writeToFile = false; + } else { + _randomTest = false; + printf("\nPrint Tests (y/n)? "); + EXPECT_TRUE(fgets(print, 10, stdin) != NULL); + print[10] = '\0'; + if (strstr(print, "y") == NULL) { + EXPECT_TRUE(freopen("APITest_log.txt", "w", stdout) != 0); + _verbose = false; + } + } + + _vadCallbackA = new VADCallback; + _vadCallbackB = new VADCallback; + + return 0; +} + +bool APITest::PushAudioThreadA(void* obj) { + return static_cast(obj)->PushAudioRunA(); +} + +bool APITest::PushAudioThreadB(void* obj) { + return static_cast(obj)->PushAudioRunB(); +} + +bool APITest::PullAudioThreadA(void* obj) { + return static_cast(obj)->PullAudioRunA(); +} + +bool APITest::PullAudioThreadB(void* obj) { + return static_cast(obj)->PullAudioRunB(); +} + +bool APITest::ProcessThreadA(void* obj) { + return static_cast(obj)->ProcessRunA(); +} + +bool APITest::ProcessThreadB(void* obj) { + return static_cast(obj)->ProcessRunB(); +} + +bool APITest::APIThreadA(void* obj) { + return static_cast(obj)->APIRunA(); +} + +bool APITest::APIThreadB(void* obj) { + return static_cast(obj)->APIRunB(); +} + +bool APITest::PullAudioRunA() { + _pullEventA->Wait(100); + AudioFrame audioFrame; + bool muted; + if (_acmA->PlayoutData10Ms(_outFreqHzA, &audioFrame, &muted) < 0) { + bool thereIsDecoder; + { + ReadLockScoped rl(_apiTestRWLock); + thereIsDecoder = _thereIsDecoderA; + } + if (thereIsDecoder) { + fprintf(stderr, "\n>>>>>> cannot pull audio A <<<<<<<< \n"); + } + } else { + if (_writeToFile) { + _outFileA.Write10MsData(audioFrame); + } + } + return true; +} + +bool APITest::PullAudioRunB() { + _pullEventB->Wait(100); + AudioFrame audioFrame; + bool muted; + if (_acmB->PlayoutData10Ms(_outFreqHzB, &audioFrame, &muted) < 0) { + bool thereIsDecoder; + { + ReadLockScoped rl(_apiTestRWLock); + thereIsDecoder = _thereIsDecoderB; + } + if (thereIsDecoder) { + fprintf(stderr, "\n>>>>>> cannot pull audio B <<<<<<<< \n"); + fprintf(stderr, "%d %d\n", _testNumA, _testNumB); + } + } else { + if (_writeToFile) { + _outFileB.Write10MsData(audioFrame); + } + } + return true; +} + +bool APITest::PushAudioRunA() { + _pushEventA->Wait(100); + AudioFrame audioFrame; + _inFileA.Read10MsData(audioFrame); + if (_acmA->Add10MsData(audioFrame) < 0) { + bool thereIsEncoder; + { + ReadLockScoped rl(_apiTestRWLock); + thereIsEncoder = _thereIsEncoderA; + } + if (thereIsEncoder) { + fprintf(stderr, "\n>>>> add10MsData at A failed <<<<\n"); + } + } + return true; +} + +bool APITest::PushAudioRunB() { + _pushEventB->Wait(100); + AudioFrame audioFrame; + _inFileB.Read10MsData(audioFrame); + if (_acmB->Add10MsData(audioFrame) < 0) { + bool thereIsEncoder; + { + ReadLockScoped rl(_apiTestRWLock); + thereIsEncoder = _thereIsEncoderB; + } + + if (thereIsEncoder) { + fprintf(stderr, "\n>>>> cannot add audio to B <<<<"); + } + } + + return true; +} + +bool APITest::ProcessRunA() { + _processEventA->Wait(100); + return true; +} + +bool APITest::ProcessRunB() { + _processEventB->Wait(100); + return true; +} + +/*/ + * + * In side A we test the APIs which are related to sender Side. + * +/*/ + +void APITest::RunTest(char thread) { + int testNum; + { + WriteLockScoped cs(_apiTestRWLock); + if (thread == 'A') { + _testNumA = (_testNumB + 1 + (rand() % 3)) % 4; + testNum = _testNumA; + + _movingDot[_dotPositionA] = ' '; + if (_dotPositionA == 0) { + _dotMoveDirectionA = 1; + } + if (_dotPositionA == 19) { + _dotMoveDirectionA = -1; + } + _dotPositionA += _dotMoveDirectionA; + _movingDot[_dotPositionA] = (_dotMoveDirectionA > 0) ? '>' : '<'; + } else { + _testNumB = (_testNumA + 1 + (rand() % 3)) % 4; + testNum = _testNumB; + + _movingDot[_dotPositionB] = ' '; + if (_dotPositionB == 20) { + _dotMoveDirectionB = 1; + } + if (_dotPositionB == 39) { + _dotMoveDirectionB = -1; + } + _dotPositionB += _dotMoveDirectionB; + _movingDot[_dotPositionB] = (_dotMoveDirectionB > 0) ? '>' : '<'; + } + //fprintf(stderr, "%c: %d \n", thread, testNum); + //fflush(stderr); + } + switch (testNum) { + case 0: + CurrentCodec('A'); + ChangeCodec('A'); + break; + case 1: + if (!_randomTest) { + fprintf(stdout, "\nTesting Delay ...\n"); + } + TestDelay('A'); + break; + case 2: + TestSendVAD('A'); + break; + case 3: + TestRegisteration('A'); + break; + default: + fprintf(stderr, "Wrong Test Number\n"); + getc(stdin); + exit(1); + } +} + +bool APITest::APIRunA() { + _apiEventA->Wait(50); + + bool randomTest; + { + ReadLockScoped rl(_apiTestRWLock); + randomTest = _randomTest; + } + if (randomTest) { + RunTest('A'); + } else { + CurrentCodec('A'); + ChangeCodec('A'); + if (_codecCntrA == 0) { + fprintf(stdout, "\nTesting Delay ...\n"); + TestDelay('A'); + } + // VAD TEST + TestSendVAD('A'); + TestRegisteration('A'); + } + return true; +} + +bool APITest::APIRunB() { + _apiEventB->Wait(50); + bool randomTest; + { + ReadLockScoped rl(_apiTestRWLock); + randomTest = _randomTest; + } + //_apiEventB->Wait(2000); + if (randomTest) { + RunTest('B'); + } + + return true; +} + +void APITest::Perform() { + SetUp(); + + //--- THREADS + // A + // PUSH + rtc::PlatformThread myPushAudioThreadA(PushAudioThreadA, this, + "PushAudioThreadA"); + myPushAudioThreadA.Start(); + // PULL + rtc::PlatformThread myPullAudioThreadA(PullAudioThreadA, this, + "PullAudioThreadA"); + myPullAudioThreadA.Start(); + // Process + rtc::PlatformThread myProcessThreadA(ProcessThreadA, this, "ProcessThreadA"); + myProcessThreadA.Start(); + // API + rtc::PlatformThread myAPIThreadA(APIThreadA, this, "APIThreadA"); + myAPIThreadA.Start(); + // B + // PUSH + rtc::PlatformThread myPushAudioThreadB(PushAudioThreadB, this, + "PushAudioThreadB"); + myPushAudioThreadB.Start(); + // PULL + rtc::PlatformThread myPullAudioThreadB(PullAudioThreadB, this, + "PullAudioThreadB"); + myPullAudioThreadB.Start(); + // Process + rtc::PlatformThread myProcessThreadB(ProcessThreadB, this, "ProcessThreadB"); + myProcessThreadB.Start(); + // API + rtc::PlatformThread myAPIThreadB(APIThreadB, this, "APIThreadB"); + myAPIThreadB.Start(); + + //_apiEventA->StartTimer(true, 5000); + //_apiEventB->StartTimer(true, 5000); + + _processEventA->StartTimer(true, 10); + _processEventB->StartTimer(true, 10); + + _pullEventA->StartTimer(true, 10); + _pullEventB->StartTimer(true, 10); + + _pushEventA->StartTimer(true, 10); + _pushEventB->StartTimer(true, 10); + + // Keep main thread waiting for sender/receiver + // threads to complete + EventWrapper* completeEvent = EventWrapper::Create(); + uint64_t startTime = rtc::TimeMillis(); + uint64_t currentTime; + // Run test in 2 minutes (120000 ms). + do { + { + //ReadLockScoped rl(_apiTestRWLock); + //fprintf(stderr, "\r%s", _movingDot); + } + //fflush(stderr); + completeEvent->Wait(50); + currentTime = rtc::TimeMillis(); + } while ((currentTime - startTime) < 120000); + + //completeEvent->Wait(0xFFFFFFFF); + //(unsigned long)((unsigned long)TEST_DURATION_SEC * (unsigned long)1000)); + delete completeEvent; + + myPushAudioThreadA.Stop(); + myPullAudioThreadA.Stop(); + myProcessThreadA.Stop(); + myAPIThreadA.Stop(); + + myPushAudioThreadB.Stop(); + myPullAudioThreadB.Stop(); + myProcessThreadB.Stop(); + myAPIThreadB.Stop(); +} + +void APITest::CheckVADStatus(char side) { + + bool dtxEnabled; + bool vadEnabled; + ACMVADMode vadMode; + + if (side == 'A') { + _acmA->VAD(&dtxEnabled, &vadEnabled, &vadMode); + _acmA->RegisterVADCallback(NULL); + _vadCallbackA->Reset(); + _acmA->RegisterVADCallback(_vadCallbackA); + + if (!_randomTest) { + if (_verbose) { + fprintf(stdout, "DTX %3s, VAD %3s, Mode %d", dtxEnabled ? "ON" : "OFF", + vadEnabled ? "ON" : "OFF", (int) vadMode); + Wait(5000); + fprintf(stdout, " => bit-rate %3.0f kbps\n", _channel_A2B->BitRate()); + } else { + Wait(5000); + fprintf(stdout, "DTX %3s, VAD %3s, Mode %d => bit-rate %3.0f kbps\n", + dtxEnabled ? "ON" : "OFF", vadEnabled ? "ON" : "OFF", + (int) vadMode, _channel_A2B->BitRate()); + } + _vadCallbackA->PrintFrameTypes(); + } + + if (dtxEnabled != _sendDTXA) { + fprintf(stderr, ">>> Error Enabling DTX <<<\n"); + } + if ((vadEnabled != _sendVADA) && (!dtxEnabled)) { + fprintf(stderr, ">>> Error Enabling VAD <<<\n"); + } + if ((vadMode != _sendVADModeA) && vadEnabled) { + fprintf(stderr, ">>> Error setting VAD-mode <<<\n"); + } + } else { + _acmB->VAD(&dtxEnabled, &vadEnabled, &vadMode); + + _acmB->RegisterVADCallback(NULL); + _vadCallbackB->Reset(); + _acmB->RegisterVADCallback(_vadCallbackB); + + if (!_randomTest) { + if (_verbose) { + fprintf(stdout, "DTX %3s, VAD %3s, Mode %d", dtxEnabled ? "ON" : "OFF", + vadEnabled ? "ON" : "OFF", (int) vadMode); + Wait(5000); + fprintf(stdout, " => bit-rate %3.0f kbps\n", _channel_B2A->BitRate()); + } else { + Wait(5000); + fprintf(stdout, "DTX %3s, VAD %3s, Mode %d => bit-rate %3.0f kbps\n", + dtxEnabled ? "ON" : "OFF", vadEnabled ? "ON" : "OFF", + (int) vadMode, _channel_B2A->BitRate()); + } + _vadCallbackB->PrintFrameTypes(); + } + + if (dtxEnabled != _sendDTXB) { + fprintf(stderr, ">>> Error Enabling DTX <<<\n"); + } + if ((vadEnabled != _sendVADB) && (!dtxEnabled)) { + fprintf(stderr, ">>> Error Enabling VAD <<<\n"); + } + if ((vadMode != _sendVADModeB) && vadEnabled) { + fprintf(stderr, ">>> Error setting VAD-mode <<<\n"); + } + } +} + +// Set Min delay, get delay, playout timestamp +void APITest::TestDelay(char side) { + AudioCodingModule* myACM; + Channel* myChannel; + int32_t* myMinDelay; + EventTimerWrapper* myEvent = EventTimerWrapper::Create(); + + uint32_t inTimestamp = 0; + double estimDelay = 0; + + double averageEstimDelay = 0; + double averageDelay = 0; + + test::CircularBuffer estimDelayCB(100); + estimDelayCB.SetArithMean(true); + + if (side == 'A') { + myACM = _acmA.get(); + myChannel = _channel_B2A; + myMinDelay = &_minDelayA; + } else { + myACM = _acmB.get(); + myChannel = _channel_A2B; + myMinDelay = &_minDelayB; + } + + CHECK_ERROR_MT(myACM->SetMinimumPlayoutDelay(*myMinDelay)); + + inTimestamp = myChannel->LastInTimestamp(); + rtc::Optional outTimestamp = myACM->PlayoutTimestamp(); + CHECK_ERROR_MT(outTimestamp ? 0 : -1); + + if (!_randomTest) { + myEvent->StartTimer(true, 30); + int n = 0; + int settlePoint = 5000; + while (n < settlePoint + 400) { + myEvent->Wait(1000); + + inTimestamp = myChannel->LastInTimestamp(); + outTimestamp = myACM->PlayoutTimestamp(); + CHECK_ERROR_MT(outTimestamp ? 0 : -1); + + //std::cout << outTimestamp << std::endl << std::flush; + estimDelay = (double)((uint32_t)(inTimestamp - *outTimestamp)) / + ((double)myACM->ReceiveFrequency() / 1000.0); + + estimDelayCB.Update(estimDelay); + + estimDelayCB.ArithMean(averageEstimDelay); + //printf("\n %6.1f \n", estimDelay); + //std::cout << " " << std::flush; + + if (_verbose) { + fprintf(stdout, + "\rExpected: %4d, retreived: %6.1f, measured: %6.1f", + *myMinDelay, averageDelay, averageEstimDelay); + std::cout << " " << std::flush; + } + if ((averageDelay > *myMinDelay) && (n < settlePoint)) { + settlePoint = n; + } + n++; + } + myEvent->StopTimer(); + } + + if ((!_verbose) && (!_randomTest)) { + fprintf(stdout, "\nExpected: %4d, retreived: %6.1f, measured: %6.1f", + *myMinDelay, averageDelay, averageEstimDelay); + } + + *myMinDelay = (rand() % 1000) + 1; + + NetworkStatistics networkStat; + CHECK_ERROR_MT(myACM->GetNetworkStatistics(&networkStat)); + + if (!_randomTest) { + fprintf(stdout, "\n\nJitter Statistics at Side %c\n", side); + fprintf(stdout, "--------------------------------------\n"); + fprintf(stdout, "buffer-size............. %d\n", + networkStat.currentBufferSize); + fprintf(stdout, "Preferred buffer-size... %d\n", + networkStat.preferredBufferSize); + fprintf(stdout, "Peaky jitter mode........%d\n", + networkStat.jitterPeaksFound); + fprintf(stdout, "packet-size rate........ %d\n", + networkStat.currentPacketLossRate); + fprintf(stdout, "expand rate............. %d\n", + networkStat.currentExpandRate); + fprintf(stdout, "speech expand rate...... %d\n", + networkStat.currentSpeechExpandRate); + fprintf(stdout, "Preemptive rate......... %d\n", + networkStat.currentPreemptiveRate); + fprintf(stdout, "Accelerate rate......... %d\n", + networkStat.currentAccelerateRate); + fprintf(stdout, "Secondary decoded rate.. %d\n", + networkStat.currentSecondaryDecodedRate); + fprintf(stdout, "Secondary discarded rate.%d\n", + networkStat.currentSecondaryDiscardedRate); + fprintf(stdout, "Clock-drift............. %d\n", networkStat.clockDriftPPM); + fprintf(stdout, "Mean waiting time....... %d\n", + networkStat.meanWaitingTimeMs); + fprintf(stdout, "Median waiting time..... %d\n", + networkStat.medianWaitingTimeMs); + fprintf(stdout, "Min waiting time........ %d\n", + networkStat.minWaitingTimeMs); + fprintf(stdout, "Max waiting time........ %d\n", + networkStat.maxWaitingTimeMs); + } + + CHECK_ERROR_MT(myACM->SetMinimumPlayoutDelay(*myMinDelay)); + + if (!_randomTest) { + myEvent->Wait(500); + fprintf(stdout, "\n"); + fprintf(stdout, "\n"); + } + delete myEvent; +} + +// Unregister a codec & register again. +void APITest::TestRegisteration(char sendSide) { + AudioCodingModule* sendACM; + AudioCodingModule* receiveACM; + bool* thereIsDecoder; + EventWrapper* myEvent = EventWrapper::Create(); + + if (!_randomTest) { + fprintf(stdout, "\n\n"); + fprintf(stdout, + "---------------------------------------------------------\n"); + fprintf(stdout, " Unregister/register Receive Codec\n"); + fprintf(stdout, + "---------------------------------------------------------\n"); + } + + switch (sendSide) { + case 'A': { + sendACM = _acmA.get(); + receiveACM = _acmB.get(); + thereIsDecoder = &_thereIsDecoderB; + break; + } + case 'B': { + sendACM = _acmB.get(); + receiveACM = _acmA.get(); + thereIsDecoder = &_thereIsDecoderA; + break; + } + default: + fprintf(stderr, "Invalid sender-side in TestRegistration(%c)\n", + sendSide); + exit(-1); + } + + auto myCodec = sendACM->SendCodec(); + if (!myCodec) { + CodecInst ci; + AudioCodingModule::Codec(_codecCntrA, &ci); + myCodec = ci; + } + + if (!_randomTest) { + fprintf(stdout, "Unregistering reveive codec, NO AUDIO.\n"); + fflush (stdout); + } + { + WriteLockScoped wl(_apiTestRWLock); + *thereIsDecoder = false; + } + //myEvent->Wait(20); + CHECK_ERROR_MT(receiveACM->UnregisterReceiveCodec(myCodec->pltype)); + Wait(1000); + + int currentPayload = myCodec->pltype; + + if (!FixedPayloadTypeCodec(myCodec->plname)) { + int32_t i; + for (i = 0; i < 32; i++) { + if (!_payloadUsed[i]) { + if (!_randomTest) { + fprintf(stdout, + "Register receive codec with new Payload, AUDIO BACK.\n"); + } + //myCodec->pltype = i + 96; + //CHECK_ERROR_MT(receiveACM->RegisterReceiveCodec(*myCodec)); + //CHECK_ERROR_MT(sendACM->RegisterSendCodec(*myCodec)); + //myEvent->Wait(20); + //{ + // WriteLockScoped wl(_apiTestRWLock); + // *thereIsDecoder = true; + //} + Wait(1000); + + if (!_randomTest) { + fprintf(stdout, "Unregistering reveive codec, NO AUDIO.\n"); + } + //{ + // WriteLockScoped wl(_apiTestRWLock); + // *thereIsDecoder = false; + //} + //myEvent->Wait(20); + //CHECK_ERROR_MT(receiveACM->UnregisterReceiveCodec(myCodec->pltype)); + Wait(1000); + + myCodec->pltype = currentPayload; + if (!_randomTest) { + fprintf(stdout, + "Register receive codec with default Payload, AUDIO BACK.\n"); + fflush (stdout); + } + EXPECT_EQ(true, receiveACM->RegisterReceiveCodec( + myCodec->pltype, CodecInstToSdp(*myCodec))); + //CHECK_ERROR_MT(sendACM->RegisterSendCodec(*myCodec)); + myEvent->Wait(20); + { + WriteLockScoped wl(_apiTestRWLock); + *thereIsDecoder = true; + } + Wait(1000); + + break; + } + } + if (i == 32) { + EXPECT_EQ(true, receiveACM->RegisterReceiveCodec( + myCodec->pltype, CodecInstToSdp(*myCodec))); + { + WriteLockScoped wl(_apiTestRWLock); + *thereIsDecoder = true; + } + } + } else { + if (!_randomTest) { + fprintf(stdout, + "Register receive codec with fixed Payload, AUDIO BACK.\n"); + fflush (stdout); + } + EXPECT_EQ(true, receiveACM->RegisterReceiveCodec(myCodec->pltype, + CodecInstToSdp(*myCodec))); + //CHECK_ERROR_MT(receiveACM->UnregisterReceiveCodec(myCodec->pltype)); + //CHECK_ERROR_MT(receiveACM->RegisterReceiveCodec(*myCodec)); + myEvent->Wait(20); + { + WriteLockScoped wl(_apiTestRWLock); + *thereIsDecoder = true; + } + } + delete myEvent; + if (!_randomTest) { + fprintf(stdout, + "---------------------------------------------------------\n"); + } +} + +void APITest::TestSendVAD(char side) { + if (_randomTest) { + return; + } + + bool* vad; + bool* dtx; + ACMVADMode* mode; + Channel* myChannel; + AudioCodingModule* myACM; + + CodecInst myCodec; + if (!_randomTest) { + fprintf(stdout, "\n\n"); + fprintf(stdout, "-----------------------------------------------\n"); + fprintf(stdout, " Test VAD API\n"); + fprintf(stdout, "-----------------------------------------------\n"); + } + + if (side == 'A') { + AudioCodingModule::Codec(_codecCntrA, &myCodec); + vad = &_sendVADA; + dtx = &_sendDTXA; + mode = &_sendVADModeA; + myChannel = _channel_A2B; + myACM = _acmA.get(); + } else { + AudioCodingModule::Codec(_codecCntrB, &myCodec); + vad = &_sendVADB; + dtx = &_sendDTXB; + mode = &_sendVADModeB; + myChannel = _channel_B2A; + myACM = _acmB.get(); + } + + CheckVADStatus(side); + if (!_randomTest) { + fprintf(stdout, "\n\n"); + } + + switch (*mode) { + case VADNormal: + *vad = true; + *dtx = true; + *mode = VADAggr; + break; + case VADLowBitrate: + *vad = true; + *dtx = true; + *mode = VADVeryAggr; + break; + case VADAggr: + *vad = true; + *dtx = true; + *mode = VADLowBitrate; + break; + case VADVeryAggr: + *vad = false; + *dtx = false; + *mode = VADNormal; + break; + default: + *mode = VADNormal; + } + + *dtx = (myCodec.plfreq == 32000) ? false : *dtx; + + CHECK_ERROR_MT(myACM->SetVAD(*dtx, *vad, *mode)); + myChannel->ResetStats(); + + CheckVADStatus(side); + if (!_randomTest) { + fprintf(stdout, "\n"); + fprintf(stdout, "-----------------------------------------------\n"); + } + + // Fault Test + CHECK_PROTECTED_MT(myACM->SetVAD(false, true, (ACMVADMode) - 1)); + CHECK_PROTECTED_MT(myACM->SetVAD(false, true, (ACMVADMode) 4)); + +} + +void APITest::CurrentCodec(char side) { + auto myCodec = (side == 'A' ? _acmA : _acmB)->SendCodec(); + + if (!_randomTest) { + fprintf(stdout, "\n\n"); + fprintf(stdout, "Send codec in Side A\n"); + fprintf(stdout, "----------------------------\n"); + fprintf(stdout, "Name................. %s\n", myCodec->plname); + fprintf(stdout, "Sampling Frequency... %d\n", myCodec->plfreq); + fprintf(stdout, "Rate................. %d\n", myCodec->rate); + fprintf(stdout, "Payload-type......... %d\n", myCodec->pltype); + fprintf(stdout, "Packet-size.......... %d\n", myCodec->pacsize); + } + + Wait(100); +} + +void APITest::ChangeCodec(char side) { + CodecInst myCodec; + AudioCodingModule* myACM; + uint8_t* codecCntr; + bool* thereIsEncoder; + bool* vad; + bool* dtx; + ACMVADMode* mode; + Channel* myChannel; + // Reset and Wait + if (!_randomTest) { + fprintf(stdout, "Reset Encoder Side A \n"); + } + if (side == 'A') { + myACM = _acmA.get(); + codecCntr = &_codecCntrA; + { + WriteLockScoped wl(_apiTestRWLock); + thereIsEncoder = &_thereIsEncoderA; + } + vad = &_sendVADA; + dtx = &_sendDTXA; + mode = &_sendVADModeA; + myChannel = _channel_A2B; + } else { + myACM = _acmB.get(); + codecCntr = &_codecCntrB; + { + WriteLockScoped wl(_apiTestRWLock); + thereIsEncoder = &_thereIsEncoderB; + } + vad = &_sendVADB; + dtx = &_sendDTXB; + mode = &_sendVADModeB; + myChannel = _channel_B2A; + } + + Wait(100); + + // Register the next codec + do { + *codecCntr = + (*codecCntr < AudioCodingModule::NumberOfCodecs() - 1) ? + (*codecCntr + 1) : 0; + + if (*codecCntr == 0) { + //printf("Initialize Sender Side A \n"); + { + WriteLockScoped wl(_apiTestRWLock); + *thereIsEncoder = false; + } + // After Initialization CN is lost, re-register them + if (AudioCodingModule::Codec("CN", &myCodec, 8000, 1) >= 0) { + CHECK_ERROR_MT(myACM->RegisterSendCodec(myCodec)); + } + if (AudioCodingModule::Codec("CN", &myCodec, 16000, 1) >= 0) { + CHECK_ERROR_MT(myACM->RegisterSendCodec(myCodec)); + } + // VAD & DTX are disabled after initialization + *vad = false; + *dtx = false; + _writeToFile = false; + } + + AudioCodingModule::Codec(*codecCntr, &myCodec); + } while (!STR_CASE_CMP(myCodec.plname, "CN") + || !STR_CASE_CMP(myCodec.plname, "telephone-event") + || !STR_CASE_CMP(myCodec.plname, "RED")); + + if (!_randomTest) { + fprintf(stdout,"\n=====================================================\n"); + fprintf(stdout, " Registering New Codec %s, %d kHz, %d kbps\n", + myCodec.plname, myCodec.plfreq / 1000, myCodec.rate / 1000); + } + //std::cout<< std::flush; + + // NO DTX for supe-wideband codec at this point + if (myCodec.plfreq == 32000) { + *dtx = false; + CHECK_ERROR_MT(myACM->SetVAD(*dtx, *vad, *mode)); + + } + + CHECK_ERROR_MT(myACM->RegisterSendCodec(myCodec)); + myChannel->ResetStats(); + { + WriteLockScoped wl(_apiTestRWLock); + *thereIsEncoder = true; + } + Wait(500); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.h new file mode 100644 index 0000000000..652d7c401c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/APITest.h @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_APITEST_H_ +#define MODULES_AUDIO_CODING_TEST_APITEST_H_ + +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/utility.h" +#include "system_wrappers/include/event_wrapper.h" +#include "system_wrappers/include/rw_lock_wrapper.h" + +namespace webrtc { + +enum APITESTAction { + TEST_CHANGE_CODEC_ONLY = 0, + DTX_TEST = 1 +}; + +class APITest : public ACMTest { + public: + APITest(); + ~APITest(); + + void Perform(); + private: + int16_t SetUp(); + + static bool PushAudioThreadA(void* obj); + static bool PullAudioThreadA(void* obj); + static bool ProcessThreadA(void* obj); + static bool APIThreadA(void* obj); + + static bool PushAudioThreadB(void* obj); + static bool PullAudioThreadB(void* obj); + static bool ProcessThreadB(void* obj); + static bool APIThreadB(void* obj); + + void CheckVADStatus(char side); + + // Set Min delay, get delay, playout timestamp + void TestDelay(char side); + + // Unregister a codec & register again. + void TestRegisteration(char side); + + // Playout Mode, background noise mode. + // Receiver Frequency, playout frequency. + void TestPlayout(char receiveSide); + + // + void TestSendVAD(char side); + + void CurrentCodec(char side); + + void ChangeCodec(char side); + + void Wait(uint32_t waitLengthMs); + + void RunTest(char thread); + + bool PushAudioRunA(); + bool PullAudioRunA(); + bool ProcessRunA(); + bool APIRunA(); + + bool PullAudioRunB(); + bool PushAudioRunB(); + bool ProcessRunB(); + bool APIRunB(); + + //--- ACMs + std::unique_ptr _acmA; + std::unique_ptr _acmB; + + //--- Channels + Channel* _channel_A2B; + Channel* _channel_B2A; + + //--- I/O files + // A + PCMFile _inFileA; + PCMFile _outFileA; + // B + PCMFile _outFileB; + PCMFile _inFileB; + + //--- I/O params + // A + int32_t _outFreqHzA; + // B + int32_t _outFreqHzB; + + // Should we write to file. + // we might skip writing to file if we + // run the test for a long time. + bool _writeToFile; + //--- Events + // A + EventTimerWrapper* _pullEventA; // pulling data from ACM + EventTimerWrapper* _pushEventA; // pushing data to ACM + EventTimerWrapper* _processEventA; // process + EventWrapper* _apiEventA; // API calls + // B + EventTimerWrapper* _pullEventB; // pulling data from ACM + EventTimerWrapper* _pushEventB; // pushing data to ACM + EventTimerWrapper* _processEventB; // process + EventWrapper* _apiEventB; // API calls + + // keep track of the codec in either side. + uint8_t _codecCntrA; + uint8_t _codecCntrB; + + // Is set to true if there is no encoder in either side + bool _thereIsEncoderA; + bool _thereIsEncoderB; + bool _thereIsDecoderA; + bool _thereIsDecoderB; + + bool _sendVADA; + bool _sendDTXA; + ACMVADMode _sendVADModeA; + + bool _sendVADB; + bool _sendDTXB; + ACMVADMode _sendVADModeB; + + int32_t _minDelayA; + int32_t _minDelayB; + bool _payloadUsed[32]; + + bool _verbose; + + int _dotPositionA; + int _dotMoveDirectionA; + int _dotPositionB; + int _dotMoveDirectionB; + + char _movingDot[41]; + + VADCallback* _vadCallbackA; + VADCallback* _vadCallbackB; + RWLockWrapper& _apiTestRWLock; + bool _randomTest; + int _testNumA; + int _testNumB; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_APITEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.cc new file mode 100644 index 0000000000..7d5e6e291f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.cc @@ -0,0 +1,421 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/Channel.h" + +#include +#include + +#include "rtc_base/format_macros.h" +#include "rtc_base/timeutils.h" + +namespace webrtc { + +int32_t Channel::SendData(FrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + const uint8_t* payloadData, + size_t payloadSize, + const RTPFragmentationHeader* fragmentation) { + WebRtcRTPHeader rtpInfo; + int32_t status; + size_t payloadDataSize = payloadSize; + + rtpInfo.header.markerBit = false; + rtpInfo.header.ssrc = 0; + rtpInfo.header.sequenceNumber = (external_sequence_number_ < 0) ? + _seqNo++ : static_cast(external_sequence_number_); + rtpInfo.header.payloadType = payloadType; + rtpInfo.header.timestamp = (external_send_timestamp_ < 0) ? timeStamp : + static_cast(external_send_timestamp_); + + if (frameType == kAudioFrameCN) { + rtpInfo.type.Audio.isCNG = true; + } else { + rtpInfo.type.Audio.isCNG = false; + } + if (frameType == kEmptyFrame) { + // When frame is empty, we should not transmit it. The frame size of the + // next non-empty frame will be based on the previous frame size. + _useLastFrameSize = _lastFrameSizeSample > 0; + return 0; + } + + rtpInfo.type.Audio.channel = 1; + // Treat fragmentation separately + if (fragmentation != NULL) { + // If silence for too long, send only new data. + if ((fragmentation->fragmentationVectorSize == 2) && + (fragmentation->fragmentationTimeDiff[1] <= 0x3fff)) { + // only 0x80 if we have multiple blocks + _payloadData[0] = 0x80 + fragmentation->fragmentationPlType[1]; + size_t REDheader = (fragmentation->fragmentationTimeDiff[1] << 10) + + fragmentation->fragmentationLength[1]; + _payloadData[1] = uint8_t((REDheader >> 16) & 0x000000FF); + _payloadData[2] = uint8_t((REDheader >> 8) & 0x000000FF); + _payloadData[3] = uint8_t(REDheader & 0x000000FF); + + _payloadData[4] = fragmentation->fragmentationPlType[0]; + // copy the RED data + memcpy(_payloadData + 5, + payloadData + fragmentation->fragmentationOffset[1], + fragmentation->fragmentationLength[1]); + // copy the normal data + memcpy(_payloadData + 5 + fragmentation->fragmentationLength[1], + payloadData + fragmentation->fragmentationOffset[0], + fragmentation->fragmentationLength[0]); + payloadDataSize += 5; + } else { + // single block (newest one) + memcpy(_payloadData, payloadData + fragmentation->fragmentationOffset[0], + fragmentation->fragmentationLength[0]); + payloadDataSize = fragmentation->fragmentationLength[0]; + rtpInfo.header.payloadType = fragmentation->fragmentationPlType[0]; + } + } else { + memcpy(_payloadData, payloadData, payloadDataSize); + if (_isStereo) { + if (_leftChannel) { + memcpy(&_rtpInfo, &rtpInfo, sizeof(WebRtcRTPHeader)); + _leftChannel = false; + rtpInfo.type.Audio.channel = 1; + } else { + memcpy(&rtpInfo, &_rtpInfo, sizeof(WebRtcRTPHeader)); + _leftChannel = true; + rtpInfo.type.Audio.channel = 2; + } + } + } + + _channelCritSect.Enter(); + if (_saveBitStream) { + //fwrite(payloadData, sizeof(uint8_t), payloadSize, _bitStreamFile); + } + + if (!_isStereo) { + CalcStatistics(rtpInfo, payloadSize); + } + _useLastFrameSize = false; + _lastInTimestamp = timeStamp; + _totalBytes += payloadDataSize; + _channelCritSect.Leave(); + + if (_useFECTestWithPacketLoss) { + _packetLoss += 1; + if (_packetLoss == 3) { + _packetLoss = 0; + return 0; + } + } + + if (num_packets_to_drop_ > 0) { + num_packets_to_drop_--; + return 0; + } + + status = _receiverACM->IncomingPacket(_payloadData, payloadDataSize, rtpInfo); + + return status; +} + +// TODO(turajs): rewite this method. +void Channel::CalcStatistics(WebRtcRTPHeader& rtpInfo, size_t payloadSize) { + int n; + if ((rtpInfo.header.payloadType != _lastPayloadType) + && (_lastPayloadType != -1)) { + // payload-type is changed. + // we have to terminate the calculations on the previous payload type + // we ignore the last packet in that payload type just to make things + // easier. + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + if (_lastPayloadType == _payloadStats[n].payloadType) { + _payloadStats[n].newPacket = true; + break; + } + } + } + _lastPayloadType = rtpInfo.header.payloadType; + + bool newPayload = true; + ACMTestPayloadStats* currentPayloadStr = NULL; + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + if (rtpInfo.header.payloadType == _payloadStats[n].payloadType) { + newPayload = false; + currentPayloadStr = &_payloadStats[n]; + break; + } + } + + if (!newPayload) { + if (!currentPayloadStr->newPacket) { + if (!_useLastFrameSize) { + _lastFrameSizeSample = (uint32_t) ((uint32_t) rtpInfo.header.timestamp - + (uint32_t) currentPayloadStr->lastTimestamp); + } + assert(_lastFrameSizeSample > 0); + int k = 0; + for (; k < MAX_NUM_FRAMESIZES; ++k) { + if ((currentPayloadStr->frameSizeStats[k].frameSizeSample == + _lastFrameSizeSample) || + (currentPayloadStr->frameSizeStats[k].frameSizeSample == 0)) { + break; + } + } + if (k == MAX_NUM_FRAMESIZES) { + // New frame size found but no space to count statistics on it. Skip it. + printf("No memory to store statistics for payload %d : frame size %d\n", + _lastPayloadType, _lastFrameSizeSample); + return; + } + ACMTestFrameSizeStats* currentFrameSizeStats = &(currentPayloadStr + ->frameSizeStats[k]); + currentFrameSizeStats->frameSizeSample = (int16_t) _lastFrameSizeSample; + + // increment the number of encoded samples. + currentFrameSizeStats->totalEncodedSamples += _lastFrameSizeSample; + // increment the number of recveived packets + currentFrameSizeStats->numPackets++; + // increment the total number of bytes (this is based on + // the previous payload we don't know the frame-size of + // the current payload. + currentFrameSizeStats->totalPayloadLenByte += currentPayloadStr + ->lastPayloadLenByte; + // store the maximum payload-size (this is based on + // the previous payload we don't know the frame-size of + // the current payload. + if (currentFrameSizeStats->maxPayloadLen + < currentPayloadStr->lastPayloadLenByte) { + currentFrameSizeStats->maxPayloadLen = currentPayloadStr + ->lastPayloadLenByte; + } + // store the current values for the next time + currentPayloadStr->lastTimestamp = rtpInfo.header.timestamp; + currentPayloadStr->lastPayloadLenByte = payloadSize; + } else { + currentPayloadStr->newPacket = false; + currentPayloadStr->lastPayloadLenByte = payloadSize; + currentPayloadStr->lastTimestamp = rtpInfo.header.timestamp; + currentPayloadStr->payloadType = rtpInfo.header.payloadType; + memset(currentPayloadStr->frameSizeStats, 0, MAX_NUM_FRAMESIZES * + sizeof(ACMTestFrameSizeStats)); + } + } else { + n = 0; + while (_payloadStats[n].payloadType != -1) { + n++; + } + // first packet + _payloadStats[n].newPacket = false; + _payloadStats[n].lastPayloadLenByte = payloadSize; + _payloadStats[n].lastTimestamp = rtpInfo.header.timestamp; + _payloadStats[n].payloadType = rtpInfo.header.payloadType; + memset(_payloadStats[n].frameSizeStats, 0, MAX_NUM_FRAMESIZES * + sizeof(ACMTestFrameSizeStats)); + } +} + +Channel::Channel(int16_t chID) + : _receiverACM(NULL), + _seqNo(0), + _bitStreamFile(NULL), + _saveBitStream(false), + _lastPayloadType(-1), + _isStereo(false), + _leftChannel(true), + _lastInTimestamp(0), + _useLastFrameSize(false), + _lastFrameSizeSample(0), + _packetLoss(0), + _useFECTestWithPacketLoss(false), + _beginTime(rtc::TimeMillis()), + _totalBytes(0), + external_send_timestamp_(-1), + external_sequence_number_(-1), + num_packets_to_drop_(0) { + int n; + int k; + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + _payloadStats[n].payloadType = -1; + _payloadStats[n].newPacket = true; + for (k = 0; k < MAX_NUM_FRAMESIZES; k++) { + _payloadStats[n].frameSizeStats[k].frameSizeSample = 0; + _payloadStats[n].frameSizeStats[k].maxPayloadLen = 0; + _payloadStats[n].frameSizeStats[k].numPackets = 0; + _payloadStats[n].frameSizeStats[k].totalPayloadLenByte = 0; + _payloadStats[n].frameSizeStats[k].totalEncodedSamples = 0; + } + } + if (chID >= 0) { + _saveBitStream = true; + char bitStreamFileName[500]; + sprintf(bitStreamFileName, "bitStream_%d.dat", chID); + _bitStreamFile = fopen(bitStreamFileName, "wb"); + } else { + _saveBitStream = false; + } +} + +Channel::~Channel() { +} + +void Channel::RegisterReceiverACM(AudioCodingModule* acm) { + _receiverACM = acm; + return; +} + +void Channel::ResetStats() { + int n; + int k; + _channelCritSect.Enter(); + _lastPayloadType = -1; + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + _payloadStats[n].payloadType = -1; + _payloadStats[n].newPacket = true; + for (k = 0; k < MAX_NUM_FRAMESIZES; k++) { + _payloadStats[n].frameSizeStats[k].frameSizeSample = 0; + _payloadStats[n].frameSizeStats[k].maxPayloadLen = 0; + _payloadStats[n].frameSizeStats[k].numPackets = 0; + _payloadStats[n].frameSizeStats[k].totalPayloadLenByte = 0; + _payloadStats[n].frameSizeStats[k].totalEncodedSamples = 0; + } + } + _beginTime = rtc::TimeMillis(); + _totalBytes = 0; + _channelCritSect.Leave(); +} + +int16_t Channel::Stats(CodecInst& codecInst, + ACMTestPayloadStats& payloadStats) { + _channelCritSect.Enter(); + int n; + payloadStats.payloadType = -1; + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + if (_payloadStats[n].payloadType == codecInst.pltype) { + memcpy(&payloadStats, &_payloadStats[n], sizeof(ACMTestPayloadStats)); + break; + } + } + if (payloadStats.payloadType == -1) { + _channelCritSect.Leave(); + return -1; + } + for (n = 0; n < MAX_NUM_FRAMESIZES; n++) { + if (payloadStats.frameSizeStats[n].frameSizeSample == 0) { + _channelCritSect.Leave(); + return 0; + } + payloadStats.frameSizeStats[n].usageLenSec = (double) payloadStats + .frameSizeStats[n].totalEncodedSamples / (double) codecInst.plfreq; + + payloadStats.frameSizeStats[n].rateBitPerSec = + payloadStats.frameSizeStats[n].totalPayloadLenByte * 8 + / payloadStats.frameSizeStats[n].usageLenSec; + + } + _channelCritSect.Leave(); + return 0; +} + +void Channel::Stats(uint32_t* numPackets) { + _channelCritSect.Enter(); + int k; + int n; + memset(numPackets, 0, MAX_NUM_PAYLOADS * sizeof(uint32_t)); + for (k = 0; k < MAX_NUM_PAYLOADS; k++) { + if (_payloadStats[k].payloadType == -1) { + break; + } + numPackets[k] = 0; + for (n = 0; n < MAX_NUM_FRAMESIZES; n++) { + if (_payloadStats[k].frameSizeStats[n].frameSizeSample == 0) { + break; + } + numPackets[k] += _payloadStats[k].frameSizeStats[n].numPackets; + } + } + _channelCritSect.Leave(); +} + +void Channel::Stats(uint8_t* payloadType, uint32_t* payloadLenByte) { + _channelCritSect.Enter(); + + int k; + int n; + memset(payloadLenByte, 0, MAX_NUM_PAYLOADS * sizeof(uint32_t)); + for (k = 0; k < MAX_NUM_PAYLOADS; k++) { + if (_payloadStats[k].payloadType == -1) { + break; + } + payloadType[k] = (uint8_t) _payloadStats[k].payloadType; + payloadLenByte[k] = 0; + for (n = 0; n < MAX_NUM_FRAMESIZES; n++) { + if (_payloadStats[k].frameSizeStats[n].frameSizeSample == 0) { + break; + } + payloadLenByte[k] += (uint16_t) _payloadStats[k].frameSizeStats[n] + .totalPayloadLenByte; + } + } + + _channelCritSect.Leave(); +} + +void Channel::PrintStats(CodecInst& codecInst) { + ACMTestPayloadStats payloadStats; + Stats(codecInst, payloadStats); + printf("%s %d kHz\n", codecInst.plname, codecInst.plfreq / 1000); + printf("=====================================================\n"); + if (payloadStats.payloadType == -1) { + printf("No Packets are sent with payload-type %d (%s)\n\n", + codecInst.pltype, codecInst.plname); + return; + } + for (int k = 0; k < MAX_NUM_FRAMESIZES; k++) { + if (payloadStats.frameSizeStats[k].frameSizeSample == 0) { + break; + } + printf("Frame-size.................... %d samples\n", + payloadStats.frameSizeStats[k].frameSizeSample); + printf("Average Rate.................. %.0f bits/sec\n", + payloadStats.frameSizeStats[k].rateBitPerSec); + printf("Maximum Payload-Size.......... %" PRIuS " Bytes\n", + payloadStats.frameSizeStats[k].maxPayloadLen); + printf( + "Maximum Instantaneous Rate.... %.0f bits/sec\n", + ((double) payloadStats.frameSizeStats[k].maxPayloadLen * 8.0 + * (double) codecInst.plfreq) + / (double) payloadStats.frameSizeStats[k].frameSizeSample); + printf("Number of Packets............. %u\n", + (unsigned int) payloadStats.frameSizeStats[k].numPackets); + printf("Duration...................... %0.3f sec\n\n", + payloadStats.frameSizeStats[k].usageLenSec); + + } + +} + +uint32_t Channel::LastInTimestamp() { + uint32_t timestamp; + _channelCritSect.Enter(); + timestamp = _lastInTimestamp; + _channelCritSect.Leave(); + return timestamp; +} + +double Channel::BitRate() { + double rate; + uint64_t currTime = rtc::TimeMillis(); + _channelCritSect.Enter(); + rate = ((double) _totalBytes * 8.0) / (double) (currTime - _beginTime); + _channelCritSect.Leave(); + return rate; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.h new file mode 100644 index 0000000000..e01e33e43b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/Channel.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_CHANNEL_H_ +#define MODULES_AUDIO_CODING_TEST_CHANNEL_H_ + +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/criticalsection.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#define MAX_NUM_PAYLOADS 50 +#define MAX_NUM_FRAMESIZES 6 + +// TODO(turajs): Write constructor for this structure. +struct ACMTestFrameSizeStats { + uint16_t frameSizeSample; + size_t maxPayloadLen; + uint32_t numPackets; + uint64_t totalPayloadLenByte; + uint64_t totalEncodedSamples; + double rateBitPerSec; + double usageLenSec; +}; + +// TODO(turajs): Write constructor for this structure. +struct ACMTestPayloadStats { + bool newPacket; + int16_t payloadType; + size_t lastPayloadLenByte; + uint32_t lastTimestamp; + ACMTestFrameSizeStats frameSizeStats[MAX_NUM_FRAMESIZES]; +}; + +class Channel : public AudioPacketizationCallback { + public: + + Channel(int16_t chID = -1); + ~Channel() override; + + int32_t SendData(FrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + const uint8_t* payloadData, + size_t payloadSize, + const RTPFragmentationHeader* fragmentation) override; + + void RegisterReceiverACM(AudioCodingModule *acm); + + void ResetStats(); + + int16_t Stats(CodecInst& codecInst, ACMTestPayloadStats& payloadStats); + + void Stats(uint32_t* numPackets); + + void Stats(uint8_t* payloadType, uint32_t* payloadLenByte); + + void PrintStats(CodecInst& codecInst); + + void SetIsStereo(bool isStereo) { + _isStereo = isStereo; + } + + uint32_t LastInTimestamp(); + + void SetFECTestWithPacketLoss(bool usePacketLoss) { + _useFECTestWithPacketLoss = usePacketLoss; + } + + double BitRate(); + + void set_send_timestamp(uint32_t new_send_ts) { + external_send_timestamp_ = new_send_ts; + } + + void set_sequence_number(uint16_t new_sequence_number) { + external_sequence_number_ = new_sequence_number; + } + + void set_num_packets_to_drop(int new_num_packets_to_drop) { + num_packets_to_drop_ = new_num_packets_to_drop; + } + + private: + void CalcStatistics(WebRtcRTPHeader& rtpInfo, size_t payloadSize); + + AudioCodingModule* _receiverACM; + uint16_t _seqNo; + // 60msec * 32 sample(max)/msec * 2 description (maybe) * 2 bytes/sample + uint8_t _payloadData[60 * 32 * 2 * 2]; + + rtc::CriticalSection _channelCritSect; + FILE* _bitStreamFile; + bool _saveBitStream; + int16_t _lastPayloadType; + ACMTestPayloadStats _payloadStats[MAX_NUM_PAYLOADS]; + bool _isStereo; + WebRtcRTPHeader _rtpInfo; + bool _leftChannel; + uint32_t _lastInTimestamp; + bool _useLastFrameSize; + uint32_t _lastFrameSizeSample; + // FEC Test variables + int16_t _packetLoss; + bool _useFECTestWithPacketLoss; + uint64_t _beginTime; + uint64_t _totalBytes; + + // External timing info, defaulted to -1. Only used if they are + // non-negative. + int64_t external_send_timestamp_; + int32_t external_sequence_number_; + int num_packets_to_drop_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_CHANNEL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.cc new file mode 100644 index 0000000000..c765f68502 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.cc @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/EncodeDecodeTest.h" + +#include +#include +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/utility.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +TestPacketization::TestPacketization(RTPStream *rtpStream, uint16_t frequency) + : _rtpStream(rtpStream), + _frequency(frequency), + _seqNo(0) { +} + +TestPacketization::~TestPacketization() { +} + +int32_t TestPacketization::SendData( + const FrameType /* frameType */, const uint8_t payloadType, + const uint32_t timeStamp, const uint8_t* payloadData, + const size_t payloadSize, + const RTPFragmentationHeader* /* fragmentation */) { + _rtpStream->Write(payloadType, timeStamp, _seqNo++, payloadData, payloadSize, + _frequency); + return 1; +} + +Sender::Sender() + : _acm(NULL), + _pcmFile(), + _audioFrame(), + _packetization(NULL) { +} + +void Sender::Setup(AudioCodingModule *acm, RTPStream *rtpStream, + std::string in_file_name, int sample_rate, size_t channels) { + struct CodecInst sendCodec; + int noOfCodecs = acm->NumberOfCodecs(); + int codecNo; + + // Open input file + const std::string file_name = webrtc::test::ResourcePath(in_file_name, "pcm"); + _pcmFile.Open(file_name, sample_rate, "rb"); + if (channels == 2) { + _pcmFile.ReadStereo(true); + } + // Set test length to 500 ms (50 blocks of 10 ms each). + _pcmFile.SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the file starts with silence. + _pcmFile.FastForward(100); + + // Set the codec for the current test. + if ((testMode == 0) || (testMode == 1)) { + // Set the codec id. + codecNo = codeId; + } else { + // Choose codec on command line. + printf("List of supported codec.\n"); + for (int n = 0; n < noOfCodecs; n++) { + EXPECT_EQ(0, acm->Codec(n, &sendCodec)); + printf("%d %s\n", n, sendCodec.plname); + } + printf("Choose your codec:"); + ASSERT_GT(scanf("%d", &codecNo), 0); + } + + EXPECT_EQ(0, acm->Codec(codecNo, &sendCodec)); + + sendCodec.channels = channels; + + EXPECT_EQ(0, acm->RegisterSendCodec(sendCodec)); + _packetization = new TestPacketization(rtpStream, sendCodec.plfreq); + EXPECT_EQ(0, acm->RegisterTransportCallback(_packetization)); + + _acm = acm; +} + +void Sender::Teardown() { + _pcmFile.Close(); + delete _packetization; +} + +bool Sender::Add10MsData() { + if (!_pcmFile.EndOfFile()) { + EXPECT_GT(_pcmFile.Read10MsData(_audioFrame), 0); + int32_t ok = _acm->Add10MsData(_audioFrame); + EXPECT_GE(ok, 0); + return ok >= 0 ? true : false; + } + return false; +} + +void Sender::Run() { + while (true) { + if (!Add10MsData()) { + break; + } + } +} + +Receiver::Receiver() + : _playoutLengthSmpls(WEBRTC_10MS_PCM_AUDIO), + _payloadSizeBytes(MAX_INCOMING_PAYLOAD) { +} + +void Receiver::Setup(AudioCodingModule *acm, RTPStream *rtpStream, + std::string out_file_name, size_t channels) { + struct CodecInst recvCodec = CodecInst(); + int noOfCodecs; + EXPECT_EQ(0, acm->InitializeReceiver()); + + noOfCodecs = acm->NumberOfCodecs(); + for (int i = 0; i < noOfCodecs; i++) { + EXPECT_EQ(0, acm->Codec(i, &recvCodec)); + if (recvCodec.channels == channels) + EXPECT_EQ(true, acm->RegisterReceiveCodec(recvCodec.pltype, + CodecInstToSdp(recvCodec))); + // Forces mono/stereo for Opus. + if (!strcmp(recvCodec.plname, "opus")) { + recvCodec.channels = channels; + EXPECT_EQ(true, acm->RegisterReceiveCodec(recvCodec.pltype, + CodecInstToSdp(recvCodec))); + } + } + + int playSampFreq; + std::string file_name; + std::stringstream file_stream; + file_stream << webrtc::test::OutputPath() << out_file_name + << static_cast(codeId) << ".pcm"; + file_name = file_stream.str(); + _rtpStream = rtpStream; + + if (testMode == 1) { + playSampFreq = recvCodec.plfreq; + _pcmFile.Open(file_name, recvCodec.plfreq, "wb+"); + } else if (testMode == 0) { + playSampFreq = 32000; + _pcmFile.Open(file_name, 32000, "wb+"); + } else { + printf("\nValid output frequencies:\n"); + printf("8000\n16000\n32000\n-1,"); + printf("which means output frequency equal to received signal frequency"); + printf("\n\nChoose output sampling frequency: "); + ASSERT_GT(scanf("%d", &playSampFreq), 0); + file_name = webrtc::test::OutputPath() + out_file_name + ".pcm"; + _pcmFile.Open(file_name, playSampFreq, "wb+"); + } + + _realPayloadSizeBytes = 0; + _playoutBuffer = new int16_t[WEBRTC_10MS_PCM_AUDIO]; + _frequency = playSampFreq; + _acm = acm; + _firstTime = true; +} + +void Receiver::Teardown() { + delete[] _playoutBuffer; + _pcmFile.Close(); +} + +bool Receiver::IncomingPacket() { + if (!_rtpStream->EndOfFile()) { + if (_firstTime) { + _firstTime = false; + _realPayloadSizeBytes = _rtpStream->Read(&_rtpInfo, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0) { + if (_rtpStream->EndOfFile()) { + _firstTime = true; + return true; + } else { + return false; + } + } + } + + EXPECT_EQ(0, _acm->IncomingPacket(_incomingPayload, _realPayloadSizeBytes, + _rtpInfo)); + _realPayloadSizeBytes = _rtpStream->Read(&_rtpInfo, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0 && _rtpStream->EndOfFile()) { + _firstTime = true; + } + } + return true; +} + +bool Receiver::PlayoutData() { + AudioFrame audioFrame; + bool muted; + int32_t ok = _acm->PlayoutData10Ms(_frequency, &audioFrame, &muted); + if (muted) { + ADD_FAILURE(); + return false; + } + EXPECT_EQ(0, ok); + if (ok < 0){ + return false; + } + if (_playoutLengthSmpls == 0) { + return false; + } + _pcmFile.Write10MsData(audioFrame.data(), + audioFrame.samples_per_channel_ * audioFrame.num_channels_); + return true; +} + +void Receiver::Run() { + uint8_t counter500Ms = 50; + uint32_t clock = 0; + + while (counter500Ms > 0) { + if (clock == 0 || clock >= _nextTime) { + EXPECT_TRUE(IncomingPacket()); + if (clock == 0) { + clock = _nextTime; + } + } + if ((clock % 10) == 0) { + if (!PlayoutData()) { + clock++; + continue; + } + } + if (_rtpStream->EndOfFile()) { + counter500Ms--; + } + clock++; + } +} + +EncodeDecodeTest::EncodeDecodeTest() { + _testMode = 2; +} + +EncodeDecodeTest::EncodeDecodeTest(int testMode) { + //testMode == 0 for autotest + //testMode == 1 for testing all codecs/parameters + //testMode > 1 for specific user-input test (as it was used before) + _testMode = testMode; +} + +void EncodeDecodeTest::Perform() { + int numCodecs = 1; + int codePars[3]; // Frequency, packet size, rate. + int numPars[52]; // Number of codec parameters sets (freq, pacsize, rate) + // to test, for a given codec. + + codePars[0] = 0; + codePars[1] = 0; + codePars[2] = 0; + + std::unique_ptr acm(AudioCodingModule::Create()); + struct CodecInst sendCodecTmp; + numCodecs = acm->NumberOfCodecs(); + + if (_testMode != 2) { + for (int n = 0; n < numCodecs; n++) { + EXPECT_EQ(0, acm->Codec(n, &sendCodecTmp)); + if (STR_CASE_CMP(sendCodecTmp.plname, "telephone-event") == 0) { + numPars[n] = 0; + } else if (STR_CASE_CMP(sendCodecTmp.plname, "cn") == 0) { + numPars[n] = 0; + } else if (STR_CASE_CMP(sendCodecTmp.plname, "red") == 0) { + numPars[n] = 0; + } else if (sendCodecTmp.channels == 2) { + numPars[n] = 0; + } else { + numPars[n] = 1; + } + } + } else { + numCodecs = 1; + numPars[0] = 1; + } + + _receiver.testMode = _testMode; + + // Loop over all mono codecs: + for (int codeId = 0; codeId < numCodecs; codeId++) { + // Only encode using real mono encoders, not telephone-event and cng. + for (int loopPars = 1; loopPars <= numPars[codeId]; loopPars++) { + // Encode all data to file. + std::string fileName = EncodeToFile(1, codeId, codePars, _testMode); + + RTPFile rtpFile; + rtpFile.Open(fileName.c_str(), "rb"); + + _receiver.codeId = codeId; + + rtpFile.ReadHeader(); + _receiver.Setup(acm.get(), &rtpFile, "encodeDecode_out", 1); + _receiver.Run(); + _receiver.Teardown(); + rtpFile.Close(); + } + } +} + +std::string EncodeDecodeTest::EncodeToFile(int fileType, + int codeId, + int* codePars, + int testMode) { + std::unique_ptr acm(AudioCodingModule::Create()); + RTPFile rtpFile; + std::string fileName = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "encode_decode_rtp"); + rtpFile.Open(fileName.c_str(), "wb+"); + rtpFile.WriteHeader(); + + // Store for auto_test and logging. + _sender.testMode = testMode; + _sender.codeId = codeId; + + _sender.Setup(acm.get(), &rtpFile, "audio_coding/testfile32kHz", 32000, 1); + if (acm->SendCodec()) { + _sender.Run(); + } + _sender.Teardown(); + rtpFile.Close(); + + return fileName; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.h new file mode 100644 index 0000000000..5351a2a4bf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/EncodeDecodeTest.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_ENCODEDECODETEST_H_ +#define MODULES_AUDIO_CODING_TEST_ENCODEDECODETEST_H_ + +#include +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/RTPFile.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#define MAX_INCOMING_PAYLOAD 8096 + +// TestPacketization callback which writes the encoded payloads to file +class TestPacketization : public AudioPacketizationCallback { + public: + TestPacketization(RTPStream *rtpStream, uint16_t frequency); + ~TestPacketization(); + int32_t SendData(const FrameType frameType, + const uint8_t payloadType, + const uint32_t timeStamp, + const uint8_t* payloadData, + const size_t payloadSize, + const RTPFragmentationHeader* fragmentation) override; + + private: + static void MakeRTPheader(uint8_t* rtpHeader, uint8_t payloadType, + int16_t seqNo, uint32_t timeStamp, uint32_t ssrc); + RTPStream* _rtpStream; + int32_t _frequency; + int16_t _seqNo; +}; + +class Sender { + public: + Sender(); + void Setup(AudioCodingModule *acm, RTPStream *rtpStream, + std::string in_file_name, int sample_rate, size_t channels); + void Teardown(); + void Run(); + bool Add10MsData(); + + //for auto_test and logging + uint8_t testMode; + uint8_t codeId; + + protected: + AudioCodingModule* _acm; + + private: + PCMFile _pcmFile; + AudioFrame _audioFrame; + TestPacketization* _packetization; +}; + +class Receiver { + public: + Receiver(); + virtual ~Receiver() {}; + void Setup(AudioCodingModule *acm, RTPStream *rtpStream, + std::string out_file_name, size_t channels); + void Teardown(); + void Run(); + virtual bool IncomingPacket(); + bool PlayoutData(); + + //for auto_test and logging + uint8_t codeId; + uint8_t testMode; + + private: + PCMFile _pcmFile; + int16_t* _playoutBuffer; + uint16_t _playoutLengthSmpls; + int32_t _frequency; + bool _firstTime; + + protected: + AudioCodingModule* _acm; + uint8_t _incomingPayload[MAX_INCOMING_PAYLOAD]; + RTPStream* _rtpStream; + WebRtcRTPHeader _rtpInfo; + size_t _realPayloadSizeBytes; + size_t _payloadSizeBytes; + uint32_t _nextTime; +}; + +class EncodeDecodeTest : public ACMTest { + public: + EncodeDecodeTest(); + explicit EncodeDecodeTest(int testMode); + void Perform() override; + + uint16_t _playoutFreq; + uint8_t _testMode; + + private: + std::string EncodeToFile(int fileType, + int codeId, + int* codePars, + int testMode); + + protected: + Sender _sender; + Receiver _receiver; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_ENCODEDECODETEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.cc new file mode 100644 index 0000000000..e9e9430a33 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/PCMFile.h" + +#include +#include +#include + +#include "modules/include/module_common_types.h" +#include "test/gtest.h" + +namespace webrtc { + +#define MAX_FILE_NAME_LENGTH_BYTE 500 + +PCMFile::PCMFile() + : pcm_file_(NULL), + samples_10ms_(160), + frequency_(16000), + end_of_file_(false), + auto_rewind_(false), + rewinded_(false), + read_stereo_(false), + save_stereo_(false) { + timestamp_ = (((uint32_t) rand() & 0x0000FFFF) << 16) | + ((uint32_t) rand() & 0x0000FFFF); +} + +PCMFile::PCMFile(uint32_t timestamp) + : pcm_file_(NULL), + samples_10ms_(160), + frequency_(16000), + end_of_file_(false), + auto_rewind_(false), + rewinded_(false), + read_stereo_(false), + save_stereo_(false) { + timestamp_ = timestamp; +} + +PCMFile::~PCMFile() { + if (pcm_file_) { + fclose(pcm_file_); + } +} + +int16_t PCMFile::ChooseFile(std::string* file_name, int16_t max_len, + uint16_t* frequency_hz) { + char tmp_name[MAX_FILE_NAME_LENGTH_BYTE]; + + EXPECT_TRUE(fgets(tmp_name, MAX_FILE_NAME_LENGTH_BYTE, stdin) != NULL); + tmp_name[MAX_FILE_NAME_LENGTH_BYTE - 1] = '\0'; + int16_t n = 0; + + // Removing trailing spaces. + while ((isspace(tmp_name[n]) || iscntrl(tmp_name[n])) && (tmp_name[n] != 0) + && (n < MAX_FILE_NAME_LENGTH_BYTE)) { + n++; + } + if (n > 0) { + memmove(tmp_name, &tmp_name[n], MAX_FILE_NAME_LENGTH_BYTE - n); + } + + // Removing trailing spaces. + n = (int16_t)(strlen(tmp_name) - 1); + if (n >= 0) { + while ((isspace(tmp_name[n]) || iscntrl(tmp_name[n])) && (n >= 0)) { + n--; + } + } + if (n >= 0) { + tmp_name[n + 1] = '\0'; + } + + int16_t len = (int16_t) strlen(tmp_name); + if (len > max_len) { + return -1; + } + if (len > 0) { + std::string tmp_string(tmp_name, len + 1); + *file_name = tmp_string; + } + printf("Enter the sampling frequency (in Hz) of the above file [%u]: ", + *frequency_hz); + EXPECT_TRUE(fgets(tmp_name, 10, stdin) != NULL); + uint16_t tmp_frequency = (uint16_t) atoi(tmp_name); + if (tmp_frequency > 0) { + *frequency_hz = tmp_frequency; + } + return 0; +} + +void PCMFile::Open(const std::string& file_name, uint16_t frequency, + const char* mode, bool auto_rewind) { + if ((pcm_file_ = fopen(file_name.c_str(), mode)) == NULL) { + printf("Cannot open file %s.\n", file_name.c_str()); + ADD_FAILURE() << "Unable to read file"; + } + frequency_ = frequency; + samples_10ms_ = (uint16_t)(frequency_ / 100); + auto_rewind_ = auto_rewind; + end_of_file_ = false; + rewinded_ = false; +} + +int32_t PCMFile::SamplingFrequency() const { + return frequency_; +} + +uint16_t PCMFile::PayloadLength10Ms() const { + return samples_10ms_; +} + +int32_t PCMFile::Read10MsData(AudioFrame& audio_frame) { + uint16_t channels = 1; + if (read_stereo_) { + channels = 2; + } + + int32_t payload_size = (int32_t) fread(audio_frame.mutable_data(), + sizeof(uint16_t), + samples_10ms_ * channels, pcm_file_); + if (payload_size < samples_10ms_ * channels) { + int16_t* frame_data = audio_frame.mutable_data(); + for (int k = payload_size; k < samples_10ms_ * channels; k++) { + frame_data[k] = 0; + } + if (auto_rewind_) { + rewind(pcm_file_); + rewinded_ = true; + } else { + end_of_file_ = true; + } + } + audio_frame.samples_per_channel_ = samples_10ms_; + audio_frame.sample_rate_hz_ = frequency_; + audio_frame.num_channels_ = channels; + audio_frame.timestamp_ = timestamp_; + timestamp_ += samples_10ms_; + ++blocks_read_; + if (num_10ms_blocks_to_read_ && blocks_read_ >= *num_10ms_blocks_to_read_) + end_of_file_ = true; + return samples_10ms_; +} + +void PCMFile::Write10MsData(const AudioFrame& audio_frame) { + if (audio_frame.num_channels_ == 1) { + if (!save_stereo_) { + if (fwrite(audio_frame.data(), sizeof(uint16_t), + audio_frame.samples_per_channel_, pcm_file_) != + static_cast(audio_frame.samples_per_channel_)) { + return; + } + } else { + const int16_t* frame_data = audio_frame.data(); + int16_t* stereo_audio = new int16_t[2 * audio_frame.samples_per_channel_]; + for (size_t k = 0; k < audio_frame.samples_per_channel_; k++) { + stereo_audio[k << 1] = frame_data[k]; + stereo_audio[(k << 1) + 1] = frame_data[k]; + } + if (fwrite(stereo_audio, sizeof(int16_t), + 2 * audio_frame.samples_per_channel_, pcm_file_) != + static_cast(2 * audio_frame.samples_per_channel_)) { + return; + } + delete[] stereo_audio; + } + } else { + if (fwrite(audio_frame.data(), sizeof(int16_t), + audio_frame.num_channels_ * audio_frame.samples_per_channel_, + pcm_file_) != + static_cast(audio_frame.num_channels_ * + audio_frame.samples_per_channel_)) { + return; + } + } +} + +void PCMFile::Write10MsData(const int16_t* playout_buffer, + size_t length_smpls) { + if (fwrite(playout_buffer, sizeof(uint16_t), length_smpls, pcm_file_) != + length_smpls) { + return; + } +} + +void PCMFile::Close() { + fclose(pcm_file_); + pcm_file_ = NULL; + blocks_read_ = 0; +} + +void PCMFile::FastForward(int num_10ms_blocks) { + const int channels = read_stereo_ ? 2 : 1; + long num_bytes_to_move = + num_10ms_blocks * sizeof(int16_t) * samples_10ms_ * channels; + int error = fseek(pcm_file_, num_bytes_to_move, SEEK_CUR); + RTC_DCHECK_EQ(error, 0); +} + +void PCMFile::Rewind() { + rewind(pcm_file_); + end_of_file_ = false; + blocks_read_ = 0; +} + +bool PCMFile::Rewinded() { + return rewinded_; +} + +void PCMFile::SaveStereo(bool is_stereo) { + save_stereo_ = is_stereo; +} + +void PCMFile::ReadStereo(bool is_stereo) { + read_stereo_ = is_stereo; +} + +void PCMFile::SetNum10MsBlocksToRead(int value) { + num_10ms_blocks_to_read_ = value; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.h new file mode 100644 index 0000000000..bbf2571980 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PCMFile.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_PCMFILE_H_ +#define MODULES_AUDIO_CODING_TEST_PCMFILE_H_ + +#include +#include + +#include + +#include "api/optional.h" +#include "modules/include/module_common_types.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class PCMFile { + public: + PCMFile(); + PCMFile(uint32_t timestamp); + ~PCMFile(); + + void Open(const std::string& filename, uint16_t frequency, const char* mode, + bool auto_rewind = false); + + int32_t Read10MsData(AudioFrame& audio_frame); + + void Write10MsData(const int16_t *playout_buffer, size_t length_smpls); + void Write10MsData(const AudioFrame& audio_frame); + + uint16_t PayloadLength10Ms() const; + int32_t SamplingFrequency() const; + void Close(); + bool EndOfFile() const { + return end_of_file_; + } + // Moves forward the specified number of 10 ms blocks. If a limit has been set + // with SetNum10MsBlocksToRead, fast-forwarding does not count towards this + // limit. + void FastForward(int num_10ms_blocks); + void Rewind(); + static int16_t ChooseFile(std::string* file_name, int16_t max_len, + uint16_t* frequency_hz); + bool Rewinded(); + void SaveStereo(bool is_stereo = true); + void ReadStereo(bool is_stereo = true); + // If set, the reading will stop after the specified number of blocks have + // been read. When that has happened, EndOfFile() will return true. Calling + // Rewind() will reset the counter and start over. + void SetNum10MsBlocksToRead(int value); + + private: + FILE* pcm_file_; + uint16_t samples_10ms_; + int32_t frequency_; + bool end_of_file_; + bool auto_rewind_; + bool rewinded_; + uint32_t timestamp_; + bool read_stereo_; + bool save_stereo_; + rtc::Optional num_10ms_blocks_to_read_; + int blocks_read_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_PCMFILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.cc new file mode 100644 index 0000000000..a6c56fa780 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/PacketLossTest.h" + +#include + +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +ReceiverWithPacketLoss::ReceiverWithPacketLoss() + : loss_rate_(0), + burst_length_(1), + packet_counter_(0), + lost_packet_counter_(0), + burst_lost_counter_(burst_length_) { +} + +void ReceiverWithPacketLoss::Setup(AudioCodingModule *acm, + RTPStream *rtpStream, + std::string out_file_name, + int channels, + int loss_rate, + int burst_length) { + loss_rate_ = loss_rate; + burst_length_ = burst_length; + burst_lost_counter_ = burst_length_; // To prevent first packet gets lost. + std::stringstream ss; + ss << out_file_name << "_" << loss_rate_ << "_" << burst_length_ << "_"; + Receiver::Setup(acm, rtpStream, ss.str(), channels); +} + +bool ReceiverWithPacketLoss::IncomingPacket() { + if (!_rtpStream->EndOfFile()) { + if (packet_counter_ == 0) { + _realPayloadSizeBytes = _rtpStream->Read(&_rtpInfo, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0) { + if (_rtpStream->EndOfFile()) { + packet_counter_ = 0; + return true; + } else { + return false; + } + } + } + + if (!PacketLost()) { + _acm->IncomingPacket(_incomingPayload, _realPayloadSizeBytes, _rtpInfo); + } + packet_counter_++; + _realPayloadSizeBytes = _rtpStream->Read(&_rtpInfo, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0 && _rtpStream->EndOfFile()) { + packet_counter_ = 0; + lost_packet_counter_ = 0; + } + } + return true; +} + +bool ReceiverWithPacketLoss::PacketLost() { + if (burst_lost_counter_ < burst_length_) { + lost_packet_counter_++; + burst_lost_counter_++; + return true; + } + + if (lost_packet_counter_ * 100 < loss_rate_ * packet_counter_) { + lost_packet_counter_++; + burst_lost_counter_ = 1; + return true; + } + return false; +} + +SenderWithFEC::SenderWithFEC() + : expected_loss_rate_(0) { +} + +void SenderWithFEC::Setup(AudioCodingModule *acm, RTPStream *rtpStream, + std::string in_file_name, int sample_rate, + int channels, int expected_loss_rate) { + Sender::Setup(acm, rtpStream, in_file_name, sample_rate, channels); + EXPECT_TRUE(SetFEC(true)); + EXPECT_TRUE(SetPacketLossRate(expected_loss_rate)); +} + +bool SenderWithFEC::SetFEC(bool enable_fec) { + if (_acm->SetCodecFEC(enable_fec) == 0) { + return true; + } + return false; +} + +bool SenderWithFEC::SetPacketLossRate(int expected_loss_rate) { + if (_acm->SetPacketLossRate(expected_loss_rate) == 0) { + expected_loss_rate_ = expected_loss_rate; + return true; + } + return false; +} + +PacketLossTest::PacketLossTest(int channels, int expected_loss_rate, + int actual_loss_rate, int burst_length) + : channels_(channels), + in_file_name_(channels_ == 1 ? "audio_coding/testfile32kHz" : + "audio_coding/teststereo32kHz"), + sample_rate_hz_(32000), + sender_(new SenderWithFEC), + receiver_(new ReceiverWithPacketLoss), + expected_loss_rate_(expected_loss_rate), + actual_loss_rate_(actual_loss_rate), + burst_length_(burst_length) { +} + +void PacketLossTest::Perform() { +#ifndef WEBRTC_CODEC_OPUS + return; +#else + std::unique_ptr acm(AudioCodingModule::Create()); + + int codec_id = acm->Codec("opus", 48000, channels_); + + RTPFile rtpFile; + std::string fileName = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "packet_loss_test"); + + // Encode to file + rtpFile.Open(fileName.c_str(), "wb+"); + rtpFile.WriteHeader(); + + sender_->testMode = 0; + sender_->codeId = codec_id; + + sender_->Setup(acm.get(), &rtpFile, in_file_name_, sample_rate_hz_, channels_, + expected_loss_rate_); + if (acm->SendCodec()) { + sender_->Run(); + } + sender_->Teardown(); + rtpFile.Close(); + + // Decode to file + rtpFile.Open(fileName.c_str(), "rb"); + rtpFile.ReadHeader(); + + receiver_->testMode = 0; + receiver_->codeId = codec_id; + + receiver_->Setup(acm.get(), &rtpFile, "packetLoss_out", channels_, + actual_loss_rate_, burst_length_); + receiver_->Run(); + receiver_->Teardown(); + rtpFile.Close(); +#endif +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.h new file mode 100644 index 0000000000..7eab442b59 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/PacketLossTest.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_PACKETLOSSTEST_H_ +#define MODULES_AUDIO_CODING_TEST_PACKETLOSSTEST_H_ + +#include +#include +#include "modules/audio_coding/test/EncodeDecodeTest.h" + +namespace webrtc { + +class ReceiverWithPacketLoss : public Receiver { + public: + ReceiverWithPacketLoss(); + void Setup(AudioCodingModule *acm, RTPStream *rtpStream, + std::string out_file_name, int channels, int loss_rate, + int burst_length); + bool IncomingPacket() override; + + protected: + bool PacketLost(); + int loss_rate_; + int burst_length_; + int packet_counter_; + int lost_packet_counter_; + int burst_lost_counter_; +}; + +class SenderWithFEC : public Sender { + public: + SenderWithFEC(); + void Setup(AudioCodingModule *acm, RTPStream *rtpStream, + std::string in_file_name, int sample_rate, int channels, + int expected_loss_rate); + bool SetPacketLossRate(int expected_loss_rate); + bool SetFEC(bool enable_fec); + protected: + int expected_loss_rate_; +}; + +class PacketLossTest : public ACMTest { + public: + PacketLossTest(int channels, int expected_loss_rate_, int actual_loss_rate, + int burst_length); + void Perform(); + protected: + int channels_; + std::string in_file_name_; + int sample_rate_hz_; + std::unique_ptr sender_; + std::unique_ptr receiver_; + int expected_loss_rate_; + int actual_loss_rate_; + int burst_length_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_PACKETLOSSTEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.cc new file mode 100644 index 0000000000..38f304ea7c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.cc @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "RTPFile.h" + +#include +#include + +#ifdef WIN32 +# include +#else +# include +#endif + +#include "audio_coding_module.h" +#include "system_wrappers/include/rw_lock_wrapper.h" +// TODO(tlegrand): Consider removing usage of gtest. +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +void RTPStream::ParseRTPHeader(WebRtcRTPHeader* rtpInfo, + const uint8_t* rtpHeader) { + rtpInfo->header.payloadType = rtpHeader[1]; + rtpInfo->header.sequenceNumber = (static_cast(rtpHeader[2]) << 8) | + rtpHeader[3]; + rtpInfo->header.timestamp = (static_cast(rtpHeader[4]) << 24) | + (static_cast(rtpHeader[5]) << 16) | + (static_cast(rtpHeader[6]) << 8) | rtpHeader[7]; + rtpInfo->header.ssrc = (static_cast(rtpHeader[8]) << 24) | + (static_cast(rtpHeader[9]) << 16) | + (static_cast(rtpHeader[10]) << 8) | rtpHeader[11]; +} + +void RTPStream::MakeRTPheader(uint8_t* rtpHeader, uint8_t payloadType, + int16_t seqNo, uint32_t timeStamp, + uint32_t ssrc) { + rtpHeader[0] = 0x80; + rtpHeader[1] = payloadType; + rtpHeader[2] = (seqNo >> 8) & 0xFF; + rtpHeader[3] = seqNo & 0xFF; + rtpHeader[4] = timeStamp >> 24; + rtpHeader[5] = (timeStamp >> 16) & 0xFF; + rtpHeader[6] = (timeStamp >> 8) & 0xFF; + rtpHeader[7] = timeStamp & 0xFF; + rtpHeader[8] = ssrc >> 24; + rtpHeader[9] = (ssrc >> 16) & 0xFF; + rtpHeader[10] = (ssrc >> 8) & 0xFF; + rtpHeader[11] = ssrc & 0xFF; +} + +RTPPacket::RTPPacket(uint8_t payloadType, uint32_t timeStamp, int16_t seqNo, + const uint8_t* payloadData, size_t payloadSize, + uint32_t frequency) + : payloadType(payloadType), + timeStamp(timeStamp), + seqNo(seqNo), + payloadSize(payloadSize), + frequency(frequency) { + if (payloadSize > 0) { + this->payloadData = new uint8_t[payloadSize]; + memcpy(this->payloadData, payloadData, payloadSize); + } +} + +RTPPacket::~RTPPacket() { + delete[] payloadData; +} + +RTPBuffer::RTPBuffer() { + _queueRWLock = RWLockWrapper::CreateRWLock(); +} + +RTPBuffer::~RTPBuffer() { + delete _queueRWLock; +} + +void RTPBuffer::Write(const uint8_t payloadType, const uint32_t timeStamp, + const int16_t seqNo, const uint8_t* payloadData, + const size_t payloadSize, uint32_t frequency) { + RTPPacket *packet = new RTPPacket(payloadType, timeStamp, seqNo, payloadData, + payloadSize, frequency); + _queueRWLock->AcquireLockExclusive(); + _rtpQueue.push(packet); + _queueRWLock->ReleaseLockExclusive(); +} + +size_t RTPBuffer::Read(WebRtcRTPHeader* rtpInfo, uint8_t* payloadData, + size_t payloadSize, uint32_t* offset) { + _queueRWLock->AcquireLockShared(); + RTPPacket *packet = _rtpQueue.front(); + _rtpQueue.pop(); + _queueRWLock->ReleaseLockShared(); + rtpInfo->header.markerBit = 1; + rtpInfo->header.payloadType = packet->payloadType; + rtpInfo->header.sequenceNumber = packet->seqNo; + rtpInfo->header.ssrc = 0; + rtpInfo->header.timestamp = packet->timeStamp; + if (packet->payloadSize > 0 && payloadSize >= packet->payloadSize) { + memcpy(payloadData, packet->payloadData, packet->payloadSize); + } else { + return 0; + } + *offset = (packet->timeStamp / (packet->frequency / 1000)); + + return packet->payloadSize; +} + +bool RTPBuffer::EndOfFile() const { + _queueRWLock->AcquireLockShared(); + bool eof = _rtpQueue.empty(); + _queueRWLock->ReleaseLockShared(); + return eof; +} + +void RTPFile::Open(const char *filename, const char *mode) { + if ((_rtpFile = fopen(filename, mode)) == NULL) { + printf("Cannot write file %s.\n", filename); + ADD_FAILURE() << "Unable to write file"; + exit(1); + } +} + +void RTPFile::Close() { + if (_rtpFile != NULL) { + fclose(_rtpFile); + _rtpFile = NULL; + } +} + +void RTPFile::WriteHeader() { + // Write data in a format that NetEQ and RTP Play can parse + fprintf(_rtpFile, "#!RTPencode%s\n", "1.0"); + uint32_t dummy_variable = 0; + // should be converted to network endian format, but does not matter when 0 + EXPECT_EQ(1u, fwrite(&dummy_variable, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 2, 1, _rtpFile)); + fflush(_rtpFile); +} + +void RTPFile::ReadHeader() { + uint32_t start_sec, start_usec, source; + uint16_t port, padding; + char fileHeader[40]; + EXPECT_TRUE(fgets(fileHeader, 40, _rtpFile) != 0); + EXPECT_EQ(1u, fread(&start_sec, 4, 1, _rtpFile)); + start_sec = ntohl(start_sec); + EXPECT_EQ(1u, fread(&start_usec, 4, 1, _rtpFile)); + start_usec = ntohl(start_usec); + EXPECT_EQ(1u, fread(&source, 4, 1, _rtpFile)); + source = ntohl(source); + EXPECT_EQ(1u, fread(&port, 2, 1, _rtpFile)); + port = ntohs(port); + EXPECT_EQ(1u, fread(&padding, 2, 1, _rtpFile)); + padding = ntohs(padding); +} + +void RTPFile::Write(const uint8_t payloadType, const uint32_t timeStamp, + const int16_t seqNo, const uint8_t* payloadData, + const size_t payloadSize, uint32_t frequency) { + /* write RTP packet to file */ + uint8_t rtpHeader[12]; + MakeRTPheader(rtpHeader, payloadType, seqNo, timeStamp, 0); + ASSERT_LE(12 + payloadSize + 8, std::numeric_limits::max()); + uint16_t lengthBytes = htons(static_cast(12 + payloadSize + 8)); + uint16_t plen = htons(static_cast(12 + payloadSize)); + uint32_t offsetMs; + + offsetMs = (timeStamp / (frequency / 1000)); + offsetMs = htonl(offsetMs); + EXPECT_EQ(1u, fwrite(&lengthBytes, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&plen, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&offsetMs, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&rtpHeader, 12, 1, _rtpFile)); + EXPECT_EQ(payloadSize, fwrite(payloadData, 1, payloadSize, _rtpFile)); +} + +size_t RTPFile::Read(WebRtcRTPHeader* rtpInfo, uint8_t* payloadData, + size_t payloadSize, uint32_t* offset) { + uint16_t lengthBytes; + uint16_t plen; + uint8_t rtpHeader[12]; + size_t read_len = fread(&lengthBytes, 2, 1, _rtpFile); + /* Check if we have reached end of file. */ + if ((read_len == 0) && feof(_rtpFile)) { + _rtpEOF = true; + return 0; + } + EXPECT_EQ(1u, fread(&plen, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fread(offset, 4, 1, _rtpFile)); + lengthBytes = ntohs(lengthBytes); + plen = ntohs(plen); + *offset = ntohl(*offset); + EXPECT_GT(plen, 11); + + EXPECT_EQ(1u, fread(rtpHeader, 12, 1, _rtpFile)); + ParseRTPHeader(rtpInfo, rtpHeader); + rtpInfo->type.Audio.isCNG = false; + rtpInfo->type.Audio.channel = 1; + EXPECT_EQ(lengthBytes, plen + 8); + + if (plen == 0) { + return 0; + } + if (lengthBytes < 20) { + return 0; + } + if (payloadSize < static_cast((lengthBytes - 20))) { + return 0; + } + lengthBytes -= 20; + EXPECT_EQ(lengthBytes, fread(payloadData, 1, lengthBytes, _rtpFile)); + return lengthBytes; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.h new file mode 100644 index 0000000000..34dae63b2a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/RTPFile.h @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_RTPFILE_H_ +#define MODULES_AUDIO_CODING_TEST_RTPFILE_H_ + +#include +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/include/module_common_types.h" +#include "system_wrappers/include/rw_lock_wrapper.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class RTPStream { + public: + virtual ~RTPStream() { + } + + virtual void Write(const uint8_t payloadType, const uint32_t timeStamp, + const int16_t seqNo, const uint8_t* payloadData, + const size_t payloadSize, uint32_t frequency) = 0; + + // Returns the packet's payload size. Zero should be treated as an + // end-of-stream (in the case that EndOfFile() is true) or an error. + virtual size_t Read(WebRtcRTPHeader* rtpInfo, uint8_t* payloadData, + size_t payloadSize, uint32_t* offset) = 0; + virtual bool EndOfFile() const = 0; + + protected: + void MakeRTPheader(uint8_t* rtpHeader, uint8_t payloadType, int16_t seqNo, + uint32_t timeStamp, uint32_t ssrc); + + void ParseRTPHeader(WebRtcRTPHeader* rtpInfo, const uint8_t* rtpHeader); +}; + +class RTPPacket { + public: + RTPPacket(uint8_t payloadType, uint32_t timeStamp, int16_t seqNo, + const uint8_t* payloadData, size_t payloadSize, + uint32_t frequency); + + ~RTPPacket(); + + uint8_t payloadType; + uint32_t timeStamp; + int16_t seqNo; + uint8_t* payloadData; + size_t payloadSize; + uint32_t frequency; +}; + +class RTPBuffer : public RTPStream { + public: + RTPBuffer(); + + ~RTPBuffer(); + + void Write(const uint8_t payloadType, + const uint32_t timeStamp, + const int16_t seqNo, + const uint8_t* payloadData, + const size_t payloadSize, + uint32_t frequency) override; + + size_t Read(WebRtcRTPHeader* rtpInfo, + uint8_t* payloadData, + size_t payloadSize, + uint32_t* offset) override; + + bool EndOfFile() const override; + + private: + RWLockWrapper* _queueRWLock; + std::queue _rtpQueue; +}; + +class RTPFile : public RTPStream { + public: + ~RTPFile() { + } + + RTPFile() + : _rtpFile(NULL), + _rtpEOF(false) { + } + + void Open(const char *outFilename, const char *mode); + + void Close(); + + void WriteHeader(); + + void ReadHeader(); + + void Write(const uint8_t payloadType, + const uint32_t timeStamp, + const int16_t seqNo, + const uint8_t* payloadData, + const size_t payloadSize, + uint32_t frequency) override; + + size_t Read(WebRtcRTPHeader* rtpInfo, + uint8_t* payloadData, + size_t payloadSize, + uint32_t* offset) override; + + bool EndOfFile() const override { return _rtpEOF; } + + private: + FILE* _rtpFile; + bool _rtpEOF; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_RTPFILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.cc new file mode 100644 index 0000000000..3a968c99cc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.cc @@ -0,0 +1,485 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestAllCodecs.h" + +#include +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/utility.h" +#include "rtc_base/logging.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +// Description of the test: +// In this test we set up a one-way communication channel from a participant +// called "a" to a participant called "b". +// a -> channel_a_to_b -> b +// +// The test loops through all available mono codecs, encode at "a" sends over +// the channel, and decodes at "b". + +namespace { +const size_t kVariableSize = std::numeric_limits::max(); +} + +namespace webrtc { + +// Class for simulating packet handling. +TestPack::TestPack() + : receiver_acm_(NULL), + sequence_number_(0), + timestamp_diff_(0), + last_in_timestamp_(0), + total_bytes_(0), + payload_size_(0) { +} + +TestPack::~TestPack() { +} + +void TestPack::RegisterReceiverACM(AudioCodingModule* acm) { + receiver_acm_ = acm; + return; +} + +int32_t TestPack::SendData(FrameType frame_type, uint8_t payload_type, + uint32_t timestamp, const uint8_t* payload_data, + size_t payload_size, + const RTPFragmentationHeader* fragmentation) { + WebRtcRTPHeader rtp_info; + int32_t status; + + rtp_info.header.markerBit = false; + rtp_info.header.ssrc = 0; + rtp_info.header.sequenceNumber = sequence_number_++; + rtp_info.header.payloadType = payload_type; + rtp_info.header.timestamp = timestamp; + if (frame_type == kAudioFrameCN) { + rtp_info.type.Audio.isCNG = true; + } else { + rtp_info.type.Audio.isCNG = false; + } + if (frame_type == kEmptyFrame) { + // Skip this frame. + return 0; + } + + // Only run mono for all test cases. + rtp_info.type.Audio.channel = 1; + memcpy(payload_data_, payload_data, payload_size); + + status = receiver_acm_->IncomingPacket(payload_data_, payload_size, rtp_info); + + payload_size_ = payload_size; + timestamp_diff_ = timestamp - last_in_timestamp_; + last_in_timestamp_ = timestamp; + total_bytes_ += payload_size; + return status; +} + +size_t TestPack::payload_size() { + return payload_size_; +} + +uint32_t TestPack::timestamp_diff() { + return timestamp_diff_; +} + +void TestPack::reset_payload_size() { + payload_size_ = 0; +} + +TestAllCodecs::TestAllCodecs(int test_mode) + : acm_a_(AudioCodingModule::Create()), + acm_b_(AudioCodingModule::Create()), + channel_a_to_b_(NULL), + test_count_(0), + packet_size_samples_(0), + packet_size_bytes_(0) { + // test_mode = 0 for silent test (auto test) + test_mode_ = test_mode; +} + +TestAllCodecs::~TestAllCodecs() { + if (channel_a_to_b_ != NULL) { + delete channel_a_to_b_; + channel_a_to_b_ = NULL; + } +} + +void TestAllCodecs::Perform() { + const std::string file_name = webrtc::test::ResourcePath( + "audio_coding/testfile32kHz", "pcm"); + infile_a_.Open(file_name, 32000, "rb"); + + if (test_mode_ == 0) { + RTC_LOG(LS_INFO) << "---------- TestAllCodecs ----------"; + } + + acm_a_->InitializeReceiver(); + acm_b_->InitializeReceiver(); + + uint8_t num_encoders = acm_a_->NumberOfCodecs(); + CodecInst my_codec_param; + for (uint8_t n = 0; n < num_encoders; n++) { + acm_b_->Codec(n, &my_codec_param); + if (!strcmp(my_codec_param.plname, "opus")) { + my_codec_param.channels = 1; + } + acm_b_->RegisterReceiveCodec(my_codec_param.pltype, + CodecInstToSdp(my_codec_param)); + } + + // Create and connect the channel + channel_a_to_b_ = new TestPack; + acm_a_->RegisterTransportCallback(channel_a_to_b_); + channel_a_to_b_->RegisterReceiverACM(acm_b_.get()); + + // All codecs are tested for all allowed sampling frequencies, rates and + // packet sizes. + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + char codec_g722[] = "G722"; + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 480, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 640, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 800, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 960, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); +#ifdef WEBRTC_CODEC_ILBC + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + char codec_ilbc[] = "ILBC"; + RegisterSendCodec('A', codec_ilbc, 8000, 13300, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_ilbc, 8000, 13300, 480, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_ilbc, 8000, 15200, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_ilbc, 8000, 15200, 320, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); +#endif +#if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + char codec_isac[] = "ISAC"; + RegisterSendCodec('A', codec_isac, 16000, -1, 480, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_isac, 16000, -1, 960, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_isac, 16000, 15000, 480, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_isac, 16000, 32000, 960, kVariableSize); + Run(channel_a_to_b_); + outfile_b_.Close(); +#endif +#ifdef WEBRTC_CODEC_ISAC + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + RegisterSendCodec('A', codec_isac, 32000, -1, 960, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_isac, 32000, 56000, 960, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_isac, 32000, 37000, 960, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_isac, 32000, 32000, 960, kVariableSize); + Run(channel_a_to_b_); + outfile_b_.Close(); +#endif + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + char codec_l16[] = "L16"; + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 320, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 480, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 640, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 640, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + char codec_pcma[] = "PCMA"; + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 400, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 480, 0); + Run(channel_a_to_b_); + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + char codec_pcmu[] = "PCMU"; + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 400, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 480, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); +#ifdef WEBRTC_CODEC_OPUS + if (test_mode_ != 0) { + printf("===============================================================\n"); + } + test_count_++; + OpenOutFile(test_count_); + char codec_opus[] = "OPUS"; + RegisterSendCodec('A', codec_opus, 48000, 6000, 480, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 20000, 480*2, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 32000, 480*4, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 48000, 480, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 64000, 480*4, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 96000, 480*6, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 500000, 480*2, kVariableSize); + Run(channel_a_to_b_); + outfile_b_.Close(); +#endif + if (test_mode_ != 0) { + printf("===============================================================\n"); + + /* Print out all codecs that were not tested in the run */ + printf("The following codecs was not included in the test:\n"); +#ifndef WEBRTC_CODEC_ILBC + printf(" iLBC\n"); +#endif +#ifndef WEBRTC_CODEC_ISAC + printf(" ISAC float\n"); +#endif +#ifndef WEBRTC_CODEC_ISACFX + printf(" ISAC fix\n"); +#endif + + printf("\nTo complete the test, listen to the %d number of output files.\n", + test_count_); + } +} + +// Register Codec to use in the test +// +// Input: side - which ACM to use, 'A' or 'B' +// codec_name - name to use when register the codec +// sampling_freq_hz - sampling frequency in Herz +// rate - bitrate in bytes +// packet_size - packet size in samples +// extra_byte - if extra bytes needed compared to the bitrate +// used when registering, can be an internal header +// set to kVariableSize if the codec is a variable +// rate codec +void TestAllCodecs::RegisterSendCodec(char side, char* codec_name, + int32_t sampling_freq_hz, int rate, + int packet_size, size_t extra_byte) { + if (test_mode_ != 0) { + // Print out codec and settings. + printf("codec: %s Freq: %d Rate: %d PackSize: %d\n", codec_name, + sampling_freq_hz, rate, packet_size); + } + + // Store packet-size in samples, used to validate the received packet. + // If G.722, store half the size to compensate for the timestamp bug in the + // RFC for G.722. + // If iSAC runs in adaptive mode, packet size in samples can change on the + // fly, so we exclude this test by setting |packet_size_samples_| to -1. + if (!strcmp(codec_name, "G722")) { + packet_size_samples_ = packet_size / 2; + } else if (!strcmp(codec_name, "ISAC") && (rate == -1)) { + packet_size_samples_ = -1; + } else { + packet_size_samples_ = packet_size; + } + + // Store the expected packet size in bytes, used to validate the received + // packet. If variable rate codec (extra_byte == -1), set to -1. + if (extra_byte != kVariableSize) { + // Add 0.875 to always round up to a whole byte + packet_size_bytes_ = static_cast( + static_cast(packet_size * rate) / + static_cast(sampling_freq_hz * 8) + 0.875) + extra_byte; + } else { + // Packets will have a variable size. + packet_size_bytes_ = kVariableSize; + } + + // Set pointer to the ACM where to register the codec. + AudioCodingModule* my_acm = NULL; + switch (side) { + case 'A': { + my_acm = acm_a_.get(); + break; + } + case 'B': { + my_acm = acm_b_.get(); + break; + } + default: { + break; + } + } + ASSERT_TRUE(my_acm != NULL); + + // Get all codec parameters before registering + CodecInst my_codec_param; + CHECK_ERROR(AudioCodingModule::Codec(codec_name, &my_codec_param, + sampling_freq_hz, 1)); + my_codec_param.rate = rate; + my_codec_param.pacsize = packet_size; + CHECK_ERROR(my_acm->RegisterSendCodec(my_codec_param)); +} + +void TestAllCodecs::Run(TestPack* channel) { + AudioFrame audio_frame; + + int32_t out_freq_hz = outfile_b_.SamplingFrequency(); + size_t receive_size; + uint32_t timestamp_diff; + channel->reset_payload_size(); + int error_count = 0; + int counter = 0; + // Set test length to 500 ms (50 blocks of 10 ms each). + infile_a_.SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the file starts with silence. + infile_a_.FastForward(100); + + while (!infile_a_.EndOfFile()) { + // Add 10 msec to ACM. + infile_a_.Read10MsData(audio_frame); + CHECK_ERROR(acm_a_->Add10MsData(audio_frame)); + + // Verify that the received packet size matches the settings. + receive_size = channel->payload_size(); + if (receive_size) { + if ((receive_size != packet_size_bytes_) && + (packet_size_bytes_ != kVariableSize)) { + error_count++; + } + + // Verify that the timestamp is updated with expected length. The counter + // is used to avoid problems when switching codec or frame size in the + // test. + timestamp_diff = channel->timestamp_diff(); + if ((counter > 10) && + (static_cast(timestamp_diff) != packet_size_samples_) && + (packet_size_samples_ > -1)) + error_count++; + } + + // Run received side of ACM. + bool muted; + CHECK_ERROR(acm_b_->PlayoutData10Ms(out_freq_hz, &audio_frame, &muted)); + ASSERT_FALSE(muted); + + // Write output speech to file. + outfile_b_.Write10MsData(audio_frame.data(), + audio_frame.samples_per_channel_); + + // Update loop counter + counter++; + } + + EXPECT_EQ(0, error_count); + + if (infile_a_.EndOfFile()) { + infile_a_.Rewind(); + } +} + +void TestAllCodecs::OpenOutFile(int test_number) { + std::string filename = webrtc::test::OutputPath(); + std::ostringstream test_number_str; + test_number_str << test_number; + filename += "testallcodecs_out_"; + filename += test_number_str.str(); + filename += ".pcm"; + outfile_b_.Open(filename, 32000, "wb"); +} + +void TestAllCodecs::DisplaySendReceiveCodec() { + CodecInst my_codec_param; + printf("%s -> ", acm_a_->SendCodec()->plname); + acm_b_->ReceiveCodec(&my_codec_param); + printf("%s\n", my_codec_param.plname); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.h new file mode 100644 index 0000000000..e0285e95e0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestAllCodecs.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTALLCODECS_H_ +#define MODULES_AUDIO_CODING_TEST_TESTALLCODECS_H_ + +#include + +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class TestPack : public AudioPacketizationCallback { + public: + TestPack(); + ~TestPack(); + + void RegisterReceiverACM(AudioCodingModule* acm); + + int32_t SendData(FrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_size, + const RTPFragmentationHeader* fragmentation) override; + + size_t payload_size(); + uint32_t timestamp_diff(); + void reset_payload_size(); + + private: + AudioCodingModule* receiver_acm_; + uint16_t sequence_number_; + uint8_t payload_data_[60 * 32 * 2 * 2]; + uint32_t timestamp_diff_; + uint32_t last_in_timestamp_; + uint64_t total_bytes_; + size_t payload_size_; +}; + +class TestAllCodecs : public ACMTest { + public: + explicit TestAllCodecs(int test_mode); + ~TestAllCodecs(); + + void Perform() override; + + private: + // The default value of '-1' indicates that the registration is based only on + // codec name, and a sampling frequency matching is not required. + // This is useful for codecs which support several sampling frequency. + // Note! Only mono mode is tested in this test. + void RegisterSendCodec(char side, char* codec_name, int32_t sampling_freq_hz, + int rate, int packet_size, size_t extra_byte); + + void Run(TestPack* channel); + void OpenOutFile(int test_number); + void DisplaySendReceiveCodec(); + + int test_mode_; + std::unique_ptr acm_a_; + std::unique_ptr acm_b_; + TestPack* channel_a_to_b_; + PCMFile infile_a_; + PCMFile outfile_b_; + int test_count_; + int packet_size_samples_; + size_t packet_size_bytes_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTALLCODECS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.cc new file mode 100644 index 0000000000..034c5954af --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.cc @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestRedFec.h" + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/utility.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +#ifdef SUPPORT_RED_WB +#undef SUPPORT_RED_WB +#endif + +#ifdef SUPPORT_RED_SWB +#undef SUPPORT_RED_SWB +#endif + +#ifdef SUPPORT_RED_FB +#undef SUPPORT_RED_FB +#endif + +namespace webrtc { + +namespace { + const char kNameL16[] = "L16"; + const char kNamePCMU[] = "PCMU"; + const char kNameCN[] = "CN"; + const char kNameRED[] = "RED"; + const char kNameISAC[] = "ISAC"; + const char kNameG722[] = "G722"; + const char kNameOPUS[] = "opus"; +} + +TestRedFec::TestRedFec() + : _acmA(AudioCodingModule::Create()), + _acmB(AudioCodingModule::Create()), + _channelA2B(NULL), + _testCntr(0) { +} + +TestRedFec::~TestRedFec() { + if (_channelA2B != NULL) { + delete _channelA2B; + _channelA2B = NULL; + } +} + +void TestRedFec::Perform() { + const std::string file_name = webrtc::test::ResourcePath( + "audio_coding/testfile32kHz", "pcm"); + _inFileA.Open(file_name, 32000, "rb"); + + ASSERT_EQ(0, _acmA->InitializeReceiver()); + ASSERT_EQ(0, _acmB->InitializeReceiver()); + + uint8_t numEncoders = _acmA->NumberOfCodecs(); + CodecInst myCodecParam; + for (uint8_t n = 0; n < numEncoders; n++) { + EXPECT_EQ(0, _acmB->Codec(n, &myCodecParam)); + // Default number of channels is 2 for opus, so we change to 1 in this test. + if (!strcmp(myCodecParam.plname, "opus")) { + myCodecParam.channels = 1; + } + EXPECT_EQ(true, _acmB->RegisterReceiveCodec(myCodecParam.pltype, + CodecInstToSdp(myCodecParam))); + } + + // Create and connect the channel + _channelA2B = new Channel; + _acmA->RegisterTransportCallback(_channelA2B); + _channelA2B->RegisterReceiverACM(_acmB.get()); + + EXPECT_EQ(0, RegisterSendCodec('A', kNameL16, 8000)); + EXPECT_EQ(0, RegisterSendCodec('A', kNameCN, 8000)); + EXPECT_EQ(0, RegisterSendCodec('A', kNameRED)); + EXPECT_EQ(0, SetVAD(true, true, VADAggr)); + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); + + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + RegisterSendCodec('A', kNamePCMU, 8000); + // Switch to another 8 kHz codec, RED should remain switched on. + EXPECT_TRUE(_acmA->REDStatus()); + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + EXPECT_EQ(0, RegisterSendCodec('A', kNameG722, 16000)); + EXPECT_EQ(0, RegisterSendCodec('A', kNameCN, 16000)); + +#ifdef SUPPORT_RED_WB + // Switch codec, RED should remain. + EXPECT_TRUE(_acmA->REDStatus()); +#else + // Switch to a 16 kHz codec, RED should have been switched off. + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + OpenOutFile(_testCntr); + EXPECT_EQ(0, SetVAD(true, true, VADAggr)); + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_FALSE(_acmA->REDStatus()); + Run(); +#ifdef SUPPORT_RED_WB + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + Run(); + _outFileB.Close(); + + RegisterSendCodec('A', kNameISAC, 16000); + +#ifdef SUPPORT_RED_WB + // Switch codec, RED should remain. + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + OpenOutFile(_testCntr); + EXPECT_EQ(0, SetVAD(true, true, VADVeryAggr)); + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_FALSE(_acmA->REDStatus()); + Run(); + _outFileB.Close(); + +#ifdef SUPPORT_RED_WB + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + RegisterSendCodec('A', kNameISAC, 32000); + +#if defined(SUPPORT_RED_SWB) && defined(SUPPORT_RED_WB) + // Switch codec, RED should remain. + EXPECT_TRUE(_acmA->REDStatus()); +#else + // Switch to a 32 kHz codec, RED should have been switched off. + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + OpenOutFile(_testCntr); + EXPECT_EQ(0, SetVAD(true, true, VADVeryAggr)); + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_FALSE(_acmA->REDStatus()); + Run(); + _outFileB.Close(); + +#ifdef SUPPORT_RED_SWB + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + RegisterSendCodec('A', kNameISAC, 32000); + EXPECT_EQ(0, SetVAD(false, false, VADNormal)); + +#if defined(SUPPORT_RED_SWB) && defined(SUPPORT_RED_WB) + OpenOutFile(_testCntr); + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + + RegisterSendCodec('A', kNameISAC, 16000); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + + RegisterSendCodec('A', kNameISAC, 32000); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + + RegisterSendCodec('A', kNameISAC, 16000); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + _outFileB.Close(); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + _channelA2B->SetFECTestWithPacketLoss(true); + // Following tests are under packet losses. + + EXPECT_EQ(0, RegisterSendCodec('A', kNameG722)); + EXPECT_EQ(0, RegisterSendCodec('A', kNameCN, 16000)); + +#if defined(SUPPORT_RED_WB) && defined(SUPPORT_RED_SWB) + // Switch codec, RED should remain. + EXPECT_TRUE(_acmA->REDStatus()); +#else + // Switch to a 16 kHz codec, RED should have been switched off. + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + OpenOutFile(_testCntr); + EXPECT_EQ(0, SetVAD(true, true, VADAggr)); + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_FALSE(_acmA->REDStatus()); + Run(); + _outFileB.Close(); + +#ifdef SUPPORT_RED_WB + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + RegisterSendCodec('A', kNameISAC, 16000); + +#ifdef SUPPORT_RED_WB + // Switch codec, RED should remain. + EXPECT_TRUE(_acmA->REDStatus()); +#else + // Switch to a 16 kHz codec, RED should have been switched off. + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + OpenOutFile(_testCntr); + EXPECT_EQ(0, SetVAD(true, true, VADVeryAggr)); + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_FALSE(_acmA->REDStatus()); + Run(); + _outFileB.Close(); +#ifdef SUPPORT_RED_WB + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + RegisterSendCodec('A', kNameISAC, 32000); + +#if defined(SUPPORT_RED_SWB) && defined(SUPPORT_RED_WB) + // Switch codec, RED should remain. + EXPECT_TRUE(_acmA->REDStatus()); +#else + // Switch to a 32 kHz codec, RED should have been switched off. + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + OpenOutFile(_testCntr); + EXPECT_EQ(0, SetVAD(true, true, VADVeryAggr)); + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_FALSE(_acmA->REDStatus()); +#ifdef SUPPORT_RED_SWB + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + RegisterSendCodec('A', kNameISAC, 32000); + EXPECT_EQ(0, SetVAD(false, false, VADNormal)); +#if defined(SUPPORT_RED_SWB) && defined(SUPPORT_RED_WB) + OpenOutFile(_testCntr); + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + + RegisterSendCodec('A', kNameISAC, 16000); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + + RegisterSendCodec('A', kNameISAC, 32000); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + + RegisterSendCodec('A', kNameISAC, 16000); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + _outFileB.Close(); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); +#endif + +#ifndef WEBRTC_CODEC_OPUS + EXPECT_TRUE(false); + printf("Opus needs to be activated to run this test\n"); + return; +#endif + + RegisterSendCodec('A', kNameOPUS, 48000); + +#if defined(SUPPORT_RED_FB) && defined(SUPPORT_RED_SWB) &&\ + defined(SUPPORT_RED_WB) + // Switch to codec, RED should remain switched on. + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_FALSE(_acmA->REDStatus()); +#endif + + // _channelA2B imposes 25% packet loss rate. + EXPECT_EQ(0, _acmA->SetPacketLossRate(25)); + +#ifdef SUPPORT_RED_FB + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); + // Codec FEC and RED are mutually exclusive. + EXPECT_EQ(-1, _acmA->SetCodecFEC(true)); + + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_EQ(0, _acmA->SetCodecFEC(true)); + + // Codec FEC and RED are mutually exclusive. + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); +#else + EXPECT_EQ(-1, _acmA->SetREDStatus(true)); + EXPECT_FALSE(_acmA->REDStatus()); + EXPECT_EQ(0, _acmA->SetCodecFEC(true)); +#endif + + EXPECT_TRUE(_acmA->CodecFEC()); + OpenOutFile(_testCntr); + Run(); + + // Switch to L16 with RED. + RegisterSendCodec('A', kNameL16, 8000); + EXPECT_EQ(0, SetVAD(false, false, VADNormal)); + + // L16 does not support FEC, so FEC should be turned off automatically. + EXPECT_FALSE(_acmA->CodecFEC()); + + EXPECT_EQ(0, _acmA->SetREDStatus(true)); + EXPECT_TRUE(_acmA->REDStatus()); + Run(); + + // Switch to Opus again. + RegisterSendCodec('A', kNameOPUS, 48000); +#ifdef SUPPORT_RED_FB + // Switch to codec, RED should remain switched on. + EXPECT_TRUE(_acmA->REDStatus()); +#else + EXPECT_FALSE(_acmA->REDStatus()); +#endif + EXPECT_EQ(0, _acmA->SetREDStatus(false)); + EXPECT_EQ(0, _acmA->SetCodecFEC(false)); + Run(); + + EXPECT_EQ(0, _acmA->SetCodecFEC(true)); + _outFileB.Close(); + + // Codecs does not support internal FEC, cannot enable FEC. + RegisterSendCodec('A', kNameG722, 16000); + EXPECT_FALSE(_acmA->REDStatus()); + EXPECT_EQ(-1, _acmA->SetCodecFEC(true)); + EXPECT_FALSE(_acmA->CodecFEC()); + + RegisterSendCodec('A', kNameISAC, 16000); + EXPECT_FALSE(_acmA->REDStatus()); + EXPECT_EQ(-1, _acmA->SetCodecFEC(true)); + EXPECT_FALSE(_acmA->CodecFEC()); + + // Codecs does not support internal FEC, disable FEC does not trigger failure. + RegisterSendCodec('A', kNameG722, 16000); + EXPECT_FALSE(_acmA->REDStatus()); + EXPECT_EQ(0, _acmA->SetCodecFEC(false)); + EXPECT_FALSE(_acmA->CodecFEC()); + + RegisterSendCodec('A', kNameISAC, 16000); + EXPECT_FALSE(_acmA->REDStatus()); + EXPECT_EQ(0, _acmA->SetCodecFEC(false)); + EXPECT_FALSE(_acmA->CodecFEC()); +} + +int32_t TestRedFec::SetVAD(bool enableDTX, bool enableVAD, ACMVADMode vadMode) { + return _acmA->SetVAD(enableDTX, enableVAD, vadMode); +} + +int16_t TestRedFec::RegisterSendCodec(char side, const char* codecName, + int32_t samplingFreqHz) { + std::cout << std::flush; + AudioCodingModule* myACM; + switch (side) { + case 'A': { + myACM = _acmA.get(); + break; + } + case 'B': { + myACM = _acmB.get(); + break; + } + default: + return -1; + } + + if (myACM == NULL) { + assert(false); + return -1; + } + CodecInst myCodecParam; + EXPECT_GT(AudioCodingModule::Codec(codecName, &myCodecParam, + samplingFreqHz, 1), -1); + EXPECT_GT(myACM->RegisterSendCodec(myCodecParam), -1); + + // Initialization was successful. + return 0; +} + +void TestRedFec::Run() { + AudioFrame audioFrame; + int32_t outFreqHzB = _outFileB.SamplingFrequency(); + // Set test length to 500 ms (50 blocks of 10 ms each). + _inFileA.SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the file starts with silence. + _inFileA.FastForward(100); + + while (!_inFileA.EndOfFile()) { + EXPECT_GT(_inFileA.Read10MsData(audioFrame), 0); + EXPECT_GE(_acmA->Add10MsData(audioFrame), 0); + bool muted; + EXPECT_EQ(0, _acmB->PlayoutData10Ms(outFreqHzB, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileB.Write10MsData(audioFrame.data(), audioFrame.samples_per_channel_); + } + _inFileA.Rewind(); +} + +void TestRedFec::OpenOutFile(int16_t test_number) { + std::string file_name; + std::stringstream file_stream; + file_stream << webrtc::test::OutputPath(); + file_stream << "TestRedFec_outFile_"; + file_stream << test_number << ".pcm"; + file_name = file_stream.str(); + _outFileB.Open(file_name, 16000, "wb"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.h new file mode 100644 index 0000000000..98aa0080ec --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestRedFec.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTREDFEC_H_ +#define MODULES_AUDIO_CODING_TEST_TESTREDFEC_H_ + +#include +#include + +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" + +namespace webrtc { + +class TestRedFec : public ACMTest { + public: + explicit TestRedFec(); + ~TestRedFec(); + + void Perform(); + private: + // The default value of '-1' indicates that the registration is based only on + // codec name and a sampling frequency matching is not required. This is + // useful for codecs which support several sampling frequency. + int16_t RegisterSendCodec(char side, const char* codecName, + int32_t sampFreqHz = -1); + void Run(); + void OpenOutFile(int16_t testNumber); + int32_t SetVAD(bool enableDTX, bool enableVAD, ACMVADMode vadMode); + std::unique_ptr _acmA; + std::unique_ptr _acmB; + + Channel* _channelA2B; + + PCMFile _inFileA; + PCMFile _outFileB; + int16_t _testCntr; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTREDFEC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.cc new file mode 100644 index 0000000000..ba86719fe8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.cc @@ -0,0 +1,847 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestStereo.h" + +#include + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/utility.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Class for simulating packet handling +TestPackStereo::TestPackStereo() + : receiver_acm_(NULL), + seq_no_(0), + timestamp_diff_(0), + last_in_timestamp_(0), + total_bytes_(0), + payload_size_(0), + codec_mode_(kNotSet), + lost_packet_(false) { +} + +TestPackStereo::~TestPackStereo() { +} + +void TestPackStereo::RegisterReceiverACM(AudioCodingModule* acm) { + receiver_acm_ = acm; + return; +} + +int32_t TestPackStereo::SendData(const FrameType frame_type, + const uint8_t payload_type, + const uint32_t timestamp, + const uint8_t* payload_data, + const size_t payload_size, + const RTPFragmentationHeader* fragmentation) { + WebRtcRTPHeader rtp_info; + int32_t status = 0; + + rtp_info.header.markerBit = false; + rtp_info.header.ssrc = 0; + rtp_info.header.sequenceNumber = seq_no_++; + rtp_info.header.payloadType = payload_type; + rtp_info.header.timestamp = timestamp; + if (frame_type == kEmptyFrame) { + // Skip this frame + return 0; + } + + if (lost_packet_ == false) { + if (frame_type != kAudioFrameCN) { + rtp_info.type.Audio.isCNG = false; + rtp_info.type.Audio.channel = static_cast(codec_mode_); + } else { + rtp_info.type.Audio.isCNG = true; + rtp_info.type.Audio.channel = static_cast(kMono); + } + status = receiver_acm_->IncomingPacket(payload_data, payload_size, + rtp_info); + + if (frame_type != kAudioFrameCN) { + payload_size_ = static_cast(payload_size); + } else { + payload_size_ = -1; + } + + timestamp_diff_ = timestamp - last_in_timestamp_; + last_in_timestamp_ = timestamp; + total_bytes_ += payload_size; + } + return status; +} + +uint16_t TestPackStereo::payload_size() { + return static_cast(payload_size_); +} + +uint32_t TestPackStereo::timestamp_diff() { + return timestamp_diff_; +} + +void TestPackStereo::reset_payload_size() { + payload_size_ = 0; +} + +void TestPackStereo::set_codec_mode(enum StereoMonoMode mode) { + codec_mode_ = mode; +} + +void TestPackStereo::set_lost_packet(bool lost) { + lost_packet_ = lost; +} + +TestStereo::TestStereo(int test_mode) + : acm_a_(AudioCodingModule::Create()), + acm_b_(AudioCodingModule::Create()), + channel_a2b_(NULL), + test_cntr_(0), + pack_size_samp_(0), + pack_size_bytes_(0), + counter_(0), + g722_pltype_(0), + l16_8khz_pltype_(-1), + l16_16khz_pltype_(-1), + l16_32khz_pltype_(-1) +#ifdef PCMA_AND_PCMU + , pcma_pltype_(-1) + , pcmu_pltype_(-1) +#endif +#ifdef WEBRTC_CODEC_OPUS + , opus_pltype_(-1) +#endif +{ + // test_mode = 0 for silent test (auto test) + test_mode_ = test_mode; +} + +TestStereo::~TestStereo() { + if (channel_a2b_ != NULL) { + delete channel_a2b_; + channel_a2b_ = NULL; + } +} + +void TestStereo::Perform() { + uint16_t frequency_hz; + int audio_channels; + int codec_channels; + bool dtx; + bool vad; + ACMVADMode vad_mode; + + // Open both mono and stereo test files in 32 kHz. + const std::string file_name_stereo = webrtc::test::ResourcePath( + "audio_coding/teststereo32kHz", "pcm"); + const std::string file_name_mono = webrtc::test::ResourcePath( + "audio_coding/testfile32kHz", "pcm"); + frequency_hz = 32000; + in_file_stereo_ = new PCMFile(); + in_file_mono_ = new PCMFile(); + in_file_stereo_->Open(file_name_stereo, frequency_hz, "rb"); + in_file_stereo_->ReadStereo(true); + in_file_mono_->Open(file_name_mono, frequency_hz, "rb"); + in_file_mono_->ReadStereo(false); + + // Create and initialize two ACMs, one for each side of a one-to-one call. + ASSERT_TRUE((acm_a_.get() != NULL) && (acm_b_.get() != NULL)); + EXPECT_EQ(0, acm_a_->InitializeReceiver()); + EXPECT_EQ(0, acm_b_->InitializeReceiver()); + + // Register all available codes as receiving codecs. + uint8_t num_encoders = acm_a_->NumberOfCodecs(); + CodecInst my_codec_param; + for (uint8_t n = 0; n < num_encoders; n++) { + EXPECT_EQ(0, acm_b_->Codec(n, &my_codec_param)); + EXPECT_EQ(true, acm_b_->RegisterReceiveCodec( + my_codec_param.pltype, CodecInstToSdp(my_codec_param))); + } + + // Test that unregister all receive codecs works. + for (uint8_t n = 0; n < num_encoders; n++) { + EXPECT_EQ(0, acm_b_->Codec(n, &my_codec_param)); + EXPECT_EQ(0, acm_b_->UnregisterReceiveCodec(my_codec_param.pltype)); + } + + // Register all available codes as receiving codecs once more. + for (uint8_t n = 0; n < num_encoders; n++) { + EXPECT_EQ(0, acm_b_->Codec(n, &my_codec_param)); + EXPECT_EQ(true, acm_b_->RegisterReceiveCodec( + my_codec_param.pltype, CodecInstToSdp(my_codec_param))); + } + + // Create and connect the channel. + channel_a2b_ = new TestPackStereo; + EXPECT_EQ(0, acm_a_->RegisterTransportCallback(channel_a2b_)); + channel_a2b_->RegisterReceiverACM(acm_b_.get()); + + // Start with setting VAD/DTX, before we know we will send stereo. + // Continue with setting a stereo codec as send codec and verify that + // VAD/DTX gets turned off. + EXPECT_EQ(0, acm_a_->SetVAD(true, true, VADNormal)); + EXPECT_EQ(0, acm_a_->VAD(&dtx, &vad, &vad_mode)); + EXPECT_TRUE(dtx); + EXPECT_TRUE(vad); + char codec_pcma_temp[] = "PCMA"; + RegisterSendCodec('A', codec_pcma_temp, 8000, 64000, 80, 2, pcma_pltype_); + EXPECT_EQ(0, acm_a_->VAD(&dtx, &vad, &vad_mode)); + EXPECT_FALSE(dtx); + EXPECT_FALSE(vad); + if (test_mode_ != 0) { + printf("\n"); + } + + // + // Test Stereo-To-Stereo for all codecs. + // + audio_channels = 2; + codec_channels = 2; + + // All codecs are tested for all allowed sampling frequencies, rates and + // packet sizes. + if (test_mode_ != 0) { + printf("===========================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-stereo\n"); + } + channel_a2b_->set_codec_mode(kStereo); + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_g722[] = "G722"; + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, codec_channels, + g722_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 320, codec_channels, + g722_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 480, codec_channels, + g722_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 640, codec_channels, + g722_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 800, codec_channels, + g722_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 960, codec_channels, + g722_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + if (test_mode_ != 0) { + printf("===========================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-stereo\n"); + } + channel_a2b_->set_codec_mode(kStereo); + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_l16[] = "L16"; + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, codec_channels, + l16_8khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 8000, 128000, 160, codec_channels, + l16_8khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 8000, 128000, 240, codec_channels, + l16_8khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 8000, 128000, 320, codec_channels, + l16_8khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + if (test_mode_ != 0) { + printf("===========================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-stereo\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, codec_channels, + l16_16khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 16000, 256000, 320, codec_channels, + l16_16khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 16000, 256000, 480, codec_channels, + l16_16khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 16000, 256000, 640, codec_channels, + l16_16khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + if (test_mode_ != 0) { + printf("===========================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-stereo\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, codec_channels, + l16_32khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 32000, 512000, 640, codec_channels, + l16_32khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#ifdef PCMA_AND_PCMU + if (test_mode_ != 0) { + printf("===========================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-stereo\n"); + } + channel_a2b_->set_codec_mode(kStereo); + audio_channels = 2; + codec_channels = 2; + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_pcma[] = "PCMA"; + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 160, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 240, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 320, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 400, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 480, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + + // Test that VAD/DTX cannot be turned on while sending stereo. + EXPECT_EQ(-1, acm_a_->SetVAD(true, true, VADNormal)); + EXPECT_EQ(0, acm_a_->VAD(&dtx, &vad, &vad_mode)); + EXPECT_FALSE(dtx); + EXPECT_FALSE(vad); + EXPECT_EQ(0, acm_a_->SetVAD(false, false, VADNormal)); + EXPECT_EQ(0, acm_a_->VAD(&dtx, &vad, &vad_mode)); + EXPECT_FALSE(dtx); + EXPECT_FALSE(vad); + + out_file_.Close(); + if (test_mode_ != 0) { + printf("===========================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-stereo\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_pcmu[] = "PCMU"; + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 160, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 240, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 320, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 400, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 480, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif +#ifdef WEBRTC_CODEC_OPUS + if (test_mode_ != 0) { + printf("===========================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-stereo\n"); + } + channel_a2b_->set_codec_mode(kStereo); + audio_channels = 2; + codec_channels = 2; + test_cntr_++; + OpenOutFile(test_cntr_); + + char codec_opus[] = "opus"; + // Run Opus with 10 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480, codec_channels, + opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 20 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480*2, codec_channels, + opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 40 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480*4, codec_channels, + opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 60 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480*6, codec_channels, + opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 20 ms frame size and different bitrates. + RegisterSendCodec('A', codec_opus, 48000, 40000, 960, codec_channels, + opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_opus, 48000, 510000, 960, codec_channels, + opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif + // + // Test Mono-To-Stereo for all codecs. + // + audio_channels = 1; + codec_channels = 2; + + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Mono-to-stereo\n"); + } + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, codec_channels, + g722_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Mono-to-stereo\n"); + } + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, codec_channels, + l16_8khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Mono-to-stereo\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, codec_channels, + l16_16khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Mono-to-stereo\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, codec_channels, + l16_32khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#ifdef PCMA_AND_PCMU + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Mono-to-stereo\n"); + } + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif +#ifdef WEBRTC_CODEC_OPUS + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Mono-to-stereo\n"); + } + + // Keep encode and decode in stereo. + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_opus, 48000, 64000, 960, codec_channels, + opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + + // Encode in mono, decode in stereo mode. + RegisterSendCodec('A', codec_opus, 48000, 64000, 960, 1, opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif + + // + // Test Stereo-To-Mono for all codecs. + // + audio_channels = 2; + codec_channels = 1; + channel_a2b_->set_codec_mode(kMono); + + // Run stereo audio and mono codec. + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-mono\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, codec_channels, + g722_pltype_); + + // Make sure it is possible to set VAD/CNG, now that we are sending mono + // again. + EXPECT_EQ(0, acm_a_->SetVAD(true, true, VADNormal)); + EXPECT_EQ(0, acm_a_->VAD(&dtx, &vad, &vad_mode)); + EXPECT_TRUE(dtx); + EXPECT_TRUE(vad); + EXPECT_EQ(0, acm_a_->SetVAD(false, false, VADNormal)); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-mono\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, codec_channels, + l16_8khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-mono\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, codec_channels, + l16_16khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + if (test_mode_ != 0) { + printf("==============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-mono\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, codec_channels, + l16_32khz_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#ifdef PCMA_AND_PCMU + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-mono\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, codec_channels, + pcmu_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, codec_channels, + pcma_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif +#ifdef WEBRTC_CODEC_OPUS + if (test_mode_ != 0) { + printf("===============================================================\n"); + printf("Test number: %d\n", test_cntr_ + 1); + printf("Test type: Stereo-to-mono\n"); + } + test_cntr_++; + OpenOutFile(test_cntr_); + // Encode and decode in mono. + RegisterSendCodec('A', codec_opus, 48000, 32000, 960, codec_channels, + opus_pltype_); + CodecInst opus_codec_param; + for (uint8_t n = 0; n < num_encoders; n++) { + EXPECT_EQ(0, acm_b_->Codec(n, &opus_codec_param)); + if (!strcmp(opus_codec_param.plname, "opus")) { + opus_codec_param.channels = 1; + EXPECT_EQ(true, + acm_b_->RegisterReceiveCodec(opus_codec_param.pltype, + CodecInstToSdp(opus_codec_param))); + break; + } + } + Run(channel_a2b_, audio_channels, codec_channels); + + // Encode in stereo, decode in mono. + RegisterSendCodec('A', codec_opus, 48000, 32000, 960, 2, opus_pltype_); + Run(channel_a2b_, audio_channels, codec_channels); + + out_file_.Close(); + + // Test switching between decoding mono and stereo for Opus. + + // Decode in mono. + test_cntr_++; + OpenOutFile(test_cntr_); + if (test_mode_ != 0) { + // Print out codec and settings + printf("Test number: %d\nCodec: Opus Freq: 48000 Rate :32000 PackSize: 960" + " Decode: mono\n", test_cntr_); + } + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + // Decode in stereo. + test_cntr_++; + OpenOutFile(test_cntr_); + if (test_mode_ != 0) { + // Print out codec and settings + printf("Test number: %d\nCodec: Opus Freq: 48000 Rate :32000 PackSize: 960" + " Decode: stereo\n", test_cntr_); + } + opus_codec_param.channels = 2; + EXPECT_EQ(true, + acm_b_->RegisterReceiveCodec(opus_codec_param.pltype, + CodecInstToSdp(opus_codec_param))); + Run(channel_a2b_, audio_channels, 2); + out_file_.Close(); + // Decode in mono. + test_cntr_++; + OpenOutFile(test_cntr_); + if (test_mode_ != 0) { + // Print out codec and settings + printf("Test number: %d\nCodec: Opus Freq: 48000 Rate :32000 PackSize: 960" + " Decode: mono\n", test_cntr_); + } + opus_codec_param.channels = 1; + EXPECT_EQ(true, + acm_b_->RegisterReceiveCodec(opus_codec_param.pltype, + CodecInstToSdp(opus_codec_param))); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + +#endif + + // Print out which codecs were tested, and which were not, in the run. + if (test_mode_ != 0) { + printf("\nThe following codecs was INCLUDED in the test:\n"); + printf(" G.722\n"); + printf(" PCM16\n"); + printf(" G.711\n"); +#ifdef WEBRTC_CODEC_OPUS + printf(" Opus\n"); +#endif + printf("\nTo complete the test, listen to the %d number of output " + "files.\n", + test_cntr_); + } + + // Delete the file pointers. + delete in_file_stereo_; + delete in_file_mono_; +} + +// Register Codec to use in the test +// +// Input: side - which ACM to use, 'A' or 'B' +// codec_name - name to use when register the codec +// sampling_freq_hz - sampling frequency in Herz +// rate - bitrate in bytes +// pack_size - packet size in samples +// channels - number of channels; 1 for mono, 2 for stereo +// payload_type - payload type for the codec +void TestStereo::RegisterSendCodec(char side, char* codec_name, + int32_t sampling_freq_hz, int rate, + int pack_size, int channels, + int payload_type) { + if (test_mode_ != 0) { + // Print out codec and settings + printf("Codec: %s Freq: %d Rate: %d PackSize: %d\n", codec_name, + sampling_freq_hz, rate, pack_size); + } + + // Store packet size in samples, used to validate the received packet + pack_size_samp_ = pack_size; + + // Store the expected packet size in bytes, used to validate the received + // packet. Add 0.875 to always round up to a whole byte. + pack_size_bytes_ = (uint16_t)(static_cast(pack_size * rate) / + static_cast(sampling_freq_hz * 8) + + 0.875); + + // Set pointer to the ACM where to register the codec + AudioCodingModule* my_acm = NULL; + switch (side) { + case 'A': { + my_acm = acm_a_.get(); + break; + } + case 'B': { + my_acm = acm_b_.get(); + break; + } + default: + break; + } + ASSERT_TRUE(my_acm != NULL); + + CodecInst my_codec_param; + // Get all codec parameters before registering + EXPECT_GT(AudioCodingModule::Codec(codec_name, &my_codec_param, + sampling_freq_hz, channels), -1); + my_codec_param.rate = rate; + my_codec_param.pacsize = pack_size; + EXPECT_EQ(0, my_acm->RegisterSendCodec(my_codec_param)); + + send_codec_name_ = codec_name; +} + +void TestStereo::Run(TestPackStereo* channel, int in_channels, int out_channels, + int percent_loss) { + AudioFrame audio_frame; + + int32_t out_freq_hz_b = out_file_.SamplingFrequency(); + uint16_t rec_size; + uint32_t time_stamp_diff; + channel->reset_payload_size(); + int error_count = 0; + int variable_bytes = 0; + int variable_packets = 0; + // Set test length to 500 ms (50 blocks of 10 ms each). + in_file_mono_->SetNum10MsBlocksToRead(50); + in_file_stereo_->SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the files start with silence. + in_file_stereo_->FastForward(100); + in_file_mono_->FastForward(100); + + while (1) { + // Simulate packet loss by setting |packet_loss_| to "true" in + // |percent_loss| percent of the loops. + if (percent_loss > 0) { + if (counter_ == floor((100 / percent_loss) + 0.5)) { + counter_ = 0; + channel->set_lost_packet(true); + } else { + channel->set_lost_packet(false); + } + counter_++; + } + + // Add 10 msec to ACM + if (in_channels == 1) { + if (in_file_mono_->EndOfFile()) { + break; + } + in_file_mono_->Read10MsData(audio_frame); + } else { + if (in_file_stereo_->EndOfFile()) { + break; + } + in_file_stereo_->Read10MsData(audio_frame); + } + EXPECT_GE(acm_a_->Add10MsData(audio_frame), 0); + + // Verify that the received packet size matches the settings. + rec_size = channel->payload_size(); + if ((0 < rec_size) & (rec_size < 65535)) { + if (strcmp(send_codec_name_, "opus") == 0) { + // Opus is a variable rate codec, hence calculate the average packet + // size, and later make sure the average is in the right range. + variable_bytes += rec_size; + variable_packets++; + } else { + // For fixed rate codecs, check that packet size is correct. + if ((rec_size != pack_size_bytes_ * out_channels) + && (pack_size_bytes_ < 65535)) { + error_count++; + } + } + // Verify that the timestamp is updated with expected length + time_stamp_diff = channel->timestamp_diff(); + if ((counter_ > 10) && (time_stamp_diff != pack_size_samp_)) { + error_count++; + } + } + + // Run received side of ACM + bool muted; + EXPECT_EQ(0, acm_b_->PlayoutData10Ms(out_freq_hz_b, &audio_frame, &muted)); + ASSERT_FALSE(muted); + + // Write output speech to file + out_file_.Write10MsData( + audio_frame.data(), + audio_frame.samples_per_channel_ * audio_frame.num_channels_); + } + + EXPECT_EQ(0, error_count); + + // Check that packet size is in the right range for variable rate codecs, + // such as Opus. + if (variable_packets > 0) { + variable_bytes /= variable_packets; + EXPECT_NEAR(variable_bytes, pack_size_bytes_, 18); + } + + if (in_file_mono_->EndOfFile()) { + in_file_mono_->Rewind(); + } + if (in_file_stereo_->EndOfFile()) { + in_file_stereo_->Rewind(); + } + // Reset in case we ended with a lost packet + channel->set_lost_packet(false); +} + +void TestStereo::OpenOutFile(int16_t test_number) { + std::string file_name; + std::stringstream file_stream; + file_stream << webrtc::test::OutputPath() << "teststereo_out_" << test_number + << ".pcm"; + file_name = file_stream.str(); + out_file_.Open(file_name, 32000, "wb"); +} + +void TestStereo::DisplaySendReceiveCodec() { + auto send_codec = acm_a_->SendCodec(); + if (test_mode_ != 0) { + ASSERT_TRUE(send_codec); + printf("%s -> ", send_codec->plname); + } + CodecInst receive_codec; + acm_b_->ReceiveCodec(&receive_codec); + if (test_mode_ != 0) { + printf("%s\n", receive_codec.plname); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.h new file mode 100644 index 0000000000..a454f25639 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestStereo.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTSTEREO_H_ +#define MODULES_AUDIO_CODING_TEST_TESTSTEREO_H_ + +#include + +#include + +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" + +#define PCMA_AND_PCMU + +namespace webrtc { + +enum StereoMonoMode { + kNotSet, + kMono, + kStereo +}; + +class TestPackStereo : public AudioPacketizationCallback { + public: + TestPackStereo(); + ~TestPackStereo(); + + void RegisterReceiverACM(AudioCodingModule* acm); + + int32_t SendData(const FrameType frame_type, + const uint8_t payload_type, + const uint32_t timestamp, + const uint8_t* payload_data, + const size_t payload_size, + const RTPFragmentationHeader* fragmentation) override; + + uint16_t payload_size(); + uint32_t timestamp_diff(); + void reset_payload_size(); + void set_codec_mode(StereoMonoMode mode); + void set_lost_packet(bool lost); + + private: + AudioCodingModule* receiver_acm_; + int16_t seq_no_; + uint32_t timestamp_diff_; + uint32_t last_in_timestamp_; + uint64_t total_bytes_; + int payload_size_; + StereoMonoMode codec_mode_; + // Simulate packet losses + bool lost_packet_; +}; + +class TestStereo : public ACMTest { + public: + explicit TestStereo(int test_mode); + ~TestStereo(); + + void Perform() override; + + private: + // The default value of '-1' indicates that the registration is based only on + // codec name and a sampling frequncy matching is not required. This is useful + // for codecs which support several sampling frequency. + void RegisterSendCodec(char side, char* codec_name, int32_t samp_freq_hz, + int rate, int pack_size, int channels, + int payload_type); + + void Run(TestPackStereo* channel, int in_channels, int out_channels, + int percent_loss = 0); + void OpenOutFile(int16_t test_number); + void DisplaySendReceiveCodec(); + + int test_mode_; + + std::unique_ptr acm_a_; + std::unique_ptr acm_b_; + + TestPackStereo* channel_a2b_; + + PCMFile* in_file_stereo_; + PCMFile* in_file_mono_; + PCMFile out_file_; + int16_t test_cntr_; + uint16_t pack_size_samp_; + uint16_t pack_size_bytes_; + int counter_; + char* send_codec_name_; + + // Payload types for stereo codecs and CNG + int g722_pltype_; + int l16_8khz_pltype_; + int l16_16khz_pltype_; + int l16_32khz_pltype_; +#ifdef PCMA_AND_PCMU + int pcma_pltype_; + int pcmu_pltype_; +#endif +#ifdef WEBRTC_CODEC_OPUS + int opus_pltype_; +#endif +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTSTEREO_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.cc new file mode 100644 index 0000000000..8064448989 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.cc @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestVADDTX.h" + +#include + +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/utility.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#ifdef WEBRTC_CODEC_ISAC +const CodecInst kIsacWb = {103, "ISAC", 16000, 480, 1, 32000}; +const CodecInst kIsacSwb = {104, "ISAC", 32000, 960, 1, 56000}; +#endif + +#ifdef WEBRTC_CODEC_ILBC +const CodecInst kIlbc = {102, "ILBC", 8000, 240, 1, 13300}; +#endif + +#ifdef WEBRTC_CODEC_OPUS +const CodecInst kOpus = {120, "opus", 48000, 960, 1, 64000}; +const CodecInst kOpusStereo = {120, "opus", 48000, 960, 2, 64000}; +#endif + +ActivityMonitor::ActivityMonitor() { + ResetStatistics(); +} + +int32_t ActivityMonitor::InFrameType(FrameType frame_type) { + counter_[frame_type]++; + return 0; +} + +void ActivityMonitor::PrintStatistics() { + printf("\n"); + printf("kEmptyFrame %u\n", counter_[kEmptyFrame]); + printf("kAudioFrameSpeech %u\n", counter_[kAudioFrameSpeech]); + printf("kAudioFrameCN %u\n", counter_[kAudioFrameCN]); + printf("kVideoFrameKey %u\n", counter_[kVideoFrameKey]); + printf("kVideoFrameDelta %u\n", counter_[kVideoFrameDelta]); + printf("\n\n"); +} + +void ActivityMonitor::ResetStatistics() { + memset(counter_, 0, sizeof(counter_)); +} + +void ActivityMonitor::GetStatistics(uint32_t* counter) { + memcpy(counter, counter_, sizeof(counter_)); +} + +TestVadDtx::TestVadDtx() + : acm_send_(AudioCodingModule::Create()), + acm_receive_(AudioCodingModule::Create()), + channel_(new Channel), + monitor_(new ActivityMonitor) { + EXPECT_EQ(0, acm_send_->RegisterTransportCallback(channel_.get())); + channel_->RegisterReceiverACM(acm_receive_.get()); + EXPECT_EQ(0, acm_send_->RegisterVADCallback(monitor_.get())); +} + +void TestVadDtx::RegisterCodec(CodecInst codec_param) { + // Set the codec for sending and receiving. + EXPECT_EQ(0, acm_send_->RegisterSendCodec(codec_param)); + EXPECT_EQ(true, acm_receive_->RegisterReceiveCodec( + codec_param.pltype, CodecInstToSdp(codec_param))); + channel_->SetIsStereo(codec_param.channels > 1); +} + +// Encoding a file and see if the numbers that various packets occur follow +// the expectation. +void TestVadDtx::Run(std::string in_filename, int frequency, int channels, + std::string out_filename, bool append, + const int* expects) { + monitor_->ResetStatistics(); + + PCMFile in_file; + in_file.Open(in_filename, frequency, "rb"); + in_file.ReadStereo(channels > 1); + // Set test length to 1000 ms (100 blocks of 10 ms each). + in_file.SetNum10MsBlocksToRead(100); + // Fast-forward both files 500 ms (50 blocks). The first second of the file is + // silence, but we want to keep half of that to test silence periods. + in_file.FastForward(50); + + PCMFile out_file; + if (append) { + out_file.Open(out_filename, kOutputFreqHz, "ab"); + } else { + out_file.Open(out_filename, kOutputFreqHz, "wb"); + } + + uint16_t frame_size_samples = in_file.PayloadLength10Ms(); + AudioFrame audio_frame; + while (!in_file.EndOfFile()) { + in_file.Read10MsData(audio_frame); + audio_frame.timestamp_ = time_stamp_; + time_stamp_ += frame_size_samples; + EXPECT_GE(acm_send_->Add10MsData(audio_frame), 0); + bool muted; + acm_receive_->PlayoutData10Ms(kOutputFreqHz, &audio_frame, &muted); + ASSERT_FALSE(muted); + out_file.Write10MsData(audio_frame); + } + + in_file.Close(); + out_file.Close(); + +#ifdef PRINT_STAT + monitor_->PrintStatistics(); +#endif + + uint32_t stats[5]; + monitor_->GetStatistics(stats); + monitor_->ResetStatistics(); + + for (const auto& st : stats) { + int i = &st - stats; // Calculate the current position in stats. + switch (expects[i]) { + case 0: { + EXPECT_EQ(0u, st) << "stats[" << i << "] error."; + break; + } + case 1: { + EXPECT_GT(st, 0u) << "stats[" << i << "] error."; + break; + } + } + } +} + +// Following is the implementation of TestWebRtcVadDtx. +TestWebRtcVadDtx::TestWebRtcVadDtx() + : vad_enabled_(false), + dtx_enabled_(false), + output_file_num_(0) { +} + +void TestWebRtcVadDtx::Perform() { + // Go through various test cases. +#ifdef WEBRTC_CODEC_ISAC + // Register iSAC WB as send codec + RegisterCodec(kIsacWb); + RunTestCases(); + + // Register iSAC SWB as send codec + RegisterCodec(kIsacSwb); + RunTestCases(); +#endif + +#ifdef WEBRTC_CODEC_ILBC + // Register iLBC as send codec + RegisterCodec(kIlbc); + RunTestCases(); +#endif + +#ifdef WEBRTC_CODEC_OPUS + // Register Opus as send codec + RegisterCodec(kOpus); + RunTestCases(); +#endif +} + +// Test various configurations on VAD/DTX. +void TestWebRtcVadDtx::RunTestCases() { + // #1 DTX = OFF, VAD = OFF, VADNormal + SetVAD(false, false, VADNormal); + Test(true); + + // #2 DTX = ON, VAD = ON, VADAggr + SetVAD(true, true, VADAggr); + Test(false); + + // #3 DTX = ON, VAD = ON, VADLowBitrate + SetVAD(true, true, VADLowBitrate); + Test(false); + + // #4 DTX = ON, VAD = ON, VADVeryAggr + SetVAD(true, true, VADVeryAggr); + Test(false); + + // #5 DTX = ON, VAD = ON, VADNormal + SetVAD(true, true, VADNormal); + Test(false); +} + +// Set the expectation and run the test. +void TestWebRtcVadDtx::Test(bool new_outfile) { + int expects[] = {-1, 1, dtx_enabled_, 0, 0}; + if (new_outfile) { + output_file_num_++; + } + std::stringstream out_filename; + out_filename << webrtc::test::OutputPath() + << "testWebRtcVadDtx_outFile_" + << output_file_num_ + << ".pcm"; + Run(webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000, 1, out_filename.str(), !new_outfile, expects); +} + +void TestWebRtcVadDtx::SetVAD(bool enable_dtx, bool enable_vad, + ACMVADMode vad_mode) { + ACMVADMode mode; + EXPECT_EQ(0, acm_send_->SetVAD(enable_dtx, enable_vad, vad_mode)); + EXPECT_EQ(0, acm_send_->VAD(&dtx_enabled_, &vad_enabled_, &mode)); + + auto codec_param = acm_send_->SendCodec(); + ASSERT_TRUE(codec_param); + if (STR_CASE_CMP(codec_param->plname, "opus") == 0) { + // If send codec is Opus, WebRTC VAD/DTX cannot be used. + enable_dtx = enable_vad = false; + } + + EXPECT_EQ(dtx_enabled_ , enable_dtx); // DTX should be set as expected. + + if (dtx_enabled_) { + EXPECT_TRUE(vad_enabled_); // WebRTC DTX cannot run without WebRTC VAD. + } else { + // Using no DTX should not affect setting of VAD. + EXPECT_EQ(enable_vad, vad_enabled_); + } +} + +// Following is the implementation of TestOpusDtx. +void TestOpusDtx::Perform() { +#ifdef WEBRTC_CODEC_ISAC + // If we set other codec than Opus, DTX cannot be switched on. + RegisterCodec(kIsacWb); + EXPECT_EQ(-1, acm_send_->EnableOpusDtx()); + EXPECT_EQ(0, acm_send_->DisableOpusDtx()); +#endif + +#ifdef WEBRTC_CODEC_OPUS + int expects[] = {0, 1, 0, 0, 0}; + + // Register Opus as send codec + std::string out_filename = webrtc::test::OutputPath() + + "testOpusDtx_outFile_mono.pcm"; + RegisterCodec(kOpus); + EXPECT_EQ(0, acm_send_->DisableOpusDtx()); + + Run(webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000, 1, out_filename, false, expects); + + EXPECT_EQ(0, acm_send_->EnableOpusDtx()); + expects[kEmptyFrame] = 1; + expects[kAudioFrameCN] = 1; + Run(webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000, 1, out_filename, true, expects); + + // Register stereo Opus as send codec + out_filename = webrtc::test::OutputPath() + "testOpusDtx_outFile_stereo.pcm"; + RegisterCodec(kOpusStereo); + EXPECT_EQ(0, acm_send_->DisableOpusDtx()); + expects[kEmptyFrame] = 0; + expects[kAudioFrameCN] = 0; + Run(webrtc::test::ResourcePath("audio_coding/teststereo32kHz", "pcm"), + 32000, 2, out_filename, false, expects); + + EXPECT_EQ(0, acm_send_->EnableOpusDtx()); + + expects[kEmptyFrame] = 1; + expects[kAudioFrameCN] = 1; + Run(webrtc::test::ResourcePath("audio_coding/teststereo32kHz", "pcm"), + 32000, 2, out_filename, true, expects); +#endif +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.h new file mode 100644 index 0000000000..8cd4444f3d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TestVADDTX.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTVADDTX_H_ +#define MODULES_AUDIO_CODING_TEST_TESTVADDTX_H_ + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" + +namespace webrtc { + +class ActivityMonitor : public ACMVADCallback { + public: + ActivityMonitor(); + int32_t InFrameType(FrameType frame_type); + void PrintStatistics(); + void ResetStatistics(); + void GetStatistics(uint32_t* stats); + private: + // 0 - kEmptyFrame + // 1 - kAudioFrameSpeech + // 2 - kAudioFrameCN + // 3 - kVideoFrameKey (not used by audio) + // 4 - kVideoFrameDelta (not used by audio) + uint32_t counter_[5]; +}; + + +// TestVadDtx is to verify that VAD/DTX perform as they should. It runs through +// an audio file and check if the occurrence of various packet types follows +// expectation. TestVadDtx needs its derived class to implement the Perform() +// to put the test together. +class TestVadDtx : public ACMTest { + public: + static const int kOutputFreqHz = 16000; + + TestVadDtx(); + + virtual void Perform() = 0; + + protected: + void RegisterCodec(CodecInst codec_param); + + // Encoding a file and see if the numbers that various packets occur follow + // the expectation. Saves result to a file. + // expects[x] means + // -1 : do not care, + // 0 : there have been no packets of type |x|, + // 1 : there have been packets of type |x|, + // with |x| indicates the following packet types + // 0 - kEmptyFrame + // 1 - kAudioFrameSpeech + // 2 - kAudioFrameCN + // 3 - kVideoFrameKey (not used by audio) + // 4 - kVideoFrameDelta (not used by audio) + void Run(std::string in_filename, int frequency, int channels, + std::string out_filename, bool append, const int* expects); + + std::unique_ptr acm_send_; + std::unique_ptr acm_receive_; + std::unique_ptr channel_; + std::unique_ptr monitor_; + uint32_t time_stamp_ = 0x12345678; +}; + +// TestWebRtcVadDtx is to verify that the WebRTC VAD/DTX perform as they should. +class TestWebRtcVadDtx final : public TestVadDtx { + public: + TestWebRtcVadDtx(); + + void Perform() override; + + private: + void RunTestCases(); + void Test(bool new_outfile); + void SetVAD(bool enable_dtx, bool enable_vad, ACMVADMode vad_mode); + + bool vad_enabled_; + bool dtx_enabled_; + int output_file_num_; +}; + +// TestOpusDtx is to verify that the Opus DTX performs as it should. +class TestOpusDtx final : public TestVadDtx { + public: + void Perform() override; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTVADDTX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/Tester.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/Tester.cc new file mode 100644 index 0000000000..9cd774dd15 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/Tester.cc @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/APITest.h" +#include "modules/audio_coding/test/EncodeDecodeTest.h" +#include "modules/audio_coding/test/PacketLossTest.h" +#include "modules/audio_coding/test/TestAllCodecs.h" +#include "modules/audio_coding/test/TestRedFec.h" +#include "modules/audio_coding/test/TestStereo.h" +#include "modules/audio_coding/test/TestVADDTX.h" +#include "modules/audio_coding/test/TwoWayCommunication.h" +#include "modules/audio_coding/test/iSACTest.h" +#include "modules/audio_coding/test/opus_test.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +// This parameter is used to describe how to run the tests. It is normally +// set to 0, and all tests are run in quite mode. +#define ACM_TEST_MODE 0 + +TEST(AudioCodingModuleTest, TestAllCodecs) { + webrtc::TestAllCodecs(ACM_TEST_MODE).Perform(); +} + +#if defined(WEBRTC_ANDROID) +TEST(AudioCodingModuleTest, DISABLED_TestEncodeDecode) { +#else +TEST(AudioCodingModuleTest, TestEncodeDecode) { +#endif + webrtc::EncodeDecodeTest(ACM_TEST_MODE).Perform(); +} + +#if defined(WEBRTC_CODEC_RED) +#if defined(WEBRTC_ANDROID) +TEST(AudioCodingModuleTest, DISABLED_TestRedFec) { +#else +TEST(AudioCodingModuleTest, TestRedFec) { +#endif + webrtc::TestRedFec().Perform(); +} +#endif + +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +#if defined(WEBRTC_ANDROID) +TEST(AudioCodingModuleTest, DISABLED_TestIsac) { +#else +TEST(AudioCodingModuleTest, TestIsac) { +#endif + webrtc::ISACTest(ACM_TEST_MODE).Perform(); +} +#endif + +#if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) && \ + defined(WEBRTC_CODEC_ILBC) +#if defined(WEBRTC_ANDROID) +TEST(AudioCodingModuleTest, DISABLED_TwoWayCommunication) { +#else +TEST(AudioCodingModuleTest, TwoWayCommunication) { +#endif + webrtc::TwoWayCommunication(ACM_TEST_MODE).Perform(); +} +#endif + +// Disabled on ios as flaky, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestStereo) { +#else +TEST(AudioCodingModuleTest, TestStereo) { +#endif + webrtc::TestStereo(ACM_TEST_MODE).Perform(); +} + +// Disabled on ios as flaky, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestWebRtcVadDtx) { +#else +TEST(AudioCodingModuleTest, TestWebRtcVadDtx) { +#endif + webrtc::TestWebRtcVadDtx().Perform(); +} + +TEST(AudioCodingModuleTest, TestOpusDtx) { + webrtc::TestOpusDtx().Perform(); +} + +// Disabled on ios as flaky, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestOpus) { +#else +TEST(AudioCodingModuleTest, TestOpus) { +#endif + webrtc::OpusTest().Perform(); +} + +TEST(AudioCodingModuleTest, TestPacketLoss) { + webrtc::PacketLossTest(1, 10, 10, 1).Perform(); +} + +TEST(AudioCodingModuleTest, TestPacketLossBurst) { + webrtc::PacketLossTest(1, 10, 10, 2).Perform(); +} + +// Disabled on ios as flake, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestPacketLossStereo) { +#else + TEST(AudioCodingModuleTest, TestPacketLossStereo) { +#endif + webrtc::PacketLossTest(2, 10, 10, 1).Perform(); +} + +// Disabled on ios as flake, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestPacketLossStereoBurst) { +#else +TEST(AudioCodingModuleTest, TestPacketLossStereoBurst) { +#endif + webrtc::PacketLossTest(2, 10, 10, 2).Perform(); +} + +// The full API test is too long to run automatically on bots, but can be used +// for offline testing. User interaction is needed. +#ifdef ACM_TEST_FULL_API + TEST(AudioCodingModuleTest, TestAPI) { + webrtc::APITest().Perform(); + } +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.cc new file mode 100644 index 0000000000..8049436cd0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.cc @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "TwoWayCommunication.h" + +#include +#include +#include + +#include + +#ifdef WIN32 +#include +#endif + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/utility.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#define MAX_FILE_NAME_LENGTH_BYTE 500 + +TwoWayCommunication::TwoWayCommunication(int testMode) + : _acmA(AudioCodingModule::Create()), + _acmRefA(AudioCodingModule::Create()), + _testMode(testMode) { + AudioCodingModule::Config config; + // The clicks will be more obvious in FAX mode. TODO(henrik.lundin) Really? + config.neteq_config.playout_mode = kPlayoutFax; + config.decoder_factory = CreateBuiltinAudioDecoderFactory(); + _acmB.reset(AudioCodingModule::Create(config)); + _acmRefB.reset(AudioCodingModule::Create(config)); +} + +TwoWayCommunication::~TwoWayCommunication() { + delete _channel_A2B; + delete _channel_B2A; + delete _channelRef_A2B; + delete _channelRef_B2A; + _inFileA.Close(); + _inFileB.Close(); + _outFileA.Close(); + _outFileB.Close(); + _outFileRefA.Close(); + _outFileRefB.Close(); +} + +void TwoWayCommunication::ChooseCodec(uint8_t* codecID_A, + uint8_t* codecID_B) { + std::unique_ptr tmpACM(AudioCodingModule::Create()); + uint8_t noCodec = tmpACM->NumberOfCodecs(); + CodecInst codecInst; + printf("List of Supported Codecs\n"); + printf("========================\n"); + for (uint8_t codecCntr = 0; codecCntr < noCodec; codecCntr++) { + EXPECT_EQ(tmpACM->Codec(codecCntr, &codecInst), 0); + printf("%d- %s\n", codecCntr, codecInst.plname); + } + printf("\nChoose a send codec for side A [0]: "); + char myStr[15] = ""; + EXPECT_TRUE(fgets(myStr, 10, stdin) != NULL); + *codecID_A = (uint8_t) atoi(myStr); + + printf("\nChoose a send codec for side B [0]: "); + EXPECT_TRUE(fgets(myStr, 10, stdin) != NULL); + *codecID_B = (uint8_t) atoi(myStr); + + printf("\n"); +} + +void TwoWayCommunication::SetUp() { + uint8_t codecID_A; + uint8_t codecID_B; + + ChooseCodec(&codecID_A, &codecID_B); + CodecInst codecInst_A; + CodecInst codecInst_B; + CodecInst dummyCodec; + EXPECT_EQ(0, _acmA->Codec(codecID_A, &codecInst_A)); + EXPECT_EQ(0, _acmB->Codec(codecID_B, &codecInst_B)); + EXPECT_EQ(0, _acmA->Codec(6, &dummyCodec)); + + //--- Set A codecs + EXPECT_EQ(0, _acmA->RegisterSendCodec(codecInst_A)); + EXPECT_EQ(true, _acmA->RegisterReceiveCodec(codecInst_B.pltype, + CodecInstToSdp(codecInst_B))); + //--- Set ref-A codecs + EXPECT_EQ(0, _acmRefA->RegisterSendCodec(codecInst_A)); + EXPECT_EQ(true, _acmRefA->RegisterReceiveCodec(codecInst_B.pltype, + CodecInstToSdp(codecInst_B))); + + //--- Set B codecs + EXPECT_EQ(0, _acmB->RegisterSendCodec(codecInst_B)); + EXPECT_EQ(true, _acmB->RegisterReceiveCodec(codecInst_A.pltype, + CodecInstToSdp(codecInst_A))); + + //--- Set ref-B codecs + EXPECT_EQ(0, _acmRefB->RegisterSendCodec(codecInst_B)); + EXPECT_EQ(true, _acmRefB->RegisterReceiveCodec(codecInst_A.pltype, + CodecInstToSdp(codecInst_A))); + + uint16_t frequencyHz; + + //--- Input A + std::string in_file_name = webrtc::test::ResourcePath( + "audio_coding/testfile32kHz", "pcm"); + frequencyHz = 32000; + printf("Enter input file at side A [%s]: ", in_file_name.c_str()); + PCMFile::ChooseFile(&in_file_name, 499, &frequencyHz); + _inFileA.Open(in_file_name, frequencyHz, "rb"); + + //--- Output A + std::string out_file_a = webrtc::test::OutputPath() + "outA.pcm"; + printf("Output file at side A: %s\n", out_file_a.c_str()); + printf("Sampling frequency (in Hz) of the above file: %u\n", frequencyHz); + _outFileA.Open(out_file_a, frequencyHz, "wb"); + std::string ref_file_name = webrtc::test::OutputPath() + "ref_outA.pcm"; + _outFileRefA.Open(ref_file_name, frequencyHz, "wb"); + + //--- Input B + in_file_name = webrtc::test::ResourcePath("audio_coding/testfile32kHz", + "pcm"); + frequencyHz = 32000; + printf("\n\nEnter input file at side B [%s]: ", in_file_name.c_str()); + PCMFile::ChooseFile(&in_file_name, 499, &frequencyHz); + _inFileB.Open(in_file_name, frequencyHz, "rb"); + + //--- Output B + std::string out_file_b = webrtc::test::OutputPath() + "outB.pcm"; + printf("Output file at side B: %s\n", out_file_b.c_str()); + printf("Sampling frequency (in Hz) of the above file: %u\n", frequencyHz); + _outFileB.Open(out_file_b, frequencyHz, "wb"); + ref_file_name = webrtc::test::OutputPath() + "ref_outB.pcm"; + _outFileRefB.Open(ref_file_name, frequencyHz, "wb"); + + //--- Set A-to-B channel + _channel_A2B = new Channel; + _acmA->RegisterTransportCallback(_channel_A2B); + _channel_A2B->RegisterReceiverACM(_acmB.get()); + //--- Do the same for the reference + _channelRef_A2B = new Channel; + _acmRefA->RegisterTransportCallback(_channelRef_A2B); + _channelRef_A2B->RegisterReceiverACM(_acmRefB.get()); + + //--- Set B-to-A channel + _channel_B2A = new Channel; + _acmB->RegisterTransportCallback(_channel_B2A); + _channel_B2A->RegisterReceiverACM(_acmA.get()); + //--- Do the same for reference + _channelRef_B2A = new Channel; + _acmRefB->RegisterTransportCallback(_channelRef_B2A); + _channelRef_B2A->RegisterReceiverACM(_acmRefA.get()); +} + +void TwoWayCommunication::SetUpAutotest() { + CodecInst codecInst_A; + CodecInst codecInst_B; + CodecInst dummyCodec; + + EXPECT_EQ(0, _acmA->Codec("ISAC", &codecInst_A, 16000, 1)); + EXPECT_EQ(0, _acmB->Codec("L16", &codecInst_B, 8000, 1)); + EXPECT_EQ(0, _acmA->Codec(6, &dummyCodec)); + + //--- Set A codecs + EXPECT_EQ(0, _acmA->RegisterSendCodec(codecInst_A)); + EXPECT_EQ(true, _acmA->RegisterReceiveCodec(codecInst_B.pltype, + CodecInstToSdp(codecInst_B))); + + //--- Set ref-A codecs + EXPECT_GT(_acmRefA->RegisterSendCodec(codecInst_A), -1); + EXPECT_EQ(true, _acmRefA->RegisterReceiveCodec(codecInst_B.pltype, + CodecInstToSdp(codecInst_B))); + + //--- Set B codecs + EXPECT_GT(_acmB->RegisterSendCodec(codecInst_B), -1); + EXPECT_EQ(true, _acmB->RegisterReceiveCodec(codecInst_A.pltype, + CodecInstToSdp(codecInst_A))); + + //--- Set ref-B codecs + EXPECT_EQ(0, _acmRefB->RegisterSendCodec(codecInst_B)); + EXPECT_EQ(true, _acmRefB->RegisterReceiveCodec(codecInst_A.pltype, + CodecInstToSdp(codecInst_A))); + + uint16_t frequencyHz; + + //--- Input A and B + std::string in_file_name = webrtc::test::ResourcePath( + "audio_coding/testfile32kHz", "pcm"); + frequencyHz = 16000; + _inFileA.Open(in_file_name, frequencyHz, "rb"); + _inFileB.Open(in_file_name, frequencyHz, "rb"); + + //--- Output A + std::string output_file_a = webrtc::test::OutputPath() + "outAutotestA.pcm"; + frequencyHz = 16000; + _outFileA.Open(output_file_a, frequencyHz, "wb"); + std::string output_ref_file_a = webrtc::test::OutputPath() + + "ref_outAutotestA.pcm"; + _outFileRefA.Open(output_ref_file_a, frequencyHz, "wb"); + + //--- Output B + std::string output_file_b = webrtc::test::OutputPath() + "outAutotestB.pcm"; + frequencyHz = 16000; + _outFileB.Open(output_file_b, frequencyHz, "wb"); + std::string output_ref_file_b = webrtc::test::OutputPath() + + "ref_outAutotestB.pcm"; + _outFileRefB.Open(output_ref_file_b, frequencyHz, "wb"); + + //--- Set A-to-B channel + _channel_A2B = new Channel; + _acmA->RegisterTransportCallback(_channel_A2B); + _channel_A2B->RegisterReceiverACM(_acmB.get()); + //--- Do the same for the reference + _channelRef_A2B = new Channel; + _acmRefA->RegisterTransportCallback(_channelRef_A2B); + _channelRef_A2B->RegisterReceiverACM(_acmRefB.get()); + + //--- Set B-to-A channel + _channel_B2A = new Channel; + _acmB->RegisterTransportCallback(_channel_B2A); + _channel_B2A->RegisterReceiverACM(_acmA.get()); + //--- Do the same for reference + _channelRef_B2A = new Channel; + _acmRefB->RegisterTransportCallback(_channelRef_B2A); + _channelRef_B2A->RegisterReceiverACM(_acmRefA.get()); +} + +void TwoWayCommunication::Perform() { + if (_testMode == 0) { + SetUpAutotest(); + } else { + SetUp(); + } + unsigned int msecPassed = 0; + unsigned int secPassed = 0; + + int32_t outFreqHzA = _outFileA.SamplingFrequency(); + int32_t outFreqHzB = _outFileB.SamplingFrequency(); + + AudioFrame audioFrame; + + auto codecInst_B = _acmB->SendCodec(); + ASSERT_TRUE(codecInst_B); + + // In the following loop we tests that the code can handle misuse of the APIs. + // In the middle of a session with data flowing between two sides, called A + // and B, APIs will be called, and the code should continue to run, and be + // able to recover. + while (!_inFileA.EndOfFile() && !_inFileB.EndOfFile()) { + msecPassed += 10; + EXPECT_GT(_inFileA.Read10MsData(audioFrame), 0); + EXPECT_GE(_acmA->Add10MsData(audioFrame), 0); + EXPECT_GE(_acmRefA->Add10MsData(audioFrame), 0); + + EXPECT_GT(_inFileB.Read10MsData(audioFrame), 0); + + EXPECT_GE(_acmB->Add10MsData(audioFrame), 0); + EXPECT_GE(_acmRefB->Add10MsData(audioFrame), 0); + bool muted; + EXPECT_EQ(0, _acmA->PlayoutData10Ms(outFreqHzA, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileA.Write10MsData(audioFrame); + EXPECT_EQ(0, _acmRefA->PlayoutData10Ms(outFreqHzA, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileRefA.Write10MsData(audioFrame); + EXPECT_EQ(0, _acmB->PlayoutData10Ms(outFreqHzB, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileB.Write10MsData(audioFrame); + EXPECT_EQ(0, _acmRefB->PlayoutData10Ms(outFreqHzB, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileRefB.Write10MsData(audioFrame); + + // Update time counters each time a second of data has passed. + if (msecPassed >= 1000) { + msecPassed = 0; + secPassed++; + } + // Re-register send codec on side B. + if (((secPassed % 5) == 4) && (msecPassed >= 990)) { + EXPECT_EQ(0, _acmB->RegisterSendCodec(*codecInst_B)); + EXPECT_TRUE(_acmB->SendCodec()); + } + // Initialize receiver on side A. + if (((secPassed % 7) == 6) && (msecPassed == 0)) + EXPECT_EQ(0, _acmA->InitializeReceiver()); + // Re-register codec on side A. + if (((secPassed % 7) == 6) && (msecPassed >= 990)) { + EXPECT_EQ(true, _acmA->RegisterReceiveCodec( + codecInst_B->pltype, CodecInstToSdp(*codecInst_B))); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.h new file mode 100644 index 0000000000..fb23275a9a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/TwoWayCommunication.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TWOWAYCOMMUNICATION_H_ +#define MODULES_AUDIO_CODING_TEST_TWOWAYCOMMUNICATION_H_ + +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/utility.h" + +namespace webrtc { + +class TwoWayCommunication : public ACMTest { + public: + explicit TwoWayCommunication(int testMode); + ~TwoWayCommunication(); + + void Perform(); + private: + void ChooseCodec(uint8_t* codecID_A, uint8_t* codecID_B); + void SetUp(); + void SetUpAutotest(); + + std::unique_ptr _acmA; + std::unique_ptr _acmB; + + std::unique_ptr _acmRefA; + std::unique_ptr _acmRefB; + + Channel* _channel_A2B; + Channel* _channel_B2A; + + Channel* _channelRef_A2B; + Channel* _channelRef_B2A; + + PCMFile _inFileA; + PCMFile _inFileB; + + PCMFile _outFileA; + PCMFile _outFileB; + + PCMFile _outFileRefA; + PCMFile _outFileRefB; + + int _testMode; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TWOWAYCOMMUNICATION_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/delay_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/delay_test.cc new file mode 100644 index 0000000000..407f7095cb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/delay_test.cc @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/utility.h" +#include "rtc_base/flags.h" +#include "system_wrappers/include/event_wrapper.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +DEFINE_string(codec, "isac", "Codec Name"); +DEFINE_int(sample_rate_hz, 16000, "Sampling rate in Hertz."); +DEFINE_int(num_channels, 1, "Number of Channels."); +DEFINE_string(input_file, "", "Input file, PCM16 32 kHz, optional."); +DEFINE_int(delay, 0, "Delay in millisecond."); +DEFINE_bool(dtx, false, "Enable DTX at the sender side."); +DEFINE_bool(packet_loss, false, "Apply packet loss, c.f. Channel{.cc, .h}."); +DEFINE_bool(fec, false, "Use Forward Error Correction (FEC)."); +DEFINE_bool(help, false, "Print this message."); + +namespace webrtc { + +namespace { + +struct CodecSettings { + char name[50]; + int sample_rate_hz; + int num_channels; +}; + +struct AcmSettings { + bool dtx; + bool fec; +}; + +struct TestSettings { + CodecSettings codec; + AcmSettings acm; + bool packet_loss; +}; + +} // namespace + +class DelayTest { + public: + DelayTest() + : acm_a_(AudioCodingModule::Create()), + acm_b_(AudioCodingModule::Create()), + channel_a2b_(new Channel), + test_cntr_(0), + encoding_sample_rate_hz_(8000) {} + + ~DelayTest() { + if (channel_a2b_ != NULL) { + delete channel_a2b_; + channel_a2b_ = NULL; + } + in_file_a_.Close(); + } + + void Initialize() { + test_cntr_ = 0; + std::string file_name = webrtc::test::ResourcePath( + "audio_coding/testfile32kHz", "pcm"); + if (strlen(FLAG_input_file) > 0) + file_name = FLAG_input_file; + in_file_a_.Open(file_name, 32000, "rb"); + ASSERT_EQ(0, acm_a_->InitializeReceiver()) << + "Couldn't initialize receiver.\n"; + ASSERT_EQ(0, acm_b_->InitializeReceiver()) << + "Couldn't initialize receiver.\n"; + + if (FLAG_delay > 0) { + ASSERT_EQ(0, acm_b_->SetMinimumPlayoutDelay(FLAG_delay)) << + "Failed to set minimum delay.\n"; + } + + int num_encoders = acm_a_->NumberOfCodecs(); + CodecInst my_codec_param; + for (int n = 0; n < num_encoders; n++) { + EXPECT_EQ(0, acm_b_->Codec(n, &my_codec_param)) << + "Failed to get codec."; + if (STR_CASE_CMP(my_codec_param.plname, "opus") == 0) + my_codec_param.channels = 1; + else if (my_codec_param.channels > 1) + continue; + if (STR_CASE_CMP(my_codec_param.plname, "CN") == 0 && + my_codec_param.plfreq == 48000) + continue; + if (STR_CASE_CMP(my_codec_param.plname, "telephone-event") == 0) + continue; + ASSERT_EQ(true, + acm_b_->RegisterReceiveCodec(my_codec_param.pltype, + CodecInstToSdp(my_codec_param))); + } + + // Create and connect the channel + ASSERT_EQ(0, acm_a_->RegisterTransportCallback(channel_a2b_)) << + "Couldn't register Transport callback.\n"; + channel_a2b_->RegisterReceiverACM(acm_b_.get()); + } + + void Perform(const TestSettings* config, size_t num_tests, int duration_sec, + const char* output_prefix) { + for (size_t n = 0; n < num_tests; ++n) { + ApplyConfig(config[n]); + Run(duration_sec, output_prefix); + } + } + + private: + void ApplyConfig(const TestSettings& config) { + printf("====================================\n"); + printf("Test %d \n" + "Codec: %s, %d kHz, %d channel(s)\n" + "ACM: DTX %s, FEC %s\n" + "Channel: %s\n", + ++test_cntr_, config.codec.name, config.codec.sample_rate_hz, + config.codec.num_channels, config.acm.dtx ? "on" : "off", + config.acm.fec ? "on" : "off", + config.packet_loss ? "with packet-loss" : "no packet-loss"); + SendCodec(config.codec); + ConfigAcm(config.acm); + ConfigChannel(config.packet_loss); + } + + void SendCodec(const CodecSettings& config) { + CodecInst my_codec_param; + ASSERT_EQ(0, AudioCodingModule::Codec( + config.name, &my_codec_param, config.sample_rate_hz, + config.num_channels)) << "Specified codec is not supported.\n"; + + encoding_sample_rate_hz_ = my_codec_param.plfreq; + ASSERT_EQ(0, acm_a_->RegisterSendCodec(my_codec_param)) << + "Failed to register send-codec.\n"; + } + + void ConfigAcm(const AcmSettings& config) { + ASSERT_EQ(0, acm_a_->SetVAD(config.dtx, config.dtx, VADAggr)) << + "Failed to set VAD.\n"; + ASSERT_EQ(0, acm_a_->SetREDStatus(config.fec)) << + "Failed to set RED.\n"; + } + + void ConfigChannel(bool packet_loss) { + channel_a2b_->SetFECTestWithPacketLoss(packet_loss); + } + + void OpenOutFile(const char* output_id) { + std::stringstream file_stream; + file_stream << "delay_test_" << FLAG_codec << "_" << FLAG_sample_rate_hz + << "Hz" << "_" << FLAG_delay << "ms.pcm"; + std::cout << "Output file: " << file_stream.str() << std::endl << std::endl; + std::string file_name = webrtc::test::OutputPath() + file_stream.str(); + out_file_b_.Open(file_name.c_str(), 32000, "wb"); + } + + void Run(int duration_sec, const char* output_prefix) { + OpenOutFile(output_prefix); + AudioFrame audio_frame; + uint32_t out_freq_hz_b = out_file_b_.SamplingFrequency(); + + int num_frames = 0; + int in_file_frames = 0; + uint32_t received_ts; + double average_delay = 0; + double inst_delay_sec = 0; + while (num_frames < (duration_sec * 100)) { + if (in_file_a_.EndOfFile()) { + in_file_a_.Rewind(); + } + + // Print delay information every 16 frame + if ((num_frames & 0x3F) == 0x3F) { + NetworkStatistics statistics; + acm_b_->GetNetworkStatistics(&statistics); + fprintf(stdout, "delay: min=%3d max=%3d mean=%3d median=%3d" + " ts-based average = %6.3f, " + "curr buff-lev = %4u opt buff-lev = %4u \n", + statistics.minWaitingTimeMs, statistics.maxWaitingTimeMs, + statistics.meanWaitingTimeMs, statistics.medianWaitingTimeMs, + average_delay, statistics.currentBufferSize, + statistics.preferredBufferSize); + fflush (stdout); + } + + in_file_a_.Read10MsData(audio_frame); + ASSERT_GE(acm_a_->Add10MsData(audio_frame), 0); + bool muted; + ASSERT_EQ(0, + acm_b_->PlayoutData10Ms(out_freq_hz_b, &audio_frame, &muted)); + RTC_DCHECK(!muted); + out_file_b_.Write10MsData( + audio_frame.data(), + audio_frame.samples_per_channel_ * audio_frame.num_channels_); + received_ts = channel_a2b_->LastInTimestamp(); + rtc::Optional playout_timestamp = acm_b_->PlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + inst_delay_sec = static_cast(received_ts - *playout_timestamp) / + static_cast(encoding_sample_rate_hz_); + + if (num_frames > 10) + average_delay = 0.95 * average_delay + 0.05 * inst_delay_sec; + + ++num_frames; + ++in_file_frames; + } + out_file_b_.Close(); + } + + std::unique_ptr acm_a_; + std::unique_ptr acm_b_; + + Channel* channel_a2b_; + + PCMFile in_file_a_; + PCMFile out_file_b_; + int test_cntr_; + int encoding_sample_rate_hz_; +}; + +} // namespace webrtc + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true)) { + return 1; + } + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + + webrtc::TestSettings test_setting; + strcpy(test_setting.codec.name, FLAG_codec); + + if (FLAG_sample_rate_hz != 8000 && + FLAG_sample_rate_hz != 16000 && + FLAG_sample_rate_hz != 32000 && + FLAG_sample_rate_hz != 48000) { + std::cout << "Invalid sampling rate.\n"; + return 1; + } + test_setting.codec.sample_rate_hz = FLAG_sample_rate_hz; + if (FLAG_num_channels < 1 || FLAG_num_channels > 2) { + std::cout << "Only mono and stereo are supported.\n"; + return 1; + } + test_setting.codec.num_channels = FLAG_num_channels; + test_setting.acm.dtx = FLAG_dtx; + test_setting.acm.fec = FLAG_fec; + test_setting.packet_loss = FLAG_packet_loss; + + webrtc::DelayTest delay_test; + delay_test.Initialize(); + delay_test.Perform(&test_setting, 1, 240, "delay_test"); + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.cc new file mode 100644 index 0000000000..a44259fbbe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.cc @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/iSACTest.h" + +#include +#include +#include + +#ifdef _WIN32 +#include +#elif defined(WEBRTC_LINUX) +#include +#else +#include +#include +#endif + +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/test/utility.h" +#include "system_wrappers/include/event_wrapper.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +void SetISACConfigDefault(ACMTestISACConfig& isacConfig) { + isacConfig.currentRateBitPerSec = 0; + isacConfig.currentFrameSizeMsec = 0; + isacConfig.encodingMode = -1; + isacConfig.initRateBitPerSec = 0; + isacConfig.initFrameSizeInMsec = 0; + isacConfig.enforceFrameSize = false; + return; +} + +int16_t SetISAConfig(ACMTestISACConfig& isacConfig, AudioCodingModule* acm, + int testMode) { + + if ((isacConfig.currentRateBitPerSec != 0) + || (isacConfig.currentFrameSizeMsec != 0)) { + auto sendCodec = acm->SendCodec(); + EXPECT_TRUE(sendCodec); + if (isacConfig.currentRateBitPerSec < 0) { + // Register iSAC in adaptive (channel-dependent) mode. + sendCodec->rate = -1; + EXPECT_EQ(0, acm->RegisterSendCodec(*sendCodec)); + } else { + if (isacConfig.currentRateBitPerSec != 0) { + sendCodec->rate = isacConfig.currentRateBitPerSec; + } + if (isacConfig.currentFrameSizeMsec != 0) { + sendCodec->pacsize = isacConfig.currentFrameSizeMsec + * (sendCodec->plfreq / 1000); + } + EXPECT_EQ(0, acm->RegisterSendCodec(*sendCodec)); + } + } + + return 0; +} + +ISACTest::ISACTest(int testMode) + : _acmA(AudioCodingModule::Create()), + _acmB(AudioCodingModule::Create()), + _testMode(testMode) {} + +ISACTest::~ISACTest() {} + +void ISACTest::Setup() { + int codecCntr; + CodecInst codecParam; + + for (codecCntr = 0; codecCntr < AudioCodingModule::NumberOfCodecs(); + codecCntr++) { + EXPECT_EQ(0, AudioCodingModule::Codec(codecCntr, &codecParam)); + if (!STR_CASE_CMP(codecParam.plname, "ISAC") + && codecParam.plfreq == 16000) { + memcpy(&_paramISAC16kHz, &codecParam, sizeof(CodecInst)); + _idISAC16kHz = codecCntr; + } + if (!STR_CASE_CMP(codecParam.plname, "ISAC") + && codecParam.plfreq == 32000) { + memcpy(&_paramISAC32kHz, &codecParam, sizeof(CodecInst)); + _idISAC32kHz = codecCntr; + } + } + + // Register both iSAC-wb & iSAC-swb in both sides as receiver codecs. + EXPECT_EQ(true, _acmA->RegisterReceiveCodec(_paramISAC16kHz.pltype, + CodecInstToSdp(_paramISAC16kHz))); + EXPECT_EQ(true, _acmA->RegisterReceiveCodec(_paramISAC32kHz.pltype, + CodecInstToSdp(_paramISAC32kHz))); + EXPECT_EQ(true, _acmB->RegisterReceiveCodec(_paramISAC16kHz.pltype, + CodecInstToSdp(_paramISAC16kHz))); + EXPECT_EQ(true, _acmB->RegisterReceiveCodec(_paramISAC32kHz.pltype, + CodecInstToSdp(_paramISAC32kHz))); + + //--- Set A-to-B channel + _channel_A2B.reset(new Channel); + EXPECT_EQ(0, _acmA->RegisterTransportCallback(_channel_A2B.get())); + _channel_A2B->RegisterReceiverACM(_acmB.get()); + + //--- Set B-to-A channel + _channel_B2A.reset(new Channel); + EXPECT_EQ(0, _acmB->RegisterTransportCallback(_channel_B2A.get())); + _channel_B2A->RegisterReceiverACM(_acmA.get()); + + file_name_swb_ = webrtc::test::ResourcePath("audio_coding/testfile32kHz", + "pcm"); + + EXPECT_EQ(0, _acmB->RegisterSendCodec(_paramISAC16kHz)); + EXPECT_EQ(0, _acmA->RegisterSendCodec(_paramISAC32kHz)); + + _inFileA.Open(file_name_swb_, 32000, "rb"); + // Set test length to 500 ms (50 blocks of 10 ms each). + _inFileA.SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the files start with silence. + _inFileA.FastForward(100); + std::string fileNameA = webrtc::test::OutputPath() + "testisac_a.pcm"; + std::string fileNameB = webrtc::test::OutputPath() + "testisac_b.pcm"; + _outFileA.Open(fileNameA, 32000, "wb"); + _outFileB.Open(fileNameB, 32000, "wb"); + + while (!_inFileA.EndOfFile()) { + Run10ms(); + } + CodecInst receiveCodec; + EXPECT_EQ(0, _acmA->ReceiveCodec(&receiveCodec)); + EXPECT_EQ(0, _acmB->ReceiveCodec(&receiveCodec)); + + _inFileA.Close(); + _outFileA.Close(); + _outFileB.Close(); +} + +void ISACTest::Perform() { + Setup(); + + int16_t testNr = 0; + ACMTestISACConfig wbISACConfig; + ACMTestISACConfig swbISACConfig; + + SetISACConfigDefault(wbISACConfig); + SetISACConfigDefault(swbISACConfig); + + wbISACConfig.currentRateBitPerSec = -1; + swbISACConfig.currentRateBitPerSec = -1; + testNr++; + EncodeDecode(testNr, wbISACConfig, swbISACConfig); + + if (_testMode != 0) { + SetISACConfigDefault(wbISACConfig); + SetISACConfigDefault(swbISACConfig); + + wbISACConfig.currentRateBitPerSec = -1; + swbISACConfig.currentRateBitPerSec = -1; + wbISACConfig.initRateBitPerSec = 13000; + wbISACConfig.initFrameSizeInMsec = 60; + swbISACConfig.initRateBitPerSec = 20000; + swbISACConfig.initFrameSizeInMsec = 30; + testNr++; + EncodeDecode(testNr, wbISACConfig, swbISACConfig); + + SetISACConfigDefault(wbISACConfig); + SetISACConfigDefault(swbISACConfig); + + wbISACConfig.currentRateBitPerSec = 20000; + swbISACConfig.currentRateBitPerSec = 48000; + testNr++; + EncodeDecode(testNr, wbISACConfig, swbISACConfig); + + wbISACConfig.currentRateBitPerSec = 16000; + swbISACConfig.currentRateBitPerSec = 30000; + wbISACConfig.currentFrameSizeMsec = 60; + testNr++; + EncodeDecode(testNr, wbISACConfig, swbISACConfig); + } + + SetISACConfigDefault(wbISACConfig); + SetISACConfigDefault(swbISACConfig); + testNr++; + EncodeDecode(testNr, wbISACConfig, swbISACConfig); + + testNr++; + if (_testMode == 0) { + SwitchingSamplingRate(testNr, 4); + } else { + SwitchingSamplingRate(testNr, 80); + } +} + +void ISACTest::Run10ms() { + AudioFrame audioFrame; + EXPECT_GT(_inFileA.Read10MsData(audioFrame), 0); + EXPECT_GE(_acmA->Add10MsData(audioFrame), 0); + EXPECT_GE(_acmB->Add10MsData(audioFrame), 0); + bool muted; + EXPECT_EQ(0, _acmA->PlayoutData10Ms(32000, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileA.Write10MsData(audioFrame); + EXPECT_EQ(0, _acmB->PlayoutData10Ms(32000, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileB.Write10MsData(audioFrame); +} + +void ISACTest::EncodeDecode(int testNr, ACMTestISACConfig& wbISACConfig, + ACMTestISACConfig& swbISACConfig) { + // Files in Side A and B + _inFileA.Open(file_name_swb_, 32000, "rb", true); + _inFileB.Open(file_name_swb_, 32000, "rb", true); + + std::string file_name_out; + std::stringstream file_stream_a; + std::stringstream file_stream_b; + file_stream_a << webrtc::test::OutputPath(); + file_stream_b << webrtc::test::OutputPath(); + file_stream_a << "out_iSACTest_A_" << testNr << ".pcm"; + file_stream_b << "out_iSACTest_B_" << testNr << ".pcm"; + file_name_out = file_stream_a.str(); + _outFileA.Open(file_name_out, 32000, "wb"); + file_name_out = file_stream_b.str(); + _outFileB.Open(file_name_out, 32000, "wb"); + + EXPECT_EQ(0, _acmA->RegisterSendCodec(_paramISAC16kHz)); + EXPECT_EQ(0, _acmA->RegisterSendCodec(_paramISAC32kHz)); + EXPECT_EQ(0, _acmB->RegisterSendCodec(_paramISAC32kHz)); + EXPECT_EQ(0, _acmB->RegisterSendCodec(_paramISAC16kHz)); + + // Side A is sending super-wideband, and side B is sending wideband. + SetISAConfig(swbISACConfig, _acmA.get(), _testMode); + SetISAConfig(wbISACConfig, _acmB.get(), _testMode); + + bool adaptiveMode = false; + if ((swbISACConfig.currentRateBitPerSec == -1) + || (wbISACConfig.currentRateBitPerSec == -1)) { + adaptiveMode = true; + } + _myTimer.Reset(); + _channel_A2B->ResetStats(); + _channel_B2A->ResetStats(); + + char currentTime[500]; + EventTimerWrapper* myEvent = EventTimerWrapper::Create(); + EXPECT_TRUE(myEvent->StartTimer(true, 10)); + while (!(_inFileA.EndOfFile() || _inFileA.Rewinded())) { + Run10ms(); + _myTimer.Tick10ms(); + _myTimer.CurrentTimeHMS(currentTime); + + if ((adaptiveMode) && (_testMode != 0)) { + myEvent->Wait(5000); + EXPECT_TRUE(_acmA->SendCodec()); + EXPECT_TRUE(_acmB->SendCodec()); + } + } + + if (_testMode != 0) { + printf("\n\nSide A statistics\n\n"); + _channel_A2B->PrintStats(_paramISAC32kHz); + + printf("\n\nSide B statistics\n\n"); + _channel_B2A->PrintStats(_paramISAC16kHz); + } + + _channel_A2B->ResetStats(); + _channel_B2A->ResetStats(); + + _outFileA.Close(); + _outFileB.Close(); + _inFileA.Close(); + _inFileB.Close(); +} + +void ISACTest::SwitchingSamplingRate(int testNr, int maxSampRateChange) { + // Files in Side A + _inFileA.Open(file_name_swb_, 32000, "rb"); + _inFileB.Open(file_name_swb_, 32000, "rb"); + + std::string file_name_out; + std::stringstream file_stream_a; + std::stringstream file_stream_b; + file_stream_a << webrtc::test::OutputPath(); + file_stream_b << webrtc::test::OutputPath(); + file_stream_a << "out_iSACTest_A_" << testNr << ".pcm"; + file_stream_b << "out_iSACTest_B_" << testNr << ".pcm"; + file_name_out = file_stream_a.str(); + _outFileA.Open(file_name_out, 32000, "wb"); + file_name_out = file_stream_b.str(); + _outFileB.Open(file_name_out, 32000, "wb"); + + // Start with side A sending super-wideband and side B seding wideband. + // Toggle sending wideband/super-wideband in this test. + EXPECT_EQ(0, _acmA->RegisterSendCodec(_paramISAC32kHz)); + EXPECT_EQ(0, _acmB->RegisterSendCodec(_paramISAC16kHz)); + + int numSendCodecChanged = 0; + _myTimer.Reset(); + char currentTime[50]; + while (numSendCodecChanged < (maxSampRateChange << 1)) { + Run10ms(); + _myTimer.Tick10ms(); + _myTimer.CurrentTimeHMS(currentTime); + if (_testMode == 2) + printf("\r%s", currentTime); + if (_inFileA.EndOfFile()) { + if (_inFileA.SamplingFrequency() == 16000) { + // Switch side A to send super-wideband. + _inFileA.Close(); + _inFileA.Open(file_name_swb_, 32000, "rb"); + EXPECT_EQ(0, _acmA->RegisterSendCodec(_paramISAC32kHz)); + } else { + // Switch side A to send wideband. + _inFileA.Close(); + _inFileA.Open(file_name_swb_, 32000, "rb"); + EXPECT_EQ(0, _acmA->RegisterSendCodec(_paramISAC16kHz)); + } + numSendCodecChanged++; + } + + if (_inFileB.EndOfFile()) { + if (_inFileB.SamplingFrequency() == 16000) { + // Switch side B to send super-wideband. + _inFileB.Close(); + _inFileB.Open(file_name_swb_, 32000, "rb"); + EXPECT_EQ(0, _acmB->RegisterSendCodec(_paramISAC32kHz)); + } else { + // Switch side B to send wideband. + _inFileB.Close(); + _inFileB.Open(file_name_swb_, 32000, "rb"); + EXPECT_EQ(0, _acmB->RegisterSendCodec(_paramISAC16kHz)); + } + numSendCodecChanged++; + } + } + _outFileA.Close(); + _outFileB.Close(); + _inFileA.Close(); + _inFileB.Close(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.h new file mode 100644 index 0000000000..d0e7d5954f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/iSACTest.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_ISACTEST_H_ +#define MODULES_AUDIO_CODING_TEST_ISACTEST_H_ + +#include + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/utility.h" + +#define MAX_FILE_NAME_LENGTH_BYTE 500 +#define NO_OF_CLIENTS 15 + +namespace webrtc { + +struct ACMTestISACConfig { + int32_t currentRateBitPerSec; + int16_t currentFrameSizeMsec; + int16_t encodingMode; + uint32_t initRateBitPerSec; + int16_t initFrameSizeInMsec; + bool enforceFrameSize; +}; + +class ISACTest : public ACMTest { + public: + explicit ISACTest(int testMode); + ~ISACTest(); + + void Perform(); + private: + void Setup(); + + void Run10ms(); + + void EncodeDecode(int testNr, ACMTestISACConfig& wbISACConfig, + ACMTestISACConfig& swbISACConfig); + + void SwitchingSamplingRate(int testNr, int maxSampRateChange); + + std::unique_ptr _acmA; + std::unique_ptr _acmB; + + std::unique_ptr _channel_A2B; + std::unique_ptr _channel_B2A; + + PCMFile _inFileA; + PCMFile _inFileB; + + PCMFile _outFileA; + PCMFile _outFileB; + + uint8_t _idISAC16kHz; + uint8_t _idISAC32kHz; + CodecInst _paramISAC16kHz; + CodecInst _paramISAC32kHz; + + std::string file_name_swb_; + + ACMTestTimer _myTimer; + int _testMode; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_ISACTEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/insert_packet_with_timing.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/insert_packet_with_timing.cc new file mode 100644 index 0000000000..2c0e54b0cb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/insert_packet_with_timing.cc @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/flags.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +// Codec. +DEFINE_string(codec, "opus", "Codec Name"); +DEFINE_int(codec_sample_rate_hz, 48000, "Sampling rate in Hertz."); +DEFINE_int(codec_channels, 1, "Number of channels of the codec."); + +// PCM input/output. +DEFINE_string(input, "", "Input PCM file at 16 kHz."); +DEFINE_bool(input_stereo, false, "Input is stereo."); +DEFINE_int(input_fs_hz, 32000, "Input sample rate Hz."); +DEFINE_string(output, "insert_rtp_with_timing_out.pcm", "OutputFile"); +DEFINE_int(output_fs_hz, 32000, "Output sample rate Hz"); + +// Timing files +DEFINE_string(seq_num, "seq_num", "Sequence number file."); +DEFINE_string(send_ts, "send_timestamp", "Send timestamp file."); +DEFINE_string(receive_ts, "last_rec_timestamp", "Receive timestamp file"); + +// Delay logging +DEFINE_string(delay, "", "Log for delay."); + +// Other setups +DEFINE_bool(verbose, false, "Verbosity."); +DEFINE_float(loss_rate, 0, "Rate of packet loss < 1"); + +DEFINE_bool(help, false, "Prints this message."); + +const int32_t kAudioPlayedOut = 0x00000001; +const int32_t kPacketPushedIn = 0x00000001 << 1; +const int kPlayoutPeriodMs = 10; + +namespace webrtc { + +class InsertPacketWithTiming { + public: + InsertPacketWithTiming() + : sender_clock_(new SimulatedClock(0)), + receiver_clock_(new SimulatedClock(0)), + send_acm_(AudioCodingModule::Create(sender_clock_)), + receive_acm_(AudioCodingModule::Create(receiver_clock_)), + channel_(new Channel), + seq_num_fid_(fopen(FLAG_seq_num, "rt")), + send_ts_fid_(fopen(FLAG_send_ts, "rt")), + receive_ts_fid_(fopen(FLAG_receive_ts, "rt")), + pcm_out_fid_(fopen(FLAG_output, "wb")), + samples_in_1ms_(48), + num_10ms_in_codec_frame_(2), // Typical 20 ms frames. + time_to_insert_packet_ms_(3), // An arbitrary offset on pushing packet. + next_receive_ts_(0), + time_to_playout_audio_ms_(kPlayoutPeriodMs), + loss_threshold_(0), + playout_timing_fid_(fopen("playout_timing.txt", "wt")) {} + + void SetUp() { + ASSERT_TRUE(sender_clock_ != NULL); + ASSERT_TRUE(receiver_clock_ != NULL); + + ASSERT_TRUE(send_acm_.get() != NULL); + ASSERT_TRUE(receive_acm_.get() != NULL); + ASSERT_TRUE(channel_ != NULL); + + ASSERT_TRUE(seq_num_fid_ != NULL); + ASSERT_TRUE(send_ts_fid_ != NULL); + ASSERT_TRUE(receive_ts_fid_ != NULL); + + ASSERT_TRUE(playout_timing_fid_ != NULL); + + next_receive_ts_ = ReceiveTimestamp(); + + CodecInst codec; + ASSERT_EQ(0, AudioCodingModule::Codec(FLAG_codec, &codec, + FLAG_codec_sample_rate_hz, + FLAG_codec_channels)); + ASSERT_EQ(0, receive_acm_->InitializeReceiver()); + ASSERT_EQ(0, send_acm_->RegisterSendCodec(codec)); + ASSERT_EQ(true, receive_acm_->RegisterReceiveCodec(codec.pltype, + CodecInstToSdp(codec))); + + // Set codec-dependent parameters. + samples_in_1ms_ = codec.plfreq / 1000; + num_10ms_in_codec_frame_ = codec.pacsize / (codec.plfreq / 100); + + channel_->RegisterReceiverACM(receive_acm_.get()); + send_acm_->RegisterTransportCallback(channel_); + + if (strlen(FLAG_input) == 0) { + std::string file_name = test::ResourcePath("audio_coding/testfile32kHz", + "pcm"); + pcm_in_fid_.Open(file_name, 32000, "r", true); // auto-rewind + std::cout << "Input file " << file_name << " 32 kHz mono." << std::endl; + } else { + pcm_in_fid_.Open(FLAG_input, static_cast(FLAG_input_fs_hz), + "r", true); // auto-rewind + std::cout << "Input file " << FLAG_input << "at " << FLAG_input_fs_hz + << " Hz in " << ((FLAG_input_stereo) ? "stereo." : "mono.") + << std::endl; + pcm_in_fid_.ReadStereo(FLAG_input_stereo); + } + + ASSERT_TRUE(pcm_out_fid_ != NULL); + std::cout << "Output file " << FLAG_output << " at " << FLAG_output_fs_hz + << " Hz." << std::endl; + + // Other setups + if (FLAG_loss_rate > 0) + loss_threshold_ = RAND_MAX * FLAG_loss_rate; + else + loss_threshold_ = 0; + } + + void TickOneMillisecond(uint32_t* action) { + // One millisecond passed. + time_to_insert_packet_ms_--; + time_to_playout_audio_ms_--; + sender_clock_->AdvanceTimeMilliseconds(1); + receiver_clock_->AdvanceTimeMilliseconds(1); + + // Reset action. + *action = 0; + + // Is it time to pull audio? + if (time_to_playout_audio_ms_ == 0) { + time_to_playout_audio_ms_ = kPlayoutPeriodMs; + bool muted; + receive_acm_->PlayoutData10Ms(static_cast(FLAG_output_fs_hz), + &frame_, &muted); + ASSERT_FALSE(muted); + fwrite(frame_.data(), sizeof(*frame_.data()), + frame_.samples_per_channel_ * frame_.num_channels_, pcm_out_fid_); + *action |= kAudioPlayedOut; + } + + // Is it time to push in next packet? + if (time_to_insert_packet_ms_ <= .5) { + *action |= kPacketPushedIn; + + // Update time-to-insert packet. + uint32_t t = next_receive_ts_; + next_receive_ts_ = ReceiveTimestamp(); + time_to_insert_packet_ms_ += static_cast(next_receive_ts_ - t) / + samples_in_1ms_; + + // Push in just enough audio. + for (int n = 0; n < num_10ms_in_codec_frame_; n++) { + pcm_in_fid_.Read10MsData(frame_); + EXPECT_GE(send_acm_->Add10MsData(frame_), 0); + } + + // Set the parameters for the packet to be pushed in receiver ACM right + // now. + uint32_t ts = SendTimestamp(); + int seq_num = SequenceNumber(); + bool lost = false; + channel_->set_send_timestamp(ts); + channel_->set_sequence_number(seq_num); + if (loss_threshold_ > 0 && rand() < loss_threshold_) { + channel_->set_num_packets_to_drop(1); + lost = true; + } + + if (FLAG_verbose) { + if (!lost) { + std::cout << "\nInserting packet number " << seq_num + << " timestamp " << ts << std::endl; + } else { + std::cout << "\nLost packet number " << seq_num + << " timestamp " << ts << std::endl; + } + } + } + } + + void TearDown() { + delete channel_; + + fclose(seq_num_fid_); + fclose(send_ts_fid_); + fclose(receive_ts_fid_); + fclose(pcm_out_fid_); + pcm_in_fid_.Close(); + } + + ~InsertPacketWithTiming() { + delete sender_clock_; + delete receiver_clock_; + } + + // Are there more info to simulate. + bool HasPackets() { + if (feof(seq_num_fid_) || feof(send_ts_fid_) || feof(receive_ts_fid_)) + return false; + return true; + } + + // Jitter buffer delay. + void Delay(int* optimal_delay, int* current_delay) { + NetworkStatistics statistics; + receive_acm_->GetNetworkStatistics(&statistics); + *optimal_delay = statistics.preferredBufferSize; + *current_delay = statistics.currentBufferSize; + } + + private: + uint32_t SendTimestamp() { + uint32_t t; + EXPECT_EQ(1, fscanf(send_ts_fid_, "%u\n", &t)); + return t; + } + + uint32_t ReceiveTimestamp() { + uint32_t t; + EXPECT_EQ(1, fscanf(receive_ts_fid_, "%u\n", &t)); + return t; + } + + int SequenceNumber() { + int n; + EXPECT_EQ(1, fscanf(seq_num_fid_, "%d\n", &n)); + return n; + } + + // This class just creates these pointers, not deleting them. They are deleted + // by the associated ACM. + SimulatedClock* sender_clock_; + SimulatedClock* receiver_clock_; + + std::unique_ptr send_acm_; + std::unique_ptr receive_acm_; + Channel* channel_; + + FILE* seq_num_fid_; // Input (text), one sequence number per line. + FILE* send_ts_fid_; // Input (text), one send timestamp per line. + FILE* receive_ts_fid_; // Input (text), one receive timestamp per line. + FILE* pcm_out_fid_; // Output PCM16. + + PCMFile pcm_in_fid_; // Input PCM16. + + int samples_in_1ms_; + + // TODO(turajs): this can be computed from the send timestamp, but there is + // some complication to account for lost and reordered packets. + int num_10ms_in_codec_frame_; + + float time_to_insert_packet_ms_; + uint32_t next_receive_ts_; + uint32_t time_to_playout_audio_ms_; + + AudioFrame frame_; + + double loss_threshold_; + + // Output (text), sequence number, playout timestamp, time (ms) of playout, + // per line. + FILE* playout_timing_fid_; +}; + +} // webrtc + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true)) { + return 1; + } + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + + webrtc::InsertPacketWithTiming test; + test.SetUp(); + + FILE* delay_log = NULL; + if (strlen(FLAG_delay) > 0) { + delay_log = fopen(FLAG_delay, "wt"); + if (delay_log == NULL) { + std::cout << "Cannot open the file to log delay values." << std::endl; + exit(1); + } + } + + uint32_t action_taken; + int optimal_delay_ms; + int current_delay_ms; + while (test.HasPackets()) { + test.TickOneMillisecond(&action_taken); + + if (action_taken != 0) { + test.Delay(&optimal_delay_ms, ¤t_delay_ms); + if (delay_log != NULL) { + fprintf(delay_log, "%3d %3d\n", optimal_delay_ms, current_delay_ms); + } + } + } + std::cout << std::endl; + test.TearDown(); + if (delay_log != NULL) + fclose(delay_log); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.cc new file mode 100644 index 0000000000..b7acc0f7c6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.cc @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/opus_test.h" + +#include + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/audio_format_conversion.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/TestStereo.h" +#include "modules/audio_coding/test/utility.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +OpusTest::OpusTest() + : acm_receiver_(AudioCodingModule::Create()), + channel_a2b_(NULL), + counter_(0), + payload_type_(255), + rtp_timestamp_(0) {} + +OpusTest::~OpusTest() { + if (channel_a2b_ != NULL) { + delete channel_a2b_; + channel_a2b_ = NULL; + } + if (opus_mono_encoder_ != NULL) { + WebRtcOpus_EncoderFree(opus_mono_encoder_); + opus_mono_encoder_ = NULL; + } + if (opus_stereo_encoder_ != NULL) { + WebRtcOpus_EncoderFree(opus_stereo_encoder_); + opus_stereo_encoder_ = NULL; + } + if (opus_mono_decoder_ != NULL) { + WebRtcOpus_DecoderFree(opus_mono_decoder_); + opus_mono_decoder_ = NULL; + } + if (opus_stereo_decoder_ != NULL) { + WebRtcOpus_DecoderFree(opus_stereo_decoder_); + opus_stereo_decoder_ = NULL; + } +} + +void OpusTest::Perform() { +#ifndef WEBRTC_CODEC_OPUS + // Opus isn't defined, exit. + return; +#else + uint16_t frequency_hz; + size_t audio_channels; + int16_t test_cntr = 0; + + // Open both mono and stereo test files in 32 kHz. + const std::string file_name_stereo = + webrtc::test::ResourcePath("audio_coding/teststereo32kHz", "pcm"); + const std::string file_name_mono = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + frequency_hz = 32000; + in_file_stereo_.Open(file_name_stereo, frequency_hz, "rb"); + in_file_stereo_.ReadStereo(true); + in_file_mono_.Open(file_name_mono, frequency_hz, "rb"); + in_file_mono_.ReadStereo(false); + + // Create Opus encoders for mono and stereo. + ASSERT_GT(WebRtcOpus_EncoderCreate(&opus_mono_encoder_, 1, 0), -1); + ASSERT_GT(WebRtcOpus_EncoderCreate(&opus_stereo_encoder_, 2, 1), -1); + + // Create Opus decoders for mono and stereo for stand-alone testing of Opus. + ASSERT_GT(WebRtcOpus_DecoderCreate(&opus_mono_decoder_, 1), -1); + ASSERT_GT(WebRtcOpus_DecoderCreate(&opus_stereo_decoder_, 2), -1); + WebRtcOpus_DecoderInit(opus_mono_decoder_); + WebRtcOpus_DecoderInit(opus_stereo_decoder_); + + ASSERT_TRUE(acm_receiver_.get() != NULL); + EXPECT_EQ(0, acm_receiver_->InitializeReceiver()); + + // Register Opus stereo as receiving codec. + CodecInst opus_codec_param; + int codec_id = acm_receiver_->Codec("opus", 48000, 2); + EXPECT_EQ(0, acm_receiver_->Codec(codec_id, &opus_codec_param)); + payload_type_ = opus_codec_param.pltype; + EXPECT_EQ(true, + acm_receiver_->RegisterReceiveCodec( + opus_codec_param.pltype, CodecInstToSdp(opus_codec_param))); + + // Create and connect the channel. + channel_a2b_ = new TestPackStereo; + channel_a2b_->RegisterReceiverACM(acm_receiver_.get()); + + // + // Test Stereo. + // + + channel_a2b_->set_codec_mode(kStereo); + audio_channels = 2; + test_cntr++; + OpenOutFile(test_cntr); + + // Run Opus with 2.5 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 120); + + // Run Opus with 5 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 240); + + // Run Opus with 10 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 480); + + // Run Opus with 20 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 960); + + // Run Opus with 40 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 1920); + + // Run Opus with 60 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 2880); + + out_file_.Close(); + out_file_standalone_.Close(); + + // + // Test Opus stereo with packet-losses. + // + + test_cntr++; + OpenOutFile(test_cntr); + + // Run Opus with 20 ms frame size, 1% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 1); + + // Run Opus with 20 ms frame size, 5% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 5); + + // Run Opus with 20 ms frame size, 10% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 10); + + out_file_.Close(); + out_file_standalone_.Close(); + + // + // Test Mono. + // + channel_a2b_->set_codec_mode(kMono); + audio_channels = 1; + test_cntr++; + OpenOutFile(test_cntr); + + // Register Opus mono as receiving codec. + opus_codec_param.channels = 1; + EXPECT_EQ(true, + acm_receiver_->RegisterReceiveCodec( + opus_codec_param.pltype, CodecInstToSdp(opus_codec_param))); + + // Run Opus with 2.5 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 120); + + // Run Opus with 5 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 240); + + // Run Opus with 10 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 480); + + // Run Opus with 20 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 960); + + // Run Opus with 40 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 1920); + + // Run Opus with 60 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 2880); + + out_file_.Close(); + out_file_standalone_.Close(); + + // + // Test Opus mono with packet-losses. + // + test_cntr++; + OpenOutFile(test_cntr); + + // Run Opus with 20 ms frame size, 1% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 1); + + // Run Opus with 20 ms frame size, 5% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 5); + + // Run Opus with 20 ms frame size, 10% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 10); + + // Close the files. + in_file_stereo_.Close(); + in_file_mono_.Close(); + out_file_.Close(); + out_file_standalone_.Close(); +#endif +} + +void OpusTest::Run(TestPackStereo* channel, size_t channels, int bitrate, + size_t frame_length, int percent_loss) { + AudioFrame audio_frame; + int32_t out_freq_hz_b = out_file_.SamplingFrequency(); + const size_t kBufferSizeSamples = 480 * 12 * 2; // 120 ms stereo audio. + int16_t audio[kBufferSizeSamples]; + int16_t out_audio[kBufferSizeSamples]; + int16_t audio_type; + size_t written_samples = 0; + size_t read_samples = 0; + size_t decoded_samples = 0; + bool first_packet = true; + uint32_t start_time_stamp = 0; + + channel->reset_payload_size(); + counter_ = 0; + + // Set encoder rate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_mono_encoder_, bitrate)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_stereo_encoder_, bitrate)); + +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM) + // If we are on Android, iOS and/or ARM, use a lower complexity setting as + // default. + const int kOpusComplexity5 = 5; + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_mono_encoder_, kOpusComplexity5)); + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_stereo_encoder_, + kOpusComplexity5)); +#endif + + // Fast-forward 1 second (100 blocks) since the files start with silence. + in_file_stereo_.FastForward(100); + in_file_mono_.FastForward(100); + + // Limit the runtime to 1000 blocks of 10 ms each. + for (size_t audio_length = 0; audio_length < 1000; audio_length += 10) { + bool lost_packet = false; + + // Get 10 msec of audio. + if (channels == 1) { + if (in_file_mono_.EndOfFile()) { + break; + } + in_file_mono_.Read10MsData(audio_frame); + } else { + if (in_file_stereo_.EndOfFile()) { + break; + } + in_file_stereo_.Read10MsData(audio_frame); + } + + // If input audio is sampled at 32 kHz, resampling to 48 kHz is required. + EXPECT_EQ(480, + resampler_.Resample10Msec(audio_frame.data(), + audio_frame.sample_rate_hz_, + 48000, + channels, + kBufferSizeSamples - written_samples, + &audio[written_samples])); + written_samples += 480 * channels; + + // Sometimes we need to loop over the audio vector to produce the right + // number of packets. + size_t loop_encode = (written_samples - read_samples) / + (channels * frame_length); + + if (loop_encode > 0) { + const size_t kMaxBytes = 1000; // Maximum number of bytes for one packet. + size_t bitstream_len_byte; + uint8_t bitstream[kMaxBytes]; + for (size_t i = 0; i < loop_encode; i++) { + int bitstream_len_byte_int = WebRtcOpus_Encode( + (channels == 1) ? opus_mono_encoder_ : opus_stereo_encoder_, + &audio[read_samples], frame_length, kMaxBytes, bitstream); + ASSERT_GE(bitstream_len_byte_int, 0); + bitstream_len_byte = static_cast(bitstream_len_byte_int); + + // Simulate packet loss by setting |packet_loss_| to "true" in + // |percent_loss| percent of the loops. + // TODO(tlegrand): Move handling of loss simulation to TestPackStereo. + if (percent_loss > 0) { + if (counter_ == floor((100 / percent_loss) + 0.5)) { + counter_ = 0; + lost_packet = true; + channel->set_lost_packet(true); + } else { + lost_packet = false; + channel->set_lost_packet(false); + } + counter_++; + } + + // Run stand-alone Opus decoder, or decode PLC. + if (channels == 1) { + if (!lost_packet) { + decoded_samples += WebRtcOpus_Decode( + opus_mono_decoder_, bitstream, bitstream_len_byte, + &out_audio[decoded_samples * channels], &audio_type); + } else { + decoded_samples += WebRtcOpus_DecodePlc( + opus_mono_decoder_, &out_audio[decoded_samples * channels], 1); + } + } else { + if (!lost_packet) { + decoded_samples += WebRtcOpus_Decode( + opus_stereo_decoder_, bitstream, bitstream_len_byte, + &out_audio[decoded_samples * channels], &audio_type); + } else { + decoded_samples += WebRtcOpus_DecodePlc( + opus_stereo_decoder_, &out_audio[decoded_samples * channels], + 1); + } + } + + // Send data to the channel. "channel" will handle the loss simulation. + channel->SendData(kAudioFrameSpeech, payload_type_, rtp_timestamp_, + bitstream, bitstream_len_byte, NULL); + if (first_packet) { + first_packet = false; + start_time_stamp = rtp_timestamp_; + } + rtp_timestamp_ += static_cast(frame_length); + read_samples += frame_length * channels; + } + if (read_samples == written_samples) { + read_samples = 0; + written_samples = 0; + } + } + + // Run received side of ACM. + bool muted; + ASSERT_EQ( + 0, acm_receiver_->PlayoutData10Ms(out_freq_hz_b, &audio_frame, &muted)); + ASSERT_FALSE(muted); + + // Write output speech to file. + out_file_.Write10MsData( + audio_frame.data(), + audio_frame.samples_per_channel_ * audio_frame.num_channels_); + + // Write stand-alone speech to file. + out_file_standalone_.Write10MsData(out_audio, decoded_samples * channels); + + if (audio_frame.timestamp_ > start_time_stamp) { + // Number of channels should be the same for both stand-alone and + // ACM-decoding. + EXPECT_EQ(audio_frame.num_channels_, channels); + } + + decoded_samples = 0; + } + + if (in_file_mono_.EndOfFile()) { + in_file_mono_.Rewind(); + } + if (in_file_stereo_.EndOfFile()) { + in_file_stereo_.Rewind(); + } + // Reset in case we ended with a lost packet. + channel->set_lost_packet(false); +} + +void OpusTest::OpenOutFile(int test_number) { + std::string file_name; + std::stringstream file_stream; + file_stream << webrtc::test::OutputPath() << "opustest_out_" + << test_number << ".pcm"; + file_name = file_stream.str(); + out_file_.Open(file_name, 48000, "wb"); + file_stream.str(""); + file_name = file_stream.str(); + file_stream << webrtc::test::OutputPath() << "opusstandalone_out_" + << test_number << ".pcm"; + file_name = file_stream.str(); + out_file_standalone_.Open(file_name, 48000, "wb"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.h new file mode 100644 index 0000000000..3e9d9a7187 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/opus_test.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_OPUS_TEST_H_ +#define MODULES_AUDIO_CODING_TEST_OPUS_TEST_H_ + +#include + +#include + +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/audio_coding/test/ACMTest.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/TestStereo.h" + +namespace webrtc { + +class OpusTest : public ACMTest { + public: + OpusTest(); + ~OpusTest(); + + void Perform(); + + private: + void Run(TestPackStereo* channel, + size_t channels, + int bitrate, + size_t frame_length, + int percent_loss = 0); + + void OpenOutFile(int test_number); + + std::unique_ptr acm_receiver_; + TestPackStereo* channel_a2b_; + PCMFile in_file_stereo_; + PCMFile in_file_mono_; + PCMFile out_file_; + PCMFile out_file_standalone_; + int counter_; + uint8_t payload_type_; + uint32_t rtp_timestamp_; + acm2::ACMResampler resampler_; + WebRtcOpusEncInst* opus_mono_encoder_; + WebRtcOpusEncInst* opus_stereo_encoder_; + WebRtcOpusDecInst* opus_mono_decoder_; + WebRtcOpusDecInst* opus_stereo_decoder_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_OPUS_TEST_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/target_delay_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/target_delay_unittest.cc new file mode 100644 index 0000000000..03135da6c4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/target_delay_unittest.cc @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/utility.h" +#include "modules/include/module_common_types.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +class TargetDelayTest : public ::testing::Test { + protected: + TargetDelayTest() : acm_(AudioCodingModule::Create()) {} + + ~TargetDelayTest() {} + + void SetUp() { + EXPECT_TRUE(acm_.get() != NULL); + + ASSERT_EQ(0, acm_->InitializeReceiver()); + constexpr int pltype = 108; + ASSERT_EQ(true, + acm_->RegisterReceiveCodec(pltype, {"L16", kSampleRateHz, 1})); + + rtp_info_.header.payloadType = pltype; + rtp_info_.header.timestamp = 0; + rtp_info_.header.ssrc = 0x12345678; + rtp_info_.header.markerBit = false; + rtp_info_.header.sequenceNumber = 0; + rtp_info_.type.Audio.channel = 1; + rtp_info_.type.Audio.isCNG = false; + rtp_info_.frameType = kAudioFrameSpeech; + + int16_t audio[kFrameSizeSamples]; + const int kRange = 0x7FF; // 2047, easy for masking. + for (size_t n = 0; n < kFrameSizeSamples; ++n) + audio[n] = (rand() & kRange) - kRange / 2; + WebRtcPcm16b_Encode(audio, kFrameSizeSamples, payload_); + } + + void OutOfRangeInput() { + EXPECT_EQ(-1, SetMinimumDelay(-1)); + EXPECT_EQ(-1, SetMinimumDelay(10001)); + } + + void NoTargetDelayBufferSizeChanges() { + for (int n = 0; n < 30; ++n) // Run enough iterations. + Run(true); + int clean_optimal_delay = GetCurrentOptimalDelayMs(); + Run(false); // Run with jitter. + int jittery_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_GT(jittery_optimal_delay, clean_optimal_delay); + int required_delay = RequiredDelay(); + EXPECT_GT(required_delay, 0); + EXPECT_NEAR(required_delay, jittery_optimal_delay, 1); + } + + void WithTargetDelayBufferNotChanging() { + // A target delay that is one packet larger than jitter. + const int kTargetDelayMs = (kInterarrivalJitterPacket + 1) * + kNum10msPerFrame * 10; + ASSERT_EQ(0, SetMinimumDelay(kTargetDelayMs)); + for (int n = 0; n < 30; ++n) // Run enough iterations to fill the buffer. + Run(true); + int clean_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_EQ(kTargetDelayMs, clean_optimal_delay); + Run(false); // Run with jitter. + int jittery_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_EQ(jittery_optimal_delay, clean_optimal_delay); + } + + void RequiredDelayAtCorrectRange() { + for (int n = 0; n < 30; ++n) // Run clean and store delay. + Run(true); + int clean_optimal_delay = GetCurrentOptimalDelayMs(); + + // A relatively large delay. + const int kTargetDelayMs = (kInterarrivalJitterPacket + 10) * + kNum10msPerFrame * 10; + ASSERT_EQ(0, SetMinimumDelay(kTargetDelayMs)); + for (int n = 0; n < 300; ++n) // Run enough iterations to fill the buffer. + Run(true); + Run(false); // Run with jitter. + + int jittery_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_EQ(kTargetDelayMs, jittery_optimal_delay); + + int required_delay = RequiredDelay(); + + // Checking |required_delay| is in correct range. + EXPECT_GT(required_delay, 0); + EXPECT_GT(jittery_optimal_delay, required_delay); + EXPECT_GT(required_delay, clean_optimal_delay); + + // A tighter check for the value of |required_delay|. + // The jitter forces a delay of + // |kInterarrivalJitterPacket * kNum10msPerFrame * 10| milliseconds. So we + // expect |required_delay| be close to that. + EXPECT_NEAR(kInterarrivalJitterPacket * kNum10msPerFrame * 10, + required_delay, 1); + } + + void TargetDelayBufferMinMax() { + const int kTargetMinDelayMs = kNum10msPerFrame * 10; + ASSERT_EQ(0, SetMinimumDelay(kTargetMinDelayMs)); + for (int m = 0; m < 30; ++m) // Run enough iterations to fill the buffer. + Run(true); + int clean_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_EQ(kTargetMinDelayMs, clean_optimal_delay); + + const int kTargetMaxDelayMs = 2 * (kNum10msPerFrame * 10); + ASSERT_EQ(0, SetMaximumDelay(kTargetMaxDelayMs)); + for (int n = 0; n < 30; ++n) // Run enough iterations to fill the buffer. + Run(false); + + int capped_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_EQ(kTargetMaxDelayMs, capped_optimal_delay); + } + + private: + static const int kSampleRateHz = 16000; + static const int kNum10msPerFrame = 2; + static const size_t kFrameSizeSamples = 320; // 20 ms @ 16 kHz. + // payload-len = frame-samples * 2 bytes/sample. + static const int kPayloadLenBytes = 320 * 2; + // Inter-arrival time in number of packets in a jittery channel. One is no + // jitter. + static const int kInterarrivalJitterPacket = 2; + + void Push() { + rtp_info_.header.timestamp += kFrameSizeSamples; + rtp_info_.header.sequenceNumber++; + ASSERT_EQ(0, acm_->IncomingPacket(payload_, kFrameSizeSamples * 2, + rtp_info_)); + } + + // Pull audio equivalent to the amount of audio in one RTP packet. + void Pull() { + AudioFrame frame; + bool muted; + for (int k = 0; k < kNum10msPerFrame; ++k) { // Pull one frame. + ASSERT_EQ(0, acm_->PlayoutData10Ms(-1, &frame, &muted)); + ASSERT_FALSE(muted); + // Had to use ASSERT_TRUE, ASSERT_EQ generated error. + ASSERT_TRUE(kSampleRateHz == frame.sample_rate_hz_); + ASSERT_EQ(1u, frame.num_channels_); + ASSERT_TRUE(kSampleRateHz / 100 == frame.samples_per_channel_); + } + } + + void Run(bool clean) { + for (int n = 0; n < 10; ++n) { + for (int m = 0; m < 5; ++m) { + Push(); + Pull(); + } + + if (!clean) { + for (int m = 0; m < 10; ++m) { // Long enough to trigger delay change. + Push(); + for (int n = 0; n < kInterarrivalJitterPacket; ++n) + Pull(); + } + } + } + } + + int SetMinimumDelay(int delay_ms) { + return acm_->SetMinimumPlayoutDelay(delay_ms); + } + + int SetMaximumDelay(int delay_ms) { + return acm_->SetMaximumPlayoutDelay(delay_ms); + } + + int GetCurrentOptimalDelayMs() { + NetworkStatistics stats; + acm_->GetNetworkStatistics(&stats); + return stats.preferredBufferSize; + } + + int RequiredDelay() { + return acm_->LeastRequiredDelayMs(); + } + + std::unique_ptr acm_; + WebRtcRTPHeader rtp_info_; + uint8_t payload_[kPayloadLenBytes]; +}; + +// Flaky on iOS: webrtc:7057. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_OutOfRangeInput DISABLED_OutOfRangeInput +#else +#define MAYBE_OutOfRangeInput OutOfRangeInput +#endif +TEST_F(TargetDelayTest, MAYBE_OutOfRangeInput) { + OutOfRangeInput(); +} + +// Flaky on iOS: webrtc:7057. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_NoTargetDelayBufferSizeChanges \ + DISABLED_NoTargetDelayBufferSizeChanges +#else +#define MAYBE_NoTargetDelayBufferSizeChanges NoTargetDelayBufferSizeChanges +#endif +TEST_F(TargetDelayTest, MAYBE_NoTargetDelayBufferSizeChanges) { + NoTargetDelayBufferSizeChanges(); +} + +// Flaky on iOS: webrtc:7057. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_WithTargetDelayBufferNotChanging \ + DISABLED_WithTargetDelayBufferNotChanging +#else +#define MAYBE_WithTargetDelayBufferNotChanging WithTargetDelayBufferNotChanging +#endif +TEST_F(TargetDelayTest, MAYBE_WithTargetDelayBufferNotChanging) { + WithTargetDelayBufferNotChanging(); +} + +// Flaky on iOS: webrtc:7057. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_RequiredDelayAtCorrectRange DISABLED_RequiredDelayAtCorrectRange +#else +#define MAYBE_RequiredDelayAtCorrectRange RequiredDelayAtCorrectRange +#endif +TEST_F(TargetDelayTest, MAYBE_RequiredDelayAtCorrectRange) { + RequiredDelayAtCorrectRange(); +} + +// Flaky on iOS: webrtc:7057. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_TargetDelayBufferMinMax DISABLED_TargetDelayBufferMinMax +#else +#define MAYBE_TargetDelayBufferMinMax TargetDelayBufferMinMax +#endif +TEST_F(TargetDelayTest, MAYBE_TargetDelayBufferMinMax) { + TargetDelayBufferMinMax(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.cc b/third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.cc new file mode 100644 index 0000000000..3c64620d3f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.cc @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "utility.h" + +#include +#include +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_coding/include/audio_coding_module.h" +#include "test/gtest.h" + +#define NUM_CODECS_WITH_FIXED_PAYLOAD_TYPE 13 + +namespace webrtc { + +ACMTestTimer::ACMTestTimer() + : _msec(0), + _sec(0), + _min(0), + _hour(0) { + return; +} + +ACMTestTimer::~ACMTestTimer() { + return; +} + +void ACMTestTimer::Reset() { + _msec = 0; + _sec = 0; + _min = 0; + _hour = 0; + return; +} +void ACMTestTimer::Tick10ms() { + _msec += 10; + Adjust(); + return; +} + +void ACMTestTimer::Tick1ms() { + _msec++; + Adjust(); + return; +} + +void ACMTestTimer::Tick100ms() { + _msec += 100; + Adjust(); + return; +} + +void ACMTestTimer::Tick1sec() { + _sec++; + Adjust(); + return; +} + +void ACMTestTimer::CurrentTimeHMS(char* currTime) { + sprintf(currTime, "%4lu:%02u:%06.3f", _hour, _min, + (double) _sec + (double) _msec / 1000.); + return; +} + +void ACMTestTimer::CurrentTime(unsigned long& h, unsigned char& m, + unsigned char& s, unsigned short& ms) { + h = _hour; + m = _min; + s = _sec; + ms = _msec; + return; +} + +void ACMTestTimer::Adjust() { + unsigned int n; + if (_msec >= 1000) { + n = _msec / 1000; + _msec -= (1000 * n); + _sec += n; + } + if (_sec >= 60) { + n = _sec / 60; + _sec -= (n * 60); + _min += n; + } + if (_min >= 60) { + n = _min / 60; + _min -= (n * 60); + _hour += n; + } +} + +int16_t ChooseCodec(CodecInst& codecInst) { + + PrintCodecs(); + //AudioCodingModule* tmpACM = AudioCodingModule::Create(0); + uint8_t noCodec = AudioCodingModule::NumberOfCodecs(); + int8_t codecID; + bool outOfRange = false; + char myStr[15] = ""; + do { + printf("\nChoose a codec [0]: "); + EXPECT_TRUE(fgets(myStr, 10, stdin) != NULL); + codecID = atoi(myStr); + if ((codecID < 0) || (codecID >= noCodec)) { + printf("\nOut of range.\n"); + outOfRange = true; + } + } while (outOfRange); + + CHECK_ERROR(AudioCodingModule::Codec((uint8_t )codecID, &codecInst)); + return 0; +} + +void PrintCodecs() { + uint8_t noCodec = AudioCodingModule::NumberOfCodecs(); + + CodecInst codecInst; + printf("No Name [Hz] [bps]\n"); + for (uint8_t codecCntr = 0; codecCntr < noCodec; codecCntr++) { + AudioCodingModule::Codec(codecCntr, &codecInst); + printf("%2d- %-18s %5d %6d\n", codecCntr, codecInst.plname, + codecInst.plfreq, codecInst.rate); + } + +} + +namespace test { + +CircularBuffer::CircularBuffer(uint32_t len) + : _buff(NULL), + _idx(0), + _buffIsFull(false), + _calcAvg(false), + _calcVar(false), + _sum(0), + _sumSqr(0) { + _buff = new double[len]; + if (_buff == NULL) { + _buffLen = 0; + } else { + for (uint32_t n = 0; n < len; n++) { + _buff[n] = 0; + } + _buffLen = len; + } +} + +CircularBuffer::~CircularBuffer() { + if (_buff != NULL) { + delete[] _buff; + _buff = NULL; + } +} + +void CircularBuffer::Update(const double newVal) { + assert(_buffLen > 0); + + // store the value that is going to be overwritten + double oldVal = _buff[_idx]; + // record the new value + _buff[_idx] = newVal; + // increment the index, to point to where we would + // write next + _idx++; + // it is a circular buffer, if we are at the end + // we have to cycle to the beginning + if (_idx >= _buffLen) { + // flag that the buffer is filled up. + _buffIsFull = true; + _idx = 0; + } + + // Update + + if (_calcAvg) { + // for the average we have to update + // the sum + _sum += (newVal - oldVal); + } + + if (_calcVar) { + // to calculate variance we have to update + // the sum of squares + _sumSqr += (double) (newVal - oldVal) * (double) (newVal + oldVal); + } +} + +void CircularBuffer::SetArithMean(bool enable) { + assert(_buffLen > 0); + + if (enable && !_calcAvg) { + uint32_t lim; + if (_buffIsFull) { + lim = _buffLen; + } else { + lim = _idx; + } + _sum = 0; + for (uint32_t n = 0; n < lim; n++) { + _sum += _buff[n]; + } + } + _calcAvg = enable; +} + +void CircularBuffer::SetVariance(bool enable) { + assert(_buffLen > 0); + + if (enable && !_calcVar) { + uint32_t lim; + if (_buffIsFull) { + lim = _buffLen; + } else { + lim = _idx; + } + _sumSqr = 0; + for (uint32_t n = 0; n < lim; n++) { + _sumSqr += _buff[n] * _buff[n]; + } + } + _calcAvg = enable; +} + +int16_t CircularBuffer::ArithMean(double& mean) { + assert(_buffLen > 0); + + if (_buffIsFull) { + + mean = _sum / (double) _buffLen; + return 0; + } else { + if (_idx > 0) { + mean = _sum / (double) _idx; + return 0; + } else { + return -1; + } + + } +} + +int16_t CircularBuffer::Variance(double& var) { + assert(_buffLen > 0); + + if (_buffIsFull) { + var = _sumSqr / (double) _buffLen; + return 0; + } else { + if (_idx > 0) { + var = _sumSqr / (double) _idx; + return 0; + } else { + return -1; + } + } +} + +} // namespace test + +bool FixedPayloadTypeCodec(const char* payloadName) { + char fixPayloadTypeCodecs[NUM_CODECS_WITH_FIXED_PAYLOAD_TYPE][32] = { "PCMU", + "PCMA", "GSM", "G723", "DVI4", "LPC", "PCMA", "G722", "QCELP", "CN", + "MPA", "G728", "G729" }; + + for (int n = 0; n < NUM_CODECS_WITH_FIXED_PAYLOAD_TYPE; n++) { + if (!STR_CASE_CMP(payloadName, fixPayloadTypeCodecs[n])) { + return true; + } + } + return false; +} + +void VADCallback::Reset() { + memset(_numFrameTypes, 0, sizeof(_numFrameTypes)); +} + +VADCallback::VADCallback() { + memset(_numFrameTypes, 0, sizeof(_numFrameTypes)); +} + +void VADCallback::PrintFrameTypes() { + printf("kEmptyFrame......... %d\n", _numFrameTypes[kEmptyFrame]); + printf("kAudioFrameSpeech... %d\n", _numFrameTypes[kAudioFrameSpeech]); + printf("kAudioFrameCN....... %d\n", _numFrameTypes[kAudioFrameCN]); + printf("kVideoFrameKey...... %d\n", _numFrameTypes[kVideoFrameKey]); + printf("kVideoFrameDelta.... %d\n", _numFrameTypes[kVideoFrameDelta]); +} + +int32_t VADCallback::InFrameType(FrameType frame_type) { + _numFrameTypes[frame_type]++; + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.h b/third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.h new file mode 100644 index 0000000000..07cbe7170d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/test/utility.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_UTILITY_H_ +#define MODULES_AUDIO_CODING_TEST_UTILITY_H_ + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "test/gtest.h" + +namespace webrtc { + +//----------------------------- +#define CHECK_ERROR(f) \ + do { \ + EXPECT_GE(f, 0) << "Error Calling API"; \ + } while(0) + +//----------------------------- +#define CHECK_PROTECTED(f) \ + do { \ + if (f >= 0) { \ + ADD_FAILURE() << "Error Calling API"; \ + } else { \ + printf("An expected error is caught.\n"); \ + } \ + } while(0) + +//---------------------------- +#define CHECK_ERROR_MT(f) \ + do { \ + if (f < 0) { \ + fprintf(stderr, "Error Calling API in file %s at line %d \n", \ + __FILE__, __LINE__); \ + } \ + } while(0) + +//---------------------------- +#define CHECK_PROTECTED_MT(f) \ + do { \ + if (f >= 0) { \ + fprintf(stderr, "Error Calling API in file %s at line %d \n", \ + __FILE__, __LINE__); \ + } else { \ + printf("An expected error is caught.\n"); \ + } \ + } while(0) + +#define DELETE_POINTER(p) \ + do { \ + if (p != NULL) { \ + delete p; \ + p = NULL; \ + } \ + } while(0) + +class ACMTestTimer { + public: + ACMTestTimer(); + ~ACMTestTimer(); + + void Reset(); + void Tick10ms(); + void Tick1ms(); + void Tick100ms(); + void Tick1sec(); + void CurrentTimeHMS(char* currTime); + void CurrentTime(unsigned long& h, unsigned char& m, unsigned char& s, + unsigned short& ms); + + private: + void Adjust(); + + unsigned short _msec; + unsigned char _sec; + unsigned char _min; + unsigned long _hour; +}; + +// To avoid clashes with CircularBuffer in APM. +namespace test { + +class CircularBuffer { + public: + CircularBuffer(uint32_t len); + ~CircularBuffer(); + + void SetArithMean(bool enable); + void SetVariance(bool enable); + + void Update(const double newVal); + void IsBufferFull(); + + int16_t Variance(double& var); + int16_t ArithMean(double& mean); + + protected: + double* _buff; + uint32_t _idx; + uint32_t _buffLen; + + bool _buffIsFull; + bool _calcAvg; + bool _calcVar; + double _sum; + double _sumSqr; +}; + +} // namespace test + +int16_t ChooseCodec(CodecInst& codecInst); + +void PrintCodecs(); + +bool FixedPayloadTypeCodec(const char* payloadName); + +class VADCallback : public ACMVADCallback { + public: + VADCallback(); + + int32_t InFrameType(FrameType frame_type) override; + + void PrintFrameTypes(); + void Reset(); + + private: + uint32_t _numFrameTypes[5]; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_UTILITY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_c_gn/moz.build new file mode 100644 index 0000000000..7e0c63a8ad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_c_gn/moz.build @@ -0,0 +1,280 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_CODEC_ILBC"] = True +DEFINES["WEBRTC_CODEC_OPUS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_OPUS_SUPPORT_120MS_PTIME"] = "1" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/media/libopus/include/", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/opus_interface.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_CODEC_ISACFX"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "ppc64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +Library("webrtc_opus_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_gn/moz.build new file mode 100644 index 0000000000..cc3990ed69 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_coding/webrtc_opus_gn/moz.build @@ -0,0 +1,292 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_CODEC_ILBC"] = True +DEFINES["WEBRTC_CODEC_OPUS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_OPUS_SUPPORT_120MS_PTIME"] = "1" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/media/libopus/include/", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_CODEC_ISACFX"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "ppc64": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_CODEC_ISAC"] = True + +Library("webrtc_opus_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/BUILD.gn b/third_party/libwebrtc/webrtc/modules/audio_device/BUILD.gn new file mode 100644 index 0000000000..70447c69c6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/BUILD.gn @@ -0,0 +1,378 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +if (is_android) { + import("//build/config/android/config.gni") + import("//build/config/android/rules.gni") +} + +config("audio_device_config") { + include_dirs = [ + "../include", + "include", + "dummy", # Contains dummy audio device implementations. + ] +} + +config("audio_device_warnings_config") { + if (is_win && is_clang) { + cflags = [ + # Disable warnings failing when compiling with Clang on Windows. + # https://bugs.chromium.org/p/webrtc/issues/detail?id=5366 + "-Wno-bool-conversion", + "-Wno-delete-non-virtual-dtor", + "-Wno-logical-op-parentheses", + "-Wno-microsoft-extra-qualification", + "-Wno-microsoft-goto", + "-Wno-missing-braces", + "-Wno-parentheses-equality", + "-Wno-reorder", + "-Wno-shift-overflow", + "-Wno-tautological-compare", + + # See https://bugs.chromium.org/p/webrtc/issues/detail?id=6265 + # for -Wno-thread-safety-analysis + "-Wno-thread-safety-analysis", + "-Wno-unused-private-field", + ] + } +} + +rtc_source_set("audio_device") { + public_deps = [ + ":audio_device_generic", + ] + if (rtc_include_internal_audio_device && is_ios) { + public_deps += [ ":audio_device_ios_objc" ] + } +} + +if (rtc_include_internal_audio_device && is_ios) { + rtc_source_set("audio_device_ios_objc") { + visibility = [ ":audio_device" ] + sources = [ + "ios/audio_device_ios.h", + "ios/audio_device_ios.mm", + "ios/audio_device_not_implemented_ios.mm", + "ios/audio_session_observer.h", + "ios/objc/RTCAudioSessionDelegateAdapter.h", + "ios/objc/RTCAudioSessionDelegateAdapter.mm", + "ios/voice_processing_audio_unit.h", + "ios/voice_processing_audio_unit.mm", + ] + libs = [ + "AudioToolbox.framework", + "AVFoundation.framework", + "Foundation.framework", + "UIKit.framework", + ] + deps = [ + ":audio_device_generic", + "../../api:array_view", + "../../rtc_base:gtest_prod", + "../../rtc_base:rtc_base", + "../../sdk:audio_objc", + "../../sdk:common_objc", + "../../system_wrappers:metrics_api", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } +} + +rtc_source_set("audio_device_generic") { + visibility = [ + ":audio_device", + ":audio_device_ios_objc", + ] + public_configs = [ ":audio_device_config" ] + + deps = [ + "..:module_api", + "../..:webrtc_common", + "../../api:array_view", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_task_queue", + "../../system_wrappers", + "../utility", + ] + + sources = [ + "audio_device_buffer.cc", + "audio_device_buffer.h", + "audio_device_config.h", + "audio_device_generic.cc", + "audio_device_generic.h", + "dummy/audio_device_dummy.cc", + "dummy/audio_device_dummy.h", + "dummy/file_audio_device.cc", + "dummy/file_audio_device.h", + "fine_audio_buffer.cc", + "fine_audio_buffer.h", + "include/audio_device.h", + "include/audio_device_defines.h", + ] + + include_dirs = [] + if (is_linux) { + include_dirs += [ "linux" ] + } + if (is_ios) { + include_dirs += [ "ios" ] + } + if (is_mac) { + include_dirs += [ "mac" ] + } + if (is_win) { + include_dirs += [ "win" ] + } + if (is_android) { + include_dirs += [ "android" ] + } + defines = [] + cflags = [] + if (rtc_audio_device_plays_sinus_tone) { + defines += [ "AUDIO_DEVICE_PLAYS_SINUS_TONE" ] + } + if (rtc_include_internal_audio_device) { + sources += [ + "audio_device_data_observer.cc", + "audio_device_impl.cc", + "audio_device_impl.h", + "include/audio_device_data_observer.h", + ] + if (is_android) { + sources += [ + "android/audio_device_template.h", + "android/audio_manager.cc", + "android/audio_manager.h", + "android/audio_record_jni.cc", + "android/audio_record_jni.h", + "android/audio_track_jni.cc", + "android/audio_track_jni.h", + "android/build_info.cc", + "android/build_info.h", + "android/opensles_common.cc", + "android/opensles_common.h", + "android/opensles_player.cc", + "android/opensles_player.h", + "android/opensles_recorder.cc", + "android/opensles_recorder.h", + ] + libs = [ + "log", + "OpenSLES", + ] + } + if (rtc_use_dummy_audio_file_devices) { + defines += [ "WEBRTC_DUMMY_FILE_DEVICES" ] + } else { + if (is_linux) { + sources += [ + "linux/alsasymboltable_linux.cc", + "linux/alsasymboltable_linux.h", + "linux/audio_device_alsa_linux.cc", + "linux/audio_device_alsa_linux.h", + "linux/audio_mixer_manager_alsa_linux.cc", + "linux/audio_mixer_manager_alsa_linux.h", + "linux/latebindingsymboltable_linux.cc", + "linux/latebindingsymboltable_linux.h", + ] + defines += [ "LINUX_ALSA" ] + libs = [ "dl" ] + if (use_x11) { + libs += [ "X11" ] + } + if (rtc_include_pulse_audio) { + sources += [ + "linux/audio_device_pulse_linux.cc", + "linux/audio_device_pulse_linux.h", + "linux/audio_mixer_manager_pulse_linux.cc", + "linux/audio_mixer_manager_pulse_linux.h", + "linux/pulseaudiosymboltable_linux.cc", + "linux/pulseaudiosymboltable_linux.h", + ] + defines += [ "LINUX_PULSE" ] + } + } + if (is_mac) { + sources += [ + "mac/audio_device_mac.cc", + "mac/audio_device_mac.h", + "mac/audio_mixer_manager_mac.cc", + "mac/audio_mixer_manager_mac.h", + ] + deps += [ ":mac_portaudio" ] + libs = [ + # Needed for CoreGraphics: + "ApplicationServices.framework", + + "AudioToolbox.framework", + "CoreAudio.framework", + + # Needed for CGEventSourceKeyState in audio_device_mac.cc: + "CoreGraphics.framework", + ] + } + if (is_win) { + sources += [ + "win/audio_device_core_win.cc", + "win/audio_device_core_win.h", + ] + libs = [ + # Required for the built-in WASAPI AEC. + "dmoguids.lib", + "wmcodecdspuuid.lib", + "amstrmid.lib", + "msdmo.lib", + ] + } + configs += [ ":audio_device_warnings_config" ] + } + } else { + defines = [ "WEBRTC_DUMMY_AUDIO_BUILD" ] + } + + if (!build_with_chromium) { + sources += [ + # Do not link these into Chrome since they contain static data. + "dummy/file_audio_device_factory.cc", + "dummy/file_audio_device_factory.h", + ] + } + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } +} + +rtc_source_set("mac_portaudio") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "mac/portaudio/pa_memorybarrier.h", + "mac/portaudio/pa_ringbuffer.c", + "mac/portaudio/pa_ringbuffer.h", + ] +} + +config("mock_audio_device_config") { + if (is_win) { + cflags = [ + # TODO(phoglund): get rid of 4373 supression when + # http://code.google.com/p/webrtc/issues/detail?id=261 is solved. + # legacy warning for ignoring const / volatile in signatures. + "/wd4373", + ] + } +} + +rtc_source_set("mock_audio_device") { + testonly = true + sources = [ + "include/mock_audio_device.h", + "include/mock_audio_transport.h", + ] + deps = [ + ":audio_device", + "../../test:test_support", + ] + all_dependent_configs = [ ":mock_audio_device_config" ] +} + +if (rtc_include_tests) { + # TODO(kthelgason): Reenable these tests on simulator. + # See bugs.webrtc.org/7812 + if (is_ios && !use_ios_simulator) { + rtc_source_set("audio_device_ios_objc_unittests") { + testonly = true + visibility = [ ":*" ] + sources = [ + "ios/audio_device_unittest_ios.mm", + ] + deps = [ + ":audio_device", + ":mock_audio_device", + "../../rtc_base:rtc_base_approved", + "../../sdk:audio_objc", + "../../system_wrappers", + "../../test:test_support", + "//testing/gmock", + "//third_party/ocmock", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + } + + rtc_source_set("audio_device_unittests") { + testonly = true + + sources = [ + "fine_audio_buffer_unittest.cc", + ] + deps = [ + ":audio_device", + ":mock_audio_device", + "../../api:array_view", + "../../api:optional", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../../test:test_support", + "../utility:utility", + "//testing/gmock", + ] + if (is_linux || is_mac || is_win) { + sources += [ "audio_device_unittest.cc" ] + } + if (is_android) { + # Need to disable error due to the line in + # base/android/jni_android.h triggering it: + # const BASE_EXPORT jobject GetApplicationContext() + # error: type qualifiers ignored on function return type + cflags = [ "-Wno-ignored-qualifiers" ] + sources += [ + "android/audio_device_unittest.cc", + "android/audio_manager_unittest.cc", + "android/ensure_initialized.cc", + "android/ensure_initialized.h", + ] + deps += [ + "../../../base", + "../../sdk/android:libjingle_peerconnection_java", + ] + } + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } +} + +if ((!build_with_chromium && !build_with_mozilla) && is_android) { + rtc_android_library("audio_device_java") { + java_files = [ + "android/java/src/org/webrtc/voiceengine/BuildInfo.java", + "android/java/src/org/webrtc/voiceengine/WebRtcAudioEffects.java", + "android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java", + "android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java", + "android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java", + "android/java/src/org/webrtc/voiceengine/WebRtcAudioUtils.java", + ] + deps = [ + "../../rtc_base:base_java", + ] + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/DEPS b/third_party/libwebrtc/webrtc/modules/audio_device/DEPS new file mode 100644 index 0000000000..cb7a541b8d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/DEPS @@ -0,0 +1,31 @@ +include_rules = [ + "+common_audio", + "+system_wrappers", +] + +specific_include_rules = { + "ensure_initialized\.cc": [ + "+base/android", + ], + "audio_device_ios\.h": [ + "+sdk/objc", + ], + "audio_device_ios\.mm": [ + "+sdk/objc", + ], + "audio_device_unittest_ios\.mm": [ + "+sdk/objc", + ], + "RTCAudioSession\.h": [ + "+sdk/objc", + ], + "RTCAudioSessionConfiguration\.h": [ + "+sdk/objc", + ], + "RTCAudioSessionDelegateAdapter\.h": [ + "+sdk/objc", + ], + "voice_processing_audio_unit\.mm": [ + "+sdk/objc", + ], +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_device/OWNERS new file mode 100644 index 0000000000..f216978e2f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/OWNERS @@ -0,0 +1,9 @@ +henrikg@webrtc.org +henrika@webrtc.org +niklas.enbom@webrtc.org +tkchin@webrtc.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_common.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_common.h new file mode 100644 index 0000000000..81ea733aa4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_common.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_AUDIO_COMMON_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_AUDIO_COMMON_H_ + +namespace webrtc { + +const int kDefaultSampleRate = 44100; +// Delay estimates for the two different supported modes. These values are based +// on real-time round-trip delay estimates on a large set of devices and they +// are lower bounds since the filter length is 128 ms, so the AEC works for +// delays in the range [50, ~170] ms and [150, ~270] ms. Note that, in most +// cases, the lowest delay estimate will not be utilized since devices that +// support low-latency output audio often supports HW AEC as well. +const int kLowLatencyModeDelayEstimateInMilliseconds = 50; +const int kHighLatencyModeDelayEstimateInMilliseconds = 150; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_AUDIO_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_template.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_template.h new file mode 100644 index 0000000000..75dbaf315f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_template.h @@ -0,0 +1,466 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_AUDIO_DEVICE_TEMPLATE_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_AUDIO_DEVICE_TEMPLATE_H_ + +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/audio_device_generic.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +// InputType/OutputType can be any class that implements the capturing/rendering +// part of the AudioDeviceGeneric API. +// Construction and destruction must be done on one and the same thread. Each +// internal implementation of InputType and OutputType will RTC_DCHECK if that +// is not the case. All implemented methods must also be called on the same +// thread. See comments in each InputType/OutputType class for more info. +// It is possible to call the two static methods (SetAndroidAudioDeviceObjects +// and ClearAndroidAudioDeviceObjects) from a different thread but both will +// RTC_CHECK that the calling thread is attached to a Java VM. + +template +class AudioDeviceTemplate : public AudioDeviceGeneric { + public: + AudioDeviceTemplate(AudioDeviceModule::AudioLayer audio_layer, + AudioManager* audio_manager) + : audio_layer_(audio_layer), + audio_manager_(audio_manager), + output_(audio_manager_), + input_(audio_manager_), + initialized_(false) { + RTC_LOG(INFO) << __FUNCTION__; + RTC_CHECK(audio_manager); + audio_manager_->SetActiveAudioLayer(audio_layer); + } + + virtual ~AudioDeviceTemplate() { RTC_LOG(INFO) << __FUNCTION__; } + + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override { + RTC_LOG(INFO) << __FUNCTION__; + audioLayer = audio_layer_; + return 0; + } + + InitStatus Init() override { + RTC_LOG(INFO) << __FUNCTION__; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!initialized_); + if (!audio_manager_->Init()) { + return InitStatus::OTHER_ERROR; + } + if (output_.Init() != 0) { + audio_manager_->Close(); + return InitStatus::PLAYOUT_ERROR; + } + if (input_.Init() != 0) { + output_.Terminate(); + audio_manager_->Close(); + return InitStatus::RECORDING_ERROR; + } + initialized_ = true; + return InitStatus::OK; + } + + int32_t Terminate() override { + RTC_LOG(INFO) << __FUNCTION__; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + int32_t err = input_.Terminate(); + err |= output_.Terminate(); + err |= !audio_manager_->Close(); + initialized_ = false; + RTC_DCHECK_EQ(err, 0); + return err; + } + + bool Initialized() const override { + RTC_LOG(INFO) << __FUNCTION__; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return initialized_; + } + + int16_t PlayoutDevices() override { + RTC_LOG(INFO) << __FUNCTION__; + return 1; + } + + int16_t RecordingDevices() override { + RTC_LOG(INFO) << __FUNCTION__; + return 1; + } + + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t SetPlayoutDevice(uint16_t index) override { + // OK to use but it has no effect currently since device selection is + // done using Andoid APIs instead. + RTC_LOG(INFO) << __FUNCTION__; + return 0; + } + + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t SetRecordingDevice(uint16_t index) override { + // OK to use but it has no effect currently since device selection is + // done using Andoid APIs instead. + RTC_LOG(INFO) << __FUNCTION__; + return 0; + } + + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t PlayoutIsAvailable(bool& available) override { + RTC_LOG(INFO) << __FUNCTION__; + available = true; + return 0; + } + + int32_t InitPlayout() override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.InitPlayout(); + } + + bool PlayoutIsInitialized() const override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.PlayoutIsInitialized(); + } + + int32_t RecordingIsAvailable(bool& available) override { + RTC_LOG(INFO) << __FUNCTION__; + available = true; + return 0; + } + + int32_t InitRecording() override { + RTC_LOG(INFO) << __FUNCTION__; + return input_.InitRecording(); + } + + bool RecordingIsInitialized() const override { + RTC_LOG(INFO) << __FUNCTION__; + return input_.RecordingIsInitialized(); + } + + int32_t StartPlayout() override { + RTC_LOG(INFO) << __FUNCTION__; + if (!audio_manager_->IsCommunicationModeEnabled()) { + RTC_LOG(WARNING) + << "The application should use MODE_IN_COMMUNICATION audio mode!"; + } + return output_.StartPlayout(); + } + + int32_t StopPlayout() override { + // Avoid using audio manger (JNI/Java cost) if playout was inactive. + if (!Playing()) + return 0; + RTC_LOG(INFO) << __FUNCTION__; + int32_t err = output_.StopPlayout(); + return err; + } + + bool Playing() const override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.Playing(); + } + + int32_t StartRecording() override { + RTC_LOG(INFO) << __FUNCTION__; + if (!audio_manager_->IsCommunicationModeEnabled()) { + RTC_LOG(WARNING) + << "The application should use MODE_IN_COMMUNICATION audio mode!"; + } + return input_.StartRecording(); + } + + int32_t StopRecording() override { + // Avoid using audio manger (JNI/Java cost) if recording was inactive. + RTC_LOG(INFO) << __FUNCTION__; + if (!Recording()) + return 0; + int32_t err = input_.StopRecording(); + return err; + } + + bool Recording() const override { return input_.Recording(); } + + int32_t SetAGC(bool enable) override { + if (enable) { + FATAL() << "Should never be called"; + } + return -1; + } + + bool AGC() const override { + RTC_LOG(INFO) << __FUNCTION__; + return false; + } + + int32_t InitSpeaker() override { + RTC_LOG(INFO) << __FUNCTION__; + return 0; + } + + bool SpeakerIsInitialized() const override { + RTC_LOG(INFO) << __FUNCTION__; + return true; + } + + int32_t InitMicrophone() override { + RTC_LOG(INFO) << __FUNCTION__; + return 0; + } + + bool MicrophoneIsInitialized() const override { + RTC_LOG(INFO) << __FUNCTION__; + return true; + } + + int32_t SpeakerVolumeIsAvailable(bool& available) override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.SpeakerVolumeIsAvailable(available); + } + + int32_t SetSpeakerVolume(uint32_t volume) override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.SetSpeakerVolume(volume); + } + + int32_t SpeakerVolume(uint32_t& volume) const override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.SpeakerVolume(volume); + } + + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.MaxSpeakerVolume(maxVolume); + } + + int32_t MinSpeakerVolume(uint32_t& minVolume) const override { + RTC_LOG(INFO) << __FUNCTION__; + return output_.MinSpeakerVolume(minVolume); + } + + int32_t MicrophoneVolumeIsAvailable(bool& available) override { + available = false; + return -1; + } + + int32_t SetMicrophoneVolume(uint32_t volume) override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t MicrophoneVolume(uint32_t& volume) const override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t SpeakerMuteIsAvailable(bool& available) override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t SetSpeakerMute(bool enable) override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t SpeakerMute(bool& enabled) const override { + FATAL() << "Should never be called"; + return -1; + } + + int32_t MicrophoneMuteIsAvailable(bool& available) override { + FATAL() << "Not implemented"; + return -1; + } + + int32_t SetMicrophoneMute(bool enable) override { + FATAL() << "Not implemented"; + return -1; + } + + int32_t MicrophoneMute(bool& enabled) const override { + FATAL() << "Not implemented"; + return -1; + } + + // Returns true if the audio manager has been configured to support stereo + // and false otherwised. Default is mono. + int32_t StereoPlayoutIsAvailable(bool& available) override { + RTC_LOG(INFO) << __FUNCTION__; + available = audio_manager_->IsStereoPlayoutSupported(); + return 0; + } + + int32_t SetStereoPlayout(bool enable) override { + RTC_LOG(INFO) << __FUNCTION__; + bool available = audio_manager_->IsStereoPlayoutSupported(); + // Android does not support changes between mono and stero on the fly. + // Instead, the native audio layer is configured via the audio manager + // to either support mono or stereo. It is allowed to call this method + // if that same state is not modified. + return (enable == available) ? 0 : -1; + } + + int32_t StereoPlayout(bool& enabled) const override { + enabled = audio_manager_->IsStereoPlayoutSupported(); + return 0; + } + + int32_t StereoRecordingIsAvailable(bool& available) override { + RTC_LOG(INFO) << __FUNCTION__; + available = audio_manager_->IsStereoRecordSupported(); + return 0; + } + + int32_t SetStereoRecording(bool enable) override { + RTC_LOG(INFO) << __FUNCTION__; + bool available = audio_manager_->IsStereoRecordSupported(); + // Android does not support changes between mono and stero on the fly. + // Instead, the native audio layer is configured via the audio manager + // to either support mono or stereo. It is allowed to call this method + // if that same state is not modified. + return (enable == available) ? 0 : -1; + } + + int32_t StereoRecording(bool& enabled) const override { + RTC_LOG(INFO) << __FUNCTION__; + enabled = audio_manager_->IsStereoRecordSupported(); + return 0; + } + + int32_t PlayoutDelay(uint16_t& delay_ms) const override { + // Best guess we can do is to use half of the estimated total delay. + delay_ms = audio_manager_->GetDelayEstimateInMilliseconds() / 2; + RTC_DCHECK_GT(delay_ms, 0); + return 0; + } + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override { + RTC_LOG(INFO) << __FUNCTION__; + output_.AttachAudioBuffer(audioBuffer); + input_.AttachAudioBuffer(audioBuffer); + } + + // Returns true if the device both supports built in AEC and the device + // is not blacklisted. + // Currently, if OpenSL ES is used in both directions, this method will still + // report the correct value and it has the correct effect. As an example: + // a device supports built in AEC and this method returns true. Libjingle + // will then disable the WebRTC based AEC and that will work for all devices + // (mainly Nexus) even when OpenSL ES is used for input since our current + // implementation will enable built-in AEC by default also for OpenSL ES. + // The only "bad" thing that happens today is that when Libjingle calls + // OpenSLESRecorder::EnableBuiltInAEC() it will not have any real effect and + // a "Not Implemented" log will be filed. This non-perfect state will remain + // until I have added full support for audio effects based on OpenSL ES APIs. + bool BuiltInAECIsAvailable() const override { + RTC_LOG(INFO) << __FUNCTION__; + return audio_manager_->IsAcousticEchoCancelerSupported(); + } + + // TODO(henrika): add implementation for OpenSL ES based audio as well. + int32_t EnableBuiltInAEC(bool enable) override { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + RTC_CHECK(BuiltInAECIsAvailable()) << "HW AEC is not available"; + return input_.EnableBuiltInAEC(enable); + } + + // Returns true if the device both supports built in AGC and the device + // is not blacklisted. + // TODO(henrika): add implementation for OpenSL ES based audio as well. + // In addition, see comments for BuiltInAECIsAvailable(). + bool BuiltInAGCIsAvailable() const override { + RTC_LOG(INFO) << __FUNCTION__; + return audio_manager_->IsAutomaticGainControlSupported(); + } + + // TODO(henrika): add implementation for OpenSL ES based audio as well. + int32_t EnableBuiltInAGC(bool enable) override { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + RTC_CHECK(BuiltInAGCIsAvailable()) << "HW AGC is not available"; + return input_.EnableBuiltInAGC(enable); + } + + // Returns true if the device both supports built in NS and the device + // is not blacklisted. + // TODO(henrika): add implementation for OpenSL ES based audio as well. + // In addition, see comments for BuiltInAECIsAvailable(). + bool BuiltInNSIsAvailable() const override { + RTC_LOG(INFO) << __FUNCTION__; + return audio_manager_->IsNoiseSuppressorSupported(); + } + + // TODO(henrika): add implementation for OpenSL ES based audio as well. + int32_t EnableBuiltInNS(bool enable) override { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + RTC_CHECK(BuiltInNSIsAvailable()) << "HW NS is not available"; + return input_.EnableBuiltInNS(enable); + } + + private: + rtc::ThreadChecker thread_checker_; + + // Local copy of the audio layer set during construction of the + // AudioDeviceModuleImpl instance. Read only value. + const AudioDeviceModule::AudioLayer audio_layer_; + + // Non-owning raw pointer to AudioManager instance given to use at + // construction. The real object is owned by AudioDeviceModuleImpl and the + // life time is the same as that of the AudioDeviceModuleImpl, hence there + // is no risk of reading a NULL pointer at any time in this class. + AudioManager* const audio_manager_; + + OutputType output_; + + InputType input_; + + bool initialized_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_AUDIO_DEVICE_TEMPLATE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_unittest.cc new file mode 100644 index 0000000000..4a9262b044 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_device_unittest.cc @@ -0,0 +1,1028 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "modules/audio_device/android/audio_common.h" +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/android/build_info.h" +#include "modules/audio_device/android/ensure_initialized.h" +#include "modules/audio_device/audio_device_impl.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/mock_audio_transport.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/event_wrapper.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +using std::cout; +using std::endl; +using ::testing::_; +using ::testing::AtLeast; +using ::testing::Gt; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::NotNull; +using ::testing::Return; + +// #define ENABLE_DEBUG_PRINTF +#ifdef ENABLE_DEBUG_PRINTF +#define PRINTD(...) fprintf(stderr, __VA_ARGS__); +#else +#define PRINTD(...) ((void)0) +#endif +#define PRINT(...) fprintf(stderr, __VA_ARGS__); + +namespace webrtc { + +// Number of callbacks (input or output) the tests waits for before we set +// an event indicating that the test was OK. +static const size_t kNumCallbacks = 10; +// Max amount of time we wait for an event to be set while counting callbacks. +static const int kTestTimeOutInMilliseconds = 10 * 1000; +// Average number of audio callbacks per second assuming 10ms packet size. +static const size_t kNumCallbacksPerSecond = 100; +// Play out a test file during this time (unit is in seconds). +static const int kFilePlayTimeInSec = 5; +static const size_t kBitsPerSample = 16; +static const size_t kBytesPerSample = kBitsPerSample / 8; +// Run the full-duplex test during this time (unit is in seconds). +// Note that first |kNumIgnoreFirstCallbacks| are ignored. +static const int kFullDuplexTimeInSec = 5; +// Wait for the callback sequence to stabilize by ignoring this amount of the +// initial callbacks (avoids initial FIFO access). +// Only used in the RunPlayoutAndRecordingInFullDuplex test. +static const size_t kNumIgnoreFirstCallbacks = 50; +// Sets the number of impulses per second in the latency test. +static const int kImpulseFrequencyInHz = 1; +// Length of round-trip latency measurements. Number of transmitted impulses +// is kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1. +static const int kMeasureLatencyTimeInSec = 11; +// Utilized in round-trip latency measurements to avoid capturing noise samples. +static const int kImpulseThreshold = 1000; +static const char kTag[] = "[..........] "; + +enum TransportType { + kPlayout = 0x1, + kRecording = 0x2, +}; + +// Interface for processing the audio stream. Real implementations can e.g. +// run audio in loopback, read audio from a file or perform latency +// measurements. +class AudioStreamInterface { + public: + virtual void Write(const void* source, size_t num_frames) = 0; + virtual void Read(void* destination, size_t num_frames) = 0; + protected: + virtual ~AudioStreamInterface() {} +}; + +// Reads audio samples from a PCM file where the file is stored in memory at +// construction. +class FileAudioStream : public AudioStreamInterface { + public: + FileAudioStream( + size_t num_callbacks, const std::string& file_name, int sample_rate) + : file_size_in_bytes_(0), + sample_rate_(sample_rate), + file_pos_(0) { + file_size_in_bytes_ = test::GetFileSize(file_name); + sample_rate_ = sample_rate; + EXPECT_GE(file_size_in_callbacks(), num_callbacks) + << "Size of test file is not large enough to last during the test."; + const size_t num_16bit_samples = + test::GetFileSize(file_name) / kBytesPerSample; + file_.reset(new int16_t[num_16bit_samples]); + FILE* audio_file = fopen(file_name.c_str(), "rb"); + EXPECT_NE(audio_file, nullptr); + size_t num_samples_read = fread( + file_.get(), sizeof(int16_t), num_16bit_samples, audio_file); + EXPECT_EQ(num_samples_read, num_16bit_samples); + fclose(audio_file); + } + + // AudioStreamInterface::Write() is not implemented. + void Write(const void* source, size_t num_frames) override {} + + // Read samples from file stored in memory (at construction) and copy + // |num_frames| (<=> 10ms) to the |destination| byte buffer. + void Read(void* destination, size_t num_frames) override { + memcpy(destination, + static_cast (&file_[file_pos_]), + num_frames * sizeof(int16_t)); + file_pos_ += num_frames; + } + + int file_size_in_seconds() const { + return static_cast( + file_size_in_bytes_ / (kBytesPerSample * sample_rate_)); + } + size_t file_size_in_callbacks() const { + return file_size_in_seconds() * kNumCallbacksPerSecond; + } + + private: + size_t file_size_in_bytes_; + int sample_rate_; + std::unique_ptr file_; + size_t file_pos_; +}; + +// Simple first in first out (FIFO) class that wraps a list of 16-bit audio +// buffers of fixed size and allows Write and Read operations. The idea is to +// store recorded audio buffers (using Write) and then read (using Read) these +// stored buffers with as short delay as possible when the audio layer needs +// data to play out. The number of buffers in the FIFO will stabilize under +// normal conditions since there will be a balance between Write and Read calls. +// The container is a std::list container and access is protected with a lock +// since both sides (playout and recording) are driven by its own thread. +class FifoAudioStream : public AudioStreamInterface { + public: + explicit FifoAudioStream(size_t frames_per_buffer) + : frames_per_buffer_(frames_per_buffer), + bytes_per_buffer_(frames_per_buffer_ * sizeof(int16_t)), + fifo_(new AudioBufferList), + largest_size_(0), + total_written_elements_(0), + write_count_(0) { + EXPECT_NE(fifo_.get(), nullptr); + } + + ~FifoAudioStream() { + Flush(); + } + + // Allocate new memory, copy |num_frames| samples from |source| into memory + // and add pointer to the memory location to end of the list. + // Increases the size of the FIFO by one element. + void Write(const void* source, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + PRINTD("+"); + if (write_count_++ < kNumIgnoreFirstCallbacks) { + return; + } + int16_t* memory = new int16_t[frames_per_buffer_]; + memcpy(static_cast (&memory[0]), + source, + bytes_per_buffer_); + rtc::CritScope lock(&lock_); + fifo_->push_back(memory); + const size_t size = fifo_->size(); + if (size > largest_size_) { + largest_size_ = size; + PRINTD("(%" PRIuS ")", largest_size_); + } + total_written_elements_ += size; + } + + // Read pointer to data buffer from front of list, copy |num_frames| of stored + // data into |destination| and delete the utilized memory allocation. + // Decreases the size of the FIFO by one element. + void Read(void* destination, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + PRINTD("-"); + rtc::CritScope lock(&lock_); + if (fifo_->empty()) { + memset(destination, 0, bytes_per_buffer_); + } else { + int16_t* memory = fifo_->front(); + fifo_->pop_front(); + memcpy(destination, + static_cast (&memory[0]), + bytes_per_buffer_); + delete memory; + } + } + + size_t size() const { + return fifo_->size(); + } + + size_t largest_size() const { + return largest_size_; + } + + size_t average_size() const { + return (total_written_elements_ == 0) ? 0.0 : 0.5 + static_cast ( + total_written_elements_) / (write_count_ - kNumIgnoreFirstCallbacks); + } + + private: + void Flush() { + for (auto it = fifo_->begin(); it != fifo_->end(); ++it) { + delete *it; + } + fifo_->clear(); + } + + using AudioBufferList = std::list; + rtc::CriticalSection lock_; + const size_t frames_per_buffer_; + const size_t bytes_per_buffer_; + std::unique_ptr fifo_; + size_t largest_size_; + size_t total_written_elements_; + size_t write_count_; +}; + +// Inserts periodic impulses and measures the latency between the time of +// transmission and time of receiving the same impulse. +// Usage requires a special hardware called Audio Loopback Dongle. +// See http://source.android.com/devices/audio/loopback.html for details. +class LatencyMeasuringAudioStream : public AudioStreamInterface { + public: + explicit LatencyMeasuringAudioStream(size_t frames_per_buffer) + : frames_per_buffer_(frames_per_buffer), + bytes_per_buffer_(frames_per_buffer_ * sizeof(int16_t)), + play_count_(0), + rec_count_(0), + pulse_time_(0) { + } + + // Insert periodic impulses in first two samples of |destination|. + void Read(void* destination, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + if (play_count_ == 0) { + PRINT("["); + } + play_count_++; + memset(destination, 0, bytes_per_buffer_); + if (play_count_ % (kNumCallbacksPerSecond / kImpulseFrequencyInHz) == 0) { + if (pulse_time_ == 0) { + pulse_time_ = rtc::TimeMillis(); + } + PRINT("."); + const int16_t impulse = std::numeric_limits::max(); + int16_t* ptr16 = static_cast (destination); + for (size_t i = 0; i < 2; ++i) { + ptr16[i] = impulse; + } + } + } + + // Detect received impulses in |source|, derive time between transmission and + // detection and add the calculated delay to list of latencies. + void Write(const void* source, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + rec_count_++; + if (pulse_time_ == 0) { + // Avoid detection of new impulse response until a new impulse has + // been transmitted (sets |pulse_time_| to value larger than zero). + return; + } + const int16_t* ptr16 = static_cast (source); + std::vector vec(ptr16, ptr16 + num_frames); + // Find max value in the audio buffer. + int max = *std::max_element(vec.begin(), vec.end()); + // Find index (element position in vector) of the max element. + int index_of_max = std::distance(vec.begin(), + std::find(vec.begin(), vec.end(), + max)); + if (max > kImpulseThreshold) { + PRINTD("(%d,%d)", max, index_of_max); + int64_t now_time = rtc::TimeMillis(); + int extra_delay = IndexToMilliseconds(static_cast (index_of_max)); + PRINTD("[%d]", static_cast (now_time - pulse_time_)); + PRINTD("[%d]", extra_delay); + // Total latency is the difference between transmit time and detection + // tome plus the extra delay within the buffer in which we detected the + // received impulse. It is transmitted at sample 0 but can be received + // at sample N where N > 0. The term |extra_delay| accounts for N and it + // is a value between 0 and 10ms. + latencies_.push_back(now_time - pulse_time_ + extra_delay); + pulse_time_ = 0; + } else { + PRINTD("-"); + } + } + + size_t num_latency_values() const { + return latencies_.size(); + } + + int min_latency() const { + if (latencies_.empty()) + return 0; + return *std::min_element(latencies_.begin(), latencies_.end()); + } + + int max_latency() const { + if (latencies_.empty()) + return 0; + return *std::max_element(latencies_.begin(), latencies_.end()); + } + + int average_latency() const { + if (latencies_.empty()) + return 0; + return 0.5 + static_cast ( + std::accumulate(latencies_.begin(), latencies_.end(), 0)) / + latencies_.size(); + } + + void PrintResults() const { + PRINT("] "); + for (auto it = latencies_.begin(); it != latencies_.end(); ++it) { + PRINT("%d ", *it); + } + PRINT("\n"); + PRINT("%s[min, max, avg]=[%d, %d, %d] ms\n", kTag, + min_latency(), max_latency(), average_latency()); + } + + int IndexToMilliseconds(double index) const { + return static_cast(10.0 * (index / frames_per_buffer_) + 0.5); + } + + private: + const size_t frames_per_buffer_; + const size_t bytes_per_buffer_; + size_t play_count_; + size_t rec_count_; + int64_t pulse_time_; + std::vector latencies_; +}; + +// Mocks the AudioTransport object and proxies actions for the two callbacks +// (RecordedDataIsAvailable and NeedMorePlayData) to different implementations +// of AudioStreamInterface. +class MockAudioTransportAndroid : public test::MockAudioTransport { + public: + explicit MockAudioTransportAndroid(int type) + : num_callbacks_(0), + type_(type), + play_count_(0), + rec_count_(0), + audio_stream_(nullptr) {} + + virtual ~MockAudioTransportAndroid() {} + + // Set default actions of the mock object. We are delegating to fake + // implementations (of AudioStreamInterface) here. + void HandleCallbacks(EventWrapper* test_is_done, + AudioStreamInterface* audio_stream, + int num_callbacks) { + test_is_done_ = test_is_done; + audio_stream_ = audio_stream; + num_callbacks_ = num_callbacks; + if (play_mode()) { + ON_CALL(*this, NeedMorePlayData(_, _, _, _, _, _, _, _)) + .WillByDefault( + Invoke(this, &MockAudioTransportAndroid::RealNeedMorePlayData)); + } + if (rec_mode()) { + ON_CALL(*this, RecordedDataIsAvailable(_, _, _, _, _, _, _, _, _, _)) + .WillByDefault(Invoke( + this, &MockAudioTransportAndroid::RealRecordedDataIsAvailable)); + } + } + + int32_t RealRecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + EXPECT_TRUE(rec_mode()) << "No test is expecting these callbacks."; + rec_count_++; + // Process the recorded audio stream if an AudioStreamInterface + // implementation exists. + if (audio_stream_) { + audio_stream_->Write(audioSamples, nSamples); + } + if (ReceivedEnoughCallbacks()) { + test_is_done_->Set(); + } + return 0; + } + + int32_t RealNeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + EXPECT_TRUE(play_mode()) << "No test is expecting these callbacks."; + play_count_++; + nSamplesOut = nSamples; + // Read (possibly processed) audio stream samples to be played out if an + // AudioStreamInterface implementation exists. + if (audio_stream_) { + audio_stream_->Read(audioSamples, nSamples); + } + if (ReceivedEnoughCallbacks()) { + test_is_done_->Set(); + } + return 0; + } + + bool ReceivedEnoughCallbacks() { + bool recording_done = false; + if (rec_mode()) + recording_done = rec_count_ >= num_callbacks_; + else + recording_done = true; + + bool playout_done = false; + if (play_mode()) + playout_done = play_count_ >= num_callbacks_; + else + playout_done = true; + + return recording_done && playout_done; + } + + bool play_mode() const { return type_ & kPlayout; } + bool rec_mode() const { return type_ & kRecording; } + + private: + EventWrapper* test_is_done_; + size_t num_callbacks_; + int type_; + size_t play_count_; + size_t rec_count_; + AudioStreamInterface* audio_stream_; + std::unique_ptr latency_audio_stream_; +}; + +// AudioDeviceTest test fixture. +class AudioDeviceTest : public ::testing::Test { + protected: + AudioDeviceTest() + : test_is_done_(EventWrapper::Create()) { + // One-time initialization of JVM and application context. Ensures that we + // can do calls between C++ and Java. Initializes both Java and OpenSL ES + // implementations. + webrtc::audiodevicemodule::EnsureInitialized(); + // Creates an audio device using a default audio layer. + audio_device_ = CreateAudioDevice(AudioDeviceModule::kPlatformDefaultAudio); + EXPECT_NE(audio_device_.get(), nullptr); + EXPECT_EQ(0, audio_device_->Init()); + playout_parameters_ = audio_manager()->GetPlayoutAudioParameters(); + record_parameters_ = audio_manager()->GetRecordAudioParameters(); + build_info_.reset(new BuildInfo()); + } + virtual ~AudioDeviceTest() { + EXPECT_EQ(0, audio_device_->Terminate()); + } + + int playout_sample_rate() const { + return playout_parameters_.sample_rate(); + } + int record_sample_rate() const { + return record_parameters_.sample_rate(); + } + size_t playout_channels() const { + return playout_parameters_.channels(); + } + size_t record_channels() const { + return record_parameters_.channels(); + } + size_t playout_frames_per_10ms_buffer() const { + return playout_parameters_.frames_per_10ms_buffer(); + } + size_t record_frames_per_10ms_buffer() const { + return record_parameters_.frames_per_10ms_buffer(); + } + + int total_delay_ms() const { + return audio_manager()->GetDelayEstimateInMilliseconds(); + } + + rtc::scoped_refptr audio_device() const { + return audio_device_; + } + + AudioDeviceModuleImpl* audio_device_impl() const { + return static_cast(audio_device_.get()); + } + + AudioManager* audio_manager() const { + return audio_device_impl()->GetAndroidAudioManagerForTest(); + } + + AudioManager* GetAudioManager(AudioDeviceModule* adm) const { + return static_cast(adm)-> + GetAndroidAudioManagerForTest(); + } + + AudioDeviceBuffer* audio_device_buffer() const { + return audio_device_impl()->GetAudioDeviceBuffer(); + } + + rtc::scoped_refptr CreateAudioDevice( + AudioDeviceModule::AudioLayer audio_layer) { + rtc::scoped_refptr module( + AudioDeviceModule::Create(0, audio_layer)); + return module; + } + + // Returns file name relative to the resource root given a sample rate. + std::string GetFileName(int sample_rate) { + EXPECT_TRUE(sample_rate == 48000 || sample_rate == 44100); + char fname[64]; + snprintf(fname, + sizeof(fname), + "audio_device/audio_short%d", + sample_rate / 1000); + std::string file_name(webrtc::test::ResourcePath(fname, "pcm")); + EXPECT_TRUE(test::FileExists(file_name)); +#ifdef ENABLE_PRINTF + PRINT("file name: %s\n", file_name.c_str()); + const size_t bytes = test::GetFileSize(file_name); + PRINT("file size: %" PRIuS " [bytes]\n", bytes); + PRINT("file size: %" PRIuS " [samples]\n", bytes / kBytesPerSample); + const int seconds = + static_cast(bytes / (sample_rate * kBytesPerSample)); + PRINT("file size: %d [secs]\n", seconds); + PRINT("file size: %" PRIuS " [callbacks]\n", + seconds * kNumCallbacksPerSecond); +#endif + return file_name; + } + + AudioDeviceModule::AudioLayer GetActiveAudioLayer() const { + AudioDeviceModule::AudioLayer audio_layer; + EXPECT_EQ(0, audio_device()->ActiveAudioLayer(&audio_layer)); + return audio_layer; + } + + int TestDelayOnAudioLayer( + const AudioDeviceModule::AudioLayer& layer_to_test) { + rtc::scoped_refptr audio_device; + audio_device = CreateAudioDevice(layer_to_test); + EXPECT_NE(audio_device.get(), nullptr); + AudioManager* audio_manager = GetAudioManager(audio_device.get()); + EXPECT_NE(audio_manager, nullptr); + return audio_manager->GetDelayEstimateInMilliseconds(); + } + + AudioDeviceModule::AudioLayer TestActiveAudioLayer( + const AudioDeviceModule::AudioLayer& layer_to_test) { + rtc::scoped_refptr audio_device; + audio_device = CreateAudioDevice(layer_to_test); + EXPECT_NE(audio_device.get(), nullptr); + AudioDeviceModule::AudioLayer active; + EXPECT_EQ(0, audio_device->ActiveAudioLayer(&active)); + return active; + } + + bool DisableTestForThisDevice(const std::string& model) { + return (build_info_->GetDeviceModel() == model); + } + + // Volume control is currently only supported for the Java output audio layer. + // For OpenSL ES, the internal stream volume is always on max level and there + // is no need for this test to set it to max. + bool AudioLayerSupportsVolumeControl() const { + return GetActiveAudioLayer() == AudioDeviceModule::kAndroidJavaAudio; + } + + void SetMaxPlayoutVolume() { + if (!AudioLayerSupportsVolumeControl()) + return; + uint32_t max_volume; + EXPECT_EQ(0, audio_device()->MaxSpeakerVolume(&max_volume)); + EXPECT_EQ(0, audio_device()->SetSpeakerVolume(max_volume)); + } + + void DisableBuiltInAECIfAvailable() { + if (audio_device()->BuiltInAECIsAvailable()) { + EXPECT_EQ(0, audio_device()->EnableBuiltInAEC(false)); + } + } + + void StartPlayout() { + EXPECT_FALSE(audio_device()->PlayoutIsInitialized()); + EXPECT_FALSE(audio_device()->Playing()); + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_TRUE(audio_device()->PlayoutIsInitialized()); + EXPECT_EQ(0, audio_device()->StartPlayout()); + EXPECT_TRUE(audio_device()->Playing()); + } + + void StopPlayout() { + EXPECT_EQ(0, audio_device()->StopPlayout()); + EXPECT_FALSE(audio_device()->Playing()); + EXPECT_FALSE(audio_device()->PlayoutIsInitialized()); + } + + void StartRecording() { + EXPECT_FALSE(audio_device()->RecordingIsInitialized()); + EXPECT_FALSE(audio_device()->Recording()); + EXPECT_EQ(0, audio_device()->InitRecording()); + EXPECT_TRUE(audio_device()->RecordingIsInitialized()); + EXPECT_EQ(0, audio_device()->StartRecording()); + EXPECT_TRUE(audio_device()->Recording()); + } + + void StopRecording() { + EXPECT_EQ(0, audio_device()->StopRecording()); + EXPECT_FALSE(audio_device()->Recording()); + } + + int GetMaxSpeakerVolume() const { + uint32_t max_volume(0); + EXPECT_EQ(0, audio_device()->MaxSpeakerVolume(&max_volume)); + return max_volume; + } + + int GetMinSpeakerVolume() const { + uint32_t min_volume(0); + EXPECT_EQ(0, audio_device()->MinSpeakerVolume(&min_volume)); + return min_volume; + } + + int GetSpeakerVolume() const { + uint32_t volume(0); + EXPECT_EQ(0, audio_device()->SpeakerVolume(&volume)); + return volume; + } + + std::unique_ptr test_is_done_; + rtc::scoped_refptr audio_device_; + AudioParameters playout_parameters_; + AudioParameters record_parameters_; + std::unique_ptr build_info_; +}; + +TEST_F(AudioDeviceTest, ConstructDestruct) { + // Using the test fixture to create and destruct the audio device module. +} + +// We always ask for a default audio layer when the ADM is constructed. But the +// ADM will then internally set the best suitable combination of audio layers, +// for input and output based on if low-latency output and/or input audio in +// combination with OpenSL ES is supported or not. This test ensures that the +// correct selection is done. +TEST_F(AudioDeviceTest, VerifyDefaultAudioLayer) { + const AudioDeviceModule::AudioLayer audio_layer = GetActiveAudioLayer(); + bool low_latency_output = audio_manager()->IsLowLatencyPlayoutSupported(); + bool low_latency_input = audio_manager()->IsLowLatencyRecordSupported(); + AudioDeviceModule::AudioLayer expected_audio_layer; + if (low_latency_output && low_latency_input) { + expected_audio_layer = AudioDeviceModule::kAndroidOpenSLESAudio; + } else if (low_latency_output && !low_latency_input) { + expected_audio_layer = + AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio; + } else { + expected_audio_layer = AudioDeviceModule::kAndroidJavaAudio; + } + EXPECT_EQ(expected_audio_layer, audio_layer); +} + +// Verify that it is possible to explicitly create the two types of supported +// ADMs. These two tests overrides the default selection of native audio layer +// by ignoring if the device supports low-latency output or not. +TEST_F(AudioDeviceTest, CorrectAudioLayerIsUsedForCombinedJavaOpenSLCombo) { + AudioDeviceModule::AudioLayer expected_layer = + AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio; + AudioDeviceModule::AudioLayer active_layer = TestActiveAudioLayer( + expected_layer); + EXPECT_EQ(expected_layer, active_layer); +} + +TEST_F(AudioDeviceTest, CorrectAudioLayerIsUsedForJavaInBothDirections) { + AudioDeviceModule::AudioLayer expected_layer = + AudioDeviceModule::kAndroidJavaAudio; + AudioDeviceModule::AudioLayer active_layer = TestActiveAudioLayer( + expected_layer); + EXPECT_EQ(expected_layer, active_layer); +} + +TEST_F(AudioDeviceTest, CorrectAudioLayerIsUsedForOpenSLInBothDirections) { + AudioDeviceModule::AudioLayer expected_layer = + AudioDeviceModule::kAndroidOpenSLESAudio; + AudioDeviceModule::AudioLayer active_layer = + TestActiveAudioLayer(expected_layer); + EXPECT_EQ(expected_layer, active_layer); +} + +// The Android ADM supports two different delay reporting modes. One for the +// low-latency output path (in combination with OpenSL ES), and one for the +// high-latency output path (Java backends in both directions). These two tests +// verifies that the audio manager reports correct delay estimate given the +// selected audio layer. Note that, this delay estimate will only be utilized +// if the HW AEC is disabled. +TEST_F(AudioDeviceTest, UsesCorrectDelayEstimateForHighLatencyOutputPath) { + EXPECT_EQ(kHighLatencyModeDelayEstimateInMilliseconds, + TestDelayOnAudioLayer(AudioDeviceModule::kAndroidJavaAudio)); +} + +TEST_F(AudioDeviceTest, UsesCorrectDelayEstimateForLowLatencyOutputPath) { + EXPECT_EQ(kLowLatencyModeDelayEstimateInMilliseconds, + TestDelayOnAudioLayer( + AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio)); +} + +// Ensure that the ADM internal audio device buffer is configured to use the +// correct set of parameters. +TEST_F(AudioDeviceTest, VerifyAudioDeviceBufferParameters) { + EXPECT_EQ(playout_parameters_.sample_rate(), + audio_device_buffer()->PlayoutSampleRate()); + EXPECT_EQ(record_parameters_.sample_rate(), + audio_device_buffer()->RecordingSampleRate()); + EXPECT_EQ(playout_parameters_.channels(), + audio_device_buffer()->PlayoutChannels()); + EXPECT_EQ(record_parameters_.channels(), + audio_device_buffer()->RecordingChannels()); +} + + +TEST_F(AudioDeviceTest, InitTerminate) { + // Initialization is part of the test fixture. + EXPECT_TRUE(audio_device()->Initialized()); + EXPECT_EQ(0, audio_device()->Terminate()); + EXPECT_FALSE(audio_device()->Initialized()); +} + +TEST_F(AudioDeviceTest, Devices) { + // Device enumeration is not supported. Verify fixed values only. + EXPECT_EQ(1, audio_device()->PlayoutDevices()); + EXPECT_EQ(1, audio_device()->RecordingDevices()); +} + +TEST_F(AudioDeviceTest, SpeakerVolumeShouldBeAvailable) { + // The OpenSL ES output audio path does not support volume control. + if (!AudioLayerSupportsVolumeControl()) + return; + bool available; + EXPECT_EQ(0, audio_device()->SpeakerVolumeIsAvailable(&available)); + EXPECT_TRUE(available); +} + +TEST_F(AudioDeviceTest, MaxSpeakerVolumeIsPositive) { + // The OpenSL ES output audio path does not support volume control. + if (!AudioLayerSupportsVolumeControl()) + return; + StartPlayout(); + EXPECT_GT(GetMaxSpeakerVolume(), 0); + StopPlayout(); +} + +TEST_F(AudioDeviceTest, MinSpeakerVolumeIsZero) { + // The OpenSL ES output audio path does not support volume control. + if (!AudioLayerSupportsVolumeControl()) + return; + EXPECT_EQ(GetMinSpeakerVolume(), 0); +} + +TEST_F(AudioDeviceTest, DefaultSpeakerVolumeIsWithinMinMax) { + // The OpenSL ES output audio path does not support volume control. + if (!AudioLayerSupportsVolumeControl()) + return; + const int default_volume = GetSpeakerVolume(); + EXPECT_GE(default_volume, GetMinSpeakerVolume()); + EXPECT_LE(default_volume, GetMaxSpeakerVolume()); +} + +TEST_F(AudioDeviceTest, SetSpeakerVolumeActuallySetsVolume) { + // The OpenSL ES output audio path does not support volume control. + if (!AudioLayerSupportsVolumeControl()) + return; + const int default_volume = GetSpeakerVolume(); + const int max_volume = GetMaxSpeakerVolume(); + EXPECT_EQ(0, audio_device()->SetSpeakerVolume(max_volume)); + int new_volume = GetSpeakerVolume(); + EXPECT_EQ(new_volume, max_volume); + EXPECT_EQ(0, audio_device()->SetSpeakerVolume(default_volume)); +} + +// Tests that playout can be initiated, started and stopped. No audio callback +// is registered in this test. +TEST_F(AudioDeviceTest, StartStopPlayout) { + StartPlayout(); + StopPlayout(); + StartPlayout(); + StopPlayout(); +} + +// Tests that recording can be initiated, started and stopped. No audio callback +// is registered in this test. +TEST_F(AudioDeviceTest, StartStopRecording) { + StartRecording(); + StopRecording(); + StartRecording(); + StopRecording(); +} + +// Verify that calling StopPlayout() will leave us in an uninitialized state +// which will require a new call to InitPlayout(). This test does not call +// StartPlayout() while being uninitialized since doing so will hit a +// RTC_DCHECK and death tests are not supported on Android. +TEST_F(AudioDeviceTest, StopPlayoutRequiresInitToRestart) { + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_EQ(0, audio_device()->StartPlayout()); + EXPECT_EQ(0, audio_device()->StopPlayout()); + EXPECT_FALSE(audio_device()->PlayoutIsInitialized()); +} + +// Verify that calling StopRecording() will leave us in an uninitialized state +// which will require a new call to InitRecording(). This test does not call +// StartRecording() while being uninitialized since doing so will hit a +// RTC_DCHECK and death tests are not supported on Android. +TEST_F(AudioDeviceTest, StopRecordingRequiresInitToRestart) { + EXPECT_EQ(0, audio_device()->InitRecording()); + EXPECT_EQ(0, audio_device()->StartRecording()); + EXPECT_EQ(0, audio_device()->StopRecording()); + EXPECT_FALSE(audio_device()->RecordingIsInitialized()); +} + +// Start playout and verify that the native audio layer starts asking for real +// audio samples to play out using the NeedMorePlayData callback. +TEST_F(AudioDeviceTest, StartPlayoutVerifyCallbacks) { + MockAudioTransportAndroid mock(kPlayout); + mock.HandleCallbacks(test_is_done_.get(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), + kBytesPerSample, + playout_channels(), + playout_sample_rate(), + NotNull(), + _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopPlayout(); +} + +// Start recording and verify that the native audio layer starts feeding real +// audio samples via the RecordedDataIsAvailable callback. +TEST_F(AudioDeviceTest, StartRecordingVerifyCallbacks) { + MockAudioTransportAndroid mock(kRecording); + mock.HandleCallbacks(test_is_done_.get(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, RecordedDataIsAvailable(NotNull(), + record_frames_per_10ms_buffer(), + kBytesPerSample, + record_channels(), + record_sample_rate(), + total_delay_ms(), + 0, + 0, + false, + _)) + .Times(AtLeast(kNumCallbacks)); + + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartRecording(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopRecording(); +} + + +// Start playout and recording (full-duplex audio) and verify that audio is +// active in both directions. +TEST_F(AudioDeviceTest, StartPlayoutAndRecordingVerifyCallbacks) { + MockAudioTransportAndroid mock(kPlayout | kRecording); + mock.HandleCallbacks(test_is_done_.get(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), + kBytesPerSample, + playout_channels(), + playout_sample_rate(), + NotNull(), + _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_CALL(mock, RecordedDataIsAvailable(NotNull(), + record_frames_per_10ms_buffer(), + kBytesPerSample, + record_channels(), + record_sample_rate(), + total_delay_ms(), + 0, + 0, + false, + _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + StartRecording(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopRecording(); + StopPlayout(); +} + +// Start playout and read audio from an external PCM file when the audio layer +// asks for data to play out. Real audio is played out in this test but it does +// not contain any explicit verification that the audio quality is perfect. +TEST_F(AudioDeviceTest, RunPlayoutWithFileAsSource) { + // TODO(henrika): extend test when mono output is supported. + EXPECT_EQ(1u, playout_channels()); + NiceMock mock(kPlayout); + const int num_callbacks = kFilePlayTimeInSec * kNumCallbacksPerSecond; + std::string file_name = GetFileName(playout_sample_rate()); + std::unique_ptr file_audio_stream( + new FileAudioStream(num_callbacks, file_name, playout_sample_rate())); + mock.HandleCallbacks(test_is_done_.get(), + file_audio_stream.get(), + num_callbacks); + // SetMaxPlayoutVolume(); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopPlayout(); +} + +// Start playout and recording and store recorded data in an intermediate FIFO +// buffer from which the playout side then reads its samples in the same order +// as they were stored. Under ideal circumstances, a callback sequence would +// look like: ...+-+-+-+-+-+-+-..., where '+' means 'packet recorded' and '-' +// means 'packet played'. Under such conditions, the FIFO would only contain +// one packet on average. However, under more realistic conditions, the size +// of the FIFO will vary more due to an unbalance between the two sides. +// This test tries to verify that the device maintains a balanced callback- +// sequence by running in loopback for ten seconds while measuring the size +// (max and average) of the FIFO. The size of the FIFO is increased by the +// recording side and decreased by the playout side. +// TODO(henrika): tune the final test parameters after running tests on several +// different devices. +// Disabling this test on bots since it is difficult to come up with a robust +// test condition that all worked as intended. The main issue is that, when +// swarming is used, an initial latency can be built up when the both sides +// starts at different times. Hence, the test can fail even if audio works +// as intended. Keeping the test so it can be enabled manually. +// http://bugs.webrtc.org/7744 +TEST_F(AudioDeviceTest, DISABLED_RunPlayoutAndRecordingInFullDuplex) { + EXPECT_EQ(record_channels(), playout_channels()); + EXPECT_EQ(record_sample_rate(), playout_sample_rate()); + NiceMock mock(kPlayout | kRecording); + std::unique_ptr fifo_audio_stream( + new FifoAudioStream(playout_frames_per_10ms_buffer())); + mock.HandleCallbacks(test_is_done_.get(), + fifo_audio_stream.get(), + kFullDuplexTimeInSec * kNumCallbacksPerSecond); + SetMaxPlayoutVolume(); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartRecording(); + StartPlayout(); + test_is_done_->Wait(std::max(kTestTimeOutInMilliseconds, + 1000 * kFullDuplexTimeInSec)); + StopPlayout(); + StopRecording(); + + // These thresholds are set rather high to accomodate differences in hardware + // in several devices, so this test can be used in swarming. + // See http://bugs.webrtc.org/6464 + EXPECT_LE(fifo_audio_stream->average_size(), 60u); + EXPECT_LE(fifo_audio_stream->largest_size(), 70u); +} + +// Measures loopback latency and reports the min, max and average values for +// a full duplex audio session. +// The latency is measured like so: +// - Insert impulses periodically on the output side. +// - Detect the impulses on the input side. +// - Measure the time difference between the transmit time and receive time. +// - Store time differences in a vector and calculate min, max and average. +// This test requires a special hardware called Audio Loopback Dongle. +// See http://source.android.com/devices/audio/loopback.html for details. +TEST_F(AudioDeviceTest, DISABLED_MeasureLoopbackLatency) { + EXPECT_EQ(record_channels(), playout_channels()); + EXPECT_EQ(record_sample_rate(), playout_sample_rate()); + NiceMock mock(kPlayout | kRecording); + std::unique_ptr latency_audio_stream( + new LatencyMeasuringAudioStream(playout_frames_per_10ms_buffer())); + mock.HandleCallbacks(test_is_done_.get(), + latency_audio_stream.get(), + kMeasureLatencyTimeInSec * kNumCallbacksPerSecond); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + SetMaxPlayoutVolume(); + DisableBuiltInAECIfAvailable(); + StartRecording(); + StartPlayout(); + test_is_done_->Wait(std::max(kTestTimeOutInMilliseconds, + 1000 * kMeasureLatencyTimeInSec)); + StopPlayout(); + StopRecording(); + // Verify that the correct number of transmitted impulses are detected. + EXPECT_EQ(latency_audio_stream->num_latency_values(), + static_cast( + kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1)); + latency_audio_stream->PrintResults(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.cc new file mode 100644 index 0000000000..6f385a3073 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.cc @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/audio_manager.h" + +#include + +#include + +#include "modules/audio_device/android/audio_common.h" +#include "modules/utility/include/helpers_android.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" + +#define TAG "AudioManager" +#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) +#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) +#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) + +namespace webrtc { + +// AudioManager::JavaAudioManager implementation +AudioManager::JavaAudioManager::JavaAudioManager( + NativeRegistration* native_reg, + std::unique_ptr audio_manager) + : audio_manager_(std::move(audio_manager)), + init_(native_reg->GetMethodId("init", "()Z")), + dispose_(native_reg->GetMethodId("dispose", "()V")), + is_communication_mode_enabled_( + native_reg->GetMethodId("isCommunicationModeEnabled", "()Z")), + is_device_blacklisted_for_open_sles_usage_( + native_reg->GetMethodId("isDeviceBlacklistedForOpenSLESUsage", + "()Z")) { + ALOGD("JavaAudioManager::ctor%s", GetThreadInfo().c_str()); +} + +AudioManager::JavaAudioManager::~JavaAudioManager() { + ALOGD("JavaAudioManager::dtor%s", GetThreadInfo().c_str()); +} + +bool AudioManager::JavaAudioManager::Init() { + return audio_manager_->CallBooleanMethod(init_); +} + +void AudioManager::JavaAudioManager::Close() { + audio_manager_->CallVoidMethod(dispose_); +} + +bool AudioManager::JavaAudioManager::IsCommunicationModeEnabled() { + return audio_manager_->CallBooleanMethod(is_communication_mode_enabled_); +} + +bool AudioManager::JavaAudioManager::IsDeviceBlacklistedForOpenSLESUsage() { + return audio_manager_->CallBooleanMethod( + is_device_blacklisted_for_open_sles_usage_); +} + +// AudioManager implementation +AudioManager::AudioManager() + : j_environment_(JVM::GetInstance()->environment()), + audio_layer_(AudioDeviceModule::kPlatformDefaultAudio), + initialized_(false), + hardware_aec_(false), + hardware_agc_(false), + hardware_ns_(false), + low_latency_playout_(false), + low_latency_record_(false), + delay_estimate_in_milliseconds_(0) { + ALOGD("ctor%s", GetThreadInfo().c_str()); + RTC_CHECK(j_environment_); + JNINativeMethod native_methods[] = { + {"nativeCacheAudioParameters", "(IIIZZZZZZIIJ)V", + reinterpret_cast(&webrtc::AudioManager::CacheAudioParameters)}}; + j_native_registration_ = j_environment_->RegisterNatives( + "org/webrtc/voiceengine/WebRtcAudioManager", native_methods, + arraysize(native_methods)); + j_audio_manager_.reset( + new JavaAudioManager(j_native_registration_.get(), + j_native_registration_->NewObject( + "", "(J)V", PointerTojlong(this)))); +} + +AudioManager::~AudioManager() { + ALOGD("~dtor%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Close(); +} + +void AudioManager::SetActiveAudioLayer( + AudioDeviceModule::AudioLayer audio_layer) { + ALOGD("SetActiveAudioLayer(%d)%s", audio_layer, GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!initialized_); + // Store the currently utilized audio layer. + audio_layer_ = audio_layer; + // The delay estimate can take one of two fixed values depending on if the + // device supports low-latency output or not. However, it is also possible + // that the user explicitly selects the high-latency audio path, hence we use + // the selected |audio_layer| here to set the delay estimate. + delay_estimate_in_milliseconds_ = + (audio_layer == AudioDeviceModule::kAndroidJavaAudio) + ? kHighLatencyModeDelayEstimateInMilliseconds + : kLowLatencyModeDelayEstimateInMilliseconds; + ALOGD("delay_estimate_in_milliseconds: %d", delay_estimate_in_milliseconds_); +} + +SLObjectItf AudioManager::GetOpenSLEngine() { + ALOGD("GetOpenSLEngine%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + // Only allow usage of OpenSL ES if such an audio layer has been specified. + if (audio_layer_ != AudioDeviceModule::kAndroidOpenSLESAudio && + audio_layer_ != + AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio) { + ALOGI("Unable to create OpenSL engine for the current audio layer: %d", + audio_layer_); + return nullptr; + } + // OpenSL ES for Android only supports a single engine per application. + // If one already has been created, return existing object instead of + // creating a new. + if (engine_object_.Get() != nullptr) { + ALOGI("The OpenSL ES engine object has already been created"); + return engine_object_.Get(); + } + // Create the engine object in thread safe mode. + const SLEngineOption option[] = { + {SL_ENGINEOPTION_THREADSAFE, static_cast(SL_BOOLEAN_TRUE)}}; + SLresult result = + slCreateEngine(engine_object_.Receive(), 1, option, 0, NULL, NULL); + if (result != SL_RESULT_SUCCESS) { + ALOGE("slCreateEngine() failed: %s", GetSLErrorString(result)); + engine_object_.Reset(); + return nullptr; + } + // Realize the SL Engine in synchronous mode. + result = engine_object_->Realize(engine_object_.Get(), SL_BOOLEAN_FALSE); + if (result != SL_RESULT_SUCCESS) { + ALOGE("Realize() failed: %s", GetSLErrorString(result)); + engine_object_.Reset(); + return nullptr; + } + // Finally return the SLObjectItf interface of the engine object. + return engine_object_.Get(); +} + +bool AudioManager::Init() { + ALOGD("Init%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!initialized_); + RTC_DCHECK_NE(audio_layer_, AudioDeviceModule::kPlatformDefaultAudio); + if (!j_audio_manager_->Init()) { + ALOGE("init failed!"); + return false; + } + initialized_ = true; + return true; +} + +bool AudioManager::Close() { + ALOGD("Close%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!initialized_) + return true; + j_audio_manager_->Close(); + initialized_ = false; + return true; +} + +bool AudioManager::IsCommunicationModeEnabled() const { + ALOGD("IsCommunicationModeEnabled()"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return j_audio_manager_->IsCommunicationModeEnabled(); +} + +bool AudioManager::IsAcousticEchoCancelerSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return hardware_aec_; +} + +bool AudioManager::IsAutomaticGainControlSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return hardware_agc_; +} + +bool AudioManager::IsNoiseSuppressorSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return hardware_ns_; +} + +bool AudioManager::IsLowLatencyPlayoutSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGD("IsLowLatencyPlayoutSupported()"); + // Some devices are blacklisted for usage of OpenSL ES even if they report + // that low-latency playout is supported. See b/21485703 for details. + return j_audio_manager_->IsDeviceBlacklistedForOpenSLESUsage() + ? false + : low_latency_playout_; +} + +bool AudioManager::IsLowLatencyRecordSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGD("IsLowLatencyRecordSupported()"); + return low_latency_record_; +} + +bool AudioManager::IsProAudioSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGD("IsProAudioSupported()"); + // TODO(henrika): return the state independently of if OpenSL ES is + // blacklisted or not for now. We could use the same approach as in + // IsLowLatencyPlayoutSupported() but I can't see the need for it yet. + return pro_audio_; +} + +bool AudioManager::IsStereoPlayoutSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGD("IsStereoPlayoutSupported()"); + return (playout_parameters_.channels() == 2); +} + +bool AudioManager::IsStereoRecordSupported() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGD("IsStereoRecordSupported()"); + return (record_parameters_.channels() == 2); +} + +int AudioManager::GetDelayEstimateInMilliseconds() const { + return delay_estimate_in_milliseconds_; +} + +void JNICALL AudioManager::CacheAudioParameters(JNIEnv* env, + jobject obj, + jint sample_rate, + jint output_channels, + jint input_channels, + jboolean hardware_aec, + jboolean hardware_agc, + jboolean hardware_ns, + jboolean low_latency_output, + jboolean low_latency_input, + jboolean pro_audio, + jint output_buffer_size, + jint input_buffer_size, + jlong native_audio_manager) { + webrtc::AudioManager* this_object = + reinterpret_cast(native_audio_manager); + this_object->OnCacheAudioParameters( + env, sample_rate, output_channels, input_channels, hardware_aec, + hardware_agc, hardware_ns, low_latency_output, low_latency_input, + pro_audio, output_buffer_size, input_buffer_size); +} + +void AudioManager::OnCacheAudioParameters(JNIEnv* env, + jint sample_rate, + jint output_channels, + jint input_channels, + jboolean hardware_aec, + jboolean hardware_agc, + jboolean hardware_ns, + jboolean low_latency_output, + jboolean low_latency_input, + jboolean pro_audio, + jint output_buffer_size, + jint input_buffer_size) { + ALOGD("OnCacheAudioParameters%s", GetThreadInfo().c_str()); + ALOGD("hardware_aec: %d", hardware_aec); + ALOGD("hardware_agc: %d", hardware_agc); + ALOGD("hardware_ns: %d", hardware_ns); + ALOGD("low_latency_output: %d", low_latency_output); + ALOGD("low_latency_input: %d", low_latency_input); + ALOGD("pro_audio: %d", pro_audio); + ALOGD("sample_rate: %d", sample_rate); + ALOGD("output_channels: %d", output_channels); + ALOGD("input_channels: %d", input_channels); + ALOGD("output_buffer_size: %d", output_buffer_size); + ALOGD("input_buffer_size: %d", input_buffer_size); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + hardware_aec_ = hardware_aec; + hardware_agc_ = hardware_agc; + hardware_ns_ = hardware_ns; + low_latency_playout_ = low_latency_output; + low_latency_record_ = low_latency_input; + pro_audio_ = pro_audio; + playout_parameters_.reset(sample_rate, static_cast(output_channels), + static_cast(output_buffer_size)); + record_parameters_.reset(sample_rate, static_cast(input_channels), + static_cast(input_buffer_size)); +} + +const AudioParameters& AudioManager::GetPlayoutAudioParameters() { + RTC_CHECK(playout_parameters_.is_valid()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return playout_parameters_; +} + +const AudioParameters& AudioManager::GetRecordAudioParameters() { + RTC_CHECK(record_parameters_.is_valid()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return record_parameters_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.h new file mode 100644 index 0000000000..dd6b51ff90 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager.h @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_AUDIO_MANAGER_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_AUDIO_MANAGER_H_ + +#include + +#include +#include + +#include "modules/audio_device/android/audio_common.h" +#include "modules/audio_device/android/opensles_common.h" +#include "modules/audio_device/audio_device_config.h" +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/include/audio_device_defines.h" +#include "modules/utility/include/helpers_android.h" +#include "modules/utility/include/jvm_android.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +// Implements support for functions in the WebRTC audio stack for Android that +// relies on the AudioManager in android.media. It also populates an +// AudioParameter structure with native audio parameters detected at +// construction. This class does not make any audio-related modifications +// unless Init() is called. Caching audio parameters makes no changes but only +// reads data from the Java side. +class AudioManager { + public: + // Wraps the Java specific parts of the AudioManager into one helper class. + // Stores method IDs for all supported methods at construction and then + // allows calls like JavaAudioManager::Close() while hiding the Java/JNI + // parts that are associated with this call. + class JavaAudioManager { + public: + JavaAudioManager(NativeRegistration* native_registration, + std::unique_ptr audio_manager); + ~JavaAudioManager(); + + bool Init(); + void Close(); + bool IsCommunicationModeEnabled(); + bool IsDeviceBlacklistedForOpenSLESUsage(); + + private: + std::unique_ptr audio_manager_; + jmethodID init_; + jmethodID dispose_; + jmethodID is_communication_mode_enabled_; + jmethodID is_device_blacklisted_for_open_sles_usage_; + }; + + AudioManager(); + ~AudioManager(); + + // Sets the currently active audio layer combination. Must be called before + // Init(). + void SetActiveAudioLayer(AudioDeviceModule::AudioLayer audio_layer); + + // Creates and realizes the main (global) Open SL engine object and returns + // a reference to it. The engine object is only created at the first call + // since OpenSL ES for Android only supports a single engine per application. + // Subsequent calls returns the already created engine. The SL engine object + // is destroyed when the AudioManager object is deleted. It means that the + // engine object will be the first OpenSL ES object to be created and last + // object to be destroyed. + // Note that NULL will be returned unless the audio layer is specified as + // AudioDeviceModule::kAndroidOpenSLESAudio or + // AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio. + SLObjectItf GetOpenSLEngine(); + + // Initializes the audio manager and stores the current audio mode. + bool Init(); + // Revert any setting done by Init(). + bool Close(); + + // Returns true if current audio mode is AudioManager.MODE_IN_COMMUNICATION. + bool IsCommunicationModeEnabled() const; + + // Native audio parameters stored during construction. + const AudioParameters& GetPlayoutAudioParameters(); + const AudioParameters& GetRecordAudioParameters(); + + // Returns true if the device supports built-in audio effects for AEC, AGC + // and NS. Some devices can also be blacklisted for use in combination with + // platform effects and these devices will return false. + // Can currently only be used in combination with a Java based audio backend + // for the recoring side (i.e. using the android.media.AudioRecord API). + bool IsAcousticEchoCancelerSupported() const; + bool IsAutomaticGainControlSupported() const; + bool IsNoiseSuppressorSupported() const; + + // Returns true if the device supports the low-latency audio paths in + // combination with OpenSL ES. + bool IsLowLatencyPlayoutSupported() const; + bool IsLowLatencyRecordSupported() const; + + // Returns true if the device supports (and has been configured for) stereo. + // Call the Java API WebRtcAudioManager.setStereoOutput/Input() with true as + // paramter to enable stereo. Default is mono in both directions and the + // setting is set once and for all when the audio manager object is created. + // TODO(henrika): stereo is not supported in combination with OpenSL ES. + bool IsStereoPlayoutSupported() const; + bool IsStereoRecordSupported() const; + + // Returns true if the device supports pro-audio features in combination with + // OpenSL ES. + bool IsProAudioSupported() const; + + // Returns the estimated total delay of this device. Unit is in milliseconds. + // The vaule is set once at construction and never changes after that. + // Possible values are webrtc::kLowLatencyModeDelayEstimateInMilliseconds and + // webrtc::kHighLatencyModeDelayEstimateInMilliseconds. + int GetDelayEstimateInMilliseconds() const; + + private: + // Called from Java side so we can cache the native audio parameters. + // This method will be called by the WebRtcAudioManager constructor, i.e. + // on the same thread that this object is created on. + static void JNICALL CacheAudioParameters(JNIEnv* env, + jobject obj, + jint sample_rate, + jint output_channels, + jint input_channels, + jboolean hardware_aec, + jboolean hardware_agc, + jboolean hardware_ns, + jboolean low_latency_output, + jboolean low_latency_input, + jboolean pro_audio, + jint output_buffer_size, + jint input_buffer_size, + jlong native_audio_manager); + void OnCacheAudioParameters(JNIEnv* env, + jint sample_rate, + jint output_channels, + jint input_channels, + jboolean hardware_aec, + jboolean hardware_agc, + jboolean hardware_ns, + jboolean low_latency_output, + jboolean low_latency_input, + jboolean pro_audio, + jint output_buffer_size, + jint input_buffer_size); + + // Stores thread ID in the constructor. + // We can then use ThreadChecker::CalledOnValidThread() to ensure that + // other methods are called from the same thread. + rtc::ThreadChecker thread_checker_; + + // Calls AttachCurrentThread() if this thread is not attached at construction. + // Also ensures that DetachCurrentThread() is called at destruction. + AttachCurrentThreadIfNeeded attach_thread_if_needed_; + + // Wraps the JNI interface pointer and methods associated with it. + std::unique_ptr j_environment_; + + // Contains factory method for creating the Java object. + std::unique_ptr j_native_registration_; + + // Wraps the Java specific parts of the AudioManager. + std::unique_ptr j_audio_manager_; + + // Contains the selected audio layer specified by the AudioLayer enumerator + // in the AudioDeviceModule class. + AudioDeviceModule::AudioLayer audio_layer_; + + // This object is the global entry point of the OpenSL ES API. + // After creating the engine object, the application can obtain this object‘s + // SLEngineItf interface. This interface contains creation methods for all + // the other object types in the API. None of these interface are realized + // by this class. It only provides access to the global engine object. + webrtc::ScopedSLObjectItf engine_object_; + + // Set to true by Init() and false by Close(). + bool initialized_; + + // True if device supports hardware (or built-in) AEC. + bool hardware_aec_; + // True if device supports hardware (or built-in) AGC. + bool hardware_agc_; + // True if device supports hardware (or built-in) NS. + bool hardware_ns_; + + // True if device supports the low-latency OpenSL ES audio path for output. + bool low_latency_playout_; + + // True if device supports the low-latency OpenSL ES audio path for input. + bool low_latency_record_; + + // True if device supports the low-latency OpenSL ES pro-audio path. + bool pro_audio_; + + // The delay estimate can take one of two fixed values depending on if the + // device supports low-latency output or not. + int delay_estimate_in_milliseconds_; + + // Contains native parameters (e.g. sample rate, channel configuration). + // Set at construction in OnCacheAudioParameters() which is called from + // Java on the same thread as this object is created on. + AudioParameters playout_parameters_; + AudioParameters record_parameters_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_AUDIO_MANAGER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager_unittest.cc new file mode 100644 index 0000000000..00314ebfeb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_manager_unittest.cc @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/android/build_info.h" +#include "modules/audio_device/android/ensure_initialized.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/format_macros.h" +#include "test/gtest.h" + +#define PRINT(...) fprintf(stderr, __VA_ARGS__); + +namespace webrtc { + +static const char kTag[] = " "; + +class AudioManagerTest : public ::testing::Test { + protected: + AudioManagerTest() { + // One-time initialization of JVM and application context. Ensures that we + // can do calls between C++ and Java. + webrtc::audiodevicemodule::EnsureInitialized(); + audio_manager_.reset(new AudioManager()); + SetActiveAudioLayer(); + playout_parameters_ = audio_manager()->GetPlayoutAudioParameters(); + record_parameters_ = audio_manager()->GetRecordAudioParameters(); + } + + AudioManager* audio_manager() const { return audio_manager_.get(); } + + // A valid audio layer must always be set before calling Init(), hence we + // might as well make it a part of the test fixture. + void SetActiveAudioLayer() { + EXPECT_EQ(0, audio_manager()->GetDelayEstimateInMilliseconds()); + audio_manager()->SetActiveAudioLayer(AudioDeviceModule::kAndroidJavaAudio); + EXPECT_NE(0, audio_manager()->GetDelayEstimateInMilliseconds()); + } + + // One way to ensure that the engine object is valid is to create an + // SL Engine interface since it exposes creation methods of all the OpenSL ES + // object types and it is only supported on the engine object. This method + // also verifies that the engine interface supports at least one interface. + // Note that, the test below is not a full test of the SLEngineItf object + // but only a simple sanity test to check that the global engine object is OK. + void ValidateSLEngine(SLObjectItf engine_object) { + EXPECT_NE(nullptr, engine_object); + // Get the SL Engine interface which is exposed by the engine object. + SLEngineItf engine; + SLresult result = + (*engine_object)->GetInterface(engine_object, SL_IID_ENGINE, &engine); + EXPECT_EQ(result, SL_RESULT_SUCCESS) << "GetInterface() on engine failed"; + // Ensure that the SL Engine interface exposes at least one interface. + SLuint32 object_id = SL_OBJECTID_ENGINE; + SLuint32 num_supported_interfaces = 0; + result = (*engine)->QueryNumSupportedInterfaces(engine, object_id, + &num_supported_interfaces); + EXPECT_EQ(result, SL_RESULT_SUCCESS) + << "QueryNumSupportedInterfaces() failed"; + EXPECT_GE(num_supported_interfaces, 1u); + } + + std::unique_ptr audio_manager_; + AudioParameters playout_parameters_; + AudioParameters record_parameters_; +}; + +TEST_F(AudioManagerTest, ConstructDestruct) { +} + +// It should not be possible to create an OpenSL engine object if Java based +// audio is requested in both directions. +TEST_F(AudioManagerTest, GetOpenSLEngineShouldFailForJavaAudioLayer) { + audio_manager()->SetActiveAudioLayer(AudioDeviceModule::kAndroidJavaAudio); + SLObjectItf engine_object = audio_manager()->GetOpenSLEngine(); + EXPECT_EQ(nullptr, engine_object); +} + +// It should be possible to create an OpenSL engine object if OpenSL ES based +// audio is requested in any direction. +TEST_F(AudioManagerTest, GetOpenSLEngineShouldSucceedForOpenSLESAudioLayer) { + // List of supported audio layers that uses OpenSL ES audio. + const AudioDeviceModule::AudioLayer opensles_audio[] = { + AudioDeviceModule::kAndroidOpenSLESAudio, + AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio}; + // Verify that the global (singleton) OpenSL Engine can be acquired for all + // audio layes that uses OpenSL ES. Note that the engine is only created once. + for (const AudioDeviceModule::AudioLayer audio_layer : opensles_audio) { + audio_manager()->SetActiveAudioLayer(audio_layer); + SLObjectItf engine_object = audio_manager()->GetOpenSLEngine(); + EXPECT_NE(nullptr, engine_object); + // Perform a simple sanity check of the created engine object. + ValidateSLEngine(engine_object); + } +} + +TEST_F(AudioManagerTest, InitClose) { + EXPECT_TRUE(audio_manager()->Init()); + EXPECT_TRUE(audio_manager()->Close()); +} + +TEST_F(AudioManagerTest, IsAcousticEchoCancelerSupported) { + PRINT("%sAcoustic Echo Canceler support: %s\n", kTag, + audio_manager()->IsAcousticEchoCancelerSupported() ? "Yes" : "No"); +} + +TEST_F(AudioManagerTest, IsAutomaticGainControlSupported) { + EXPECT_FALSE(audio_manager()->IsAutomaticGainControlSupported()); +} + +TEST_F(AudioManagerTest, IsNoiseSuppressorSupported) { + PRINT("%sNoise Suppressor support: %s\n", kTag, + audio_manager()->IsNoiseSuppressorSupported() ? "Yes" : "No"); +} + +TEST_F(AudioManagerTest, IsLowLatencyPlayoutSupported) { + PRINT("%sLow latency output support: %s\n", kTag, + audio_manager()->IsLowLatencyPlayoutSupported() ? "Yes" : "No"); +} + +TEST_F(AudioManagerTest, IsLowLatencyRecordSupported) { + PRINT("%sLow latency input support: %s\n", kTag, + audio_manager()->IsLowLatencyRecordSupported() ? "Yes" : "No"); +} + +TEST_F(AudioManagerTest, IsProAudioSupported) { + PRINT("%sPro audio support: %s\n", kTag, + audio_manager()->IsProAudioSupported() ? "Yes" : "No"); +} + +// Verify that playout side is configured for mono by default. +TEST_F(AudioManagerTest, IsStereoPlayoutSupported) { + EXPECT_FALSE(audio_manager()->IsStereoPlayoutSupported()); +} + +// Verify that recording side is configured for mono by default. +TEST_F(AudioManagerTest, IsStereoRecordSupported) { + EXPECT_FALSE(audio_manager()->IsStereoRecordSupported()); +} + +TEST_F(AudioManagerTest, ShowAudioParameterInfo) { + const bool low_latency_out = audio_manager()->IsLowLatencyPlayoutSupported(); + const bool low_latency_in = audio_manager()->IsLowLatencyRecordSupported(); + PRINT("PLAYOUT:\n"); + PRINT("%saudio layer: %s\n", kTag, + low_latency_out ? "Low latency OpenSL" : "Java/JNI based AudioTrack"); + PRINT("%ssample rate: %d Hz\n", kTag, playout_parameters_.sample_rate()); + PRINT("%schannels: %" PRIuS "\n", kTag, playout_parameters_.channels()); + PRINT("%sframes per buffer: %" PRIuS " <=> %.2f ms\n", kTag, + playout_parameters_.frames_per_buffer(), + playout_parameters_.GetBufferSizeInMilliseconds()); + PRINT("RECORD: \n"); + PRINT("%saudio layer: %s\n", kTag, + low_latency_in ? "Low latency OpenSL" : "Java/JNI based AudioRecord"); + PRINT("%ssample rate: %d Hz\n", kTag, record_parameters_.sample_rate()); + PRINT("%schannels: %" PRIuS "\n", kTag, record_parameters_.channels()); + PRINT("%sframes per buffer: %" PRIuS " <=> %.2f ms\n", kTag, + record_parameters_.frames_per_buffer(), + record_parameters_.GetBufferSizeInMilliseconds()); +} + +// The audio device module only suppors the same sample rate in both directions. +// In addition, in full-duplex low-latency mode (OpenSL ES), both input and +// output must use the same native buffer size to allow for usage of the fast +// audio track in Android. +TEST_F(AudioManagerTest, VerifyAudioParameters) { + const bool low_latency_out = audio_manager()->IsLowLatencyPlayoutSupported(); + const bool low_latency_in = audio_manager()->IsLowLatencyRecordSupported(); + EXPECT_EQ(playout_parameters_.sample_rate(), + record_parameters_.sample_rate()); + if (low_latency_out && low_latency_in) { + EXPECT_EQ(playout_parameters_.frames_per_buffer(), + record_parameters_.frames_per_buffer()); + } +} + +// Add device-specific information to the test for logging purposes. +TEST_F(AudioManagerTest, ShowDeviceInfo) { + BuildInfo build_info; + PRINT("%smodel: %s\n", kTag, build_info.GetDeviceModel().c_str()); + PRINT("%sbrand: %s\n", kTag, build_info.GetBrand().c_str()); + PRINT("%smanufacturer: %s\n", + kTag, build_info.GetDeviceManufacturer().c_str()); +} + +// Add Android build information to the test for logging purposes. +TEST_F(AudioManagerTest, ShowBuildInfo) { + BuildInfo build_info; + PRINT("%sbuild release: %s\n", kTag, build_info.GetBuildRelease().c_str()); + PRINT("%sbuild id: %s\n", kTag, build_info.GetAndroidBuildId().c_str()); + PRINT("%sbuild type: %s\n", kTag, build_info.GetBuildType().c_str()); + PRINT("%sSDK version: %d\n", kTag, build_info.GetSdkVersion()); +} + +// Basic test of the AudioParameters class using default construction where +// all members are set to zero. +TEST_F(AudioManagerTest, AudioParametersWithDefaultConstruction) { + AudioParameters params; + EXPECT_FALSE(params.is_valid()); + EXPECT_EQ(0, params.sample_rate()); + EXPECT_EQ(0U, params.channels()); + EXPECT_EQ(0U, params.frames_per_buffer()); + EXPECT_EQ(0U, params.frames_per_10ms_buffer()); + EXPECT_EQ(0U, params.GetBytesPerFrame()); + EXPECT_EQ(0U, params.GetBytesPerBuffer()); + EXPECT_EQ(0U, params.GetBytesPer10msBuffer()); + EXPECT_EQ(0.0f, params.GetBufferSizeInMilliseconds()); +} + +// Basic test of the AudioParameters class using non default construction. +TEST_F(AudioManagerTest, AudioParametersWithNonDefaultConstruction) { + const int kSampleRate = 48000; + const size_t kChannels = 1; + const size_t kFramesPerBuffer = 480; + const size_t kFramesPer10msBuffer = 480; + const size_t kBytesPerFrame = 2; + const float kBufferSizeInMs = 10.0f; + AudioParameters params(kSampleRate, kChannels, kFramesPerBuffer); + EXPECT_TRUE(params.is_valid()); + EXPECT_EQ(kSampleRate, params.sample_rate()); + EXPECT_EQ(kChannels, params.channels()); + EXPECT_EQ(kFramesPerBuffer, params.frames_per_buffer()); + EXPECT_EQ(static_cast(kSampleRate / 100), + params.frames_per_10ms_buffer()); + EXPECT_EQ(kBytesPerFrame, params.GetBytesPerFrame()); + EXPECT_EQ(kBytesPerFrame * kFramesPerBuffer, params.GetBytesPerBuffer()); + EXPECT_EQ(kBytesPerFrame * kFramesPer10msBuffer, + params.GetBytesPer10msBuffer()); + EXPECT_EQ(kBufferSizeInMs, params.GetBufferSizeInMilliseconds()); +} + +} // namespace webrtc + diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.cc new file mode 100644 index 0000000000..79f8c8b0f9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.cc @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/audio_record_jni.h" + +#include + +#include + +#include "modules/audio_device/android/audio_common.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" + +#define TAG "AudioRecordJni" +#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) +#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) +#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) + +namespace webrtc { + +// AudioRecordJni::JavaAudioRecord implementation. +AudioRecordJni::JavaAudioRecord::JavaAudioRecord( + NativeRegistration* native_reg, + std::unique_ptr audio_record) + : audio_record_(std::move(audio_record)), + init_recording_(native_reg->GetMethodId("initRecording", "(II)I")), + start_recording_(native_reg->GetMethodId("startRecording", "()Z")), + stop_recording_(native_reg->GetMethodId("stopRecording", "()Z")), + enable_built_in_aec_(native_reg->GetMethodId("enableBuiltInAEC", "(Z)Z")), + enable_built_in_ns_(native_reg->GetMethodId("enableBuiltInNS", "(Z)Z")) {} + +AudioRecordJni::JavaAudioRecord::~JavaAudioRecord() {} + +int AudioRecordJni::JavaAudioRecord::InitRecording(int sample_rate, + size_t channels) { + return audio_record_->CallIntMethod(init_recording_, + static_cast(sample_rate), + static_cast(channels)); +} + +bool AudioRecordJni::JavaAudioRecord::StartRecording() { + return audio_record_->CallBooleanMethod(start_recording_); +} + +bool AudioRecordJni::JavaAudioRecord::StopRecording() { + return audio_record_->CallBooleanMethod(stop_recording_); +} + +bool AudioRecordJni::JavaAudioRecord::EnableBuiltInAEC(bool enable) { + return audio_record_->CallBooleanMethod(enable_built_in_aec_, + static_cast(enable)); +} + +bool AudioRecordJni::JavaAudioRecord::EnableBuiltInNS(bool enable) { + return audio_record_->CallBooleanMethod(enable_built_in_ns_, + static_cast(enable)); +} + +// AudioRecordJni implementation. +AudioRecordJni::AudioRecordJni(AudioManager* audio_manager) + : j_environment_(JVM::GetInstance()->environment()), + audio_manager_(audio_manager), + audio_parameters_(audio_manager->GetRecordAudioParameters()), + total_delay_in_milliseconds_(0), + direct_buffer_address_(nullptr), + direct_buffer_capacity_in_bytes_(0), + frames_per_buffer_(0), + initialized_(false), + recording_(false), + audio_device_buffer_(nullptr) { + ALOGD("ctor%s", GetThreadInfo().c_str()); + RTC_DCHECK(audio_parameters_.is_valid()); + RTC_CHECK(j_environment_); + JNINativeMethod native_methods[] = { + {"nativeCacheDirectBufferAddress", "(Ljava/nio/ByteBuffer;J)V", + reinterpret_cast( + &webrtc::AudioRecordJni::CacheDirectBufferAddress)}, + {"nativeDataIsRecorded", "(IJ)V", + reinterpret_cast(&webrtc::AudioRecordJni::DataIsRecorded)}}; + j_native_registration_ = j_environment_->RegisterNatives( + "org/webrtc/voiceengine/WebRtcAudioRecord", native_methods, + arraysize(native_methods)); + j_audio_record_.reset( + new JavaAudioRecord(j_native_registration_.get(), + j_native_registration_->NewObject( + "", "(J)V", PointerTojlong(this)))); + // Detach from this thread since we want to use the checker to verify calls + // from the Java based audio thread. + thread_checker_java_.DetachFromThread(); +} + +AudioRecordJni::~AudioRecordJni() { + ALOGD("~dtor%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Terminate(); +} + +int32_t AudioRecordJni::Init() { + ALOGD("Init%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return 0; +} + +int32_t AudioRecordJni::Terminate() { + ALOGD("Terminate%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + StopRecording(); + return 0; +} + +int32_t AudioRecordJni::InitRecording() { + ALOGD("InitRecording%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!initialized_); + RTC_DCHECK(!recording_); + int frames_per_buffer = j_audio_record_->InitRecording( + audio_parameters_.sample_rate(), audio_parameters_.channels()); + if (frames_per_buffer < 0) { + ALOGE("InitRecording failed!"); + return -1; + } + frames_per_buffer_ = static_cast(frames_per_buffer); + ALOGD("frames_per_buffer: %" PRIuS, frames_per_buffer_); + const size_t bytes_per_frame = audio_parameters_.channels() * sizeof(int16_t); + RTC_CHECK_EQ(direct_buffer_capacity_in_bytes_, + frames_per_buffer_ * bytes_per_frame); + RTC_CHECK_EQ(frames_per_buffer_, audio_parameters_.frames_per_10ms_buffer()); + initialized_ = true; + return 0; +} + +int32_t AudioRecordJni::StartRecording() { + ALOGD("StartRecording%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(initialized_); + RTC_DCHECK(!recording_); + if (!j_audio_record_->StartRecording()) { + ALOGE("StartRecording failed!"); + return -1; + } + recording_ = true; + return 0; +} + +int32_t AudioRecordJni::StopRecording() { + ALOGD("StopRecording%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!initialized_ || !recording_) { + return 0; + } + if (!j_audio_record_->StopRecording()) { + ALOGE("StopRecording failed!"); + return -1; + } + // If we don't detach here, we will hit a RTC_DCHECK in OnDataIsRecorded() + // next time StartRecording() is called since it will create a new Java + // thread. + thread_checker_java_.DetachFromThread(); + initialized_ = false; + recording_ = false; + direct_buffer_address_ = nullptr; + return 0; +} + +void AudioRecordJni::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + ALOGD("AttachAudioBuffer"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + audio_device_buffer_ = audioBuffer; + const int sample_rate_hz = audio_parameters_.sample_rate(); + ALOGD("SetRecordingSampleRate(%d)", sample_rate_hz); + audio_device_buffer_->SetRecordingSampleRate(sample_rate_hz); + const size_t channels = audio_parameters_.channels(); + ALOGD("SetRecordingChannels(%" PRIuS ")", channels); + audio_device_buffer_->SetRecordingChannels(channels); + total_delay_in_milliseconds_ = + audio_manager_->GetDelayEstimateInMilliseconds(); + RTC_DCHECK_GT(total_delay_in_milliseconds_, 0); + ALOGD("total_delay_in_milliseconds: %d", total_delay_in_milliseconds_); +} + +int32_t AudioRecordJni::EnableBuiltInAEC(bool enable) { + ALOGD("EnableBuiltInAEC%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return j_audio_record_->EnableBuiltInAEC(enable) ? 0 : -1; +} + +int32_t AudioRecordJni::EnableBuiltInAGC(bool enable) { + // TODO(henrika): possibly remove when no longer used by any client. + FATAL() << "Should never be called"; + return -1; +} + +int32_t AudioRecordJni::EnableBuiltInNS(bool enable) { + ALOGD("EnableBuiltInNS%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return j_audio_record_->EnableBuiltInNS(enable) ? 0 : -1; +} + +void JNICALL AudioRecordJni::CacheDirectBufferAddress(JNIEnv* env, + jobject obj, + jobject byte_buffer, + jlong nativeAudioRecord) { + webrtc::AudioRecordJni* this_object = + reinterpret_cast(nativeAudioRecord); + this_object->OnCacheDirectBufferAddress(env, byte_buffer); +} + +void AudioRecordJni::OnCacheDirectBufferAddress(JNIEnv* env, + jobject byte_buffer) { + ALOGD("OnCacheDirectBufferAddress"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!direct_buffer_address_); + direct_buffer_address_ = env->GetDirectBufferAddress(byte_buffer); + jlong capacity = env->GetDirectBufferCapacity(byte_buffer); + ALOGD("direct buffer capacity: %lld", capacity); + direct_buffer_capacity_in_bytes_ = static_cast(capacity); +} + +void JNICALL AudioRecordJni::DataIsRecorded(JNIEnv* env, + jobject obj, + jint length, + jlong nativeAudioRecord) { + webrtc::AudioRecordJni* this_object = + reinterpret_cast(nativeAudioRecord); + this_object->OnDataIsRecorded(length); +} + +// This method is called on a high-priority thread from Java. The name of +// the thread is 'AudioRecordThread'. +void AudioRecordJni::OnDataIsRecorded(int length) { + RTC_DCHECK(thread_checker_java_.CalledOnValidThread()); + if (!audio_device_buffer_) { + ALOGE("AttachAudioBuffer has not been called!"); + return; + } + audio_device_buffer_->SetRecordedBuffer(direct_buffer_address_, + frames_per_buffer_); + // We provide one (combined) fixed delay estimate for the APM and use the + // |playDelayMs| parameter only. Components like the AEC only sees the sum + // of |playDelayMs| and |recDelayMs|, hence the distributions does not matter. + audio_device_buffer_->SetVQEData(total_delay_in_milliseconds_, + 0, // recDelayMs + 0); // clockDrift + if (audio_device_buffer_->DeliverRecordedData() == -1) { + ALOGE("AudioDeviceBuffer::DeliverRecordedData failed!"); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.h new file mode 100644 index 0000000000..f508280ad0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_record_jni.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_AUDIO_RECORD_JNI_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_AUDIO_RECORD_JNI_H_ + +#include + +#include + +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/include/audio_device_defines.h" +#include "modules/utility/include/helpers_android.h" +#include "modules/utility/include/jvm_android.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +// Implements 16-bit mono PCM audio input support for Android using the Java +// AudioRecord interface. Most of the work is done by its Java counterpart in +// WebRtcAudioRecord.java. This class is created and lives on a thread in +// C++-land, but recorded audio buffers are delivered on a high-priority +// thread managed by the Java class. +// +// The Java class makes use of AudioEffect features (mainly AEC) which are +// first available in Jelly Bean. If it is instantiated running against earlier +// SDKs, the AEC provided by the APM in WebRTC must be used and enabled +// separately instead. +// +// An instance must be created and destroyed on one and the same thread. +// All public methods must also be called on the same thread. A thread checker +// will RTC_DCHECK if any method is called on an invalid thread. +// +// This class uses AttachCurrentThreadIfNeeded to attach to a Java VM if needed +// and detach when the object goes out of scope. Additional thread checking +// guarantees that no other (possibly non attached) thread is used. +class AudioRecordJni { + public: + // Wraps the Java specific parts of the AudioRecordJni into one helper class. + class JavaAudioRecord { + public: + JavaAudioRecord(NativeRegistration* native_registration, + std::unique_ptr audio_track); + ~JavaAudioRecord(); + + int InitRecording(int sample_rate, size_t channels); + bool StartRecording(); + bool StopRecording(); + bool EnableBuiltInAEC(bool enable); + bool EnableBuiltInNS(bool enable); + + private: + std::unique_ptr audio_record_; + jmethodID init_recording_; + jmethodID start_recording_; + jmethodID stop_recording_; + jmethodID enable_built_in_aec_; + jmethodID enable_built_in_ns_; + }; + + explicit AudioRecordJni(AudioManager* audio_manager); + ~AudioRecordJni(); + + int32_t Init(); + int32_t Terminate(); + + int32_t InitRecording(); + bool RecordingIsInitialized() const { return initialized_; } + + int32_t StartRecording(); + int32_t StopRecording(); + bool Recording() const { return recording_; } + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); + + int32_t EnableBuiltInAEC(bool enable); + int32_t EnableBuiltInAGC(bool enable); + int32_t EnableBuiltInNS(bool enable); + + private: + // Called from Java side so we can cache the address of the Java-manged + // |byte_buffer| in |direct_buffer_address_|. The size of the buffer + // is also stored in |direct_buffer_capacity_in_bytes_|. + // This method will be called by the WebRtcAudioRecord constructor, i.e., + // on the same thread that this object is created on. + static void JNICALL CacheDirectBufferAddress( + JNIEnv* env, jobject obj, jobject byte_buffer, jlong nativeAudioRecord); + void OnCacheDirectBufferAddress(JNIEnv* env, jobject byte_buffer); + + // Called periodically by the Java based WebRtcAudioRecord object when + // recording has started. Each call indicates that there are |length| new + // bytes recorded in the memory area |direct_buffer_address_| and it is + // now time to send these to the consumer. + // This method is called on a high-priority thread from Java. The name of + // the thread is 'AudioRecordThread'. + static void JNICALL DataIsRecorded( + JNIEnv* env, jobject obj, jint length, jlong nativeAudioRecord); + void OnDataIsRecorded(int length); + + // Stores thread ID in constructor. + rtc::ThreadChecker thread_checker_; + + // Stores thread ID in first call to OnDataIsRecorded() from high-priority + // thread in Java. Detached during construction of this object. + rtc::ThreadChecker thread_checker_java_; + + // Calls AttachCurrentThread() if this thread is not attached at construction. + // Also ensures that DetachCurrentThread() is called at destruction. + AttachCurrentThreadIfNeeded attach_thread_if_needed_; + + // Wraps the JNI interface pointer and methods associated with it. + std::unique_ptr j_environment_; + + // Contains factory method for creating the Java object. + std::unique_ptr j_native_registration_; + + // Wraps the Java specific parts of the AudioRecordJni class. + std::unique_ptr j_audio_record_; + + // Raw pointer to the audio manger. + const AudioManager* audio_manager_; + + // Contains audio parameters provided to this class at construction by the + // AudioManager. + const AudioParameters audio_parameters_; + + // Delay estimate of the total round-trip delay (input + output). + // Fixed value set once in AttachAudioBuffer() and it can take one out of two + // possible values. See audio_common.h for details. + int total_delay_in_milliseconds_; + + // Cached copy of address to direct audio buffer owned by |j_audio_record_|. + void* direct_buffer_address_; + + // Number of bytes in the direct audio buffer owned by |j_audio_record_|. + size_t direct_buffer_capacity_in_bytes_; + + // Number audio frames per audio buffer. Each audio frame corresponds to + // one sample of PCM mono data at 16 bits per sample. Hence, each audio + // frame contains 2 bytes (given that the Java layer only supports mono). + // Example: 480 for 48000 Hz or 441 for 44100 Hz. + size_t frames_per_buffer_; + + bool initialized_; + + bool recording_; + + // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the + // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). + AudioDeviceBuffer* audio_device_buffer_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_AUDIO_RECORD_JNI_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.cc new file mode 100644 index 0000000000..89d4af0508 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.cc @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/audio_track_jni.h" +#include "modules/audio_device/android/audio_manager.h" + +#include + +#include + +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" + +#define TAG "AudioTrackJni" +#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) +#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) +#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) + +namespace webrtc { + +// AudioTrackJni::JavaAudioTrack implementation. +AudioTrackJni::JavaAudioTrack::JavaAudioTrack( + NativeRegistration* native_reg, + std::unique_ptr audio_track) + : audio_track_(std::move(audio_track)), + init_playout_(native_reg->GetMethodId("initPlayout", "(II)Z")), + start_playout_(native_reg->GetMethodId("startPlayout", "()Z")), + stop_playout_(native_reg->GetMethodId("stopPlayout", "()Z")), + set_stream_volume_(native_reg->GetMethodId("setStreamVolume", "(I)Z")), + get_stream_max_volume_( + native_reg->GetMethodId("getStreamMaxVolume", "()I")), + get_stream_volume_(native_reg->GetMethodId("getStreamVolume", "()I")) {} + +AudioTrackJni::JavaAudioTrack::~JavaAudioTrack() {} + +bool AudioTrackJni::JavaAudioTrack::InitPlayout(int sample_rate, int channels) { + return audio_track_->CallBooleanMethod(init_playout_, sample_rate, channels); +} + +bool AudioTrackJni::JavaAudioTrack::StartPlayout() { + return audio_track_->CallBooleanMethod(start_playout_); +} + +bool AudioTrackJni::JavaAudioTrack::StopPlayout() { + return audio_track_->CallBooleanMethod(stop_playout_); +} + +bool AudioTrackJni::JavaAudioTrack::SetStreamVolume(int volume) { + return audio_track_->CallBooleanMethod(set_stream_volume_, volume); +} + +int AudioTrackJni::JavaAudioTrack::GetStreamMaxVolume() { + return audio_track_->CallIntMethod(get_stream_max_volume_); +} + +int AudioTrackJni::JavaAudioTrack::GetStreamVolume() { + return audio_track_->CallIntMethod(get_stream_volume_); +} + +// TODO(henrika): possible extend usage of AudioManager and add it as member. +AudioTrackJni::AudioTrackJni(AudioManager* audio_manager) + : j_environment_(JVM::GetInstance()->environment()), + audio_parameters_(audio_manager->GetPlayoutAudioParameters()), + direct_buffer_address_(nullptr), + direct_buffer_capacity_in_bytes_(0), + frames_per_buffer_(0), + initialized_(false), + playing_(false), + audio_device_buffer_(nullptr) { + ALOGD("ctor%s", GetThreadInfo().c_str()); + RTC_DCHECK(audio_parameters_.is_valid()); + RTC_CHECK(j_environment_); + JNINativeMethod native_methods[] = { + {"nativeCacheDirectBufferAddress", "(Ljava/nio/ByteBuffer;J)V", + reinterpret_cast( + &webrtc::AudioTrackJni::CacheDirectBufferAddress)}, + {"nativeGetPlayoutData", "(IJ)V", + reinterpret_cast(&webrtc::AudioTrackJni::GetPlayoutData)}}; + j_native_registration_ = j_environment_->RegisterNatives( + "org/webrtc/voiceengine/WebRtcAudioTrack", native_methods, + arraysize(native_methods)); + j_audio_track_.reset( + new JavaAudioTrack(j_native_registration_.get(), + j_native_registration_->NewObject( + "", "(J)V", PointerTojlong(this)))); + // Detach from this thread since we want to use the checker to verify calls + // from the Java based audio thread. + thread_checker_java_.DetachFromThread(); +} + +AudioTrackJni::~AudioTrackJni() { + ALOGD("~dtor%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Terminate(); +} + +int32_t AudioTrackJni::Init() { + ALOGD("Init%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return 0; +} + +int32_t AudioTrackJni::Terminate() { + ALOGD("Terminate%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + StopPlayout(); + return 0; +} + +int32_t AudioTrackJni::InitPlayout() { + ALOGD("InitPlayout%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!initialized_); + RTC_DCHECK(!playing_); + if (!j_audio_track_->InitPlayout(audio_parameters_.sample_rate(), + audio_parameters_.channels())) { + ALOGE("InitPlayout failed!"); + return -1; + } + initialized_ = true; + return 0; +} + +int32_t AudioTrackJni::StartPlayout() { + ALOGD("StartPlayout%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(initialized_); + RTC_DCHECK(!playing_); + if (!j_audio_track_->StartPlayout()) { + ALOGE("StartPlayout failed!"); + return -1; + } + playing_ = true; + return 0; +} + +int32_t AudioTrackJni::StopPlayout() { + ALOGD("StopPlayout%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!initialized_ || !playing_) { + return 0; + } + if (!j_audio_track_->StopPlayout()) { + ALOGE("StopPlayout failed!"); + return -1; + } + // If we don't detach here, we will hit a RTC_DCHECK in OnDataIsRecorded() + // next time StartRecording() is called since it will create a new Java + // thread. + thread_checker_java_.DetachFromThread(); + initialized_ = false; + playing_ = false; + direct_buffer_address_ = nullptr; + return 0; +} + +int AudioTrackJni::SpeakerVolumeIsAvailable(bool& available) { + available = true; + return 0; +} + +int AudioTrackJni::SetSpeakerVolume(uint32_t volume) { + ALOGD("SetSpeakerVolume(%d)%s", volume, GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return j_audio_track_->SetStreamVolume(volume) ? 0 : -1; +} + +int AudioTrackJni::MaxSpeakerVolume(uint32_t& max_volume) const { + ALOGD("MaxSpeakerVolume%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + max_volume = j_audio_track_->GetStreamMaxVolume(); + return 0; +} + +int AudioTrackJni::MinSpeakerVolume(uint32_t& min_volume) const { + ALOGD("MaxSpeakerVolume%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + min_volume = 0; + return 0; +} + +int AudioTrackJni::SpeakerVolume(uint32_t& volume) const { + ALOGD("SpeakerVolume%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + volume = j_audio_track_->GetStreamVolume(); + return 0; +} + +// TODO(henrika): possibly add stereo support. +void AudioTrackJni::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + ALOGD("AttachAudioBuffer%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + audio_device_buffer_ = audioBuffer; + const int sample_rate_hz = audio_parameters_.sample_rate(); + ALOGD("SetPlayoutSampleRate(%d)", sample_rate_hz); + audio_device_buffer_->SetPlayoutSampleRate(sample_rate_hz); + const size_t channels = audio_parameters_.channels(); + ALOGD("SetPlayoutChannels(%" PRIuS ")", channels); + audio_device_buffer_->SetPlayoutChannels(channels); +} + +void JNICALL AudioTrackJni::CacheDirectBufferAddress(JNIEnv* env, + jobject obj, + jobject byte_buffer, + jlong nativeAudioTrack) { + webrtc::AudioTrackJni* this_object = + reinterpret_cast(nativeAudioTrack); + this_object->OnCacheDirectBufferAddress(env, byte_buffer); +} + +void AudioTrackJni::OnCacheDirectBufferAddress(JNIEnv* env, + jobject byte_buffer) { + ALOGD("OnCacheDirectBufferAddress"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!direct_buffer_address_); + direct_buffer_address_ = env->GetDirectBufferAddress(byte_buffer); + jlong capacity = env->GetDirectBufferCapacity(byte_buffer); + ALOGD("direct buffer capacity: %lld", capacity); + direct_buffer_capacity_in_bytes_ = static_cast(capacity); + const size_t bytes_per_frame = audio_parameters_.channels() * sizeof(int16_t); + frames_per_buffer_ = direct_buffer_capacity_in_bytes_ / bytes_per_frame; + ALOGD("frames_per_buffer: %" PRIuS, frames_per_buffer_); +} + +void JNICALL AudioTrackJni::GetPlayoutData(JNIEnv* env, + jobject obj, + jint length, + jlong nativeAudioTrack) { + webrtc::AudioTrackJni* this_object = + reinterpret_cast(nativeAudioTrack); + this_object->OnGetPlayoutData(static_cast(length)); +} + +// This method is called on a high-priority thread from Java. The name of +// the thread is 'AudioRecordTrack'. +void AudioTrackJni::OnGetPlayoutData(size_t length) { + RTC_DCHECK(thread_checker_java_.CalledOnValidThread()); + const size_t bytes_per_frame = audio_parameters_.channels() * sizeof(int16_t); + RTC_DCHECK_EQ(frames_per_buffer_, length / bytes_per_frame); + if (!audio_device_buffer_) { + ALOGE("AttachAudioBuffer has not been called!"); + return; + } + // Pull decoded data (in 16-bit PCM format) from jitter buffer. + int samples = audio_device_buffer_->RequestPlayoutData(frames_per_buffer_); + if (samples <= 0) { + ALOGE("AudioDeviceBuffer::RequestPlayoutData failed!"); + return; + } + RTC_DCHECK_EQ(samples, frames_per_buffer_); + // Copy decoded data into common byte buffer to ensure that it can be + // written to the Java based audio track. + samples = audio_device_buffer_->GetPlayoutData(direct_buffer_address_); + RTC_DCHECK_EQ(length, bytes_per_frame * samples); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.h new file mode 100644 index 0000000000..fd9c61e1ff --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/audio_track_jni.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_AUDIO_TRACK_JNI_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_AUDIO_TRACK_JNI_H_ + +#include + +#include + +#include "modules/audio_device/android/audio_common.h" +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/include/audio_device_defines.h" +#include "modules/utility/include/helpers_android.h" +#include "modules/utility/include/jvm_android.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +// Implements 16-bit mono PCM audio output support for Android using the Java +// AudioTrack interface. Most of the work is done by its Java counterpart in +// WebRtcAudioTrack.java. This class is created and lives on a thread in +// C++-land, but decoded audio buffers are requested on a high-priority +// thread managed by the Java class. +// +// An instance must be created and destroyed on one and the same thread. +// All public methods must also be called on the same thread. A thread checker +// will RTC_DCHECK if any method is called on an invalid thread. +// +// This class uses AttachCurrentThreadIfNeeded to attach to a Java VM if needed +// and detach when the object goes out of scope. Additional thread checking +// guarantees that no other (possibly non attached) thread is used. +class AudioTrackJni { + public: + // Wraps the Java specific parts of the AudioTrackJni into one helper class. + class JavaAudioTrack { + public: + JavaAudioTrack(NativeRegistration* native_registration, + std::unique_ptr audio_track); + ~JavaAudioTrack(); + + bool InitPlayout(int sample_rate, int channels); + bool StartPlayout(); + bool StopPlayout(); + bool SetStreamVolume(int volume); + int GetStreamMaxVolume(); + int GetStreamVolume(); + + private: + std::unique_ptr audio_track_; + jmethodID init_playout_; + jmethodID start_playout_; + jmethodID stop_playout_; + jmethodID set_stream_volume_; + jmethodID get_stream_max_volume_; + jmethodID get_stream_volume_; + }; + + explicit AudioTrackJni(AudioManager* audio_manager); + ~AudioTrackJni(); + + int32_t Init(); + int32_t Terminate(); + + int32_t InitPlayout(); + bool PlayoutIsInitialized() const { return initialized_; } + + int32_t StartPlayout(); + int32_t StopPlayout(); + bool Playing() const { return playing_; } + + int SpeakerVolumeIsAvailable(bool& available); + int SetSpeakerVolume(uint32_t volume); + int SpeakerVolume(uint32_t& volume) const; + int MaxSpeakerVolume(uint32_t& max_volume) const; + int MinSpeakerVolume(uint32_t& min_volume) const; + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); + + private: + // Called from Java side so we can cache the address of the Java-manged + // |byte_buffer| in |direct_buffer_address_|. The size of the buffer + // is also stored in |direct_buffer_capacity_in_bytes_|. + // Called on the same thread as the creating thread. + static void JNICALL CacheDirectBufferAddress( + JNIEnv* env, jobject obj, jobject byte_buffer, jlong nativeAudioTrack); + void OnCacheDirectBufferAddress(JNIEnv* env, jobject byte_buffer); + + // Called periodically by the Java based WebRtcAudioTrack object when + // playout has started. Each call indicates that |length| new bytes should + // be written to the memory area |direct_buffer_address_| for playout. + // This method is called on a high-priority thread from Java. The name of + // the thread is 'AudioTrackThread'. + static void JNICALL GetPlayoutData( + JNIEnv* env, jobject obj, jint length, jlong nativeAudioTrack); + void OnGetPlayoutData(size_t length); + + // Stores thread ID in constructor. + rtc::ThreadChecker thread_checker_; + + // Stores thread ID in first call to OnGetPlayoutData() from high-priority + // thread in Java. Detached during construction of this object. + rtc::ThreadChecker thread_checker_java_; + + // Calls AttachCurrentThread() if this thread is not attached at construction. + // Also ensures that DetachCurrentThread() is called at destruction. + AttachCurrentThreadIfNeeded attach_thread_if_needed_; + + // Wraps the JNI interface pointer and methods associated with it. + std::unique_ptr j_environment_; + + // Contains factory method for creating the Java object. + std::unique_ptr j_native_registration_; + + // Wraps the Java specific parts of the AudioTrackJni class. + std::unique_ptr j_audio_track_; + + // Contains audio parameters provided to this class at construction by the + // AudioManager. + const AudioParameters audio_parameters_; + + // Cached copy of address to direct audio buffer owned by |j_audio_track_|. + void* direct_buffer_address_; + + // Number of bytes in the direct audio buffer owned by |j_audio_track_|. + size_t direct_buffer_capacity_in_bytes_; + + // Number of audio frames per audio buffer. Each audio frame corresponds to + // one sample of PCM mono data at 16 bits per sample. Hence, each audio + // frame contains 2 bytes (given that the Java layer only supports mono). + // Example: 480 for 48000 Hz or 441 for 44100 Hz. + size_t frames_per_buffer_; + + bool initialized_; + + bool playing_; + + // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the + // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). + // The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance + // and therefore outlives this object. + AudioDeviceBuffer* audio_device_buffer_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_AUDIO_TRACK_JNI_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.cc new file mode 100644 index 0000000000..9eab4b33ef --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/build_info.h" + +#include "modules/utility/include/helpers_android.h" + +namespace webrtc { + +BuildInfo::BuildInfo() + : j_environment_(JVM::GetInstance()->environment()), + j_build_info_( + JVM::GetInstance()->GetClass("org/webrtc/voiceengine/BuildInfo")) {} + +std::string BuildInfo::GetStringFromJava(const char* name) { + jmethodID id = j_build_info_.GetStaticMethodId(name, "()Ljava/lang/String;"); + jstring j_string = static_cast( + j_build_info_.CallStaticObjectMethod(id)); + return j_environment_->JavaToStdString(j_string); +} + +std::string BuildInfo::GetDeviceModel() { + return GetStringFromJava("getDeviceModel"); +} + +std::string BuildInfo::GetBrand() { + return GetStringFromJava("getBrand"); +} + +std::string BuildInfo::GetDeviceManufacturer() { + return GetStringFromJava("getDeviceManufacturer"); +} + +std::string BuildInfo::GetAndroidBuildId() { + return GetStringFromJava("getAndroidBuildId"); +} + +std::string BuildInfo::GetBuildType() { + return GetStringFromJava("getBuildType"); +} + +std::string BuildInfo::GetBuildRelease() { + return GetStringFromJava("getBuildRelease"); +} + +SdkCode BuildInfo::GetSdkVersion() { + jmethodID id = j_build_info_.GetStaticMethodId("getSdkVersion", "()I"); + jint j_version = j_build_info_.CallStaticIntMethod(id); + return static_cast(j_version); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.h new file mode 100644 index 0000000000..2e292e8f08 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/build_info.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_BUILD_INFO_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_BUILD_INFO_H_ + +#include +#include +#include + +#include "modules/utility/include/jvm_android.h" + +namespace webrtc { + +// This enumeration maps to the values returned by BuildInfo::GetSdkVersion(), +// indicating the Android release associated with a given SDK version. +// See https://developer.android.com/guide/topics/manifest/uses-sdk-element.html +// for details. +enum SdkCode { + SDK_CODE_JELLY_BEAN = 16, // Android 4.1 + SDK_CODE_JELLY_BEAN_MR1 = 17, // Android 4.2 + SDK_CODE_JELLY_BEAN_MR2 = 18, // Android 4.3 + SDK_CODE_KITKAT = 19, // Android 4.4 + SDK_CODE_WATCH = 20, // Android 4.4W + SDK_CODE_LOLLIPOP = 21, // Android 5.0 + SDK_CODE_LOLLIPOP_MR1 = 22, // Android 5.1 + SDK_CODE_MARSHMALLOW = 23, // Android 6.0 + SDK_CODE_N = 24, +}; + +// Utility class used to query the Java class (org/webrtc/voiceengine/BuildInfo) +// for device and Android build information. +// The calling thread is attached to the JVM at construction if needed and a +// valid Java environment object is also created. +// All Get methods must be called on the creating thread. If not, the code will +// hit RTC_DCHECKs when calling JNIEnvironment::JavaToStdString(). +class BuildInfo { + public: + BuildInfo(); + ~BuildInfo() {} + + // End-user-visible name for the end product (e.g. "Nexus 6"). + std::string GetDeviceModel(); + // Consumer-visible brand (e.g. "google"). + std::string GetBrand(); + // Manufacturer of the product/hardware (e.g. "motorola"). + std::string GetDeviceManufacturer(); + // Android build ID (e.g. LMY47D). + std::string GetAndroidBuildId(); + // The type of build (e.g. "user" or "eng"). + std::string GetBuildType(); + // The user-visible version string (e.g. "5.1"). + std::string GetBuildRelease(); + // The user-visible SDK version of the framework (e.g. 21). See SdkCode enum + // for translation. + SdkCode GetSdkVersion(); + + private: + // Helper method which calls a static getter method with |name| and returns + // a string from Java. + std::string GetStringFromJava(const char* name); + + // Ensures that this class can access a valid JNI interface pointer even + // if the creating thread was not attached to the JVM. + AttachCurrentThreadIfNeeded attach_thread_if_needed_; + + // Provides access to the JNIEnv interface pointer and the JavaToStdString() + // method which is used to translate Java strings to std strings. + std::unique_ptr j_environment_; + + // Holds the jclass object and provides access to CallStaticObjectMethod(). + // Used by GetStringFromJava() during construction only. + JavaClass j_build_info_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_BUILD_INFO_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.cc new file mode 100644 index 0000000000..37086cc9eb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.cc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/ensure_initialized.h" + +#include + +#include "rtc_base/ignore_wundef.h" + +// Note: this dependency is dangerous since it reaches into Chromium's base. +// There's a risk of e.g. macro clashes. This file may only be used in tests. +RTC_PUSH_IGNORING_WUNDEF() +#include "base/android/jni_android.h" +RTC_POP_IGNORING_WUNDEF() +#include "modules/audio_device/android/audio_record_jni.h" +#include "modules/audio_device/android/audio_track_jni.h" +#include "modules/utility/include/jvm_android.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace audiodevicemodule { + +static pthread_once_t g_initialize_once = PTHREAD_ONCE_INIT; + +void EnsureInitializedOnce() { + RTC_CHECK(::base::android::IsVMInitialized()); + JNIEnv* jni = ::base::android::AttachCurrentThread(); + JavaVM* jvm = NULL; + RTC_CHECK_EQ(0, jni->GetJavaVM(&jvm)); + + // Initialize the Java environment (currently only used by the audio manager). + webrtc::JVM::Initialize(jvm); +} + +void EnsureInitialized() { + RTC_CHECK_EQ(0, pthread_once(&g_initialize_once, &EnsureInitializedOnce)); +} + +} // namespace audiodevicemodule +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.h new file mode 100644 index 0000000000..c1997b4acd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/ensure_initialized.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +namespace webrtc { +namespace audiodevicemodule { + +void EnsureInitialized(); + +} // namespace audiodevicemodule +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/BuildInfo.java b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/BuildInfo.java new file mode 100644 index 0000000000..aed8a06454 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/BuildInfo.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.voiceengine; + +import android.os.Build; + +public final class BuildInfo { + public static String getDevice() { + return Build.DEVICE; + } + + public static String getDeviceModel() { + return Build.MODEL; + } + + public static String getProduct() { + return Build.PRODUCT; + } + + public static String getBrand() { + return Build.BRAND; + } + + public static String getDeviceManufacturer() { + return Build.MANUFACTURER; + } + + public static String getAndroidBuildId() { + return Build.ID; + } + + public static String getBuildType() { + return Build.TYPE; + } + + public static String getBuildRelease() { + return Build.VERSION.RELEASE; + } + + public static int getSdkVersion() { + return Build.VERSION.SDK_INT; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioEffects.java b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioEffects.java new file mode 100644 index 0000000000..90a48d3bc2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioEffects.java @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.voiceengine; + +import android.annotation.TargetApi; +import android.media.audiofx.AcousticEchoCanceler; +import android.media.audiofx.AudioEffect; +import android.media.audiofx.AudioEffect.Descriptor; +import android.media.audiofx.AutomaticGainControl; +import android.media.audiofx.NoiseSuppressor; +import android.os.Build; +import java.util.List; +import java.util.UUID; +import org.webrtc.Logging; + +// This class wraps control of three different platform effects. Supported +// effects are: AcousticEchoCanceler (AEC) and NoiseSuppressor (NS). +// Calling enable() will active all effects that are +// supported by the device if the corresponding |shouldEnableXXX| member is set. +public class WebRtcAudioEffects { + private static final boolean DEBUG = false; + + private static final String TAG = "WebRtcAudioEffects"; + + // UUIDs for Software Audio Effects that we want to avoid using. + // The implementor field will be set to "The Android Open Source Project". + private static final UUID AOSP_ACOUSTIC_ECHO_CANCELER = + UUID.fromString("bb392ec0-8d4d-11e0-a896-0002a5d5c51b"); + private static final UUID AOSP_NOISE_SUPPRESSOR = + UUID.fromString("c06c8400-8e06-11e0-9cb6-0002a5d5c51b"); + + // Contains the available effect descriptors returned from the + // AudioEffect.getEffects() call. This result is cached to avoid doing the + // slow OS call multiple times. + private static Descriptor[] cachedEffects = null; + + // Contains the audio effect objects. Created in enable() and destroyed + // in release(). + private AcousticEchoCanceler aec = null; + private NoiseSuppressor ns = null; + + // Affects the final state given to the setEnabled() method on each effect. + // The default state is set to "disabled" but each effect can also be enabled + // by calling setAEC() and setNS(). + // To enable an effect, both the shouldEnableXXX member and the static + // canUseXXX() must be true. + private boolean shouldEnableAec = false; + private boolean shouldEnableNs = false; + + // Checks if the device implements Acoustic Echo Cancellation (AEC). + // Returns true if the device implements AEC, false otherwise. + public static boolean isAcousticEchoCancelerSupported() { + // Note: we're using isAcousticEchoCancelerEffectAvailable() instead of + // AcousticEchoCanceler.isAvailable() to avoid the expensive getEffects() + // OS API call. + return isAcousticEchoCancelerEffectAvailable(); + } + + // Checks if the device implements Noise Suppression (NS). + // Returns true if the device implements NS, false otherwise. + public static boolean isNoiseSuppressorSupported() { + // Note: we're using isNoiseSuppressorEffectAvailable() instead of + // NoiseSuppressor.isAvailable() to avoid the expensive getEffects() + // OS API call. + return isNoiseSuppressorEffectAvailable(); + } + + // Returns true if the device is blacklisted for HW AEC usage. + public static boolean isAcousticEchoCancelerBlacklisted() { + List blackListedModels = WebRtcAudioUtils.getBlackListedModelsForAecUsage(); + boolean isBlacklisted = blackListedModels.contains(Build.MODEL); + if (isBlacklisted) { + Logging.w(TAG, Build.MODEL + " is blacklisted for HW AEC usage!"); + } + return isBlacklisted; + } + + // Returns true if the device is blacklisted for HW NS usage. + public static boolean isNoiseSuppressorBlacklisted() { + List blackListedModels = WebRtcAudioUtils.getBlackListedModelsForNsUsage(); + boolean isBlacklisted = blackListedModels.contains(Build.MODEL); + if (isBlacklisted) { + Logging.w(TAG, Build.MODEL + " is blacklisted for HW NS usage!"); + } + return isBlacklisted; + } + + // Returns true if the platform AEC should be excluded based on its UUID. + // AudioEffect.queryEffects() can throw IllegalStateException. + @TargetApi(18) + private static boolean isAcousticEchoCancelerExcludedByUUID() { + for (Descriptor d : getAvailableEffects()) { + if (d.type.equals(AudioEffect.EFFECT_TYPE_AEC) + && d.uuid.equals(AOSP_ACOUSTIC_ECHO_CANCELER)) { + return true; + } + } + return false; + } + + // Returns true if the platform NS should be excluded based on its UUID. + // AudioEffect.queryEffects() can throw IllegalStateException. + @TargetApi(18) + private static boolean isNoiseSuppressorExcludedByUUID() { + for (Descriptor d : getAvailableEffects()) { + if (d.type.equals(AudioEffect.EFFECT_TYPE_NS) && d.uuid.equals(AOSP_NOISE_SUPPRESSOR)) { + return true; + } + } + return false; + } + + // Returns true if the device supports Acoustic Echo Cancellation (AEC). + @TargetApi(18) + private static boolean isAcousticEchoCancelerEffectAvailable() { + return isEffectTypeAvailable(AudioEffect.EFFECT_TYPE_AEC); + } + + // Returns true if the device supports Noise Suppression (NS). + @TargetApi(18) + private static boolean isNoiseSuppressorEffectAvailable() { + return isEffectTypeAvailable(AudioEffect.EFFECT_TYPE_NS); + } + + // Returns true if all conditions for supporting the HW AEC are fulfilled. + // It will not be possible to enable the HW AEC if this method returns false. + public static boolean canUseAcousticEchoCanceler() { + boolean canUseAcousticEchoCanceler = isAcousticEchoCancelerSupported() + && !WebRtcAudioUtils.useWebRtcBasedAcousticEchoCanceler() + && !isAcousticEchoCancelerBlacklisted() && !isAcousticEchoCancelerExcludedByUUID(); + Logging.d(TAG, "canUseAcousticEchoCanceler: " + canUseAcousticEchoCanceler); + return canUseAcousticEchoCanceler; + } + + // Returns true if all conditions for supporting the HW NS are fulfilled. + // It will not be possible to enable the HW NS if this method returns false. + public static boolean canUseNoiseSuppressor() { + boolean canUseNoiseSuppressor = isNoiseSuppressorSupported() + && !WebRtcAudioUtils.useWebRtcBasedNoiseSuppressor() && !isNoiseSuppressorBlacklisted() + && !isNoiseSuppressorExcludedByUUID(); + Logging.d(TAG, "canUseNoiseSuppressor: " + canUseNoiseSuppressor); + return canUseNoiseSuppressor; + } + + public static WebRtcAudioEffects create() { + return new WebRtcAudioEffects(); + } + + private WebRtcAudioEffects() { + Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); + } + + // Call this method to enable or disable the platform AEC. It modifies + // |shouldEnableAec| which is used in enable() where the actual state + // of the AEC effect is modified. Returns true if HW AEC is supported and + // false otherwise. + public boolean setAEC(boolean enable) { + Logging.d(TAG, "setAEC(" + enable + ")"); + if (!canUseAcousticEchoCanceler()) { + Logging.w(TAG, "Platform AEC is not supported"); + shouldEnableAec = false; + return false; + } + if (aec != null && (enable != shouldEnableAec)) { + Logging.e(TAG, "Platform AEC state can't be modified while recording"); + return false; + } + shouldEnableAec = enable; + return true; + } + + // Call this method to enable or disable the platform NS. It modifies + // |shouldEnableNs| which is used in enable() where the actual state + // of the NS effect is modified. Returns true if HW NS is supported and + // false otherwise. + public boolean setNS(boolean enable) { + Logging.d(TAG, "setNS(" + enable + ")"); + if (!canUseNoiseSuppressor()) { + Logging.w(TAG, "Platform NS is not supported"); + shouldEnableNs = false; + return false; + } + if (ns != null && (enable != shouldEnableNs)) { + Logging.e(TAG, "Platform NS state can't be modified while recording"); + return false; + } + shouldEnableNs = enable; + return true; + } + + public void enable(int audioSession) { + Logging.d(TAG, "enable(audioSession=" + audioSession + ")"); + assertTrue(aec == null); + assertTrue(ns == null); + + if (DEBUG) { + // Add logging of supported effects but filter out "VoIP effects", i.e., + // AEC, AEC and NS. Avoid calling AudioEffect.queryEffects() unless the + // DEBUG flag is set since we have seen crashes in this API. + for (Descriptor d : AudioEffect.queryEffects()) { + if (effectTypeIsVoIP(d.type)) { + Logging.d(TAG, "name: " + d.name + ", " + + "mode: " + d.connectMode + ", " + + "implementor: " + d.implementor + ", " + + "UUID: " + d.uuid); + } + } + } + + if (isAcousticEchoCancelerSupported()) { + // Create an AcousticEchoCanceler and attach it to the AudioRecord on + // the specified audio session. + aec = AcousticEchoCanceler.create(audioSession); + if (aec != null) { + boolean enabled = aec.getEnabled(); + boolean enable = shouldEnableAec && canUseAcousticEchoCanceler(); + if (aec.setEnabled(enable) != AudioEffect.SUCCESS) { + Logging.e(TAG, "Failed to set the AcousticEchoCanceler state"); + } + Logging.d(TAG, "AcousticEchoCanceler: was " + (enabled ? "enabled" : "disabled") + + ", enable: " + enable + ", is now: " + + (aec.getEnabled() ? "enabled" : "disabled")); + } else { + Logging.e(TAG, "Failed to create the AcousticEchoCanceler instance"); + } + } + + if (isNoiseSuppressorSupported()) { + // Create an NoiseSuppressor and attach it to the AudioRecord on the + // specified audio session. + ns = NoiseSuppressor.create(audioSession); + if (ns != null) { + boolean enabled = ns.getEnabled(); + boolean enable = shouldEnableNs && canUseNoiseSuppressor(); + if (ns.setEnabled(enable) != AudioEffect.SUCCESS) { + Logging.e(TAG, "Failed to set the NoiseSuppressor state"); + } + Logging.d(TAG, "NoiseSuppressor: was " + (enabled ? "enabled" : "disabled") + ", enable: " + + enable + ", is now: " + (ns.getEnabled() ? "enabled" : "disabled")); + } else { + Logging.e(TAG, "Failed to create the NoiseSuppressor instance"); + } + } + } + + // Releases all native audio effect resources. It is a good practice to + // release the effect engine when not in use as control can be returned + // to other applications or the native resources released. + public void release() { + Logging.d(TAG, "release"); + if (aec != null) { + aec.release(); + aec = null; + } + if (ns != null) { + ns.release(); + ns = null; + } + } + + // Returns true for effect types in |type| that are of "VoIP" types: + // Acoustic Echo Canceler (AEC) or Automatic Gain Control (AGC) or + // Noise Suppressor (NS). Note that, an extra check for support is needed + // in each comparison since some devices includes effects in the + // AudioEffect.Descriptor array that are actually not available on the device. + // As an example: Samsung Galaxy S6 includes an AGC in the descriptor but + // AutomaticGainControl.isAvailable() returns false. + @TargetApi(18) + private boolean effectTypeIsVoIP(UUID type) { + if (!WebRtcAudioUtils.runningOnJellyBeanMR2OrHigher()) + return false; + + return (AudioEffect.EFFECT_TYPE_AEC.equals(type) && isAcousticEchoCancelerSupported()) + || (AudioEffect.EFFECT_TYPE_NS.equals(type) && isNoiseSuppressorSupported()); + } + + // Helper method which throws an exception when an assertion has failed. + private static void assertTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + // Returns the cached copy of the audio effects array, if available, or + // queries the operating system for the list of effects. + private static Descriptor[] getAvailableEffects() { + if (cachedEffects != null) { + return cachedEffects; + } + // The caching is best effort only - if this method is called from several + // threads in parallel, they may end up doing the underlying OS call + // multiple times. It's normally only called on one thread so there's no + // real need to optimize for the multiple threads case. + cachedEffects = AudioEffect.queryEffects(); + return cachedEffects; + } + + // Returns true if an effect of the specified type is available. Functionally + // equivalent to (NoiseSuppressor|AutomaticGainControl|...).isAvailable(), but + // faster as it avoids the expensive OS call to enumerate effects. + private static boolean isEffectTypeAvailable(UUID effectType) { + Descriptor[] effects = getAvailableEffects(); + if (effects == null) { + return false; + } + for (Descriptor d : effects) { + if (d.type.equals(effectType)) { + return true; + } + } + return false; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java new file mode 100644 index 0000000000..12b2fad129 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.voiceengine; + +import android.annotation.TargetApi; +import android.content.Context; +import android.content.pm.PackageManager; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioRecord; +import android.media.AudioTrack; +import android.os.Build; +import java.util.Timer; +import java.util.TimerTask; +import org.webrtc.ContextUtils; +import org.webrtc.Logging; + +// WebRtcAudioManager handles tasks that uses android.media.AudioManager. +// At construction, storeAudioParameters() is called and it retrieves +// fundamental audio parameters like native sample rate and number of channels. +// The result is then provided to the caller by nativeCacheAudioParameters(). +// It is also possible to call init() to set up the audio environment for best +// possible "VoIP performance". All settings done in init() are reverted by +// dispose(). This class can also be used without calling init() if the user +// prefers to set up the audio environment separately. However, it is +// recommended to always use AudioManager.MODE_IN_COMMUNICATION. +public class WebRtcAudioManager { + private static final boolean DEBUG = false; + + private static final String TAG = "WebRtcAudioManager"; + + // Use mono as default for both audio directions. + private static boolean useStereoOutput = false; + private static boolean useStereoInput = false; + + private static boolean blacklistDeviceForOpenSLESUsage = false; + private static boolean blacklistDeviceForOpenSLESUsageIsOverridden = false; + + // Call this method to override the default list of blacklisted devices + // specified in WebRtcAudioUtils.BLACKLISTED_OPEN_SL_ES_MODELS. + // Allows an app to take control over which devices to exclude from using + // the OpenSL ES audio output path + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setBlacklistDeviceForOpenSLESUsage(boolean enable) { + blacklistDeviceForOpenSLESUsageIsOverridden = true; + blacklistDeviceForOpenSLESUsage = enable; + } + + // Call these methods to override the default mono audio modes for the specified direction(s) + // (input and/or output). + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setStereoOutput(boolean enable) { + Logging.w(TAG, "Overriding default output behavior: setStereoOutput(" + enable + ')'); + useStereoOutput = enable; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setStereoInput(boolean enable) { + Logging.w(TAG, "Overriding default input behavior: setStereoInput(" + enable + ')'); + useStereoInput = enable; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized boolean getStereoOutput() { + return useStereoOutput; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized boolean getStereoInput() { + return useStereoInput; + } + + // Default audio data format is PCM 16 bit per sample. + // Guaranteed to be supported by all devices. + private static final int BITS_PER_SAMPLE = 16; + + private static final int DEFAULT_FRAME_PER_BUFFER = 256; + + // List of possible audio modes. + private static final String[] AUDIO_MODES = new String[] { + "MODE_NORMAL", "MODE_RINGTONE", "MODE_IN_CALL", "MODE_IN_COMMUNICATION", + }; + + // Private utility class that periodically checks and logs the volume level + // of the audio stream that is currently controlled by the volume control. + // A timer triggers logs once every 30 seconds and the timer's associated + // thread is named "WebRtcVolumeLevelLoggerThread". + private static class VolumeLogger { + private static final String THREAD_NAME = "WebRtcVolumeLevelLoggerThread"; + private static final int TIMER_PERIOD_IN_SECONDS = 30; + + private final AudioManager audioManager; + private Timer timer; + + public VolumeLogger(AudioManager audioManager) { + this.audioManager = audioManager; + } + + public void start() { + timer = new Timer(THREAD_NAME); + timer.schedule(new LogVolumeTask(audioManager.getStreamMaxVolume(AudioManager.STREAM_RING), + audioManager.getStreamMaxVolume(AudioManager.STREAM_VOICE_CALL)), + 0, TIMER_PERIOD_IN_SECONDS * 1000); + } + + private class LogVolumeTask extends TimerTask { + private final int maxRingVolume; + private final int maxVoiceCallVolume; + + LogVolumeTask(int maxRingVolume, int maxVoiceCallVolume) { + this.maxRingVolume = maxRingVolume; + this.maxVoiceCallVolume = maxVoiceCallVolume; + } + + @Override + public void run() { + final int mode = audioManager.getMode(); + if (mode == AudioManager.MODE_RINGTONE) { + Logging.d(TAG, "STREAM_RING stream volume: " + + audioManager.getStreamVolume(AudioManager.STREAM_RING) + " (max=" + + maxRingVolume + ")"); + } else if (mode == AudioManager.MODE_IN_COMMUNICATION) { + Logging.d(TAG, "VOICE_CALL stream volume: " + + audioManager.getStreamVolume(AudioManager.STREAM_VOICE_CALL) + " (max=" + + maxVoiceCallVolume + ")"); + } + } + } + + private void stop() { + if (timer != null) { + timer.cancel(); + timer = null; + } + } + } + + private final long nativeAudioManager; + private final AudioManager audioManager; + + private boolean initialized = false; + private int nativeSampleRate; + private int nativeChannels; + + private boolean hardwareAEC; + private boolean hardwareAGC; + private boolean hardwareNS; + private boolean lowLatencyOutput; + private boolean lowLatencyInput; + private boolean proAudio; + private int sampleRate; + private int outputChannels; + private int inputChannels; + private int outputBufferSize; + private int inputBufferSize; + + private final VolumeLogger volumeLogger; + + WebRtcAudioManager(long nativeAudioManager) { + Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); + this.nativeAudioManager = nativeAudioManager; + audioManager = + (AudioManager) ContextUtils.getApplicationContext().getSystemService(Context.AUDIO_SERVICE); + if (DEBUG) { + WebRtcAudioUtils.logDeviceInfo(TAG); + } + volumeLogger = new VolumeLogger(audioManager); + storeAudioParameters(); + nativeCacheAudioParameters(sampleRate, outputChannels, inputChannels, hardwareAEC, hardwareAGC, + hardwareNS, lowLatencyOutput, lowLatencyInput, proAudio, outputBufferSize, inputBufferSize, + nativeAudioManager); + } + + private boolean init() { + Logging.d(TAG, "init" + WebRtcAudioUtils.getThreadInfo()); + if (initialized) { + return true; + } + Logging.d(TAG, "audio mode is: " + AUDIO_MODES[audioManager.getMode()]); + initialized = true; + volumeLogger.start(); + return true; + } + + private void dispose() { + Logging.d(TAG, "dispose" + WebRtcAudioUtils.getThreadInfo()); + if (!initialized) { + return; + } + volumeLogger.stop(); + } + + private boolean isCommunicationModeEnabled() { + return (audioManager.getMode() == AudioManager.MODE_IN_COMMUNICATION); + } + + private boolean isDeviceBlacklistedForOpenSLESUsage() { + boolean blacklisted = blacklistDeviceForOpenSLESUsageIsOverridden + ? blacklistDeviceForOpenSLESUsage + : WebRtcAudioUtils.deviceIsBlacklistedForOpenSLESUsage(); + if (blacklisted) { + Logging.d(TAG, Build.MODEL + " is blacklisted for OpenSL ES usage!"); + } + return blacklisted; + } + + private void storeAudioParameters() { + outputChannels = getStereoOutput() ? 2 : 1; + inputChannels = getStereoInput() ? 2 : 1; + sampleRate = getNativeOutputSampleRate(); + hardwareAEC = isAcousticEchoCancelerSupported(); + // TODO(henrika): use of hardware AGC is no longer supported. Currently + // hardcoded to false. To be removed. + hardwareAGC = false; + hardwareNS = isNoiseSuppressorSupported(); + lowLatencyOutput = isLowLatencyOutputSupported(); + lowLatencyInput = isLowLatencyInputSupported(); + proAudio = isProAudioSupported(); + outputBufferSize = lowLatencyOutput ? getLowLatencyOutputFramesPerBuffer() + : getMinOutputFrameSize(sampleRate, outputChannels); + inputBufferSize = lowLatencyInput ? getLowLatencyInputFramesPerBuffer() + : getMinInputFrameSize(sampleRate, inputChannels); + } + + // Gets the current earpiece state. + private boolean hasEarpiece() { + return ContextUtils.getApplicationContext().getPackageManager().hasSystemFeature( + PackageManager.FEATURE_TELEPHONY); + } + + // Returns true if low-latency audio output is supported. + private boolean isLowLatencyOutputSupported() { + return ContextUtils.getApplicationContext().getPackageManager().hasSystemFeature( + PackageManager.FEATURE_AUDIO_LOW_LATENCY); + } + + // Returns true if low-latency audio input is supported. + // TODO(henrika): remove the hardcoded false return value when OpenSL ES + // input performance has been evaluated and tested more. + public boolean isLowLatencyInputSupported() { + // TODO(henrika): investigate if some sort of device list is needed here + // as well. The NDK doc states that: "As of API level 21, lower latency + // audio input is supported on select devices. To take advantage of this + // feature, first confirm that lower latency output is available". + return WebRtcAudioUtils.runningOnLollipopOrHigher() && isLowLatencyOutputSupported(); + } + + // Returns true if the device has professional audio level of functionality + // and therefore supports the lowest possible round-trip latency. + @TargetApi(23) + private boolean isProAudioSupported() { + return WebRtcAudioUtils.runningOnMarshmallowOrHigher() + && ContextUtils.getApplicationContext().getPackageManager().hasSystemFeature( + PackageManager.FEATURE_AUDIO_PRO); + } + + // Returns the native output sample rate for this device's output stream. + private int getNativeOutputSampleRate() { + // Override this if we're running on an old emulator image which only + // supports 8 kHz and doesn't support PROPERTY_OUTPUT_SAMPLE_RATE. + if (WebRtcAudioUtils.runningOnEmulator()) { + Logging.d(TAG, "Running emulator, overriding sample rate to 8 kHz."); + return 8000; + } + // Default can be overriden by WebRtcAudioUtils.setDefaultSampleRateHz(). + // If so, use that value and return here. + if (WebRtcAudioUtils.isDefaultSampleRateOverridden()) { + Logging.d(TAG, "Default sample rate is overriden to " + + WebRtcAudioUtils.getDefaultSampleRateHz() + " Hz"); + return WebRtcAudioUtils.getDefaultSampleRateHz(); + } + // No overrides available. Deliver best possible estimate based on default + // Android AudioManager APIs. + final int sampleRateHz; + if (WebRtcAudioUtils.runningOnJellyBeanMR1OrHigher()) { + sampleRateHz = getSampleRateOnJellyBeanMR10OrHigher(); + } else { + sampleRateHz = WebRtcAudioUtils.getDefaultSampleRateHz(); + } + Logging.d(TAG, "Sample rate is set to " + sampleRateHz + " Hz"); + return sampleRateHz; + } + + @TargetApi(17) + private int getSampleRateOnJellyBeanMR10OrHigher() { + String sampleRateString = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE); + return (sampleRateString == null) ? WebRtcAudioUtils.getDefaultSampleRateHz() + : Integer.parseInt(sampleRateString); + } + + // Returns the native output buffer size for low-latency output streams. + @TargetApi(17) + private int getLowLatencyOutputFramesPerBuffer() { + assertTrue(isLowLatencyOutputSupported()); + if (!WebRtcAudioUtils.runningOnJellyBeanMR1OrHigher()) { + return DEFAULT_FRAME_PER_BUFFER; + } + String framesPerBuffer = + audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER); + return framesPerBuffer == null ? DEFAULT_FRAME_PER_BUFFER : Integer.parseInt(framesPerBuffer); + } + + // Returns true if the device supports an audio effect (AEC or NS). + // Four conditions must be fulfilled if functions are to return true: + // 1) the platform must support the built-in (HW) effect, + // 2) explicit use (override) of a WebRTC based version must not be set, + // 3) the device must not be blacklisted for use of the effect, and + // 4) the UUID of the effect must be approved (some UUIDs can be excluded). + private static boolean isAcousticEchoCancelerSupported() { + return WebRtcAudioEffects.canUseAcousticEchoCanceler(); + } + private static boolean isNoiseSuppressorSupported() { + return WebRtcAudioEffects.canUseNoiseSuppressor(); + } + + // Returns the minimum output buffer size for Java based audio (AudioTrack). + // This size can also be used for OpenSL ES implementations on devices that + // lacks support of low-latency output. + private static int getMinOutputFrameSize(int sampleRateInHz, int numChannels) { + final int bytesPerFrame = numChannels * (BITS_PER_SAMPLE / 8); + final int channelConfig = + (numChannels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO); + return AudioTrack.getMinBufferSize( + sampleRateInHz, channelConfig, AudioFormat.ENCODING_PCM_16BIT) + / bytesPerFrame; + } + + // Returns the native input buffer size for input streams. + private int getLowLatencyInputFramesPerBuffer() { + assertTrue(isLowLatencyInputSupported()); + return getLowLatencyOutputFramesPerBuffer(); + } + + // Returns the minimum input buffer size for Java based audio (AudioRecord). + // This size can calso be used for OpenSL ES implementations on devices that + // lacks support of low-latency input. + private static int getMinInputFrameSize(int sampleRateInHz, int numChannels) { + final int bytesPerFrame = numChannels * (BITS_PER_SAMPLE / 8); + final int channelConfig = + (numChannels == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO); + return AudioRecord.getMinBufferSize( + sampleRateInHz, channelConfig, AudioFormat.ENCODING_PCM_16BIT) + / bytesPerFrame; + } + + // Helper method which throws an exception when an assertion has failed. + private static void assertTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + private native void nativeCacheAudioParameters(int sampleRate, int outputChannels, + int inputChannels, boolean hardwareAEC, boolean hardwareAGC, boolean hardwareNS, + boolean lowLatencyOutput, boolean lowLatencyInput, boolean proAudio, int outputBufferSize, + int inputBufferSize, long nativeAudioManager); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java new file mode 100644 index 0000000000..f9bdc5d212 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.voiceengine; + +import android.annotation.TargetApi; +import android.content.Context; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaRecorder.AudioSource; +import android.os.Process; +import java.lang.System; +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; +import org.webrtc.ContextUtils; +import org.webrtc.Logging; +import org.webrtc.ThreadUtils; + +public class WebRtcAudioRecord { + private static final boolean DEBUG = false; + + private static final String TAG = "WebRtcAudioRecord"; + + // Default audio data format is PCM 16 bit per sample. + // Guaranteed to be supported by all devices. + private static final int BITS_PER_SAMPLE = 16; + + // Requested size of each recorded buffer provided to the client. + private static final int CALLBACK_BUFFER_SIZE_MS = 10; + + // Average number of callbacks per second. + private static final int BUFFERS_PER_SECOND = 1000 / CALLBACK_BUFFER_SIZE_MS; + + // We ask for a native buffer size of BUFFER_SIZE_FACTOR * (minimum required + // buffer size). The extra space is allocated to guard against glitches under + // high load. + private static final int BUFFER_SIZE_FACTOR = 2; + + // The AudioRecordJavaThread is allowed to wait for successful call to join() + // but the wait times out afther this amount of time. + private static final long AUDIO_RECORD_THREAD_JOIN_TIMEOUT_MS = 2000; + + private static final int DEFAULT_AUDIO_SOURCE = getDefaultAudioSource(); + private static int audioSource = DEFAULT_AUDIO_SOURCE; + + private final long nativeAudioRecord; + + private WebRtcAudioEffects effects = null; + + private ByteBuffer byteBuffer; + + private AudioRecord audioRecord = null; + private AudioRecordThread audioThread = null; + + private static volatile boolean microphoneMute = false; + private byte[] emptyBytes; + + // Audio recording error handler functions. + public enum AudioRecordStartErrorCode { + AUDIO_RECORD_START_EXCEPTION, + AUDIO_RECORD_START_STATE_MISMATCH, + } + + public static interface WebRtcAudioRecordErrorCallback { + void onWebRtcAudioRecordInitError(String errorMessage); + void onWebRtcAudioRecordStartError(AudioRecordStartErrorCode errorCode, String errorMessage); + void onWebRtcAudioRecordError(String errorMessage); + } + + private static WebRtcAudioRecordErrorCallback errorCallback = null; + + public static void setErrorCallback(WebRtcAudioRecordErrorCallback errorCallback) { + Logging.d(TAG, "Set error callback"); + WebRtcAudioRecord.errorCallback = errorCallback; + } + + /** + * Audio thread which keeps calling ByteBuffer.read() waiting for audio + * to be recorded. Feeds recorded data to the native counterpart as a + * periodic sequence of callbacks using DataIsRecorded(). + * This thread uses a Process.THREAD_PRIORITY_URGENT_AUDIO priority. + */ + private class AudioRecordThread extends Thread { + private volatile boolean keepAlive = true; + + public AudioRecordThread(String name) { + super(name); + } + + @Override + public void run() { + Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO); + Logging.d(TAG, "AudioRecordThread" + WebRtcAudioUtils.getThreadInfo()); + assertTrue(audioRecord.getRecordingState() == AudioRecord.RECORDSTATE_RECORDING); + + long lastTime = System.nanoTime(); + while (keepAlive) { + int bytesRead = audioRecord.read(byteBuffer, byteBuffer.capacity()); + if (bytesRead == byteBuffer.capacity()) { + if (microphoneMute) { + byteBuffer.clear(); + byteBuffer.put(emptyBytes); + } + nativeDataIsRecorded(bytesRead, nativeAudioRecord); + } else { + String errorMessage = "AudioRecord.read failed: " + bytesRead; + Logging.e(TAG, errorMessage); + if (bytesRead == AudioRecord.ERROR_INVALID_OPERATION) { + keepAlive = false; + reportWebRtcAudioRecordError(errorMessage); + } + } + if (DEBUG) { + long nowTime = System.nanoTime(); + long durationInMs = TimeUnit.NANOSECONDS.toMillis((nowTime - lastTime)); + lastTime = nowTime; + Logging.d(TAG, "bytesRead[" + durationInMs + "] " + bytesRead); + } + } + + try { + if (audioRecord != null) { + audioRecord.stop(); + } + } catch (IllegalStateException e) { + Logging.e(TAG, "AudioRecord.stop failed: " + e.getMessage()); + } + } + + // Stops the inner thread loop and also calls AudioRecord.stop(). + // Does not block the calling thread. + public void stopThread() { + Logging.d(TAG, "stopThread"); + keepAlive = false; + } + } + + WebRtcAudioRecord(long nativeAudioRecord) { + Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); + this.nativeAudioRecord = nativeAudioRecord; + if (DEBUG) { + WebRtcAudioUtils.logDeviceInfo(TAG); + } + effects = WebRtcAudioEffects.create(); + } + + private boolean enableBuiltInAEC(boolean enable) { + Logging.d(TAG, "enableBuiltInAEC(" + enable + ')'); + if (effects == null) { + Logging.e(TAG, "Built-in AEC is not supported on this platform"); + return false; + } + return effects.setAEC(enable); + } + + private boolean enableBuiltInNS(boolean enable) { + Logging.d(TAG, "enableBuiltInNS(" + enable + ')'); + if (effects == null) { + Logging.e(TAG, "Built-in NS is not supported on this platform"); + return false; + } + return effects.setNS(enable); + } + + private int initRecording(int sampleRate, int channels) { + Logging.d(TAG, "initRecording(sampleRate=" + sampleRate + ", channels=" + channels + ")"); + if (audioRecord != null) { + reportWebRtcAudioRecordInitError("InitRecording called twice without StopRecording."); + return -1; + } + final int bytesPerFrame = channels * (BITS_PER_SAMPLE / 8); + final int framesPerBuffer = sampleRate / BUFFERS_PER_SECOND; + byteBuffer = ByteBuffer.allocateDirect(bytesPerFrame * framesPerBuffer); + Logging.d(TAG, "byteBuffer.capacity: " + byteBuffer.capacity()); + emptyBytes = new byte[byteBuffer.capacity()]; + // Rather than passing the ByteBuffer with every callback (requiring + // the potentially expensive GetDirectBufferAddress) we simply have the + // the native class cache the address to the memory once. + nativeCacheDirectBufferAddress(byteBuffer, nativeAudioRecord); + + // Get the minimum buffer size required for the successful creation of + // an AudioRecord object, in byte units. + // Note that this size doesn't guarantee a smooth recording under load. + final int channelConfig = channelCountToConfiguration(channels); + int minBufferSize = + AudioRecord.getMinBufferSize(sampleRate, channelConfig, AudioFormat.ENCODING_PCM_16BIT); + if (minBufferSize == AudioRecord.ERROR || minBufferSize == AudioRecord.ERROR_BAD_VALUE) { + reportWebRtcAudioRecordInitError("AudioRecord.getMinBufferSize failed: " + minBufferSize); + return -1; + } + Logging.d(TAG, "AudioRecord.getMinBufferSize: " + minBufferSize); + + // Use a larger buffer size than the minimum required when creating the + // AudioRecord instance to ensure smooth recording under load. It has been + // verified that it does not increase the actual recording latency. + int bufferSizeInBytes = Math.max(BUFFER_SIZE_FACTOR * minBufferSize, byteBuffer.capacity()); + Logging.d(TAG, "bufferSizeInBytes: " + bufferSizeInBytes); + try { + audioRecord = new AudioRecord(audioSource, sampleRate, channelConfig, + AudioFormat.ENCODING_PCM_16BIT, bufferSizeInBytes); + } catch (IllegalArgumentException e) { + reportWebRtcAudioRecordInitError("AudioRecord ctor error: " + e.getMessage()); + releaseAudioResources(); + return -1; + } + if (audioRecord == null || audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { + reportWebRtcAudioRecordInitError("Failed to create a new AudioRecord instance"); + releaseAudioResources(); + return -1; + } + if (effects != null) { + effects.enable(audioRecord.getAudioSessionId()); + } + logMainParameters(); + logMainParametersExtended(); + return framesPerBuffer; + } + + private boolean startRecording() { + Logging.d(TAG, "startRecording"); + assertTrue(audioRecord != null); + assertTrue(audioThread == null); + try { + audioRecord.startRecording(); + } catch (IllegalStateException e) { + reportWebRtcAudioRecordStartError(AudioRecordStartErrorCode.AUDIO_RECORD_START_EXCEPTION, + "AudioRecord.startRecording failed: " + e.getMessage()); + return false; + } + if (audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) { + reportWebRtcAudioRecordStartError( + AudioRecordStartErrorCode.AUDIO_RECORD_START_STATE_MISMATCH, + "AudioRecord.startRecording failed - incorrect state :" + + audioRecord.getRecordingState()); + return false; + } + audioThread = new AudioRecordThread("AudioRecordJavaThread"); + audioThread.start(); + return true; + } + + private boolean stopRecording() { + Logging.d(TAG, "stopRecording"); + assertTrue(audioThread != null); + audioThread.stopThread(); + if (!ThreadUtils.joinUninterruptibly(audioThread, AUDIO_RECORD_THREAD_JOIN_TIMEOUT_MS)) { + Logging.e(TAG, "Join of AudioRecordJavaThread timed out"); + } + audioThread = null; + if (effects != null) { + effects.release(); + } + releaseAudioResources(); + return true; + } + + private void logMainParameters() { + Logging.d(TAG, "AudioRecord: " + + "session ID: " + audioRecord.getAudioSessionId() + ", " + + "channels: " + audioRecord.getChannelCount() + ", " + + "sample rate: " + audioRecord.getSampleRate()); + } + + @TargetApi(23) + private void logMainParametersExtended() { + if (WebRtcAudioUtils.runningOnMarshmallowOrHigher()) { + Logging.d(TAG, "AudioRecord: " + // The frame count of the native AudioRecord buffer. + + "buffer size in frames: " + audioRecord.getBufferSizeInFrames()); + } + } + + // Helper method which throws an exception when an assertion has failed. + private static void assertTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + private int channelCountToConfiguration(int channels) { + return (channels == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO); + } + + private native void nativeCacheDirectBufferAddress(ByteBuffer byteBuffer, long nativeAudioRecord); + + private native void nativeDataIsRecorded(int bytes, long nativeAudioRecord); + + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setAudioSource(int source) { + Logging.w(TAG, "Audio source is changed from: " + audioSource + + " to " + source); + audioSource = source; + } + + private static int getDefaultAudioSource() { + return AudioSource.VOICE_COMMUNICATION; + } + + // Sets all recorded samples to zero if |mute| is true, i.e., ensures that + // the microphone is muted. + public static void setMicrophoneMute(boolean mute) { + Logging.w(TAG, "setMicrophoneMute(" + mute + ")"); + microphoneMute = mute; + } + + // Releases the native AudioRecord resources. + private void releaseAudioResources() { + if (audioRecord != null) { + audioRecord.release(); + audioRecord = null; + } + } + + private void reportWebRtcAudioRecordInitError(String errorMessage) { + Logging.e(TAG, "Init recording error: " + errorMessage); + if (errorCallback != null) { + errorCallback.onWebRtcAudioRecordInitError(errorMessage); + } + } + + private void reportWebRtcAudioRecordStartError( + AudioRecordStartErrorCode errorCode, String errorMessage) { + Logging.e(TAG, "Start recording error: " + errorCode + ". " + errorMessage); + if (errorCallback != null) { + errorCallback.onWebRtcAudioRecordStartError(errorCode, errorMessage); + } + } + + private void reportWebRtcAudioRecordError(String errorMessage) { + Logging.e(TAG, "Run-time recording error: " + errorMessage); + if (errorCallback != null) { + errorCallback.onWebRtcAudioRecordError(errorMessage); + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java new file mode 100644 index 0000000000..ae7a16f8f0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java @@ -0,0 +1,522 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.voiceengine; + +import android.annotation.TargetApi; +import android.content.Context; +import android.media.AudioAttributes; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioTrack; +import android.os.Process; +import java.lang.Thread; +import java.nio.ByteBuffer; +import org.webrtc.ContextUtils; +import org.webrtc.Logging; +import org.webrtc.ThreadUtils; + +public class WebRtcAudioTrack { + private static final boolean DEBUG = false; + + private static final String TAG = "WebRtcAudioTrack"; + + // Default audio data format is PCM 16 bit per sample. + // Guaranteed to be supported by all devices. + private static final int BITS_PER_SAMPLE = 16; + + // Requested size of each recorded buffer provided to the client. + private static final int CALLBACK_BUFFER_SIZE_MS = 10; + + // Average number of callbacks per second. + private static final int BUFFERS_PER_SECOND = 1000 / CALLBACK_BUFFER_SIZE_MS; + + // The AudioTrackThread is allowed to wait for successful call to join() + // but the wait times out afther this amount of time. + private static final long AUDIO_TRACK_THREAD_JOIN_TIMEOUT_MS = 2000; + + // By default, WebRTC creates audio tracks with a usage attribute + // corresponding to voice communications, such as telephony or VoIP. + private static final int DEFAULT_USAGE = getDefaultUsageAttribute(); + private static int usageAttribute = DEFAULT_USAGE; + + // This method overrides the default usage attribute and allows the user + // to set it to something else than AudioAttributes.USAGE_VOICE_COMMUNICATION. + // NOTE: calling this method will most likely break existing VoIP tuning. + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setAudioTrackUsageAttribute(int usage) { + Logging.w(TAG, "Default usage attribute is changed from: " + + DEFAULT_USAGE + " to " + usage); + usageAttribute = usage; + } + + private static int getDefaultUsageAttribute() { + if (WebRtcAudioUtils.runningOnLollipopOrHigher()) { + return getDefaultUsageAttributeOnLollipopOrHigher(); + } else { + // Not used on SDKs lower than L. + return 0; + } + } + + @TargetApi(21) + private static int getDefaultUsageAttributeOnLollipopOrHigher() { + return AudioAttributes.USAGE_VOICE_COMMUNICATION; + } + + private final long nativeAudioTrack; + private final AudioManager audioManager; + private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); + + private ByteBuffer byteBuffer; + + private AudioTrack audioTrack = null; + private AudioTrackThread audioThread = null; + + // Samples to be played are replaced by zeros if |speakerMute| is set to true. + // Can be used to ensure that the speaker is fully muted. + private static volatile boolean speakerMute = false; + private byte[] emptyBytes; + + // Audio playout/track error handler functions. + public enum AudioTrackStartErrorCode { + AUDIO_TRACK_START_EXCEPTION, + AUDIO_TRACK_START_STATE_MISMATCH, + } + + @Deprecated + public static interface WebRtcAudioTrackErrorCallback { + void onWebRtcAudioTrackInitError(String errorMessage); + void onWebRtcAudioTrackStartError(String errorMessage); + void onWebRtcAudioTrackError(String errorMessage); + } + + // TODO(henrika): upgrade all clients to use this new interface instead. + public static interface ErrorCallback { + void onWebRtcAudioTrackInitError(String errorMessage); + void onWebRtcAudioTrackStartError(AudioTrackStartErrorCode errorCode, String errorMessage); + void onWebRtcAudioTrackError(String errorMessage); + } + + private static WebRtcAudioTrackErrorCallback errorCallbackOld = null; + private static ErrorCallback errorCallback = null; + + @Deprecated + public static void setErrorCallback(WebRtcAudioTrackErrorCallback errorCallback) { + Logging.d(TAG, "Set error callback (deprecated"); + WebRtcAudioTrack.errorCallbackOld = errorCallback; + } + + public static void setErrorCallback(ErrorCallback errorCallback) { + Logging.d(TAG, "Set extended error callback"); + WebRtcAudioTrack.errorCallback = errorCallback; + } + + /** + * Audio thread which keeps calling AudioTrack.write() to stream audio. + * Data is periodically acquired from the native WebRTC layer using the + * nativeGetPlayoutData callback function. + * This thread uses a Process.THREAD_PRIORITY_URGENT_AUDIO priority. + */ + private class AudioTrackThread extends Thread { + private volatile boolean keepAlive = true; + + public AudioTrackThread(String name) { + super(name); + } + + @Override + public void run() { + Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO); + Logging.d(TAG, "AudioTrackThread" + WebRtcAudioUtils.getThreadInfo()); + assertTrue(audioTrack.getPlayState() == AudioTrack.PLAYSTATE_PLAYING); + + // Fixed size in bytes of each 10ms block of audio data that we ask for + // using callbacks to the native WebRTC client. + final int sizeInBytes = byteBuffer.capacity(); + + while (keepAlive) { + // Get 10ms of PCM data from the native WebRTC client. Audio data is + // written into the common ByteBuffer using the address that was + // cached at construction. + nativeGetPlayoutData(sizeInBytes, nativeAudioTrack); + // Write data until all data has been written to the audio sink. + // Upon return, the buffer position will have been advanced to reflect + // the amount of data that was successfully written to the AudioTrack. + assertTrue(sizeInBytes <= byteBuffer.remaining()); + if (speakerMute) { + byteBuffer.clear(); + byteBuffer.put(emptyBytes); + byteBuffer.position(0); + } + int bytesWritten = 0; + if (WebRtcAudioUtils.runningOnLollipopOrHigher()) { + bytesWritten = writeOnLollipop(audioTrack, byteBuffer, sizeInBytes); + } else { + bytesWritten = writePreLollipop(audioTrack, byteBuffer, sizeInBytes); + } + if (bytesWritten != sizeInBytes) { + Logging.e(TAG, "AudioTrack.write played invalid number of bytes: " + bytesWritten); + // If a write() returns a negative value, an error has occurred. + // Stop playing and report an error in this case. + if (bytesWritten < 0) { + keepAlive = false; + reportWebRtcAudioTrackError("AudioTrack.write failed: " + bytesWritten); + } + } + // The byte buffer must be rewinded since byteBuffer.position() is + // increased at each call to AudioTrack.write(). If we don't do this, + // next call to AudioTrack.write() will fail. + byteBuffer.rewind(); + + // TODO(henrika): it is possible to create a delay estimate here by + // counting number of written frames and subtracting the result from + // audioTrack.getPlaybackHeadPosition(). + } + + // Stops playing the audio data. Since the instance was created in + // MODE_STREAM mode, audio will stop playing after the last buffer that + // was written has been played. + if (audioTrack != null) { + Logging.d(TAG, "Calling AudioTrack.stop..."); + try { + audioTrack.stop(); + Logging.d(TAG, "AudioTrack.stop is done."); + } catch (IllegalStateException e) { + Logging.e(TAG, "AudioTrack.stop failed: " + e.getMessage()); + } + } + } + + @TargetApi(21) + private int writeOnLollipop(AudioTrack audioTrack, ByteBuffer byteBuffer, int sizeInBytes) { + return audioTrack.write(byteBuffer, sizeInBytes, AudioTrack.WRITE_BLOCKING); + } + + private int writePreLollipop(AudioTrack audioTrack, ByteBuffer byteBuffer, int sizeInBytes) { + return audioTrack.write(byteBuffer.array(), byteBuffer.arrayOffset(), sizeInBytes); + } + + // Stops the inner thread loop which results in calling AudioTrack.stop(). + // Does not block the calling thread. + public void stopThread() { + Logging.d(TAG, "stopThread"); + keepAlive = false; + } + } + + WebRtcAudioTrack(long nativeAudioTrack) { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); + this.nativeAudioTrack = nativeAudioTrack; + audioManager = + (AudioManager) ContextUtils.getApplicationContext().getSystemService(Context.AUDIO_SERVICE); + if (DEBUG) { + WebRtcAudioUtils.logDeviceInfo(TAG); + } + } + + private boolean initPlayout(int sampleRate, int channels) { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "initPlayout(sampleRate=" + sampleRate + ", channels=" + channels + ")"); + final int bytesPerFrame = channels * (BITS_PER_SAMPLE / 8); + byteBuffer = ByteBuffer.allocateDirect(bytesPerFrame * (sampleRate / BUFFERS_PER_SECOND)); + Logging.d(TAG, "byteBuffer.capacity: " + byteBuffer.capacity()); + emptyBytes = new byte[byteBuffer.capacity()]; + // Rather than passing the ByteBuffer with every callback (requiring + // the potentially expensive GetDirectBufferAddress) we simply have the + // the native class cache the address to the memory once. + nativeCacheDirectBufferAddress(byteBuffer, nativeAudioTrack); + + // Get the minimum buffer size required for the successful creation of an + // AudioTrack object to be created in the MODE_STREAM mode. + // Note that this size doesn't guarantee a smooth playback under load. + // TODO(henrika): should we extend the buffer size to avoid glitches? + final int channelConfig = channelCountToConfiguration(channels); + final int minBufferSizeInBytes = + AudioTrack.getMinBufferSize(sampleRate, channelConfig, AudioFormat.ENCODING_PCM_16BIT); + Logging.d(TAG, "AudioTrack.getMinBufferSize: " + minBufferSizeInBytes); + // For the streaming mode, data must be written to the audio sink in + // chunks of size (given by byteBuffer.capacity()) less than or equal + // to the total buffer size |minBufferSizeInBytes|. But, we have seen + // reports of "getMinBufferSize(): error querying hardware". Hence, it + // can happen that |minBufferSizeInBytes| contains an invalid value. + if (minBufferSizeInBytes < byteBuffer.capacity()) { + reportWebRtcAudioTrackInitError("AudioTrack.getMinBufferSize returns an invalid value."); + return false; + } + + // Ensure that prevision audio session was stopped correctly before trying + // to create a new AudioTrack. + if (audioTrack != null) { + reportWebRtcAudioTrackInitError("Conflict with existing AudioTrack."); + return false; + } + try { + // Create an AudioTrack object and initialize its associated audio buffer. + // The size of this buffer determines how long an AudioTrack can play + // before running out of data. + if (WebRtcAudioUtils.runningOnLollipopOrHigher()) { + // If we are on API level 21 or higher, it is possible to use a special AudioTrack + // constructor that uses AudioAttributes and AudioFormat as input. It allows us to + // supersede the notion of stream types for defining the behavior of audio playback, + // and to allow certain platforms or routing policies to use this information for more + // refined volume or routing decisions. + audioTrack = createAudioTrackOnLollipopOrHigher( + sampleRate, channelConfig, minBufferSizeInBytes); + } else { + // Use default constructor for API levels below 21. + audioTrack = + createAudioTrackOnLowerThanLollipop(sampleRate, channelConfig, minBufferSizeInBytes); + } + } catch (IllegalArgumentException e) { + reportWebRtcAudioTrackInitError(e.getMessage()); + releaseAudioResources(); + return false; + } + + // It can happen that an AudioTrack is created but it was not successfully + // initialized upon creation. Seems to be the case e.g. when the maximum + // number of globally available audio tracks is exceeded. + if (audioTrack == null || audioTrack.getState() != AudioTrack.STATE_INITIALIZED) { + reportWebRtcAudioTrackInitError("Initialization of audio track failed."); + releaseAudioResources(); + return false; + } + logMainParameters(); + logMainParametersExtended(); + return true; + } + + private boolean startPlayout() { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "startPlayout"); + assertTrue(audioTrack != null); + assertTrue(audioThread == null); + + // Starts playing an audio track. + try { + audioTrack.play(); + } catch (IllegalStateException e) { + reportWebRtcAudioTrackStartError(AudioTrackStartErrorCode.AUDIO_TRACK_START_EXCEPTION, + "AudioTrack.play failed: " + e.getMessage()); + releaseAudioResources(); + return false; + } + if (audioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) { + reportWebRtcAudioTrackStartError( + AudioTrackStartErrorCode.AUDIO_TRACK_START_STATE_MISMATCH, + "AudioTrack.play failed - incorrect state :" + + audioTrack.getPlayState()); + releaseAudioResources(); + return false; + } + + // Create and start new high-priority thread which calls AudioTrack.write() + // and where we also call the native nativeGetPlayoutData() callback to + // request decoded audio from WebRTC. + audioThread = new AudioTrackThread("AudioTrackJavaThread"); + audioThread.start(); + return true; + } + + private boolean stopPlayout() { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "stopPlayout"); + assertTrue(audioThread != null); + logUnderrunCount(); + audioThread.stopThread(); + + Logging.d(TAG, "Stopping the AudioTrackThread..."); + audioThread.interrupt(); + if (!ThreadUtils.joinUninterruptibly(audioThread, AUDIO_TRACK_THREAD_JOIN_TIMEOUT_MS)) { + Logging.e(TAG, "Join of AudioTrackThread timed out."); + } + Logging.d(TAG, "AudioTrackThread has now been stopped."); + audioThread = null; + releaseAudioResources(); + return true; + } + + // Get max possible volume index for a phone call audio stream. + private int getStreamMaxVolume() { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "getStreamMaxVolume"); + assertTrue(audioManager != null); + return audioManager.getStreamMaxVolume(AudioManager.STREAM_VOICE_CALL); + } + + // Set current volume level for a phone call audio stream. + private boolean setStreamVolume(int volume) { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "setStreamVolume(" + volume + ")"); + assertTrue(audioManager != null); + if (isVolumeFixed()) { + Logging.e(TAG, "The device implements a fixed volume policy."); + return false; + } + audioManager.setStreamVolume(AudioManager.STREAM_VOICE_CALL, volume, 0); + return true; + } + + private boolean isVolumeFixed() { + if (!WebRtcAudioUtils.runningOnLollipopOrHigher()) + return false; + return audioManager.isVolumeFixed(); + } + + /** Get current volume level for a phone call audio stream. */ + private int getStreamVolume() { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "getStreamVolume"); + assertTrue(audioManager != null); + return audioManager.getStreamVolume(AudioManager.STREAM_VOICE_CALL); + } + + private void logMainParameters() { + Logging.d(TAG, "AudioTrack: " + + "session ID: " + audioTrack.getAudioSessionId() + ", " + + "channels: " + audioTrack.getChannelCount() + ", " + + "sample rate: " + audioTrack.getSampleRate() + ", " + // Gain (>=1.0) expressed as linear multiplier on sample values. + + "max gain: " + AudioTrack.getMaxVolume()); + } + + // Creates and AudioTrack instance using AudioAttributes and AudioFormat as input. + // It allows certain platforms or routing policies to use this information for more + // refined volume or routing decisions. + @TargetApi(21) + private static AudioTrack createAudioTrackOnLollipopOrHigher( + int sampleRateInHz, int channelConfig, int bufferSizeInBytes) { + Logging.d(TAG, "createAudioTrackOnLollipopOrHigher"); + // TODO(henrika): use setPerformanceMode(int) with PERFORMANCE_MODE_LOW_LATENCY to control + // performance when Android O is supported. Add some logging in the mean time. + final int nativeOutputSampleRate = + AudioTrack.getNativeOutputSampleRate(AudioManager.STREAM_VOICE_CALL); + Logging.d(TAG, "nativeOutputSampleRate: " + nativeOutputSampleRate); + if (sampleRateInHz != nativeOutputSampleRate) { + Logging.w(TAG, "Unable to use fast mode since requested sample rate is not native"); + } + if (usageAttribute != DEFAULT_USAGE) { + Logging.w(TAG, "A non default usage attribute is used: " + usageAttribute); + } + // Create an audio track where the audio usage is for VoIP and the content type is speech. + return new AudioTrack( + new AudioAttributes.Builder() + .setUsage(usageAttribute) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build(), + new AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(sampleRateInHz) + .setChannelMask(channelConfig) + .build(), + bufferSizeInBytes, + AudioTrack.MODE_STREAM, + AudioManager.AUDIO_SESSION_ID_GENERATE); + } + + @SuppressWarnings("deprecation") // Deprecated in API level 25. + private static AudioTrack createAudioTrackOnLowerThanLollipop( + int sampleRateInHz, int channelConfig, int bufferSizeInBytes) { + return new AudioTrack(AudioManager.STREAM_VOICE_CALL, sampleRateInHz, channelConfig, + AudioFormat.ENCODING_PCM_16BIT, bufferSizeInBytes, AudioTrack.MODE_STREAM); + } + + @TargetApi(24) + private void logMainParametersExtended() { + if (WebRtcAudioUtils.runningOnMarshmallowOrHigher()) { + Logging.d(TAG, "AudioTrack: " + // The effective size of the AudioTrack buffer that the app writes to. + + "buffer size in frames: " + audioTrack.getBufferSizeInFrames()); + } + if (WebRtcAudioUtils.runningOnNougatOrHigher()) { + Logging.d(TAG, "AudioTrack: " + // Maximum size of the AudioTrack buffer in frames. + + "buffer capacity in frames: " + audioTrack.getBufferCapacityInFrames()); + } + } + + // Prints the number of underrun occurrences in the application-level write + // buffer since the AudioTrack was created. An underrun occurs if the app does + // not write audio data quickly enough, causing the buffer to underflow and a + // potential audio glitch. + // TODO(henrika): keep track of this value in the field and possibly add new + // UMA stat if needed. + @TargetApi(24) + private void logUnderrunCount() { + if (WebRtcAudioUtils.runningOnNougatOrHigher()) { + Logging.d(TAG, "underrun count: " + audioTrack.getUnderrunCount()); + } + } + + // Helper method which throws an exception when an assertion has failed. + private static void assertTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + private int channelCountToConfiguration(int channels) { + return (channels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO); + } + + private native void nativeCacheDirectBufferAddress(ByteBuffer byteBuffer, long nativeAudioRecord); + + private native void nativeGetPlayoutData(int bytes, long nativeAudioRecord); + + // Sets all samples to be played out to zero if |mute| is true, i.e., + // ensures that the speaker is muted. + public static void setSpeakerMute(boolean mute) { + Logging.w(TAG, "setSpeakerMute(" + mute + ")"); + speakerMute = mute; + } + + // Releases the native AudioTrack resources. + private void releaseAudioResources() { + Logging.d(TAG, "releaseAudioResources"); + if (audioTrack != null) { + audioTrack.release(); + audioTrack = null; + } + } + + private void reportWebRtcAudioTrackInitError(String errorMessage) { + Logging.e(TAG, "Init playout error: " + errorMessage); + if (errorCallback != null) { + errorCallbackOld.onWebRtcAudioTrackInitError(errorMessage); + } + if (errorCallback != null) { + errorCallback.onWebRtcAudioTrackInitError(errorMessage); + } + } + + private void reportWebRtcAudioTrackStartError( + AudioTrackStartErrorCode errorCode, String errorMessage) { + Logging.e(TAG, "Start playout error: " + errorCode + ". " + errorMessage); + if (errorCallback != null) { + errorCallbackOld.onWebRtcAudioTrackStartError(errorMessage); + } + if (errorCallback != null) { + errorCallback.onWebRtcAudioTrackStartError(errorCode, errorMessage); + } + } + + private void reportWebRtcAudioTrackError(String errorMessage) { + Logging.e(TAG, "Run-time playback error: " + errorMessage); + if (errorCallback != null) { + errorCallbackOld.onWebRtcAudioTrackError(errorMessage); + } + if (errorCallback != null) { + errorCallback.onWebRtcAudioTrackError(errorMessage); + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioUtils.java b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioUtils.java new file mode 100644 index 0000000000..3d66923a01 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioUtils.java @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.voiceengine; + +import android.content.Context; +import android.content.pm.PackageManager; +import android.os.Build; +import android.os.Process; +import java.lang.Thread; +import java.util.Arrays; +import java.util.List; +import org.webrtc.Logging; + +public final class WebRtcAudioUtils { + private static final String TAG = "WebRtcAudioUtils"; + + // List of devices where we have seen issues (e.g. bad audio quality) using + // the low latency output mode in combination with OpenSL ES. + // The device name is given by Build.MODEL. + private static final String[] BLACKLISTED_OPEN_SL_ES_MODELS = new String[] { + // It is recommended to maintain a list of blacklisted models outside + // this package and instead call + // WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true) + // from the client for devices where OpenSL ES shall be disabled. + }; + + // List of devices where it has been verified that the built-in effect + // bad and where it makes sense to avoid using it and instead rely on the + // native WebRTC version instead. The device name is given by Build.MODEL. + private static final String[] BLACKLISTED_AEC_MODELS = new String[] { + // It is recommended to maintain a list of blacklisted models outside + // this package and instead call setWebRtcBasedAcousticEchoCanceler(true) + // from the client for devices where the built-in AEC shall be disabled. + }; + private static final String[] BLACKLISTED_NS_MODELS = new String[] { + // It is recommended to maintain a list of blacklisted models outside + // this package and instead call setWebRtcBasedNoiseSuppressor(true) + // from the client for devices where the built-in NS shall be disabled. + }; + + // Use 16kHz as the default sample rate. A higher sample rate might prevent + // us from supporting communication mode on some older (e.g. ICS) devices. + private static final int DEFAULT_SAMPLE_RATE_HZ = 16000; + private static int defaultSampleRateHz = DEFAULT_SAMPLE_RATE_HZ; + // Set to true if setDefaultSampleRateHz() has been called. + private static boolean isDefaultSampleRateOverridden = false; + + // By default, utilize hardware based audio effects for AEC and NS when + // available. + private static boolean useWebRtcBasedAcousticEchoCanceler = false; + private static boolean useWebRtcBasedNoiseSuppressor = false; + + // Call these methods if any hardware based effect shall be replaced by a + // software based version provided by the WebRTC stack instead. + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setWebRtcBasedAcousticEchoCanceler(boolean enable) { + useWebRtcBasedAcousticEchoCanceler = enable; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setWebRtcBasedNoiseSuppressor(boolean enable) { + useWebRtcBasedNoiseSuppressor = enable; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setWebRtcBasedAutomaticGainControl(boolean enable) { + // TODO(henrika): deprecated; remove when no longer used by any client. + Logging.w(TAG, "setWebRtcBasedAutomaticGainControl() is deprecated"); + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized boolean useWebRtcBasedAcousticEchoCanceler() { + if (useWebRtcBasedAcousticEchoCanceler) { + Logging.w(TAG, "Overriding default behavior; now using WebRTC AEC!"); + } + return useWebRtcBasedAcousticEchoCanceler; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized boolean useWebRtcBasedNoiseSuppressor() { + if (useWebRtcBasedNoiseSuppressor) { + Logging.w(TAG, "Overriding default behavior; now using WebRTC NS!"); + } + return useWebRtcBasedNoiseSuppressor; + } + + // TODO(henrika): deprecated; remove when no longer used by any client. + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized boolean useWebRtcBasedAutomaticGainControl() { + // Always return true here to avoid trying to use any built-in AGC. + return true; + } + + // Returns true if the device supports an audio effect (AEC or NS). + // Four conditions must be fulfilled if functions are to return true: + // 1) the platform must support the built-in (HW) effect, + // 2) explicit use (override) of a WebRTC based version must not be set, + // 3) the device must not be blacklisted for use of the effect, and + // 4) the UUID of the effect must be approved (some UUIDs can be excluded). + public static boolean isAcousticEchoCancelerSupported() { + return WebRtcAudioEffects.canUseAcousticEchoCanceler(); + } + public static boolean isNoiseSuppressorSupported() { + return WebRtcAudioEffects.canUseNoiseSuppressor(); + } + // TODO(henrika): deprecated; remove when no longer used by any client. + public static boolean isAutomaticGainControlSupported() { + // Always return false here to avoid trying to use any built-in AGC. + return false; + } + + // Call this method if the default handling of querying the native sample + // rate shall be overridden. Can be useful on some devices where the + // available Android APIs are known to return invalid results. + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized void setDefaultSampleRateHz(int sampleRateHz) { + isDefaultSampleRateOverridden = true; + defaultSampleRateHz = sampleRateHz; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized boolean isDefaultSampleRateOverridden() { + return isDefaultSampleRateOverridden; + } + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public static synchronized int getDefaultSampleRateHz() { + return defaultSampleRateHz; + } + + public static List getBlackListedModelsForAecUsage() { + return Arrays.asList(WebRtcAudioUtils.BLACKLISTED_AEC_MODELS); + } + + public static List getBlackListedModelsForNsUsage() { + return Arrays.asList(WebRtcAudioUtils.BLACKLISTED_NS_MODELS); + } + + public static boolean runningOnJellyBeanMR1OrHigher() { + // November 2012: Android 4.2. API Level 17. + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1; + } + + public static boolean runningOnJellyBeanMR2OrHigher() { + // July 24, 2013: Android 4.3. API Level 18. + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2; + } + + public static boolean runningOnLollipopOrHigher() { + // API Level 21. + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP; + } + + public static boolean runningOnMarshmallowOrHigher() { + // API Level 23. + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.M; + } + + public static boolean runningOnNougatOrHigher() { + // API Level 24. + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.N; + } + + // Helper method for building a string of thread information. + public static String getThreadInfo() { + return "@[name=" + Thread.currentThread().getName() + ", id=" + Thread.currentThread().getId() + + "]"; + } + + // Returns true if we're running on emulator. + public static boolean runningOnEmulator() { + return Build.HARDWARE.equals("goldfish") && Build.BRAND.startsWith("generic_"); + } + + // Returns true if the device is blacklisted for OpenSL ES usage. + public static boolean deviceIsBlacklistedForOpenSLESUsage() { + List blackListedModels = Arrays.asList(BLACKLISTED_OPEN_SL_ES_MODELS); + return blackListedModels.contains(Build.MODEL); + } + + // Information about the current build, taken from system properties. + public static void logDeviceInfo(String tag) { + Logging.d(tag, "Android SDK: " + Build.VERSION.SDK_INT + ", " + + "Release: " + Build.VERSION.RELEASE + ", " + + "Brand: " + Build.BRAND + ", " + + "Device: " + Build.DEVICE + ", " + + "Id: " + Build.ID + ", " + + "Hardware: " + Build.HARDWARE + ", " + + "Manufacturer: " + Build.MANUFACTURER + ", " + + "Model: " + Build.MODEL + ", " + + "Product: " + Build.PRODUCT); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.cc new file mode 100644 index 0000000000..019714dae4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/opensles_common.h" + +#include + +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// Returns a string representation given an integer SL_RESULT_XXX code. +// The mapping can be found in . +const char* GetSLErrorString(size_t code) { + static const char* sl_error_strings[] = { + "SL_RESULT_SUCCESS", // 0 + "SL_RESULT_PRECONDITIONS_VIOLATED", // 1 + "SL_RESULT_PARAMETER_INVALID", // 2 + "SL_RESULT_MEMORY_FAILURE", // 3 + "SL_RESULT_RESOURCE_ERROR", // 4 + "SL_RESULT_RESOURCE_LOST", // 5 + "SL_RESULT_IO_ERROR", // 6 + "SL_RESULT_BUFFER_INSUFFICIENT", // 7 + "SL_RESULT_CONTENT_CORRUPTED", // 8 + "SL_RESULT_CONTENT_UNSUPPORTED", // 9 + "SL_RESULT_CONTENT_NOT_FOUND", // 10 + "SL_RESULT_PERMISSION_DENIED", // 11 + "SL_RESULT_FEATURE_UNSUPPORTED", // 12 + "SL_RESULT_INTERNAL_ERROR", // 13 + "SL_RESULT_UNKNOWN_ERROR", // 14 + "SL_RESULT_OPERATION_ABORTED", // 15 + "SL_RESULT_CONTROL_LOST", // 16 + }; + + if (code >= arraysize(sl_error_strings)) { + return "SL_RESULT_UNKNOWN_ERROR"; + } + return sl_error_strings[code]; +} + +SLDataFormat_PCM CreatePCMConfiguration(size_t channels, + int sample_rate, + size_t bits_per_sample) { + RTC_CHECK_EQ(bits_per_sample, SL_PCMSAMPLEFORMAT_FIXED_16); + SLDataFormat_PCM format; + format.formatType = SL_DATAFORMAT_PCM; + format.numChannels = static_cast(channels); + // Note that, the unit of sample rate is actually in milliHertz and not Hertz. + switch (sample_rate) { + case 8000: + format.samplesPerSec = SL_SAMPLINGRATE_8; + break; + case 16000: + format.samplesPerSec = SL_SAMPLINGRATE_16; + break; + case 22050: + format.samplesPerSec = SL_SAMPLINGRATE_22_05; + break; + case 32000: + format.samplesPerSec = SL_SAMPLINGRATE_32; + break; + case 44100: + format.samplesPerSec = SL_SAMPLINGRATE_44_1; + break; + case 48000: + format.samplesPerSec = SL_SAMPLINGRATE_48; + break; + case 64000: + format.samplesPerSec = SL_SAMPLINGRATE_64; + break; + case 88200: + format.samplesPerSec = SL_SAMPLINGRATE_88_2; + break; + case 96000: + format.samplesPerSec = SL_SAMPLINGRATE_96; + break; + default: + RTC_CHECK(false) << "Unsupported sample rate: " << sample_rate; + break; + } + format.bitsPerSample = SL_PCMSAMPLEFORMAT_FIXED_16; + format.containerSize = SL_PCMSAMPLEFORMAT_FIXED_16; + format.endianness = SL_BYTEORDER_LITTLEENDIAN; + if (format.numChannels == 1) { + format.channelMask = SL_SPEAKER_FRONT_CENTER; + } else if (format.numChannels == 2) { + format.channelMask = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT; + } else { + RTC_CHECK(false) << "Unsupported number of channels: " + << format.numChannels; + } + return format; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.h new file mode 100644 index 0000000000..90988eef5a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_common.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_COMMON_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_COMMON_H_ + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +// Returns a string representation given an integer SL_RESULT_XXX code. +// The mapping can be found in . +const char* GetSLErrorString(size_t code); + +// Configures an SL_DATAFORMAT_PCM structure based on native audio parameters. +SLDataFormat_PCM CreatePCMConfiguration(size_t channels, + int sample_rate, + size_t bits_per_sample); + +// Helper class for using SLObjectItf interfaces. +template +class ScopedSLObject { + public: + ScopedSLObject() : obj_(nullptr) {} + + ~ScopedSLObject() { Reset(); } + + SLType* Receive() { + RTC_DCHECK(!obj_); + return &obj_; + } + + SLDerefType operator->() { return *obj_; } + + SLType Get() const { return obj_; } + + void Reset() { + if (obj_) { + (*obj_)->Destroy(obj_); + obj_ = nullptr; + } + } + + private: + SLType obj_; +}; + +typedef ScopedSLObject ScopedSLObjectItf; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.cc new file mode 100644 index 0000000000..7ac691236b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.cc @@ -0,0 +1,430 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/opensles_player.h" + +#include + +#include "api/array_view.h" +#include "modules/audio_device/android/audio_common.h" +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/fine_audio_buffer.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/timeutils.h" + +#define TAG "OpenSLESPlayer" +#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) +#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) +#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) + +#define RETURN_ON_ERROR(op, ...) \ + do { \ + SLresult err = (op); \ + if (err != SL_RESULT_SUCCESS) { \ + ALOGE("%s failed: %s", #op, GetSLErrorString(err)); \ + return __VA_ARGS__; \ + } \ + } while (0) + +namespace webrtc { + +OpenSLESPlayer::OpenSLESPlayer(AudioManager* audio_manager) + : audio_manager_(audio_manager), + audio_parameters_(audio_manager->GetPlayoutAudioParameters()), + audio_device_buffer_(nullptr), + initialized_(false), + playing_(false), + buffer_index_(0), + engine_(nullptr), + player_(nullptr), + simple_buffer_queue_(nullptr), + volume_(nullptr), + last_play_time_(0) { + ALOGD("ctor%s", GetThreadInfo().c_str()); + // Use native audio output parameters provided by the audio manager and + // define the PCM format structure. + pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(), + audio_parameters_.sample_rate(), + audio_parameters_.bits_per_sample()); + // Detach from this thread since we want to use the checker to verify calls + // from the internal audio thread. + thread_checker_opensles_.DetachFromThread(); +} + +OpenSLESPlayer::~OpenSLESPlayer() { + ALOGD("dtor%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Terminate(); + DestroyAudioPlayer(); + DestroyMix(); + engine_ = nullptr; + RTC_DCHECK(!engine_); + RTC_DCHECK(!output_mix_.Get()); + RTC_DCHECK(!player_); + RTC_DCHECK(!simple_buffer_queue_); + RTC_DCHECK(!volume_); +} + +int OpenSLESPlayer::Init() { + ALOGD("Init%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (audio_parameters_.channels() == 2) { + // TODO(henrika): FineAudioBuffer needs more work to support stereo. + ALOGE("OpenSLESPlayer does not support stereo"); + return -1; + } + return 0; +} + +int OpenSLESPlayer::Terminate() { + ALOGD("Terminate%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + StopPlayout(); + return 0; +} + +int OpenSLESPlayer::InitPlayout() { + ALOGD("InitPlayout%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!initialized_); + RTC_DCHECK(!playing_); + if (!ObtainEngineInterface()) { + ALOGE("Failed to obtain SL Engine interface"); + return -1; + } + CreateMix(); + initialized_ = true; + buffer_index_ = 0; + return 0; +} + +int OpenSLESPlayer::StartPlayout() { + ALOGD("StartPlayout%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(initialized_); + RTC_DCHECK(!playing_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetPlayout(); + } + // The number of lower latency audio players is limited, hence we create the + // audio player in Start() and destroy it in Stop(). + CreateAudioPlayer(); + // Fill up audio buffers to avoid initial glitch and to ensure that playback + // starts when mode is later changed to SL_PLAYSTATE_PLAYING. + // TODO(henrika): we can save some delay by only making one call to + // EnqueuePlayoutData. Most likely not worth the risk of adding a glitch. + last_play_time_ = rtc::Time(); + for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) { + EnqueuePlayoutData(true); + } + // Start streaming data by setting the play state to SL_PLAYSTATE_PLAYING. + // For a player object, when the object is in the SL_PLAYSTATE_PLAYING + // state, adding buffers will implicitly start playback. + RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_PLAYING), -1); + playing_ = (GetPlayState() == SL_PLAYSTATE_PLAYING); + RTC_DCHECK(playing_); + return 0; +} + +int OpenSLESPlayer::StopPlayout() { + ALOGD("StopPlayout%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!initialized_ || !playing_) { + return 0; + } + // Stop playing by setting the play state to SL_PLAYSTATE_STOPPED. + RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_STOPPED), -1); + // Clear the buffer queue to flush out any remaining data. + RETURN_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_), -1); +#if RTC_DCHECK_IS_ON + // Verify that the buffer queue is in fact cleared as it should. + SLAndroidSimpleBufferQueueState buffer_queue_state; + (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &buffer_queue_state); + RTC_DCHECK_EQ(0, buffer_queue_state.count); + RTC_DCHECK_EQ(0, buffer_queue_state.index); +#endif + // The number of lower latency audio players is limited, hence we create the + // audio player in Start() and destroy it in Stop(). + DestroyAudioPlayer(); + thread_checker_opensles_.DetachFromThread(); + initialized_ = false; + playing_ = false; + return 0; +} + +int OpenSLESPlayer::SpeakerVolumeIsAvailable(bool& available) { + available = false; + return 0; +} + +int OpenSLESPlayer::MaxSpeakerVolume(uint32_t& maxVolume) const { + return -1; +} + +int OpenSLESPlayer::MinSpeakerVolume(uint32_t& minVolume) const { + return -1; +} + +int OpenSLESPlayer::SetSpeakerVolume(uint32_t volume) { + return -1; +} + +int OpenSLESPlayer::SpeakerVolume(uint32_t& volume) const { + return -1; +} + +void OpenSLESPlayer::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + ALOGD("AttachAudioBuffer"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + audio_device_buffer_ = audioBuffer; + const int sample_rate_hz = audio_parameters_.sample_rate(); + ALOGD("SetPlayoutSampleRate(%d)", sample_rate_hz); + audio_device_buffer_->SetPlayoutSampleRate(sample_rate_hz); + const size_t channels = audio_parameters_.channels(); + ALOGD("SetPlayoutChannels(%" PRIuS ")", channels); + audio_device_buffer_->SetPlayoutChannels(channels); + RTC_CHECK(audio_device_buffer_); + AllocateDataBuffers(); +} + +void OpenSLESPlayer::AllocateDataBuffers() { + ALOGD("AllocateDataBuffers"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!simple_buffer_queue_); + RTC_CHECK(audio_device_buffer_); + // Create a modified audio buffer class which allows us to ask for any number + // of samples (and not only multiple of 10ms) to match the native OpenSL ES + // buffer size. The native buffer size corresponds to the + // PROPERTY_OUTPUT_FRAMES_PER_BUFFER property which is the number of audio + // frames that the HAL (Hardware Abstraction Layer) buffer can hold. It is + // recommended to construct audio buffers so that they contain an exact + // multiple of this number. If so, callbacks will occur at regular intervals, + // which reduces jitter. + const size_t buffer_size_in_bytes = audio_parameters_.GetBytesPerBuffer(); + ALOGD("native buffer size: %" PRIuS, buffer_size_in_bytes); + ALOGD("native buffer size in ms: %.2f", + audio_parameters_.GetBufferSizeInMilliseconds()); + fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_, + audio_parameters_.sample_rate(), + 2 * buffer_size_in_bytes)); + // Allocated memory for audio buffers. + for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) { + audio_buffers_[i].reset(new SLint8[buffer_size_in_bytes]); + } +} + +bool OpenSLESPlayer::ObtainEngineInterface() { + ALOGD("ObtainEngineInterface"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (engine_) + return true; + // Get access to (or create if not already existing) the global OpenSL Engine + // object. + SLObjectItf engine_object = audio_manager_->GetOpenSLEngine(); + if (engine_object == nullptr) { + ALOGE("Failed to access the global OpenSL engine"); + return false; + } + // Get the SL Engine Interface which is implicit. + RETURN_ON_ERROR( + (*engine_object)->GetInterface(engine_object, SL_IID_ENGINE, &engine_), + false); + return true; +} + +bool OpenSLESPlayer::CreateMix() { + ALOGD("CreateMix"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(engine_); + if (output_mix_.Get()) + return true; + + // Create the ouput mix on the engine object. No interfaces will be used. + RETURN_ON_ERROR((*engine_)->CreateOutputMix(engine_, output_mix_.Receive(), 0, + nullptr, nullptr), + false); + RETURN_ON_ERROR(output_mix_->Realize(output_mix_.Get(), SL_BOOLEAN_FALSE), + false); + return true; +} + +void OpenSLESPlayer::DestroyMix() { + ALOGD("DestroyMix"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!output_mix_.Get()) + return; + output_mix_.Reset(); +} + +bool OpenSLESPlayer::CreateAudioPlayer() { + ALOGD("CreateAudioPlayer"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(output_mix_.Get()); + if (player_object_.Get()) + return true; + RTC_DCHECK(!player_); + RTC_DCHECK(!simple_buffer_queue_); + RTC_DCHECK(!volume_); + + // source: Android Simple Buffer Queue Data Locator is source. + SLDataLocator_AndroidSimpleBufferQueue simple_buffer_queue = { + SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, + static_cast(kNumOfOpenSLESBuffers)}; + SLDataSource audio_source = {&simple_buffer_queue, &pcm_format_}; + + // sink: OutputMix-based data is sink. + SLDataLocator_OutputMix locator_output_mix = {SL_DATALOCATOR_OUTPUTMIX, + output_mix_.Get()}; + SLDataSink audio_sink = {&locator_output_mix, nullptr}; + + // Define interfaces that we indend to use and realize. + const SLInterfaceID interface_ids[] = {SL_IID_ANDROIDCONFIGURATION, + SL_IID_BUFFERQUEUE, SL_IID_VOLUME}; + const SLboolean interface_required[] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, + SL_BOOLEAN_TRUE}; + + // Create the audio player on the engine interface. + RETURN_ON_ERROR( + (*engine_)->CreateAudioPlayer( + engine_, player_object_.Receive(), &audio_source, &audio_sink, + arraysize(interface_ids), interface_ids, interface_required), + false); + + // Use the Android configuration interface to set platform-specific + // parameters. Should be done before player is realized. + SLAndroidConfigurationItf player_config; + RETURN_ON_ERROR( + player_object_->GetInterface(player_object_.Get(), + SL_IID_ANDROIDCONFIGURATION, &player_config), + false); + // Set audio player configuration to SL_ANDROID_STREAM_VOICE which + // corresponds to android.media.AudioManager.STREAM_VOICE_CALL. + SLint32 stream_type = SL_ANDROID_STREAM_VOICE; + RETURN_ON_ERROR( + (*player_config) + ->SetConfiguration(player_config, SL_ANDROID_KEY_STREAM_TYPE, + &stream_type, sizeof(SLint32)), + false); + + // Realize the audio player object after configuration has been set. + RETURN_ON_ERROR( + player_object_->Realize(player_object_.Get(), SL_BOOLEAN_FALSE), false); + + // Get the SLPlayItf interface on the audio player. + RETURN_ON_ERROR( + player_object_->GetInterface(player_object_.Get(), SL_IID_PLAY, &player_), + false); + + // Get the SLAndroidSimpleBufferQueueItf interface on the audio player. + RETURN_ON_ERROR( + player_object_->GetInterface(player_object_.Get(), SL_IID_BUFFERQUEUE, + &simple_buffer_queue_), + false); + + // Register callback method for the Android Simple Buffer Queue interface. + // This method will be called when the native audio layer needs audio data. + RETURN_ON_ERROR((*simple_buffer_queue_) + ->RegisterCallback(simple_buffer_queue_, + SimpleBufferQueueCallback, this), + false); + + // Get the SLVolumeItf interface on the audio player. + RETURN_ON_ERROR(player_object_->GetInterface(player_object_.Get(), + SL_IID_VOLUME, &volume_), + false); + + // TODO(henrika): might not be required to set volume to max here since it + // seems to be default on most devices. Might be required for unit tests. + // RETURN_ON_ERROR((*volume_)->SetVolumeLevel(volume_, 0), false); + + return true; +} + +void OpenSLESPlayer::DestroyAudioPlayer() { + ALOGD("DestroyAudioPlayer"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!player_object_.Get()) + return; + (*simple_buffer_queue_) + ->RegisterCallback(simple_buffer_queue_, nullptr, nullptr); + player_object_.Reset(); + player_ = nullptr; + simple_buffer_queue_ = nullptr; + volume_ = nullptr; +} + +// static +void OpenSLESPlayer::SimpleBufferQueueCallback( + SLAndroidSimpleBufferQueueItf caller, + void* context) { + OpenSLESPlayer* stream = reinterpret_cast(context); + stream->FillBufferQueue(); +} + +void OpenSLESPlayer::FillBufferQueue() { + RTC_DCHECK(thread_checker_opensles_.CalledOnValidThread()); + SLuint32 state = GetPlayState(); + if (state != SL_PLAYSTATE_PLAYING) { + ALOGW("Buffer callback in non-playing state!"); + return; + } + EnqueuePlayoutData(false); +} + +void OpenSLESPlayer::EnqueuePlayoutData(bool silence) { + // Check delta time between two successive callbacks and provide a warning + // if it becomes very large. + // TODO(henrika): using 150ms as upper limit but this value is rather random. + const uint32_t current_time = rtc::Time(); + const uint32_t diff = current_time - last_play_time_; + if (diff > 150) { + ALOGW("Bad OpenSL ES playout timing, dT=%u [ms]", diff); + } + last_play_time_ = current_time; + SLint8* audio_ptr = audio_buffers_[buffer_index_].get(); + if (silence) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + // Avoid aquiring real audio data from WebRTC and fill the buffer with + // zeros instead. Used to prime the buffer with silence and to avoid asking + // for audio data from two different threads. + memset(audio_ptr, 0, audio_parameters_.GetBytesPerBuffer()); + } else { + RTC_DCHECK(thread_checker_opensles_.CalledOnValidThread()); + // Read audio data from the WebRTC source using the FineAudioBuffer object + // to adjust for differences in buffer size between WebRTC (10ms) and native + // OpenSL ES. + fine_audio_buffer_->GetPlayoutData(rtc::ArrayView( + audio_ptr, audio_parameters_.GetBytesPerBuffer())); + } + // Enqueue the decoded audio buffer for playback. + SLresult err = (*simple_buffer_queue_) + ->Enqueue(simple_buffer_queue_, audio_ptr, + audio_parameters_.GetBytesPerBuffer()); + if (SL_RESULT_SUCCESS != err) { + ALOGE("Enqueue failed: %d", err); + } + buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers; +} + +SLuint32 OpenSLESPlayer::GetPlayState() const { + RTC_DCHECK(player_); + SLuint32 state; + SLresult err = (*player_)->GetPlayState(player_, &state); + if (SL_RESULT_SUCCESS != err) { + ALOGE("GetPlayState failed: %d", err); + } + return state; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.h new file mode 100644 index 0000000000..167c356330 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_player.h @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_PLAYER_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_PLAYER_H_ + +#include +#include +#include + +#include "modules/audio_device/android/audio_common.h" +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/android/opensles_common.h" +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/include/audio_device_defines.h" +#include "modules/utility/include/helpers_android.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +class FineAudioBuffer; + +// Implements 16-bit mono PCM audio output support for Android using the +// C based OpenSL ES API. No calls from C/C++ to Java using JNI is done. +// +// An instance must be created and destroyed on one and the same thread. +// All public methods must also be called on the same thread. A thread checker +// will RTC_DCHECK if any method is called on an invalid thread. Decoded audio +// buffers are requested on a dedicated internal thread managed by the OpenSL +// ES layer. +// +// The existing design forces the user to call InitPlayout() after Stoplayout() +// to be able to call StartPlayout() again. This is inline with how the Java- +// based implementation works. +// +// OpenSL ES is a native C API which have no Dalvik-related overhead such as +// garbage collection pauses and it supports reduced audio output latency. +// If the device doesn't claim this feature but supports API level 9 (Android +// platform version 2.3) or later, then we can still use the OpenSL ES APIs but +// the output latency may be higher. +class OpenSLESPlayer { + public: + // Beginning with API level 17 (Android 4.2), a buffer count of 2 or more is + // required for lower latency. Beginning with API level 18 (Android 4.3), a + // buffer count of 1 is sufficient for lower latency. In addition, the buffer + // size and sample rate must be compatible with the device's native output + // configuration provided via the audio manager at construction. + // TODO(henrika): perhaps set this value dynamically based on OS version. + static const int kNumOfOpenSLESBuffers = 2; + + explicit OpenSLESPlayer(AudioManager* audio_manager); + ~OpenSLESPlayer(); + + int Init(); + int Terminate(); + + int InitPlayout(); + bool PlayoutIsInitialized() const { return initialized_; } + + int StartPlayout(); + int StopPlayout(); + bool Playing() const { return playing_; } + + int SpeakerVolumeIsAvailable(bool& available); + int SetSpeakerVolume(uint32_t volume); + int SpeakerVolume(uint32_t& volume) const; + int MaxSpeakerVolume(uint32_t& maxVolume) const; + int MinSpeakerVolume(uint32_t& minVolume) const; + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); + + private: + // These callback methods are called when data is required for playout. + // They are both called from an internal "OpenSL ES thread" which is not + // attached to the Dalvik VM. + static void SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf caller, + void* context); + void FillBufferQueue(); + // Reads audio data in PCM format using the AudioDeviceBuffer. + // Can be called both on the main thread (during Start()) and from the + // internal audio thread while output streaming is active. + // If the |silence| flag is set, the audio is filled with zeros instead of + // asking the WebRTC layer for real audio data. This procedure is also known + // as audio priming. + void EnqueuePlayoutData(bool silence); + + // Allocate memory for audio buffers which will be used to render audio + // via the SLAndroidSimpleBufferQueueItf interface. + void AllocateDataBuffers(); + + // Obtaines the SL Engine Interface from the existing global Engine object. + // The interface exposes creation methods of all the OpenSL ES object types. + // This method defines the |engine_| member variable. + bool ObtainEngineInterface(); + + // Creates/destroys the output mix object. + bool CreateMix(); + void DestroyMix(); + + // Creates/destroys the audio player and the simple-buffer object. + // Also creates the volume object. + bool CreateAudioPlayer(); + void DestroyAudioPlayer(); + + SLuint32 GetPlayState() const; + + // Ensures that methods are called from the same thread as this object is + // created on. + rtc::ThreadChecker thread_checker_; + + // Stores thread ID in first call to SimpleBufferQueueCallback() from internal + // non-application thread which is not attached to the Dalvik JVM. + // Detached during construction of this object. + rtc::ThreadChecker thread_checker_opensles_; + + // Raw pointer to the audio manager injected at construction. Used to cache + // audio parameters and to access the global SL engine object needed by the + // ObtainEngineInterface() method. The audio manager outlives any instance of + // this class. + AudioManager* audio_manager_; + + // Contains audio parameters provided to this class at construction by the + // AudioManager. + const AudioParameters audio_parameters_; + + // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the + // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). + AudioDeviceBuffer* audio_device_buffer_; + + bool initialized_; + bool playing_; + + // PCM-type format definition. + // TODO(henrika): add support for SLAndroidDataFormat_PCM_EX (android-21) if + // 32-bit float representation is needed. + SLDataFormat_PCM pcm_format_; + + // Queue of audio buffers to be used by the player object for rendering + // audio. They will be used in a Round-robin way and the size of each buffer + // is given by FineAudioBuffer::RequiredBufferSizeBytes(). + std::unique_ptr audio_buffers_[kNumOfOpenSLESBuffers]; + + // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data + // in chunks of 10ms. It then allows for this data to be pulled in + // a finer or coarser granularity. I.e. interacting with this class instead + // of directly with the AudioDeviceBuffer one can ask for any number of + // audio data samples. + // Example: native buffer size can be 192 audio frames at 48kHz sample rate. + // WebRTC will provide 480 audio frames per 10ms but OpenSL ES asks for 192 + // in each callback (one every 4th ms). This class can then ask for 192 and + // the FineAudioBuffer will ask WebRTC for new data approximately only every + // second callback and also cache non-utilized audio. + std::unique_ptr fine_audio_buffer_; + + // Keeps track of active audio buffer 'n' in the audio_buffers_[n] queue. + // Example (kNumOfOpenSLESBuffers = 2): counts 0, 1, 0, 1, ... + int buffer_index_; + + // This interface exposes creation methods for all the OpenSL ES object types. + // It is the OpenSL ES API entry point. + SLEngineItf engine_; + + // Output mix object to be used by the player object. + webrtc::ScopedSLObjectItf output_mix_; + + // The audio player media object plays out audio to the speakers. It also + // supports volume control. + webrtc::ScopedSLObjectItf player_object_; + + // This interface is supported on the audio player and it controls the state + // of the audio player. + SLPlayItf player_; + + // The Android Simple Buffer Queue interface is supported on the audio player + // and it provides methods to send audio data from the source to the audio + // player for rendering. + SLAndroidSimpleBufferQueueItf simple_buffer_queue_; + + // This interface exposes controls for manipulating the object’s audio volume + // properties. This interface is supported on the Audio Player object. + SLVolumeItf volume_; + + // Last time the OpenSL ES layer asked for audio data to play out. + uint32_t last_play_time_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_PLAYER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.cc b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.cc new file mode 100644 index 0000000000..123406889c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.cc @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/android/opensles_recorder.h" + +#include + +#include "api/array_view.h" +#include "modules/audio_device/android/audio_common.h" +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/fine_audio_buffer.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/timeutils.h" + +#define TAG "OpenSLESRecorder" +#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) +#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) +#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) + +#define LOG_ON_ERROR(op) \ + [](SLresult err) { \ + if (err != SL_RESULT_SUCCESS) { \ + ALOGE("%s:%d %s failed: %s", __FILE__, __LINE__, #op, \ + GetSLErrorString(err)); \ + return true; \ + } \ + return false; \ + }(op) + +namespace webrtc { + +OpenSLESRecorder::OpenSLESRecorder(AudioManager* audio_manager) + : audio_manager_(audio_manager), + audio_parameters_(audio_manager->GetRecordAudioParameters()), + audio_device_buffer_(nullptr), + initialized_(false), + recording_(false), + engine_(nullptr), + recorder_(nullptr), + simple_buffer_queue_(nullptr), + buffer_index_(0), + last_rec_time_(0) { + ALOGD("ctor%s", GetThreadInfo().c_str()); + // Detach from this thread since we want to use the checker to verify calls + // from the internal audio thread. + thread_checker_opensles_.DetachFromThread(); + // Use native audio output parameters provided by the audio manager and + // define the PCM format structure. + pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(), + audio_parameters_.sample_rate(), + audio_parameters_.bits_per_sample()); +} + +OpenSLESRecorder::~OpenSLESRecorder() { + ALOGD("dtor%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Terminate(); + DestroyAudioRecorder(); + engine_ = nullptr; + RTC_DCHECK(!engine_); + RTC_DCHECK(!recorder_); + RTC_DCHECK(!simple_buffer_queue_); +} + +int OpenSLESRecorder::Init() { + ALOGD("Init%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (audio_parameters_.channels() == 2) { + // TODO(henrika): FineAudioBuffer needs more work to support stereo. + ALOGE("OpenSLESRecorder does not support stereo"); + return -1; + } + return 0; +} + +int OpenSLESRecorder::Terminate() { + ALOGD("Terminate%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + StopRecording(); + return 0; +} + +int OpenSLESRecorder::InitRecording() { + ALOGD("InitRecording%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!initialized_); + RTC_DCHECK(!recording_); + if (!ObtainEngineInterface()) { + ALOGE("Failed to obtain SL Engine interface"); + return -1; + } + CreateAudioRecorder(); + initialized_ = true; + buffer_index_ = 0; + return 0; +} + +int OpenSLESRecorder::StartRecording() { + ALOGD("StartRecording%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(initialized_); + RTC_DCHECK(!recording_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetRecord(); + } + // Add buffers to the queue before changing state to SL_RECORDSTATE_RECORDING + // to ensure that recording starts as soon as the state is modified. On some + // devices, SLAndroidSimpleBufferQueue::Clear() used in Stop() does not flush + // the buffers as intended and we therefore check the number of buffers + // already queued first. Enqueue() can return SL_RESULT_BUFFER_INSUFFICIENT + // otherwise. + int num_buffers_in_queue = GetBufferCount(); + for (int i = 0; i < kNumOfOpenSLESBuffers - num_buffers_in_queue; ++i) { + if (!EnqueueAudioBuffer()) { + recording_ = false; + return -1; + } + } + num_buffers_in_queue = GetBufferCount(); + RTC_DCHECK_EQ(num_buffers_in_queue, kNumOfOpenSLESBuffers); + LogBufferState(); + // Start audio recording by changing the state to SL_RECORDSTATE_RECORDING. + // Given that buffers are already enqueued, recording should start at once. + // The macro returns -1 if recording fails to start. + last_rec_time_ = rtc::Time(); + if (LOG_ON_ERROR( + (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_RECORDING))) { + return -1; + } + recording_ = (GetRecordState() == SL_RECORDSTATE_RECORDING); + RTC_DCHECK(recording_); + return 0; +} + +int OpenSLESRecorder::StopRecording() { + ALOGD("StopRecording%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!initialized_ || !recording_) { + return 0; + } + // Stop recording by setting the record state to SL_RECORDSTATE_STOPPED. + if (LOG_ON_ERROR( + (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_STOPPED))) { + return -1; + } + // Clear the buffer queue to get rid of old data when resuming recording. + if (LOG_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_))) { + return -1; + } + thread_checker_opensles_.DetachFromThread(); + initialized_ = false; + recording_ = false; + return 0; +} + +void OpenSLESRecorder::AttachAudioBuffer(AudioDeviceBuffer* audio_buffer) { + ALOGD("AttachAudioBuffer"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_CHECK(audio_buffer); + audio_device_buffer_ = audio_buffer; + // Ensure that the audio device buffer is informed about the native sample + // rate used on the recording side. + const int sample_rate_hz = audio_parameters_.sample_rate(); + ALOGD("SetRecordingSampleRate(%d)", sample_rate_hz); + audio_device_buffer_->SetRecordingSampleRate(sample_rate_hz); + // Ensure that the audio device buffer is informed about the number of + // channels preferred by the OS on the recording side. + const size_t channels = audio_parameters_.channels(); + ALOGD("SetRecordingChannels(%" PRIuS ")", channels); + audio_device_buffer_->SetRecordingChannels(channels); + // Allocated memory for internal data buffers given existing audio parameters. + AllocateDataBuffers(); +} + +int OpenSLESRecorder::EnableBuiltInAEC(bool enable) { + ALOGD("EnableBuiltInAEC(%d)", enable); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGE("Not implemented"); + return 0; +} + +int OpenSLESRecorder::EnableBuiltInAGC(bool enable) { + ALOGD("EnableBuiltInAGC(%d)", enable); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGE("Not implemented"); + return 0; +} + +int OpenSLESRecorder::EnableBuiltInNS(bool enable) { + ALOGD("EnableBuiltInNS(%d)", enable); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + ALOGE("Not implemented"); + return 0; +} + +bool OpenSLESRecorder::ObtainEngineInterface() { + ALOGD("ObtainEngineInterface"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (engine_) + return true; + // Get access to (or create if not already existing) the global OpenSL Engine + // object. + SLObjectItf engine_object = audio_manager_->GetOpenSLEngine(); + if (engine_object == nullptr) { + ALOGE("Failed to access the global OpenSL engine"); + return false; + } + // Get the SL Engine Interface which is implicit. + if (LOG_ON_ERROR( + (*engine_object) + ->GetInterface(engine_object, SL_IID_ENGINE, &engine_))) { + return false; + } + return true; +} + +bool OpenSLESRecorder::CreateAudioRecorder() { + ALOGD("CreateAudioRecorder"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (recorder_object_.Get()) + return true; + RTC_DCHECK(!recorder_); + RTC_DCHECK(!simple_buffer_queue_); + + // Audio source configuration. + SLDataLocator_IODevice mic_locator = {SL_DATALOCATOR_IODEVICE, + SL_IODEVICE_AUDIOINPUT, + SL_DEFAULTDEVICEID_AUDIOINPUT, NULL}; + SLDataSource audio_source = {&mic_locator, NULL}; + + // Audio sink configuration. + SLDataLocator_AndroidSimpleBufferQueue buffer_queue = { + SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, + static_cast(kNumOfOpenSLESBuffers)}; + SLDataSink audio_sink = {&buffer_queue, &pcm_format_}; + + // Create the audio recorder object (requires the RECORD_AUDIO permission). + // Do not realize the recorder yet. Set the configuration first. + const SLInterfaceID interface_id[] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE, + SL_IID_ANDROIDCONFIGURATION}; + const SLboolean interface_required[] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE}; + if (LOG_ON_ERROR((*engine_)->CreateAudioRecorder( + engine_, recorder_object_.Receive(), &audio_source, &audio_sink, + arraysize(interface_id), interface_id, interface_required))) { + return false; + } + + // Configure the audio recorder (before it is realized). + SLAndroidConfigurationItf recorder_config; + if (LOG_ON_ERROR((recorder_object_->GetInterface(recorder_object_.Get(), + SL_IID_ANDROIDCONFIGURATION, + &recorder_config)))) { + return false; + } + + // Uses the default microphone tuned for audio communication. + // Note that, SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION leads to a fast + // track but also excludes usage of required effects like AEC, AGC and NS. + // SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION + SLint32 stream_type = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION; + if (LOG_ON_ERROR(((*recorder_config) + ->SetConfiguration(recorder_config, + SL_ANDROID_KEY_RECORDING_PRESET, + &stream_type, sizeof(SLint32))))) { + return false; + } + + // The audio recorder can now be realized (in synchronous mode). + if (LOG_ON_ERROR((recorder_object_->Realize(recorder_object_.Get(), + SL_BOOLEAN_FALSE)))) { + return false; + } + + // Get the implicit recorder interface (SL_IID_RECORD). + if (LOG_ON_ERROR((recorder_object_->GetInterface( + recorder_object_.Get(), SL_IID_RECORD, &recorder_)))) { + return false; + } + + // Get the simple buffer queue interface (SL_IID_ANDROIDSIMPLEBUFFERQUEUE). + // It was explicitly requested. + if (LOG_ON_ERROR((recorder_object_->GetInterface( + recorder_object_.Get(), SL_IID_ANDROIDSIMPLEBUFFERQUEUE, + &simple_buffer_queue_)))) { + return false; + } + + // Register the input callback for the simple buffer queue. + // This callback will be called when receiving new data from the device. + if (LOG_ON_ERROR(((*simple_buffer_queue_) + ->RegisterCallback(simple_buffer_queue_, + SimpleBufferQueueCallback, this)))) { + return false; + } + return true; +} + +void OpenSLESRecorder::DestroyAudioRecorder() { + ALOGD("DestroyAudioRecorder"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!recorder_object_.Get()) + return; + (*simple_buffer_queue_) + ->RegisterCallback(simple_buffer_queue_, nullptr, nullptr); + recorder_object_.Reset(); + recorder_ = nullptr; + simple_buffer_queue_ = nullptr; +} + +void OpenSLESRecorder::SimpleBufferQueueCallback( + SLAndroidSimpleBufferQueueItf buffer_queue, + void* context) { + OpenSLESRecorder* stream = static_cast(context); + stream->ReadBufferQueue(); +} + +void OpenSLESRecorder::AllocateDataBuffers() { + ALOGD("AllocateDataBuffers"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!simple_buffer_queue_); + RTC_CHECK(audio_device_buffer_); + // Create a modified audio buffer class which allows us to deliver any number + // of samples (and not only multiple of 10ms) to match the native audio unit + // buffer size. + ALOGD("frames per native buffer: %" PRIuS, + audio_parameters_.frames_per_buffer()); + ALOGD("frames per 10ms buffer: %" PRIuS, + audio_parameters_.frames_per_10ms_buffer()); + ALOGD("bytes per native buffer: %" PRIuS, + audio_parameters_.GetBytesPerBuffer()); + ALOGD("native sample rate: %d", audio_parameters_.sample_rate()); + RTC_DCHECK(audio_device_buffer_); + fine_audio_buffer_.reset( + new FineAudioBuffer(audio_device_buffer_, audio_parameters_.sample_rate(), + 2 * audio_parameters_.GetBytesPerBuffer())); + // Allocate queue of audio buffers that stores recorded audio samples. + const int data_size_bytes = audio_parameters_.GetBytesPerBuffer(); + audio_buffers_.reset(new std::unique_ptr[kNumOfOpenSLESBuffers]); + for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) { + audio_buffers_[i].reset(new SLint8[data_size_bytes]); + } +} + +void OpenSLESRecorder::ReadBufferQueue() { + RTC_DCHECK(thread_checker_opensles_.CalledOnValidThread()); + SLuint32 state = GetRecordState(); + if (state != SL_RECORDSTATE_RECORDING) { + ALOGW("Buffer callback in non-recording state!"); + return; + } + // Check delta time between two successive callbacks and provide a warning + // if it becomes very large. + // TODO(henrika): using 150ms as upper limit but this value is rather random. + const uint32_t current_time = rtc::Time(); + const uint32_t diff = current_time - last_rec_time_; + if (diff > 150) { + ALOGW("Bad OpenSL ES record timing, dT=%u [ms]", diff); + } + last_rec_time_ = current_time; + // Send recorded audio data to the WebRTC sink. + // TODO(henrika): fix delay estimates. It is OK to use fixed values for now + // since there is no support to turn off built-in EC in combination with + // OpenSL ES anyhow. Hence, as is, the WebRTC based AEC (which would use + // these estimates) will never be active. + const size_t size_in_bytes = + static_cast(audio_parameters_.GetBytesPerBuffer()); + const int8_t* data = + static_cast(audio_buffers_[buffer_index_].get()); + fine_audio_buffer_->DeliverRecordedData( + rtc::ArrayView(data, size_in_bytes), 25, 25); + // Enqueue the utilized audio buffer and use if for recording again. + EnqueueAudioBuffer(); +} + +bool OpenSLESRecorder::EnqueueAudioBuffer() { + SLresult err = + (*simple_buffer_queue_) + ->Enqueue(simple_buffer_queue_, audio_buffers_[buffer_index_].get(), + audio_parameters_.GetBytesPerBuffer()); + if (SL_RESULT_SUCCESS != err) { + ALOGE("Enqueue failed: %s", GetSLErrorString(err)); + return false; + } + buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers; + return true; +} + +SLuint32 OpenSLESRecorder::GetRecordState() const { + RTC_DCHECK(recorder_); + SLuint32 state; + SLresult err = (*recorder_)->GetRecordState(recorder_, &state); + if (SL_RESULT_SUCCESS != err) { + ALOGE("GetRecordState failed: %s", GetSLErrorString(err)); + } + return state; +} + +SLAndroidSimpleBufferQueueState OpenSLESRecorder::GetBufferQueueState() const { + RTC_DCHECK(simple_buffer_queue_); + // state.count: Number of buffers currently in the queue. + // state.index: Index of the currently filling buffer. This is a linear index + // that keeps a cumulative count of the number of buffers recorded. + SLAndroidSimpleBufferQueueState state; + SLresult err = + (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &state); + if (SL_RESULT_SUCCESS != err) { + ALOGE("GetState failed: %s", GetSLErrorString(err)); + } + return state; +} + +void OpenSLESRecorder::LogBufferState() const { + SLAndroidSimpleBufferQueueState state = GetBufferQueueState(); + ALOGD("state.count:%d state.index:%d", state.count, state.index); +} + +SLuint32 OpenSLESRecorder::GetBufferCount() { + SLAndroidSimpleBufferQueueState state = GetBufferQueueState(); + return state.count; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.h b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.h new file mode 100644 index 0000000000..478c0340af --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/android/opensles_recorder.h @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_RECORDER_H_ +#define MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_RECORDER_H_ + +#include +#include +#include + +#include + +#include "modules/audio_device/android/audio_common.h" +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/android/opensles_common.h" +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/include/audio_device_defines.h" +#include "modules/utility/include/helpers_android.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +class FineAudioBuffer; + +// Implements 16-bit mono PCM audio input support for Android using the +// C based OpenSL ES API. No calls from C/C++ to Java using JNI is done. +// +// An instance must be created and destroyed on one and the same thread. +// All public methods must also be called on the same thread. A thread checker +// will RTC_DCHECK if any method is called on an invalid thread. Recorded audio +// buffers are provided on a dedicated internal thread managed by the OpenSL +// ES layer. +// +// The existing design forces the user to call InitRecording() after +// StopRecording() to be able to call StartRecording() again. This is inline +// with how the Java-based implementation works. +// +// As of API level 21, lower latency audio input is supported on select devices. +// To take advantage of this feature, first confirm that lower latency output is +// available. The capability for lower latency output is a prerequisite for the +// lower latency input feature. Then, create an AudioRecorder with the same +// sample rate and buffer size as would be used for output. OpenSL ES interfaces +// for input effects preclude the lower latency path. +// See https://developer.android.com/ndk/guides/audio/opensl-prog-notes.html +// for more details. +class OpenSLESRecorder { + public: + // Beginning with API level 17 (Android 4.2), a buffer count of 2 or more is + // required for lower latency. Beginning with API level 18 (Android 4.3), a + // buffer count of 1 is sufficient for lower latency. In addition, the buffer + // size and sample rate must be compatible with the device's native input + // configuration provided via the audio manager at construction. + // TODO(henrika): perhaps set this value dynamically based on OS version. + static const int kNumOfOpenSLESBuffers = 2; + + explicit OpenSLESRecorder(AudioManager* audio_manager); + ~OpenSLESRecorder(); + + int Init(); + int Terminate(); + + int InitRecording(); + bool RecordingIsInitialized() const { return initialized_; } + + int StartRecording(); + int StopRecording(); + bool Recording() const { return recording_; } + + void AttachAudioBuffer(AudioDeviceBuffer* audio_buffer); + + // TODO(henrika): add support using OpenSL ES APIs when available. + int EnableBuiltInAEC(bool enable); + int EnableBuiltInAGC(bool enable); + int EnableBuiltInNS(bool enable); + + private: + // Obtaines the SL Engine Interface from the existing global Engine object. + // The interface exposes creation methods of all the OpenSL ES object types. + // This method defines the |engine_| member variable. + bool ObtainEngineInterface(); + + // Creates/destroys the audio recorder and the simple-buffer queue object. + bool CreateAudioRecorder(); + void DestroyAudioRecorder(); + + // Allocate memory for audio buffers which will be used to capture audio + // via the SLAndroidSimpleBufferQueueItf interface. + void AllocateDataBuffers(); + + // These callback methods are called when data has been written to the input + // buffer queue. They are both called from an internal "OpenSL ES thread" + // which is not attached to the Dalvik VM. + static void SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf caller, + void* context); + void ReadBufferQueue(); + + // Wraps calls to SLAndroidSimpleBufferQueueState::Enqueue() and it can be + // called both on the main thread (but before recording has started) and from + // the internal audio thread while input streaming is active. It uses + // |simple_buffer_queue_| but no lock is needed since the initial calls from + // the main thread and the native callback thread are mutually exclusive. + bool EnqueueAudioBuffer(); + + // Returns the current recorder state. + SLuint32 GetRecordState() const; + + // Returns the current buffer queue state. + SLAndroidSimpleBufferQueueState GetBufferQueueState() const; + + // Number of buffers currently in the queue. + SLuint32 GetBufferCount(); + + // Prints a log message of the current queue state. Can be used for debugging + // purposes. + void LogBufferState() const; + + // Ensures that methods are called from the same thread as this object is + // created on. + rtc::ThreadChecker thread_checker_; + + // Stores thread ID in first call to SimpleBufferQueueCallback() from internal + // non-application thread which is not attached to the Dalvik JVM. + // Detached during construction of this object. + rtc::ThreadChecker thread_checker_opensles_; + + // Raw pointer to the audio manager injected at construction. Used to cache + // audio parameters and to access the global SL engine object needed by the + // ObtainEngineInterface() method. The audio manager outlives any instance of + // this class. + AudioManager* const audio_manager_; + + // Contains audio parameters provided to this class at construction by the + // AudioManager. + const AudioParameters audio_parameters_; + + // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the + // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). + AudioDeviceBuffer* audio_device_buffer_; + + // PCM-type format definition. + // TODO(henrika): add support for SLAndroidDataFormat_PCM_EX (android-21) if + // 32-bit float representation is needed. + SLDataFormat_PCM pcm_format_; + + bool initialized_; + bool recording_; + + // This interface exposes creation methods for all the OpenSL ES object types. + // It is the OpenSL ES API entry point. + SLEngineItf engine_; + + // The audio recorder media object records audio to the destination specified + // by the data sink capturing it from the input specified by the data source. + webrtc::ScopedSLObjectItf recorder_object_; + + // This interface is supported on the audio recorder object and it controls + // the state of the audio recorder. + SLRecordItf recorder_; + + // The Android Simple Buffer Queue interface is supported on the audio + // recorder. For recording, an app should enqueue empty buffers. When a + // registered callback sends notification that the system has finished writing + // data to the buffer, the app can read the buffer. + SLAndroidSimpleBufferQueueItf simple_buffer_queue_; + + // Consumes audio of native buffer size and feeds the WebRTC layer with 10ms + // chunks of audio. + std::unique_ptr fine_audio_buffer_; + + // Queue of audio buffers to be used by the recorder object for capturing + // audio. They will be used in a Round-robin way and the size of each buffer + // is given by AudioParameters::GetBytesPerBuffer(), i.e., it corresponds to + // the native OpenSL ES buffer size. + std::unique_ptr[]> audio_buffers_; + + // Keeps track of active audio buffer 'n' in the audio_buffers_[n] queue. + // Example (kNumOfOpenSLESBuffers = 2): counts 0, 1, 0, 1, ... + int buffer_index_; + + // Last time the OpenSL ES layer delivered recorded audio data. + uint32_t last_rec_time_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_RECORDER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.cc new file mode 100644 index 0000000000..158a86ad6c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.cc @@ -0,0 +1,500 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_device/audio_device_buffer.h" + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_device/audio_device_config.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/bind.h" +#include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/logging.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +static const char kTimerQueueName[] = "AudioDeviceBufferTimer"; + +// Time between two sucessive calls to LogStats(). +static const size_t kTimerIntervalInSeconds = 10; +static const size_t kTimerIntervalInMilliseconds = + kTimerIntervalInSeconds * rtc::kNumMillisecsPerSec; +// Min time required to qualify an audio session as a "call". If playout or +// recording has been active for less than this time we will not store any +// logs or UMA stats but instead consider the call as too short. +static const size_t kMinValidCallTimeTimeInSeconds = 10; +static const size_t kMinValidCallTimeTimeInMilliseconds = + kMinValidCallTimeTimeInSeconds * rtc::kNumMillisecsPerSec; +#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE +static const double k2Pi = 6.28318530717959; +#endif + +AudioDeviceBuffer::AudioDeviceBuffer() + : task_queue_(kTimerQueueName), + audio_transport_cb_(nullptr), + rec_sample_rate_(0), + play_sample_rate_(0), + rec_channels_(0), + play_channels_(0), + playing_(false), + recording_(false), + current_mic_level_(0), + new_mic_level_(0), + typing_status_(false), + play_delay_ms_(0), + rec_delay_ms_(0), + clock_drift_(0), + num_stat_reports_(0), + last_timer_task_time_(0), + rec_stat_count_(0), + play_stat_count_(0), + play_start_time_(0), + only_silence_recorded_(true), + log_stats_(false) { + RTC_LOG(INFO) << "AudioDeviceBuffer::ctor"; +#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE + phase_ = 0.0; + RTC_LOG(WARNING) << "AUDIO_DEVICE_PLAYS_SINUS_TONE is defined!"; +#endif + WebRtcSpl_Init(); + playout_thread_checker_.DetachFromThread(); + recording_thread_checker_.DetachFromThread(); +} + +AudioDeviceBuffer::~AudioDeviceBuffer() { + RTC_DCHECK_RUN_ON(&main_thread_checker_); + RTC_DCHECK(!playing_); + RTC_DCHECK(!recording_); + RTC_LOG(INFO) << "AudioDeviceBuffer::~dtor"; +} + +int32_t AudioDeviceBuffer::RegisterAudioCallback( + AudioTransport* audio_callback) { + RTC_DCHECK_RUN_ON(&main_thread_checker_); + RTC_LOG(INFO) << __FUNCTION__; + if (playing_ || recording_) { + RTC_LOG(LS_ERROR) << "Failed to set audio transport since media was active"; + return -1; + } + audio_transport_cb_ = audio_callback; + return 0; +} + +void AudioDeviceBuffer::StartPlayout() { + RTC_DCHECK_RUN_ON(&main_thread_checker_); + // TODO(henrika): allow for usage of DCHECK(!playing_) here instead. Today the + // ADM allows calling Start(), Start() by ignoring the second call but it + // makes more sense to only allow one call. + if (playing_) { + return; + } + RTC_LOG(INFO) << __FUNCTION__; + playout_thread_checker_.DetachFromThread(); + // Clear members tracking playout stats and do it on the task queue. + task_queue_.PostTask([this] { ResetPlayStats(); }); + // Start a periodic timer based on task queue if not already done by the + // recording side. + if (!recording_) { + StartPeriodicLogging(); + } + const int64_t now_time = rtc::TimeMillis(); + // Clear members that are only touched on the main (creating) thread. + play_start_time_ = now_time; + playing_ = true; +} + +void AudioDeviceBuffer::StartRecording() { + RTC_DCHECK_RUN_ON(&main_thread_checker_); + if (recording_) { + return; + } + RTC_LOG(INFO) << __FUNCTION__; + recording_thread_checker_.DetachFromThread(); + // Clear members tracking recording stats and do it on the task queue. + task_queue_.PostTask([this] { ResetRecStats(); }); + // Start a periodic timer based on task queue if not already done by the + // playout side. + if (!playing_) { + StartPeriodicLogging(); + } + // Clear members that will be touched on the main (creating) thread. + rec_start_time_ = rtc::TimeMillis(); + recording_ = true; + // And finally a member which can be modified on the native audio thread. + // It is safe to do so since we know by design that the owning ADM has not + // yet started the native audio recording. + only_silence_recorded_ = true; +} + +void AudioDeviceBuffer::StopPlayout() { + RTC_DCHECK_RUN_ON(&main_thread_checker_); + if (!playing_) { + return; + } + RTC_LOG(INFO) << __FUNCTION__; + playing_ = false; + // Stop periodic logging if no more media is active. + if (!recording_) { + StopPeriodicLogging(); + } + RTC_LOG(INFO) << "total playout time: " << rtc::TimeSince(play_start_time_); +} + +void AudioDeviceBuffer::StopRecording() { + RTC_DCHECK_RUN_ON(&main_thread_checker_); + if (!recording_) { + return; + } + RTC_LOG(INFO) << __FUNCTION__; + recording_ = false; + // Stop periodic logging if no more media is active. + if (!playing_) { + StopPeriodicLogging(); + } + // Add UMA histogram to keep track of the case when only zeros have been + // recorded. Measurements (max of absolute level) are taken twice per second, + // which means that if e.g 10 seconds of audio has been recorded, a total of + // 20 level estimates must all be identical to zero to trigger the histogram. + // |only_silence_recorded_| can only be cleared on the native audio thread + // that drives audio capture but we know by design that the audio has stopped + // when this method is called, hence there should not be aby conflicts. Also, + // the fact that |only_silence_recorded_| can be affected during the complete + // call makes chances of conflicts with potentially one last callback very + // small. + const size_t time_since_start = rtc::TimeSince(rec_start_time_); + if (time_since_start > kMinValidCallTimeTimeInMilliseconds) { + const int only_zeros = static_cast(only_silence_recorded_); + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.RecordedOnlyZeros", only_zeros); + RTC_LOG(INFO) << "HISTOGRAM(WebRTC.Audio.RecordedOnlyZeros): " + << only_zeros; + } + RTC_LOG(INFO) << "total recording time: " << time_since_start; +} + +int32_t AudioDeviceBuffer::SetRecordingSampleRate(uint32_t fsHz) { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + RTC_LOG(INFO) << "SetRecordingSampleRate(" << fsHz << ")"; + rec_sample_rate_ = fsHz; + return 0; +} + +int32_t AudioDeviceBuffer::SetPlayoutSampleRate(uint32_t fsHz) { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + RTC_LOG(INFO) << "SetPlayoutSampleRate(" << fsHz << ")"; + play_sample_rate_ = fsHz; + return 0; +} + +int32_t AudioDeviceBuffer::RecordingSampleRate() const { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + return rec_sample_rate_; +} + +int32_t AudioDeviceBuffer::PlayoutSampleRate() const { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + return play_sample_rate_; +} + +int32_t AudioDeviceBuffer::SetRecordingChannels(size_t channels) { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + RTC_LOG(INFO) << "SetRecordingChannels(" << channels << ")"; + rec_channels_ = channels; + return 0; +} + +int32_t AudioDeviceBuffer::SetPlayoutChannels(size_t channels) { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + RTC_LOG(INFO) << "SetPlayoutChannels(" << channels << ")"; + play_channels_ = channels; + return 0; +} + +size_t AudioDeviceBuffer::RecordingChannels() const { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + return rec_channels_; +} + +size_t AudioDeviceBuffer::PlayoutChannels() const { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + return play_channels_; +} + +int32_t AudioDeviceBuffer::SetCurrentMicLevel(uint32_t level) { +#if !defined(WEBRTC_WIN) + // Windows uses a dedicated thread for volume APIs. + RTC_DCHECK_RUN_ON(&recording_thread_checker_); +#endif + current_mic_level_ = level; + return 0; +} + +int32_t AudioDeviceBuffer::SetTypingStatus(bool typing_status) { + RTC_DCHECK_RUN_ON(&recording_thread_checker_); + typing_status_ = typing_status; + return 0; +} + +void AudioDeviceBuffer::NativeAudioInterrupted() { + RTC_DCHECK(main_thread_checker_.CalledOnValidThread()); + playout_thread_checker_.DetachFromThread(); + recording_thread_checker_.DetachFromThread(); +} + +uint32_t AudioDeviceBuffer::NewMicLevel() const { + RTC_DCHECK_RUN_ON(&recording_thread_checker_); + return new_mic_level_; +} + +void AudioDeviceBuffer::SetVQEData(int play_delay_ms, + int rec_delay_ms, + int clock_drift) { + RTC_DCHECK_RUN_ON(&recording_thread_checker_); + play_delay_ms_ = play_delay_ms; + rec_delay_ms_ = rec_delay_ms; + clock_drift_ = clock_drift; +} + +int32_t AudioDeviceBuffer::SetRecordedBuffer(const void* audio_buffer, + size_t samples_per_channel) { + RTC_DCHECK_RUN_ON(&recording_thread_checker_); + // Copy the complete input buffer to the local buffer. + const size_t old_size = rec_buffer_.size(); + rec_buffer_.SetData(static_cast(audio_buffer), + rec_channels_ * samples_per_channel); + // Keep track of the size of the recording buffer. Only updated when the + // size changes, which is a rare event. + if (old_size != rec_buffer_.size()) { + RTC_LOG(LS_INFO) << "Size of recording buffer: " << rec_buffer_.size(); + } + + // Derive a new level value twice per second and check if it is non-zero. + int16_t max_abs = 0; + RTC_DCHECK_LT(rec_stat_count_, 50); + if (++rec_stat_count_ >= 50) { + // Returns the largest absolute value in a signed 16-bit vector. + max_abs = WebRtcSpl_MaxAbsValueW16(rec_buffer_.data(), rec_buffer_.size()); + rec_stat_count_ = 0; + // Set |only_silence_recorded_| to false as soon as at least one detection + // of a non-zero audio packet is found. It can only be restored to true + // again by restarting the call. + if (max_abs > 0) { + only_silence_recorded_ = false; + } + } + // Update recording stats which is used as base for periodic logging of the + // audio input state. + UpdateRecStats(max_abs, samples_per_channel); + return 0; +} + +int32_t AudioDeviceBuffer::DeliverRecordedData() { + RTC_DCHECK_RUN_ON(&recording_thread_checker_); + if (!audio_transport_cb_) { + RTC_LOG(LS_WARNING) << "Invalid audio transport"; + return 0; + } + const size_t frames = rec_buffer_.size() / rec_channels_; + const size_t bytes_per_frame = rec_channels_ * sizeof(int16_t); + uint32_t new_mic_level(0); + uint32_t total_delay_ms = play_delay_ms_ + rec_delay_ms_; + int32_t res = audio_transport_cb_->RecordedDataIsAvailable( + rec_buffer_.data(), frames, bytes_per_frame, rec_channels_, + rec_sample_rate_, total_delay_ms, clock_drift_, current_mic_level_, + typing_status_, new_mic_level); + if (res != -1) { + new_mic_level_ = new_mic_level; + } else { + RTC_LOG(LS_ERROR) << "RecordedDataIsAvailable() failed"; + } + return 0; +} + +int32_t AudioDeviceBuffer::RequestPlayoutData(size_t samples_per_channel) { + RTC_DCHECK_RUN_ON(&playout_thread_checker_); + // The consumer can change the requested size on the fly and we therefore + // resize the buffer accordingly. Also takes place at the first call to this + // method. + const size_t total_samples = play_channels_ * samples_per_channel; + if (play_buffer_.size() != total_samples) { + play_buffer_.SetSize(total_samples); + RTC_LOG(LS_INFO) << "Size of playout buffer: " << play_buffer_.size(); + } + + size_t num_samples_out(0); + // It is currently supported to start playout without a valid audio + // transport object. Leads to warning and silence. + if (!audio_transport_cb_) { + RTC_LOG(LS_WARNING) << "Invalid audio transport"; + return 0; + } + + // Retrieve new 16-bit PCM audio data using the audio transport instance. + int64_t elapsed_time_ms = -1; + int64_t ntp_time_ms = -1; + const size_t bytes_per_frame = play_channels_ * sizeof(int16_t); + uint32_t res = audio_transport_cb_->NeedMorePlayData( + samples_per_channel, bytes_per_frame, play_channels_, play_sample_rate_, + play_buffer_.data(), num_samples_out, &elapsed_time_ms, &ntp_time_ms); + if (res != 0) { + RTC_LOG(LS_ERROR) << "NeedMorePlayData() failed"; + } + + // Derive a new level value twice per second. + int16_t max_abs = 0; + RTC_DCHECK_LT(play_stat_count_, 50); + if (++play_stat_count_ >= 50) { + // Returns the largest absolute value in a signed 16-bit vector. + max_abs = + WebRtcSpl_MaxAbsValueW16(play_buffer_.data(), play_buffer_.size()); + play_stat_count_ = 0; + } + // Update playout stats which is used as base for periodic logging of the + // audio output state. + UpdatePlayStats(max_abs, num_samples_out / play_channels_); + return static_cast(num_samples_out / play_channels_); +} + +int32_t AudioDeviceBuffer::GetPlayoutData(void* audio_buffer) { + RTC_DCHECK_RUN_ON(&playout_thread_checker_); + RTC_DCHECK_GT(play_buffer_.size(), 0); +#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE + const double phase_increment = + k2Pi * 440.0 / static_cast(play_sample_rate_); + int16_t* destination_r = reinterpret_cast(audio_buffer); + for (size_t i = 0; i < play_buffer_.size(); ++i) { + destination_r[i] = static_cast((sin(phase_) * (1 << 14))); + phase_ += phase_increment; + } +#else + memcpy(audio_buffer, play_buffer_.data(), + play_buffer_.size() * sizeof(int16_t)); +#endif + // Return samples per channel or number of frames. + return static_cast(play_buffer_.size() / play_channels_); +} + +void AudioDeviceBuffer::StartPeriodicLogging() { + task_queue_.PostTask(rtc::Bind(&AudioDeviceBuffer::LogStats, this, + AudioDeviceBuffer::LOG_START)); +} + +void AudioDeviceBuffer::StopPeriodicLogging() { + task_queue_.PostTask(rtc::Bind(&AudioDeviceBuffer::LogStats, this, + AudioDeviceBuffer::LOG_STOP)); +} + +void AudioDeviceBuffer::LogStats(LogState state) { + RTC_DCHECK_RUN_ON(&task_queue_); + int64_t now_time = rtc::TimeMillis(); + + if (state == AudioDeviceBuffer::LOG_START) { + // Reset counters at start. We will not add any logging in this state but + // the timer will started by posting a new (delayed) task. + num_stat_reports_ = 0; + last_timer_task_time_ = now_time; + log_stats_ = true; + } else if (state == AudioDeviceBuffer::LOG_STOP) { + // Stop logging and posting new tasks. + log_stats_ = false; + } else if (state == AudioDeviceBuffer::LOG_ACTIVE) { + // Keep logging unless logging was disabled while task was posted. + } + + // Avoid adding more logs since we are in STOP mode. + if (!log_stats_) { + return; + } + + int64_t next_callback_time = now_time + kTimerIntervalInMilliseconds; + int64_t time_since_last = rtc::TimeDiff(now_time, last_timer_task_time_); + last_timer_task_time_ = now_time; + + Stats stats; + { + rtc::CritScope cs(&lock_); + stats = stats_; + stats_.max_rec_level = 0; + stats_.max_play_level = 0; + } + + // Log the latest statistics but skip the first round just after state was + // set to LOG_START. Hence, first printed log will be after ~10 seconds. + if (++num_stat_reports_ > 1 && time_since_last > 0) { + uint32_t diff_samples = stats.rec_samples - last_stats_.rec_samples; + float rate = diff_samples / (static_cast(time_since_last) / 1000.0); + RTC_LOG(INFO) << "[REC : " << time_since_last << "msec, " + << rec_sample_rate_ / 1000 << "kHz] callbacks: " + << stats.rec_callbacks - last_stats_.rec_callbacks << ", " + << "samples: " << diff_samples << ", " + << "rate: " << static_cast(rate + 0.5) << ", " + << "level: " << stats.max_rec_level; + + diff_samples = stats.play_samples - last_stats_.play_samples; + rate = diff_samples / (static_cast(time_since_last) / 1000.0); + RTC_LOG(INFO) << "[PLAY: " << time_since_last << "msec, " + << play_sample_rate_ / 1000 << "kHz] callbacks: " + << stats.play_callbacks - last_stats_.play_callbacks << ", " + << "samples: " << diff_samples << ", " + << "rate: " << static_cast(rate + 0.5) << ", " + << "level: " << stats.max_play_level; + last_stats_ = stats; + } + + int64_t time_to_wait_ms = next_callback_time - rtc::TimeMillis(); + RTC_DCHECK_GT(time_to_wait_ms, 0) << "Invalid timer interval"; + + // Keep posting new (delayed) tasks until state is changed to kLogStop. + task_queue_.PostDelayedTask(rtc::Bind(&AudioDeviceBuffer::LogStats, this, + AudioDeviceBuffer::LOG_ACTIVE), + time_to_wait_ms); +} + +void AudioDeviceBuffer::ResetRecStats() { + RTC_DCHECK_RUN_ON(&task_queue_); + last_stats_.ResetRecStats(); + rtc::CritScope cs(&lock_); + stats_.ResetRecStats(); +} + +void AudioDeviceBuffer::ResetPlayStats() { + RTC_DCHECK_RUN_ON(&task_queue_); + last_stats_.ResetPlayStats(); + rtc::CritScope cs(&lock_); + stats_.ResetPlayStats(); +} + +void AudioDeviceBuffer::UpdateRecStats(int16_t max_abs, + size_t samples_per_channel) { + RTC_DCHECK_RUN_ON(&recording_thread_checker_); + rtc::CritScope cs(&lock_); + ++stats_.rec_callbacks; + stats_.rec_samples += samples_per_channel; + if (max_abs > stats_.max_rec_level) { + stats_.max_rec_level = max_abs; + } +} + +void AudioDeviceBuffer::UpdatePlayStats(int16_t max_abs, + size_t samples_per_channel) { + RTC_DCHECK_RUN_ON(&playout_thread_checker_); + rtc::CritScope cs(&lock_); + ++stats_.play_callbacks; + stats_.play_samples += samples_per_channel; + if (max_abs > stats_.max_play_level) { + stats_.max_play_level = max_abs; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.h new file mode 100644 index 0000000000..38e799672a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.h @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_AUDIO_DEVICE_BUFFER_H_ +#define MODULES_AUDIO_DEVICE_AUDIO_DEVICE_BUFFER_H_ + +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/buffer.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" +#include "system_wrappers/include/file_wrapper.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Delta times between two successive playout callbacks are limited to this +// value before added to an internal array. +const size_t kMaxDeltaTimeInMs = 500; +// TODO(henrika): remove when no longer used by external client. +const size_t kMaxBufferSizeBytes = 3840; // 10ms in stereo @ 96kHz + +class AudioDeviceBuffer { + public: + enum LogState { + LOG_START = 0, + LOG_STOP, + LOG_ACTIVE, + }; + + struct Stats { + void ResetRecStats() { + rec_callbacks = 0; + rec_samples = 0; + max_rec_level = 0; + } + + void ResetPlayStats() { + play_callbacks = 0; + play_samples = 0; + max_play_level = 0; + } + + // Total number of recording callbacks where the source provides 10ms audio + // data each time. + uint64_t rec_callbacks = 0; + + // Total number of playback callbacks where the sink asks for 10ms audio + // data each time. + uint64_t play_callbacks = 0; + + // Total number of recorded audio samples. + uint64_t rec_samples = 0; + + // Total number of played audio samples. + uint64_t play_samples = 0; + + // Contains max level (max(abs(x))) of recorded audio packets over the last + // 10 seconds where a new measurement is done twice per second. The level + // is reset to zero at each call to LogStats(). + int16_t max_rec_level = 0; + + // Contains max level of recorded audio packets over the last 10 seconds + // where a new measurement is done twice per second. + int16_t max_play_level = 0; + }; + + AudioDeviceBuffer(); + virtual ~AudioDeviceBuffer(); + + int32_t RegisterAudioCallback(AudioTransport* audio_callback); + + void StartPlayout(); + void StartRecording(); + void StopPlayout(); + void StopRecording(); + + int32_t SetRecordingSampleRate(uint32_t fsHz); + int32_t SetPlayoutSampleRate(uint32_t fsHz); + int32_t RecordingSampleRate() const; + int32_t PlayoutSampleRate() const; + + int32_t SetRecordingChannels(size_t channels); + int32_t SetPlayoutChannels(size_t channels); + size_t RecordingChannels() const; + size_t PlayoutChannels() const; + + virtual int32_t SetRecordedBuffer(const void* audio_buffer, + size_t samples_per_channel); + int32_t SetCurrentMicLevel(uint32_t level); + virtual void SetVQEData(int play_delay_ms, int rec_delay_ms, int clock_drift); + virtual int32_t DeliverRecordedData(); + uint32_t NewMicLevel() const; + + virtual int32_t RequestPlayoutData(size_t samples_per_channel); + virtual int32_t GetPlayoutData(void* audio_buffer); + + int32_t SetTypingStatus(bool typing_status); + + // Called on iOS where the native audio layer can be interrupted by other + // audio applications. This method can then be used to reset internal + // states and detach thread checkers to allow for a new audio session where + // native callbacks may come from a new set of I/O threads. + void NativeAudioInterrupted(); + + private: + // Starts/stops periodic logging of audio stats. + void StartPeriodicLogging(); + void StopPeriodicLogging(); + + // Called periodically on the internal thread created by the TaskQueue. + // Updates some stats but dooes it on the task queue to ensure that access of + // members is serialized hence avoiding usage of locks. + // state = LOG_START => members are initialized and the timer starts. + // state = LOG_STOP => no logs are printed and the timer stops. + // state = LOG_ACTIVE => logs are printed and the timer is kept alive. + void LogStats(LogState state); + + // Updates counters in each play/record callback. These counters are later + // (periodically) read by LogStats() using a lock. + void UpdateRecStats(int16_t max_abs, size_t samples_per_channel); + void UpdatePlayStats(int16_t max_abs, size_t samples_per_channel); + + // Clears all members tracking stats for recording and playout. + // These methods both run on the task queue. + void ResetRecStats(); + void ResetPlayStats(); + + // This object lives on the main (creating) thread and most methods are + // called on that same thread. When audio has started some methods will be + // called on either a native audio thread for playout or a native thread for + // recording. Some members are not annotated since they are "protected by + // design" and adding e.g. a race checker can cause failuries for very few + // edge cases and it is IMHO not worth the risk to use them in this class. + // TODO(henrika): see if it is possible to refactor and annotate all members. + + // Main thread on which this object is created. + rtc::ThreadChecker main_thread_checker_; + + // Native (platform specific) audio thread driving the playout side. + rtc::ThreadChecker playout_thread_checker_; + + // Native (platform specific) audio thread driving the recording side. + rtc::ThreadChecker recording_thread_checker_; + + rtc::CriticalSection lock_; + + // Task queue used to invoke LogStats() periodically. Tasks are executed on a + // worker thread but it does not necessarily have to be the same thread for + // each task. + rtc::TaskQueue task_queue_; + + // Raw pointer to AudioTransport instance. Supplied to RegisterAudioCallback() + // and it must outlive this object. It is not possible to change this member + // while any media is active. It is possible to start media without calling + // RegisterAudioCallback() but that will lead to ignored audio callbacks in + // both directions where native audio will be acive but no audio samples will + // be transported. + AudioTransport* audio_transport_cb_; + + // The members below that are not annotated are protected by design. They are + // all set on the main thread (verified by |main_thread_checker_|) and then + // read on either the playout or recording audio thread. But, media will never + // be active when the member is set; hence no conflict exists. It is too + // complex to ensure and verify that this is actually the case. + + // Sample rate in Hertz. + uint32_t rec_sample_rate_; + uint32_t play_sample_rate_; + + // Number of audio channels. + size_t rec_channels_; + size_t play_channels_; + + // Keeps track of if playout/recording are active or not. A combination + // of these states are used to determine when to start and stop the timer. + // Only used on the creating thread and not used to control any media flow. + bool playing_ RTC_ACCESS_ON(main_thread_checker_); + bool recording_ RTC_ACCESS_ON(main_thread_checker_); + + // Buffer used for audio samples to be played out. Size can be changed + // dynamically. The 16-bit samples are interleaved, hence the size is + // proportional to the number of channels. + rtc::BufferT play_buffer_ RTC_ACCESS_ON(playout_thread_checker_); + + // Byte buffer used for recorded audio samples. Size can be changed + // dynamically. + rtc::BufferT rec_buffer_ RTC_ACCESS_ON(recording_thread_checker_); + +// AGC parameters. +#if !defined(WEBRTC_WIN) + uint32_t current_mic_level_ RTC_ACCESS_ON(recording_thread_checker_); +#else + // Windows uses a dedicated thread for volume APIs. + uint32_t current_mic_level_; +#endif + uint32_t new_mic_level_ RTC_ACCESS_ON(recording_thread_checker_); + + // Contains true of a key-press has been detected. + bool typing_status_ RTC_ACCESS_ON(recording_thread_checker_); + + // Delay values used by the AEC. + int play_delay_ms_ RTC_ACCESS_ON(recording_thread_checker_); + int rec_delay_ms_ RTC_ACCESS_ON(recording_thread_checker_); + + // Contains a clock-drift measurement. + int clock_drift_ RTC_ACCESS_ON(recording_thread_checker_); + + // Counts number of times LogStats() has been called. + size_t num_stat_reports_ RTC_ACCESS_ON(task_queue_); + + // Time stamp of last timer task (drives logging). + int64_t last_timer_task_time_ RTC_ACCESS_ON(task_queue_); + + // Counts number of audio callbacks modulo 50 to create a signal when + // a new storage of audio stats shall be done. + int16_t rec_stat_count_ RTC_ACCESS_ON(recording_thread_checker_); + int16_t play_stat_count_ RTC_ACCESS_ON(playout_thread_checker_); + + // Time stamps of when playout and recording starts. + int64_t play_start_time_ RTC_ACCESS_ON(main_thread_checker_); + int64_t rec_start_time_ RTC_ACCESS_ON(main_thread_checker_); + + // Contains counters for playout and recording statistics. + Stats stats_ RTC_GUARDED_BY(lock_); + + // Stores current stats at each timer task. Used to calculate differences + // between two successive timer events. + Stats last_stats_ RTC_ACCESS_ON(task_queue_); + + // Set to true at construction and modified to false as soon as one audio- + // level estimate larger than zero is detected. + bool only_silence_recorded_; + + // Set to true when logging of audio stats is enabled for the first time in + // StartPeriodicLogging() and set to false by StopPeriodicLogging(). + // Setting this member to false prevents (possiby invalid) log messages from + // being printed in the LogStats() task. + bool log_stats_ RTC_ACCESS_ON(task_queue_); + +// Should *never* be defined in production builds. Only used for testing. +// When defined, the output signal will be replaced by a sinus tone at 440Hz. +#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE + double phase_ RTC_ACCESS_ON(playout_thread_checker_); +#endif +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_AUDIO_DEVICE_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_config.h b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_config.h new file mode 100644 index 0000000000..fd12c8bedd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_config.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_DEVICE_CONFIG_H_ +#define AUDIO_DEVICE_AUDIO_DEVICE_CONFIG_H_ + +// Enumerators +// +enum { GET_MIC_VOLUME_INTERVAL_MS = 1000 }; + +// Platform specifics +// +#if defined(_WIN32) +#if (_MSC_VER >= 1400) +#if !defined(WEBRTC_DUMMY_FILE_DEVICES) +// Windows Core Audio is the default audio layer in Windows. +// Only supported for VS 2005 and higher. +#define WEBRTC_WINDOWS_CORE_AUDIO_BUILD +#endif +#endif +#endif + +#endif // AUDIO_DEVICE_AUDIO_DEVICE_CONFIG_H_ + diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_data_observer.cc b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_data_observer.cc new file mode 100644 index 0000000000..823fbfed8c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_data_observer.cc @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/include/audio_device_data_observer.h" +#include "rtc_base/checks.h" +#include "rtc_base/refcountedobject.h" + +namespace webrtc { + +namespace { + +// A wrapper over AudioDeviceModule that registers itself as AudioTransport +// callback and redirects the PCM data to AudioDeviceDataObserver callback. +class ADMWrapper : public AudioDeviceModule, public AudioTransport { + public: + ADMWrapper(const AudioLayer audio_layer, AudioDeviceDataObserver* observer) + : impl_(AudioDeviceModule::Create(audio_layer)), observer_(observer) { + // Register self as the audio transport callback for underlying ADM impl. + auto res = impl_->RegisterAudioCallback(this); + is_valid_ = (impl_.get() != nullptr) && (res == 0); + } + virtual ~ADMWrapper() { + audio_transport_ = nullptr; + observer_ = nullptr; + } + + // Make sure we have a valid ADM before returning it to user. + bool IsValid() { return is_valid_; } + + // AudioTransport methods overrides. + int32_t RecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samples_per_sec, + const uint32_t total_delay_ms, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) override { + int32_t res = 0; + // Capture PCM data of locally captured audio. + if (observer_) { + observer_->OnCaptureData(audioSamples, nSamples, nBytesPerSample, + nChannels, samples_per_sec); + } + + // Send to the actual audio transport. + if (audio_transport_) { + res = audio_transport_->RecordedDataIsAvailable( + audioSamples, nSamples, nBytesPerSample, nChannels, samples_per_sec, + total_delay_ms, clockDrift, currentMicLevel, keyPressed, newMicLevel); + } + + return res; + } + + int32_t NeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samples_per_sec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override { + int32_t res = 0; + // Request data from audio transport. + if (audio_transport_) { + res = audio_transport_->NeedMorePlayData( + nSamples, nBytesPerSample, nChannels, samples_per_sec, audioSamples, + nSamplesOut, elapsed_time_ms, ntp_time_ms); + } + + // Capture rendered data. + if (observer_) { + observer_->OnRenderData(audioSamples, nSamples, nBytesPerSample, + nChannels, samples_per_sec); + } + + return res; + } + + void PushCaptureData(int voe_channel, + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) override { + RTC_NOTREACHED(); + } + + void PullRenderData(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override { + RTC_NOTREACHED(); + } + + // Override AudioDeviceModule's RegisterAudioCallback method to remember the + // actual audio transport (e.g.: voice engine). + int32_t RegisterAudioCallback(AudioTransport* audio_callback) override { + // Remember the audio callback to forward PCM data + audio_transport_ = audio_callback; + return 0; + } + + // AudioDeviceModule pass through method overrides. + int32_t ActiveAudioLayer(AudioLayer* audio_layer) const override { + return impl_->ActiveAudioLayer(audio_layer); + } + int32_t Init() override { return impl_->Init(); } + int32_t Terminate() override { return impl_->Terminate(); } + bool Initialized() const override { return impl_->Initialized(); } + int16_t PlayoutDevices() override { return impl_->PlayoutDevices(); } + int16_t RecordingDevices() override { return impl_->RecordingDevices(); } + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override { + return impl_->PlayoutDeviceName(index, name, guid); + } + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override { + return impl_->RecordingDeviceName(index, name, guid); + } + int32_t SetPlayoutDevice(uint16_t index) override { + return impl_->SetPlayoutDevice(index); + } + int32_t SetPlayoutDevice(WindowsDeviceType device) override { + return impl_->SetPlayoutDevice(device); + } + int32_t SetRecordingDevice(uint16_t index) override { + return impl_->SetRecordingDevice(index); + } + int32_t SetRecordingDevice(WindowsDeviceType device) override { + return impl_->SetRecordingDevice(device); + } + int32_t PlayoutIsAvailable(bool* available) override { + return impl_->PlayoutIsAvailable(available); + } + int32_t InitPlayout() override { return impl_->InitPlayout(); } + bool PlayoutIsInitialized() const override { + return impl_->PlayoutIsInitialized(); + } + int32_t RecordingIsAvailable(bool* available) override { + return impl_->RecordingIsAvailable(available); + } + int32_t InitRecording() override { return impl_->InitRecording(); } + bool RecordingIsInitialized() const override { + return impl_->RecordingIsInitialized(); + } + int32_t StartPlayout() override { return impl_->StartPlayout(); } + int32_t StopPlayout() override { return impl_->StopPlayout(); } + bool Playing() const override { return impl_->Playing(); } + int32_t StartRecording() override { return impl_->StartRecording(); } + int32_t StopRecording() override { return impl_->StopRecording(); } + bool Recording() const override { return impl_->Recording(); } + int32_t SetAGC(bool enable) override { return impl_->SetAGC(enable); } + bool AGC() const override { return impl_->AGC(); } + int32_t InitSpeaker() override { return impl_->InitSpeaker(); } + bool SpeakerIsInitialized() const override { + return impl_->SpeakerIsInitialized(); + } + int32_t InitMicrophone() override { return impl_->InitMicrophone(); } + bool MicrophoneIsInitialized() const override { + return impl_->MicrophoneIsInitialized(); + } + int32_t SpeakerVolumeIsAvailable(bool* available) override { + return impl_->SpeakerVolumeIsAvailable(available); + } + int32_t SetSpeakerVolume(uint32_t volume) override { + return impl_->SetSpeakerVolume(volume); + } + int32_t SpeakerVolume(uint32_t* volume) const override { + return impl_->SpeakerVolume(volume); + } + int32_t MaxSpeakerVolume(uint32_t* max_volume) const override { + return impl_->MaxSpeakerVolume(max_volume); + } + int32_t MinSpeakerVolume(uint32_t* min_volume) const override { + return impl_->MinSpeakerVolume(min_volume); + } + int32_t MicrophoneVolumeIsAvailable(bool* available) override { + return impl_->MicrophoneVolumeIsAvailable(available); + } + int32_t SetMicrophoneVolume(uint32_t volume) override { + return impl_->SetMicrophoneVolume(volume); + } + int32_t MicrophoneVolume(uint32_t* volume) const override { + return impl_->MicrophoneVolume(volume); + } + int32_t MaxMicrophoneVolume(uint32_t* max_volume) const override { + return impl_->MaxMicrophoneVolume(max_volume); + } + int32_t MinMicrophoneVolume(uint32_t* min_volume) const override { + return impl_->MinMicrophoneVolume(min_volume); + } + int32_t SpeakerMuteIsAvailable(bool* available) override { + return impl_->SpeakerMuteIsAvailable(available); + } + int32_t SetSpeakerMute(bool enable) override { + return impl_->SetSpeakerMute(enable); + } + int32_t SpeakerMute(bool* enabled) const override { + return impl_->SpeakerMute(enabled); + } + int32_t MicrophoneMuteIsAvailable(bool* available) override { + return impl_->MicrophoneMuteIsAvailable(available); + } + int32_t SetMicrophoneMute(bool enable) override { + return impl_->SetMicrophoneMute(enable); + } + int32_t MicrophoneMute(bool* enabled) const override { + return impl_->MicrophoneMute(enabled); + } + int32_t StereoPlayoutIsAvailable(bool* available) const override { + return impl_->StereoPlayoutIsAvailable(available); + } + int32_t SetStereoPlayout(bool enable) override { + return impl_->SetStereoPlayout(enable); + } + int32_t StereoPlayout(bool* enabled) const override { + return impl_->StereoPlayout(enabled); + } + int32_t StereoRecordingIsAvailable(bool* available) const override { + return impl_->StereoRecordingIsAvailable(available); + } + int32_t SetStereoRecording(bool enable) override { + return impl_->SetStereoRecording(enable); + } + int32_t StereoRecording(bool* enabled) const override { + return impl_->StereoRecording(enabled); + } + int32_t PlayoutDelay(uint16_t* delay_ms) const override { + return impl_->PlayoutDelay(delay_ms); + } + bool BuiltInAECIsAvailable() const override { + return impl_->BuiltInAECIsAvailable(); + } + bool BuiltInAGCIsAvailable() const override { + return impl_->BuiltInAGCIsAvailable(); + } + bool BuiltInNSIsAvailable() const override { + return impl_->BuiltInNSIsAvailable(); + } + int32_t EnableBuiltInAEC(bool enable) override { + return impl_->EnableBuiltInAEC(enable); + } + int32_t EnableBuiltInAGC(bool enable) override { + return impl_->EnableBuiltInAGC(enable); + } + int32_t EnableBuiltInNS(bool enable) override { + return impl_->EnableBuiltInNS(enable); + } +// Only supported on iOS. +#if defined(WEBRTC_IOS) + int GetPlayoutAudioParameters(AudioParameters* params) const override { + return impl_->GetPlayoutAudioParameters(params); + } + int GetRecordAudioParameters(AudioParameters* params) const override { + return impl_->GetRecordAudioParameters(params); + } +#endif // WEBRTC_IOS + + protected: + rtc::scoped_refptr impl_; + AudioDeviceDataObserver* observer_ = nullptr; + AudioTransport* audio_transport_ = nullptr; + bool is_valid_ = false; +}; + +} // namespace + +rtc::scoped_refptr CreateAudioDeviceWithDataObserver( + const AudioDeviceModule::AudioLayer audio_layer, + AudioDeviceDataObserver* observer) { + rtc::scoped_refptr audio_device( + new rtc::RefCountedObject(audio_layer, observer)); + + if (!audio_device->IsValid()) { + return nullptr; + } + + return audio_device; +} + +// TODO(bugs.webrtc.org/7306): deprecated. +rtc::scoped_refptr CreateAudioDeviceWithDataObserver( + const int32_t id, + const AudioDeviceModule::AudioLayer audio_layer, + AudioDeviceDataObserver* observer) { + return CreateAudioDeviceWithDataObserver(audio_layer, observer); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.cc b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.cc new file mode 100644 index 0000000000..2baea2a3a9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/audio_device_generic.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +bool AudioDeviceGeneric::BuiltInAECIsAvailable() const { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return false; +} + +int32_t AudioDeviceGeneric::EnableBuiltInAEC(bool enable) { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return -1; +} + +bool AudioDeviceGeneric::BuiltInAGCIsAvailable() const { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return false; +} + +int32_t AudioDeviceGeneric::EnableBuiltInAGC(bool enable) { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return -1; +} + +bool AudioDeviceGeneric::BuiltInNSIsAvailable() const { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return false; +} + +int32_t AudioDeviceGeneric::EnableBuiltInNS(bool enable) { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return -1; +} + +#if defined(WEBRTC_IOS) +int AudioDeviceGeneric::GetPlayoutAudioParameters( + AudioParameters* params) const { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return -1; +} + +int AudioDeviceGeneric::GetRecordAudioParameters( + AudioParameters* params) const { + RTC_LOG_F(LS_ERROR) << "Not supported on this platform"; + return -1; +} +#endif // WEBRTC_IOS + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.h b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.h new file mode 100644 index 0000000000..2cd8874671 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_DEVICE_GENERIC_H_ +#define AUDIO_DEVICE_AUDIO_DEVICE_GENERIC_H_ + +#include "modules/audio_device/audio_device_buffer.h" +#include "modules/audio_device/include/audio_device.h" + +namespace webrtc { + +class AudioDeviceGeneric { + public: + // For use with UMA logging. Must be kept in sync with histograms.xml in + // Chrome, located at + // https://cs.chromium.org/chromium/src/tools/metrics/histograms/histograms.xml + enum class InitStatus { + OK = 0, + PLAYOUT_ERROR = 1, + RECORDING_ERROR = 2, + OTHER_ERROR = 3, + NUM_STATUSES = 4 + }; + // Retrieve the currently utilized audio layer + virtual int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const = 0; + + // Main initializaton and termination + virtual InitStatus Init() = 0; + virtual int32_t Terminate() = 0; + virtual bool Initialized() const = 0; + + // Device enumeration + virtual int16_t PlayoutDevices() = 0; + virtual int16_t RecordingDevices() = 0; + virtual int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) = 0; + virtual int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) = 0; + + // Device selection + virtual int32_t SetPlayoutDevice(uint16_t index) = 0; + virtual int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) = 0; + virtual int32_t SetRecordingDevice(uint16_t index) = 0; + virtual int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) = 0; + + // Audio transport initialization + virtual int32_t PlayoutIsAvailable(bool& available) = 0; + virtual int32_t InitPlayout() = 0; + virtual bool PlayoutIsInitialized() const = 0; + virtual int32_t RecordingIsAvailable(bool& available) = 0; + virtual int32_t InitRecording() = 0; + virtual bool RecordingIsInitialized() const = 0; + + // Audio transport control + virtual int32_t StartPlayout() = 0; + virtual int32_t StopPlayout() = 0; + virtual bool Playing() const = 0; + virtual int32_t StartRecording() = 0; + virtual int32_t StopRecording() = 0; + virtual bool Recording() const = 0; + + // Microphone Automatic Gain Control (AGC) + virtual int32_t SetAGC(bool enable) = 0; + virtual bool AGC() const = 0; + + // Audio mixer initialization + virtual int32_t InitSpeaker() = 0; + virtual bool SpeakerIsInitialized() const = 0; + virtual int32_t InitMicrophone() = 0; + virtual bool MicrophoneIsInitialized() const = 0; + + // Speaker volume controls + virtual int32_t SpeakerVolumeIsAvailable(bool& available) = 0; + virtual int32_t SetSpeakerVolume(uint32_t volume) = 0; + virtual int32_t SpeakerVolume(uint32_t& volume) const = 0; + virtual int32_t MaxSpeakerVolume(uint32_t& maxVolume) const = 0; + virtual int32_t MinSpeakerVolume(uint32_t& minVolume) const = 0; + + // Microphone volume controls + virtual int32_t MicrophoneVolumeIsAvailable(bool& available) = 0; + virtual int32_t SetMicrophoneVolume(uint32_t volume) = 0; + virtual int32_t MicrophoneVolume(uint32_t& volume) const = 0; + virtual int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const = 0; + virtual int32_t MinMicrophoneVolume(uint32_t& minVolume) const = 0; + + // Speaker mute control + virtual int32_t SpeakerMuteIsAvailable(bool& available) = 0; + virtual int32_t SetSpeakerMute(bool enable) = 0; + virtual int32_t SpeakerMute(bool& enabled) const = 0; + + // Microphone mute control + virtual int32_t MicrophoneMuteIsAvailable(bool& available) = 0; + virtual int32_t SetMicrophoneMute(bool enable) = 0; + virtual int32_t MicrophoneMute(bool& enabled) const = 0; + + // Stereo support + virtual int32_t StereoPlayoutIsAvailable(bool& available) = 0; + virtual int32_t SetStereoPlayout(bool enable) = 0; + virtual int32_t StereoPlayout(bool& enabled) const = 0; + virtual int32_t StereoRecordingIsAvailable(bool& available) = 0; + virtual int32_t SetStereoRecording(bool enable) = 0; + virtual int32_t StereoRecording(bool& enabled) const = 0; + + // Delay information and control + virtual int32_t PlayoutDelay(uint16_t& delayMS) const = 0; + + // Android only + virtual bool BuiltInAECIsAvailable() const; + virtual bool BuiltInAGCIsAvailable() const; + virtual bool BuiltInNSIsAvailable() const; + + // Windows Core Audio and Android only. + virtual int32_t EnableBuiltInAEC(bool enable); + virtual int32_t EnableBuiltInAGC(bool enable); + virtual int32_t EnableBuiltInNS(bool enable); + + // iOS only. + // TODO(henrika): add Android support. +#if defined(WEBRTC_IOS) + virtual int GetPlayoutAudioParameters(AudioParameters* params) const; + virtual int GetRecordAudioParameters(AudioParameters* params) const; +#endif // WEBRTC_IOS + + virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) = 0; + + virtual ~AudioDeviceGeneric() {} +}; + +} // namespace webrtc + +#endif // AUDIO_DEVICE_AUDIO_DEVICE_GENERIC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic_gn/moz.build new file mode 100644 index 0000000000..ecf1f4e546 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic_gn/moz.build @@ -0,0 +1,253 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_DUMMY_AUDIO_BUILD"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/", + "/third_party/libwebrtc/webrtc/modules/audio_device/dummy/", + "/third_party/libwebrtc/webrtc/modules/audio_device/include/", + "/third_party/libwebrtc/webrtc/modules/include/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_generic.cc", + "/third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.cc", + "/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.cc", + "/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.cc", + "/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_device/android/" + ] + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_device/mac/" + ] + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_device/linux/" + ] + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/webrtc/modules/audio_device/win/" + ] + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_device_generic_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_gn/moz.build new file mode 100644 index 0000000000..68f22c1ee2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_gn/moz.build @@ -0,0 +1,190 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/modules/audio_device/dummy/", + "/third_party/libwebrtc/webrtc/modules/audio_device/include/", + "/third_party/libwebrtc/webrtc/modules/include/" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +Library("audio_device_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.cc new file mode 100644 index 0000000000..f3aceffc42 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.cc @@ -0,0 +1,903 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/audio_device_impl.h" + +#include "modules/audio_device/audio_device_config.h" +#include "modules/audio_device/audio_device_generic.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/refcount.h" +#include "rtc_base/refcountedobject.h" +#include "system_wrappers/include/metrics.h" + +#if defined(_WIN32) +#if defined(WEBRTC_WINDOWS_CORE_AUDIO_BUILD) +#include "audio_device_core_win.h" +#endif +#elif defined(WEBRTC_ANDROID) +#include +#include "modules/audio_device/android/audio_device_template.h" +#include "modules/audio_device/android/audio_manager.h" +#include "modules/audio_device/android/audio_record_jni.h" +#include "modules/audio_device/android/audio_track_jni.h" +#include "modules/audio_device/android/opensles_player.h" +#include "modules/audio_device/android/opensles_recorder.h" +#elif defined(WEBRTC_LINUX) +#if defined(LINUX_ALSA) +#include "audio_device_alsa_linux.h" +#endif +#if defined(LINUX_PULSE) +#include "audio_device_pulse_linux.h" +#endif +#elif defined(WEBRTC_IOS) +#include "audio_device_ios.h" +#elif defined(WEBRTC_MAC) +#include "audio_device_mac.h" +#endif +#if defined(WEBRTC_DUMMY_FILE_DEVICES) +#include "modules/audio_device/dummy/file_audio_device_factory.h" +#endif +#include "modules/audio_device/dummy/audio_device_dummy.h" +#include "modules/audio_device/dummy/file_audio_device.h" + +#define CHECKinitialized_() \ + { \ + if (!initialized_) { \ + return -1; \ + }; \ + } + +#define CHECKinitialized__BOOL() \ + { \ + if (!initialized_) { \ + return false; \ + }; \ + } + +namespace webrtc { + +// static +rtc::scoped_refptr AudioDeviceModule::Create( + const AudioLayer audio_layer) { + RTC_LOG(INFO) << __FUNCTION__; + // Create the generic reference counted (platform independent) implementation. + rtc::scoped_refptr audioDevice( + new rtc::RefCountedObject(audio_layer)); + + // Ensure that the current platform is supported. + if (audioDevice->CheckPlatform() == -1) { + return nullptr; + } + + // Create the platform-dependent implementation. + if (audioDevice->CreatePlatformSpecificObjects() == -1) { + return nullptr; + } + + // Ensure that the generic audio buffer can communicate with the platform + // specific parts. + if (audioDevice->AttachAudioBuffer() == -1) { + return nullptr; + } + + return audioDevice; +} + +// TODO(bugs.webrtc.org/7306): deprecated. +rtc::scoped_refptr AudioDeviceModule::Create( + const int32_t id, + const AudioLayer audio_layer) { + RTC_LOG(INFO) << __FUNCTION__; + return AudioDeviceModule::Create(audio_layer); +} + +AudioDeviceModuleImpl::AudioDeviceModuleImpl(const AudioLayer audioLayer) + : audio_layer_(audioLayer) { + RTC_LOG(INFO) << __FUNCTION__; +} + +int32_t AudioDeviceModuleImpl::CheckPlatform() { + RTC_LOG(INFO) << __FUNCTION__; + // Ensure that the current platform is supported + PlatformType platform(kPlatformNotSupported); +#if defined(_WIN32) + platform = kPlatformWin32; + RTC_LOG(INFO) << "current platform is Win32"; +#elif defined(WEBRTC_ANDROID) + platform = kPlatformAndroid; + RTC_LOG(INFO) << "current platform is Android"; +#elif defined(WEBRTC_LINUX) + platform = kPlatformLinux; + RTC_LOG(INFO) << "current platform is Linux"; +#elif defined(WEBRTC_IOS) + platform = kPlatformIOS; + RTC_LOG(INFO) << "current platform is IOS"; +#elif defined(WEBRTC_MAC) + platform = kPlatformMac; + RTC_LOG(INFO) << "current platform is Mac"; +#endif + if (platform == kPlatformNotSupported) { + RTC_LOG(LERROR) + << "current platform is not supported => this module will self " + "destruct!"; + return -1; + } + platform_type_ = platform; + return 0; +} + +int32_t AudioDeviceModuleImpl::CreatePlatformSpecificObjects() { + RTC_LOG(INFO) << __FUNCTION__; +// Dummy ADM implementations if build flags are set. +#if defined(WEBRTC_DUMMY_AUDIO_BUILD) + audio_device_.reset(new AudioDeviceDummy()); + RTC_LOG(INFO) << "Dummy Audio APIs will be utilized"; +#elif defined(WEBRTC_DUMMY_FILE_DEVICES) + audio_device_.reset(FileAudioDeviceFactory::CreateFileAudioDevice()); + if (audio_device_) { + RTC_LOG(INFO) << "Will use file-playing dummy device."; + } else { + // Create a dummy device instead. + audio_device_.reset(new AudioDeviceDummy()); + RTC_LOG(INFO) << "Dummy Audio APIs will be utilized"; + } + +// Real (non-dummy) ADM implementations. +#else + AudioLayer audio_layer(PlatformAudioLayer()); +// Windows ADM implementation. +#if defined(WEBRTC_WINDOWS_CORE_AUDIO_BUILD) + if ((audio_layer == kWindowsCoreAudio) || + (audio_layer == kPlatformDefaultAudio)) { + RTC_LOG(INFO) << "Attempting to use the Windows Core Audio APIs..."; + if (AudioDeviceWindowsCore::CoreAudioIsSupported()) { + audio_device_.reset(new AudioDeviceWindowsCore()); + RTC_LOG(INFO) << "Windows Core Audio APIs will be utilized"; + } + } +#endif // defined(WEBRTC_WINDOWS_CORE_AUDIO_BUILD) + +#if defined(WEBRTC_ANDROID) + // Create an Android audio manager. + audio_manager_android_.reset(new AudioManager()); + // Select best possible combination of audio layers. + if (audio_layer == kPlatformDefaultAudio) { + if (audio_manager_android_->IsLowLatencyPlayoutSupported() && + audio_manager_android_->IsLowLatencyRecordSupported()) { + // Use OpenSL ES for both playout and recording. + audio_layer = kAndroidOpenSLESAudio; + } else if (audio_manager_android_->IsLowLatencyPlayoutSupported() && + !audio_manager_android_->IsLowLatencyRecordSupported()) { + // Use OpenSL ES for output on devices that only supports the + // low-latency output audio path. + audio_layer = kAndroidJavaInputAndOpenSLESOutputAudio; + } else { + // Use Java-based audio in both directions when low-latency output is + // not supported. + audio_layer = kAndroidJavaAudio; + } + } + AudioManager* audio_manager = audio_manager_android_.get(); + if (audio_layer == kAndroidJavaAudio) { + // Java audio for both input and output audio. + audio_device_.reset(new AudioDeviceTemplate( + audio_layer, audio_manager)); + } else if (audio_layer == kAndroidOpenSLESAudio) { + // OpenSL ES based audio for both input and output audio. + audio_device_.reset( + new AudioDeviceTemplate( + audio_layer, audio_manager)); + } else if (audio_layer == kAndroidJavaInputAndOpenSLESOutputAudio) { + // Java audio for input and OpenSL ES for output audio (i.e. mixed APIs). + // This combination provides low-latency output audio and at the same + // time support for HW AEC using the AudioRecord Java API. + audio_device_.reset(new AudioDeviceTemplate( + audio_layer, audio_manager)); + } else { + // Invalid audio layer. + audio_device_.reset(nullptr); + } +// END #if defined(WEBRTC_ANDROID) + +// Linux ADM implementation. +#elif defined(WEBRTC_LINUX) + if ((audio_layer == kLinuxPulseAudio) || + (audio_layer == kPlatformDefaultAudio)) { +#if defined(LINUX_PULSE) + RTC_LOG(INFO) << "Attempting to use Linux PulseAudio APIs..."; + // Linux PulseAudio implementation. + audio_device_.reset(new AudioDeviceLinuxPulse()); + RTC_LOG(INFO) << "Linux PulseAudio APIs will be utilized"; +#endif +#if defined(LINUX_PULSE) +#endif + } else if (audio_layer == kLinuxAlsaAudio) { +#if defined(LINUX_ALSA) + // Linux ALSA implementation. + audio_device_.reset(new AudioDeviceLinuxALSA()); + RTC_LOG(INFO) << "Linux ALSA APIs will be utilized."; +#endif + } +#endif // #if defined(WEBRTC_LINUX) + +// iOS ADM implementation. +#if defined(WEBRTC_IOS) + if (audio_layer == kPlatformDefaultAudio) { + audio_device_.reset(new AudioDeviceIOS()); + RTC_LOG(INFO) << "iPhone Audio APIs will be utilized."; + } +// END #if defined(WEBRTC_IOS) + +// Mac OS X ADM implementation. +#elif defined(WEBRTC_MAC) + if (audio_layer == kPlatformDefaultAudio) { + audio_device_.reset(new AudioDeviceMac()); + RTC_LOG(INFO) << "Mac OS X Audio APIs will be utilized."; + } +#endif // WEBRTC_MAC + + // Dummy ADM implementation. + if (audio_layer == kDummyAudio) { + audio_device_.reset(new AudioDeviceDummy()); + RTC_LOG(INFO) << "Dummy Audio APIs will be utilized."; + } +#endif // if defined(WEBRTC_DUMMY_AUDIO_BUILD) + + if (!audio_device_) { + RTC_LOG(LS_ERROR) + << "Failed to create the platform specific ADM implementation."; + return -1; + } + return 0; +} + +int32_t AudioDeviceModuleImpl::AttachAudioBuffer() { + RTC_LOG(INFO) << __FUNCTION__; + audio_device_->AttachAudioBuffer(&audio_device_buffer_); + return 0; +} + +AudioDeviceModuleImpl::~AudioDeviceModuleImpl() { + RTC_LOG(INFO) << __FUNCTION__; +} + +int32_t AudioDeviceModuleImpl::ActiveAudioLayer(AudioLayer* audioLayer) const { + RTC_LOG(INFO) << __FUNCTION__; + AudioLayer activeAudio; + if (audio_device_->ActiveAudioLayer(activeAudio) == -1) { + return -1; + } + *audioLayer = activeAudio; + return 0; +} + +int32_t AudioDeviceModuleImpl::Init() { + RTC_LOG(INFO) << __FUNCTION__; + if (initialized_) + return 0; + RTC_CHECK(audio_device_); + AudioDeviceGeneric::InitStatus status = audio_device_->Init(); + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.InitializationResult", static_cast(status), + static_cast(AudioDeviceGeneric::InitStatus::NUM_STATUSES)); + if (status != AudioDeviceGeneric::InitStatus::OK) { + RTC_LOG(LS_ERROR) << "Audio device initialization failed."; + return -1; + } + initialized_ = true; + return 0; +} + +int32_t AudioDeviceModuleImpl::Terminate() { + RTC_LOG(INFO) << __FUNCTION__; + if (!initialized_) + return 0; + if (audio_device_->Terminate() == -1) { + return -1; + } + initialized_ = false; + return 0; +} + +bool AudioDeviceModuleImpl::Initialized() const { + RTC_LOG(INFO) << __FUNCTION__ << ": " << initialized_; + return initialized_; +} + +int32_t AudioDeviceModuleImpl::InitSpeaker() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->InitSpeaker(); +} + +int32_t AudioDeviceModuleImpl::InitMicrophone() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->InitMicrophone(); +} + +int32_t AudioDeviceModuleImpl::SpeakerVolumeIsAvailable(bool* available) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->SpeakerVolumeIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::SetSpeakerVolume(uint32_t volume) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << volume << ")"; + CHECKinitialized_(); + return audio_device_->SetSpeakerVolume(volume); +} + +int32_t AudioDeviceModuleImpl::SpeakerVolume(uint32_t* volume) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + uint32_t level = 0; + if (audio_device_->SpeakerVolume(level) == -1) { + return -1; + } + *volume = level; + RTC_LOG(INFO) << "output: " << *volume; + return 0; +} + +bool AudioDeviceModuleImpl::SpeakerIsInitialized() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isInitialized = audio_device_->SpeakerIsInitialized(); + RTC_LOG(INFO) << "output: " << isInitialized; + return isInitialized; +} + +bool AudioDeviceModuleImpl::MicrophoneIsInitialized() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isInitialized = audio_device_->MicrophoneIsInitialized(); + RTC_LOG(INFO) << "output: " << isInitialized; + return isInitialized; +} + +int32_t AudioDeviceModuleImpl::MaxSpeakerVolume(uint32_t* maxVolume) const { + CHECKinitialized_(); + uint32_t maxVol = 0; + if (audio_device_->MaxSpeakerVolume(maxVol) == -1) { + return -1; + } + *maxVolume = maxVol; + return 0; +} + +int32_t AudioDeviceModuleImpl::MinSpeakerVolume(uint32_t* minVolume) const { + CHECKinitialized_(); + uint32_t minVol = 0; + if (audio_device_->MinSpeakerVolume(minVol) == -1) { + return -1; + } + *minVolume = minVol; + return 0; +} + +int32_t AudioDeviceModuleImpl::SpeakerMuteIsAvailable(bool* available) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->SpeakerMuteIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::SetSpeakerMute(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + return audio_device_->SetSpeakerMute(enable); +} + +int32_t AudioDeviceModuleImpl::SpeakerMute(bool* enabled) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool muted = false; + if (audio_device_->SpeakerMute(muted) == -1) { + return -1; + } + *enabled = muted; + RTC_LOG(INFO) << "output: " << muted; + return 0; +} + +int32_t AudioDeviceModuleImpl::MicrophoneMuteIsAvailable(bool* available) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->MicrophoneMuteIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::SetMicrophoneMute(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + return (audio_device_->SetMicrophoneMute(enable)); +} + +int32_t AudioDeviceModuleImpl::MicrophoneMute(bool* enabled) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool muted = false; + if (audio_device_->MicrophoneMute(muted) == -1) { + return -1; + } + *enabled = muted; + RTC_LOG(INFO) << "output: " << muted; + return 0; +} + +int32_t AudioDeviceModuleImpl::MicrophoneVolumeIsAvailable(bool* available) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->MicrophoneVolumeIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::SetMicrophoneVolume(uint32_t volume) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << volume << ")"; + CHECKinitialized_(); + return (audio_device_->SetMicrophoneVolume(volume)); +} + +int32_t AudioDeviceModuleImpl::MicrophoneVolume(uint32_t* volume) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + uint32_t level = 0; + if (audio_device_->MicrophoneVolume(level) == -1) { + return -1; + } + *volume = level; + RTC_LOG(INFO) << "output: " << *volume; + return 0; +} + +int32_t AudioDeviceModuleImpl::StereoRecordingIsAvailable( + bool* available) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->StereoRecordingIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::SetStereoRecording(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + if (audio_device_->RecordingIsInitialized()) { + RTC_LOG(WARNING) << "recording in stereo is not supported"; + return -1; + } + if (audio_device_->SetStereoRecording(enable) == -1) { + RTC_LOG(WARNING) << "failed to change stereo recording"; + return -1; + } + int8_t nChannels(1); + if (enable) { + nChannels = 2; + } + audio_device_buffer_.SetRecordingChannels(nChannels); + return 0; +} + +int32_t AudioDeviceModuleImpl::StereoRecording(bool* enabled) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool stereo = false; + if (audio_device_->StereoRecording(stereo) == -1) { + return -1; + } + *enabled = stereo; + RTC_LOG(INFO) << "output: " << stereo; + return 0; +} + +int32_t AudioDeviceModuleImpl::StereoPlayoutIsAvailable(bool* available) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->StereoPlayoutIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::SetStereoPlayout(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + if (audio_device_->PlayoutIsInitialized()) { + RTC_LOG(LERROR) + << "unable to set stereo mode while playing side is initialized"; + return -1; + } + if (audio_device_->SetStereoPlayout(enable)) { + RTC_LOG(WARNING) << "stereo playout is not supported"; + return -1; + } + int8_t nChannels(1); + if (enable) { + nChannels = 2; + } + audio_device_buffer_.SetPlayoutChannels(nChannels); + return 0; +} + +int32_t AudioDeviceModuleImpl::StereoPlayout(bool* enabled) const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool stereo = false; + if (audio_device_->StereoPlayout(stereo) == -1) { + return -1; + } + *enabled = stereo; + RTC_LOG(INFO) << "output: " << stereo; + return 0; +} + +int32_t AudioDeviceModuleImpl::SetAGC(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + return (audio_device_->SetAGC(enable)); +} + +bool AudioDeviceModuleImpl::AGC() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->AGC(); +} + +int32_t AudioDeviceModuleImpl::PlayoutIsAvailable(bool* available) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->PlayoutIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::RecordingIsAvailable(bool* available) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->RecordingIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_LOG(INFO) << "output: " << isAvailable; + return 0; +} + +int32_t AudioDeviceModuleImpl::MaxMicrophoneVolume(uint32_t* maxVolume) const { + CHECKinitialized_(); + uint32_t maxVol(0); + if (audio_device_->MaxMicrophoneVolume(maxVol) == -1) { + return -1; + } + *maxVolume = maxVol; + return 0; +} + +int32_t AudioDeviceModuleImpl::MinMicrophoneVolume(uint32_t* minVolume) const { + CHECKinitialized_(); + uint32_t minVol(0); + if (audio_device_->MinMicrophoneVolume(minVol) == -1) { + return -1; + } + *minVolume = minVol; + return 0; +} + +int16_t AudioDeviceModuleImpl::PlayoutDevices() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + uint16_t nPlayoutDevices = audio_device_->PlayoutDevices(); + RTC_LOG(INFO) << "output: " << nPlayoutDevices; + return (int16_t)(nPlayoutDevices); +} + +int32_t AudioDeviceModuleImpl::SetPlayoutDevice(uint16_t index) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ")"; + CHECKinitialized_(); + return audio_device_->SetPlayoutDevice(index); +} + +int32_t AudioDeviceModuleImpl::SetPlayoutDevice(WindowsDeviceType device) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->SetPlayoutDevice(device); +} + +int32_t AudioDeviceModuleImpl::PlayoutDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ", ...)"; + CHECKinitialized_(); + if (name == NULL) { + return -1; + } + if (audio_device_->PlayoutDeviceName(index, name, guid) == -1) { + return -1; + } + if (name != NULL) { + RTC_LOG(INFO) << "output: name = " << name; + } + if (guid != NULL) { + RTC_LOG(INFO) << "output: guid = " << guid; + } + return 0; +} + +int32_t AudioDeviceModuleImpl::RecordingDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ", ...)"; + CHECKinitialized_(); + if (name == NULL) { + return -1; + } + if (audio_device_->RecordingDeviceName(index, name, guid) == -1) { + return -1; + } + if (name != NULL) { + RTC_LOG(INFO) << "output: name = " << name; + } + if (guid != NULL) { + RTC_LOG(INFO) << "output: guid = " << guid; + } + return 0; +} + +int16_t AudioDeviceModuleImpl::RecordingDevices() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + uint16_t nRecordingDevices = audio_device_->RecordingDevices(); + RTC_LOG(INFO) << "output: " << nRecordingDevices; + return (int16_t)nRecordingDevices; +} + +int32_t AudioDeviceModuleImpl::SetRecordingDevice(uint16_t index) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ")"; + CHECKinitialized_(); + return audio_device_->SetRecordingDevice(index); +} + +int32_t AudioDeviceModuleImpl::SetRecordingDevice(WindowsDeviceType device) { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->SetRecordingDevice(device); +} + +int32_t AudioDeviceModuleImpl::InitPlayout() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + if (PlayoutIsInitialized()) { + return 0; + } + int32_t result = audio_device_->InitPlayout(); + RTC_LOG(INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitPlayoutSuccess", + static_cast(result == 0)); + return result; +} + +int32_t AudioDeviceModuleImpl::InitRecording() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + if (RecordingIsInitialized()) { + return 0; + } + int32_t result = audio_device_->InitRecording(); + RTC_LOG(INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitRecordingSuccess", + static_cast(result == 0)); + return result; +} + +bool AudioDeviceModuleImpl::PlayoutIsInitialized() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->PlayoutIsInitialized(); +} + +bool AudioDeviceModuleImpl::RecordingIsInitialized() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->RecordingIsInitialized(); +} + +int32_t AudioDeviceModuleImpl::StartPlayout() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + if (Playing()) { + return 0; + } + audio_device_buffer_.StartPlayout(); + int32_t result = audio_device_->StartPlayout(); + RTC_LOG(INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartPlayoutSuccess", + static_cast(result == 0)); + return result; +} + +int32_t AudioDeviceModuleImpl::StopPlayout() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + int32_t result = audio_device_->StopPlayout(); + audio_device_buffer_.StopPlayout(); + RTC_LOG(INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopPlayoutSuccess", + static_cast(result == 0)); + return result; +} + +bool AudioDeviceModuleImpl::Playing() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->Playing(); +} + +int32_t AudioDeviceModuleImpl::StartRecording() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + if (Recording()) { + return 0; + } + audio_device_buffer_.StartRecording(); + int32_t result = audio_device_->StartRecording(); + RTC_LOG(INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartRecordingSuccess", + static_cast(result == 0)); + return result; +} + +int32_t AudioDeviceModuleImpl::StopRecording() { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized_(); + int32_t result = audio_device_->StopRecording(); + audio_device_buffer_.StopRecording(); + RTC_LOG(INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopRecordingSuccess", + static_cast(result == 0)); + return result; +} + +bool AudioDeviceModuleImpl::Recording() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->Recording(); +} + +int32_t AudioDeviceModuleImpl::RegisterAudioCallback( + AudioTransport* audioCallback) { + RTC_LOG(INFO) << __FUNCTION__; + return audio_device_buffer_.RegisterAudioCallback(audioCallback); +} + +int32_t AudioDeviceModuleImpl::PlayoutDelay(uint16_t* delayMS) const { + CHECKinitialized_(); + uint16_t delay = 0; + if (audio_device_->PlayoutDelay(delay) == -1) { + RTC_LOG(LERROR) << "failed to retrieve the playout delay"; + return -1; + } + *delayMS = delay; + return 0; +} + +bool AudioDeviceModuleImpl::BuiltInAECIsAvailable() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isAvailable = audio_device_->BuiltInAECIsAvailable(); + RTC_LOG(INFO) << "output: " << isAvailable; + return isAvailable; +} + +int32_t AudioDeviceModuleImpl::EnableBuiltInAEC(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + int32_t ok = audio_device_->EnableBuiltInAEC(enable); + RTC_LOG(INFO) << "output: " << ok; + return ok; +} + +bool AudioDeviceModuleImpl::BuiltInAGCIsAvailable() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isAvailable = audio_device_->BuiltInAGCIsAvailable(); + RTC_LOG(INFO) << "output: " << isAvailable; + return isAvailable; +} + +int32_t AudioDeviceModuleImpl::EnableBuiltInAGC(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + int32_t ok = audio_device_->EnableBuiltInAGC(enable); + RTC_LOG(INFO) << "output: " << ok; + return ok; +} + +bool AudioDeviceModuleImpl::BuiltInNSIsAvailable() const { + RTC_LOG(INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isAvailable = audio_device_->BuiltInNSIsAvailable(); + RTC_LOG(INFO) << "output: " << isAvailable; + return isAvailable; +} + +int32_t AudioDeviceModuleImpl::EnableBuiltInNS(bool enable) { + RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + int32_t ok = audio_device_->EnableBuiltInNS(enable); + RTC_LOG(INFO) << "output: " << ok; + return ok; +} + +#if defined(WEBRTC_IOS) +int AudioDeviceModuleImpl::GetPlayoutAudioParameters( + AudioParameters* params) const { + RTC_LOG(INFO) << __FUNCTION__; + int r = audio_device_->GetPlayoutAudioParameters(params); + RTC_LOG(INFO) << "output: " << r; + return r; +} + +int AudioDeviceModuleImpl::GetRecordAudioParameters( + AudioParameters* params) const { + RTC_LOG(INFO) << __FUNCTION__; + int r = audio_device_->GetRecordAudioParameters(params); + RTC_LOG(INFO) << "output: " << r; + return r; +} +#endif // WEBRTC_IOS + +AudioDeviceModuleImpl::PlatformType AudioDeviceModuleImpl::Platform() const { + RTC_LOG(INFO) << __FUNCTION__; + return platform_type_; +} + +AudioDeviceModule::AudioLayer AudioDeviceModuleImpl::PlatformAudioLayer() + const { + RTC_LOG(INFO) << __FUNCTION__; + return audio_layer_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.h b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.h new file mode 100644 index 0000000000..dce6b0c8c8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_impl.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_AUDIO_DEVICE_IMPL_H_ +#define MODULES_AUDIO_DEVICE_AUDIO_DEVICE_IMPL_H_ + +#if defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE) + +#include + +#include "modules/audio_device/audio_device_buffer.h" +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/checks.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +class AudioDeviceGeneric; +class AudioManager; + +class AudioDeviceModuleImpl : public AudioDeviceModule { + public: + enum PlatformType { + kPlatformNotSupported = 0, + kPlatformWin32 = 1, + kPlatformWinCe = 2, + kPlatformLinux = 3, + kPlatformMac = 4, + kPlatformAndroid = 5, + kPlatformIOS = 6 + }; + + int32_t CheckPlatform(); + int32_t CreatePlatformSpecificObjects(); + int32_t AttachAudioBuffer(); + + AudioDeviceModuleImpl(const AudioLayer audioLayer); + ~AudioDeviceModuleImpl() override; + + // Retrieve the currently utilized audio layer + int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override; + + // Full-duplex transportation of PCM audio + int32_t RegisterAudioCallback(AudioTransport* audioCallback) override; + + // Main initializaton and termination + int32_t Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + // Device enumeration + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + + // Device selection + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice(WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice(WindowsDeviceType device) override; + + // Audio transport initialization + int32_t PlayoutIsAvailable(bool* available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool* available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + // Audio transport control + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // Microphone Automatic Gain Control (AGC) + int32_t SetAGC(bool enable) override; + bool AGC() const override; + + // Audio mixer initialization + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Speaker volume controls + int32_t SpeakerVolumeIsAvailable(bool* available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t* volume) const override; + int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t* minVolume) const override; + + // Microphone volume controls + int32_t MicrophoneVolumeIsAvailable(bool* available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t* volume) const override; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const override; + + // Speaker mute control + int32_t SpeakerMuteIsAvailable(bool* available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool* enabled) const override; + + // Microphone mute control + int32_t MicrophoneMuteIsAvailable(bool* available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool* enabled) const override; + + // Stereo support + int32_t StereoPlayoutIsAvailable(bool* available) const override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool* enabled) const override; + int32_t StereoRecordingIsAvailable(bool* available) const override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool* enabled) const override; + + // Delay information and control + int32_t PlayoutDelay(uint16_t* delayMS) const override; + + bool BuiltInAECIsAvailable() const override; + int32_t EnableBuiltInAEC(bool enable) override; + bool BuiltInAGCIsAvailable() const override; + int32_t EnableBuiltInAGC(bool enable) override; + bool BuiltInNSIsAvailable() const override; + int32_t EnableBuiltInNS(bool enable) override; + +#if defined(WEBRTC_IOS) + int GetPlayoutAudioParameters(AudioParameters* params) const override; + int GetRecordAudioParameters(AudioParameters* params) const override; +#endif // WEBRTC_IOS + +#if defined(WEBRTC_ANDROID) + // Only use this acccessor for test purposes on Android. + AudioManager* GetAndroidAudioManagerForTest() { + return audio_manager_android_.get(); + } +#endif + AudioDeviceBuffer* GetAudioDeviceBuffer() { return &audio_device_buffer_; } + + private: + PlatformType Platform() const; + AudioLayer PlatformAudioLayer() const; + + AudioLayer audio_layer_; + PlatformType platform_type_ = kPlatformNotSupported; + bool initialized_ = false; +#if defined(WEBRTC_ANDROID) + // Should be declared first to ensure that it outlives other resources. + std::unique_ptr audio_manager_android_; +#endif + AudioDeviceBuffer audio_device_buffer_; + std::unique_ptr audio_device_; +}; + +} // namespace webrtc + +#endif // defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE) + +#endif // MODULES_AUDIO_DEVICE_AUDIO_DEVICE_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_unittest.cc new file mode 100644 index 0000000000..d7df357259 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/audio_device_unittest.cc @@ -0,0 +1,750 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "api/array_view.h" +#include "api/optional.h" +#include "modules/audio_device/audio_device_impl.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/mock_audio_transport.h" +#include "rtc_base/buffer.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/event.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" +#include "rtc_base/timeutils.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::Ge; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::NotNull; + +namespace webrtc { +namespace { + +// #define ENABLE_DEBUG_PRINTF +#ifdef ENABLE_DEBUG_PRINTF +#define PRINTD(...) fprintf(stderr, __VA_ARGS__); +#else +#define PRINTD(...) ((void)0) +#endif +#define PRINT(...) fprintf(stderr, __VA_ARGS__); + +// Don't run these tests in combination with sanitizers. +#if !defined(ADDRESS_SANITIZER) && !defined(MEMORY_SANITIZER) +#define SKIP_TEST_IF_NOT(requirements_satisfied) \ + do { \ + if (!requirements_satisfied) { \ + return; \ + } \ + } while (false) +#else +// Or if other audio-related requirements are not met. +#define SKIP_TEST_IF_NOT(requirements_satisfied) \ + do { \ + return; \ + } while (false) +#endif + +// Number of callbacks (input or output) the tests waits for before we set +// an event indicating that the test was OK. +static constexpr size_t kNumCallbacks = 10; +// Max amount of time we wait for an event to be set while counting callbacks. +static constexpr size_t kTestTimeOutInMilliseconds = 10 * 1000; +// Average number of audio callbacks per second assuming 10ms packet size. +static constexpr size_t kNumCallbacksPerSecond = 100; +// Run the full-duplex test during this time (unit is in seconds). +static constexpr size_t kFullDuplexTimeInSec = 5; +// Length of round-trip latency measurements. Number of deteced impulses +// shall be kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1 since the +// last transmitted pulse is not used. +static constexpr size_t kMeasureLatencyTimeInSec = 10; +// Sets the number of impulses per second in the latency test. +static constexpr size_t kImpulseFrequencyInHz = 1; +// Utilized in round-trip latency measurements to avoid capturing noise samples. +static constexpr int kImpulseThreshold = 1000; + +enum class TransportType { + kInvalid, + kPlay, + kRecord, + kPlayAndRecord, +}; + +// Interface for processing the audio stream. Real implementations can e.g. +// run audio in loopback, read audio from a file or perform latency +// measurements. +class AudioStream { + public: + virtual void Write(rtc::ArrayView source, size_t channels) = 0; + virtual void Read(rtc::ArrayView destination, size_t channels) = 0; + + virtual ~AudioStream() = default; +}; + +// Converts index corresponding to position within a 10ms buffer into a +// delay value in milliseconds. +// Example: index=240, frames_per_10ms_buffer=480 => 5ms as output. +int IndexToMilliseconds(size_t index, size_t frames_per_10ms_buffer) { + return rtc::checked_cast( + 10.0 * (static_cast(index) / frames_per_10ms_buffer) + 0.5); +} + +} // namespace + +// Simple first in first out (FIFO) class that wraps a list of 16-bit audio +// buffers of fixed size and allows Write and Read operations. The idea is to +// store recorded audio buffers (using Write) and then read (using Read) these +// stored buffers with as short delay as possible when the audio layer needs +// data to play out. The number of buffers in the FIFO will stabilize under +// normal conditions since there will be a balance between Write and Read calls. +// The container is a std::list container and access is protected with a lock +// since both sides (playout and recording) are driven by its own thread. +// Note that, we know by design that the size of the audio buffer will not +// change over time and that both sides will use the same size. +class FifoAudioStream : public AudioStream { + public: + void Write(rtc::ArrayView source, size_t channels) override { + EXPECT_EQ(channels, 1u); + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + const size_t size = [&] { + rtc::CritScope lock(&lock_); + fifo_.push_back(Buffer16(source.data(), source.size())); + return fifo_.size(); + }(); + if (size > max_size_) { + max_size_ = size; + } + // Add marker once per second to signal that audio is active. + if (write_count_++ % 100 == 0) { + PRINT("."); + } + written_elements_ += size; + } + + void Read(rtc::ArrayView destination, size_t channels) override { + EXPECT_EQ(channels, 1u); + rtc::CritScope lock(&lock_); + if (fifo_.empty()) { + std::fill(destination.begin(), destination.end(), 0); + } else { + const Buffer16& buffer = fifo_.front(); + RTC_CHECK_EQ(buffer.size(), destination.size()); + std::copy(buffer.begin(), buffer.end(), destination.begin()); + fifo_.pop_front(); + } + } + + size_t size() const { + rtc::CritScope lock(&lock_); + return fifo_.size(); + } + + size_t max_size() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + return max_size_; + } + + size_t average_size() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + return 0.5 + static_cast(written_elements_ / write_count_); + } + + using Buffer16 = rtc::BufferT; + + rtc::CriticalSection lock_; + rtc::RaceChecker race_checker_; + + std::list fifo_ RTC_GUARDED_BY(lock_); + size_t write_count_ RTC_GUARDED_BY(race_checker_) = 0; + size_t max_size_ RTC_GUARDED_BY(race_checker_) = 0; + size_t written_elements_ RTC_GUARDED_BY(race_checker_) = 0; +}; + +// Inserts periodic impulses and measures the latency between the time of +// transmission and time of receiving the same impulse. +class LatencyAudioStream : public AudioStream { + public: + LatencyAudioStream() { + // Delay thread checkers from being initialized until first callback from + // respective thread. + read_thread_checker_.DetachFromThread(); + write_thread_checker_.DetachFromThread(); + } + + // Insert periodic impulses in first two samples of |destination|. + void Read(rtc::ArrayView destination, size_t channels) override { + RTC_DCHECK_RUN_ON(&read_thread_checker_); + EXPECT_EQ(channels, 1u); + if (read_count_ == 0) { + PRINT("["); + } + read_count_++; + std::fill(destination.begin(), destination.end(), 0); + if (read_count_ % (kNumCallbacksPerSecond / kImpulseFrequencyInHz) == 0) { + PRINT("."); + { + rtc::CritScope lock(&lock_); + if (!pulse_time_) { + pulse_time_ = rtc::Optional(rtc::TimeMillis()); + } + } + constexpr int16_t impulse = std::numeric_limits::max(); + std::fill_n(destination.begin(), 2, impulse); + } + } + + // Detect received impulses in |source|, derive time between transmission and + // detection and add the calculated delay to list of latencies. + void Write(rtc::ArrayView source, size_t channels) override { + EXPECT_EQ(channels, 1u); + RTC_DCHECK_RUN_ON(&write_thread_checker_); + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + rtc::CritScope lock(&lock_); + write_count_++; + if (!pulse_time_) { + // Avoid detection of new impulse response until a new impulse has + // been transmitted (sets |pulse_time_| to value larger than zero). + return; + } + // Find index (element position in vector) of the max element. + const size_t index_of_max = + std::max_element(source.begin(), source.end()) - source.begin(); + // Derive time between transmitted pulse and received pulse if the level + // is high enough (removes noise). + const size_t max = source[index_of_max]; + if (max > kImpulseThreshold) { + PRINTD("(%zu, %zu)", max, index_of_max); + int64_t now_time = rtc::TimeMillis(); + int extra_delay = IndexToMilliseconds(index_of_max, source.size()); + PRINTD("[%d]", rtc::checked_cast(now_time - pulse_time_)); + PRINTD("[%d]", extra_delay); + // Total latency is the difference between transmit time and detection + // tome plus the extra delay within the buffer in which we detected the + // received impulse. It is transmitted at sample 0 but can be received + // at sample N where N > 0. The term |extra_delay| accounts for N and it + // is a value between 0 and 10ms. + latencies_.push_back(now_time - *pulse_time_ + extra_delay); + pulse_time_.reset(); + } else { + PRINTD("-"); + } + } + + size_t num_latency_values() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + return latencies_.size(); + } + + int min_latency() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + if (latencies_.empty()) + return 0; + return *std::min_element(latencies_.begin(), latencies_.end()); + } + + int max_latency() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + if (latencies_.empty()) + return 0; + return *std::max_element(latencies_.begin(), latencies_.end()); + } + + int average_latency() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + if (latencies_.empty()) + return 0; + return 0.5 + static_cast( + std::accumulate(latencies_.begin(), latencies_.end(), 0)) / + latencies_.size(); + } + + void PrintResults() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + PRINT("] "); + for (auto it = latencies_.begin(); it != latencies_.end(); ++it) { + PRINTD("%d ", *it); + } + PRINT("\n"); + PRINT("[..........] [min, max, avg]=[%d, %d, %d] ms\n", min_latency(), + max_latency(), average_latency()); + } + + rtc::CriticalSection lock_; + rtc::RaceChecker race_checker_; + rtc::ThreadChecker read_thread_checker_; + rtc::ThreadChecker write_thread_checker_; + + rtc::Optional pulse_time_ RTC_GUARDED_BY(lock_); + std::vector latencies_ RTC_GUARDED_BY(race_checker_); + size_t read_count_ RTC_ACCESS_ON(read_thread_checker_) = 0; + size_t write_count_ RTC_ACCESS_ON(write_thread_checker_) = 0; +}; + +// Mocks the AudioTransport object and proxies actions for the two callbacks +// (RecordedDataIsAvailable and NeedMorePlayData) to different implementations +// of AudioStreamInterface. +class MockAudioTransport : public test::MockAudioTransport { + public: + explicit MockAudioTransport(TransportType type) : type_(type) {} + ~MockAudioTransport() {} + + // Set default actions of the mock object. We are delegating to fake + // implementation where the number of callbacks is counted and an event + // is set after a certain number of callbacks. Audio parameters are also + // checked. + void HandleCallbacks(rtc::Event* event, + AudioStream* audio_stream, + int num_callbacks) { + event_ = event; + audio_stream_ = audio_stream; + num_callbacks_ = num_callbacks; + if (play_mode()) { + ON_CALL(*this, NeedMorePlayData(_, _, _, _, _, _, _, _)) + .WillByDefault( + Invoke(this, &MockAudioTransport::RealNeedMorePlayData)); + } + if (rec_mode()) { + ON_CALL(*this, RecordedDataIsAvailable(_, _, _, _, _, _, _, _, _, _)) + .WillByDefault( + Invoke(this, &MockAudioTransport::RealRecordedDataIsAvailable)); + } + } + + int32_t RealRecordedDataIsAvailable(const void* audio_buffer, + const size_t samples_per_channel, + const size_t bytes_per_frame, + const size_t channels, + const uint32_t sample_rate, + const uint32_t total_delay_ms, + const int32_t clock_drift, + const uint32_t current_mic_level, + const bool typing_status, + uint32_t& new_mic_level) { + EXPECT_TRUE(rec_mode()) << "No test is expecting these callbacks."; + RTC_LOG(INFO) << "+"; + // Store audio parameters once in the first callback. For all other + // callbacks, verify that the provided audio parameters are maintained and + // that each callback corresponds to 10ms for any given sample rate. + if (!record_parameters_.is_complete()) { + record_parameters_.reset(sample_rate, channels, samples_per_channel); + } else { + EXPECT_EQ(samples_per_channel, record_parameters_.frames_per_buffer()); + EXPECT_EQ(bytes_per_frame, record_parameters_.GetBytesPerFrame()); + EXPECT_EQ(channels, record_parameters_.channels()); + EXPECT_EQ(static_cast(sample_rate), + record_parameters_.sample_rate()); + EXPECT_EQ(samples_per_channel, + record_parameters_.frames_per_10ms_buffer()); + } + rec_count_++; + // Write audio data to audio stream object if one has been injected. + if (audio_stream_) { + audio_stream_->Write( + rtc::MakeArrayView(static_cast(audio_buffer), + samples_per_channel * channels), + channels); + } + // Signal the event after given amount of callbacks. + if (ReceivedEnoughCallbacks()) { + event_->Set(); + } + return 0; + } + + int32_t RealNeedMorePlayData(const size_t samples_per_channel, + const size_t bytes_per_frame, + const size_t channels, + const uint32_t sample_rate, + void* audio_buffer, + size_t& samples_per_channel_out, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + EXPECT_TRUE(play_mode()) << "No test is expecting these callbacks."; + RTC_LOG(INFO) << "-"; + // Store audio parameters once in the first callback. For all other + // callbacks, verify that the provided audio parameters are maintained and + // that each callback corresponds to 10ms for any given sample rate. + if (!playout_parameters_.is_complete()) { + playout_parameters_.reset(sample_rate, channels, samples_per_channel); + } else { + EXPECT_EQ(samples_per_channel, playout_parameters_.frames_per_buffer()); + EXPECT_EQ(bytes_per_frame, playout_parameters_.GetBytesPerFrame()); + EXPECT_EQ(channels, playout_parameters_.channels()); + EXPECT_EQ(static_cast(sample_rate), + playout_parameters_.sample_rate()); + EXPECT_EQ(samples_per_channel, + playout_parameters_.frames_per_10ms_buffer()); + } + play_count_++; + samples_per_channel_out = samples_per_channel; + // Read audio data from audio stream object if one has been injected. + if (audio_stream_) { + audio_stream_->Read( + rtc::MakeArrayView(static_cast(audio_buffer), + samples_per_channel * channels), + channels); + } else { + // Fill the audio buffer with zeros to avoid disturbing audio. + const size_t num_bytes = samples_per_channel * bytes_per_frame; + std::memset(audio_buffer, 0, num_bytes); + } + // Signal the event after given amount of callbacks. + if (ReceivedEnoughCallbacks()) { + event_->Set(); + } + return 0; + } + + bool ReceivedEnoughCallbacks() { + bool recording_done = false; + if (rec_mode()) { + recording_done = rec_count_ >= num_callbacks_; + } else { + recording_done = true; + } + bool playout_done = false; + if (play_mode()) { + playout_done = play_count_ >= num_callbacks_; + } else { + playout_done = true; + } + return recording_done && playout_done; + } + + bool play_mode() const { + return type_ == TransportType::kPlay || + type_ == TransportType::kPlayAndRecord; + } + + bool rec_mode() const { + return type_ == TransportType::kRecord || + type_ == TransportType::kPlayAndRecord; + } + + private: + TransportType type_ = TransportType::kInvalid; + rtc::Event* event_ = nullptr; + AudioStream* audio_stream_ = nullptr; + size_t num_callbacks_ = 0; + size_t play_count_ = 0; + size_t rec_count_ = 0; + AudioParameters playout_parameters_; + AudioParameters record_parameters_; +}; + +// AudioDeviceTest test fixture. +class AudioDeviceTest : public ::testing::Test { + protected: + AudioDeviceTest() : event_(false, false) { +#if !defined(ADDRESS_SANITIZER) && !defined(MEMORY_SANITIZER) + rtc::LogMessage::LogToDebug(rtc::LS_INFO); + // Add extra logging fields here if needed for debugging. + // rtc::LogMessage::LogTimestamps(); + // rtc::LogMessage::LogThreads(); + audio_device_ = + AudioDeviceModule::Create(AudioDeviceModule::kPlatformDefaultAudio); + EXPECT_NE(audio_device_.get(), nullptr); + AudioDeviceModule::AudioLayer audio_layer; + int got_platform_audio_layer = + audio_device_->ActiveAudioLayer(&audio_layer); + // First, ensure that a valid audio layer can be activated. + if (got_platform_audio_layer != 0) { + requirements_satisfied_ = false; + } + // Next, verify that the ADM can be initialized. + if (requirements_satisfied_) { + requirements_satisfied_ = (audio_device_->Init() == 0); + } + // Finally, ensure that at least one valid device exists in each direction. + if (requirements_satisfied_) { + const int16_t num_playout_devices = audio_device_->PlayoutDevices(); + const int16_t num_record_devices = audio_device_->RecordingDevices(); + requirements_satisfied_ = + num_playout_devices > 0 && num_record_devices > 0; + } +#else + requirements_satisfied_ = false; +#endif + if (requirements_satisfied_) { + EXPECT_EQ(0, audio_device_->SetPlayoutDevice(0)); + EXPECT_EQ(0, audio_device_->InitSpeaker()); + EXPECT_EQ(0, audio_device_->SetRecordingDevice(0)); + EXPECT_EQ(0, audio_device_->InitMicrophone()); + EXPECT_EQ(0, audio_device_->StereoPlayoutIsAvailable(&stereo_playout_)); + EXPECT_EQ(0, audio_device_->SetStereoPlayout(stereo_playout_)); + // Avoid asking for input stereo support and always record in mono + // since asking can cause issues in combination with remote desktop. + // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7397 for + // details. + EXPECT_EQ(0, audio_device_->SetStereoRecording(false)); + EXPECT_EQ(0, audio_device_->SetAGC(false)); + EXPECT_FALSE(audio_device_->AGC()); + } + } + + virtual ~AudioDeviceTest() { + if (audio_device_) { + EXPECT_EQ(0, audio_device_->Terminate()); + } + } + + bool requirements_satisfied() const { return requirements_satisfied_; } + rtc::Event* event() { return &event_; } + + const rtc::scoped_refptr& audio_device() const { + return audio_device_; + } + + void StartPlayout() { + EXPECT_FALSE(audio_device()->Playing()); + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_TRUE(audio_device()->PlayoutIsInitialized()); + EXPECT_EQ(0, audio_device()->StartPlayout()); + EXPECT_TRUE(audio_device()->Playing()); + } + + void StopPlayout() { + EXPECT_EQ(0, audio_device()->StopPlayout()); + EXPECT_FALSE(audio_device()->Playing()); + EXPECT_FALSE(audio_device()->PlayoutIsInitialized()); + } + + void StartRecording() { + EXPECT_FALSE(audio_device()->Recording()); + EXPECT_EQ(0, audio_device()->InitRecording()); + EXPECT_TRUE(audio_device()->RecordingIsInitialized()); + EXPECT_EQ(0, audio_device()->StartRecording()); + EXPECT_TRUE(audio_device()->Recording()); + } + + void StopRecording() { + EXPECT_EQ(0, audio_device()->StopRecording()); + EXPECT_FALSE(audio_device()->Recording()); + EXPECT_FALSE(audio_device()->RecordingIsInitialized()); + } + + private: + bool requirements_satisfied_ = true; + rtc::Event event_; + rtc::scoped_refptr audio_device_; + bool stereo_playout_ = false; +}; + +// Uses the test fixture to create, initialize and destruct the ADM. +TEST_F(AudioDeviceTest, ConstructDestruct) {} + +TEST_F(AudioDeviceTest, InitTerminate) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + // Initialization is part of the test fixture. + EXPECT_TRUE(audio_device()->Initialized()); + EXPECT_EQ(0, audio_device()->Terminate()); + EXPECT_FALSE(audio_device()->Initialized()); +} + +// Tests Start/Stop playout without any registered audio callback. +TEST_F(AudioDeviceTest, StartStopPlayout) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + StartPlayout(); + StopPlayout(); + StartPlayout(); + StopPlayout(); +} + +// Tests Start/Stop recording without any registered audio callback. +TEST_F(AudioDeviceTest, StartStopRecording) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + StartRecording(); + StopRecording(); + StartRecording(); + StopRecording(); +} + +// Tests Init/Stop/Init recording without any registered audio callback. +// See https://bugs.chromium.org/p/webrtc/issues/detail?id=8041 for details +// on why this test is useful. +TEST_F(AudioDeviceTest, InitStopInitRecording) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + EXPECT_EQ(0, audio_device()->InitRecording()); + EXPECT_TRUE(audio_device()->RecordingIsInitialized()); + StopRecording(); + EXPECT_EQ(0, audio_device()->InitRecording()); + StopRecording(); +} + +// Tests Init/Stop/Init recording while playout is active. +TEST_F(AudioDeviceTest, InitStopInitRecordingWhilePlaying) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + StartPlayout(); + EXPECT_EQ(0, audio_device()->InitRecording()); + EXPECT_TRUE(audio_device()->RecordingIsInitialized()); + StopRecording(); + EXPECT_EQ(0, audio_device()->InitRecording()); + StopRecording(); + StopPlayout(); +} + +// Tests Init/Stop/Init playout without any registered audio callback. +TEST_F(AudioDeviceTest, InitStopInitPlayout) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_TRUE(audio_device()->PlayoutIsInitialized()); + StopPlayout(); + EXPECT_EQ(0, audio_device()->InitPlayout()); + StopPlayout(); +} + +// Tests Init/Stop/Init playout while recording is active. +TEST_F(AudioDeviceTest, InitStopInitPlayoutWhileRecording) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + StartRecording(); + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_TRUE(audio_device()->PlayoutIsInitialized()); + StopPlayout(); + EXPECT_EQ(0, audio_device()->InitPlayout()); + StopPlayout(); + StopRecording(); +} + +// Start playout and verify that the native audio layer starts asking for real +// audio samples to play out using the NeedMorePlayData() callback. +// Note that we can't add expectations on audio parameters in EXPECT_CALL +// since parameter are not provided in the each callback. We therefore test and +// verify the parameters in the fake audio transport implementation instead. +TEST_F(AudioDeviceTest, StartPlayoutVerifyCallbacks) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + MockAudioTransport mock(TransportType::kPlay); + mock.HandleCallbacks(event(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(_, _, _, _, NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + event()->Wait(kTestTimeOutInMilliseconds); + StopPlayout(); +} + +// Start recording and verify that the native audio layer starts providing real +// audio samples using the RecordedDataIsAvailable() callback. +TEST_F(AudioDeviceTest, StartRecordingVerifyCallbacks) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + MockAudioTransport mock(TransportType::kRecord); + mock.HandleCallbacks(event(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, RecordedDataIsAvailable(NotNull(), _, _, _, _, Ge(0u), 0, _, + false, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartRecording(); + event()->Wait(kTestTimeOutInMilliseconds); + StopRecording(); +} + +// Start playout and recording (full-duplex audio) and verify that audio is +// active in both directions. +TEST_F(AudioDeviceTest, StartPlayoutAndRecordingVerifyCallbacks) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + MockAudioTransport mock(TransportType::kPlayAndRecord); + mock.HandleCallbacks(event(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(_, _, _, _, NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_CALL(mock, RecordedDataIsAvailable(NotNull(), _, _, _, _, Ge(0u), 0, _, + false, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + StartRecording(); + event()->Wait(kTestTimeOutInMilliseconds); + StopRecording(); + StopPlayout(); +} + +// Start playout and recording and store recorded data in an intermediate FIFO +// buffer from which the playout side then reads its samples in the same order +// as they were stored. Under ideal circumstances, a callback sequence would +// look like: ...+-+-+-+-+-+-+-..., where '+' means 'packet recorded' and '-' +// means 'packet played'. Under such conditions, the FIFO would contain max 1, +// with an average somewhere in (0,1) depending on how long the packets are +// buffered. However, under more realistic conditions, the size +// of the FIFO will vary more due to an unbalance between the two sides. +// This test tries to verify that the device maintains a balanced callback- +// sequence by running in loopback for a few seconds while measuring the size +// (max and average) of the FIFO. The size of the FIFO is increased by the +// recording side and decreased by the playout side. +TEST_F(AudioDeviceTest, RunPlayoutAndRecordingInFullDuplex) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + NiceMock mock(TransportType::kPlayAndRecord); + FifoAudioStream audio_stream; + mock.HandleCallbacks(event(), &audio_stream, + kFullDuplexTimeInSec * kNumCallbacksPerSecond); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + // Run both sides in mono to make the loopback packet handling less complex. + // The test works for stereo as well; the only requirement is that both sides + // use the same configuration. + EXPECT_EQ(0, audio_device()->SetStereoPlayout(false)); + EXPECT_EQ(0, audio_device()->SetStereoRecording(false)); + StartPlayout(); + StartRecording(); + event()->Wait(static_cast( + std::max(kTestTimeOutInMilliseconds, 1000 * kFullDuplexTimeInSec))); + StopRecording(); + StopPlayout(); + // This thresholds is set rather high to accommodate differences in hardware + // in several devices. The main idea is to capture cases where a very large + // latency is built up. See http://bugs.webrtc.org/7744 for examples on + // bots where relatively large average latencies can happen. + EXPECT_LE(audio_stream.average_size(), 25u); + PRINT("\n"); +} + +// Measures loopback latency and reports the min, max and average values for +// a full duplex audio session. +// The latency is measured like so: +// - Insert impulses periodically on the output side. +// - Detect the impulses on the input side. +// - Measure the time difference between the transmit time and receive time. +// - Store time differences in a vector and calculate min, max and average. +// This test needs the '--gtest_also_run_disabled_tests' flag to run and also +// some sort of audio feedback loop. E.g. a headset where the mic is placed +// close to the speaker to ensure highest possible echo. It is also recommended +// to run the test at highest possible output volume. +TEST_F(AudioDeviceTest, DISABLED_MeasureLoopbackLatency) { + SKIP_TEST_IF_NOT(requirements_satisfied()); + NiceMock mock(TransportType::kPlayAndRecord); + LatencyAudioStream audio_stream; + mock.HandleCallbacks(event(), &audio_stream, + kMeasureLatencyTimeInSec * kNumCallbacksPerSecond); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + EXPECT_EQ(0, audio_device()->SetStereoPlayout(false)); + EXPECT_EQ(0, audio_device()->SetStereoRecording(false)); + StartPlayout(); + StartRecording(); + event()->Wait(static_cast( + std::max(kTestTimeOutInMilliseconds, 1000 * kMeasureLatencyTimeInSec))); + StopRecording(); + StopPlayout(); + // Verify that the correct number of transmitted impulses are detected. + EXPECT_EQ(audio_stream.num_latency_values(), + static_cast( + kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1)); + // Print out min, max and average delay values for debugging purposes. + audio_stream.PrintResults(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.cc b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.cc new file mode 100644 index 0000000000..67ad5729f8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/dummy/audio_device_dummy.h" + +namespace webrtc { + +int32_t AudioDeviceDummy::ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const { + return -1; +} + +AudioDeviceGeneric::InitStatus AudioDeviceDummy::Init() { + return InitStatus::OK; +} + +int32_t AudioDeviceDummy::Terminate() { return 0; } + +bool AudioDeviceDummy::Initialized() const { return true; } + +int16_t AudioDeviceDummy::PlayoutDevices() { return -1; } + +int16_t AudioDeviceDummy::RecordingDevices() { return -1; } + +int32_t AudioDeviceDummy::PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + return -1; +} + +int32_t AudioDeviceDummy::RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + return -1; +} + +int32_t AudioDeviceDummy::SetPlayoutDevice(uint16_t index) { return -1; } + +int32_t AudioDeviceDummy::SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) { + return -1; +} + +int32_t AudioDeviceDummy::SetRecordingDevice(uint16_t index) { return -1; } + +int32_t AudioDeviceDummy::SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) { + return -1; +} + +int32_t AudioDeviceDummy::PlayoutIsAvailable(bool& available) { return -1; } + +int32_t AudioDeviceDummy::InitPlayout() { return -1; } + +bool AudioDeviceDummy::PlayoutIsInitialized() const { return false; } + +int32_t AudioDeviceDummy::RecordingIsAvailable(bool& available) { return -1; } + +int32_t AudioDeviceDummy::InitRecording() { return -1; } + +bool AudioDeviceDummy::RecordingIsInitialized() const { return false; } + +int32_t AudioDeviceDummy::StartPlayout() { return -1; } + +int32_t AudioDeviceDummy::StopPlayout() { return 0; } + +bool AudioDeviceDummy::Playing() const { return false; } + +int32_t AudioDeviceDummy::StartRecording() { return -1; } + +int32_t AudioDeviceDummy::StopRecording() { return 0; } + +bool AudioDeviceDummy::Recording() const { return false; } + +int32_t AudioDeviceDummy::SetAGC(bool enable) { return -1; } + +bool AudioDeviceDummy::AGC() const { return false; } + +int32_t AudioDeviceDummy::InitSpeaker() { return -1; } + +bool AudioDeviceDummy::SpeakerIsInitialized() const { return false; } + +int32_t AudioDeviceDummy::InitMicrophone() { return -1; } + +bool AudioDeviceDummy::MicrophoneIsInitialized() const { return false; } + +int32_t AudioDeviceDummy::SpeakerVolumeIsAvailable(bool& available) { + return -1; +} + +int32_t AudioDeviceDummy::SetSpeakerVolume(uint32_t volume) { return -1; } + +int32_t AudioDeviceDummy::SpeakerVolume(uint32_t& volume) const { return -1; } + +int32_t AudioDeviceDummy::MaxSpeakerVolume(uint32_t& maxVolume) const { + return -1; +} + +int32_t AudioDeviceDummy::MinSpeakerVolume(uint32_t& minVolume) const { + return -1; +} + +int32_t AudioDeviceDummy::MicrophoneVolumeIsAvailable(bool& available) { + return -1; +} + +int32_t AudioDeviceDummy::SetMicrophoneVolume(uint32_t volume) { return -1; } + +int32_t AudioDeviceDummy::MicrophoneVolume(uint32_t& volume) const { + return -1; +} + +int32_t AudioDeviceDummy::MaxMicrophoneVolume(uint32_t& maxVolume) const { + return -1; +} + +int32_t AudioDeviceDummy::MinMicrophoneVolume(uint32_t& minVolume) const { + return -1; +} + +int32_t AudioDeviceDummy::SpeakerMuteIsAvailable(bool& available) { return -1; } + +int32_t AudioDeviceDummy::SetSpeakerMute(bool enable) { return -1; } + +int32_t AudioDeviceDummy::SpeakerMute(bool& enabled) const { return -1; } + +int32_t AudioDeviceDummy::MicrophoneMuteIsAvailable(bool& available) { + return -1; +} + +int32_t AudioDeviceDummy::SetMicrophoneMute(bool enable) { return -1; } + +int32_t AudioDeviceDummy::MicrophoneMute(bool& enabled) const { return -1; } + +int32_t AudioDeviceDummy::StereoPlayoutIsAvailable(bool& available) { + return -1; +} +int32_t AudioDeviceDummy::SetStereoPlayout(bool enable) { return -1; } + +int32_t AudioDeviceDummy::StereoPlayout(bool& enabled) const { return -1; } + +int32_t AudioDeviceDummy::StereoRecordingIsAvailable(bool& available) { + return -1; +} + +int32_t AudioDeviceDummy::SetStereoRecording(bool enable) { return -1; } + +int32_t AudioDeviceDummy::StereoRecording(bool& enabled) const { return -1; } + +int32_t AudioDeviceDummy::PlayoutDelay(uint16_t& delayMS) const { return -1; } + +void AudioDeviceDummy::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.h b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.h new file mode 100644 index 0000000000..18c94fc914 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/audio_device_dummy.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_DEVICE_DUMMY_H_ +#define AUDIO_DEVICE_AUDIO_DEVICE_DUMMY_H_ + +#include + +#include "modules/audio_device/audio_device_generic.h" + +namespace webrtc { + +class AudioDeviceDummy : public AudioDeviceGeneric { + public: + AudioDeviceDummy() {} + virtual ~AudioDeviceDummy() {} + + // Retrieve the currently utilized audio layer + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override; + + // Main initializaton and termination + InitStatus Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + // Device enumeration + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + + // Device selection + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override; + + // Audio transport initialization + int32_t PlayoutIsAvailable(bool& available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool& available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + // Audio transport control + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // Microphone Automatic Gain Control (AGC) + int32_t SetAGC(bool enable) override; + bool AGC() const override; + + // Audio mixer initialization + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Speaker volume controls + int32_t SpeakerVolumeIsAvailable(bool& available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; + + // Microphone volume controls + int32_t MicrophoneVolumeIsAvailable(bool& available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; + + // Speaker mute control + int32_t SpeakerMuteIsAvailable(bool& available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool& enabled) const override; + + // Microphone mute control + int32_t MicrophoneMuteIsAvailable(bool& available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool& enabled) const override; + + // Stereo support + int32_t StereoPlayoutIsAvailable(bool& available) override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool& enabled) const override; + + // Delay information and control + int32_t PlayoutDelay(uint16_t& delayMS) const override; + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; +}; + +} // namespace webrtc + +#endif // AUDIO_DEVICE_AUDIO_DEVICE_DUMMY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.cc b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.cc new file mode 100644 index 0000000000..af9b569257 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.cc @@ -0,0 +1,511 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/dummy/file_audio_device.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/platform_thread.h" +#include "system_wrappers/include/sleep.h" + +namespace webrtc { + +const int kRecordingFixedSampleRate = 48000; +const size_t kRecordingNumChannels = 2; +const int kPlayoutFixedSampleRate = 48000; +const size_t kPlayoutNumChannels = 2; +const size_t kPlayoutBufferSize = + kPlayoutFixedSampleRate / 100 * kPlayoutNumChannels * 2; +const size_t kRecordingBufferSize = + kRecordingFixedSampleRate / 100 * kRecordingNumChannels * 2; + +FileAudioDevice::FileAudioDevice(const char* inputFilename, + const char* outputFilename) + : _ptrAudioBuffer(NULL), + _recordingBuffer(NULL), + _playoutBuffer(NULL), + _recordingFramesLeft(0), + _playoutFramesLeft(0), + _recordingBufferSizeIn10MS(0), + _recordingFramesIn10MS(0), + _playoutFramesIn10MS(0), + _playing(false), + _recording(false), + _lastCallPlayoutMillis(0), + _lastCallRecordMillis(0), + _outputFile(*FileWrapper::Create()), + _inputFile(*FileWrapper::Create()), + _outputFilename(outputFilename), + _inputFilename(inputFilename) {} + +FileAudioDevice::~FileAudioDevice() { + delete &_outputFile; + delete &_inputFile; +} + +int32_t FileAudioDevice::ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const { + return -1; +} + +AudioDeviceGeneric::InitStatus FileAudioDevice::Init() { + return InitStatus::OK; +} + +int32_t FileAudioDevice::Terminate() { + return 0; +} + +bool FileAudioDevice::Initialized() const { + return true; +} + +int16_t FileAudioDevice::PlayoutDevices() { + return 1; +} + +int16_t FileAudioDevice::RecordingDevices() { + return 1; +} + +int32_t FileAudioDevice::PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + const char* kName = "dummy_device"; + const char* kGuid = "dummy_device_unique_id"; + if (index < 1) { + memset(name, 0, kAdmMaxDeviceNameSize); + memset(guid, 0, kAdmMaxGuidSize); + memcpy(name, kName, strlen(kName)); + memcpy(guid, kGuid, strlen(guid)); + return 0; + } + return -1; +} + +int32_t FileAudioDevice::RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + const char* kName = "dummy_device"; + const char* kGuid = "dummy_device_unique_id"; + if (index < 1) { + memset(name, 0, kAdmMaxDeviceNameSize); + memset(guid, 0, kAdmMaxGuidSize); + memcpy(name, kName, strlen(kName)); + memcpy(guid, kGuid, strlen(guid)); + return 0; + } + return -1; +} + +int32_t FileAudioDevice::SetPlayoutDevice(uint16_t index) { + if (index == 0) { + _playout_index = index; + return 0; + } + return -1; +} + +int32_t FileAudioDevice::SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) { + return -1; +} + +int32_t FileAudioDevice::SetRecordingDevice(uint16_t index) { + if (index == 0) { + _record_index = index; + return _record_index; + } + return -1; +} + +int32_t FileAudioDevice::SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) { + return -1; +} + +int32_t FileAudioDevice::PlayoutIsAvailable(bool& available) { + if (_playout_index == 0) { + available = true; + return _playout_index; + } + available = false; + return -1; +} + +int32_t FileAudioDevice::InitPlayout() { + if (_ptrAudioBuffer) { + // Update webrtc audio buffer with the selected parameters + _ptrAudioBuffer->SetPlayoutSampleRate(kPlayoutFixedSampleRate); + _ptrAudioBuffer->SetPlayoutChannels(kPlayoutNumChannels); + } + return 0; +} + +bool FileAudioDevice::PlayoutIsInitialized() const { + return true; +} + +int32_t FileAudioDevice::RecordingIsAvailable(bool& available) { + if (_record_index == 0) { + available = true; + return _record_index; + } + available = false; + return -1; +} + +int32_t FileAudioDevice::InitRecording() { + rtc::CritScope lock(&_critSect); + + if (_recording) { + return -1; + } + + _recordingFramesIn10MS = static_cast(kRecordingFixedSampleRate / 100); + + if (_ptrAudioBuffer) { + _ptrAudioBuffer->SetRecordingSampleRate(kRecordingFixedSampleRate); + _ptrAudioBuffer->SetRecordingChannels(kRecordingNumChannels); + } + return 0; +} + +bool FileAudioDevice::RecordingIsInitialized() const { + return _recordingFramesIn10MS != 0; +} + +int32_t FileAudioDevice::StartPlayout() { + if (_playing) { + return 0; + } + + _playoutFramesIn10MS = static_cast(kPlayoutFixedSampleRate / 100); + _playing = true; + _playoutFramesLeft = 0; + + if (!_playoutBuffer) { + _playoutBuffer = new int8_t[kPlayoutBufferSize]; + } + if (!_playoutBuffer) { + _playing = false; + return -1; + } + + // PLAYOUT + if (!_outputFilename.empty() && + !_outputFile.OpenFile(_outputFilename.c_str(), false)) { + RTC_LOG(LS_ERROR) << "Failed to open playout file: " << _outputFilename; + _playing = false; + delete[] _playoutBuffer; + _playoutBuffer = NULL; + return -1; + } + + _ptrThreadPlay.reset(new rtc::PlatformThread( + PlayThreadFunc, this, "webrtc_audio_module_play_thread")); + _ptrThreadPlay->Start(); + _ptrThreadPlay->SetPriority(rtc::kRealtimePriority); + + RTC_LOG(LS_INFO) << "Started playout capture to output file: " + << _outputFilename; + return 0; +} + +int32_t FileAudioDevice::StopPlayout() { + { + rtc::CritScope lock(&_critSect); + _playing = false; + } + + // stop playout thread first + if (_ptrThreadPlay) { + _ptrThreadPlay->Stop(); + _ptrThreadPlay.reset(); + } + + rtc::CritScope lock(&_critSect); + + _playoutFramesLeft = 0; + delete[] _playoutBuffer; + _playoutBuffer = NULL; + _outputFile.CloseFile(); + + RTC_LOG(LS_INFO) << "Stopped playout capture to output file: " + << _outputFilename; + return 0; +} + +bool FileAudioDevice::Playing() const { + return true; +} + +int32_t FileAudioDevice::StartRecording() { + _recording = true; + + // Make sure we only create the buffer once. + _recordingBufferSizeIn10MS = + _recordingFramesIn10MS * kRecordingNumChannels * 2; + if (!_recordingBuffer) { + _recordingBuffer = new int8_t[_recordingBufferSizeIn10MS]; + } + + if (!_inputFilename.empty() && + !_inputFile.OpenFile(_inputFilename.c_str(), true)) { + RTC_LOG(LS_ERROR) << "Failed to open audio input file: " << _inputFilename; + _recording = false; + delete[] _recordingBuffer; + _recordingBuffer = NULL; + return -1; + } + + _ptrThreadRec.reset(new rtc::PlatformThread( + RecThreadFunc, this, "webrtc_audio_module_capture_thread")); + + _ptrThreadRec->Start(); + _ptrThreadRec->SetPriority(rtc::kRealtimePriority); + + RTC_LOG(LS_INFO) << "Started recording from input file: " << _inputFilename; + + return 0; +} + +int32_t FileAudioDevice::StopRecording() { + { + rtc::CritScope lock(&_critSect); + _recording = false; + } + + if (_ptrThreadRec) { + _ptrThreadRec->Stop(); + _ptrThreadRec.reset(); + } + + rtc::CritScope lock(&_critSect); + _recordingFramesLeft = 0; + if (_recordingBuffer) { + delete[] _recordingBuffer; + _recordingBuffer = NULL; + } + _inputFile.CloseFile(); + + RTC_LOG(LS_INFO) << "Stopped recording from input file: " << _inputFilename; + return 0; +} + +bool FileAudioDevice::Recording() const { + return _recording; +} + +int32_t FileAudioDevice::SetAGC(bool enable) { + return -1; +} + +bool FileAudioDevice::AGC() const { + return false; +} + +int32_t FileAudioDevice::InitSpeaker() { + return -1; +} + +bool FileAudioDevice::SpeakerIsInitialized() const { + return false; +} + +int32_t FileAudioDevice::InitMicrophone() { + return 0; +} + +bool FileAudioDevice::MicrophoneIsInitialized() const { + return true; +} + +int32_t FileAudioDevice::SpeakerVolumeIsAvailable(bool& available) { + return -1; +} + +int32_t FileAudioDevice::SetSpeakerVolume(uint32_t volume) { + return -1; +} + +int32_t FileAudioDevice::SpeakerVolume(uint32_t& volume) const { + return -1; +} + +int32_t FileAudioDevice::MaxSpeakerVolume(uint32_t& maxVolume) const { + return -1; +} + +int32_t FileAudioDevice::MinSpeakerVolume(uint32_t& minVolume) const { + return -1; +} + +int32_t FileAudioDevice::MicrophoneVolumeIsAvailable(bool& available) { + return -1; +} + +int32_t FileAudioDevice::SetMicrophoneVolume(uint32_t volume) { + return -1; +} + +int32_t FileAudioDevice::MicrophoneVolume(uint32_t& volume) const { + return -1; +} + +int32_t FileAudioDevice::MaxMicrophoneVolume(uint32_t& maxVolume) const { + return -1; +} + +int32_t FileAudioDevice::MinMicrophoneVolume(uint32_t& minVolume) const { + return -1; +} + +int32_t FileAudioDevice::SpeakerMuteIsAvailable(bool& available) { + return -1; +} + +int32_t FileAudioDevice::SetSpeakerMute(bool enable) { + return -1; +} + +int32_t FileAudioDevice::SpeakerMute(bool& enabled) const { + return -1; +} + +int32_t FileAudioDevice::MicrophoneMuteIsAvailable(bool& available) { + return -1; +} + +int32_t FileAudioDevice::SetMicrophoneMute(bool enable) { + return -1; +} + +int32_t FileAudioDevice::MicrophoneMute(bool& enabled) const { + return -1; +} + +int32_t FileAudioDevice::StereoPlayoutIsAvailable(bool& available) { + available = true; + return 0; +} +int32_t FileAudioDevice::SetStereoPlayout(bool enable) { + return 0; +} + +int32_t FileAudioDevice::StereoPlayout(bool& enabled) const { + enabled = true; + return 0; +} + +int32_t FileAudioDevice::StereoRecordingIsAvailable(bool& available) { + available = true; + return 0; +} + +int32_t FileAudioDevice::SetStereoRecording(bool enable) { + return 0; +} + +int32_t FileAudioDevice::StereoRecording(bool& enabled) const { + enabled = true; + return 0; +} + +int32_t FileAudioDevice::PlayoutDelay(uint16_t& delayMS) const { + return 0; +} + +void FileAudioDevice::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + rtc::CritScope lock(&_critSect); + + _ptrAudioBuffer = audioBuffer; + + // Inform the AudioBuffer about default settings for this implementation. + // Set all values to zero here since the actual settings will be done by + // InitPlayout and InitRecording later. + _ptrAudioBuffer->SetRecordingSampleRate(0); + _ptrAudioBuffer->SetPlayoutSampleRate(0); + _ptrAudioBuffer->SetRecordingChannels(0); + _ptrAudioBuffer->SetPlayoutChannels(0); +} + +bool FileAudioDevice::PlayThreadFunc(void* pThis) { + return (static_cast(pThis)->PlayThreadProcess()); +} + +bool FileAudioDevice::RecThreadFunc(void* pThis) { + return (static_cast(pThis)->RecThreadProcess()); +} + +bool FileAudioDevice::PlayThreadProcess() { + if (!_playing) { + return false; + } + int64_t currentTime = rtc::TimeMillis(); + _critSect.Enter(); + + if (_lastCallPlayoutMillis == 0 || + currentTime - _lastCallPlayoutMillis >= 10) { + _critSect.Leave(); + _ptrAudioBuffer->RequestPlayoutData(_playoutFramesIn10MS); + _critSect.Enter(); + + _playoutFramesLeft = _ptrAudioBuffer->GetPlayoutData(_playoutBuffer); + RTC_DCHECK_EQ(_playoutFramesIn10MS, _playoutFramesLeft); + if (_outputFile.is_open()) { + _outputFile.Write(_playoutBuffer, kPlayoutBufferSize); + } + _lastCallPlayoutMillis = currentTime; + } + _playoutFramesLeft = 0; + _critSect.Leave(); + + int64_t deltaTimeMillis = rtc::TimeMillis() - currentTime; + if (deltaTimeMillis < 10) { + SleepMs(10 - deltaTimeMillis); + } + + return true; +} + +bool FileAudioDevice::RecThreadProcess() { + if (!_recording) { + return false; + } + + int64_t currentTime = rtc::TimeMillis(); + _critSect.Enter(); + + if (_lastCallRecordMillis == 0 || currentTime - _lastCallRecordMillis >= 10) { + if (_inputFile.is_open()) { + if (_inputFile.Read(_recordingBuffer, kRecordingBufferSize) > 0) { + _ptrAudioBuffer->SetRecordedBuffer(_recordingBuffer, + _recordingFramesIn10MS); + } else { + _inputFile.Rewind(); + } + _lastCallRecordMillis = currentTime; + _critSect.Leave(); + _ptrAudioBuffer->DeliverRecordedData(); + _critSect.Enter(); + } + } + + _critSect.Leave(); + + int64_t deltaTimeMillis = rtc::TimeMillis() - currentTime; + if (deltaTimeMillis < 10) { + SleepMs(10 - deltaTimeMillis); + } + + return true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.h b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.h new file mode 100644 index 0000000000..f735d00e8e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device.h @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_FILE_AUDIO_DEVICE_H_ +#define AUDIO_DEVICE_FILE_AUDIO_DEVICE_H_ + +#include + +#include +#include + +#include "modules/audio_device/audio_device_generic.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/file_wrapper.h" + +namespace rtc { +class PlatformThread; +} // namespace rtc + +namespace webrtc { +class EventWrapper; + +// This is a fake audio device which plays audio from a file as its microphone +// and plays out into a file. +class FileAudioDevice : public AudioDeviceGeneric { + public: + // Constructs a file audio device with |id|. It will read audio from + // |inputFilename| and record output audio to |outputFilename|. + // + // The input file should be a readable 48k stereo raw file, and the output + // file should point to a writable location. The output format will also be + // 48k stereo raw audio. + FileAudioDevice(const char* inputFilename, + const char* outputFilename); + virtual ~FileAudioDevice(); + + // Retrieve the currently utilized audio layer + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override; + + // Main initializaton and termination + InitStatus Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + // Device enumeration + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + + // Device selection + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override; + + // Audio transport initialization + int32_t PlayoutIsAvailable(bool& available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool& available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + // Audio transport control + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // Microphone Automatic Gain Control (AGC) + int32_t SetAGC(bool enable) override; + bool AGC() const override; + + // Audio mixer initialization + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Speaker volume controls + int32_t SpeakerVolumeIsAvailable(bool& available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; + + // Microphone volume controls + int32_t MicrophoneVolumeIsAvailable(bool& available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; + + // Speaker mute control + int32_t SpeakerMuteIsAvailable(bool& available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool& enabled) const override; + + // Microphone mute control + int32_t MicrophoneMuteIsAvailable(bool& available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool& enabled) const override; + + // Stereo support + int32_t StereoPlayoutIsAvailable(bool& available) override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool& enabled) const override; + + // Delay information and control + int32_t PlayoutDelay(uint16_t& delayMS) const override; + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; + + private: + static bool RecThreadFunc(void*); + static bool PlayThreadFunc(void*); + bool RecThreadProcess(); + bool PlayThreadProcess(); + + int32_t _playout_index; + int32_t _record_index; + AudioDeviceBuffer* _ptrAudioBuffer; + int8_t* _recordingBuffer; // In bytes. + int8_t* _playoutBuffer; // In bytes. + uint32_t _recordingFramesLeft; + uint32_t _playoutFramesLeft; + rtc::CriticalSection _critSect; + + size_t _recordingBufferSizeIn10MS; + size_t _recordingFramesIn10MS; + size_t _playoutFramesIn10MS; + + // TODO(pbos): Make plain members instead of pointers and stop resetting them. + std::unique_ptr _ptrThreadRec; + std::unique_ptr _ptrThreadPlay; + + bool _playing; + bool _recording; + int64_t _lastCallPlayoutMillis; + int64_t _lastCallRecordMillis; + + FileWrapper& _outputFile; + FileWrapper& _inputFile; + std::string _outputFilename; + std::string _inputFilename; +}; + +} // namespace webrtc + +#endif // AUDIO_DEVICE_FILE_AUDIO_DEVICE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.cc b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.cc new file mode 100644 index 0000000000..6b38d8bc05 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/dummy/file_audio_device_factory.h" + +#include +#include + +#include "modules/audio_device/dummy/file_audio_device.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +bool FileAudioDeviceFactory::_isConfigured = false; +char FileAudioDeviceFactory::_inputAudioFilename[MAX_FILENAME_LEN] = ""; +char FileAudioDeviceFactory::_outputAudioFilename[MAX_FILENAME_LEN] = ""; + +FileAudioDevice* FileAudioDeviceFactory::CreateFileAudioDevice() { + // Bail out here if the files haven't been set explicitly. + // audio_device_impl.cc should then fall back to dummy audio. + if (!_isConfigured) { + RTC_LOG(LS_WARNING) + << "WebRTC configured with WEBRTC_DUMMY_FILE_DEVICES but " + << "no device files supplied. Will fall back to dummy " + << "audio."; + + return nullptr; + } + return new FileAudioDevice(_inputAudioFilename, _outputAudioFilename); +} + +void FileAudioDeviceFactory::SetFilenamesToUse( + const char* inputAudioFilename, + const char* outputAudioFilename) { +#ifdef WEBRTC_DUMMY_FILE_DEVICES + RTC_DCHECK_LT(strlen(inputAudioFilename), MAX_FILENAME_LEN); + RTC_DCHECK_LT(strlen(outputAudioFilename), MAX_FILENAME_LEN); + + // Copy the strings since we don't know the lifetime of the input pointers. + strncpy(_inputAudioFilename, inputAudioFilename, MAX_FILENAME_LEN); + strncpy(_outputAudioFilename, outputAudioFilename, MAX_FILENAME_LEN); + _isConfigured = true; +#else + // Sanity: must be compiled with the right define to run this. + printf( + "Trying to use dummy file devices, but is not compiled " + "with WEBRTC_DUMMY_FILE_DEVICES. Bailing out.\n"); + std::exit(1); +#endif +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.h b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.h new file mode 100644 index 0000000000..9cd5b3d625 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/dummy/file_audio_device_factory.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_FILE_AUDIO_DEVICE_FACTORY_H_ +#define AUDIO_DEVICE_FILE_AUDIO_DEVICE_FACTORY_H_ + +#include "common_types.h" // NOLINT(build/include) + +namespace webrtc { + +class FileAudioDevice; + +// This class is used by audio_device_impl.cc when WebRTC is compiled with +// WEBRTC_DUMMY_FILE_DEVICES. The application must include this file and set the +// filenames to use before the audio device module is initialized. This is +// intended for test tools which use the audio device module. +class FileAudioDeviceFactory { + public: + static FileAudioDevice* CreateFileAudioDevice(); + + // The input file must be a readable 48k stereo raw file. The output + // file must be writable. The strings will be copied. + static void SetFilenamesToUse(const char* inputAudioFilename, + const char* outputAudioFilename); + + private: + enum : uint32_t { MAX_FILENAME_LEN = 512 }; + static bool _isConfigured; + static char _inputAudioFilename[MAX_FILENAME_LEN]; + static char _outputAudioFilename[MAX_FILENAME_LEN]; +}; + +} // namespace webrtc + +#endif // AUDIO_DEVICE_FILE_AUDIO_DEVICE_FACTORY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.cc new file mode 100644 index 0000000000..f58239745a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/fine_audio_buffer.h" + +#include +#include +#include + +#include "modules/audio_device/audio_device_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +FineAudioBuffer::FineAudioBuffer(AudioDeviceBuffer* device_buffer, + int sample_rate, + size_t capacity) + : device_buffer_(device_buffer), + sample_rate_(sample_rate), + samples_per_10_ms_(static_cast(sample_rate_ * 10 / 1000)), + bytes_per_10_ms_(samples_per_10_ms_ * sizeof(int16_t)), + playout_buffer_(0, capacity), + record_buffer_(0, capacity) { + RTC_LOG(INFO) << "samples_per_10_ms_:" << samples_per_10_ms_; +} + +FineAudioBuffer::~FineAudioBuffer() {} + +void FineAudioBuffer::ResetPlayout() { + playout_buffer_.Clear(); +} + +void FineAudioBuffer::ResetRecord() { + record_buffer_.Clear(); +} + +void FineAudioBuffer::GetPlayoutData(rtc::ArrayView audio_buffer) { + // Ask WebRTC for new data in chunks of 10ms until we have enough to + // fulfill the request. It is possible that the buffer already contains + // enough samples from the last round. + const size_t num_bytes = audio_buffer.size(); + while (playout_buffer_.size() < num_bytes) { + // Get 10ms decoded audio from WebRTC. + device_buffer_->RequestPlayoutData(samples_per_10_ms_); + // Append |bytes_per_10_ms_| elements to the end of the buffer. + const size_t bytes_written = playout_buffer_.AppendData( + bytes_per_10_ms_, [&](rtc::ArrayView buf) { + const size_t samples_per_channel = + device_buffer_->GetPlayoutData(buf.data()); + // TODO(henrika): this class is only used on mobile devices and is + // currently limited to mono. Modifications are needed for stereo. + return sizeof(int16_t) * samples_per_channel; + }); + RTC_DCHECK_EQ(bytes_per_10_ms_, bytes_written); + } + // Provide the requested number of bytes to the consumer. + memcpy(audio_buffer.data(), playout_buffer_.data(), num_bytes); + // Move remaining samples to start of buffer to prepare for next round. + memmove(playout_buffer_.data(), playout_buffer_.data() + num_bytes, + playout_buffer_.size() - num_bytes); + playout_buffer_.SetSize(playout_buffer_.size() - num_bytes); +} + +void FineAudioBuffer::DeliverRecordedData( + rtc::ArrayView audio_buffer, + int playout_delay_ms, + int record_delay_ms) { + // Always append new data and grow the buffer if needed. + record_buffer_.AppendData(audio_buffer.data(), audio_buffer.size()); + // Consume samples from buffer in chunks of 10ms until there is not + // enough data left. The number of remaining bytes in the cache is given by + // the new size of the buffer. + while (record_buffer_.size() >= bytes_per_10_ms_) { + device_buffer_->SetRecordedBuffer(record_buffer_.data(), + samples_per_10_ms_); + device_buffer_->SetVQEData(playout_delay_ms, record_delay_ms, 0); + device_buffer_->DeliverRecordedData(); + memmove(record_buffer_.data(), record_buffer_.data() + bytes_per_10_ms_, + record_buffer_.size() - bytes_per_10_ms_); + record_buffer_.SetSize(record_buffer_.size() - bytes_per_10_ms_); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.h new file mode 100644 index 0000000000..6bd047eca4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_ +#define MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_ + +#include + +#include "api/array_view.h" +#include "rtc_base/buffer.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioDeviceBuffer; + +// FineAudioBuffer takes an AudioDeviceBuffer (ADB) which deals with audio data +// corresponding to 10ms of data. It then allows for this data to be pulled in +// a finer or coarser granularity. I.e. interacting with this class instead of +// directly with the AudioDeviceBuffer one can ask for any number of audio data +// samples. This class also ensures that audio data can be delivered to the ADB +// in 10ms chunks when the size of the provided audio buffers differs from 10ms. +// As an example: calling DeliverRecordedData() with 5ms buffers will deliver +// accumulated 10ms worth of data to the ADB every second call. +// TODO(henrika): add support for stereo when mobile platforms need it. +class FineAudioBuffer { + public: + // |device_buffer| is a buffer that provides 10ms of audio data. + // |sample_rate| is the sample rate of the audio data. This is needed because + // |device_buffer| delivers 10ms of data. Given the sample rate the number + // of samples can be calculated. The |capacity| ensures that the buffer size + // can be increased to at least capacity without further reallocation. + FineAudioBuffer(AudioDeviceBuffer* device_buffer, + int sample_rate, + size_t capacity); + ~FineAudioBuffer(); + + // Clears buffers and counters dealing with playour and/or recording. + void ResetPlayout(); + void ResetRecord(); + + // Copies audio samples into |audio_buffer| where number of requested + // elements is specified by |audio_buffer.size()|. The producer will always + // fill up the audio buffer and if no audio exists, the buffer will contain + // silence instead. + void GetPlayoutData(rtc::ArrayView audio_buffer); + + // Consumes the audio data in |audio_buffer| and sends it to the WebRTC layer + // in chunks of 10ms. The provided delay estimates in |playout_delay_ms| and + // |record_delay_ms| are given to the AEC in the audio processing module. + // They can be fixed values on most platforms and they are ignored if an + // external (hardware/built-in) AEC is used. + // Example: buffer size is 5ms => call #1 stores 5ms of data, call #2 stores + // 5ms of data and sends a total of 10ms to WebRTC and clears the intenal + // cache. Call #3 restarts the scheme above. + void DeliverRecordedData(rtc::ArrayView audio_buffer, + int playout_delay_ms, + int record_delay_ms); + + private: + // Device buffer that works with 10ms chunks of data both for playout and + // for recording. I.e., the WebRTC side will always be asked for audio to be + // played out in 10ms chunks and recorded audio will be sent to WebRTC in + // 10ms chunks as well. This pointer is owned by the constructor of this + // class and the owner must ensure that the pointer is valid during the life- + // time of this object. + AudioDeviceBuffer* const device_buffer_; + // Sample rate in Hertz. + const int sample_rate_; + // Number of audio samples per 10ms. + const size_t samples_per_10_ms_; + // Number of audio bytes per 10ms. + const size_t bytes_per_10_ms_; + // Storage for output samples from which a consumer can read audio buffers + // in any size using GetPlayoutData(). + rtc::BufferT playout_buffer_; + // Storage for input samples that are about to be delivered to the WebRTC + // ADB or remains from the last successful delivery of a 10ms audio buffer. + rtc::BufferT record_buffer_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc new file mode 100644 index 0000000000..43d3f6353d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/fine_audio_buffer.h" + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_device/mock_audio_device_buffer.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::InSequence; +using ::testing::Return; + +namespace webrtc { + +const int kSampleRate = 44100; +const int kSamplesPer10Ms = kSampleRate * 10 / 1000; + +// The fake audio data is 0,1,..SCHAR_MAX-1,0,1,... This is to make it easy +// to detect errors. This function verifies that the buffers contain such data. +// E.g. if there are two buffers of size 3, buffer 1 would contain 0,1,2 and +// buffer 2 would contain 3,4,5. Note that SCHAR_MAX is 127 so wrap-around +// will happen. +// |buffer| is the audio buffer to verify. +bool VerifyBuffer(const int8_t* buffer, int buffer_number, int size) { + int start_value = (buffer_number * size) % SCHAR_MAX; + for (int i = 0; i < size; ++i) { + if (buffer[i] != (i + start_value) % SCHAR_MAX) { + return false; + } + } + return true; +} + +// This function replaces the real AudioDeviceBuffer::GetPlayoutData when it's +// called (which is done implicitly when calling GetBufferData). It writes the +// sequence 0,1,..SCHAR_MAX-1,0,1,... to the buffer. Note that this is likely a +// buffer of different size than the one VerifyBuffer verifies. +// |iteration| is the number of calls made to UpdateBuffer prior to this call. +// |samples_per_10_ms| is the number of samples that should be written to the +// buffer (|arg0|). +ACTION_P2(UpdateBuffer, iteration, samples_per_10_ms) { + int8_t* buffer = static_cast(arg0); + int bytes_per_10_ms = samples_per_10_ms * static_cast(sizeof(int16_t)); + int start_value = (iteration * bytes_per_10_ms) % SCHAR_MAX; + for (int i = 0; i < bytes_per_10_ms; ++i) { + buffer[i] = (i + start_value) % SCHAR_MAX; + } + return samples_per_10_ms; +} + +// Writes a periodic ramp pattern to the supplied |buffer|. See UpdateBuffer() +// for details. +void UpdateInputBuffer(int8_t* buffer, int iteration, int size) { + int start_value = (iteration * size) % SCHAR_MAX; + for (int i = 0; i < size; ++i) { + buffer[i] = (i + start_value) % SCHAR_MAX; + } +} + +// Action macro which verifies that the recorded 10ms chunk of audio data +// (in |arg0|) contains the correct reference values even if they have been +// supplied using a buffer size that is smaller or larger than 10ms. +// See VerifyBuffer() for details. +ACTION_P2(VerifyInputBuffer, iteration, samples_per_10_ms) { + const int8_t* buffer = static_cast(arg0); + int bytes_per_10_ms = samples_per_10_ms * static_cast(sizeof(int16_t)); + int start_value = (iteration * bytes_per_10_ms) % SCHAR_MAX; + for (int i = 0; i < bytes_per_10_ms; ++i) { + EXPECT_EQ(buffer[i], (i + start_value) % SCHAR_MAX); + } + return 0; +} + +void RunFineBufferTest(int frame_size_in_samples) { + const int kFrameSizeBytes = + frame_size_in_samples * static_cast(sizeof(int16_t)); + const int kNumberOfFrames = 5; + // Ceiling of integer division: 1 + ((x - 1) / y) + const int kNumberOfUpdateBufferCalls = + 1 + ((kNumberOfFrames * frame_size_in_samples - 1) / kSamplesPer10Ms); + + MockAudioDeviceBuffer audio_device_buffer; + EXPECT_CALL(audio_device_buffer, RequestPlayoutData(_)) + .WillRepeatedly(Return(kSamplesPer10Ms)); + { + InSequence s; + for (int i = 0; i < kNumberOfUpdateBufferCalls; ++i) { + EXPECT_CALL(audio_device_buffer, GetPlayoutData(_)) + .WillOnce(UpdateBuffer(i, kSamplesPer10Ms)) + .RetiresOnSaturation(); + } + } + { + InSequence s; + for (int j = 0; j < kNumberOfUpdateBufferCalls - 1; ++j) { + EXPECT_CALL(audio_device_buffer, SetRecordedBuffer(_, kSamplesPer10Ms)) + .WillOnce(VerifyInputBuffer(j, kSamplesPer10Ms)) + .RetiresOnSaturation(); + } + } + EXPECT_CALL(audio_device_buffer, SetVQEData(_, _, _)) + .Times(kNumberOfUpdateBufferCalls - 1); + EXPECT_CALL(audio_device_buffer, DeliverRecordedData()) + .Times(kNumberOfUpdateBufferCalls - 1) + .WillRepeatedly(Return(kSamplesPer10Ms)); + + FineAudioBuffer fine_buffer(&audio_device_buffer, kSampleRate, + kFrameSizeBytes); + std::unique_ptr out_buffer(new int8_t[kFrameSizeBytes]); + std::unique_ptr in_buffer(new int8_t[kFrameSizeBytes]); + + for (int i = 0; i < kNumberOfFrames; ++i) { + fine_buffer.GetPlayoutData( + rtc::ArrayView(out_buffer.get(), kFrameSizeBytes)); + EXPECT_TRUE(VerifyBuffer(out_buffer.get(), i, kFrameSizeBytes)); + UpdateInputBuffer(in_buffer.get(), i, kFrameSizeBytes); + fine_buffer.DeliverRecordedData( + rtc::ArrayView(in_buffer.get(), kFrameSizeBytes), 0, 0); + } +} + +TEST(FineBufferTest, BufferLessThan10ms) { + const int kFrameSizeSamples = kSamplesPer10Ms - 50; + RunFineBufferTest(kFrameSizeSamples); +} + +TEST(FineBufferTest, GreaterThan10ms) { + const int kFrameSizeSamples = kSamplesPer10Ms + 50; + RunFineBufferTest(kFrameSizeSamples); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device.h b/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device.h new file mode 100644 index 0000000000..c096a8398c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device.h @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_H_ +#define MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_H_ + +#include "modules/audio_device/include/audio_device_defines.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "rtc_base/refcount.h" + +namespace webrtc { + +class AudioDeviceModule : public rtc::RefCountInterface { + public: + // Deprecated. + // TODO(henrika): to be removed. + enum ErrorCode { + kAdmErrNone = 0, + kAdmErrArgument = 1 + }; + + enum AudioLayer { + kPlatformDefaultAudio = 0, + kWindowsCoreAudio = 2, + kLinuxAlsaAudio = 3, + kLinuxPulseAudio = 4, + kAndroidJavaAudio = 5, + kAndroidOpenSLESAudio = 6, + kAndroidJavaInputAndOpenSLESOutputAudio = 7, + kDummyAudio = 8 + }; + + enum WindowsDeviceType { + kDefaultCommunicationDevice = -1, + kDefaultDevice = -2 + }; + + // TODO(bugs.webrtc.org/7306): deprecated. + enum ChannelType { + kChannelLeft = 0, + kChannelRight = 1, + kChannelBoth = 2 + }; + + public: + // Creates an ADM. + static rtc::scoped_refptr Create( + const AudioLayer audio_layer); + // TODO(bugs.webrtc.org/7306): deprecated (to be removed). + static rtc::scoped_refptr Create( + const int32_t id, + const AudioLayer audio_layer); + + // Retrieve the currently utilized audio layer + virtual int32_t ActiveAudioLayer(AudioLayer* audioLayer) const = 0; + + // Full-duplex transportation of PCM audio + virtual int32_t RegisterAudioCallback(AudioTransport* audioCallback) = 0; + + // Main initialization and termination + virtual int32_t Init() = 0; + virtual int32_t Terminate() = 0; + virtual bool Initialized() const = 0; + + // Device enumeration + virtual int16_t PlayoutDevices() = 0; + virtual int16_t RecordingDevices() = 0; + virtual int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) = 0; + virtual int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) = 0; + + // Device selection + virtual int32_t SetPlayoutDevice(uint16_t index) = 0; + virtual int32_t SetPlayoutDevice(WindowsDeviceType device) = 0; + virtual int32_t SetRecordingDevice(uint16_t index) = 0; + virtual int32_t SetRecordingDevice(WindowsDeviceType device) = 0; + + // Audio transport initialization + virtual int32_t PlayoutIsAvailable(bool* available) = 0; + virtual int32_t InitPlayout() = 0; + virtual bool PlayoutIsInitialized() const = 0; + virtual int32_t RecordingIsAvailable(bool* available) = 0; + virtual int32_t InitRecording() = 0; + virtual bool RecordingIsInitialized() const = 0; + + // Audio transport control + virtual int32_t StartPlayout() = 0; + virtual int32_t StopPlayout() = 0; + virtual bool Playing() const = 0; + virtual int32_t StartRecording() = 0; + virtual int32_t StopRecording() = 0; + virtual bool Recording() const = 0; + + // Microphone Automatic Gain Control (AGC) + virtual int32_t SetAGC(bool enable) = 0; + virtual bool AGC() const = 0; + + // Audio mixer initialization + virtual int32_t InitSpeaker() = 0; + virtual bool SpeakerIsInitialized() const = 0; + virtual int32_t InitMicrophone() = 0; + virtual bool MicrophoneIsInitialized() const = 0; + + // Speaker volume controls + virtual int32_t SpeakerVolumeIsAvailable(bool* available) = 0; + virtual int32_t SetSpeakerVolume(uint32_t volume) = 0; + virtual int32_t SpeakerVolume(uint32_t* volume) const = 0; + virtual int32_t MaxSpeakerVolume(uint32_t* maxVolume) const = 0; + virtual int32_t MinSpeakerVolume(uint32_t* minVolume) const = 0; + + // Microphone volume controls + virtual int32_t MicrophoneVolumeIsAvailable(bool* available) = 0; + virtual int32_t SetMicrophoneVolume(uint32_t volume) = 0; + virtual int32_t MicrophoneVolume(uint32_t* volume) const = 0; + virtual int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const = 0; + virtual int32_t MinMicrophoneVolume(uint32_t* minVolume) const = 0; + + // Speaker mute control + virtual int32_t SpeakerMuteIsAvailable(bool* available) = 0; + virtual int32_t SetSpeakerMute(bool enable) = 0; + virtual int32_t SpeakerMute(bool* enabled) const = 0; + + // Microphone mute control + virtual int32_t MicrophoneMuteIsAvailable(bool* available) = 0; + virtual int32_t SetMicrophoneMute(bool enable) = 0; + virtual int32_t MicrophoneMute(bool* enabled) const = 0; + + // Stereo support + virtual int32_t StereoPlayoutIsAvailable(bool* available) const = 0; + virtual int32_t SetStereoPlayout(bool enable) = 0; + virtual int32_t StereoPlayout(bool* enabled) const = 0; + virtual int32_t StereoRecordingIsAvailable(bool* available) const = 0; + virtual int32_t SetStereoRecording(bool enable) = 0; + virtual int32_t StereoRecording(bool* enabled) const = 0; + // TODO(bugs.webrtc.org/7306): deprecated. + virtual int32_t SetRecordingChannel(const ChannelType channel) { return -1; } + virtual int32_t RecordingChannel(ChannelType* channel) const { return -1; } + + // Playout delay + virtual int32_t PlayoutDelay(uint16_t* delayMS) const = 0; + + // TODO(bugs.webrtc.org/7306): deprecated (to be removed). + virtual int32_t SetRecordingSampleRate(const uint32_t samplesPerSec) { + return -1; + } + virtual int32_t RecordingSampleRate(uint32_t* samplesPerSec) const { + return -1; + } + virtual int32_t SetPlayoutSampleRate(const uint32_t samplesPerSec) { + return -1; + } + virtual int32_t PlayoutSampleRate(uint32_t* samplesPerSec) const { + return -1; + } + + // TODO(bugs.webrtc.org/7306): deprecated (to be removed). + virtual int32_t SetLoudspeakerStatus(bool enable) { return -1; } + virtual int32_t GetLoudspeakerStatus(bool* enabled) const { return -1; } + + // Only supported on Android. + virtual bool BuiltInAECIsAvailable() const = 0; + virtual bool BuiltInAGCIsAvailable() const = 0; + virtual bool BuiltInNSIsAvailable() const = 0; + + // Enables the built-in audio effects. Only supported on Android. + virtual int32_t EnableBuiltInAEC(bool enable) = 0; + virtual int32_t EnableBuiltInAGC(bool enable) = 0; + virtual int32_t EnableBuiltInNS(bool enable) = 0; + +// Only supported on iOS. +#if defined(WEBRTC_IOS) + virtual int GetPlayoutAudioParameters(AudioParameters* params) const = 0; + virtual int GetRecordAudioParameters(AudioParameters* params) const = 0; +#endif // WEBRTC_IOS + + protected: + ~AudioDeviceModule() override {} +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_data_observer.h b/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_data_observer.h new file mode 100644 index 0000000000..3a4ee99518 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_data_observer.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DATA_OBSERVER_H_ +#define MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DATA_OBSERVER_H_ + +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/scoped_ref_ptr.h" + +namespace webrtc { + +// This interface will capture the raw PCM data of both the local captured as +// well as the mixed/rendered remote audio. +class AudioDeviceDataObserver { + public: + virtual void OnCaptureData(const void* audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) = 0; + + virtual void OnRenderData(const void* audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) = 0; + + AudioDeviceDataObserver() = default; + virtual ~AudioDeviceDataObserver() = default; +}; + +// Creates an ADM instance with AudioDeviceDataObserver registered. +rtc::scoped_refptr CreateAudioDeviceWithDataObserver( + const AudioDeviceModule::AudioLayer audio_layer, + AudioDeviceDataObserver* observer); + +// TODO(bugs.webrtc.org/7306): deprecated. +rtc::scoped_refptr CreateAudioDeviceWithDataObserver( + const int32_t id, + const AudioDeviceModule::AudioLayer audio_layer, + AudioDeviceDataObserver* observer); + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DATA_OBSERVER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_defines.h b/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_defines.h new file mode 100644 index 0000000000..510b07c049 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/include/audio_device_defines.h @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DEFINES_H_ +#define MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DEFINES_H_ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +static const int kAdmMaxDeviceNameSize = 128; +static const int kAdmMaxFileNameSize = 512; +static const int kAdmMaxGuidSize = 128; + +static const int kAdmMinPlayoutBufferSizeMs = 10; +static const int kAdmMaxPlayoutBufferSizeMs = 250; + +// ---------------------------------------------------------------------------- +// AudioTransport +// ---------------------------------------------------------------------------- + +class AudioTransport { + public: + virtual int32_t RecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) = 0; + + virtual int32_t NeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) = 0; + + // Method to push the captured audio data to the specific VoE channel. + // The data will not undergo audio processing. + // |voe_channel| is the id of the VoE channel which is the sink to the + // capture data. + virtual void PushCaptureData(int voe_channel, + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) = 0; + + // Method to pull mixed render audio data from all active VoE channels. + // The data will not be passed as reference for audio processing internally. + // TODO(xians): Support getting the unmixed render data from specific VoE + // channel. + virtual void PullRenderData(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) = 0; + + protected: + virtual ~AudioTransport() {} +}; + +// Helper class for storage of fundamental audio parameters such as sample rate, +// number of channels, native buffer size etc. +// Note that one audio frame can contain more than one channel sample and each +// sample is assumed to be a 16-bit PCM sample. Hence, one audio frame in +// stereo contains 2 * (16/8) = 4 bytes of data. +class AudioParameters { + public: + // This implementation does only support 16-bit PCM samples. + static const size_t kBitsPerSample = 16; + AudioParameters() + : sample_rate_(0), + channels_(0), + frames_per_buffer_(0), + frames_per_10ms_buffer_(0) {} + AudioParameters(int sample_rate, size_t channels, size_t frames_per_buffer) + : sample_rate_(sample_rate), + channels_(channels), + frames_per_buffer_(frames_per_buffer), + frames_per_10ms_buffer_(static_cast(sample_rate / 100)) {} + void reset(int sample_rate, size_t channels, size_t frames_per_buffer) { + sample_rate_ = sample_rate; + channels_ = channels; + frames_per_buffer_ = frames_per_buffer; + frames_per_10ms_buffer_ = static_cast(sample_rate / 100); + } + size_t bits_per_sample() const { return kBitsPerSample; } + void reset(int sample_rate, size_t channels, double ms_per_buffer) { + reset(sample_rate, channels, + static_cast(sample_rate * ms_per_buffer + 0.5)); + } + void reset(int sample_rate, size_t channels) { + reset(sample_rate, channels, static_cast(0)); + } + int sample_rate() const { return sample_rate_; } + size_t channels() const { return channels_; } + size_t frames_per_buffer() const { return frames_per_buffer_; } + size_t frames_per_10ms_buffer() const { return frames_per_10ms_buffer_; } + size_t GetBytesPerFrame() const { return channels_ * kBitsPerSample / 8; } + size_t GetBytesPerBuffer() const { + return frames_per_buffer_ * GetBytesPerFrame(); + } + // The WebRTC audio device buffer (ADB) only requires that the sample rate + // and number of channels are configured. Hence, to be "valid", only these + // two attributes must be set. + bool is_valid() const { return ((sample_rate_ > 0) && (channels_ > 0)); } + // Most platforms also require that a native buffer size is defined. + // An audio parameter instance is considered to be "complete" if it is both + // "valid" (can be used by the ADB) and also has a native frame size. + bool is_complete() const { return (is_valid() && (frames_per_buffer_ > 0)); } + size_t GetBytesPer10msBuffer() const { + return frames_per_10ms_buffer_ * GetBytesPerFrame(); + } + double GetBufferSizeInMilliseconds() const { + if (sample_rate_ == 0) + return 0.0; + return frames_per_buffer_ / (sample_rate_ / 1000.0); + } + double GetBufferSizeInSeconds() const { + if (sample_rate_ == 0) + return 0.0; + return static_cast(frames_per_buffer_) / (sample_rate_); + } + + private: + int sample_rate_; + size_t channels_; + size_t frames_per_buffer_; + size_t frames_per_10ms_buffer_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DEFINES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/include/fake_audio_device.h b/third_party/libwebrtc/webrtc/modules/audio_device/include/fake_audio_device.h new file mode 100644 index 0000000000..e8e3f2cf33 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/include/fake_audio_device.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_INCLUDE_FAKE_AUDIO_DEVICE_H_ +#define MODULES_AUDIO_DEVICE_INCLUDE_FAKE_AUDIO_DEVICE_H_ + +#include "modules/audio_device/include/audio_device.h" + +namespace webrtc { + +class FakeAudioDeviceModule : public AudioDeviceModule { + public: + FakeAudioDeviceModule() {} + virtual ~FakeAudioDeviceModule() {} + + // TODO(nisse): Fix all users of this class to managed references using + // scoped_refptr. Current code doesn't always use refcounting for this class. + void AddRef() const override {} + rtc::RefCountReleaseStatus Release() const override { + return rtc::RefCountReleaseStatus::kDroppedLastRef; + } + + private: + int32_t RegisterAudioCallback(AudioTransport* audioCallback) override { + return 0; + } + int32_t Init() override { return 0; } + int32_t InitSpeaker() override { return 0; } + int32_t SetPlayoutDevice(uint16_t index) override { return 0; } + int32_t SetPlayoutDevice(WindowsDeviceType device) override { return 0; } + int32_t SetStereoPlayout(bool enable) override { return 0; } + int32_t StopPlayout() override { return 0; } + int32_t InitMicrophone() override { return 0; } + int32_t SetRecordingDevice(uint16_t index) override { return 0; } + int32_t SetRecordingDevice(WindowsDeviceType device) override { return 0; } + int32_t SetStereoRecording(bool enable) override { return 0; } + int32_t SetAGC(bool enable) override { return 0; } + int32_t StopRecording() override { return 0; } + + int32_t Terminate() override { return 0; } + + int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override { return 0; } + bool Initialized() const override { return true; } + int16_t PlayoutDevices() override { return 0; } + int16_t RecordingDevices() override { return 0; } + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override { + return 0; + } + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override { + return 0; + } + int32_t PlayoutIsAvailable(bool* available) override { return 0; } + int32_t InitPlayout() override { return 0; } + bool PlayoutIsInitialized() const override { return true; } + int32_t RecordingIsAvailable(bool* available) override { return 0; } + int32_t InitRecording() override { return 0; } + bool RecordingIsInitialized() const override { return true; } + int32_t StartPlayout() override { return 0; } + bool Playing() const override { return false; } + int32_t StartRecording() override { return 0; } + bool Recording() const override { return false; } + bool AGC() const override { return true; } + bool SpeakerIsInitialized() const override { return true; } + bool MicrophoneIsInitialized() const override { return true; } + int32_t SpeakerVolumeIsAvailable(bool* available) override { return 0; } + int32_t SetSpeakerVolume(uint32_t volume) override { return 0; } + int32_t SpeakerVolume(uint32_t* volume) const override { return 0; } + int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override { return 0; } + int32_t MinSpeakerVolume(uint32_t* minVolume) const override { return 0; } + int32_t MicrophoneVolumeIsAvailable(bool* available) override { return 0; } + int32_t SetMicrophoneVolume(uint32_t volume) override { return 0; } + int32_t MicrophoneVolume(uint32_t* volume) const override { return 0; } + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override { return 0; } + int32_t MinMicrophoneVolume(uint32_t* minVolume) const override { return 0; } + int32_t SpeakerMuteIsAvailable(bool* available) override { return 0; } + int32_t SetSpeakerMute(bool enable) override { return 0; } + int32_t SpeakerMute(bool* enabled) const override { return 0; } + int32_t MicrophoneMuteIsAvailable(bool* available) override { return 0; } + int32_t SetMicrophoneMute(bool enable) override { return 0; } + int32_t MicrophoneMute(bool* enabled) const override { return 0; } + int32_t StereoPlayoutIsAvailable(bool* available) const override { + *available = false; + return 0; + } + int32_t StereoPlayout(bool* enabled) const override { return 0; } + int32_t StereoRecordingIsAvailable(bool* available) const override { + *available = false; + return 0; + } + int32_t StereoRecording(bool* enabled) const override { return 0; } + int32_t PlayoutDelay(uint16_t* delayMS) const override { + *delayMS = 0; + return 0; + } + bool BuiltInAECIsAvailable() const override { return false; } + int32_t EnableBuiltInAEC(bool enable) override { return -1; } + bool BuiltInAGCIsAvailable() const override { return false; } + int32_t EnableBuiltInAGC(bool enable) override { return -1; } + bool BuiltInNSIsAvailable() const override { return false; } + int32_t EnableBuiltInNS(bool enable) override { return -1; } + +#if defined(WEBRTC_IOS) + int GetPlayoutAudioParameters(AudioParameters* params) const override { + return -1; + } + int GetRecordAudioParameters(AudioParameters* params) const override { + return -1; + } +#endif // WEBRTC_IOS +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_INCLUDE_FAKE_AUDIO_DEVICE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_device.h b/third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_device.h new file mode 100644 index 0000000000..7984e0b5bd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_device.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_INCLUDE_MOCK_AUDIO_DEVICE_H_ +#define MODULES_AUDIO_DEVICE_INCLUDE_MOCK_AUDIO_DEVICE_H_ + +#include + +#include "modules/audio_device/include/audio_device.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockAudioDeviceModule : public AudioDeviceModule { + public: + // RefCounted + MOCK_CONST_METHOD0(AddRef, void()); + MOCK_CONST_METHOD0(Release, rtc::RefCountReleaseStatus()); + // AudioDeviceModule. + MOCK_CONST_METHOD1(ActiveAudioLayer, int32_t(AudioLayer* audioLayer)); + MOCK_CONST_METHOD0(LastError, ErrorCode()); + MOCK_METHOD1(RegisterAudioCallback, int32_t(AudioTransport* audioCallback)); + MOCK_METHOD0(Init, int32_t()); + MOCK_METHOD0(Terminate, int32_t()); + MOCK_CONST_METHOD0(Initialized, bool()); + MOCK_METHOD0(PlayoutDevices, int16_t()); + MOCK_METHOD0(RecordingDevices, int16_t()); + MOCK_METHOD3(PlayoutDeviceName, int32_t(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize])); + MOCK_METHOD3(RecordingDeviceName, int32_t(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize])); + MOCK_METHOD1(SetPlayoutDevice, int32_t(uint16_t index)); + MOCK_METHOD1(SetPlayoutDevice, int32_t(WindowsDeviceType device)); + MOCK_METHOD1(SetRecordingDevice, int32_t(uint16_t index)); + MOCK_METHOD1(SetRecordingDevice, int32_t(WindowsDeviceType device)); + MOCK_METHOD1(PlayoutIsAvailable, int32_t(bool* available)); + MOCK_METHOD0(InitPlayout, int32_t()); + MOCK_CONST_METHOD0(PlayoutIsInitialized, bool()); + MOCK_METHOD1(RecordingIsAvailable, int32_t(bool* available)); + MOCK_METHOD0(InitRecording, int32_t()); + MOCK_CONST_METHOD0(RecordingIsInitialized, bool()); + MOCK_METHOD0(StartPlayout, int32_t()); + MOCK_METHOD0(StopPlayout, int32_t()); + MOCK_CONST_METHOD0(Playing, bool()); + MOCK_METHOD0(StartRecording, int32_t()); + MOCK_METHOD0(StopRecording, int32_t()); + MOCK_CONST_METHOD0(Recording, bool()); + MOCK_METHOD1(SetAGC, int32_t(bool enable)); + MOCK_CONST_METHOD0(AGC, bool()); + MOCK_METHOD0(InitSpeaker, int32_t()); + MOCK_CONST_METHOD0(SpeakerIsInitialized, bool()); + MOCK_METHOD0(InitMicrophone, int32_t()); + MOCK_CONST_METHOD0(MicrophoneIsInitialized, bool()); + MOCK_METHOD1(SpeakerVolumeIsAvailable, int32_t(bool* available)); + MOCK_METHOD1(SetSpeakerVolume, int32_t(uint32_t volume)); + MOCK_CONST_METHOD1(SpeakerVolume, int32_t(uint32_t* volume)); + MOCK_CONST_METHOD1(MaxSpeakerVolume, int32_t(uint32_t* maxVolume)); + MOCK_CONST_METHOD1(MinSpeakerVolume, int32_t(uint32_t* minVolume)); + MOCK_METHOD1(MicrophoneVolumeIsAvailable, int32_t(bool* available)); + MOCK_METHOD1(SetMicrophoneVolume, int32_t(uint32_t volume)); + MOCK_CONST_METHOD1(MicrophoneVolume, int32_t(uint32_t* volume)); + MOCK_CONST_METHOD1(MaxMicrophoneVolume, int32_t(uint32_t* maxVolume)); + MOCK_CONST_METHOD1(MinMicrophoneVolume, int32_t(uint32_t* minVolume)); + MOCK_METHOD1(SpeakerMuteIsAvailable, int32_t(bool* available)); + MOCK_METHOD1(SetSpeakerMute, int32_t(bool enable)); + MOCK_CONST_METHOD1(SpeakerMute, int32_t(bool* enabled)); + MOCK_METHOD1(MicrophoneMuteIsAvailable, int32_t(bool* available)); + MOCK_METHOD1(SetMicrophoneMute, int32_t(bool enable)); + MOCK_CONST_METHOD1(MicrophoneMute, int32_t(bool* enabled)); + MOCK_CONST_METHOD1(StereoPlayoutIsAvailable, int32_t(bool* available)); + MOCK_METHOD1(SetStereoPlayout, int32_t(bool enable)); + MOCK_CONST_METHOD1(StereoPlayout, int32_t(bool* enabled)); + MOCK_CONST_METHOD1(StereoRecordingIsAvailable, int32_t(bool* available)); + MOCK_METHOD1(SetStereoRecording, int32_t(bool enable)); + MOCK_CONST_METHOD1(StereoRecording, int32_t(bool* enabled)); + MOCK_CONST_METHOD1(PlayoutDelay, int32_t(uint16_t* delayMS)); + MOCK_CONST_METHOD0(BuiltInAECIsAvailable, bool()); + MOCK_CONST_METHOD0(BuiltInAGCIsAvailable, bool()); + MOCK_CONST_METHOD0(BuiltInNSIsAvailable, bool()); + MOCK_METHOD1(EnableBuiltInAEC, int32_t(bool enable)); + MOCK_METHOD1(EnableBuiltInAGC, int32_t(bool enable)); + MOCK_METHOD1(EnableBuiltInNS, int32_t(bool enable)); +#if defined(WEBRTC_IOS) + MOCK_CONST_METHOD1(GetPlayoutAudioParameters, int(AudioParameters* params)); + MOCK_CONST_METHOD1(GetRecordAudioParameters, int(AudioParameters* params)); +#endif // WEBRTC_IOS +}; +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_INCLUDE_MOCK_AUDIO_DEVICE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_transport.h b/third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_transport.h new file mode 100644 index 0000000000..baa0227b39 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/include/mock_audio_transport.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_INCLUDE_MOCK_AUDIO_TRANSPORT_H_ +#define MODULES_AUDIO_DEVICE_INCLUDE_MOCK_AUDIO_TRANSPORT_H_ + +#include "modules/audio_device/include/audio_device_defines.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockAudioTransport : public AudioTransport { + public: + MockAudioTransport() {} + ~MockAudioTransport() {} + + MOCK_METHOD10(RecordedDataIsAvailable, + int32_t(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel)); + + MOCK_METHOD8(NeedMorePlayData, + int32_t(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms)); + + MOCK_METHOD6(PushCaptureData, + void(int voe_channel, + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames)); + + MOCK_METHOD7(PullRenderData, + void(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms)); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_INCLUDE_MOCK_AUDIO_TRANSPORT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.h b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.h new file mode 100644 index 0000000000..8f30889835 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.h @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_ +#define MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_ + +#include + +#include "sdk/objc/Framework/Headers/WebRTC/RTCMacros.h" +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/ios/audio_session_observer.h" +#include "modules/audio_device/ios/voice_processing_audio_unit.h" +#include "rtc_base/buffer.h" +#include "rtc_base/gtest_prod_util.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" + +RTC_FWD_DECL_OBJC_CLASS(RTCAudioSessionDelegateAdapter); + +namespace webrtc { + +class FineAudioBuffer; + +// Implements full duplex 16-bit mono PCM audio support for iOS using a +// Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit +// supports audio echo cancellation. It also adds automatic gain control, +// adjustment of voice-processing quality and muting. +// +// An instance must be created and destroyed on one and the same thread. +// All supported public methods must also be called on the same thread. +// A thread checker will RTC_DCHECK if any supported method is called on an +// invalid thread. +// +// Recorded audio will be delivered on a real-time internal I/O thread in the +// audio unit. The audio unit will also ask for audio data to play out on this +// same thread. +class AudioDeviceIOS : public AudioDeviceGeneric, + public AudioSessionObserver, + public VoiceProcessingAudioUnitObserver, + public rtc::MessageHandler { + public: + AudioDeviceIOS(); + ~AudioDeviceIOS(); + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; + + InitStatus Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override { return playing_; } + + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override { return recording_; } + + // These methods returns hard-coded delay values and not dynamic delay + // estimates. The reason is that iOS supports a built-in AEC and the WebRTC + // AEC will always be disabled in the Libjingle layer to avoid running two + // AEC implementations at the same time. And, it saves resources to avoid + // updating these delay values continuously. + // TODO(henrika): it would be possible to mark these two methods as not + // implemented since they are only called for A/V-sync purposes today and + // A/V-sync is not supported on iOS. However, we avoid adding error messages + // the log by using these dummy implementations instead. + int32_t PlayoutDelay(uint16_t& delayMS) const override; + + // Native audio parameters stored during construction. + // These methods are unique for the iOS implementation. + int GetPlayoutAudioParameters(AudioParameters* params) const override; + int GetRecordAudioParameters(AudioParameters* params) const override; + + // These methods are currently not fully implemented on iOS: + + // See audio_device_not_implemented.cc for trivial implementations. + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override; + int32_t PlayoutIsAvailable(bool& available) override; + int32_t RecordingIsAvailable(bool& available) override; + int32_t SetAGC(bool enable) override; + bool AGC() const override; + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + int32_t SpeakerVolumeIsAvailable(bool& available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; + int32_t MicrophoneVolumeIsAvailable(bool& available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; + int32_t MicrophoneMuteIsAvailable(bool& available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool& enabled) const override; + int32_t SpeakerMuteIsAvailable(bool& available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool& enabled) const override; + int32_t StereoPlayoutIsAvailable(bool& available) override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool& enabled) const override; + + // AudioSessionObserver methods. May be called from any thread. + void OnInterruptionBegin() override; + void OnInterruptionEnd() override; + void OnValidRouteChange() override; + void OnCanPlayOrRecordChange(bool can_play_or_record) override; + void OnChangedOutputVolume() override; + + // VoiceProcessingAudioUnitObserver methods. + OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) override; + OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) override; + + // Handles messages from posts. + void OnMessage(rtc::Message *msg) override; + + private: + // Called by the relevant AudioSessionObserver methods on |thread_|. + void HandleInterruptionBegin(); + void HandleInterruptionEnd(); + void HandleValidRouteChange(); + void HandleCanPlayOrRecordChange(bool can_play_or_record); + void HandleSampleRateChange(float sample_rate); + void HandlePlayoutGlitchDetected(); + void HandleOutputVolumeChange(); + + // Uses current |playout_parameters_| and |record_parameters_| to inform the + // audio device buffer (ADB) about our internal audio parameters. + void UpdateAudioDeviceBuffer(); + + // Since the preferred audio parameters are only hints to the OS, the actual + // values may be different once the AVAudioSession has been activated. + // This method asks for the current hardware parameters and takes actions + // if they should differ from what we have asked for initially. It also + // defines |playout_parameters_| and |record_parameters_|. + void SetupAudioBuffersForActiveAudioSession(); + + // Creates the audio unit. + bool CreateAudioUnit(); + + // Updates the audio unit state based on current state. + void UpdateAudioUnit(bool can_play_or_record); + + // Configures the audio session for WebRTC. + bool ConfigureAudioSession(); + // Unconfigures the audio session. + void UnconfigureAudioSession(); + + // Activates our audio session, creates and initializes the voice-processing + // audio unit and verifies that we got the preferred native audio parameters. + bool InitPlayOrRecord(); + + // Closes and deletes the voice-processing I/O unit. + void ShutdownPlayOrRecord(); + + // Ensures that methods are called from the same thread as this object is + // created on. + rtc::ThreadChecker thread_checker_; + + // Native I/O audio thread checker. + rtc::ThreadChecker io_thread_checker_; + + // Thread that this object is created on. + rtc::Thread* thread_; + + // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the + // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). + // The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance + // and therefore outlives this object. + AudioDeviceBuffer* audio_device_buffer_; + + // Contains audio parameters (sample rate, #channels, buffer size etc.) for + // the playout and recording sides. These structure is set in two steps: + // first, native sample rate and #channels are defined in Init(). Next, the + // audio session is activated and we verify that the preferred parameters + // were granted by the OS. At this stage it is also possible to add a third + // component to the parameters; the native I/O buffer duration. + // A RTC_CHECK will be hit if we for some reason fail to open an audio session + // using the specified parameters. + AudioParameters playout_parameters_; + AudioParameters record_parameters_; + + // The AudioUnit used to play and record audio. + std::unique_ptr audio_unit_; + + // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data + // in chunks of 10ms. It then allows for this data to be pulled in + // a finer or coarser granularity. I.e. interacting with this class instead + // of directly with the AudioDeviceBuffer one can ask for any number of + // audio data samples. Is also supports a similar scheme for the recording + // side. + // Example: native buffer size can be 128 audio frames at 16kHz sample rate. + // WebRTC will provide 480 audio frames per 10ms but iOS asks for 128 + // in each callback (one every 8ms). This class can then ask for 128 and the + // FineAudioBuffer will ask WebRTC for new data only when needed and also + // cache non-utilized audio between callbacks. On the recording side, iOS + // can provide audio data frames of size 128 and these are accumulated until + // enough data to supply one 10ms call exists. This 10ms chunk is then sent + // to WebRTC and the remaining part is stored. + std::unique_ptr fine_audio_buffer_; + + // Temporary storage for recorded data. AudioUnitRender() renders into this + // array as soon as a frame of the desired buffer size has been recorded. + // On real iOS devices, the size will be fixed and set once. For iOS + // simulators, the size can vary from callback to callback and the size + // will be changed dynamically to account for this behavior. + rtc::BufferT record_audio_buffer_; + + // Set to 1 when recording is active and 0 otherwise. + volatile int recording_; + + // Set to 1 when playout is active and 0 otherwise. + volatile int playing_; + + // Set to true after successful call to Init(), false otherwise. + bool initialized_ RTC_ACCESS_ON(thread_checker_); + + // Set to true after successful call to InitRecording() or InitPlayout(), + // false otherwise. + bool audio_is_initialized_; + + // Set to true if audio session is interrupted, false otherwise. + bool is_interrupted_; + + // Audio interruption observer instance. + RTCAudioSessionDelegateAdapter* audio_session_observer_ + RTC_ACCESS_ON(thread_checker_); + + // Set to true if we've activated the audio session. + bool has_configured_session_ RTC_ACCESS_ON(thread_checker_); + + // Counts number of detected audio glitches on the playout side. + int64_t num_detected_playout_glitches_ RTC_ACCESS_ON(thread_checker_); + int64_t last_playout_time_ RTC_ACCESS_ON(io_thread_checker_); + + // Counts number of playout callbacks per call. + // The value isupdated on the native I/O thread and later read on the + // creating thread (see thread_checker_) but at this stage no audio is + // active. Hence, it is a "thread safe" design and no lock is needed. + int64_t num_playout_callbacks_; + + // Contains the time for when the last output volume change was detected. + int64_t last_output_volume_change_time_ RTC_ACCESS_ON(thread_checker_); + + // Exposes private members for testing purposes only. + FRIEND_TEST_ALL_PREFIXES(AudioDeviceTest, testInterruptedAudioSession); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.mm b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.mm new file mode 100644 index 0000000000..528e146068 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_ios.mm @@ -0,0 +1,906 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#include "modules/audio_device/ios/audio_device_ios.h" + +#include + +#include "api/array_view.h" +#include "modules/audio_device/fine_audio_buffer.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/bind.h" +#include "rtc_base/checks.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/timeutils.h" +#include "sdk/objc/Framework/Classes/Common/helpers.h" +#include "system_wrappers/include/metrics.h" + +#import "WebRTC/RTCLogging.h" +#import "modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h" +#import "sdk/objc/Framework/Classes/Audio/RTCAudioSession+Private.h" +#import "sdk/objc/Framework/Headers/WebRTC/RTCAudioSession.h" +#import "sdk/objc/Framework/Headers/WebRTC/RTCAudioSessionConfiguration.h" + +namespace webrtc { + +#define LOGI() RTC_LOG(LS_INFO) << "AudioDeviceIOS::" + +#define LOG_AND_RETURN_IF_ERROR(error, message) \ + do { \ + OSStatus err = error; \ + if (err) { \ + RTC_LOG(LS_ERROR) << message << ": " << err; \ + return false; \ + } \ + } while (0) + +#define LOG_IF_ERROR(error, message) \ + do { \ + OSStatus err = error; \ + if (err) { \ + RTC_LOG(LS_ERROR) << message << ": " << err; \ + } \ + } while (0) + +// Hardcoded delay estimates based on real measurements. +// TODO(henrika): these value is not used in combination with built-in AEC. +// Can most likely be removed. +const UInt16 kFixedPlayoutDelayEstimate = 30; +const UInt16 kFixedRecordDelayEstimate = 30; + +enum AudioDeviceMessageType : uint32_t { + kMessageTypeInterruptionBegin, + kMessageTypeInterruptionEnd, + kMessageTypeValidRouteChange, + kMessageTypeCanPlayOrRecordChange, + kMessageTypePlayoutGlitchDetected, + kMessageOutputVolumeChange, +}; + +using ios::CheckAndLogError; + +#if !defined(NDEBUG) +// Returns true when the code runs on a device simulator. +static bool DeviceIsSimulator() { + return ios::GetDeviceName() == "x86_64"; +} + +// Helper method that logs essential device information strings. +static void LogDeviceInfo() { + RTC_LOG(LS_INFO) << "LogDeviceInfo"; + @autoreleasepool { + RTC_LOG(LS_INFO) << " system name: " << ios::GetSystemName(); + RTC_LOG(LS_INFO) << " system version 1(2): " << ios::GetSystemVersionAsString(); + RTC_LOG(LS_INFO) << " system version 2(2): " << ios::GetSystemVersion(); + RTC_LOG(LS_INFO) << " device type: " << ios::GetDeviceType(); + RTC_LOG(LS_INFO) << " device name: " << ios::GetDeviceName(); + RTC_LOG(LS_INFO) << " process name: " << ios::GetProcessName(); + RTC_LOG(LS_INFO) << " process ID: " << ios::GetProcessID(); + RTC_LOG(LS_INFO) << " OS version: " << ios::GetOSVersionString(); + RTC_LOG(LS_INFO) << " processing cores: " << ios::GetProcessorCount(); +#if defined(__IPHONE_9_0) && defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && \ + __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_9_0 + RTC_LOG(LS_INFO) << " low power mode: " << ios::GetLowPowerModeEnabled(); +#endif +#if TARGET_IPHONE_SIMULATOR + RTC_LOG(LS_INFO) << " TARGET_IPHONE_SIMULATOR is defined"; +#endif + RTC_LOG(LS_INFO) << " DeviceIsSimulator: " << DeviceIsSimulator(); + } +} +#endif // !defined(NDEBUG) + +AudioDeviceIOS::AudioDeviceIOS() + : audio_device_buffer_(nullptr), + audio_unit_(nullptr), + recording_(0), + playing_(0), + initialized_(false), + audio_is_initialized_(false), + is_interrupted_(false), + has_configured_session_(false), + num_detected_playout_glitches_(0), + last_playout_time_(0), + num_playout_callbacks_(0), + last_output_volume_change_time_(0) { + LOGI() << "ctor" << ios::GetCurrentThreadDescription(); + io_thread_checker_.DetachFromThread(); + thread_ = rtc::Thread::Current(); + audio_session_observer_ = [[RTCAudioSessionDelegateAdapter alloc] initWithObserver:this]; +} + +AudioDeviceIOS::~AudioDeviceIOS() { + LOGI() << "~dtor" << ios::GetCurrentThreadDescription(); + audio_session_observer_ = nil; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Terminate(); +} + +void AudioDeviceIOS::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + LOGI() << "AttachAudioBuffer"; + RTC_DCHECK(audioBuffer); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + audio_device_buffer_ = audioBuffer; +} + +AudioDeviceGeneric::InitStatus AudioDeviceIOS::Init() { + LOGI() << "Init"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (initialized_) { + return InitStatus::OK; + } +#if !defined(NDEBUG) + LogDeviceInfo(); +#endif + // Store the preferred sample rate and preferred number of channels already + // here. They have not been set and confirmed yet since configureForWebRTC + // is not called until audio is about to start. However, it makes sense to + // store the parameters now and then verify at a later stage. + RTCAudioSessionConfiguration* config = [RTCAudioSessionConfiguration webRTCConfiguration]; + playout_parameters_.reset(config.sampleRate, config.outputNumberOfChannels); + record_parameters_.reset(config.sampleRate, config.inputNumberOfChannels); + // Ensure that the audio device buffer (ADB) knows about the internal audio + // parameters. Note that, even if we are unable to get a mono audio session, + // we will always tell the I/O audio unit to do a channel format conversion + // to guarantee mono on the "input side" of the audio unit. + UpdateAudioDeviceBuffer(); + initialized_ = true; + return InitStatus::OK; +} + +int32_t AudioDeviceIOS::Terminate() { + LOGI() << "Terminate"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!initialized_) { + return 0; + } + StopPlayout(); + StopRecording(); + initialized_ = false; + return 0; +} + +bool AudioDeviceIOS::Initialized() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return initialized_; +} + +int32_t AudioDeviceIOS::InitPlayout() { + LOGI() << "InitPlayout"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(initialized_); + RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!playing_); + if (!audio_is_initialized_) { + if (!InitPlayOrRecord()) { + RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitPlayout!"; + return -1; + } + } + audio_is_initialized_ = true; + return 0; +} + +bool AudioDeviceIOS::PlayoutIsInitialized() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return audio_is_initialized_; +} + +bool AudioDeviceIOS::RecordingIsInitialized() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return audio_is_initialized_; +} + +int32_t AudioDeviceIOS::InitRecording() { + LOGI() << "InitRecording"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(initialized_); + RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!recording_); + if (!audio_is_initialized_) { + if (!InitPlayOrRecord()) { + RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitRecording!"; + return -1; + } + } + audio_is_initialized_ = true; + return 0; +} + +int32_t AudioDeviceIOS::StartPlayout() { + LOGI() << "StartPlayout"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(!playing_); + RTC_DCHECK(audio_unit_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetPlayout(); + } + if (!recording_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + if (!audio_unit_->Start()) { + RTCLogError(@"StartPlayout failed to start audio unit."); + return -1; + } + RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + } + rtc::AtomicOps::ReleaseStore(&playing_, 1); + num_playout_callbacks_ = 0; + num_detected_playout_glitches_ = 0; + return 0; +} + +int32_t AudioDeviceIOS::StopPlayout() { + LOGI() << "StopPlayout"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!audio_is_initialized_ || !playing_) { + return 0; + } + if (!recording_) { + ShutdownPlayOrRecord(); + audio_is_initialized_ = false; + } + rtc::AtomicOps::ReleaseStore(&playing_, 0); + + // Derive average number of calls to OnGetPlayoutData() between detected + // audio glitches and add the result to a histogram. + int average_number_of_playout_callbacks_between_glitches = 100000; + RTC_DCHECK_GE(num_playout_callbacks_, num_detected_playout_glitches_); + if (num_detected_playout_glitches_ > 0) { + average_number_of_playout_callbacks_between_glitches = + num_playout_callbacks_ / num_detected_playout_glitches_; + } + RTC_HISTOGRAM_COUNTS_100000("WebRTC.Audio.AveragePlayoutCallbacksBetweenGlitches", + average_number_of_playout_callbacks_between_glitches); + RTCLog(@"Average number of playout callbacks between glitches: %d", + average_number_of_playout_callbacks_between_glitches); + return 0; +} + +int32_t AudioDeviceIOS::StartRecording() { + LOGI() << "StartRecording"; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(!recording_); + RTC_DCHECK(audio_unit_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetRecord(); + } + if (!playing_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + if (!audio_unit_->Start()) { + RTCLogError(@"StartRecording failed to start audio unit."); + return -1; + } + RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + } + rtc::AtomicOps::ReleaseStore(&recording_, 1); + return 0; +} + +int32_t AudioDeviceIOS::StopRecording() { + LOGI() << "StopRecording"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!audio_is_initialized_ || !recording_) { + return 0; + } + if (!playing_) { + ShutdownPlayOrRecord(); + audio_is_initialized_ = false; + } + rtc::AtomicOps::ReleaseStore(&recording_, 0); + return 0; +} + +int32_t AudioDeviceIOS::PlayoutDelay(uint16_t& delayMS) const { + delayMS = kFixedPlayoutDelayEstimate; + return 0; +} + +int AudioDeviceIOS::GetPlayoutAudioParameters(AudioParameters* params) const { + LOGI() << "GetPlayoutAudioParameters"; + RTC_DCHECK(playout_parameters_.is_valid()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + *params = playout_parameters_; + return 0; +} + +int AudioDeviceIOS::GetRecordAudioParameters(AudioParameters* params) const { + LOGI() << "GetRecordAudioParameters"; + RTC_DCHECK(record_parameters_.is_valid()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + *params = record_parameters_; + return 0; +} + +void AudioDeviceIOS::OnInterruptionBegin() { + RTC_DCHECK(thread_); + LOGI() << "OnInterruptionBegin"; + thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionBegin); +} + +void AudioDeviceIOS::OnInterruptionEnd() { + RTC_DCHECK(thread_); + LOGI() << "OnInterruptionEnd"; + thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionEnd); +} + +void AudioDeviceIOS::OnValidRouteChange() { + RTC_DCHECK(thread_); + thread_->Post(RTC_FROM_HERE, this, kMessageTypeValidRouteChange); +} + +void AudioDeviceIOS::OnCanPlayOrRecordChange(bool can_play_or_record) { + RTC_DCHECK(thread_); + thread_->Post(RTC_FROM_HERE, + this, + kMessageTypeCanPlayOrRecordChange, + new rtc::TypedMessageData(can_play_or_record)); +} + +void AudioDeviceIOS::OnChangedOutputVolume() { + RTC_DCHECK(thread_); + thread_->Post(RTC_FROM_HERE, this, kMessageOutputVolumeChange); +} + +OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* /* io_data */) { + RTC_DCHECK_RUN_ON(&io_thread_checker_); + OSStatus result = noErr; + // Simply return if recording is not enabled. + if (!rtc::AtomicOps::AcquireLoad(&recording_)) return result; + + const size_t num_bytes = num_frames * VoiceProcessingAudioUnit::kBytesPerSample; + // Set the size of our own audio buffer and clear it first to avoid copying + // in combination with potential reallocations. + // On real iOS devices, the size will only be set once (at first callback). + record_audio_buffer_.Clear(); + record_audio_buffer_.SetSize(num_bytes); + + // Allocate AudioBuffers to be used as storage for the received audio. + // The AudioBufferList structure works as a placeholder for the + // AudioBuffer structure, which holds a pointer to the actual data buffer + // in |record_audio_buffer_|. Recorded audio will be rendered into this memory + // at each input callback when calling AudioUnitRender(). + AudioBufferList audio_buffer_list; + audio_buffer_list.mNumberBuffers = 1; + AudioBuffer* audio_buffer = &audio_buffer_list.mBuffers[0]; + audio_buffer->mNumberChannels = record_parameters_.channels(); + audio_buffer->mDataByteSize = record_audio_buffer_.size(); + audio_buffer->mData = record_audio_buffer_.data(); + + // Obtain the recorded audio samples by initiating a rendering cycle. + // Since it happens on the input bus, the |io_data| parameter is a reference + // to the preallocated audio buffer list that the audio unit renders into. + // We can make the audio unit provide a buffer instead in io_data, but we + // currently just use our own. + // TODO(henrika): should error handling be improved? + result = audio_unit_->Render(flags, time_stamp, bus_number, num_frames, &audio_buffer_list); + if (result != noErr) { + RTCLogError(@"Failed to render audio."); + return result; + } + + // Get a pointer to the recorded audio and send it to the WebRTC ADB. + // Use the FineAudioBuffer instance to convert between native buffer size + // and the 10ms buffer size used by WebRTC. + fine_audio_buffer_->DeliverRecordedData( + record_audio_buffer_, kFixedPlayoutDelayEstimate, kFixedRecordDelayEstimate); + return noErr; +} + +OSStatus AudioDeviceIOS::OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + RTC_DCHECK_RUN_ON(&io_thread_checker_); + // Verify 16-bit, noninterleaved mono PCM signal format. + RTC_DCHECK_EQ(1, io_data->mNumberBuffers); + AudioBuffer* audio_buffer = &io_data->mBuffers[0]; + RTC_DCHECK_EQ(1, audio_buffer->mNumberChannels); + + // Get pointer to internal audio buffer to which new audio data shall be + // written. + const size_t size_in_bytes = audio_buffer->mDataByteSize; + RTC_CHECK_EQ(size_in_bytes / VoiceProcessingAudioUnit::kBytesPerSample, num_frames); + int8_t* destination = reinterpret_cast(audio_buffer->mData); + // Produce silence and give audio unit a hint about it if playout is not + // activated. + if (!rtc::AtomicOps::AcquireLoad(&playing_)) { + *flags |= kAudioUnitRenderAction_OutputIsSilence; + memset(destination, 0, size_in_bytes); + return noErr; + } + + // Measure time since last call to OnGetPlayoutData() and see if it is larger + // than a well defined threshold which depends on the current IO buffer size. + // If so, we have an indication of a glitch in the output audio since the + // core audio layer will most likely run dry in this state. + ++num_playout_callbacks_; + const int64_t now_time = rtc::TimeMillis(); + if (time_stamp->mSampleTime != num_frames) { + const int64_t delta_time = now_time - last_playout_time_; + const int glitch_threshold = 1.6 * playout_parameters_.GetBufferSizeInMilliseconds(); + if (delta_time > glitch_threshold) { + RTCLogWarning(@"Possible playout audio glitch detected.\n" + " Time since last OnGetPlayoutData was %lld ms.\n", + delta_time); + // Exclude extreme delta values since they do most likely not correspond + // to a real glitch. Instead, the most probable cause is that a headset + // has been plugged in or out. There are more direct ways to detect + // audio device changes (see HandleValidRouteChange()) but experiments + // show that using it leads to more complex implementations. + // TODO(henrika): more tests might be needed to come up with an even + // better upper limit. + if (glitch_threshold < 120 && delta_time > 120) { + RTCLog(@"Glitch warning is ignored. Probably caused by device switch."); + } else { + thread_->Post(RTC_FROM_HERE, this, kMessageTypePlayoutGlitchDetected); + } + } + } + last_playout_time_ = now_time; + + // Read decoded 16-bit PCM samples from WebRTC (using a size that matches + // the native I/O audio unit) and copy the result to the audio buffer in the + // |io_data| destination. + fine_audio_buffer_->GetPlayoutData(rtc::ArrayView(destination, size_in_bytes)); + return noErr; +} + +void AudioDeviceIOS::OnMessage(rtc::Message* msg) { + switch (msg->message_id) { + case kMessageTypeInterruptionBegin: + HandleInterruptionBegin(); + break; + case kMessageTypeInterruptionEnd: + HandleInterruptionEnd(); + break; + case kMessageTypeValidRouteChange: + HandleValidRouteChange(); + break; + case kMessageTypeCanPlayOrRecordChange: { + rtc::TypedMessageData* data = static_cast*>(msg->pdata); + HandleCanPlayOrRecordChange(data->data()); + delete data; + break; + } + case kMessageTypePlayoutGlitchDetected: + HandlePlayoutGlitchDetected(); + break; + case kMessageOutputVolumeChange: + HandleOutputVolumeChange(); + break; + } +} + +void AudioDeviceIOS::HandleInterruptionBegin() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_); + if (audio_unit_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + RTCLog(@"Stopping the audio unit due to interruption begin."); + if (!audio_unit_->Stop()) { + RTCLogError(@"Failed to stop the audio unit for interruption begin."); + } else { + // The audio unit has been stopped but will be restarted when the + // interruption ends in HandleInterruptionEnd(). It will result in audio + // callbacks from a new native I/O thread which means that we must detach + // thread checkers here to be prepared for an upcoming new audio stream. + io_thread_checker_.DetachFromThread(); + // The audio device buffer must also be informed about the interrupted + // state so it can detach its thread checkers as well. + audio_device_buffer_->NativeAudioInterrupted(); + } + } + is_interrupted_ = true; +} + +void AudioDeviceIOS::HandleInterruptionEnd() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Interruption ended. IsInterrupted changed from %d to 0. " + "Updating audio unit state.", + is_interrupted_); + is_interrupted_ = false; + UpdateAudioUnit([RTCAudioSession sharedInstance].canPlayOrRecord); +} + +void AudioDeviceIOS::HandleValidRouteChange() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + RTCLog(@"%@", session); + HandleSampleRateChange(session.sampleRate); +} + +void AudioDeviceIOS::HandleCanPlayOrRecordChange(bool can_play_or_record) { + RTCLog(@"Handling CanPlayOrRecord change to: %d", can_play_or_record); + UpdateAudioUnit(can_play_or_record); +} + +void AudioDeviceIOS::HandleSampleRateChange(float sample_rate) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Handling sample rate change to %f.", sample_rate); + + // Don't do anything if we're interrupted. + if (is_interrupted_) { + RTCLog(@"Ignoring sample rate change to %f due to interruption.", sample_rate); + return; + } + + // If we don't have an audio unit yet, or the audio unit is uninitialized, + // there is no work to do. + if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { + return; + } + + // The audio unit is already initialized or started. + // Check to see if the sample rate or buffer size has changed. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + const double session_sample_rate = session.sampleRate; + const NSTimeInterval session_buffer_duration = session.IOBufferDuration; + const size_t session_frames_per_buffer = + static_cast(session_sample_rate * session_buffer_duration + .5); + const double current_sample_rate = playout_parameters_.sample_rate(); + const size_t current_frames_per_buffer = playout_parameters_.frames_per_buffer(); + RTCLog(@"Handling playout sample rate change to: %f\n" + " Session sample rate: %f frames_per_buffer: %lu\n" + " ADM sample rate: %f frames_per_buffer: %lu", + sample_rate, + session_sample_rate, + (unsigned long)session_frames_per_buffer, + current_sample_rate, + (unsigned long)current_frames_per_buffer); + + // Sample rate and buffer size are the same, no work to do. + if (std::abs(current_sample_rate - session_sample_rate) <= DBL_EPSILON && + current_frames_per_buffer == session_frames_per_buffer) { + RTCLog(@"Ignoring sample rate change since audio parameters are intact."); + return; + } + + // Extra sanity check to ensure that the new sample rate is valid. + if (session_sample_rate <= 0.0) { + RTCLogError(@"Sample rate is invalid: %f", session_sample_rate); + return; + } + + // We need to adjust our format and buffer sizes. + // The stream format is about to be changed and it requires that we first + // stop and uninitialize the audio unit to deallocate its resources. + RTCLog(@"Stopping and uninitializing audio unit to adjust buffers."); + bool restart_audio_unit = false; + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + audio_unit_->Stop(); + restart_audio_unit = true; + } + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + audio_unit_->Uninitialize(); + } + + // Allocate new buffers given the new stream format. + SetupAudioBuffersForActiveAudioSession(); + + // Initialize the audio unit again with the new sample rate. + RTC_DCHECK_EQ(playout_parameters_.sample_rate(), session_sample_rate); + if (!audio_unit_->Initialize(session_sample_rate)) { + RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", session_sample_rate); + return; + } + + // Restart the audio unit if it was already running. + if (restart_audio_unit && !audio_unit_->Start()) { + RTCLogError(@"Failed to start audio unit with sample rate: %f", session_sample_rate); + return; + } + RTCLog(@"Successfully handled sample rate change."); +} + +void AudioDeviceIOS::HandlePlayoutGlitchDetected() { + RTC_DCHECK_RUN_ON(&thread_checker_); + // Don't update metrics if we're interrupted since a "glitch" is expected + // in this state. + if (is_interrupted_) { + RTCLog(@"Ignoring audio glitch due to interruption."); + return; + } + // Avoid doing glitch detection for two seconds after a volume change + // has been detected to reduce the risk of false alarm. + if (last_output_volume_change_time_ > 0 && + rtc::TimeSince(last_output_volume_change_time_) < 2000) { + RTCLog(@"Ignoring audio glitch due to recent output volume change."); + return; + } + num_detected_playout_glitches_++; + RTCLog(@"Number of detected playout glitches: %lld", num_detected_playout_glitches_); + + int64_t glitch_count = num_detected_playout_glitches_; + dispatch_async(dispatch_get_main_queue(), ^{ + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session notifyDidDetectPlayoutGlitch:glitch_count]; + }); +} + +void AudioDeviceIOS::HandleOutputVolumeChange() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Output volume change detected."); + // Store time of this detection so it can be used to defer detection of + // glitches too close in time to this event. + last_output_volume_change_time_ = rtc::TimeMillis(); +} + +void AudioDeviceIOS::UpdateAudioDeviceBuffer() { + LOGI() << "UpdateAudioDevicebuffer"; + // AttachAudioBuffer() is called at construction by the main class but check + // just in case. + RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; + RTC_CHECK_GT(playout_parameters_.sample_rate(), 0); + RTC_CHECK_GT(record_parameters_.sample_rate(), 0); + RTC_CHECK_EQ(playout_parameters_.channels(), 1); + RTC_CHECK_EQ(record_parameters_.channels(), 1); + // Inform the audio device buffer (ADB) about the new audio format. + audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate()); + audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels()); + audio_device_buffer_->SetRecordingSampleRate(record_parameters_.sample_rate()); + audio_device_buffer_->SetRecordingChannels(record_parameters_.channels()); +} + +void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() { + LOGI() << "SetupAudioBuffersForActiveAudioSession"; + // Verify the current values once the audio session has been activated. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + double sample_rate = session.sampleRate; + NSTimeInterval io_buffer_duration = session.IOBufferDuration; + RTCLog(@"%@", session); + + // Log a warning message for the case when we are unable to set the preferred + // hardware sample rate but continue and use the non-ideal sample rate after + // reinitializing the audio parameters. Most BT headsets only support 8kHz or + // 16kHz. + RTCAudioSessionConfiguration* webRTCConfig = [RTCAudioSessionConfiguration webRTCConfiguration]; + if (sample_rate != webRTCConfig.sampleRate) { + RTC_LOG(LS_WARNING) << "Unable to set the preferred sample rate"; + } + + // Crash reports indicates that it can happen in rare cases that the reported + // sample rate is less than or equal to zero. If that happens and if a valid + // sample rate has already been set during initialization, the best guess we + // can do is to reuse the current sample rate. + if (sample_rate <= DBL_EPSILON && playout_parameters_.sample_rate() > 0) { + RTCLogError(@"Reported rate is invalid: %f. " + "Using %d as sample rate instead.", + sample_rate, playout_parameters_.sample_rate()); + sample_rate = playout_parameters_.sample_rate(); + } + + // At this stage, we also know the exact IO buffer duration and can add + // that info to the existing audio parameters where it is converted into + // number of audio frames. + // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz. + // Hence, 128 is the size we expect to see in upcoming render callbacks. + playout_parameters_.reset(sample_rate, playout_parameters_.channels(), io_buffer_duration); + RTC_DCHECK(playout_parameters_.is_complete()); + record_parameters_.reset(sample_rate, record_parameters_.channels(), io_buffer_duration); + RTC_DCHECK(record_parameters_.is_complete()); + RTC_LOG(LS_INFO) << " frames per I/O buffer: " << playout_parameters_.frames_per_buffer(); + RTC_LOG(LS_INFO) << " bytes per I/O buffer: " << playout_parameters_.GetBytesPerBuffer(); + RTC_DCHECK_EQ(playout_parameters_.GetBytesPerBuffer(), record_parameters_.GetBytesPerBuffer()); + + // Update the ADB parameters since the sample rate might have changed. + UpdateAudioDeviceBuffer(); + + // Create a modified audio buffer class which allows us to ask for, + // or deliver, any number of samples (and not only multiple of 10ms) to match + // the native audio unit buffer size. Use a reasonable capacity to avoid + // reallocations while audio is played to reduce risk of glitches. + RTC_DCHECK(audio_device_buffer_); + const size_t capacity_in_bytes = 2 * playout_parameters_.GetBytesPerBuffer(); + fine_audio_buffer_.reset(new FineAudioBuffer( + audio_device_buffer_, playout_parameters_.sample_rate(), capacity_in_bytes)); +} + +bool AudioDeviceIOS::CreateAudioUnit() { + RTC_DCHECK(!audio_unit_); + + audio_unit_.reset(new VoiceProcessingAudioUnit(this)); + if (!audio_unit_->Init()) { + audio_unit_.reset(); + return false; + } + + return true; +} + +void AudioDeviceIOS::UpdateAudioUnit(bool can_play_or_record) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Updating audio unit state. CanPlayOrRecord=%d IsInterrupted=%d", + can_play_or_record, + is_interrupted_); + + if (is_interrupted_) { + RTCLog(@"Ignoring audio unit update due to interruption."); + return; + } + + // If we're not initialized we don't need to do anything. Audio unit will + // be initialized on initialization. + if (!audio_is_initialized_) return; + + // If we're initialized, we must have an audio unit. + RTC_DCHECK(audio_unit_); + + bool should_initialize_audio_unit = false; + bool should_uninitialize_audio_unit = false; + bool should_start_audio_unit = false; + bool should_stop_audio_unit = false; + + switch (audio_unit_->GetState()) { + case VoiceProcessingAudioUnit::kInitRequired: + RTCLog(@"VPAU state: InitRequired"); + RTC_NOTREACHED(); + break; + case VoiceProcessingAudioUnit::kUninitialized: + RTCLog(@"VPAU state: Uninitialized"); + should_initialize_audio_unit = can_play_or_record; + should_start_audio_unit = should_initialize_audio_unit && (playing_ || recording_); + break; + case VoiceProcessingAudioUnit::kInitialized: + RTCLog(@"VPAU state: Initialized"); + should_start_audio_unit = can_play_or_record && (playing_ || recording_); + should_uninitialize_audio_unit = !can_play_or_record; + break; + case VoiceProcessingAudioUnit::kStarted: + RTCLog(@"VPAU state: Started"); + RTC_DCHECK(playing_ || recording_); + should_stop_audio_unit = !can_play_or_record; + should_uninitialize_audio_unit = should_stop_audio_unit; + break; + } + + if (should_initialize_audio_unit) { + RTCLog(@"Initializing audio unit for UpdateAudioUnit"); + ConfigureAudioSession(); + SetupAudioBuffersForActiveAudioSession(); + if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + RTCLogError(@"Failed to initialize audio unit."); + return; + } + } + + if (should_start_audio_unit) { + RTCLog(@"Starting audio unit for UpdateAudioUnit"); + // Log session settings before trying to start audio streaming. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + RTCLog(@"%@", session); + if (!audio_unit_->Start()) { + RTCLogError(@"Failed to start audio unit."); + return; + } + } + + if (should_stop_audio_unit) { + RTCLog(@"Stopping audio unit for UpdateAudioUnit"); + if (!audio_unit_->Stop()) { + RTCLogError(@"Failed to stop audio unit."); + return; + } + } + + if (should_uninitialize_audio_unit) { + RTCLog(@"Uninitializing audio unit for UpdateAudioUnit"); + audio_unit_->Uninitialize(); + UnconfigureAudioSession(); + } +} + +bool AudioDeviceIOS::ConfigureAudioSession() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Configuring audio session."); + if (has_configured_session_) { + RTCLogWarning(@"Audio session already configured."); + return false; + } + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session lockForConfiguration]; + bool success = [session configureWebRTCSession:nil]; + [session unlockForConfiguration]; + if (success) { + has_configured_session_ = true; + RTCLog(@"Configured audio session."); + } else { + RTCLog(@"Failed to configure audio session."); + } + return success; +} + +void AudioDeviceIOS::UnconfigureAudioSession() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTCLog(@"Unconfiguring audio session."); + if (!has_configured_session_) { + RTCLogWarning(@"Audio session already unconfigured."); + return; + } + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session lockForConfiguration]; + [session unconfigureWebRTCSession:nil]; + [session unlockForConfiguration]; + has_configured_session_ = false; + RTCLog(@"Unconfigured audio session."); +} + +bool AudioDeviceIOS::InitPlayOrRecord() { + LOGI() << "InitPlayOrRecord"; + RTC_DCHECK_RUN_ON(&thread_checker_); + + // There should be no audio unit at this point. + if (!CreateAudioUnit()) { + return false; + } + + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + // Subscribe to audio session events. + [session pushDelegate:audio_session_observer_]; + is_interrupted_ = session.isInterrupted ? true : false; + + // Lock the session to make configuration changes. + [session lockForConfiguration]; + NSError* error = nil; + if (![session beginWebRTCSession:&error]) { + [session unlockForConfiguration]; + RTCLogError(@"Failed to begin WebRTC session: %@", error.localizedDescription); + return false; + } + + // If we are ready to play or record, initialize the audio unit. + if (session.canPlayOrRecord) { + ConfigureAudioSession(); + SetupAudioBuffersForActiveAudioSession(); + audio_unit_->Initialize(playout_parameters_.sample_rate()); + } + + // Release the lock. + [session unlockForConfiguration]; + + return true; +} + +void AudioDeviceIOS::ShutdownPlayOrRecord() { + LOGI() << "ShutdownPlayOrRecord"; + RTC_DCHECK_RUN_ON(&thread_checker_); + + // Stop the audio unit to prevent any additional audio callbacks. + audio_unit_->Stop(); + + // Close and delete the voice-processing I/O unit. + audio_unit_.reset(); + + // Detach thread checker for the AURemoteIO::IOThread to ensure that the + // next session uses a fresh thread id. + io_thread_checker_.DetachFromThread(); + + // Remove audio session notification observers. + RTCAudioSession* session = [RTCAudioSession sharedInstance]; + [session removeDelegate:audio_session_observer_]; + + // All I/O should be stopped or paused prior to deactivating the audio + // session, hence we deactivate as last action. + [session lockForConfiguration]; + UnconfigureAudioSession(); + [session endWebRTCSession:nil]; + [session unlockForConfiguration]; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_not_implemented_ios.mm b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_not_implemented_ios.mm new file mode 100644 index 0000000000..2022dddcb1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_not_implemented_ios.mm @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/ios/audio_device_ios.h" + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +int32_t AudioDeviceIOS::ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const { + audioLayer = AudioDeviceModule::kPlatformDefaultAudio; + return 0; +} + +int16_t AudioDeviceIOS::PlayoutDevices() { + // TODO(henrika): improve. + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return (int16_t)1; +} + +int16_t AudioDeviceIOS::RecordingDevices() { + // TODO(henrika): improve. + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return (int16_t)1; +} + +int32_t AudioDeviceIOS::InitSpeaker() { + return 0; +} + +bool AudioDeviceIOS::SpeakerIsInitialized() const { + return true; +} + +int32_t AudioDeviceIOS::SpeakerVolumeIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetSpeakerVolume(uint32_t volume) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerVolume(uint32_t& volume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MaxSpeakerVolume(uint32_t& maxVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MinSpeakerVolume(uint32_t& minVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerMuteIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetSpeakerMute(bool enable) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerMute(bool& enabled) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SetPlayoutDevice(uint16_t index) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return 0; +} + +int32_t AudioDeviceIOS::SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::InitMicrophone() { + return 0; +} + +bool AudioDeviceIOS::MicrophoneIsInitialized() const { + return true; +} + +int32_t AudioDeviceIOS::MicrophoneMuteIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetMicrophoneMute(bool enable) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MicrophoneMute(bool& enabled) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoRecordingIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetStereoRecording(bool enable) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoRecording(bool& enabled) const { + enabled = false; + return 0; +} + +int32_t AudioDeviceIOS::StereoPlayoutIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetStereoPlayout(bool enable) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoPlayout(bool& enabled) const { + enabled = false; + return 0; +} + +int32_t AudioDeviceIOS::SetAGC(bool enable) { + if (enable) { + RTC_NOTREACHED() << "Should never be called"; + } + return -1; +} + +bool AudioDeviceIOS::AGC() const { + return false; +} + +int32_t AudioDeviceIOS::MicrophoneVolumeIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetMicrophoneVolume(uint32_t volume) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MicrophoneVolume(uint32_t& volume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MaxMicrophoneVolume(uint32_t& maxVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MinMicrophoneVolume(uint32_t& minVolume) const { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SetRecordingDevice(uint16_t index) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return 0; +} + +int32_t AudioDeviceIOS::SetRecordingDevice(AudioDeviceModule::WindowsDeviceType) { + RTC_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::PlayoutIsAvailable(bool& available) { + available = true; + return 0; +} + +int32_t AudioDeviceIOS::RecordingIsAvailable(bool& available) { + available = true; + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_unittest_ios.mm b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_unittest_ios.mm new file mode 100644 index 0000000000..b0a356f2e6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_device_unittest_ios.mm @@ -0,0 +1,880 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "modules/audio_device/audio_device_impl.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/mock_audio_transport.h" +#include "modules/audio_device/ios/audio_device_ios.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/logging.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/event_wrapper.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +#import "sdk/objc/Framework/Classes/Audio/RTCAudioSession+Private.h" +#import "sdk/objc/Framework/Headers/WebRTC/RTCAudioSession.h" + +using std::cout; +using std::endl; +using ::testing::_; +using ::testing::AtLeast; +using ::testing::Gt; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::NotNull; +using ::testing::Return; + +// #define ENABLE_DEBUG_PRINTF +#ifdef ENABLE_DEBUG_PRINTF +#define PRINTD(...) fprintf(stderr, __VA_ARGS__); +#else +#define PRINTD(...) ((void)0) +#endif +#define PRINT(...) fprintf(stderr, __VA_ARGS__); + +namespace webrtc { + +// Number of callbacks (input or output) the tests waits for before we set +// an event indicating that the test was OK. +static const size_t kNumCallbacks = 10; +// Max amount of time we wait for an event to be set while counting callbacks. +static const int kTestTimeOutInMilliseconds = 10 * 1000; +// Number of bits per PCM audio sample. +static const size_t kBitsPerSample = 16; +// Number of bytes per PCM audio sample. +static const size_t kBytesPerSample = kBitsPerSample / 8; +// Average number of audio callbacks per second assuming 10ms packet size. +static const size_t kNumCallbacksPerSecond = 100; +// Play out a test file during this time (unit is in seconds). +static const int kFilePlayTimeInSec = 15; +// Run the full-duplex test during this time (unit is in seconds). +// Note that first |kNumIgnoreFirstCallbacks| are ignored. +static const int kFullDuplexTimeInSec = 10; +// Wait for the callback sequence to stabilize by ignoring this amount of the +// initial callbacks (avoids initial FIFO access). +// Only used in the RunPlayoutAndRecordingInFullDuplex test. +static const size_t kNumIgnoreFirstCallbacks = 50; +// Sets the number of impulses per second in the latency test. +// TODO(henrika): fine tune this setting for iOS. +static const int kImpulseFrequencyInHz = 1; +// Length of round-trip latency measurements. Number of transmitted impulses +// is kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1. +// TODO(henrika): fine tune this setting for iOS. +static const int kMeasureLatencyTimeInSec = 5; +// Utilized in round-trip latency measurements to avoid capturing noise samples. +// TODO(henrika): fine tune this setting for iOS. +static const int kImpulseThreshold = 50; +static const char kTag[] = "[..........] "; + +enum TransportType { + kPlayout = 0x1, + kRecording = 0x2, +}; + +// Interface for processing the audio stream. Real implementations can e.g. +// run audio in loopback, read audio from a file or perform latency +// measurements. +class AudioStreamInterface { + public: + virtual void Write(const void* source, size_t num_frames) = 0; + virtual void Read(void* destination, size_t num_frames) = 0; + + protected: + virtual ~AudioStreamInterface() {} +}; + +// Reads audio samples from a PCM file where the file is stored in memory at +// construction. +class FileAudioStream : public AudioStreamInterface { + public: + FileAudioStream(size_t num_callbacks, + const std::string& file_name, + int sample_rate) + : file_size_in_bytes_(0), sample_rate_(sample_rate), file_pos_(0) { + file_size_in_bytes_ = test::GetFileSize(file_name); + sample_rate_ = sample_rate; + EXPECT_GE(file_size_in_callbacks(), num_callbacks) + << "Size of test file is not large enough to last during the test."; + const size_t num_16bit_samples = + test::GetFileSize(file_name) / kBytesPerSample; + file_.reset(new int16_t[num_16bit_samples]); + FILE* audio_file = fopen(file_name.c_str(), "rb"); + EXPECT_NE(audio_file, nullptr); + size_t num_samples_read = + fread(file_.get(), sizeof(int16_t), num_16bit_samples, audio_file); + EXPECT_EQ(num_samples_read, num_16bit_samples); + fclose(audio_file); + } + + // AudioStreamInterface::Write() is not implemented. + void Write(const void* source, size_t num_frames) override {} + + // Read samples from file stored in memory (at construction) and copy + // |num_frames| (<=> 10ms) to the |destination| byte buffer. + void Read(void* destination, size_t num_frames) override { + memcpy(destination, static_cast(&file_[file_pos_]), + num_frames * sizeof(int16_t)); + file_pos_ += num_frames; + } + + int file_size_in_seconds() const { + return static_cast( + file_size_in_bytes_ / (kBytesPerSample * sample_rate_)); + } + size_t file_size_in_callbacks() const { + return file_size_in_seconds() * kNumCallbacksPerSecond; + } + + private: + size_t file_size_in_bytes_; + int sample_rate_; + std::unique_ptr file_; + size_t file_pos_; +}; + +// Simple first in first out (FIFO) class that wraps a list of 16-bit audio +// buffers of fixed size and allows Write and Read operations. The idea is to +// store recorded audio buffers (using Write) and then read (using Read) these +// stored buffers with as short delay as possible when the audio layer needs +// data to play out. The number of buffers in the FIFO will stabilize under +// normal conditions since there will be a balance between Write and Read calls. +// The container is a std::list container and access is protected with a lock +// since both sides (playout and recording) are driven by its own thread. +class FifoAudioStream : public AudioStreamInterface { + public: + explicit FifoAudioStream(size_t frames_per_buffer) + : frames_per_buffer_(frames_per_buffer), + bytes_per_buffer_(frames_per_buffer_ * sizeof(int16_t)), + fifo_(new AudioBufferList), + largest_size_(0), + total_written_elements_(0), + write_count_(0) { + EXPECT_NE(fifo_.get(), nullptr); + } + + ~FifoAudioStream() { Flush(); } + + // Allocate new memory, copy |num_frames| samples from |source| into memory + // and add pointer to the memory location to end of the list. + // Increases the size of the FIFO by one element. + void Write(const void* source, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + PRINTD("+"); + if (write_count_++ < kNumIgnoreFirstCallbacks) { + return; + } + int16_t* memory = new int16_t[frames_per_buffer_]; + memcpy(static_cast(&memory[0]), source, bytes_per_buffer_); + rtc::CritScope lock(&lock_); + fifo_->push_back(memory); + const size_t size = fifo_->size(); + if (size > largest_size_) { + largest_size_ = size; + PRINTD("(%" PRIuS ")", largest_size_); + } + total_written_elements_ += size; + } + + // Read pointer to data buffer from front of list, copy |num_frames| of stored + // data into |destination| and delete the utilized memory allocation. + // Decreases the size of the FIFO by one element. + void Read(void* destination, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + PRINTD("-"); + rtc::CritScope lock(&lock_); + if (fifo_->empty()) { + memset(destination, 0, bytes_per_buffer_); + } else { + int16_t* memory = fifo_->front(); + fifo_->pop_front(); + memcpy(destination, static_cast(&memory[0]), bytes_per_buffer_); + delete memory; + } + } + + size_t size() const { return fifo_->size(); } + + size_t largest_size() const { return largest_size_; } + + size_t average_size() const { + return (total_written_elements_ == 0) + ? 0.0 + : 0.5 + + static_cast(total_written_elements_) / + (write_count_ - kNumIgnoreFirstCallbacks); + } + + private: + void Flush() { + for (auto it = fifo_->begin(); it != fifo_->end(); ++it) { + delete *it; + } + fifo_->clear(); + } + + using AudioBufferList = std::list; + rtc::CriticalSection lock_; + const size_t frames_per_buffer_; + const size_t bytes_per_buffer_; + std::unique_ptr fifo_; + size_t largest_size_; + size_t total_written_elements_; + size_t write_count_; +}; + +// Inserts periodic impulses and measures the latency between the time of +// transmission and time of receiving the same impulse. +// Usage requires a special hardware called Audio Loopback Dongle. +// See http://source.android.com/devices/audio/loopback.html for details. +class LatencyMeasuringAudioStream : public AudioStreamInterface { + public: + explicit LatencyMeasuringAudioStream(size_t frames_per_buffer) + : frames_per_buffer_(frames_per_buffer), + bytes_per_buffer_(frames_per_buffer_ * sizeof(int16_t)), + play_count_(0), + rec_count_(0), + pulse_time_(0) {} + + // Insert periodic impulses in first two samples of |destination|. + void Read(void* destination, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + if (play_count_ == 0) { + PRINT("["); + } + play_count_++; + memset(destination, 0, bytes_per_buffer_); + if (play_count_ % (kNumCallbacksPerSecond / kImpulseFrequencyInHz) == 0) { + if (pulse_time_ == 0) { + pulse_time_ = rtc::TimeMillis(); + } + PRINT("."); + const int16_t impulse = std::numeric_limits::max(); + int16_t* ptr16 = static_cast(destination); + for (size_t i = 0; i < 2; ++i) { + ptr16[i] = impulse; + } + } + } + + // Detect received impulses in |source|, derive time between transmission and + // detection and add the calculated delay to list of latencies. + void Write(const void* source, size_t num_frames) override { + ASSERT_EQ(num_frames, frames_per_buffer_); + rec_count_++; + if (pulse_time_ == 0) { + // Avoid detection of new impulse response until a new impulse has + // been transmitted (sets |pulse_time_| to value larger than zero). + return; + } + const int16_t* ptr16 = static_cast(source); + std::vector vec(ptr16, ptr16 + num_frames); + // Find max value in the audio buffer. + int max = *std::max_element(vec.begin(), vec.end()); + // Find index (element position in vector) of the max element. + int index_of_max = + std::distance(vec.begin(), std::find(vec.begin(), vec.end(), max)); + if (max > kImpulseThreshold) { + PRINTD("(%d,%d)", max, index_of_max); + int64_t now_time = rtc::TimeMillis(); + int extra_delay = IndexToMilliseconds(static_cast(index_of_max)); + PRINTD("[%d]", static_cast(now_time - pulse_time_)); + PRINTD("[%d]", extra_delay); + // Total latency is the difference between transmit time and detection + // tome plus the extra delay within the buffer in which we detected the + // received impulse. It is transmitted at sample 0 but can be received + // at sample N where N > 0. The term |extra_delay| accounts for N and it + // is a value between 0 and 10ms. + latencies_.push_back(now_time - pulse_time_ + extra_delay); + pulse_time_ = 0; + } else { + PRINTD("-"); + } + } + + size_t num_latency_values() const { return latencies_.size(); } + + int min_latency() const { + if (latencies_.empty()) + return 0; + return *std::min_element(latencies_.begin(), latencies_.end()); + } + + int max_latency() const { + if (latencies_.empty()) + return 0; + return *std::max_element(latencies_.begin(), latencies_.end()); + } + + int average_latency() const { + if (latencies_.empty()) + return 0; + return 0.5 + + static_cast( + std::accumulate(latencies_.begin(), latencies_.end(), 0)) / + latencies_.size(); + } + + void PrintResults() const { + PRINT("] "); + for (auto it = latencies_.begin(); it != latencies_.end(); ++it) { + PRINT("%d ", *it); + } + PRINT("\n"); + PRINT("%s[min, max, avg]=[%d, %d, %d] ms\n", kTag, min_latency(), + max_latency(), average_latency()); + } + + int IndexToMilliseconds(double index) const { + return 10.0 * (index / frames_per_buffer_) + 0.5; + } + + private: + const size_t frames_per_buffer_; + const size_t bytes_per_buffer_; + size_t play_count_; + size_t rec_count_; + int64_t pulse_time_; + std::vector latencies_; +}; +// Mocks the AudioTransport object and proxies actions for the two callbacks +// (RecordedDataIsAvailable and NeedMorePlayData) to different implementations +// of AudioStreamInterface. +class MockAudioTransportIOS : public test::MockAudioTransport { + public: + explicit MockAudioTransportIOS(int type) + : num_callbacks_(0), + type_(type), + play_count_(0), + rec_count_(0), + audio_stream_(nullptr) {} + + virtual ~MockAudioTransportIOS() {} + + // Set default actions of the mock object. We are delegating to fake + // implementations (of AudioStreamInterface) here. + void HandleCallbacks(EventWrapper* test_is_done, + AudioStreamInterface* audio_stream, + size_t num_callbacks) { + test_is_done_ = test_is_done; + audio_stream_ = audio_stream; + num_callbacks_ = num_callbacks; + if (play_mode()) { + ON_CALL(*this, NeedMorePlayData(_, _, _, _, _, _, _, _)) + .WillByDefault( + Invoke(this, &MockAudioTransportIOS::RealNeedMorePlayData)); + } + if (rec_mode()) { + ON_CALL(*this, RecordedDataIsAvailable(_, _, _, _, _, _, _, _, _, _)) + .WillByDefault(Invoke( + this, &MockAudioTransportIOS::RealRecordedDataIsAvailable)); + } + } + + int32_t RealRecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + EXPECT_TRUE(rec_mode()) << "No test is expecting these callbacks."; + rec_count_++; + // Process the recorded audio stream if an AudioStreamInterface + // implementation exists. + if (audio_stream_) { + audio_stream_->Write(audioSamples, nSamples); + } + if (ReceivedEnoughCallbacks()) { + if (test_is_done_) { + test_is_done_->Set(); + } + } + return 0; + } + + int32_t RealNeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + EXPECT_TRUE(play_mode()) << "No test is expecting these callbacks."; + play_count_++; + nSamplesOut = nSamples; + // Read (possibly processed) audio stream samples to be played out if an + // AudioStreamInterface implementation exists. + if (audio_stream_) { + audio_stream_->Read(audioSamples, nSamples); + } else { + memset(audioSamples, 0, nSamples * nBytesPerSample); + } + if (ReceivedEnoughCallbacks()) { + if (test_is_done_) { + test_is_done_->Set(); + } + } + return 0; + } + + bool ReceivedEnoughCallbacks() { + bool recording_done = false; + if (rec_mode()) + recording_done = rec_count_ >= num_callbacks_; + else + recording_done = true; + + bool playout_done = false; + if (play_mode()) + playout_done = play_count_ >= num_callbacks_; + else + playout_done = true; + + return recording_done && playout_done; + } + + bool play_mode() const { return type_ & kPlayout; } + bool rec_mode() const { return type_ & kRecording; } + + private: + EventWrapper* test_is_done_; + size_t num_callbacks_; + int type_; + size_t play_count_; + size_t rec_count_; + AudioStreamInterface* audio_stream_; +}; + +// AudioDeviceTest test fixture. +class AudioDeviceTest : public ::testing::Test { + protected: + AudioDeviceTest() : test_is_done_(EventWrapper::Create()) { + old_sev_ = rtc::LogMessage::GetLogToDebug(); + // Set suitable logging level here. Change to rtc::LS_INFO for more verbose + // output. See webrtc/rtc_base/logging.h for complete list of options. + rtc::LogMessage::LogToDebug(rtc::LS_INFO); + // Add extra logging fields here (timestamps and thread id). + // rtc::LogMessage::LogTimestamps(); + rtc::LogMessage::LogThreads(); + // Creates an audio device using a default audio layer. + audio_device_ = CreateAudioDevice(AudioDeviceModule::kPlatformDefaultAudio); + EXPECT_NE(audio_device_.get(), nullptr); + EXPECT_EQ(0, audio_device_->Init()); + EXPECT_EQ(0, + audio_device()->GetPlayoutAudioParameters(&playout_parameters_)); + EXPECT_EQ(0, audio_device()->GetRecordAudioParameters(&record_parameters_)); + } + virtual ~AudioDeviceTest() { + EXPECT_EQ(0, audio_device_->Terminate()); + rtc::LogMessage::LogToDebug(old_sev_); + } + + int playout_sample_rate() const { return playout_parameters_.sample_rate(); } + int record_sample_rate() const { return record_parameters_.sample_rate(); } + int playout_channels() const { return playout_parameters_.channels(); } + int record_channels() const { return record_parameters_.channels(); } + size_t playout_frames_per_10ms_buffer() const { + return playout_parameters_.frames_per_10ms_buffer(); + } + size_t record_frames_per_10ms_buffer() const { + return record_parameters_.frames_per_10ms_buffer(); + } + + rtc::scoped_refptr audio_device() const { + return audio_device_; + } + + AudioDeviceModuleImpl* audio_device_impl() const { + return static_cast(audio_device_.get()); + } + + AudioDeviceBuffer* audio_device_buffer() const { + return audio_device_impl()->GetAudioDeviceBuffer(); + } + + rtc::scoped_refptr CreateAudioDevice( + AudioDeviceModule::AudioLayer audio_layer) { + rtc::scoped_refptr module( + AudioDeviceModule::Create(0, audio_layer)); + return module; + } + + // Returns file name relative to the resource root given a sample rate. + std::string GetFileName(int sample_rate) { + EXPECT_TRUE(sample_rate == 48000 || sample_rate == 44100 || + sample_rate == 16000); + char fname[64]; + snprintf(fname, sizeof(fname), "audio_device/audio_short%d", + sample_rate / 1000); + std::string file_name(webrtc::test::ResourcePath(fname, "pcm")); + EXPECT_TRUE(test::FileExists(file_name)); +#ifdef ENABLE_DEBUG_PRINTF + PRINTD("file name: %s\n", file_name.c_str()); + const size_t bytes = test::GetFileSize(file_name); + PRINTD("file size: %" PRIuS " [bytes]\n", bytes); + PRINTD("file size: %" PRIuS " [samples]\n", bytes / kBytesPerSample); + const int seconds = + static_cast(bytes / (sample_rate * kBytesPerSample)); + PRINTD("file size: %d [secs]\n", seconds); + PRINTD("file size: %" PRIuS " [callbacks]\n", + seconds * kNumCallbacksPerSecond); +#endif + return file_name; + } + + void StartPlayout() { + EXPECT_FALSE(audio_device()->Playing()); + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_TRUE(audio_device()->PlayoutIsInitialized()); + EXPECT_EQ(0, audio_device()->StartPlayout()); + EXPECT_TRUE(audio_device()->Playing()); + } + + void StopPlayout() { + EXPECT_EQ(0, audio_device()->StopPlayout()); + EXPECT_FALSE(audio_device()->Playing()); + } + + void StartRecording() { + EXPECT_FALSE(audio_device()->Recording()); + EXPECT_EQ(0, audio_device()->InitRecording()); + EXPECT_TRUE(audio_device()->RecordingIsInitialized()); + EXPECT_EQ(0, audio_device()->StartRecording()); + EXPECT_TRUE(audio_device()->Recording()); + } + + void StopRecording() { + EXPECT_EQ(0, audio_device()->StopRecording()); + EXPECT_FALSE(audio_device()->Recording()); + } + + std::unique_ptr test_is_done_; + rtc::scoped_refptr audio_device_; + AudioParameters playout_parameters_; + AudioParameters record_parameters_; + rtc::LoggingSeverity old_sev_; +}; + +TEST_F(AudioDeviceTest, ConstructDestruct) { + // Using the test fixture to create and destruct the audio device module. +} + +TEST_F(AudioDeviceTest, InitTerminate) { + // Initialization is part of the test fixture. + EXPECT_TRUE(audio_device()->Initialized()); + EXPECT_EQ(0, audio_device()->Terminate()); + EXPECT_FALSE(audio_device()->Initialized()); +} + +// Tests that playout can be initiated, started and stopped. No audio callback +// is registered in this test. +// Failing when running on real iOS devices: bugs.webrtc.org/6889. +TEST_F(AudioDeviceTest, DISABLED_StartStopPlayout) { + StartPlayout(); + StopPlayout(); + StartPlayout(); + StopPlayout(); +} + +// Tests that recording can be initiated, started and stopped. No audio callback +// is registered in this test. +// Can sometimes fail when running on real devices: bugs.webrtc.org/7888. +TEST_F(AudioDeviceTest, DISABLED_StartStopRecording) { + StartRecording(); + StopRecording(); + StartRecording(); + StopRecording(); +} + +// Verify that calling StopPlayout() will leave us in an uninitialized state +// which will require a new call to InitPlayout(). This test does not call +// StartPlayout() while being uninitialized since doing so will hit a +// RTC_DCHECK. +TEST_F(AudioDeviceTest, StopPlayoutRequiresInitToRestart) { + EXPECT_EQ(0, audio_device()->InitPlayout()); + EXPECT_EQ(0, audio_device()->StartPlayout()); + EXPECT_EQ(0, audio_device()->StopPlayout()); + EXPECT_FALSE(audio_device()->PlayoutIsInitialized()); +} + +// Verify that we can create two ADMs and start playing on the second ADM. +// Only the first active instance shall activate an audio session and the +// last active instance shall deactivate the audio session. The test does not +// explicitly verify correct audio session calls but instead focuses on +// ensuring that audio starts for both ADMs. + +// Failing when running on real iOS devices: bugs.webrtc.org/6889. +TEST_F(AudioDeviceTest, DISABLED_StartPlayoutOnTwoInstances) { + // Create and initialize a second/extra ADM instance. The default ADM is + // created by the test harness. + rtc::scoped_refptr second_audio_device = + CreateAudioDevice(AudioDeviceModule::kPlatformDefaultAudio); + EXPECT_NE(second_audio_device.get(), nullptr); + EXPECT_EQ(0, second_audio_device->Init()); + + // Start playout for the default ADM but don't wait here. Instead use the + // upcoming second stream for that. We set the same expectation on number + // of callbacks as for the second stream. + NiceMock mock(kPlayout); + mock.HandleCallbacks(nullptr, nullptr, 0); + EXPECT_CALL( + mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), kBytesPerSample, + playout_channels(), playout_sample_rate(), + NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + + // Initialize playout for the second ADM. If all is OK, the second ADM shall + // reuse the audio session activated when the first ADM started playing. + // This call will also ensure that we avoid a problem related to initializing + // two different audio unit instances back to back (see webrtc:5166 for + // details). + EXPECT_EQ(0, second_audio_device->InitPlayout()); + EXPECT_TRUE(second_audio_device->PlayoutIsInitialized()); + + // Start playout for the second ADM and verify that it starts as intended. + // Passing this test ensures that initialization of the second audio unit + // has been done successfully and that there is no conflict with the already + // playing first ADM. + MockAudioTransportIOS mock2(kPlayout); + mock2.HandleCallbacks(test_is_done_.get(), nullptr, kNumCallbacks); + EXPECT_CALL( + mock2, NeedMorePlayData(playout_frames_per_10ms_buffer(), kBytesPerSample, + playout_channels(), playout_sample_rate(), + NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, second_audio_device->RegisterAudioCallback(&mock2)); + EXPECT_EQ(0, second_audio_device->StartPlayout()); + EXPECT_TRUE(second_audio_device->Playing()); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + EXPECT_EQ(0, second_audio_device->StopPlayout()); + EXPECT_FALSE(second_audio_device->Playing()); + EXPECT_FALSE(second_audio_device->PlayoutIsInitialized()); + + EXPECT_EQ(0, second_audio_device->Terminate()); +} + +// Start playout and verify that the native audio layer starts asking for real +// audio samples to play out using the NeedMorePlayData callback. +TEST_F(AudioDeviceTest, StartPlayoutVerifyCallbacks) { + MockAudioTransportIOS mock(kPlayout); + mock.HandleCallbacks(test_is_done_.get(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), + kBytesPerSample, playout_channels(), + playout_sample_rate(), NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopPlayout(); +} + +// Start recording and verify that the native audio layer starts feeding real +// audio samples via the RecordedDataIsAvailable callback. +TEST_F(AudioDeviceTest, StartRecordingVerifyCallbacks) { + MockAudioTransportIOS mock(kRecording); + mock.HandleCallbacks(test_is_done_.get(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, + RecordedDataIsAvailable( + NotNull(), record_frames_per_10ms_buffer(), kBytesPerSample, + record_channels(), record_sample_rate(), + _, // TODO(henrika): fix delay + 0, 0, false, _)).Times(AtLeast(kNumCallbacks)); + + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartRecording(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopRecording(); +} + +// Start playout and recording (full-duplex audio) and verify that audio is +// active in both directions. +TEST_F(AudioDeviceTest, StartPlayoutAndRecordingVerifyCallbacks) { + MockAudioTransportIOS mock(kPlayout | kRecording); + mock.HandleCallbacks(test_is_done_.get(), nullptr, kNumCallbacks); + EXPECT_CALL(mock, NeedMorePlayData(playout_frames_per_10ms_buffer(), + kBytesPerSample, playout_channels(), + playout_sample_rate(), NotNull(), _, _, _)) + .Times(AtLeast(kNumCallbacks)); + EXPECT_CALL(mock, + RecordedDataIsAvailable( + NotNull(), record_frames_per_10ms_buffer(), kBytesPerSample, + record_channels(), record_sample_rate(), + _, // TODO(henrika): fix delay + 0, 0, false, _)).Times(AtLeast(kNumCallbacks)); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + StartRecording(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopRecording(); + StopPlayout(); +} + +// Start playout and read audio from an external PCM file when the audio layer +// asks for data to play out. Real audio is played out in this test but it does +// not contain any explicit verification that the audio quality is perfect. +TEST_F(AudioDeviceTest, RunPlayoutWithFileAsSource) { + // TODO(henrika): extend test when mono output is supported. + EXPECT_EQ(1, playout_channels()); + NiceMock mock(kPlayout); + const int num_callbacks = kFilePlayTimeInSec * kNumCallbacksPerSecond; + std::string file_name = GetFileName(playout_sample_rate()); + std::unique_ptr file_audio_stream( + new FileAudioStream(num_callbacks, file_name, playout_sample_rate())); + mock.HandleCallbacks(test_is_done_.get(), file_audio_stream.get(), + num_callbacks); + // SetMaxPlayoutVolume(); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartPlayout(); + test_is_done_->Wait(kTestTimeOutInMilliseconds); + StopPlayout(); +} + +TEST_F(AudioDeviceTest, Devices) { + // Device enumeration is not supported. Verify fixed values only. + EXPECT_EQ(1, audio_device()->PlayoutDevices()); + EXPECT_EQ(1, audio_device()->RecordingDevices()); +} + +// Start playout and recording and store recorded data in an intermediate FIFO +// buffer from which the playout side then reads its samples in the same order +// as they were stored. Under ideal circumstances, a callback sequence would +// look like: ...+-+-+-+-+-+-+-..., where '+' means 'packet recorded' and '-' +// means 'packet played'. Under such conditions, the FIFO would only contain +// one packet on average. However, under more realistic conditions, the size +// of the FIFO will vary more due to an unbalance between the two sides. +// This test tries to verify that the device maintains a balanced callback- +// sequence by running in loopback for ten seconds while measuring the size +// (max and average) of the FIFO. The size of the FIFO is increased by the +// recording side and decreased by the playout side. +// TODO(henrika): tune the final test parameters after running tests on several +// different devices. +TEST_F(AudioDeviceTest, RunPlayoutAndRecordingInFullDuplex) { + EXPECT_EQ(record_channels(), playout_channels()); + EXPECT_EQ(record_sample_rate(), playout_sample_rate()); + NiceMock mock(kPlayout | kRecording); + std::unique_ptr fifo_audio_stream( + new FifoAudioStream(playout_frames_per_10ms_buffer())); + mock.HandleCallbacks(test_is_done_.get(), fifo_audio_stream.get(), + kFullDuplexTimeInSec * kNumCallbacksPerSecond); + // SetMaxPlayoutVolume(); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + StartRecording(); + StartPlayout(); + test_is_done_->Wait( + std::max(kTestTimeOutInMilliseconds, 1000 * kFullDuplexTimeInSec)); + StopPlayout(); + StopRecording(); + EXPECT_LE(fifo_audio_stream->average_size(), 10u); + EXPECT_LE(fifo_audio_stream->largest_size(), 20u); +} + +// Measures loopback latency and reports the min, max and average values for +// a full duplex audio session. +// The latency is measured like so: +// - Insert impulses periodically on the output side. +// - Detect the impulses on the input side. +// - Measure the time difference between the transmit time and receive time. +// - Store time differences in a vector and calculate min, max and average. +// This test requires a special hardware called Audio Loopback Dongle. +// See http://source.android.com/devices/audio/loopback.html for details. +TEST_F(AudioDeviceTest, DISABLED_MeasureLoopbackLatency) { + EXPECT_EQ(record_channels(), playout_channels()); + EXPECT_EQ(record_sample_rate(), playout_sample_rate()); + NiceMock mock(kPlayout | kRecording); + std::unique_ptr latency_audio_stream( + new LatencyMeasuringAudioStream(playout_frames_per_10ms_buffer())); + mock.HandleCallbacks(test_is_done_.get(), latency_audio_stream.get(), + kMeasureLatencyTimeInSec * kNumCallbacksPerSecond); + EXPECT_EQ(0, audio_device()->RegisterAudioCallback(&mock)); + // SetMaxPlayoutVolume(); + // DisableBuiltInAECIfAvailable(); + StartRecording(); + StartPlayout(); + test_is_done_->Wait( + std::max(kTestTimeOutInMilliseconds, 1000 * kMeasureLatencyTimeInSec)); + StopPlayout(); + StopRecording(); + // Verify that the correct number of transmitted impulses are detected. + EXPECT_EQ(latency_audio_stream->num_latency_values(), + static_cast( + kImpulseFrequencyInHz * kMeasureLatencyTimeInSec - 1)); + latency_audio_stream->PrintResults(); +} + +// Verifies that the AudioDeviceIOS is_interrupted_ flag is reset correctly +// after an iOS AVAudioSessionInterruptionTypeEnded notification event. +// AudioDeviceIOS listens to RTCAudioSession interrupted notifications by: +// - In AudioDeviceIOS.InitPlayOrRecord registers its audio_session_observer_ +// callback with RTCAudioSession's delegate list. +// - When RTCAudioSession receives an iOS audio interrupted notification, it +// passes the notification to callbacks in its delegate list which sets +// AudioDeviceIOS's is_interrupted_ flag to true. +// - When AudioDeviceIOS.ShutdownPlayOrRecord is called, its +// audio_session_observer_ callback is removed from RTCAudioSessions's +// delegate list. +// So if RTCAudioSession receives an iOS end audio interruption notification, +// AudioDeviceIOS is not notified as its callback is not in RTCAudioSession's +// delegate list. This causes AudioDeviceIOS's is_interrupted_ flag to be in +// the wrong (true) state and the audio session will ignore audio changes. +// As RTCAudioSession keeps its own interrupted state, the fix is to initialize +// AudioDeviceIOS's is_interrupted_ flag to RTCAudioSession's isInterrupted +// flag in AudioDeviceIOS.InitPlayOrRecord. +TEST_F(AudioDeviceTest, testInterruptedAudioSession) { + RTCAudioSession *session = [RTCAudioSession sharedInstance]; + std::unique_ptr audio_device; + audio_device.reset(new webrtc::AudioDeviceIOS()); + std::unique_ptr audio_buffer; + audio_buffer.reset(new webrtc::AudioDeviceBuffer()); + audio_device->AttachAudioBuffer(audio_buffer.get()); + audio_device->Init(); + audio_device->InitPlayout(); + // Force interruption. + [session notifyDidBeginInterruption]; + + // Wait for notification to propagate. + rtc::MessageQueueManager::ProcessAllMessageQueues(); + EXPECT_TRUE(audio_device->is_interrupted_); + + // Force it for testing. + audio_device->playing_ = false; + audio_device->ShutdownPlayOrRecord(); + // Force it for testing. + audio_device->audio_is_initialized_ = false; + + [session notifyDidEndInterruptionWithShouldResumeSession:YES]; + // Wait for notification to propagate. + rtc::MessageQueueManager::ProcessAllMessageQueues(); + EXPECT_TRUE(audio_device->is_interrupted_); + + audio_device->Init(); + audio_device->InitPlayout(); + EXPECT_FALSE(audio_device->is_interrupted_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_session_observer.h b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_session_observer.h new file mode 100644 index 0000000000..13a979bd4e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/audio_session_observer.h @@ -0,0 +1,42 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_IOS_AUDIO_SESSION_OBSERVER_H_ +#define MODULES_AUDIO_DEVICE_IOS_AUDIO_SESSION_OBSERVER_H_ + +#include "rtc_base/asyncinvoker.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +// Observer interface for listening to AVAudioSession events. +class AudioSessionObserver { + public: + // Called when audio session interruption begins. + virtual void OnInterruptionBegin() = 0; + + // Called when audio session interruption ends. + virtual void OnInterruptionEnd() = 0; + + // Called when audio route changes. + virtual void OnValidRouteChange() = 0; + + // Called when the ability to play or record changes. + virtual void OnCanPlayOrRecordChange(bool can_play_or_record) = 0; + + virtual void OnChangedOutputVolume() = 0; + + protected: + virtual ~AudioSessionObserver() {} +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_IOS_AUDIO_SESSION_OBSERVER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSession.h b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSession.h new file mode 100644 index 0000000000..006fb81cb6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSession.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "sdk/objc/Framework/Headers/WebRTC/RTCAudioSession.h" diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h new file mode 100644 index 0000000000..b1e41d1e11 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionConfiguration.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "sdk/objc/Framework/Headers/WebRTC/RTCAudioSessionConfiguration.h" diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h new file mode 100644 index 0000000000..7a0f9d2594 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h @@ -0,0 +1,30 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "sdk/objc/Framework/Headers/WebRTC/RTCAudioSession.h" + +namespace webrtc { +class AudioSessionObserver; +} + +/** Adapter that forwards RTCAudioSessionDelegate calls to the appropriate + * methods on the AudioSessionObserver. + */ +@interface RTCAudioSessionDelegateAdapter : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +/** |observer| is a raw pointer and should be kept alive + * for this object's lifetime. + */ +- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer + NS_DESIGNATED_INITIALIZER; + +@end diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm new file mode 100644 index 0000000000..6bb3df2e67 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.mm @@ -0,0 +1,89 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h" + +#include "modules/audio_device/ios/audio_session_observer.h" + +#import "WebRTC/RTCLogging.h" + +@implementation RTCAudioSessionDelegateAdapter { + webrtc::AudioSessionObserver *_observer; +} + +- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer { + NSParameterAssert(observer); + if (self = [super init]) { + _observer = observer; + } + return self; +} + +#pragma mark - RTCAudioSessionDelegate + +- (void)audioSessionDidBeginInterruption:(RTCAudioSession *)session { + _observer->OnInterruptionBegin(); +} + +- (void)audioSessionDidEndInterruption:(RTCAudioSession *)session + shouldResumeSession:(BOOL)shouldResumeSession { + _observer->OnInterruptionEnd(); +} + +- (void)audioSessionDidChangeRoute:(RTCAudioSession *)session + reason:(AVAudioSessionRouteChangeReason)reason + previousRoute:(AVAudioSessionRouteDescription *)previousRoute { + switch (reason) { + case AVAudioSessionRouteChangeReasonUnknown: + case AVAudioSessionRouteChangeReasonNewDeviceAvailable: + case AVAudioSessionRouteChangeReasonOldDeviceUnavailable: + case AVAudioSessionRouteChangeReasonCategoryChange: + // It turns out that we see a category change (at least in iOS 9.2) + // when making a switch from a BT device to e.g. Speaker using the + // iOS Control Center and that we therefore must check if the sample + // rate has changed. And if so is the case, restart the audio unit. + case AVAudioSessionRouteChangeReasonOverride: + case AVAudioSessionRouteChangeReasonWakeFromSleep: + case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory: + _observer->OnValidRouteChange(); + break; + case AVAudioSessionRouteChangeReasonRouteConfigurationChange: + // The set of input and output ports has not changed, but their + // configuration has, e.g., a port’s selected data source has + // changed. Ignore this type of route change since we are focusing + // on detecting headset changes. + RTCLog(@"Ignoring RouteConfigurationChange"); + break; + } +} + +- (void)audioSessionMediaServerTerminated:(RTCAudioSession *)session { +} + +- (void)audioSessionMediaServerReset:(RTCAudioSession *)session { +} + +- (void)audioSession:(RTCAudioSession *)session + didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord { + _observer->OnCanPlayOrRecordChange(canPlayOrRecord); +} + +- (void)audioSessionDidStartPlayOrRecord:(RTCAudioSession *)session { +} + +- (void)audioSessionDidStopPlayOrRecord:(RTCAudioSession *)session { +} + +- (void)audioSession:(RTCAudioSession *)audioSession + didChangeOutputVolume:(float)outputVolume { + _observer->OnChangedOutputVolume(); +} + +@end diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.h b/third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.h new file mode 100644 index 0000000000..3105c129bd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.h @@ -0,0 +1,137 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_IOS_VOICE_PROCESSING_AUDIO_UNIT_H_ +#define MODULES_AUDIO_DEVICE_IOS_VOICE_PROCESSING_AUDIO_UNIT_H_ + +#include + +namespace webrtc { + +class VoiceProcessingAudioUnitObserver { + public: + // Callback function called on a real-time priority I/O thread from the audio + // unit. This method is used to signal that recorded audio is available. + virtual OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) = 0; + + // Callback function called on a real-time priority I/O thread from the audio + // unit. This method is used to provide audio samples to the audio unit. + virtual OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* io_action_flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) = 0; + + protected: + ~VoiceProcessingAudioUnitObserver() {} +}; + +// Convenience class to abstract away the management of a Voice Processing +// I/O Audio Unit. The Voice Processing I/O unit has the same characteristics +// as the Remote I/O unit (supports full duplex low-latency audio input and +// output) and adds AEC for for two-way duplex communication. It also adds AGC, +// adjustment of voice-processing quality, and muting. Hence, ideal for +// VoIP applications. +class VoiceProcessingAudioUnit { + public: + explicit VoiceProcessingAudioUnit(VoiceProcessingAudioUnitObserver* observer); + ~VoiceProcessingAudioUnit(); + + // TODO(tkchin): enum for state and state checking. + enum State : int32_t { + // Init() should be called. + kInitRequired, + // Audio unit created but not initialized. + kUninitialized, + // Initialized but not started. Equivalent to stopped. + kInitialized, + // Initialized and started. + kStarted, + }; + + // Number of bytes per audio sample for 16-bit signed integer representation. + static const UInt32 kBytesPerSample; + + // Initializes this class by creating the underlying audio unit instance. + // Creates a Voice-Processing I/O unit and configures it for full-duplex + // audio. The selected stream format is selected to avoid internal resampling + // and to match the 10ms callback rate for WebRTC as well as possible. + // Does not intialize the audio unit. + bool Init(); + + VoiceProcessingAudioUnit::State GetState() const; + + // Initializes the underlying audio unit with the given sample rate. + bool Initialize(Float64 sample_rate); + + // Starts the underlying audio unit. + bool Start(); + + // Stops the underlying audio unit. + bool Stop(); + + // Uninitializes the underlying audio unit. + bool Uninitialize(); + + // Calls render on the underlying audio unit. + OSStatus Render(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 output_bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + private: + // The C API used to set callbacks requires static functions. When these are + // called, they will invoke the relevant instance method by casting + // in_ref_con to VoiceProcessingAudioUnit*. + static OSStatus OnGetPlayoutData(void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + static OSStatus OnDeliverRecordedData(void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + // Notifies observer that samples are needed for playback. + OSStatus NotifyGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + // Notifies observer that recorded samples are available for render. + OSStatus NotifyDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + // Returns the predetermined format with a specific sample rate. See + // implementation file for details on format. + AudioStreamBasicDescription GetFormat(Float64 sample_rate) const; + + // Deletes the underlying audio unit. + void DisposeAudioUnit(); + + VoiceProcessingAudioUnitObserver* observer_; + AudioUnit vpio_unit_; + VoiceProcessingAudioUnit::State state_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_IOS_VOICE_PROCESSING_AUDIO_UNIT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.mm b/third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.mm new file mode 100644 index 0000000000..c350969335 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/ios/voice_processing_audio_unit.mm @@ -0,0 +1,467 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "modules/audio_device/ios/voice_processing_audio_unit.h" + +#include "rtc_base/checks.h" +#include "system_wrappers/include/metrics.h" + +#import "WebRTC/RTCLogging.h" +#import "sdk/objc/Framework/Headers/WebRTC/RTCAudioSessionConfiguration.h" + +#if !defined(NDEBUG) +static void LogStreamDescription(AudioStreamBasicDescription description) { + char formatIdString[5]; + UInt32 formatId = CFSwapInt32HostToBig(description.mFormatID); + bcopy(&formatId, formatIdString, 4); + formatIdString[4] = '\0'; + RTCLog(@"AudioStreamBasicDescription: {\n" + " mSampleRate: %.2f\n" + " formatIDString: %s\n" + " mFormatFlags: 0x%X\n" + " mBytesPerPacket: %u\n" + " mFramesPerPacket: %u\n" + " mBytesPerFrame: %u\n" + " mChannelsPerFrame: %u\n" + " mBitsPerChannel: %u\n" + " mReserved: %u\n}", + description.mSampleRate, formatIdString, + static_cast(description.mFormatFlags), + static_cast(description.mBytesPerPacket), + static_cast(description.mFramesPerPacket), + static_cast(description.mBytesPerFrame), + static_cast(description.mChannelsPerFrame), + static_cast(description.mBitsPerChannel), + static_cast(description.mReserved)); +} +#endif + +namespace webrtc { + +// Calls to AudioUnitInitialize() can fail if called back-to-back on different +// ADM instances. A fall-back solution is to allow multiple sequential calls +// with as small delay between each. This factor sets the max number of allowed +// initialization attempts. +static const int kMaxNumberOfAudioUnitInitializeAttempts = 5; +// A VP I/O unit's bus 1 connects to input hardware (microphone). +static const AudioUnitElement kInputBus = 1; +// A VP I/O unit's bus 0 connects to output hardware (speaker). +static const AudioUnitElement kOutputBus = 0; + +// Returns the automatic gain control (AGC) state on the processed microphone +// signal. Should be on by default for Voice Processing audio units. +static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { + RTC_DCHECK(audio_unit); + UInt32 size = sizeof(*enabled); + OSStatus result = AudioUnitGetProperty(audio_unit, + kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, + kInputBus, + enabled, + &size); + RTCLog(@"VPIO unit AGC: %u", static_cast(*enabled)); + return result; +} + +VoiceProcessingAudioUnit::VoiceProcessingAudioUnit( + VoiceProcessingAudioUnitObserver* observer) + : observer_(observer), vpio_unit_(nullptr), state_(kInitRequired) { + RTC_DCHECK(observer); +} + +VoiceProcessingAudioUnit::~VoiceProcessingAudioUnit() { + DisposeAudioUnit(); +} + +const UInt32 VoiceProcessingAudioUnit::kBytesPerSample = 2; + +bool VoiceProcessingAudioUnit::Init() { + RTC_DCHECK_EQ(state_, kInitRequired); + + // Create an audio component description to identify the Voice Processing + // I/O audio unit. + AudioComponentDescription vpio_unit_description; + vpio_unit_description.componentType = kAudioUnitType_Output; + vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO; + vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple; + vpio_unit_description.componentFlags = 0; + vpio_unit_description.componentFlagsMask = 0; + + // Obtain an audio unit instance given the description. + AudioComponent found_vpio_unit_ref = + AudioComponentFindNext(nullptr, &vpio_unit_description); + + // Create a Voice Processing IO audio unit. + OSStatus result = noErr; + result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_); + if (result != noErr) { + vpio_unit_ = nullptr; + RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result); + return false; + } + + // Enable input on the input scope of the input element. + UInt32 enable_input = 1; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &enable_input, + sizeof(enable_input)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable input on input scope of input element. " + "Error=%ld.", + (long)result); + return false; + } + + // Enable output on the output scope of the output element. + UInt32 enable_output = 1; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Output, kOutputBus, + &enable_output, sizeof(enable_output)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable output on output scope of output element. " + "Error=%ld.", + (long)result); + return false; + } + + // Specify the callback function that provides audio samples to the audio + // unit. + AURenderCallbackStruct render_callback; + render_callback.inputProc = OnGetPlayoutData; + render_callback.inputProcRefCon = this; + result = AudioUnitSetProperty( + vpio_unit_, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, + kOutputBus, &render_callback, sizeof(render_callback)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to specify the render callback on the output bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Disable AU buffer allocation for the recorder, we allocate our own. + // TODO(henrika): not sure that it actually saves resource to make this call. + UInt32 flag = 0; + result = AudioUnitSetProperty( + vpio_unit_, kAudioUnitProperty_ShouldAllocateBuffer, + kAudioUnitScope_Output, kInputBus, &flag, sizeof(flag)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to disable buffer allocation on the input bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Specify the callback to be called by the I/O thread to us when input audio + // is available. The recorded samples can then be obtained by calling the + // AudioUnitRender() method. + AURenderCallbackStruct input_callback; + input_callback.inputProc = OnDeliverRecordedData; + input_callback.inputProcRefCon = this; + result = AudioUnitSetProperty(vpio_unit_, + kAudioOutputUnitProperty_SetInputCallback, + kAudioUnitScope_Global, kInputBus, + &input_callback, sizeof(input_callback)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to specify the input callback on the input bus. " + "Error=%ld.", + (long)result); + return false; + } + + state_ = kUninitialized; + return true; +} + +VoiceProcessingAudioUnit::State VoiceProcessingAudioUnit::GetState() const { + return state_; +} + +bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate); + + OSStatus result = noErr; + AudioStreamBasicDescription format = GetFormat(sample_rate); + UInt32 size = sizeof(format); +#if !defined(NDEBUG) + LogStreamDescription(format); +#endif + + // Set the format on the output scope of the input element/bus. + result = + AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, kInputBus, &format, size); + if (result != noErr) { + RTCLogError(@"Failed to set format on output scope of input bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Set the format on the input scope of the output element/bus. + result = + AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, kOutputBus, &format, size); + if (result != noErr) { + RTCLogError(@"Failed to set format on input scope of output bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Initialize the Voice Processing I/O unit instance. + // Calls to AudioUnitInitialize() can fail if called back-to-back on + // different ADM instances. The error message in this case is -66635 which is + // undocumented. Tests have shown that calling AudioUnitInitialize a second + // time, after a short sleep, avoids this issue. + // See webrtc:5166 for details. + int failed_initalize_attempts = 0; + result = AudioUnitInitialize(vpio_unit_); + while (result != noErr) { + RTCLogError(@"Failed to initialize the Voice Processing I/O unit. " + "Error=%ld.", + (long)result); + ++failed_initalize_attempts; + if (failed_initalize_attempts == kMaxNumberOfAudioUnitInitializeAttempts) { + // Max number of initialization attempts exceeded, hence abort. + RTCLogError(@"Too many initialization attempts."); + return false; + } + RTCLog(@"Pause 100ms and try audio unit initialization again..."); + [NSThread sleepForTimeInterval:0.1f]; + result = AudioUnitInitialize(vpio_unit_); + } + if (result == noErr) { + RTCLog(@"Voice Processing I/O unit is now initialized."); + } + + // AGC should be enabled by default for Voice Processing I/O units but it is + // checked below and enabled explicitly if needed. This scheme is used + // to be absolutely sure that the AGC is enabled since we have seen cases + // where only zeros are recorded and a disabled AGC could be one of the + // reasons why it happens. + int agc_was_enabled_by_default = 0; + UInt32 agc_is_enabled = 0; + result = GetAGCState(vpio_unit_, &agc_is_enabled); + if (result != noErr) { + RTCLogError(@"Failed to get AGC state (1st attempt). " + "Error=%ld.", + (long)result); + // Example of error code: kAudioUnitErr_NoConnection (-10876). + // All error codes related to audio units are negative and are therefore + // converted into a postive value to match the UMA APIs. + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.GetAGCStateErrorCode1", (-1) * result); + } else if (agc_is_enabled) { + // Remember that the AGC was enabled by default. Will be used in UMA. + agc_was_enabled_by_default = 1; + } else { + // AGC was initially disabled => try to enable it explicitly. + UInt32 enable_agc = 1; + result = + AudioUnitSetProperty(vpio_unit_, + kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, kInputBus, &enable_agc, + sizeof(enable_agc)); + if (result != noErr) { + RTCLogError(@"Failed to enable the built-in AGC. " + "Error=%ld.", + (long)result); + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.SetAGCStateErrorCode", (-1) * result); + } + result = GetAGCState(vpio_unit_, &agc_is_enabled); + if (result != noErr) { + RTCLogError(@"Failed to get AGC state (2nd attempt). " + "Error=%ld.", + (long)result); + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.GetAGCStateErrorCode2", (-1) * result); + } + } + + // Track if the built-in AGC was enabled by default (as it should) or not. + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCWasEnabledByDefault", + agc_was_enabled_by_default); + RTCLog(@"WebRTC.Audio.BuiltInAGCWasEnabledByDefault: %d", + agc_was_enabled_by_default); + // As a final step, add an UMA histogram for tracking the AGC state. + // At this stage, the AGC should be enabled, and if it is not, more work is + // needed to find out the root cause. + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCIsEnabled", agc_is_enabled); + RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u", + static_cast(agc_is_enabled)); + + state_ = kInitialized; + return true; +} + +bool VoiceProcessingAudioUnit::Start() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Starting audio unit."); + + OSStatus result = AudioOutputUnitStart(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to start audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Started audio unit"); + } + state_ = kStarted; + return true; +} + +bool VoiceProcessingAudioUnit::Stop() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Stopping audio unit."); + + OSStatus result = AudioOutputUnitStop(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to stop audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Stopped audio unit"); + } + + state_ = kInitialized; + return true; +} + +bool VoiceProcessingAudioUnit::Uninitialize() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Unintializing audio unit."); + + OSStatus result = AudioUnitUninitialize(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to uninitialize audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Uninitialized audio unit."); + } + + state_ = kUninitialized; + return true; +} + +OSStatus VoiceProcessingAudioUnit::Render(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 output_bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + RTC_DCHECK(vpio_unit_) << "Init() not called."; + + OSStatus result = AudioUnitRender(vpio_unit_, flags, time_stamp, + output_bus_number, num_frames, io_data); + if (result != noErr) { + RTCLogError(@"Failed to render audio unit. Error=%ld", (long)result); + } + return result; +} + +OSStatus VoiceProcessingAudioUnit::OnGetPlayoutData( + void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + VoiceProcessingAudioUnit* audio_unit = + static_cast(in_ref_con); + return audio_unit->NotifyGetPlayoutData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +OSStatus VoiceProcessingAudioUnit::OnDeliverRecordedData( + void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + VoiceProcessingAudioUnit* audio_unit = + static_cast(in_ref_con); + return audio_unit->NotifyDeliverRecordedData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +OSStatus VoiceProcessingAudioUnit::NotifyGetPlayoutData( + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + return observer_->OnGetPlayoutData(flags, time_stamp, bus_number, num_frames, + io_data); +} + +OSStatus VoiceProcessingAudioUnit::NotifyDeliverRecordedData( + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + return observer_->OnDeliverRecordedData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +AudioStreamBasicDescription VoiceProcessingAudioUnit::GetFormat( + Float64 sample_rate) const { + // Set the application formats for input and output: + // - use same format in both directions + // - avoid resampling in the I/O unit by using the hardware sample rate + // - linear PCM => noncompressed audio data format with one frame per packet + // - no need to specify interleaving since only mono is supported + AudioStreamBasicDescription format; + RTC_DCHECK_EQ(1, kRTCAudioSessionPreferredNumberOfChannels); + format.mSampleRate = sample_rate; + format.mFormatID = kAudioFormatLinearPCM; + format.mFormatFlags = + kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + format.mBytesPerPacket = kBytesPerSample; + format.mFramesPerPacket = 1; // uncompressed. + format.mBytesPerFrame = kBytesPerSample; + format.mChannelsPerFrame = kRTCAudioSessionPreferredNumberOfChannels; + format.mBitsPerChannel = 8 * kBytesPerSample; + return format; +} + +void VoiceProcessingAudioUnit::DisposeAudioUnit() { + if (vpio_unit_) { + switch (state_) { + case kStarted: + Stop(); + // Fall through. + FALLTHROUGH(); + case kInitialized: + Uninitialize(); + break; + case kUninitialized: + FALLTHROUGH(); + case kInitRequired: + break; + } + + RTCLog(@"Disposing audio unit."); + OSStatus result = AudioComponentInstanceDispose(vpio_unit_); + if (result != noErr) { + RTCLogError(@"AudioComponentInstanceDispose failed. Error=%ld.", + (long)result); + } + vpio_unit_ = nullptr; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.cc b/third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.cc new file mode 100644 index 0000000000..d605dbaac7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.cc @@ -0,0 +1,41 @@ +/* + * libjingle + * Copyright 2004--2010, Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "modules/audio_device/linux/alsasymboltable_linux.h" + +namespace webrtc { +namespace adm_linux_alsa { + +LATE_BINDING_SYMBOL_TABLE_DEFINE_BEGIN(AlsaSymbolTable, "libasound.so.2") +#define X(sym) \ + LATE_BINDING_SYMBOL_TABLE_DEFINE_ENTRY(AlsaSymbolTable, sym) +ALSA_SYMBOLS_LIST +#undef X +LATE_BINDING_SYMBOL_TABLE_DEFINE_END(AlsaSymbolTable) + +} // namespace adm_linux_alsa +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.h b/third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.h new file mode 100644 index 0000000000..9c1f954522 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/alsasymboltable_linux.h @@ -0,0 +1,149 @@ +/* + * libjingle + * Copyright 2004--2010, Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef AUDIO_DEVICE_ALSASYMBOLTABLE_LINUX_H_ +#define AUDIO_DEVICE_ALSASYMBOLTABLE_LINUX_H_ + +#include "modules/audio_device/linux/latebindingsymboltable_linux.h" + +namespace webrtc { +namespace adm_linux_alsa { + +// The ALSA symbols we need, as an X-Macro list. +// This list must contain precisely every libasound function that is used in +// alsasoundsystem.cc. +#define ALSA_SYMBOLS_LIST \ + X(snd_device_name_free_hint) \ + X(snd_device_name_get_hint) \ + X(snd_device_name_hint) \ + X(snd_pcm_avail_update) \ + X(snd_pcm_close) \ + X(snd_pcm_delay) \ + X(snd_pcm_drop) \ + X(snd_pcm_open) \ + X(snd_pcm_prepare) \ + X(snd_pcm_readi) \ + X(snd_pcm_recover) \ + X(snd_pcm_resume) \ + X(snd_pcm_reset) \ + X(snd_pcm_state) \ + X(snd_pcm_set_params) \ + X(snd_pcm_get_params) \ + X(snd_pcm_start) \ + X(snd_pcm_stream) \ + X(snd_pcm_frames_to_bytes) \ + X(snd_pcm_bytes_to_frames) \ + X(snd_pcm_wait) \ + X(snd_pcm_writei) \ + X(snd_pcm_info_get_class) \ + X(snd_pcm_info_get_subdevices_avail) \ + X(snd_pcm_info_get_subdevice_name) \ + X(snd_pcm_info_set_subdevice) \ + X(snd_pcm_info_get_id) \ + X(snd_pcm_info_set_device) \ + X(snd_pcm_info_set_stream) \ + X(snd_pcm_info_get_name) \ + X(snd_pcm_info_get_subdevices_count) \ + X(snd_pcm_info_sizeof) \ + X(snd_pcm_hw_params) \ + X(snd_pcm_hw_params_malloc) \ + X(snd_pcm_hw_params_free) \ + X(snd_pcm_hw_params_any) \ + X(snd_pcm_hw_params_set_access) \ + X(snd_pcm_hw_params_set_format) \ + X(snd_pcm_hw_params_set_channels) \ + X(snd_pcm_hw_params_set_rate_near) \ + X(snd_pcm_hw_params_set_buffer_size_near) \ + X(snd_card_next) \ + X(snd_card_get_name) \ + X(snd_config_update) \ + X(snd_config_copy) \ + X(snd_config_get_id) \ + X(snd_ctl_open) \ + X(snd_ctl_close) \ + X(snd_ctl_card_info) \ + X(snd_ctl_card_info_sizeof) \ + X(snd_ctl_card_info_get_id) \ + X(snd_ctl_card_info_get_name) \ + X(snd_ctl_pcm_next_device) \ + X(snd_ctl_pcm_info) \ + X(snd_mixer_load) \ + X(snd_mixer_free) \ + X(snd_mixer_detach) \ + X(snd_mixer_close) \ + X(snd_mixer_open) \ + X(snd_mixer_attach) \ + X(snd_mixer_first_elem) \ + X(snd_mixer_elem_next) \ + X(snd_mixer_selem_get_name) \ + X(snd_mixer_selem_is_active) \ + X(snd_mixer_selem_register) \ + X(snd_mixer_selem_set_playback_volume_all) \ + X(snd_mixer_selem_get_playback_volume) \ + X(snd_mixer_selem_has_playback_volume) \ + X(snd_mixer_selem_get_playback_volume_range) \ + X(snd_mixer_selem_has_playback_switch) \ + X(snd_mixer_selem_get_playback_switch) \ + X(snd_mixer_selem_set_playback_switch_all) \ + X(snd_mixer_selem_has_capture_switch) \ + X(snd_mixer_selem_get_capture_switch) \ + X(snd_mixer_selem_set_capture_switch_all) \ + X(snd_mixer_selem_has_capture_volume) \ + X(snd_mixer_selem_set_capture_volume_all) \ + X(snd_mixer_selem_get_capture_volume) \ + X(snd_mixer_selem_get_capture_volume_range) \ + X(snd_dlopen) \ + X(snd_dlclose) \ + X(snd_config) \ + X(snd_config_search) \ + X(snd_config_get_string) \ + X(snd_config_search_definition) \ + X(snd_config_get_type) \ + X(snd_config_delete) \ + X(snd_config_iterator_entry) \ + X(snd_config_iterator_first) \ + X(snd_config_iterator_next) \ + X(snd_config_iterator_end) \ + X(snd_config_delete_compound_members) \ + X(snd_config_get_integer) \ + X(snd_config_get_bool) \ + X(snd_dlsym) \ + X(snd_strerror) \ + X(snd_lib_error) \ + X(snd_lib_error_set_handler) + +LATE_BINDING_SYMBOL_TABLE_DECLARE_BEGIN(AlsaSymbolTable) +#define X(sym) \ + LATE_BINDING_SYMBOL_TABLE_DECLARE_ENTRY(AlsaSymbolTable, sym) +ALSA_SYMBOLS_LIST +#undef X +LATE_BINDING_SYMBOL_TABLE_DECLARE_END(AlsaSymbolTable) + +} // namespace adm_linux_alsa +} // namespace webrtc + +#endif // AUDIO_DEVICE_ALSASYMBOLTABLE_LINUX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.cc b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.cc new file mode 100644 index 0000000000..1e4d1f50c8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.cc @@ -0,0 +1,1685 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_device/audio_device_config.h" +#include "modules/audio_device/linux/audio_device_alsa_linux.h" +#include "rtc_base/logging.h" + +#include "system_wrappers/include/event_wrapper.h" +#include "system_wrappers/include/sleep.h" +webrtc::adm_linux_alsa::AlsaSymbolTable AlsaSymbolTable; + +// Accesses ALSA functions through our late-binding symbol table instead of +// directly. This way we don't have to link to libasound, which means our binary +// will work on systems that don't have it. +#define LATE(sym) \ + LATESYM_GET(webrtc::adm_linux_alsa::AlsaSymbolTable, &AlsaSymbolTable, sym) + +// Redefine these here to be able to do late-binding +#undef snd_ctl_card_info_alloca +#define snd_ctl_card_info_alloca(ptr) \ + do { \ + *ptr = (snd_ctl_card_info_t*)__builtin_alloca( \ + LATE(snd_ctl_card_info_sizeof)()); \ + memset(*ptr, 0, LATE(snd_ctl_card_info_sizeof)()); \ + } while (0) + +#undef snd_pcm_info_alloca +#define snd_pcm_info_alloca(pInfo) \ + do { \ + *pInfo = (snd_pcm_info_t*)__builtin_alloca(LATE(snd_pcm_info_sizeof)()); \ + memset(*pInfo, 0, LATE(snd_pcm_info_sizeof)()); \ + } while (0) + +// snd_lib_error_handler_t +void WebrtcAlsaErrorHandler(const char* file, + int line, + const char* function, + int err, + const char* fmt, + ...){}; + +namespace webrtc { +static const unsigned int ALSA_PLAYOUT_FREQ = 48000; +static const unsigned int ALSA_PLAYOUT_CH = 2; +static const unsigned int ALSA_PLAYOUT_LATENCY = 40 * 1000; // in us +static const unsigned int ALSA_CAPTURE_FREQ = 48000; +static const unsigned int ALSA_CAPTURE_CH = 2; +static const unsigned int ALSA_CAPTURE_LATENCY = 40 * 1000; // in us +static const unsigned int ALSA_CAPTURE_WAIT_TIMEOUT = 5; // in ms + +#define FUNC_GET_NUM_OF_DEVICE 0 +#define FUNC_GET_DEVICE_NAME 1 +#define FUNC_GET_DEVICE_NAME_FOR_AN_ENUM 2 + +AudioDeviceLinuxALSA::AudioDeviceLinuxALSA() + : _ptrAudioBuffer(NULL), + _inputDeviceIndex(0), + _outputDeviceIndex(0), + _inputDeviceIsSpecified(false), + _outputDeviceIsSpecified(false), + _handleRecord(NULL), + _handlePlayout(NULL), + _recordingBuffersizeInFrame(0), + _recordingPeriodSizeInFrame(0), + _playoutBufferSizeInFrame(0), + _playoutPeriodSizeInFrame(0), + _recordingBufferSizeIn10MS(0), + _playoutBufferSizeIn10MS(0), + _recordingFramesIn10MS(0), + _playoutFramesIn10MS(0), + _recordingFreq(ALSA_CAPTURE_FREQ), + _playoutFreq(ALSA_PLAYOUT_FREQ), + _recChannels(ALSA_CAPTURE_CH), + _playChannels(ALSA_PLAYOUT_CH), + _recordingBuffer(NULL), + _playoutBuffer(NULL), + _recordingFramesLeft(0), + _playoutFramesLeft(0), + _initialized(false), + _recording(false), + _playing(false), + _recIsInitialized(false), + _playIsInitialized(false), + _AGC(false), + _recordingDelay(0), + _playoutDelay(0) { + memset(_oldKeyState, 0, sizeof(_oldKeyState)); + RTC_LOG(LS_INFO) << __FUNCTION__ << " created"; +} + +// ---------------------------------------------------------------------------- +// AudioDeviceLinuxALSA - dtor +// ---------------------------------------------------------------------------- + +AudioDeviceLinuxALSA::~AudioDeviceLinuxALSA() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " destroyed"; + + Terminate(); + + // Clean up the recording buffer and playout buffer. + if (_recordingBuffer) { + delete[] _recordingBuffer; + _recordingBuffer = NULL; + } + if (_playoutBuffer) { + delete[] _playoutBuffer; + _playoutBuffer = NULL; + } +} + +void AudioDeviceLinuxALSA::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + rtc::CritScope lock(&_critSect); + + _ptrAudioBuffer = audioBuffer; + + // Inform the AudioBuffer about default settings for this implementation. + // Set all values to zero here since the actual settings will be done by + // InitPlayout and InitRecording later. + _ptrAudioBuffer->SetRecordingSampleRate(0); + _ptrAudioBuffer->SetPlayoutSampleRate(0); + _ptrAudioBuffer->SetRecordingChannels(0); + _ptrAudioBuffer->SetPlayoutChannels(0); +} + +int32_t AudioDeviceLinuxALSA::ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const { + audioLayer = AudioDeviceModule::kLinuxAlsaAudio; + return 0; +} + +AudioDeviceGeneric::InitStatus AudioDeviceLinuxALSA::Init() { + rtc::CritScope lock(&_critSect); + + // Load libasound + if (!AlsaSymbolTable.Load()) { + // Alsa is not installed on this system + RTC_LOG(LS_ERROR) << "failed to load symbol table"; + return InitStatus::OTHER_ERROR; + } + + if (_initialized) { + return InitStatus::OK; + } +#if defined(USE_X11) + // Get X display handle for typing detection + _XDisplay = XOpenDisplay(NULL); + if (!_XDisplay) { + RTC_LOG(LS_WARNING) + << "failed to open X display, typing detection will not work"; + } +#endif + + _initialized = true; + + return InitStatus::OK; +} + +int32_t AudioDeviceLinuxALSA::Terminate() { + if (!_initialized) { + return 0; + } + + rtc::CritScope lock(&_critSect); + + _mixerManager.Close(); + + // RECORDING + if (_ptrThreadRec) { + rtc::PlatformThread* tmpThread = _ptrThreadRec.release(); + _critSect.Leave(); + + tmpThread->Stop(); + delete tmpThread; + + _critSect.Enter(); + } + + // PLAYOUT + if (_ptrThreadPlay) { + rtc::PlatformThread* tmpThread = _ptrThreadPlay.release(); + _critSect.Leave(); + + tmpThread->Stop(); + delete tmpThread; + + _critSect.Enter(); + } +#if defined(USE_X11) + if (_XDisplay) { + XCloseDisplay(_XDisplay); + _XDisplay = NULL; + } +#endif + _initialized = false; + _outputDeviceIsSpecified = false; + _inputDeviceIsSpecified = false; + + return 0; +} + +bool AudioDeviceLinuxALSA::Initialized() const { + return (_initialized); +} + +int32_t AudioDeviceLinuxALSA::InitSpeaker() { + rtc::CritScope lock(&_critSect); + + if (_playing) { + return -1; + } + + char devName[kAdmMaxDeviceNameSize] = {0}; + GetDevicesInfo(2, true, _outputDeviceIndex, devName, kAdmMaxDeviceNameSize); + return _mixerManager.OpenSpeaker(devName); +} + +int32_t AudioDeviceLinuxALSA::InitMicrophone() { + rtc::CritScope lock(&_critSect); + + if (_recording) { + return -1; + } + + char devName[kAdmMaxDeviceNameSize] = {0}; + GetDevicesInfo(2, false, _inputDeviceIndex, devName, kAdmMaxDeviceNameSize); + return _mixerManager.OpenMicrophone(devName); +} + +bool AudioDeviceLinuxALSA::SpeakerIsInitialized() const { + return (_mixerManager.SpeakerIsInitialized()); +} + +bool AudioDeviceLinuxALSA::MicrophoneIsInitialized() const { + return (_mixerManager.MicrophoneIsInitialized()); +} + +int32_t AudioDeviceLinuxALSA::SpeakerVolumeIsAvailable(bool& available) { + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + // Make an attempt to open up the + // output mixer corresponding to the currently selected output device. + if (!wasInitialized && InitSpeaker() == -1) { + // If we end up here it means that the selected speaker has no volume + // control. + available = false; + return 0; + } + + // Given that InitSpeaker was successful, we know that a volume control + // exists + available = true; + + // Close the initialized output mixer + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetSpeakerVolume(uint32_t volume) { + return (_mixerManager.SetSpeakerVolume(volume)); +} + +int32_t AudioDeviceLinuxALSA::SpeakerVolume(uint32_t& volume) const { + uint32_t level(0); + + if (_mixerManager.SpeakerVolume(level) == -1) { + return -1; + } + + volume = level; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::MaxSpeakerVolume(uint32_t& maxVolume) const { + uint32_t maxVol(0); + + if (_mixerManager.MaxSpeakerVolume(maxVol) == -1) { + return -1; + } + + maxVolume = maxVol; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::MinSpeakerVolume(uint32_t& minVolume) const { + uint32_t minVol(0); + + if (_mixerManager.MinSpeakerVolume(minVol) == -1) { + return -1; + } + + minVolume = minVol; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SpeakerMuteIsAvailable(bool& available) { + bool isAvailable(false); + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + // Make an attempt to open up the + // output mixer corresponding to the currently selected output device. + // + if (!wasInitialized && InitSpeaker() == -1) { + // If we end up here it means that the selected speaker has no volume + // control, hence it is safe to state that there is no mute control + // already at this stage. + available = false; + return 0; + } + + // Check if the selected speaker has a mute control + _mixerManager.SpeakerMuteIsAvailable(isAvailable); + + available = isAvailable; + + // Close the initialized output mixer + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetSpeakerMute(bool enable) { + return (_mixerManager.SetSpeakerMute(enable)); +} + +int32_t AudioDeviceLinuxALSA::SpeakerMute(bool& enabled) const { + bool muted(0); + + if (_mixerManager.SpeakerMute(muted) == -1) { + return -1; + } + + enabled = muted; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::MicrophoneMuteIsAvailable(bool& available) { + bool isAvailable(false); + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected input device. + // + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no volume + // control, hence it is safe to state that there is no mute control + // already at this stage. + available = false; + return 0; + } + + // Check if the selected microphone has a mute control + // + _mixerManager.MicrophoneMuteIsAvailable(isAvailable); + available = isAvailable; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetMicrophoneMute(bool enable) { + return (_mixerManager.SetMicrophoneMute(enable)); +} + +// ---------------------------------------------------------------------------- +// MicrophoneMute +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceLinuxALSA::MicrophoneMute(bool& enabled) const { + bool muted(0); + + if (_mixerManager.MicrophoneMute(muted) == -1) { + return -1; + } + + enabled = muted; + return 0; +} + +int32_t AudioDeviceLinuxALSA::StereoRecordingIsAvailable(bool& available) { + rtc::CritScope lock(&_critSect); + + // If we already have initialized in stereo it's obviously available + if (_recIsInitialized && (2 == _recChannels)) { + available = true; + return 0; + } + + // Save rec states and the number of rec channels + bool recIsInitialized = _recIsInitialized; + bool recording = _recording; + int recChannels = _recChannels; + + available = false; + + // Stop/uninitialize recording if initialized (and possibly started) + if (_recIsInitialized) { + StopRecording(); + } + + // Try init in stereo; + _recChannels = 2; + if (InitRecording() == 0) { + available = true; + } + + // Stop/uninitialize recording + StopRecording(); + + // Recover previous states + _recChannels = recChannels; + if (recIsInitialized) { + InitRecording(); + } + if (recording) { + StartRecording(); + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetStereoRecording(bool enable) { + if (enable) + _recChannels = 2; + else + _recChannels = 1; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::StereoRecording(bool& enabled) const { + if (_recChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::StereoPlayoutIsAvailable(bool& available) { + rtc::CritScope lock(&_critSect); + + // If we already have initialized in stereo it's obviously available + if (_playIsInitialized && (2 == _playChannels)) { + available = true; + return 0; + } + + // Save rec states and the number of rec channels + bool playIsInitialized = _playIsInitialized; + bool playing = _playing; + int playChannels = _playChannels; + + available = false; + + // Stop/uninitialize recording if initialized (and possibly started) + if (_playIsInitialized) { + StopPlayout(); + } + + // Try init in stereo; + _playChannels = 2; + if (InitPlayout() == 0) { + available = true; + } + + // Stop/uninitialize recording + StopPlayout(); + + // Recover previous states + _playChannels = playChannels; + if (playIsInitialized) { + InitPlayout(); + } + if (playing) { + StartPlayout(); + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetStereoPlayout(bool enable) { + if (enable) + _playChannels = 2; + else + _playChannels = 1; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::StereoPlayout(bool& enabled) const { + if (_playChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetAGC(bool enable) { + _AGC = enable; + + return 0; +} + +bool AudioDeviceLinuxALSA::AGC() const { + return _AGC; +} + +int32_t AudioDeviceLinuxALSA::MicrophoneVolumeIsAvailable(bool& available) { + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected output device. + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no volume + // control. + available = false; + return 0; + } + + // Given that InitMicrophone was successful, we know that a volume control + // exists + available = true; + + // Close the initialized input mixer + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetMicrophoneVolume(uint32_t volume) { + return (_mixerManager.SetMicrophoneVolume(volume)); + + return 0; +} + +int32_t AudioDeviceLinuxALSA::MicrophoneVolume(uint32_t& volume) const { + uint32_t level(0); + + if (_mixerManager.MicrophoneVolume(level) == -1) { + RTC_LOG(LS_WARNING) << "failed to retrive current microphone level"; + return -1; + } + + volume = level; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::MaxMicrophoneVolume(uint32_t& maxVolume) const { + uint32_t maxVol(0); + + if (_mixerManager.MaxMicrophoneVolume(maxVol) == -1) { + return -1; + } + + maxVolume = maxVol; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::MinMicrophoneVolume(uint32_t& minVolume) const { + uint32_t minVol(0); + + if (_mixerManager.MinMicrophoneVolume(minVol) == -1) { + return -1; + } + + minVolume = minVol; + + return 0; +} + +int16_t AudioDeviceLinuxALSA::PlayoutDevices() { + return (int16_t)GetDevicesInfo(0, true); +} + +int32_t AudioDeviceLinuxALSA::SetPlayoutDevice(uint16_t index) { + if (_playIsInitialized) { + return -1; + } + + uint32_t nDevices = GetDevicesInfo(0, true); + RTC_LOG(LS_VERBOSE) << "number of available audio output devices is " + << nDevices; + + if (index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + _outputDeviceIndex = index; + _outputDeviceIsSpecified = true; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType /*device*/) { + RTC_LOG(LS_ERROR) << "WindowsDeviceType not supported"; + return -1; +} + +int32_t AudioDeviceLinuxALSA::PlayoutDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + const uint16_t nDevices(PlayoutDevices()); + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + return GetDevicesInfo(1, true, index, name, kAdmMaxDeviceNameSize); +} + +int32_t AudioDeviceLinuxALSA::RecordingDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + const uint16_t nDevices(RecordingDevices()); + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + return GetDevicesInfo(1, false, index, name, kAdmMaxDeviceNameSize); +} + +int16_t AudioDeviceLinuxALSA::RecordingDevices() { + return (int16_t)GetDevicesInfo(0, false); +} + +int32_t AudioDeviceLinuxALSA::SetRecordingDevice(uint16_t index) { + if (_recIsInitialized) { + return -1; + } + + uint32_t nDevices = GetDevicesInfo(0, false); + RTC_LOG(LS_VERBOSE) << "number of availiable audio input devices is " + << nDevices; + + if (index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + _inputDeviceIndex = index; + _inputDeviceIsSpecified = true; + + return 0; +} + +// ---------------------------------------------------------------------------- +// SetRecordingDevice II (II) +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceLinuxALSA::SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType /*device*/) { + RTC_LOG(LS_ERROR) << "WindowsDeviceType not supported"; + return -1; +} + +int32_t AudioDeviceLinuxALSA::PlayoutIsAvailable(bool& available) { + available = false; + + // Try to initialize the playout side with mono + // Assumes that user set num channels after calling this function + _playChannels = 1; + int32_t res = InitPlayout(); + + // Cancel effect of initialization + StopPlayout(); + + if (res != -1) { + available = true; + } else { + // It may be possible to play out in stereo + res = StereoPlayoutIsAvailable(available); + if (available) { + // Then set channels to 2 so InitPlayout doesn't fail + _playChannels = 2; + } + } + + return res; +} + +int32_t AudioDeviceLinuxALSA::RecordingIsAvailable(bool& available) { + available = false; + + // Try to initialize the recording side with mono + // Assumes that user set num channels after calling this function + _recChannels = 1; + int32_t res = InitRecording(); + + // Cancel effect of initialization + StopRecording(); + + if (res != -1) { + available = true; + } else { + // It may be possible to record in stereo + res = StereoRecordingIsAvailable(available); + if (available) { + // Then set channels to 2 so InitPlayout doesn't fail + _recChannels = 2; + } + } + + return res; +} + +int32_t AudioDeviceLinuxALSA::InitPlayout() { + int errVal = 0; + + rtc::CritScope lock(&_critSect); + if (_playing) { + return -1; + } + + if (!_outputDeviceIsSpecified) { + return -1; + } + + if (_playIsInitialized) { + return 0; + } + // Initialize the speaker (devices might have been added or removed) + if (InitSpeaker() == -1) { + RTC_LOG(LS_WARNING) << "InitSpeaker() failed"; + } + + // Start by closing any existing wave-output devices + // + if (_handlePlayout != NULL) { + LATE(snd_pcm_close)(_handlePlayout); + _handlePlayout = NULL; + _playIsInitialized = false; + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error closing current playout sound device, error: " + << LATE(snd_strerror)(errVal); + } + } + + // Open PCM device for playout + char deviceName[kAdmMaxDeviceNameSize] = {0}; + GetDevicesInfo(2, true, _outputDeviceIndex, deviceName, + kAdmMaxDeviceNameSize); + + RTC_LOG(LS_VERBOSE) << "InitPlayout open (" << deviceName << ")"; + + errVal = LATE(snd_pcm_open)(&_handlePlayout, deviceName, + SND_PCM_STREAM_PLAYBACK, SND_PCM_NONBLOCK); + + if (errVal == -EBUSY) // Device busy - try some more! + { + for (int i = 0; i < 5; i++) { + SleepMs(1000); + errVal = LATE(snd_pcm_open)(&_handlePlayout, deviceName, + SND_PCM_STREAM_PLAYBACK, SND_PCM_NONBLOCK); + if (errVal == 0) { + break; + } + } + } + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "unable to open playback device: " + << LATE(snd_strerror)(errVal) << " (" << errVal << ")"; + _handlePlayout = NULL; + return -1; + } + + _playoutFramesIn10MS = _playoutFreq / 100; + if ((errVal = LATE(snd_pcm_set_params)( + _handlePlayout, +#if defined(WEBRTC_ARCH_BIG_ENDIAN) + SND_PCM_FORMAT_S16_BE, +#else + SND_PCM_FORMAT_S16_LE, // format +#endif + SND_PCM_ACCESS_RW_INTERLEAVED, // access + _playChannels, // channels + _playoutFreq, // rate + 1, // soft_resample + ALSA_PLAYOUT_LATENCY // 40*1000 //latency required overall latency + // in us + )) < 0) { /* 0.5sec */ + _playoutFramesIn10MS = 0; + RTC_LOG(LS_ERROR) << "unable to set playback device: " + << LATE(snd_strerror)(errVal) << " (" << errVal << ")"; + ErrorRecovery(errVal, _handlePlayout); + errVal = LATE(snd_pcm_close)(_handlePlayout); + _handlePlayout = NULL; + return -1; + } + + errVal = LATE(snd_pcm_get_params)(_handlePlayout, &_playoutBufferSizeInFrame, + &_playoutPeriodSizeInFrame); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_pcm_get_params: " << LATE(snd_strerror)(errVal) + << " (" << errVal << ")"; + _playoutBufferSizeInFrame = 0; + _playoutPeriodSizeInFrame = 0; + } else { + RTC_LOG(LS_VERBOSE) << "playout snd_pcm_get_params buffer_size:" + << _playoutBufferSizeInFrame + << " period_size :" << _playoutPeriodSizeInFrame; + } + + if (_ptrAudioBuffer) { + // Update webrtc audio buffer with the selected parameters + _ptrAudioBuffer->SetPlayoutSampleRate(_playoutFreq); + _ptrAudioBuffer->SetPlayoutChannels(_playChannels); + } + + // Set play buffer size + _playoutBufferSizeIn10MS = + LATE(snd_pcm_frames_to_bytes)(_handlePlayout, _playoutFramesIn10MS); + + // Init varaibles used for play + + if (_handlePlayout != NULL) { + _playIsInitialized = true; + return 0; + } else { + return -1; + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::InitRecording() { + int errVal = 0; + + rtc::CritScope lock(&_critSect); + + if (_recording) { + return -1; + } + + if (!_inputDeviceIsSpecified) { + return -1; + } + + if (_recIsInitialized) { + return 0; + } + + // Initialize the microphone (devices might have been added or removed) + if (InitMicrophone() == -1) { + RTC_LOG(LS_WARNING) << "InitMicrophone() failed"; + } + + // Start by closing any existing pcm-input devices + // + if (_handleRecord != NULL) { + int errVal = LATE(snd_pcm_close)(_handleRecord); + _handleRecord = NULL; + _recIsInitialized = false; + if (errVal < 0) { + RTC_LOG(LS_ERROR) + << "Error closing current recording sound device, error: " + << LATE(snd_strerror)(errVal); + } + } + + // Open PCM device for recording + // The corresponding settings for playout are made after the record settings + char deviceName[kAdmMaxDeviceNameSize] = {0}; + GetDevicesInfo(2, false, _inputDeviceIndex, deviceName, + kAdmMaxDeviceNameSize); + + RTC_LOG(LS_VERBOSE) << "InitRecording open (" << deviceName << ")"; + errVal = LATE(snd_pcm_open)(&_handleRecord, deviceName, + SND_PCM_STREAM_CAPTURE, SND_PCM_NONBLOCK); + + // Available modes: 0 = blocking, SND_PCM_NONBLOCK, SND_PCM_ASYNC + if (errVal == -EBUSY) // Device busy - try some more! + { + for (int i = 0; i < 5; i++) { + SleepMs(1000); + errVal = LATE(snd_pcm_open)(&_handleRecord, deviceName, + SND_PCM_STREAM_CAPTURE, SND_PCM_NONBLOCK); + if (errVal == 0) { + break; + } + } + } + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "unable to open record device: " + << LATE(snd_strerror)(errVal); + _handleRecord = NULL; + return -1; + } + + _recordingFramesIn10MS = _recordingFreq / 100; + if ((errVal = + LATE(snd_pcm_set_params)(_handleRecord, +#if defined(WEBRTC_ARCH_BIG_ENDIAN) + SND_PCM_FORMAT_S16_BE, // format +#else + SND_PCM_FORMAT_S16_LE, // format +#endif + SND_PCM_ACCESS_RW_INTERLEAVED, // access + _recChannels, // channels + _recordingFreq, // rate + 1, // soft_resample + ALSA_CAPTURE_LATENCY // latency in us + )) < 0) { + // Fall back to another mode then. + if (_recChannels == 1) + _recChannels = 2; + else + _recChannels = 1; + + if ((errVal = + LATE(snd_pcm_set_params)(_handleRecord, +#if defined(WEBRTC_ARCH_BIG_ENDIAN) + SND_PCM_FORMAT_S16_BE, // format +#else + SND_PCM_FORMAT_S16_LE, // format +#endif + SND_PCM_ACCESS_RW_INTERLEAVED, // access + _recChannels, // channels + _recordingFreq, // rate + 1, // soft_resample + ALSA_CAPTURE_LATENCY // latency in us + )) < 0) { + _recordingFramesIn10MS = 0; + RTC_LOG(LS_ERROR) << "unable to set record settings: " + << LATE(snd_strerror)(errVal) << " (" << errVal << ")"; + ErrorRecovery(errVal, _handleRecord); + errVal = LATE(snd_pcm_close)(_handleRecord); + _handleRecord = NULL; + return -1; + } + } + + errVal = LATE(snd_pcm_get_params)(_handleRecord, &_recordingBuffersizeInFrame, + &_recordingPeriodSizeInFrame); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_pcm_get_params " << LATE(snd_strerror)(errVal) + << " (" << errVal << ")"; + _recordingBuffersizeInFrame = 0; + _recordingPeriodSizeInFrame = 0; + } else { + RTC_LOG(LS_VERBOSE) << "capture snd_pcm_get_params, buffer_size:" + << _recordingBuffersizeInFrame + << ", period_size:" << _recordingPeriodSizeInFrame; + } + + if (_ptrAudioBuffer) { + // Update webrtc audio buffer with the selected parameters + _ptrAudioBuffer->SetRecordingSampleRate(_recordingFreq); + _ptrAudioBuffer->SetRecordingChannels(_recChannels); + } + + // Set rec buffer size and create buffer + _recordingBufferSizeIn10MS = + LATE(snd_pcm_frames_to_bytes)(_handleRecord, _recordingFramesIn10MS); + + if (_handleRecord != NULL) { + // Mark recording side as initialized + _recIsInitialized = true; + return 0; + } else { + return -1; + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::StartRecording() { + if (!_recIsInitialized) { + return -1; + } + + if (_recording) { + return 0; + } + + _recording = true; + + int errVal = 0; + _recordingFramesLeft = _recordingFramesIn10MS; + + // Make sure we only create the buffer once. + if (!_recordingBuffer) + _recordingBuffer = new int8_t[_recordingBufferSizeIn10MS]; + if (!_recordingBuffer) { + RTC_LOG(LS_ERROR) << "failed to alloc recording buffer"; + _recording = false; + return -1; + } + // RECORDING + _ptrThreadRec.reset(new rtc::PlatformThread( + RecThreadFunc, this, "webrtc_audio_module_capture_thread")); + + _ptrThreadRec->Start(); + _ptrThreadRec->SetPriority(rtc::kRealtimePriority); + + errVal = LATE(snd_pcm_prepare)(_handleRecord); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "capture snd_pcm_prepare failed (" + << LATE(snd_strerror)(errVal) << ")\n"; + // just log error + // if snd_pcm_open fails will return -1 + } + + errVal = LATE(snd_pcm_start)(_handleRecord); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "capture snd_pcm_start err: " + << LATE(snd_strerror)(errVal); + errVal = LATE(snd_pcm_start)(_handleRecord); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "capture snd_pcm_start 2nd try err: " + << LATE(snd_strerror)(errVal); + StopRecording(); + return -1; + } + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::StopRecording() { + { + rtc::CritScope lock(&_critSect); + + if (!_recIsInitialized) { + return 0; + } + + if (_handleRecord == NULL) { + return -1; + } + + // Make sure we don't start recording (it's asynchronous). + _recIsInitialized = false; + _recording = false; + } + + if (_ptrThreadRec) { + _ptrThreadRec->Stop(); + _ptrThreadRec.reset(); + } + + rtc::CritScope lock(&_critSect); + _recordingFramesLeft = 0; + if (_recordingBuffer) { + delete[] _recordingBuffer; + _recordingBuffer = NULL; + } + + // Stop and close pcm recording device. + int errVal = LATE(snd_pcm_drop)(_handleRecord); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error stop recording: " << LATE(snd_strerror)(errVal); + return -1; + } + + errVal = LATE(snd_pcm_close)(_handleRecord); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error closing record sound device, error: " + << LATE(snd_strerror)(errVal); + return -1; + } + + // Check if we have muted and unmute if so. + bool muteEnabled = false; + MicrophoneMute(muteEnabled); + if (muteEnabled) { + SetMicrophoneMute(false); + } + + // set the pcm input handle to NULL + _handleRecord = NULL; + return 0; +} + +bool AudioDeviceLinuxALSA::RecordingIsInitialized() const { + return (_recIsInitialized); +} + +bool AudioDeviceLinuxALSA::Recording() const { + return (_recording); +} + +bool AudioDeviceLinuxALSA::PlayoutIsInitialized() const { + return (_playIsInitialized); +} + +int32_t AudioDeviceLinuxALSA::StartPlayout() { + if (!_playIsInitialized) { + return -1; + } + + if (_playing) { + return 0; + } + + _playing = true; + + _playoutFramesLeft = 0; + if (!_playoutBuffer) + _playoutBuffer = new int8_t[_playoutBufferSizeIn10MS]; + if (!_playoutBuffer) { + RTC_LOG(LS_ERROR) << "failed to alloc playout buf"; + _playing = false; + return -1; + } + + // PLAYOUT + _ptrThreadPlay.reset(new rtc::PlatformThread( + PlayThreadFunc, this, "webrtc_audio_module_play_thread")); + _ptrThreadPlay->Start(); + _ptrThreadPlay->SetPriority(rtc::kRealtimePriority); + + int errVal = LATE(snd_pcm_prepare)(_handlePlayout); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "playout snd_pcm_prepare failed (" + << LATE(snd_strerror)(errVal) << ")\n"; + // just log error + // if snd_pcm_open fails will return -1 + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::StopPlayout() { + { + rtc::CritScope lock(&_critSect); + + if (!_playIsInitialized) { + return 0; + } + + if (_handlePlayout == NULL) { + return -1; + } + + _playing = false; + } + + // stop playout thread first + if (_ptrThreadPlay) { + _ptrThreadPlay->Stop(); + _ptrThreadPlay.reset(); + } + + rtc::CritScope lock(&_critSect); + + _playoutFramesLeft = 0; + delete[] _playoutBuffer; + _playoutBuffer = NULL; + + // stop and close pcm playout device + int errVal = LATE(snd_pcm_drop)(_handlePlayout); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error stop playing: " << LATE(snd_strerror)(errVal); + } + + errVal = LATE(snd_pcm_close)(_handlePlayout); + if (errVal < 0) + RTC_LOG(LS_ERROR) << "Error closing playout sound device, error: " + << LATE(snd_strerror)(errVal); + + // set the pcm input handle to NULL + _playIsInitialized = false; + _handlePlayout = NULL; + RTC_LOG(LS_VERBOSE) << "handle_playout is now set to NULL"; + + return 0; +} + +int32_t AudioDeviceLinuxALSA::PlayoutDelay(uint16_t& delayMS) const { + delayMS = (uint16_t)_playoutDelay * 1000 / _playoutFreq; + return 0; +} + +bool AudioDeviceLinuxALSA::Playing() const { + return (_playing); +} + +// ============================================================================ +// Private Methods +// ============================================================================ + +int32_t AudioDeviceLinuxALSA::GetDevicesInfo(const int32_t function, + const bool playback, + const int32_t enumDeviceNo, + char* enumDeviceName, + const int32_t ednLen) const { + // Device enumeration based on libjingle implementation + // by Tristan Schmelcher at Google Inc. + + const char* type = playback ? "Output" : "Input"; + // dmix and dsnoop are only for playback and capture, respectively, but ALSA + // stupidly includes them in both lists. + const char* ignorePrefix = playback ? "dsnoop:" : "dmix:"; + // (ALSA lists many more "devices" of questionable interest, but we show them + // just in case the weird devices may actually be desirable for some + // users/systems.) + + int err; + int enumCount(0); + bool keepSearching(true); + + // From Chromium issue 95797 + // Loop through the sound cards to get Alsa device hints. + // Don't use snd_device_name_hint(-1,..) since there is a access violation + // inside this ALSA API with libasound.so.2.0.0. + int card = -1; + while (!(LATE(snd_card_next)(&card)) && (card >= 0) && keepSearching) { + void** hints; + err = LATE(snd_device_name_hint)(card, "pcm", &hints); + if (err != 0) { + RTC_LOG(LS_ERROR) << "GetDevicesInfo - device name hint error: " + << LATE(snd_strerror)(err); + return -1; + } + + enumCount++; // default is 0 + if ((function == FUNC_GET_DEVICE_NAME || + function == FUNC_GET_DEVICE_NAME_FOR_AN_ENUM) && + enumDeviceNo == 0) { + strcpy(enumDeviceName, "default"); + + err = LATE(snd_device_name_free_hint)(hints); + if (err != 0) { + RTC_LOG(LS_ERROR) << "GetDevicesInfo - device name free hint error: " + << LATE(snd_strerror)(err); + } + + return 0; + } + + for (void** list = hints; *list != NULL; ++list) { + char* actualType = LATE(snd_device_name_get_hint)(*list, "IOID"); + if (actualType) { // NULL means it's both. + bool wrongType = (strcmp(actualType, type) != 0); + free(actualType); + if (wrongType) { + // Wrong type of device (i.e., input vs. output). + continue; + } + } + + char* name = LATE(snd_device_name_get_hint)(*list, "NAME"); + if (!name) { + RTC_LOG(LS_ERROR) << "Device has no name"; + // Skip it. + continue; + } + + // Now check if we actually want to show this device. + if (strcmp(name, "default") != 0 && strcmp(name, "null") != 0 && + strcmp(name, "pulse") != 0 && + strncmp(name, ignorePrefix, strlen(ignorePrefix)) != 0) { + // Yes, we do. + char* desc = LATE(snd_device_name_get_hint)(*list, "DESC"); + if (!desc) { + // Virtual devices don't necessarily have descriptions. + // Use their names instead. + desc = name; + } + + if (FUNC_GET_NUM_OF_DEVICE == function) { + RTC_LOG(LS_VERBOSE) << "Enum device " << enumCount << " - " << name; + } + if ((FUNC_GET_DEVICE_NAME == function) && (enumDeviceNo == enumCount)) { + // We have found the enum device, copy the name to buffer. + strncpy(enumDeviceName, desc, ednLen); + enumDeviceName[ednLen - 1] = '\0'; + keepSearching = false; + // Replace '\n' with '-'. + char* pret = strchr(enumDeviceName, '\n' /*0xa*/); // LF + if (pret) + *pret = '-'; + } + if ((FUNC_GET_DEVICE_NAME_FOR_AN_ENUM == function) && + (enumDeviceNo == enumCount)) { + // We have found the enum device, copy the name to buffer. + strncpy(enumDeviceName, name, ednLen); + enumDeviceName[ednLen - 1] = '\0'; + keepSearching = false; + } + + if (keepSearching) + ++enumCount; + + if (desc != name) + free(desc); + } + + free(name); + + if (!keepSearching) + break; + } + + err = LATE(snd_device_name_free_hint)(hints); + if (err != 0) { + RTC_LOG(LS_ERROR) << "GetDevicesInfo - device name free hint error: " + << LATE(snd_strerror)(err); + // Continue and return true anyway, since we did get the whole list. + } + } + + if (FUNC_GET_NUM_OF_DEVICE == function) { + if (enumCount == 1) // only default? + enumCount = 0; + return enumCount; // Normal return point for function 0 + } + + if (keepSearching) { + // If we get here for function 1 and 2, we didn't find the specified + // enum device. + RTC_LOG(LS_ERROR) + << "GetDevicesInfo - Could not find device name or numbers"; + return -1; + } + + return 0; +} + +int32_t AudioDeviceLinuxALSA::InputSanityCheckAfterUnlockedPeriod() const { + if (_handleRecord == NULL) { + RTC_LOG(LS_ERROR) << "input state has been modified during unlocked period"; + return -1; + } + return 0; +} + +int32_t AudioDeviceLinuxALSA::OutputSanityCheckAfterUnlockedPeriod() const { + if (_handlePlayout == NULL) { + RTC_LOG(LS_ERROR) + << "output state has been modified during unlocked period"; + return -1; + } + return 0; +} + +int32_t AudioDeviceLinuxALSA::ErrorRecovery(int32_t error, + snd_pcm_t* deviceHandle) { + int st = LATE(snd_pcm_state)(deviceHandle); + RTC_LOG(LS_VERBOSE) << "Trying to recover from " + << ((LATE(snd_pcm_stream)(deviceHandle) == + SND_PCM_STREAM_CAPTURE) + ? "capture" + : "playout") + << " error: " << LATE(snd_strerror)(error) << " (" + << error << ") (state " << st << ")"; + + // It is recommended to use snd_pcm_recover for all errors. If that function + // cannot handle the error, the input error code will be returned, otherwise + // 0 is returned. From snd_pcm_recover API doc: "This functions handles + // -EINTR (4) (interrupted system call), -EPIPE (32) (playout overrun or + // capture underrun) and -ESTRPIPE (86) (stream is suspended) error codes + // trying to prepare given stream for next I/O." + + /** Open */ + // SND_PCM_STATE_OPEN = 0, + /** Setup installed */ + // SND_PCM_STATE_SETUP, + /** Ready to start */ + // SND_PCM_STATE_PREPARED, + /** Running */ + // SND_PCM_STATE_RUNNING, + /** Stopped: underrun (playback) or overrun (capture) detected */ + // SND_PCM_STATE_XRUN,= 4 + /** Draining: running (playback) or stopped (capture) */ + // SND_PCM_STATE_DRAINING, + /** Paused */ + // SND_PCM_STATE_PAUSED, + /** Hardware is suspended */ + // SND_PCM_STATE_SUSPENDED, + // ** Hardware is disconnected */ + // SND_PCM_STATE_DISCONNECTED, + // SND_PCM_STATE_LAST = SND_PCM_STATE_DISCONNECTED + + // snd_pcm_recover isn't available in older alsa, e.g. on the FC4 machine + // in Sthlm lab. + + int res = LATE(snd_pcm_recover)(deviceHandle, error, 1); + if (0 == res) { + RTC_LOG(LS_VERBOSE) << "Recovery - snd_pcm_recover OK"; + + if ((error == -EPIPE || error == -ESTRPIPE) && // Buf underrun/overrun. + _recording && + LATE(snd_pcm_stream)(deviceHandle) == SND_PCM_STREAM_CAPTURE) { + // For capture streams we also have to repeat the explicit start() + // to get data flowing again. + int err = LATE(snd_pcm_start)(deviceHandle); + if (err != 0) { + RTC_LOG(LS_ERROR) << "Recovery - snd_pcm_start error: " << err; + return -1; + } + } + + if ((error == -EPIPE || error == -ESTRPIPE) && // Buf underrun/overrun. + _playing && + LATE(snd_pcm_stream)(deviceHandle) == SND_PCM_STREAM_PLAYBACK) { + // For capture streams we also have to repeat the explicit start() to get + // data flowing again. + int err = LATE(snd_pcm_start)(deviceHandle); + if (err != 0) { + RTC_LOG(LS_ERROR) << "Recovery - snd_pcm_start error: " + << LATE(snd_strerror)(err); + return -1; + } + } + + return -EPIPE == error ? 1 : 0; + } else { + RTC_LOG(LS_ERROR) << "Unrecoverable alsa stream error: " << res; + } + + return res; +} + +// ============================================================================ +// Thread Methods +// ============================================================================ + +bool AudioDeviceLinuxALSA::PlayThreadFunc(void* pThis) { + return (static_cast(pThis)->PlayThreadProcess()); +} + +bool AudioDeviceLinuxALSA::RecThreadFunc(void* pThis) { + return (static_cast(pThis)->RecThreadProcess()); +} + +bool AudioDeviceLinuxALSA::PlayThreadProcess() { + if (!_playing) + return false; + + int err; + snd_pcm_sframes_t frames; + snd_pcm_sframes_t avail_frames; + + Lock(); + // return a positive number of frames ready otherwise a negative error code + avail_frames = LATE(snd_pcm_avail_update)(_handlePlayout); + if (avail_frames < 0) { + RTC_LOG(LS_ERROR) << "playout snd_pcm_avail_update error: " + << LATE(snd_strerror)(avail_frames); + ErrorRecovery(avail_frames, _handlePlayout); + UnLock(); + return true; + } else if (avail_frames == 0) { + UnLock(); + + // maximum tixe in milliseconds to wait, a negative value means infinity + err = LATE(snd_pcm_wait)(_handlePlayout, 2); + if (err == 0) { // timeout occured + RTC_LOG(LS_VERBOSE) << "playout snd_pcm_wait timeout"; + } + + return true; + } + + if (_playoutFramesLeft <= 0) { + UnLock(); + _ptrAudioBuffer->RequestPlayoutData(_playoutFramesIn10MS); + Lock(); + + _playoutFramesLeft = _ptrAudioBuffer->GetPlayoutData(_playoutBuffer); + assert(_playoutFramesLeft == _playoutFramesIn10MS); + } + + if (static_cast(avail_frames) > _playoutFramesLeft) + avail_frames = _playoutFramesLeft; + + int size = LATE(snd_pcm_frames_to_bytes)(_handlePlayout, _playoutFramesLeft); + frames = LATE(snd_pcm_writei)( + _handlePlayout, &_playoutBuffer[_playoutBufferSizeIn10MS - size], + avail_frames); + + if (frames < 0) { + RTC_LOG(LS_VERBOSE) << "playout snd_pcm_writei error: " + << LATE(snd_strerror)(frames); + _playoutFramesLeft = 0; + ErrorRecovery(frames, _handlePlayout); + UnLock(); + return true; + } else { + assert(frames == avail_frames); + _playoutFramesLeft -= frames; + } + + UnLock(); + return true; +} + +bool AudioDeviceLinuxALSA::RecThreadProcess() { + if (!_recording) + return false; + + int err; + snd_pcm_sframes_t frames; + snd_pcm_sframes_t avail_frames; + int8_t buffer[_recordingBufferSizeIn10MS]; + + Lock(); + + // return a positive number of frames ready otherwise a negative error code + avail_frames = LATE(snd_pcm_avail_update)(_handleRecord); + if (avail_frames < 0) { + RTC_LOG(LS_ERROR) << "capture snd_pcm_avail_update error: " + << LATE(snd_strerror)(avail_frames); + ErrorRecovery(avail_frames, _handleRecord); + UnLock(); + return true; + } else if (avail_frames == 0) { // no frame is available now + UnLock(); + + // maximum time in milliseconds to wait, a negative value means infinity + err = LATE(snd_pcm_wait)(_handleRecord, ALSA_CAPTURE_WAIT_TIMEOUT); + if (err == 0) // timeout occured + RTC_LOG(LS_VERBOSE) << "capture snd_pcm_wait timeout"; + + return true; + } + + if (static_cast(avail_frames) > _recordingFramesLeft) + avail_frames = _recordingFramesLeft; + + frames = LATE(snd_pcm_readi)(_handleRecord, buffer, + avail_frames); // frames to be written + if (frames < 0) { + RTC_LOG(LS_ERROR) << "capture snd_pcm_readi error: " + << LATE(snd_strerror)(frames); + ErrorRecovery(frames, _handleRecord); + UnLock(); + return true; + } else if (frames > 0) { + assert(frames == avail_frames); + + int left_size = + LATE(snd_pcm_frames_to_bytes)(_handleRecord, _recordingFramesLeft); + int size = LATE(snd_pcm_frames_to_bytes)(_handleRecord, frames); + + memcpy(&_recordingBuffer[_recordingBufferSizeIn10MS - left_size], buffer, + size); + _recordingFramesLeft -= frames; + + if (!_recordingFramesLeft) { // buf is full + _recordingFramesLeft = _recordingFramesIn10MS; + + // store the recorded buffer (no action will be taken if the + // #recorded samples is not a full buffer) + _ptrAudioBuffer->SetRecordedBuffer(_recordingBuffer, + _recordingFramesIn10MS); + + uint32_t currentMicLevel = 0; + uint32_t newMicLevel = 0; + + if (AGC()) { + // store current mic level in the audio buffer if AGC is enabled + if (MicrophoneVolume(currentMicLevel) == 0) { + if (currentMicLevel == 0xffffffff) + currentMicLevel = 100; + // this call does not affect the actual microphone volume + _ptrAudioBuffer->SetCurrentMicLevel(currentMicLevel); + } + } + + // calculate delay + _playoutDelay = 0; + _recordingDelay = 0; + if (_handlePlayout) { + err = LATE(snd_pcm_delay)(_handlePlayout, + &_playoutDelay); // returned delay in frames + if (err < 0) { + // TODO(xians): Shall we call ErrorRecovery() here? + _playoutDelay = 0; + RTC_LOG(LS_ERROR) + << "playout snd_pcm_delay: " << LATE(snd_strerror)(err); + } + } + + err = LATE(snd_pcm_delay)(_handleRecord, + &_recordingDelay); // returned delay in frames + if (err < 0) { + // TODO(xians): Shall we call ErrorRecovery() here? + _recordingDelay = 0; + RTC_LOG(LS_ERROR) << "capture snd_pcm_delay: " + << LATE(snd_strerror)(err); + } + + // TODO(xians): Shall we add 10ms buffer delay to the record delay? + _ptrAudioBuffer->SetVQEData(_playoutDelay * 1000 / _playoutFreq, + _recordingDelay * 1000 / _recordingFreq, 0); + + _ptrAudioBuffer->SetTypingStatus(KeyPressed()); + + // Deliver recorded samples at specified sample rate, mic level etc. + // to the observer using callback. + UnLock(); + _ptrAudioBuffer->DeliverRecordedData(); + Lock(); + + if (AGC()) { + newMicLevel = _ptrAudioBuffer->NewMicLevel(); + if (newMicLevel != 0) { + // The VQE will only deliver non-zero microphone levels when a + // change is needed. Set this new mic level (received from the + // observer as return value in the callback). + if (SetMicrophoneVolume(newMicLevel) == -1) + RTC_LOG(LS_WARNING) + << "the required modification of the microphone volume failed"; + } + } + } + } + + UnLock(); + return true; +} + +bool AudioDeviceLinuxALSA::KeyPressed() const { +#if defined(USE_X11) + char szKey[32]; + unsigned int i = 0; + char state = 0; + + if (!_XDisplay) + return false; + + // Check key map status + XQueryKeymap(_XDisplay, szKey); + + // A bit change in keymap means a key is pressed + for (i = 0; i < sizeof(szKey); i++) + state |= (szKey[i] ^ _oldKeyState[i]) & szKey[i]; + + // Save old state + memcpy((char*)_oldKeyState, (char*)szKey, sizeof(_oldKeyState)); + return (state != 0); +#else + return false; +#endif +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.h b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.h new file mode 100644 index 0000000000..f05547b37a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_alsa_linux.h @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_DEVICE_ALSA_LINUX_H_ +#define AUDIO_DEVICE_AUDIO_DEVICE_ALSA_LINUX_H_ + +#include + +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/linux/audio_mixer_manager_alsa_linux.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/platform_thread.h" + +#if defined(USE_X11) +#include +#endif +#include +#include +#include + + +namespace webrtc +{ +class EventWrapper; + +class AudioDeviceLinuxALSA : public AudioDeviceGeneric +{ +public: + AudioDeviceLinuxALSA(); + virtual ~AudioDeviceLinuxALSA(); + + // Retrieve the currently utilized audio layer + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override; + + // Main initializaton and termination + InitStatus Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + // Device enumeration + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + + // Device selection + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override; + + // Audio transport initialization + int32_t PlayoutIsAvailable(bool& available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool& available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + // Audio transport control + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // Microphone Automatic Gain Control (AGC) + int32_t SetAGC(bool enable) override; + bool AGC() const override; + + // Audio mixer initialization + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Speaker volume controls + int32_t SpeakerVolumeIsAvailable(bool& available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; + + // Microphone volume controls + int32_t MicrophoneVolumeIsAvailable(bool& available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; + + // Speaker mute control + int32_t SpeakerMuteIsAvailable(bool& available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool& enabled) const override; + + // Microphone mute control + int32_t MicrophoneMuteIsAvailable(bool& available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool& enabled) const override; + + // Stereo support + int32_t StereoPlayoutIsAvailable(bool& available) override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool& enabled) const override; + + // Delay information and control + int32_t PlayoutDelay(uint16_t& delayMS) const override; + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; + +private: + int32_t GetDevicesInfo(const int32_t function, + const bool playback, + const int32_t enumDeviceNo = 0, + char* enumDeviceName = NULL, + const int32_t ednLen = 0) const; + int32_t ErrorRecovery(int32_t error, snd_pcm_t* deviceHandle); + + bool KeyPressed() const; + + void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION(_critSect) { _critSect.Enter(); }; + void UnLock() RTC_UNLOCK_FUNCTION(_critSect) { _critSect.Leave(); }; + + inline int32_t InputSanityCheckAfterUnlockedPeriod() const; + inline int32_t OutputSanityCheckAfterUnlockedPeriod() const; + + static bool RecThreadFunc(void*); + static bool PlayThreadFunc(void*); + bool RecThreadProcess(); + bool PlayThreadProcess(); + + AudioDeviceBuffer* _ptrAudioBuffer; + + rtc::CriticalSection _critSect; + + // TODO(pbos): Make plain members and start/stop instead of resetting these + // pointers. A thread can be reused. + std::unique_ptr _ptrThreadRec; + std::unique_ptr _ptrThreadPlay; + + AudioMixerManagerLinuxALSA _mixerManager; + + uint16_t _inputDeviceIndex; + uint16_t _outputDeviceIndex; + bool _inputDeviceIsSpecified; + bool _outputDeviceIsSpecified; + + snd_pcm_t* _handleRecord; + snd_pcm_t* _handlePlayout; + + snd_pcm_uframes_t _recordingBuffersizeInFrame; + snd_pcm_uframes_t _recordingPeriodSizeInFrame; + snd_pcm_uframes_t _playoutBufferSizeInFrame; + snd_pcm_uframes_t _playoutPeriodSizeInFrame; + + ssize_t _recordingBufferSizeIn10MS; + ssize_t _playoutBufferSizeIn10MS; + uint32_t _recordingFramesIn10MS; + uint32_t _playoutFramesIn10MS; + + uint32_t _recordingFreq; + uint32_t _playoutFreq; + uint8_t _recChannels; + uint8_t _playChannels; + + int8_t* _recordingBuffer; // in byte + int8_t* _playoutBuffer; // in byte + uint32_t _recordingFramesLeft; + uint32_t _playoutFramesLeft; + + bool _initialized; + bool _recording; + bool _playing; + bool _recIsInitialized; + bool _playIsInitialized; + bool _AGC; + + snd_pcm_sframes_t _recordingDelay; + snd_pcm_sframes_t _playoutDelay; + + char _oldKeyState[32]; +#if defined(USE_X11) + Display* _XDisplay; +#endif +}; + +} + +#endif // MODULES_AUDIO_DEVICE_MAIN_SOURCE_LINUX_AUDIO_DEVICE_ALSA_LINUX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.cc b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.cc new file mode 100644 index 0000000000..9d3b80b857 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.cc @@ -0,0 +1,2342 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_device/audio_device_config.h" +#include "modules/audio_device/linux/audio_device_pulse_linux.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/event_wrapper.h" + +webrtc::adm_linux_pulse::PulseAudioSymbolTable PaSymbolTable; + +// Accesses Pulse functions through our late-binding symbol table instead of +// directly. This way we don't have to link to libpulse, which means our binary +// will work on systems that don't have it. +#define LATE(sym) \ + LATESYM_GET(webrtc::adm_linux_pulse::PulseAudioSymbolTable, &PaSymbolTable, \ + sym) + +namespace webrtc { + +AudioDeviceLinuxPulse::AudioDeviceLinuxPulse() + : _ptrAudioBuffer(NULL), + _timeEventRec(*EventWrapper::Create()), + _timeEventPlay(*EventWrapper::Create()), + _recStartEvent(*EventWrapper::Create()), + _playStartEvent(*EventWrapper::Create()), + _inputDeviceIndex(0), + _outputDeviceIndex(0), + _inputDeviceIsSpecified(false), + _outputDeviceIsSpecified(false), + sample_rate_hz_(0), + _recChannels(1), + _playChannels(1), + _initialized(false), + _recording(false), + _playing(false), + _recIsInitialized(false), + _playIsInitialized(false), + _startRec(false), + _stopRec(false), + _startPlay(false), + _stopPlay(false), + _AGC(false), + update_speaker_volume_at_startup_(false), + _sndCardPlayDelay(0), + _sndCardRecDelay(0), + _writeErrors(0), + _deviceIndex(-1), + _numPlayDevices(0), + _numRecDevices(0), + _playDeviceName(NULL), + _recDeviceName(NULL), + _playDisplayDeviceName(NULL), + _recDisplayDeviceName(NULL), + _playBuffer(NULL), + _playbackBufferSize(0), + _playbackBufferUnused(0), + _tempBufferSpace(0), + _recBuffer(NULL), + _recordBufferSize(0), + _recordBufferUsed(0), + _tempSampleData(NULL), + _tempSampleDataSize(0), + _configuredLatencyPlay(0), + _configuredLatencyRec(0), + _paDeviceIndex(-1), + _paStateChanged(false), + _paMainloop(NULL), + _paMainloopApi(NULL), + _paContext(NULL), + _recStream(NULL), + _playStream(NULL), + _recStreamFlags(0), + _playStreamFlags(0) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " created"; + + memset(_paServerVersion, 0, sizeof(_paServerVersion)); + memset(&_playBufferAttr, 0, sizeof(_playBufferAttr)); + memset(&_recBufferAttr, 0, sizeof(_recBufferAttr)); + memset(_oldKeyState, 0, sizeof(_oldKeyState)); +} + +AudioDeviceLinuxPulse::~AudioDeviceLinuxPulse() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " destroyed"; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + Terminate(); + + if (_recBuffer) { + delete[] _recBuffer; + _recBuffer = NULL; + } + if (_playBuffer) { + delete[] _playBuffer; + _playBuffer = NULL; + } + if (_playDeviceName) { + delete[] _playDeviceName; + _playDeviceName = NULL; + } + if (_recDeviceName) { + delete[] _recDeviceName; + _recDeviceName = NULL; + } + + delete &_recStartEvent; + delete &_playStartEvent; + delete &_timeEventRec; + delete &_timeEventPlay; +} + +void AudioDeviceLinuxPulse::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + + _ptrAudioBuffer = audioBuffer; + + // Inform the AudioBuffer about default settings for this implementation. + // Set all values to zero here since the actual settings will be done by + // InitPlayout and InitRecording later. + _ptrAudioBuffer->SetRecordingSampleRate(0); + _ptrAudioBuffer->SetPlayoutSampleRate(0); + _ptrAudioBuffer->SetRecordingChannels(0); + _ptrAudioBuffer->SetPlayoutChannels(0); +} + +// ---------------------------------------------------------------------------- +// ActiveAudioLayer +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceLinuxPulse::ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const { + audioLayer = AudioDeviceModule::kLinuxPulseAudio; + return 0; +} + +AudioDeviceGeneric::InitStatus AudioDeviceLinuxPulse::Init() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_initialized) { + return InitStatus::OK; + } + + // Initialize PulseAudio + if (InitPulseAudio() < 0) { + RTC_LOG(LS_ERROR) << "failed to initialize PulseAudio"; + if (TerminatePulseAudio() < 0) { + RTC_LOG(LS_ERROR) << "failed to terminate PulseAudio"; + } + return InitStatus::OTHER_ERROR; + } + + // Get X display handle for typing detection + _XDisplay = XOpenDisplay(NULL); + if (!_XDisplay) { + RTC_LOG(LS_WARNING) + << "failed to open X display, typing detection will not work"; + } + + // RECORDING + _ptrThreadRec.reset(new rtc::PlatformThread( + RecThreadFunc, this, "webrtc_audio_module_rec_thread")); + + _ptrThreadRec->Start(); + _ptrThreadRec->SetPriority(rtc::kRealtimePriority); + + // PLAYOUT + _ptrThreadPlay.reset(new rtc::PlatformThread( + PlayThreadFunc, this, "webrtc_audio_module_play_thread")); + _ptrThreadPlay->Start(); + _ptrThreadPlay->SetPriority(rtc::kRealtimePriority); + + _initialized = true; + + return InitStatus::OK; +} + +int32_t AudioDeviceLinuxPulse::Terminate() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!_initialized) { + return 0; + } + + _mixerManager.Close(); + + // RECORDING + if (_ptrThreadRec) { + rtc::PlatformThread* tmpThread = _ptrThreadRec.release(); + + _timeEventRec.Set(); + tmpThread->Stop(); + delete tmpThread; + } + + // PLAYOUT + if (_ptrThreadPlay) { + rtc::PlatformThread* tmpThread = _ptrThreadPlay.release(); + + _timeEventPlay.Set(); + tmpThread->Stop(); + delete tmpThread; + } + + // Terminate PulseAudio + if (TerminatePulseAudio() < 0) { + RTC_LOG(LS_ERROR) << "failed to terminate PulseAudio"; + return -1; + } + + if (_XDisplay) { + XCloseDisplay(_XDisplay); + _XDisplay = NULL; + } + + _initialized = false; + _outputDeviceIsSpecified = false; + _inputDeviceIsSpecified = false; + + return 0; +} + +bool AudioDeviceLinuxPulse::Initialized() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_initialized); +} + +int32_t AudioDeviceLinuxPulse::InitSpeaker() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + + if (_playing) { + return -1; + } + + if (!_outputDeviceIsSpecified) { + return -1; + } + + // check if default device + if (_outputDeviceIndex == 0) { + uint16_t deviceIndex = 0; + GetDefaultDeviceInfo(false, NULL, deviceIndex); + _paDeviceIndex = deviceIndex; + } else { + // get the PA device index from + // the callback + _deviceIndex = _outputDeviceIndex; + + // get playout devices + PlayoutDevices(); + } + + // the callback has now set the _paDeviceIndex to + // the PulseAudio index of the device + if (_mixerManager.OpenSpeaker(_paDeviceIndex) == -1) { + return -1; + } + + // clear _deviceIndex + _deviceIndex = -1; + _paDeviceIndex = -1; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::InitMicrophone() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_recording) { + return -1; + } + + if (!_inputDeviceIsSpecified) { + return -1; + } + + // Check if default device + if (_inputDeviceIndex == 0) { + uint16_t deviceIndex = 0; + GetDefaultDeviceInfo(true, NULL, deviceIndex); + _paDeviceIndex = deviceIndex; + } else { + // Get the PA device index from + // the callback + _deviceIndex = _inputDeviceIndex; + + // get recording devices + RecordingDevices(); + } + + // The callback has now set the _paDeviceIndex to + // the PulseAudio index of the device + if (_mixerManager.OpenMicrophone(_paDeviceIndex) == -1) { + return -1; + } + + // Clear _deviceIndex + _deviceIndex = -1; + _paDeviceIndex = -1; + + return 0; +} + +bool AudioDeviceLinuxPulse::SpeakerIsInitialized() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_mixerManager.SpeakerIsInitialized()); +} + +bool AudioDeviceLinuxPulse::MicrophoneIsInitialized() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_mixerManager.MicrophoneIsInitialized()); +} + +int32_t AudioDeviceLinuxPulse::SpeakerVolumeIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + // Make an attempt to open up the + // output mixer corresponding to the currently selected output device. + if (!wasInitialized && InitSpeaker() == -1) { + // If we end up here it means that the selected speaker has no volume + // control. + available = false; + return 0; + } + + // Given that InitSpeaker was successful, we know volume control exists. + available = true; + + // Close the initialized output mixer + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SetSpeakerVolume(uint32_t volume) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!_playing) { + // Only update the volume if it's been set while we weren't playing. + update_speaker_volume_at_startup_ = true; + } + return (_mixerManager.SetSpeakerVolume(volume)); +} + +int32_t AudioDeviceLinuxPulse::SpeakerVolume(uint32_t& volume) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + uint32_t level(0); + + if (_mixerManager.SpeakerVolume(level) == -1) { + return -1; + } + + volume = level; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::MaxSpeakerVolume(uint32_t& maxVolume) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + uint32_t maxVol(0); + + if (_mixerManager.MaxSpeakerVolume(maxVol) == -1) { + return -1; + } + + maxVolume = maxVol; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::MinSpeakerVolume(uint32_t& minVolume) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + uint32_t minVol(0); + + if (_mixerManager.MinSpeakerVolume(minVol) == -1) { + return -1; + } + + minVolume = minVol; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SpeakerMuteIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + bool isAvailable(false); + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + // Make an attempt to open up the + // output mixer corresponding to the currently selected output device. + // + if (!wasInitialized && InitSpeaker() == -1) { + // If we end up here it means that the selected speaker has no volume + // control, hence it is safe to state that there is no mute control + // already at this stage. + available = false; + return 0; + } + + // Check if the selected speaker has a mute control + _mixerManager.SpeakerMuteIsAvailable(isAvailable); + + available = isAvailable; + + // Close the initialized output mixer + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SetSpeakerMute(bool enable) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_mixerManager.SetSpeakerMute(enable)); +} + +int32_t AudioDeviceLinuxPulse::SpeakerMute(bool& enabled) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + bool muted(0); + if (_mixerManager.SpeakerMute(muted) == -1) { + return -1; + } + + enabled = muted; + return 0; +} + +int32_t AudioDeviceLinuxPulse::MicrophoneMuteIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + bool isAvailable(false); + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected input device. + // + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no + // volume control, hence it is safe to state that there is no + // boost control already at this stage. + available = false; + return 0; + } + + // Check if the selected microphone has a mute control + // + _mixerManager.MicrophoneMuteIsAvailable(isAvailable); + available = isAvailable; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SetMicrophoneMute(bool enable) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_mixerManager.SetMicrophoneMute(enable)); +} + +int32_t AudioDeviceLinuxPulse::MicrophoneMute(bool& enabled) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + bool muted(0); + if (_mixerManager.MicrophoneMute(muted) == -1) { + return -1; + } + + enabled = muted; + return 0; +} + +int32_t AudioDeviceLinuxPulse::StereoRecordingIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_recChannels == 2 && _recording) { + available = true; + return 0; + } + + available = false; + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + int error = 0; + + if (!wasInitialized && InitMicrophone() == -1) { + // Cannot open the specified device + available = false; + return 0; + } + + // Check if the selected microphone can record stereo. + bool isAvailable(false); + error = _mixerManager.StereoRecordingIsAvailable(isAvailable); + if (!error) + available = isAvailable; + + // Close the initialized input mixer + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return error; +} + +int32_t AudioDeviceLinuxPulse::SetStereoRecording(bool enable) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (enable) + _recChannels = 2; + else + _recChannels = 1; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::StereoRecording(bool& enabled) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_recChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::StereoPlayoutIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_playChannels == 2 && _playing) { + available = true; + return 0; + } + + available = false; + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + int error = 0; + + if (!wasInitialized && InitSpeaker() == -1) { + // Cannot open the specified device. + return -1; + } + + // Check if the selected speaker can play stereo. + bool isAvailable(false); + error = _mixerManager.StereoPlayoutIsAvailable(isAvailable); + if (!error) + available = isAvailable; + + // Close the initialized input mixer + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return error; +} + +int32_t AudioDeviceLinuxPulse::SetStereoPlayout(bool enable) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (enable) + _playChannels = 2; + else + _playChannels = 1; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::StereoPlayout(bool& enabled) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_playChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SetAGC(bool enable) { + rtc::CritScope lock(&_critSect); + _AGC = enable; + + return 0; +} + +bool AudioDeviceLinuxPulse::AGC() const { + rtc::CritScope lock(&_critSect); + return _AGC; +} + +int32_t AudioDeviceLinuxPulse::MicrophoneVolumeIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected output device. + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no + // volume control. + available = false; + return 0; + } + + // Given that InitMicrophone was successful, we know that a volume control + // exists. + available = true; + + // Close the initialized input mixer + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SetMicrophoneVolume(uint32_t volume) { + return (_mixerManager.SetMicrophoneVolume(volume)); +} + +int32_t AudioDeviceLinuxPulse::MicrophoneVolume(uint32_t& volume) const { + uint32_t level(0); + + if (_mixerManager.MicrophoneVolume(level) == -1) { + RTC_LOG(LS_WARNING) << "failed to retrieve current microphone level"; + return -1; + } + + volume = level; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::MaxMicrophoneVolume(uint32_t& maxVolume) const { + uint32_t maxVol(0); + + if (_mixerManager.MaxMicrophoneVolume(maxVol) == -1) { + return -1; + } + + maxVolume = maxVol; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::MinMicrophoneVolume(uint32_t& minVolume) const { + uint32_t minVol(0); + + if (_mixerManager.MinMicrophoneVolume(minVol) == -1) { + return -1; + } + + minVolume = minVol; + + return 0; +} + +int16_t AudioDeviceLinuxPulse::PlayoutDevices() { + PaLock(); + + pa_operation* paOperation = NULL; + _numPlayDevices = 1; // init to 1 to account for "default" + + // get the whole list of devices and update _numPlayDevices + paOperation = + LATE(pa_context_get_sink_info_list)(_paContext, PaSinkInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + return _numPlayDevices; +} + +int32_t AudioDeviceLinuxPulse::SetPlayoutDevice(uint16_t index) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_playIsInitialized) { + return -1; + } + + const uint16_t nDevices = PlayoutDevices(); + + RTC_LOG(LS_VERBOSE) << "number of availiable output devices is " << nDevices; + + if (index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + _outputDeviceIndex = index; + _outputDeviceIsSpecified = true; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType /*device*/) { + RTC_LOG(LS_ERROR) << "WindowsDeviceType not supported"; + return -1; +} + +int32_t AudioDeviceLinuxPulse::PlayoutDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + const uint16_t nDevices = PlayoutDevices(); + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + // Check if default device + if (index == 0) { + uint16_t deviceIndex = 0; + return GetDefaultDeviceInfo(false, name, deviceIndex); + } + + // Tell the callback that we want + // The name for this device + _playDisplayDeviceName = name; + _deviceIndex = index; + + // get playout devices + PlayoutDevices(); + + // clear device name and index + _playDisplayDeviceName = NULL; + _deviceIndex = -1; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::RecordingDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + const uint16_t nDevices(RecordingDevices()); + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + // Check if default device + if (index == 0) { + uint16_t deviceIndex = 0; + return GetDefaultDeviceInfo(true, name, deviceIndex); + } + + // Tell the callback that we want + // the name for this device + _recDisplayDeviceName = name; + _deviceIndex = index; + + // Get recording devices + RecordingDevices(); + + // Clear device name and index + _recDisplayDeviceName = NULL; + _deviceIndex = -1; + + return 0; +} + +int16_t AudioDeviceLinuxPulse::RecordingDevices() { + PaLock(); + + pa_operation* paOperation = NULL; + _numRecDevices = 1; // Init to 1 to account for "default" + + // Get the whole list of devices and update _numRecDevices + paOperation = LATE(pa_context_get_source_info_list)( + _paContext, PaSourceInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + return _numRecDevices; +} + +int32_t AudioDeviceLinuxPulse::SetRecordingDevice(uint16_t index) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_recIsInitialized) { + return -1; + } + + const uint16_t nDevices(RecordingDevices()); + + RTC_LOG(LS_VERBOSE) << "number of availiable input devices is " << nDevices; + + if (index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + _inputDeviceIndex = index; + _inputDeviceIsSpecified = true; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType /*device*/) { + RTC_LOG(LS_ERROR) << "WindowsDeviceType not supported"; + return -1; +} + +int32_t AudioDeviceLinuxPulse::PlayoutIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + available = false; + + // Try to initialize the playout side + int32_t res = InitPlayout(); + + // Cancel effect of initialization + StopPlayout(); + + if (res != -1) { + available = true; + } + + return res; +} + +int32_t AudioDeviceLinuxPulse::RecordingIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + available = false; + + // Try to initialize the playout side + int32_t res = InitRecording(); + + // Cancel effect of initialization + StopRecording(); + + if (res != -1) { + available = true; + } + + return res; +} + +int32_t AudioDeviceLinuxPulse::InitPlayout() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + + if (_playing) { + return -1; + } + + if (!_outputDeviceIsSpecified) { + return -1; + } + + if (_playIsInitialized) { + return 0; + } + + // Initialize the speaker (devices might have been added or removed) + if (InitSpeaker() == -1) { + RTC_LOG(LS_WARNING) << "InitSpeaker() failed"; + } + + // Set the play sample specification + pa_sample_spec playSampleSpec; + playSampleSpec.channels = _playChannels; + playSampleSpec.format = PA_SAMPLE_S16LE; + playSampleSpec.rate = sample_rate_hz_; + + // Create a new play stream + _playStream = + LATE(pa_stream_new)(_paContext, "playStream", &playSampleSpec, NULL); + + if (!_playStream) { + RTC_LOG(LS_ERROR) << "failed to create play stream, err=" + << LATE(pa_context_errno)(_paContext); + return -1; + } + + // Provide the playStream to the mixer + _mixerManager.SetPlayStream(_playStream); + + if (_ptrAudioBuffer) { + // Update audio buffer with the selected parameters + _ptrAudioBuffer->SetPlayoutSampleRate(sample_rate_hz_); + _ptrAudioBuffer->SetPlayoutChannels((uint8_t)_playChannels); + } + + RTC_LOG(LS_VERBOSE) << "stream state " + << LATE(pa_stream_get_state)(_playStream); + + // Set stream flags + _playStreamFlags = (pa_stream_flags_t)(PA_STREAM_AUTO_TIMING_UPDATE | + PA_STREAM_INTERPOLATE_TIMING); + + if (_configuredLatencyPlay != WEBRTC_PA_NO_LATENCY_REQUIREMENTS) { + // If configuring a specific latency then we want to specify + // PA_STREAM_ADJUST_LATENCY to make the server adjust parameters + // automatically to reach that target latency. However, that flag + // doesn't exist in Ubuntu 8.04 and many people still use that, + // so we have to check the protocol version of libpulse. + if (LATE(pa_context_get_protocol_version)(_paContext) >= + WEBRTC_PA_ADJUST_LATENCY_PROTOCOL_VERSION) { + _playStreamFlags |= PA_STREAM_ADJUST_LATENCY; + } + + const pa_sample_spec* spec = LATE(pa_stream_get_sample_spec)(_playStream); + if (!spec) { + RTC_LOG(LS_ERROR) << "pa_stream_get_sample_spec()"; + return -1; + } + + size_t bytesPerSec = LATE(pa_bytes_per_second)(spec); + uint32_t latency = bytesPerSec * WEBRTC_PA_PLAYBACK_LATENCY_MINIMUM_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + // Set the play buffer attributes + _playBufferAttr.maxlength = latency; // num bytes stored in the buffer + _playBufferAttr.tlength = latency; // target fill level of play buffer + // minimum free num bytes before server request more data + _playBufferAttr.minreq = latency / WEBRTC_PA_PLAYBACK_REQUEST_FACTOR; + // prebuffer tlength before starting playout + _playBufferAttr.prebuf = _playBufferAttr.tlength - _playBufferAttr.minreq; + + _configuredLatencyPlay = latency; + } + + // num samples in bytes * num channels + _playbackBufferSize = sample_rate_hz_ / 100 * 2 * _playChannels; + _playbackBufferUnused = _playbackBufferSize; + _playBuffer = new int8_t[_playbackBufferSize]; + + // Enable underflow callback + LATE(pa_stream_set_underflow_callback) + (_playStream, PaStreamUnderflowCallback, this); + + // Set the state callback function for the stream + LATE(pa_stream_set_state_callback)(_playStream, PaStreamStateCallback, this); + + // Mark playout side as initialized + _playIsInitialized = true; + _sndCardPlayDelay = 0; + _sndCardRecDelay = 0; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::InitRecording() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + + if (_recording) { + return -1; + } + + if (!_inputDeviceIsSpecified) { + return -1; + } + + if (_recIsInitialized) { + return 0; + } + + // Initialize the microphone (devices might have been added or removed) + if (InitMicrophone() == -1) { + RTC_LOG(LS_WARNING) << "InitMicrophone() failed"; + } + + // Set the rec sample specification + pa_sample_spec recSampleSpec; + recSampleSpec.channels = _recChannels; + recSampleSpec.format = PA_SAMPLE_S16LE; + recSampleSpec.rate = sample_rate_hz_; + + // Create a new rec stream + _recStream = + LATE(pa_stream_new)(_paContext, "recStream", &recSampleSpec, NULL); + if (!_recStream) { + RTC_LOG(LS_ERROR) << "failed to create rec stream, err=" + << LATE(pa_context_errno)(_paContext); + return -1; + } + + // Provide the recStream to the mixer + _mixerManager.SetRecStream(_recStream); + + if (_ptrAudioBuffer) { + // Update audio buffer with the selected parameters + _ptrAudioBuffer->SetRecordingSampleRate(sample_rate_hz_); + _ptrAudioBuffer->SetRecordingChannels((uint8_t)_recChannels); + } + + if (_configuredLatencyRec != WEBRTC_PA_NO_LATENCY_REQUIREMENTS) { + _recStreamFlags = (pa_stream_flags_t)(PA_STREAM_AUTO_TIMING_UPDATE | + PA_STREAM_INTERPOLATE_TIMING); + + // If configuring a specific latency then we want to specify + // PA_STREAM_ADJUST_LATENCY to make the server adjust parameters + // automatically to reach that target latency. However, that flag + // doesn't exist in Ubuntu 8.04 and many people still use that, + // so we have to check the protocol version of libpulse. + if (LATE(pa_context_get_protocol_version)(_paContext) >= + WEBRTC_PA_ADJUST_LATENCY_PROTOCOL_VERSION) { + _recStreamFlags |= PA_STREAM_ADJUST_LATENCY; + } + + const pa_sample_spec* spec = LATE(pa_stream_get_sample_spec)(_recStream); + if (!spec) { + RTC_LOG(LS_ERROR) << "pa_stream_get_sample_spec(rec)"; + return -1; + } + + size_t bytesPerSec = LATE(pa_bytes_per_second)(spec); + uint32_t latency = bytesPerSec * WEBRTC_PA_LOW_CAPTURE_LATENCY_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + // Set the rec buffer attributes + // Note: fragsize specifies a maximum transfer size, not a minimum, so + // it is not possible to force a high latency setting, only a low one. + _recBufferAttr.fragsize = latency; // size of fragment + _recBufferAttr.maxlength = + latency + bytesPerSec * WEBRTC_PA_CAPTURE_BUFFER_EXTRA_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + _configuredLatencyRec = latency; + } + + _recordBufferSize = sample_rate_hz_ / 100 * 2 * _recChannels; + _recordBufferUsed = 0; + _recBuffer = new int8_t[_recordBufferSize]; + + // Enable overflow callback + LATE(pa_stream_set_overflow_callback) + (_recStream, PaStreamOverflowCallback, this); + + // Set the state callback function for the stream + LATE(pa_stream_set_state_callback)(_recStream, PaStreamStateCallback, this); + + // Mark recording side as initialized + _recIsInitialized = true; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::StartRecording() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (!_recIsInitialized) { + return -1; + } + + if (_recording) { + return 0; + } + + // Set state to ensure that the recording starts from the audio thread. + _startRec = true; + + // The audio thread will signal when recording has started. + _timeEventRec.Set(); + if (kEventTimeout == _recStartEvent.Wait(10000)) { + { + rtc::CritScope lock(&_critSect); + _startRec = false; + } + StopRecording(); + RTC_LOG(LS_ERROR) << "failed to activate recording"; + return -1; + } + + { + rtc::CritScope lock(&_critSect); + if (_recording) { + // The recording state is set by the audio thread after recording + // has started. + } else { + RTC_LOG(LS_ERROR) << "failed to activate recording"; + return -1; + } + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::StopRecording() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + rtc::CritScope lock(&_critSect); + + if (!_recIsInitialized) { + return 0; + } + + if (_recStream == NULL) { + return -1; + } + + _recIsInitialized = false; + _recording = false; + + RTC_LOG(LS_VERBOSE) << "stopping recording"; + + // Stop Recording + PaLock(); + + DisableReadCallback(); + LATE(pa_stream_set_overflow_callback)(_recStream, NULL, NULL); + + // Unset this here so that we don't get a TERMINATED callback + LATE(pa_stream_set_state_callback)(_recStream, NULL, NULL); + + if (LATE(pa_stream_get_state)(_recStream) != PA_STREAM_UNCONNECTED) { + // Disconnect the stream + if (LATE(pa_stream_disconnect)(_recStream) != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to disconnect rec stream, err=" + << LATE(pa_context_errno)(_paContext); + PaUnLock(); + return -1; + } + + RTC_LOG(LS_VERBOSE) << "disconnected recording"; + } + + LATE(pa_stream_unref)(_recStream); + _recStream = NULL; + + PaUnLock(); + + // Provide the recStream to the mixer + _mixerManager.SetRecStream(_recStream); + + if (_recBuffer) { + delete[] _recBuffer; + _recBuffer = NULL; + } + + return 0; +} + +bool AudioDeviceLinuxPulse::RecordingIsInitialized() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_recIsInitialized); +} + +bool AudioDeviceLinuxPulse::Recording() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_recording); +} + +bool AudioDeviceLinuxPulse::PlayoutIsInitialized() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_playIsInitialized); +} + +int32_t AudioDeviceLinuxPulse::StartPlayout() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + + if (!_playIsInitialized) { + return -1; + } + + if (_playing) { + return 0; + } + + // Set state to ensure that playout starts from the audio thread. + { + rtc::CritScope lock(&_critSect); + _startPlay = true; + } + + // Both |_startPlay| and |_playing| needs protction since they are also + // accessed on the playout thread. + + // The audio thread will signal when playout has started. + _timeEventPlay.Set(); + if (kEventTimeout == _playStartEvent.Wait(10000)) { + { + rtc::CritScope lock(&_critSect); + _startPlay = false; + } + StopPlayout(); + RTC_LOG(LS_ERROR) << "failed to activate playout"; + return -1; + } + + { + rtc::CritScope lock(&_critSect); + if (_playing) { + // The playing state is set by the audio thread after playout + // has started. + } else { + RTC_LOG(LS_ERROR) << "failed to activate playing"; + return -1; + } + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::StopPlayout() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + rtc::CritScope lock(&_critSect); + + if (!_playIsInitialized) { + return 0; + } + + if (_playStream == NULL) { + return -1; + } + + _playIsInitialized = false; + _playing = false; + _sndCardPlayDelay = 0; + _sndCardRecDelay = 0; + + RTC_LOG(LS_VERBOSE) << "stopping playback"; + + // Stop Playout + PaLock(); + + DisableWriteCallback(); + LATE(pa_stream_set_underflow_callback)(_playStream, NULL, NULL); + + // Unset this here so that we don't get a TERMINATED callback + LATE(pa_stream_set_state_callback)(_playStream, NULL, NULL); + + if (LATE(pa_stream_get_state)(_playStream) != PA_STREAM_UNCONNECTED) { + // Disconnect the stream + if (LATE(pa_stream_disconnect)(_playStream) != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to disconnect play stream, err=" + << LATE(pa_context_errno)(_paContext); + PaUnLock(); + return -1; + } + + RTC_LOG(LS_VERBOSE) << "disconnected playback"; + } + + LATE(pa_stream_unref)(_playStream); + _playStream = NULL; + + PaUnLock(); + + // Provide the playStream to the mixer + _mixerManager.SetPlayStream(_playStream); + + if (_playBuffer) { + delete[] _playBuffer; + _playBuffer = NULL; + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::PlayoutDelay(uint16_t& delayMS) const { + rtc::CritScope lock(&_critSect); + delayMS = (uint16_t)_sndCardPlayDelay; + return 0; +} + +bool AudioDeviceLinuxPulse::Playing() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + return (_playing); +} + +// ============================================================================ +// Private Methods +// ============================================================================ + +void AudioDeviceLinuxPulse::PaContextStateCallback(pa_context* c, void* pThis) { + static_cast(pThis)->PaContextStateCallbackHandler(c); +} + +// ---------------------------------------------------------------------------- +// PaSinkInfoCallback +// ---------------------------------------------------------------------------- + +void AudioDeviceLinuxPulse::PaSinkInfoCallback(pa_context* /*c*/, + const pa_sink_info* i, + int eol, + void* pThis) { + static_cast(pThis)->PaSinkInfoCallbackHandler(i, eol); +} + +void AudioDeviceLinuxPulse::PaSourceInfoCallback(pa_context* /*c*/, + const pa_source_info* i, + int eol, + void* pThis) { + static_cast(pThis)->PaSourceInfoCallbackHandler(i, + eol); +} + +void AudioDeviceLinuxPulse::PaServerInfoCallback(pa_context* /*c*/, + const pa_server_info* i, + void* pThis) { + static_cast(pThis)->PaServerInfoCallbackHandler(i); +} + +void AudioDeviceLinuxPulse::PaStreamStateCallback(pa_stream* p, void* pThis) { + static_cast(pThis)->PaStreamStateCallbackHandler(p); +} + +void AudioDeviceLinuxPulse::PaContextStateCallbackHandler(pa_context* c) { + RTC_LOG(LS_VERBOSE) << "context state cb"; + + pa_context_state_t state = LATE(pa_context_get_state)(c); + switch (state) { + case PA_CONTEXT_UNCONNECTED: + RTC_LOG(LS_VERBOSE) << "unconnected"; + break; + case PA_CONTEXT_CONNECTING: + case PA_CONTEXT_AUTHORIZING: + case PA_CONTEXT_SETTING_NAME: + RTC_LOG(LS_VERBOSE) << "no state"; + break; + case PA_CONTEXT_FAILED: + case PA_CONTEXT_TERMINATED: + RTC_LOG(LS_VERBOSE) << "failed"; + _paStateChanged = true; + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + break; + case PA_CONTEXT_READY: + RTC_LOG(LS_VERBOSE) << "ready"; + _paStateChanged = true; + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + break; + } +} + +void AudioDeviceLinuxPulse::PaSinkInfoCallbackHandler(const pa_sink_info* i, + int eol) { + if (eol) { + // Signal that we are done + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + return; + } + + if (_numPlayDevices == _deviceIndex) { + // Convert the device index to the one of the sink + _paDeviceIndex = i->index; + + if (_playDeviceName) { + // Copy the sink name + strncpy(_playDeviceName, i->name, kAdmMaxDeviceNameSize); + _playDeviceName[kAdmMaxDeviceNameSize - 1] = '\0'; + } + if (_playDisplayDeviceName) { + // Copy the sink display name + strncpy(_playDisplayDeviceName, i->description, kAdmMaxDeviceNameSize); + _playDisplayDeviceName[kAdmMaxDeviceNameSize - 1] = '\0'; + } + } + + _numPlayDevices++; +} + +void AudioDeviceLinuxPulse::PaSourceInfoCallbackHandler(const pa_source_info* i, + int eol) { + if (eol) { + // Signal that we are done + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + return; + } + + // We don't want to list output devices + if (i->monitor_of_sink == PA_INVALID_INDEX) { + if (_numRecDevices == _deviceIndex) { + // Convert the device index to the one of the source + _paDeviceIndex = i->index; + + if (_recDeviceName) { + // copy the source name + strncpy(_recDeviceName, i->name, kAdmMaxDeviceNameSize); + _recDeviceName[kAdmMaxDeviceNameSize - 1] = '\0'; + } + if (_recDisplayDeviceName) { + // Copy the source display name + strncpy(_recDisplayDeviceName, i->description, kAdmMaxDeviceNameSize); + _recDisplayDeviceName[kAdmMaxDeviceNameSize - 1] = '\0'; + } + } + + _numRecDevices++; + } +} + +void AudioDeviceLinuxPulse::PaServerInfoCallbackHandler( + const pa_server_info* i) { + // Use PA native sampling rate + sample_rate_hz_ = i->sample_spec.rate; + + // Copy the PA server version + strncpy(_paServerVersion, i->server_version, 31); + _paServerVersion[31] = '\0'; + + if (_recDisplayDeviceName) { + // Copy the source name + strncpy(_recDisplayDeviceName, i->default_source_name, + kAdmMaxDeviceNameSize); + _recDisplayDeviceName[kAdmMaxDeviceNameSize - 1] = '\0'; + } + + if (_playDisplayDeviceName) { + // Copy the sink name + strncpy(_playDisplayDeviceName, i->default_sink_name, + kAdmMaxDeviceNameSize); + _playDisplayDeviceName[kAdmMaxDeviceNameSize - 1] = '\0'; + } + + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); +} + +void AudioDeviceLinuxPulse::PaStreamStateCallbackHandler(pa_stream* p) { + RTC_LOG(LS_VERBOSE) << "stream state cb"; + + pa_stream_state_t state = LATE(pa_stream_get_state)(p); + switch (state) { + case PA_STREAM_UNCONNECTED: + RTC_LOG(LS_VERBOSE) << "unconnected"; + break; + case PA_STREAM_CREATING: + RTC_LOG(LS_VERBOSE) << "creating"; + break; + case PA_STREAM_FAILED: + case PA_STREAM_TERMINATED: + RTC_LOG(LS_VERBOSE) << "failed"; + break; + case PA_STREAM_READY: + RTC_LOG(LS_VERBOSE) << "ready"; + break; + } + + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); +} + +int32_t AudioDeviceLinuxPulse::CheckPulseAudioVersion() { + PaLock(); + + pa_operation* paOperation = NULL; + + // get the server info and update deviceName + paOperation = + LATE(pa_context_get_server_info)(_paContext, PaServerInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + RTC_LOG(LS_VERBOSE) << "checking PulseAudio version: " << _paServerVersion; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::InitSamplingFrequency() { + PaLock(); + + pa_operation* paOperation = NULL; + + // Get the server info and update sample_rate_hz_ + paOperation = + LATE(pa_context_get_server_info)(_paContext, PaServerInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + return 0; +} + +int32_t AudioDeviceLinuxPulse::GetDefaultDeviceInfo(bool recDevice, + char* name, + uint16_t& index) { + char tmpName[kAdmMaxDeviceNameSize] = {0}; + // subtract length of "default: " + uint16_t nameLen = kAdmMaxDeviceNameSize - 9; + char* pName = NULL; + + if (name) { + // Add "default: " + strcpy(name, "default: "); + pName = &name[9]; + } + + // Tell the callback that we want + // the name for this device + if (recDevice) { + _recDisplayDeviceName = tmpName; + } else { + _playDisplayDeviceName = tmpName; + } + + // Set members + _paDeviceIndex = -1; + _deviceIndex = 0; + _numPlayDevices = 0; + _numRecDevices = 0; + + PaLock(); + + pa_operation* paOperation = NULL; + + // Get the server info and update deviceName + paOperation = + LATE(pa_context_get_server_info)(_paContext, PaServerInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + // Get the device index + if (recDevice) { + paOperation = LATE(pa_context_get_source_info_by_name)( + _paContext, (char*)tmpName, PaSourceInfoCallback, this); + } else { + paOperation = LATE(pa_context_get_sink_info_by_name)( + _paContext, (char*)tmpName, PaSinkInfoCallback, this); + } + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + // Set the index + index = _paDeviceIndex; + + if (name) { + // Copy to name string + strncpy(pName, tmpName, nameLen); + } + + // Clear members + _playDisplayDeviceName = NULL; + _recDisplayDeviceName = NULL; + _paDeviceIndex = -1; + _deviceIndex = -1; + _numPlayDevices = 0; + _numRecDevices = 0; + + return 0; +} + +int32_t AudioDeviceLinuxPulse::InitPulseAudio() { + int retVal = 0; + + // Load libpulse + if (!PaSymbolTable.Load()) { + // Most likely the Pulse library and sound server are not installed on + // this system + RTC_LOG(LS_ERROR) << "failed to load symbol table"; + return -1; + } + + // Create a mainloop API and connection to the default server + // the mainloop is the internal asynchronous API event loop + if (_paMainloop) { + RTC_LOG(LS_ERROR) << "PA mainloop has already existed"; + return -1; + } + _paMainloop = LATE(pa_threaded_mainloop_new)(); + if (!_paMainloop) { + RTC_LOG(LS_ERROR) << "could not create mainloop"; + return -1; + } + + // Start the threaded main loop + retVal = LATE(pa_threaded_mainloop_start)(_paMainloop); + if (retVal != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to start main loop, error=" << retVal; + return -1; + } + + RTC_LOG(LS_VERBOSE) << "mainloop running!"; + + PaLock(); + + _paMainloopApi = LATE(pa_threaded_mainloop_get_api)(_paMainloop); + if (!_paMainloopApi) { + RTC_LOG(LS_ERROR) << "could not create mainloop API"; + PaUnLock(); + return -1; + } + + // Create a new PulseAudio context + if (_paContext) { + RTC_LOG(LS_ERROR) << "PA context has already existed"; + PaUnLock(); + return -1; + } + _paContext = LATE(pa_context_new)(_paMainloopApi, "WEBRTC VoiceEngine"); + + if (!_paContext) { + RTC_LOG(LS_ERROR) << "could not create context"; + PaUnLock(); + return -1; + } + + // Set state callback function + LATE(pa_context_set_state_callback)(_paContext, PaContextStateCallback, this); + + // Connect the context to a server (default) + _paStateChanged = false; + retVal = + LATE(pa_context_connect)(_paContext, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL); + + if (retVal != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to connect context, error=" << retVal; + PaUnLock(); + return -1; + } + + // Wait for state change + while (!_paStateChanged) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + // Now check to see what final state we reached. + pa_context_state_t state = LATE(pa_context_get_state)(_paContext); + + if (state != PA_CONTEXT_READY) { + if (state == PA_CONTEXT_FAILED) { + RTC_LOG(LS_ERROR) << "failed to connect to PulseAudio sound server"; + } else if (state == PA_CONTEXT_TERMINATED) { + RTC_LOG(LS_ERROR) << "PulseAudio connection terminated early"; + } else { + // Shouldn't happen, because we only signal on one of those three + // states + RTC_LOG(LS_ERROR) << "unknown problem connecting to PulseAudio"; + } + PaUnLock(); + return -1; + } + + PaUnLock(); + + // Give the objects to the mixer manager + _mixerManager.SetPulseAudioObjects(_paMainloop, _paContext); + + // Check the version + if (CheckPulseAudioVersion() < 0) { + RTC_LOG(LS_ERROR) << "PulseAudio version " << _paServerVersion + << " not supported"; + return -1; + } + + // Initialize sampling frequency + if (InitSamplingFrequency() < 0 || sample_rate_hz_ == 0) { + RTC_LOG(LS_ERROR) << "failed to initialize sampling frequency, set to " + << sample_rate_hz_ << " Hz"; + return -1; + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::TerminatePulseAudio() { + // Do nothing if the instance doesn't exist + // likely PaSymbolTable.Load() fails + if (!_paMainloop) { + return 0; + } + + PaLock(); + + // Disconnect the context + if (_paContext) { + LATE(pa_context_disconnect)(_paContext); + } + + // Unreference the context + if (_paContext) { + LATE(pa_context_unref)(_paContext); + } + + PaUnLock(); + _paContext = NULL; + + // Stop the threaded main loop + if (_paMainloop) { + LATE(pa_threaded_mainloop_stop)(_paMainloop); + } + + // Free the mainloop + if (_paMainloop) { + LATE(pa_threaded_mainloop_free)(_paMainloop); + } + + _paMainloop = NULL; + + RTC_LOG(LS_VERBOSE) << "PulseAudio terminated"; + + return 0; +} + +void AudioDeviceLinuxPulse::PaLock() { + LATE(pa_threaded_mainloop_lock)(_paMainloop); +} + +void AudioDeviceLinuxPulse::PaUnLock() { + LATE(pa_threaded_mainloop_unlock)(_paMainloop); +} + +void AudioDeviceLinuxPulse::WaitForOperationCompletion( + pa_operation* paOperation) const { + if (!paOperation) { + RTC_LOG(LS_ERROR) << "paOperation NULL in WaitForOperationCompletion"; + return; + } + + while (LATE(pa_operation_get_state)(paOperation) == PA_OPERATION_RUNNING) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + LATE(pa_operation_unref)(paOperation); +} + +// ============================================================================ +// Thread Methods +// ============================================================================ + +void AudioDeviceLinuxPulse::EnableWriteCallback() { + if (LATE(pa_stream_get_state)(_playStream) == PA_STREAM_READY) { + // May already have available space. Must check. + _tempBufferSpace = LATE(pa_stream_writable_size)(_playStream); + if (_tempBufferSpace > 0) { + // Yup, there is already space available, so if we register a + // write callback then it will not receive any event. So dispatch + // one ourself instead. + _timeEventPlay.Set(); + return; + } + } + + LATE(pa_stream_set_write_callback)(_playStream, &PaStreamWriteCallback, this); +} + +void AudioDeviceLinuxPulse::DisableWriteCallback() { + LATE(pa_stream_set_write_callback)(_playStream, NULL, NULL); +} + +void AudioDeviceLinuxPulse::PaStreamWriteCallback(pa_stream* /*unused*/, + size_t buffer_space, + void* pThis) { + static_cast(pThis)->PaStreamWriteCallbackHandler( + buffer_space); +} + +void AudioDeviceLinuxPulse::PaStreamWriteCallbackHandler(size_t bufferSpace) { + _tempBufferSpace = bufferSpace; + + // Since we write the data asynchronously on a different thread, we have + // to temporarily disable the write callback or else Pulse will call it + // continuously until we write the data. We re-enable it below. + DisableWriteCallback(); + _timeEventPlay.Set(); +} + +void AudioDeviceLinuxPulse::PaStreamUnderflowCallback(pa_stream* /*unused*/, + void* pThis) { + static_cast(pThis) + ->PaStreamUnderflowCallbackHandler(); +} + +void AudioDeviceLinuxPulse::PaStreamUnderflowCallbackHandler() { + RTC_LOG(LS_WARNING) << "Playout underflow"; + + if (_configuredLatencyPlay == WEBRTC_PA_NO_LATENCY_REQUIREMENTS) { + // We didn't configure a pa_buffer_attr before, so switching to + // one now would be questionable. + return; + } + + // Otherwise reconfigure the stream with a higher target latency. + + const pa_sample_spec* spec = LATE(pa_stream_get_sample_spec)(_playStream); + if (!spec) { + RTC_LOG(LS_ERROR) << "pa_stream_get_sample_spec()"; + return; + } + + size_t bytesPerSec = LATE(pa_bytes_per_second)(spec); + uint32_t newLatency = + _configuredLatencyPlay + bytesPerSec * + WEBRTC_PA_PLAYBACK_LATENCY_INCREMENT_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + // Set the play buffer attributes + _playBufferAttr.maxlength = newLatency; + _playBufferAttr.tlength = newLatency; + _playBufferAttr.minreq = newLatency / WEBRTC_PA_PLAYBACK_REQUEST_FACTOR; + _playBufferAttr.prebuf = _playBufferAttr.tlength - _playBufferAttr.minreq; + + pa_operation* op = LATE(pa_stream_set_buffer_attr)( + _playStream, &_playBufferAttr, NULL, NULL); + if (!op) { + RTC_LOG(LS_ERROR) << "pa_stream_set_buffer_attr()"; + return; + } + + // Don't need to wait for this to complete. + LATE(pa_operation_unref)(op); + + // Save the new latency in case we underflow again. + _configuredLatencyPlay = newLatency; +} + +void AudioDeviceLinuxPulse::EnableReadCallback() { + LATE(pa_stream_set_read_callback)(_recStream, &PaStreamReadCallback, this); +} + +void AudioDeviceLinuxPulse::DisableReadCallback() { + LATE(pa_stream_set_read_callback)(_recStream, NULL, NULL); +} + +void AudioDeviceLinuxPulse::PaStreamReadCallback(pa_stream* /*unused1*/, + size_t /*unused2*/, + void* pThis) { + static_cast(pThis)->PaStreamReadCallbackHandler(); +} + +void AudioDeviceLinuxPulse::PaStreamReadCallbackHandler() { + // We get the data pointer and size now in order to save one Lock/Unlock + // in the worker thread. + if (LATE(pa_stream_peek)(_recStream, &_tempSampleData, + &_tempSampleDataSize) != 0) { + RTC_LOG(LS_ERROR) << "Can't read data!"; + return; + } + + // Since we consume the data asynchronously on a different thread, we have + // to temporarily disable the read callback or else Pulse will call it + // continuously until we consume the data. We re-enable it below. + DisableReadCallback(); + _timeEventRec.Set(); +} + +void AudioDeviceLinuxPulse::PaStreamOverflowCallback(pa_stream* /*unused*/, + void* pThis) { + static_cast(pThis)->PaStreamOverflowCallbackHandler(); +} + +void AudioDeviceLinuxPulse::PaStreamOverflowCallbackHandler() { + RTC_LOG(LS_WARNING) << "Recording overflow"; +} + +int32_t AudioDeviceLinuxPulse::LatencyUsecs(pa_stream* stream) { + if (!WEBRTC_PA_REPORT_LATENCY) { + return 0; + } + + if (!stream) { + return 0; + } + + pa_usec_t latency; + int negative; + if (LATE(pa_stream_get_latency)(stream, &latency, &negative) != 0) { + RTC_LOG(LS_ERROR) << "Can't query latency"; + // We'd rather continue playout/capture with an incorrect delay than + // stop it altogether, so return a valid value. + return 0; + } + + if (negative) { + RTC_LOG(LS_VERBOSE) + << "warning: pa_stream_get_latency reported negative delay"; + + // The delay can be negative for monitoring streams if the captured + // samples haven't been played yet. In such a case, "latency" + // contains the magnitude, so we must negate it to get the real value. + int32_t tmpLatency = (int32_t)-latency; + if (tmpLatency < 0) { + // Make sure that we don't use a negative delay. + tmpLatency = 0; + } + + return tmpLatency; + } else { + return (int32_t)latency; + } +} + +int32_t AudioDeviceLinuxPulse::ReadRecordedData(const void* bufferData, + size_t bufferSize) + RTC_EXCLUSIVE_LOCKS_REQUIRED(_critSect) { + size_t size = bufferSize; + uint32_t numRecSamples = _recordBufferSize / (2 * _recChannels); + + // Account for the peeked data and the used data. + uint32_t recDelay = + (uint32_t)((LatencyUsecs(_recStream) / 1000) + + 10 * ((size + _recordBufferUsed) / _recordBufferSize)); + + _sndCardRecDelay = recDelay; + + if (_playStream) { + // Get the playout delay. + _sndCardPlayDelay = (uint32_t)(LatencyUsecs(_playStream) / 1000); + } + + if (_recordBufferUsed > 0) { + // Have to copy to the buffer until it is full. + size_t copy = _recordBufferSize - _recordBufferUsed; + if (size < copy) { + copy = size; + } + + memcpy(&_recBuffer[_recordBufferUsed], bufferData, copy); + _recordBufferUsed += copy; + bufferData = static_cast(bufferData) + copy; + size -= copy; + + if (_recordBufferUsed != _recordBufferSize) { + // Not enough data yet to pass to VoE. + return 0; + } + + // Provide data to VoiceEngine. + if (ProcessRecordedData(_recBuffer, numRecSamples, recDelay) == -1) { + // We have stopped recording. + return -1; + } + + _recordBufferUsed = 0; + } + + // Now process full 10ms sample sets directly from the input. + while (size >= _recordBufferSize) { + // Provide data to VoiceEngine. + if (ProcessRecordedData(static_cast(const_cast(bufferData)), + numRecSamples, recDelay) == -1) { + // We have stopped recording. + return -1; + } + + bufferData = static_cast(bufferData) + _recordBufferSize; + size -= _recordBufferSize; + + // We have consumed 10ms of data. + recDelay -= 10; + } + + // Now save any leftovers for later. + if (size > 0) { + memcpy(_recBuffer, bufferData, size); + _recordBufferUsed = size; + } + + return 0; +} + +int32_t AudioDeviceLinuxPulse::ProcessRecordedData(int8_t* bufferData, + uint32_t bufferSizeInSamples, + uint32_t recDelay) + RTC_EXCLUSIVE_LOCKS_REQUIRED(_critSect) { + uint32_t currentMicLevel(0); + uint32_t newMicLevel(0); + + _ptrAudioBuffer->SetRecordedBuffer(bufferData, bufferSizeInSamples); + + if (AGC()) { + // Store current mic level in the audio buffer if AGC is enabled + if (MicrophoneVolume(currentMicLevel) == 0) { + // This call does not affect the actual microphone volume + _ptrAudioBuffer->SetCurrentMicLevel(currentMicLevel); + } + } + + const uint32_t clockDrift(0); + // TODO(andrew): this is a temporary hack, to avoid non-causal far- and + // near-end signals at the AEC for PulseAudio. I think the system delay is + // being correctly calculated here, but for legacy reasons we add +10 ms + // to the value in the AEC. The real fix will be part of a larger + // investigation into managing system delay in the AEC. + if (recDelay > 10) + recDelay -= 10; + else + recDelay = 0; + _ptrAudioBuffer->SetVQEData(_sndCardPlayDelay, recDelay, clockDrift); + _ptrAudioBuffer->SetTypingStatus(KeyPressed()); + // Deliver recorded samples at specified sample rate, + // mic level etc. to the observer using callback. + UnLock(); + _ptrAudioBuffer->DeliverRecordedData(); + Lock(); + + // We have been unlocked - check the flag again. + if (!_recording) { + return -1; + } + + if (AGC()) { + newMicLevel = _ptrAudioBuffer->NewMicLevel(); + if (newMicLevel != 0) { + // The VQE will only deliver non-zero microphone levels when a + // change is needed. + // Set this new mic level (received from the observer as return + // value in the callback). + RTC_LOG(LS_VERBOSE) << "AGC change of volume: old=" << currentMicLevel + << " => new=" << newMicLevel; + if (SetMicrophoneVolume(newMicLevel) == -1) { + RTC_LOG(LS_WARNING) + << "the required modification of the microphone volume failed"; + } + } + } + + return 0; +} + +bool AudioDeviceLinuxPulse::PlayThreadFunc(void* pThis) { + return (static_cast(pThis)->PlayThreadProcess()); +} + +bool AudioDeviceLinuxPulse::RecThreadFunc(void* pThis) { + return (static_cast(pThis)->RecThreadProcess()); +} + +bool AudioDeviceLinuxPulse::PlayThreadProcess() { + switch (_timeEventPlay.Wait(1000)) { + case kEventSignaled: + break; + case kEventError: + RTC_LOG(LS_WARNING) << "EventWrapper::Wait() failed"; + return true; + case kEventTimeout: + return true; + } + + rtc::CritScope lock(&_critSect); + + if (_startPlay) { + RTC_LOG(LS_VERBOSE) << "_startPlay true, performing initial actions"; + + _startPlay = false; + _playDeviceName = NULL; + + // Set if not default device + if (_outputDeviceIndex > 0) { + // Get the playout device name + _playDeviceName = new char[kAdmMaxDeviceNameSize]; + _deviceIndex = _outputDeviceIndex; + PlayoutDevices(); + } + + // Start muted only supported on 0.9.11 and up + if (LATE(pa_context_get_protocol_version)(_paContext) >= + WEBRTC_PA_ADJUST_LATENCY_PROTOCOL_VERSION) { + // Get the currently saved speaker mute status + // and set the initial mute status accordingly + bool enabled(false); + _mixerManager.SpeakerMute(enabled); + if (enabled) { + _playStreamFlags |= PA_STREAM_START_MUTED; + } + } + + // Get the currently saved speaker volume + uint32_t volume = 0; + if (update_speaker_volume_at_startup_) + _mixerManager.SpeakerVolume(volume); + + PaLock(); + + // NULL gives PA the choice of startup volume. + pa_cvolume* ptr_cvolume = NULL; + if (update_speaker_volume_at_startup_) { + pa_cvolume cVolumes; + ptr_cvolume = &cVolumes; + + // Set the same volume for all channels + const pa_sample_spec* spec = LATE(pa_stream_get_sample_spec)(_playStream); + LATE(pa_cvolume_set)(&cVolumes, spec->channels, volume); + update_speaker_volume_at_startup_ = false; + } + + // Connect the stream to a sink + if (LATE(pa_stream_connect_playback)( + _playStream, _playDeviceName, &_playBufferAttr, + (pa_stream_flags_t)_playStreamFlags, ptr_cvolume, NULL) != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to connect play stream, err=" + << LATE(pa_context_errno)(_paContext); + } + + RTC_LOG(LS_VERBOSE) << "play stream connected"; + + // Wait for state change + while (LATE(pa_stream_get_state)(_playStream) != PA_STREAM_READY) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + RTC_LOG(LS_VERBOSE) << "play stream ready"; + + // We can now handle write callbacks + EnableWriteCallback(); + + PaUnLock(); + + // Clear device name + if (_playDeviceName) { + delete[] _playDeviceName; + _playDeviceName = NULL; + } + + _playing = true; + _playStartEvent.Set(); + + return true; + } + + if (_playing) { + if (!_recording) { + // Update the playout delay + _sndCardPlayDelay = (uint32_t)(LatencyUsecs(_playStream) / 1000); + } + + if (_playbackBufferUnused < _playbackBufferSize) { + size_t write = _playbackBufferSize - _playbackBufferUnused; + if (_tempBufferSpace < write) { + write = _tempBufferSpace; + } + + PaLock(); + if (LATE(pa_stream_write)( + _playStream, (void*)&_playBuffer[_playbackBufferUnused], write, + NULL, (int64_t)0, PA_SEEK_RELATIVE) != PA_OK) { + _writeErrors++; + if (_writeErrors > 10) { + RTC_LOG(LS_ERROR) << "Playout error: _writeErrors=" << _writeErrors + << ", error=" << LATE(pa_context_errno)(_paContext); + _writeErrors = 0; + } + } + PaUnLock(); + + _playbackBufferUnused += write; + _tempBufferSpace -= write; + } + + uint32_t numPlaySamples = _playbackBufferSize / (2 * _playChannels); + // Might have been reduced to zero by the above. + if (_tempBufferSpace > 0) { + // Ask for new PCM data to be played out using the + // AudioDeviceBuffer ensure that this callback is executed + // without taking the audio-thread lock. + UnLock(); + RTC_LOG(LS_VERBOSE) << "requesting data"; + uint32_t nSamples = _ptrAudioBuffer->RequestPlayoutData(numPlaySamples); + Lock(); + + // We have been unlocked - check the flag again. + if (!_playing) { + return true; + } + + nSamples = _ptrAudioBuffer->GetPlayoutData(_playBuffer); + if (nSamples != numPlaySamples) { + RTC_LOG(LS_ERROR) << "invalid number of output samples(" << nSamples + << ")"; + } + + size_t write = _playbackBufferSize; + if (_tempBufferSpace < write) { + write = _tempBufferSpace; + } + + RTC_LOG(LS_VERBOSE) << "will write"; + PaLock(); + if (LATE(pa_stream_write)(_playStream, (void*)&_playBuffer[0], write, + NULL, (int64_t)0, PA_SEEK_RELATIVE) != PA_OK) { + _writeErrors++; + if (_writeErrors > 10) { + RTC_LOG(LS_ERROR) << "Playout error: _writeErrors=" << _writeErrors + << ", error=" << LATE(pa_context_errno)(_paContext); + _writeErrors = 0; + } + } + PaUnLock(); + + _playbackBufferUnused = write; + } + + _tempBufferSpace = 0; + PaLock(); + EnableWriteCallback(); + PaUnLock(); + + } // _playing + + return true; +} + +bool AudioDeviceLinuxPulse::RecThreadProcess() { + switch (_timeEventRec.Wait(1000)) { + case kEventSignaled: + break; + case kEventError: + RTC_LOG(LS_WARNING) << "EventWrapper::Wait() failed"; + return true; + case kEventTimeout: + return true; + } + + rtc::CritScope lock(&_critSect); + + if (_startRec) { + RTC_LOG(LS_VERBOSE) << "_startRec true, performing initial actions"; + + _recDeviceName = NULL; + + // Set if not default device + if (_inputDeviceIndex > 0) { + // Get the recording device name + _recDeviceName = new char[kAdmMaxDeviceNameSize]; + _deviceIndex = _inputDeviceIndex; + RecordingDevices(); + } + + PaLock(); + + RTC_LOG(LS_VERBOSE) << "connecting stream"; + + // Connect the stream to a source + if (LATE(pa_stream_connect_record)( + _recStream, _recDeviceName, &_recBufferAttr, + (pa_stream_flags_t)_recStreamFlags) != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to connect rec stream, err=" + << LATE(pa_context_errno)(_paContext); + } + + RTC_LOG(LS_VERBOSE) << "connected"; + + // Wait for state change + while (LATE(pa_stream_get_state)(_recStream) != PA_STREAM_READY) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + RTC_LOG(LS_VERBOSE) << "done"; + + // We can now handle read callbacks + EnableReadCallback(); + + PaUnLock(); + + // Clear device name + if (_recDeviceName) { + delete[] _recDeviceName; + _recDeviceName = NULL; + } + + _startRec = false; + _recording = true; + _recStartEvent.Set(); + + return true; + } + + if (_recording) { + // Read data and provide it to VoiceEngine + if (ReadRecordedData(_tempSampleData, _tempSampleDataSize) == -1) { + return true; + } + + _tempSampleData = NULL; + _tempSampleDataSize = 0; + + PaLock(); + while (true) { + // Ack the last thing we read + if (LATE(pa_stream_drop)(_recStream) != 0) { + RTC_LOG(LS_WARNING) + << "failed to drop, err=" << LATE(pa_context_errno)(_paContext); + } + + if (LATE(pa_stream_readable_size)(_recStream) <= 0) { + // Then that was all the data + break; + } + + // Else more data. + const void* sampleData; + size_t sampleDataSize; + + if (LATE(pa_stream_peek)(_recStream, &sampleData, &sampleDataSize) != 0) { + RTC_LOG(LS_ERROR) << "RECORD_ERROR, error = " + << LATE(pa_context_errno)(_paContext); + break; + } + + _sndCardRecDelay = (uint32_t)(LatencyUsecs(_recStream) / 1000); + + // Drop lock for sigslot dispatch, which could take a while. + PaUnLock(); + // Read data and provide it to VoiceEngine + if (ReadRecordedData(sampleData, sampleDataSize) == -1) { + return true; + } + PaLock(); + + // Return to top of loop for the ack and the check for more data. + } + + EnableReadCallback(); + PaUnLock(); + + } // _recording + + return true; +} + +bool AudioDeviceLinuxPulse::KeyPressed() const { + char szKey[32]; + unsigned int i = 0; + char state = 0; + + if (!_XDisplay) + return false; + + // Check key map status + XQueryKeymap(_XDisplay, szKey); + + // A bit change in keymap means a key is pressed + for (i = 0; i < sizeof(szKey); i++) + state |= (szKey[i] ^ _oldKeyState[i]) & szKey[i]; + + // Save old state + memcpy((char*)_oldKeyState, (char*)szKey, sizeof(_oldKeyState)); + return (state != 0); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.h b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.h new file mode 100644 index 0000000000..73195c790f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_device_pulse_linux.h @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_DEVICE_PULSE_LINUX_H_ +#define AUDIO_DEVICE_AUDIO_DEVICE_PULSE_LINUX_H_ + +#include + +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/linux/audio_mixer_manager_pulse_linux.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/thread_checker.h" + +#include +#include + +// We define this flag if it's missing from our headers, because we want to be +// able to compile against old headers but still use PA_STREAM_ADJUST_LATENCY +// if run against a recent version of the library. +#ifndef PA_STREAM_ADJUST_LATENCY +#define PA_STREAM_ADJUST_LATENCY 0x2000U +#endif +#ifndef PA_STREAM_START_MUTED +#define PA_STREAM_START_MUTED 0x1000U +#endif + +// Set this constant to 0 to disable latency reading +const uint32_t WEBRTC_PA_REPORT_LATENCY = 1; + +// Constants from implementation by Tristan Schmelcher [tschmelcher@google.com] + +// First PulseAudio protocol version that supports PA_STREAM_ADJUST_LATENCY. +const uint32_t WEBRTC_PA_ADJUST_LATENCY_PROTOCOL_VERSION = 13; + +// Some timing constants for optimal operation. See +// https://tango.0pointer.de/pipermail/pulseaudio-discuss/2008-January/001170.html +// for a good explanation of some of the factors that go into this. + +// Playback. + +// For playback, there is a round-trip delay to fill the server-side playback +// buffer, so setting too low of a latency is a buffer underflow risk. We will +// automatically increase the latency if a buffer underflow does occur, but we +// also enforce a sane minimum at start-up time. Anything lower would be +// virtually guaranteed to underflow at least once, so there's no point in +// allowing lower latencies. +const uint32_t WEBRTC_PA_PLAYBACK_LATENCY_MINIMUM_MSECS = 20; + +// Every time a playback stream underflows, we will reconfigure it with target +// latency that is greater by this amount. +const uint32_t WEBRTC_PA_PLAYBACK_LATENCY_INCREMENT_MSECS = 20; + +// We also need to configure a suitable request size. Too small and we'd burn +// CPU from the overhead of transfering small amounts of data at once. Too large +// and the amount of data remaining in the buffer right before refilling it +// would be a buffer underflow risk. We set it to half of the buffer size. +const uint32_t WEBRTC_PA_PLAYBACK_REQUEST_FACTOR = 2; + +// Capture. + +// For capture, low latency is not a buffer overflow risk, but it makes us burn +// CPU from the overhead of transfering small amounts of data at once, so we set +// a recommended value that we use for the kLowLatency constant (but if the user +// explicitly requests something lower then we will honour it). +// 1ms takes about 6-7% CPU. 5ms takes about 5%. 10ms takes about 4.x%. +const uint32_t WEBRTC_PA_LOW_CAPTURE_LATENCY_MSECS = 10; + +// There is a round-trip delay to ack the data to the server, so the +// server-side buffer needs extra space to prevent buffer overflow. 20ms is +// sufficient, but there is no penalty to making it bigger, so we make it huge. +// (750ms is libpulse's default value for the _total_ buffer size in the +// kNoLatencyRequirements case.) +const uint32_t WEBRTC_PA_CAPTURE_BUFFER_EXTRA_MSECS = 750; + +const uint32_t WEBRTC_PA_MSECS_PER_SEC = 1000; + +// Init _configuredLatencyRec/Play to this value to disable latency requirements +const int32_t WEBRTC_PA_NO_LATENCY_REQUIREMENTS = -1; + +// Set this const to 1 to account for peeked and used data in latency calculation +const uint32_t WEBRTC_PA_CAPTURE_BUFFER_LATENCY_ADJUSTMENT = 0; + +namespace webrtc +{ +class EventWrapper; + +class AudioDeviceLinuxPulse: public AudioDeviceGeneric +{ +public: + AudioDeviceLinuxPulse(); + virtual ~AudioDeviceLinuxPulse(); + + // Retrieve the currently utilized audio layer + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override; + + // Main initializaton and termination + InitStatus Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + // Device enumeration + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + + // Device selection + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override; + + // Audio transport initialization + int32_t PlayoutIsAvailable(bool& available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool& available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + // Audio transport control + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // Microphone Automatic Gain Control (AGC) + int32_t SetAGC(bool enable) override; + bool AGC() const override; + + // Audio mixer initialization + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Speaker volume controls + int32_t SpeakerVolumeIsAvailable(bool& available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; + + // Microphone volume controls + int32_t MicrophoneVolumeIsAvailable(bool& available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; + + // Speaker mute control + int32_t SpeakerMuteIsAvailable(bool& available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool& enabled) const override; + + // Microphone mute control + int32_t MicrophoneMuteIsAvailable(bool& available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool& enabled) const override; + + // Stereo support + int32_t StereoPlayoutIsAvailable(bool& available) override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool& enabled) const override; + + // Delay information and control + int32_t PlayoutDelay(uint16_t& delayMS) const override; + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; + +private: + void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION(_critSect) { _critSect.Enter(); } + void UnLock() RTC_UNLOCK_FUNCTION(_critSect) { _critSect.Leave(); } + void WaitForOperationCompletion(pa_operation* paOperation) const; + void WaitForSuccess(pa_operation* paOperation) const; + + bool KeyPressed() const; + + static void PaContextStateCallback(pa_context* c, void* pThis); + static void PaSinkInfoCallback(pa_context* c, + const pa_sink_info* i, + int eol, + void* pThis); + static void PaSourceInfoCallback(pa_context* c, + const pa_source_info* i, + int eol, + void* pThis); + static void PaServerInfoCallback(pa_context* c, + const pa_server_info* i, + void* pThis); + static void PaStreamStateCallback(pa_stream* p, void* pThis); + void PaContextStateCallbackHandler(pa_context* c); + void PaSinkInfoCallbackHandler(const pa_sink_info* i, int eol); + void PaSourceInfoCallbackHandler(const pa_source_info* i, int eol); + void PaServerInfoCallbackHandler(const pa_server_info* i); + void PaStreamStateCallbackHandler(pa_stream* p); + + void EnableWriteCallback(); + void DisableWriteCallback(); + static void PaStreamWriteCallback(pa_stream* unused, + size_t buffer_space, + void* pThis); + void PaStreamWriteCallbackHandler(size_t buffer_space); + static void PaStreamUnderflowCallback(pa_stream* unused, void* pThis); + void PaStreamUnderflowCallbackHandler(); + void EnableReadCallback(); + void DisableReadCallback(); + static void PaStreamReadCallback(pa_stream* unused1, + size_t unused2, + void* pThis); + void PaStreamReadCallbackHandler(); + static void PaStreamOverflowCallback(pa_stream* unused, void* pThis); + void PaStreamOverflowCallbackHandler(); + int32_t LatencyUsecs(pa_stream* stream); + int32_t ReadRecordedData(const void* bufferData, size_t bufferSize); + int32_t ProcessRecordedData(int8_t* bufferData, + uint32_t bufferSizeInSamples, + uint32_t recDelay); + + int32_t CheckPulseAudioVersion(); + int32_t InitSamplingFrequency(); + int32_t GetDefaultDeviceInfo(bool recDevice, char* name, uint16_t& index); + int32_t InitPulseAudio(); + int32_t TerminatePulseAudio(); + + void PaLock(); + void PaUnLock(); + + static bool RecThreadFunc(void*); + static bool PlayThreadFunc(void*); + bool RecThreadProcess(); + bool PlayThreadProcess(); + + AudioDeviceBuffer* _ptrAudioBuffer; + + rtc::CriticalSection _critSect; + EventWrapper& _timeEventRec; + EventWrapper& _timeEventPlay; + EventWrapper& _recStartEvent; + EventWrapper& _playStartEvent; + + // TODO(pbos): Remove unique_ptr and use directly without resetting. + std::unique_ptr _ptrThreadPlay; + std::unique_ptr _ptrThreadRec; + + AudioMixerManagerLinuxPulse _mixerManager; + + uint16_t _inputDeviceIndex; + uint16_t _outputDeviceIndex; + bool _inputDeviceIsSpecified; + bool _outputDeviceIsSpecified; + + int sample_rate_hz_; + uint8_t _recChannels; + uint8_t _playChannels; + + // Stores thread ID in constructor. + // We can then use ThreadChecker::CalledOnValidThread() to ensure that + // other methods are called from the same thread. + // Currently only does RTC_DCHECK(thread_checker_.CalledOnValidThread()). + rtc::ThreadChecker thread_checker_; + + bool _initialized; + bool _recording; + bool _playing; + bool _recIsInitialized; + bool _playIsInitialized; + bool _startRec; + bool _stopRec; + bool _startPlay; + bool _stopPlay; + bool _AGC; + bool update_speaker_volume_at_startup_; + + uint32_t _sndCardPlayDelay; + uint32_t _sndCardRecDelay; + + int32_t _writeErrors; + + uint16_t _deviceIndex; + int16_t _numPlayDevices; + int16_t _numRecDevices; + char* _playDeviceName; + char* _recDeviceName; + char* _playDisplayDeviceName; + char* _recDisplayDeviceName; + char _paServerVersion[32]; + + int8_t* _playBuffer; + size_t _playbackBufferSize; + size_t _playbackBufferUnused; + size_t _tempBufferSpace; + int8_t* _recBuffer; + size_t _recordBufferSize; + size_t _recordBufferUsed; + const void* _tempSampleData; + size_t _tempSampleDataSize; + int32_t _configuredLatencyPlay; + int32_t _configuredLatencyRec; + + // PulseAudio + uint16_t _paDeviceIndex; + bool _paStateChanged; + + pa_threaded_mainloop* _paMainloop; + pa_mainloop_api* _paMainloopApi; + pa_context* _paContext; + + pa_stream* _recStream; + pa_stream* _playStream; + uint32_t _recStreamFlags; + uint32_t _playStreamFlags; + pa_buffer_attr _playBufferAttr; + pa_buffer_attr _recBufferAttr; + + char _oldKeyState[32]; + Display* _XDisplay; +}; + +} + +#endif // MODULES_AUDIO_DEVICE_MAIN_SOURCE_LINUX_AUDIO_DEVICE_PULSE_LINUX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.cc b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.cc new file mode 100644 index 0000000000..c1d67cfa19 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.cc @@ -0,0 +1,972 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_device/linux/audio_mixer_manager_alsa_linux.h" +#include "rtc_base/logging.h" + +extern webrtc::adm_linux_alsa::AlsaSymbolTable AlsaSymbolTable; + +// Accesses ALSA functions through our late-binding symbol table instead of +// directly. This way we don't have to link to libalsa, which means our binary +// will work on systems that don't have it. +#define LATE(sym) \ + LATESYM_GET(webrtc::adm_linux_alsa::AlsaSymbolTable, &AlsaSymbolTable, sym) + +namespace webrtc { + +AudioMixerManagerLinuxALSA::AudioMixerManagerLinuxALSA() + : _outputMixerHandle(NULL), + _inputMixerHandle(NULL), + _outputMixerElement(NULL), + _inputMixerElement(NULL) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " created"; + + memset(_outputMixerStr, 0, kAdmMaxDeviceNameSize); + memset(_inputMixerStr, 0, kAdmMaxDeviceNameSize); +} + +AudioMixerManagerLinuxALSA::~AudioMixerManagerLinuxALSA() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " destroyed"; + Close(); +} + +// ============================================================================ +// PUBLIC METHODS +// ============================================================================ + +int32_t AudioMixerManagerLinuxALSA::Close() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + rtc::CritScope lock(&_critSect); + + CloseSpeaker(); + CloseMicrophone(); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::CloseSpeaker() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + rtc::CritScope lock(&_critSect); + + int errVal = 0; + + if (_outputMixerHandle != NULL) { + RTC_LOG(LS_VERBOSE) << "Closing playout mixer"; + LATE(snd_mixer_free)(_outputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error freeing playout mixer: " + << LATE(snd_strerror)(errVal); + } + errVal = LATE(snd_mixer_detach)(_outputMixerHandle, _outputMixerStr); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error detaching playout mixer: " + << LATE(snd_strerror)(errVal); + } + errVal = LATE(snd_mixer_close)(_outputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error snd_mixer_close(handleMixer) errVal=" + << errVal; + } + _outputMixerHandle = NULL; + _outputMixerElement = NULL; + } + memset(_outputMixerStr, 0, kAdmMaxDeviceNameSize); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::CloseMicrophone() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + rtc::CritScope lock(&_critSect); + + int errVal = 0; + + if (_inputMixerHandle != NULL) { + RTC_LOG(LS_VERBOSE) << "Closing record mixer"; + + LATE(snd_mixer_free)(_inputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error freeing record mixer: " + << LATE(snd_strerror)(errVal); + } + RTC_LOG(LS_VERBOSE) << "Closing record mixer 2"; + + errVal = LATE(snd_mixer_detach)(_inputMixerHandle, _inputMixerStr); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error detaching record mixer: " + << LATE(snd_strerror)(errVal); + } + RTC_LOG(LS_VERBOSE) << "Closing record mixer 3"; + + errVal = LATE(snd_mixer_close)(_inputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error snd_mixer_close(handleMixer) errVal=" + << errVal; + } + + RTC_LOG(LS_VERBOSE) << "Closing record mixer 4"; + _inputMixerHandle = NULL; + _inputMixerElement = NULL; + } + memset(_inputMixerStr, 0, kAdmMaxDeviceNameSize); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::OpenSpeaker(char* deviceName) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxALSA::OpenSpeaker(name=" + << deviceName << ")"; + + rtc::CritScope lock(&_critSect); + + int errVal = 0; + + // Close any existing output mixer handle + // + if (_outputMixerHandle != NULL) { + RTC_LOG(LS_VERBOSE) << "Closing playout mixer"; + + LATE(snd_mixer_free)(_outputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error freeing playout mixer: " + << LATE(snd_strerror)(errVal); + } + errVal = LATE(snd_mixer_detach)(_outputMixerHandle, _outputMixerStr); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error detaching playout mixer: " + << LATE(snd_strerror)(errVal); + } + errVal = LATE(snd_mixer_close)(_outputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error snd_mixer_close(handleMixer) errVal=" + << errVal; + } + } + _outputMixerHandle = NULL; + _outputMixerElement = NULL; + + errVal = LATE(snd_mixer_open)(&_outputMixerHandle, 0); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_mixer_open(&_outputMixerHandle, 0) - error"; + return -1; + } + + char controlName[kAdmMaxDeviceNameSize] = {0}; + GetControlName(controlName, deviceName); + + RTC_LOG(LS_VERBOSE) << "snd_mixer_attach(_outputMixerHandle, " << controlName + << ")"; + + errVal = LATE(snd_mixer_attach)(_outputMixerHandle, controlName); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_mixer_attach(_outputMixerHandle, " << controlName + << ") error: " << LATE(snd_strerror)(errVal); + _outputMixerHandle = NULL; + return -1; + } + strcpy(_outputMixerStr, controlName); + + errVal = LATE(snd_mixer_selem_register)(_outputMixerHandle, NULL, NULL); + if (errVal < 0) { + RTC_LOG(LS_ERROR) + << "snd_mixer_selem_register(_outputMixerHandle, NULL, NULL), " + << "error: " << LATE(snd_strerror)(errVal); + _outputMixerHandle = NULL; + return -1; + } + + // Load and find the proper mixer element + if (LoadSpeakerMixerElement() < 0) { + return -1; + } + + if (_outputMixerHandle != NULL) { + RTC_LOG(LS_VERBOSE) << "the output mixer device is now open (" + << _outputMixerHandle << ")"; + } + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::OpenMicrophone(char* deviceName) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxALSA::OpenMicrophone(name=" + << deviceName << ")"; + + rtc::CritScope lock(&_critSect); + + int errVal = 0; + + // Close any existing input mixer handle + // + if (_inputMixerHandle != NULL) { + RTC_LOG(LS_VERBOSE) << "Closing record mixer"; + + LATE(snd_mixer_free)(_inputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error freeing record mixer: " + << LATE(snd_strerror)(errVal); + } + RTC_LOG(LS_VERBOSE) << "Closing record mixer"; + + errVal = LATE(snd_mixer_detach)(_inputMixerHandle, _inputMixerStr); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error detaching record mixer: " + << LATE(snd_strerror)(errVal); + } + RTC_LOG(LS_VERBOSE) << "Closing record mixer"; + + errVal = LATE(snd_mixer_close)(_inputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error snd_mixer_close(handleMixer) errVal=" + << errVal; + } + RTC_LOG(LS_VERBOSE) << "Closing record mixer"; + } + _inputMixerHandle = NULL; + _inputMixerElement = NULL; + + errVal = LATE(snd_mixer_open)(&_inputMixerHandle, 0); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_mixer_open(&_inputMixerHandle, 0) - error"; + return -1; + } + + char controlName[kAdmMaxDeviceNameSize] = {0}; + GetControlName(controlName, deviceName); + + RTC_LOG(LS_VERBOSE) << "snd_mixer_attach(_inputMixerHandle, " << controlName + << ")"; + + errVal = LATE(snd_mixer_attach)(_inputMixerHandle, controlName); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_mixer_attach(_inputMixerHandle, " << controlName + << ") error: " << LATE(snd_strerror)(errVal); + + _inputMixerHandle = NULL; + return -1; + } + strcpy(_inputMixerStr, controlName); + + errVal = LATE(snd_mixer_selem_register)(_inputMixerHandle, NULL, NULL); + if (errVal < 0) { + RTC_LOG(LS_ERROR) + << "snd_mixer_selem_register(_inputMixerHandle, NULL, NULL), " + << "error: " << LATE(snd_strerror)(errVal); + + _inputMixerHandle = NULL; + return -1; + } + // Load and find the proper mixer element + if (LoadMicMixerElement() < 0) { + return -1; + } + + if (_inputMixerHandle != NULL) { + RTC_LOG(LS_VERBOSE) << "the input mixer device is now open (" + << _inputMixerHandle << ")"; + } + + return 0; +} + +bool AudioMixerManagerLinuxALSA::SpeakerIsInitialized() const { + RTC_LOG(LS_INFO) << __FUNCTION__; + + return (_outputMixerHandle != NULL); +} + +bool AudioMixerManagerLinuxALSA::MicrophoneIsInitialized() const { + RTC_LOG(LS_INFO) << __FUNCTION__; + + return (_inputMixerHandle != NULL); +} + +int32_t AudioMixerManagerLinuxALSA::SetSpeakerVolume(uint32_t volume) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxALSA::SetSpeakerVolume(volume=" + << volume << ")"; + + rtc::CritScope lock(&_critSect); + + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + int errVal = LATE(snd_mixer_selem_set_playback_volume_all)( + _outputMixerElement, volume); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error changing master volume: " + << LATE(snd_strerror)(errVal); + return -1; + } + + return (0); +} + +int32_t AudioMixerManagerLinuxALSA::SpeakerVolume(uint32_t& volume) const { + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + long int vol(0); + + int errVal = LATE(snd_mixer_selem_get_playback_volume)( + _outputMixerElement, (snd_mixer_selem_channel_id_t)0, &vol); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error getting outputvolume: " + << LATE(snd_strerror)(errVal); + return -1; + } + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxALSA::SpeakerVolume() => vol=" + << vol; + + volume = static_cast(vol); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::MaxSpeakerVolume( + uint32_t& maxVolume) const { + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avilable output mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + int errVal = LATE(snd_mixer_selem_get_playback_volume_range)( + _outputMixerElement, &minVol, &maxVol); + + RTC_LOG(LS_VERBOSE) << "Playout hardware volume range, min: " << minVol + << ", max: " << maxVol; + + if (maxVol <= minVol) { + RTC_LOG(LS_ERROR) << "Error getting get_playback_volume_range: " + << LATE(snd_strerror)(errVal); + } + + maxVolume = static_cast(maxVol); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::MinSpeakerVolume( + uint32_t& minVolume) const { + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + int errVal = LATE(snd_mixer_selem_get_playback_volume_range)( + _outputMixerElement, &minVol, &maxVol); + + RTC_LOG(LS_VERBOSE) << "Playout hardware volume range, min: " << minVol + << ", max: " << maxVol; + + if (maxVol <= minVol) { + RTC_LOG(LS_ERROR) << "Error getting get_playback_volume_range: " + << LATE(snd_strerror)(errVal); + } + + minVolume = static_cast(minVol); + + return 0; +} + +// TL: Have done testnig with these but they don't seem reliable and +// they were therefore not added +/* + // ---------------------------------------------------------------------------- + // SetMaxSpeakerVolume + // ---------------------------------------------------------------------------- + + int32_t AudioMixerManagerLinuxALSA::SetMaxSpeakerVolume( + uint32_t maxVolume) + { + + if (_outputMixerElement == NULL) + { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + int errVal = snd_mixer_selem_get_playback_volume_range( + _outputMixerElement, &minVol, &maxVol); + if ((maxVol <= minVol) || (errVal != 0)) + { + RTC_LOG(LS_WARNING) << "Error getting playback volume range: " + << snd_strerror(errVal); + } + + maxVol = maxVolume; + errVal = snd_mixer_selem_set_playback_volume_range( + _outputMixerElement, minVol, maxVol); + RTC_LOG(LS_VERBOSE) << "Playout hardware volume range, min: " << minVol + << ", max: " << maxVol; + if (errVal != 0) + { + RTC_LOG(LS_ERROR) << "Error setting playback volume range: " + << snd_strerror(errVal); + return -1; + } + + return 0; + } + + // ---------------------------------------------------------------------------- + // SetMinSpeakerVolume + // ---------------------------------------------------------------------------- + + int32_t AudioMixerManagerLinuxALSA::SetMinSpeakerVolume( + uint32_t minVolume) + { + + if (_outputMixerElement == NULL) + { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + int errVal = snd_mixer_selem_get_playback_volume_range( + _outputMixerElement, &minVol, &maxVol); + if ((maxVol <= minVol) || (errVal != 0)) + { + RTC_LOG(LS_WARNING) << "Error getting playback volume range: " + << snd_strerror(errVal); + } + + minVol = minVolume; + errVal = snd_mixer_selem_set_playback_volume_range( + _outputMixerElement, minVol, maxVol); + RTC_LOG(LS_VERBOSE) << "Playout hardware volume range, min: " << minVol + << ", max: " << maxVol; + if (errVal != 0) + { + RTC_LOG(LS_ERROR) << "Error setting playback volume range: " + << snd_strerror(errVal); + return -1; + } + + return 0; + } + */ + +int32_t AudioMixerManagerLinuxALSA::SpeakerVolumeIsAvailable(bool& available) { + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + available = LATE(snd_mixer_selem_has_playback_volume)(_outputMixerElement); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::SpeakerMuteIsAvailable(bool& available) { + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + available = LATE(snd_mixer_selem_has_playback_switch)(_outputMixerElement); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::SetSpeakerMute(bool enable) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxALSA::SetSpeakerMute(enable=" + << enable << ")"; + + rtc::CritScope lock(&_critSect); + + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + // Ensure that the selected speaker destination has a valid mute control. + bool available(false); + SpeakerMuteIsAvailable(available); + if (!available) { + RTC_LOG(LS_WARNING) << "it is not possible to mute the speaker"; + return -1; + } + + // Note value = 0 (off) means muted + int errVal = LATE(snd_mixer_selem_set_playback_switch_all)( + _outputMixerElement, !enable); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error setting playback switch: " + << LATE(snd_strerror)(errVal); + return -1; + } + + return (0); +} + +int32_t AudioMixerManagerLinuxALSA::SpeakerMute(bool& enabled) const { + if (_outputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable output mixer exists"; + return -1; + } + + // Ensure that the selected speaker destination has a valid mute control. + bool available = + LATE(snd_mixer_selem_has_playback_switch)(_outputMixerElement); + if (!available) { + RTC_LOG(LS_WARNING) << "it is not possible to mute the speaker"; + return -1; + } + + int value(false); + + // Retrieve one boolean control value for a specified mute-control + // + int errVal = LATE(snd_mixer_selem_get_playback_switch)( + _outputMixerElement, (snd_mixer_selem_channel_id_t)0, &value); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error getting playback switch: " + << LATE(snd_strerror)(errVal); + return -1; + } + + // Note value = 0 (off) means muted + enabled = (bool)!value; + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::MicrophoneMuteIsAvailable(bool& available) { + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer element exists"; + return -1; + } + + available = LATE(snd_mixer_selem_has_capture_switch)(_inputMixerElement); + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::SetMicrophoneMute(bool enable) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxALSA::SetMicrophoneMute(enable=" + << enable << ")"; + + rtc::CritScope lock(&_critSect); + + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer element exists"; + return -1; + } + + // Ensure that the selected microphone destination has a valid mute control. + bool available(false); + MicrophoneMuteIsAvailable(available); + if (!available) { + RTC_LOG(LS_WARNING) << "it is not possible to mute the microphone"; + return -1; + } + + // Note value = 0 (off) means muted + int errVal = + LATE(snd_mixer_selem_set_capture_switch_all)(_inputMixerElement, !enable); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error setting capture switch: " + << LATE(snd_strerror)(errVal); + return -1; + } + + return (0); +} + +int32_t AudioMixerManagerLinuxALSA::MicrophoneMute(bool& enabled) const { + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer exists"; + return -1; + } + + // Ensure that the selected microphone destination has a valid mute control. + bool available = LATE(snd_mixer_selem_has_capture_switch)(_inputMixerElement); + if (!available) { + RTC_LOG(LS_WARNING) << "it is not possible to mute the microphone"; + return -1; + } + + int value(false); + + // Retrieve one boolean control value for a specified mute-control + // + int errVal = LATE(snd_mixer_selem_get_capture_switch)( + _inputMixerElement, (snd_mixer_selem_channel_id_t)0, &value); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error getting capture switch: " + << LATE(snd_strerror)(errVal); + return -1; + } + + // Note value = 0 (off) means muted + enabled = (bool)!value; + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::MicrophoneVolumeIsAvailable( + bool& available) { + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer element exists"; + return -1; + } + + available = LATE(snd_mixer_selem_has_capture_volume)(_inputMixerElement); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::SetMicrophoneVolume(uint32_t volume) { + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxALSA::SetMicrophoneVolume(volume=" << volume + << ")"; + + rtc::CritScope lock(&_critSect); + + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer element exists"; + return -1; + } + + int errVal = + LATE(snd_mixer_selem_set_capture_volume_all)(_inputMixerElement, volume); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error changing microphone volume: " + << LATE(snd_strerror)(errVal); + return -1; + } + + return (0); +} + +// TL: Have done testnig with these but they don't seem reliable and +// they were therefore not added +/* + // ---------------------------------------------------------------------------- + // SetMaxMicrophoneVolume + // ---------------------------------------------------------------------------- + + int32_t AudioMixerManagerLinuxALSA::SetMaxMicrophoneVolume( + uint32_t maxVolume) + { + + if (_inputMixerElement == NULL) + { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + int errVal = snd_mixer_selem_get_capture_volume_range(_inputMixerElement, + &minVol, &maxVol); + if ((maxVol <= minVol) || (errVal != 0)) + { + RTC_LOG(LS_WARNING) << "Error getting capture volume range: " + << snd_strerror(errVal); + } + + maxVol = (long int)maxVolume; + printf("min %d max %d", minVol, maxVol); + errVal = snd_mixer_selem_set_capture_volume_range(_inputMixerElement, minVol, + maxVol); RTC_LOG(LS_VERBOSE) << "Capture hardware volume range, min: " << + minVol + << ", max: " << maxVol; + if (errVal != 0) + { + RTC_LOG(LS_ERROR) << "Error setting capture volume range: " + << snd_strerror(errVal); + return -1; + } + + return 0; + } + + // ---------------------------------------------------------------------------- + // SetMinMicrophoneVolume + // ---------------------------------------------------------------------------- + + int32_t AudioMixerManagerLinuxALSA::SetMinMicrophoneVolume( + uint32_t minVolume) + { + + if (_inputMixerElement == NULL) + { + RTC_LOG(LS_WARNING) << "no avaliable output mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + int errVal = snd_mixer_selem_get_capture_volume_range( + _inputMixerElement, &minVol, &maxVol); + if (maxVol <= minVol) + { + //maxVol = 255; + RTC_LOG(LS_WARNING) << "Error getting capture volume range: " + << snd_strerror(errVal); + } + + printf("min %d max %d", minVol, maxVol); + minVol = (long int)minVolume; + errVal = snd_mixer_selem_set_capture_volume_range( + _inputMixerElement, minVol, maxVol); + RTC_LOG(LS_VERBOSE) << "Capture hardware volume range, min: " << minVol + << ", max: " << maxVol; + if (errVal != 0) + { + RTC_LOG(LS_ERROR) << "Error setting capture volume range: " + << snd_strerror(errVal); + return -1; + } + + return 0; + } + */ + +int32_t AudioMixerManagerLinuxALSA::MicrophoneVolume(uint32_t& volume) const { + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer element exists"; + return -1; + } + + long int vol(0); + + int errVal = LATE(snd_mixer_selem_get_capture_volume)( + _inputMixerElement, (snd_mixer_selem_channel_id_t)0, &vol); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "Error getting inputvolume: " + << LATE(snd_strerror)(errVal); + return -1; + } + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxALSA::MicrophoneVolume() => vol=" << vol; + + volume = static_cast(vol); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::MaxMicrophoneVolume( + uint32_t& maxVolume) const { + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + // check if we have mic volume at all + if (!LATE(snd_mixer_selem_has_capture_volume)(_inputMixerElement)) { + RTC_LOG(LS_ERROR) << "No microphone volume available"; + return -1; + } + + int errVal = LATE(snd_mixer_selem_get_capture_volume_range)( + _inputMixerElement, &minVol, &maxVol); + + RTC_LOG(LS_VERBOSE) << "Microphone hardware volume range, min: " << minVol + << ", max: " << maxVol; + if (maxVol <= minVol) { + RTC_LOG(LS_ERROR) << "Error getting microphone volume range: " + << LATE(snd_strerror)(errVal); + } + + maxVolume = static_cast(maxVol); + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::MinMicrophoneVolume( + uint32_t& minVolume) const { + if (_inputMixerElement == NULL) { + RTC_LOG(LS_WARNING) << "no avaliable input mixer element exists"; + return -1; + } + + long int minVol(0); + long int maxVol(0); + + int errVal = LATE(snd_mixer_selem_get_capture_volume_range)( + _inputMixerElement, &minVol, &maxVol); + + RTC_LOG(LS_VERBOSE) << "Microphone hardware volume range, min: " << minVol + << ", max: " << maxVol; + if (maxVol <= minVol) { + RTC_LOG(LS_ERROR) << "Error getting microphone volume range: " + << LATE(snd_strerror)(errVal); + } + + minVolume = static_cast(minVol); + + return 0; +} + +// ============================================================================ +// Private Methods +// ============================================================================ + +int32_t AudioMixerManagerLinuxALSA::LoadMicMixerElement() const { + int errVal = LATE(snd_mixer_load)(_inputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_mixer_load(_inputMixerHandle), error: " + << LATE(snd_strerror)(errVal); + _inputMixerHandle = NULL; + return -1; + } + + snd_mixer_elem_t* elem = NULL; + snd_mixer_elem_t* micElem = NULL; + unsigned mixerIdx = 0; + const char* selemName = NULL; + + // Find and store handles to the right mixer elements + for (elem = LATE(snd_mixer_first_elem)(_inputMixerHandle); elem; + elem = LATE(snd_mixer_elem_next)(elem), mixerIdx++) { + if (LATE(snd_mixer_selem_is_active)(elem)) { + selemName = LATE(snd_mixer_selem_get_name)(elem); + if (strcmp(selemName, "Capture") == 0) // "Capture", "Mic" + { + _inputMixerElement = elem; + RTC_LOG(LS_VERBOSE) << "Capture element set"; + } else if (strcmp(selemName, "Mic") == 0) { + micElem = elem; + RTC_LOG(LS_VERBOSE) << "Mic element found"; + } + } + + if (_inputMixerElement) { + // Use the first Capture element that is found + // The second one may not work + break; + } + } + + if (_inputMixerElement == NULL) { + // We didn't find a Capture handle, use Mic. + if (micElem != NULL) { + _inputMixerElement = micElem; + RTC_LOG(LS_VERBOSE) << "Using Mic as capture volume."; + } else { + _inputMixerElement = NULL; + RTC_LOG(LS_ERROR) << "Could not find capture volume on the mixer."; + + return -1; + } + } + + return 0; +} + +int32_t AudioMixerManagerLinuxALSA::LoadSpeakerMixerElement() const { + int errVal = LATE(snd_mixer_load)(_outputMixerHandle); + if (errVal < 0) { + RTC_LOG(LS_ERROR) << "snd_mixer_load(_outputMixerHandle), error: " + << LATE(snd_strerror)(errVal); + _outputMixerHandle = NULL; + return -1; + } + + snd_mixer_elem_t* elem = NULL; + snd_mixer_elem_t* masterElem = NULL; + snd_mixer_elem_t* speakerElem = NULL; + unsigned mixerIdx = 0; + const char* selemName = NULL; + + // Find and store handles to the right mixer elements + for (elem = LATE(snd_mixer_first_elem)(_outputMixerHandle); elem; + elem = LATE(snd_mixer_elem_next)(elem), mixerIdx++) { + if (LATE(snd_mixer_selem_is_active)(elem)) { + selemName = LATE(snd_mixer_selem_get_name)(elem); + RTC_LOG(LS_VERBOSE) << "snd_mixer_selem_get_name " << mixerIdx << ": " + << selemName << " =" << elem; + + // "Master", "PCM", "Wave", "Master Mono", "PC Speaker", "PCM", "Wave" + if (strcmp(selemName, "PCM") == 0) { + _outputMixerElement = elem; + RTC_LOG(LS_VERBOSE) << "PCM element set"; + } else if (strcmp(selemName, "Master") == 0) { + masterElem = elem; + RTC_LOG(LS_VERBOSE) << "Master element found"; + } else if (strcmp(selemName, "Speaker") == 0) { + speakerElem = elem; + RTC_LOG(LS_VERBOSE) << "Speaker element found"; + } + } + + if (_outputMixerElement) { + // We have found the element we want + break; + } + } + + // If we didn't find a PCM Handle, use Master or Speaker + if (_outputMixerElement == NULL) { + if (masterElem != NULL) { + _outputMixerElement = masterElem; + RTC_LOG(LS_VERBOSE) << "Using Master as output volume."; + } else if (speakerElem != NULL) { + _outputMixerElement = speakerElem; + RTC_LOG(LS_VERBOSE) << "Using Speaker as output volume."; + } else { + _outputMixerElement = NULL; + RTC_LOG(LS_ERROR) << "Could not find output volume in the mixer."; + return -1; + } + } + + return 0; +} + +void AudioMixerManagerLinuxALSA::GetControlName(char* controlName, + char* deviceName) const { + // Example + // deviceName: "front:CARD=Intel,DEV=0" + // controlName: "hw:CARD=Intel" + char* pos1 = strchr(deviceName, ':'); + char* pos2 = strchr(deviceName, ','); + if (!pos2) { + // Can also be default:CARD=Intel + pos2 = &deviceName[strlen(deviceName)]; + } + if (pos1 && pos2) { + strcpy(controlName, "hw"); + int nChar = (int)(pos2 - pos1); + strncpy(&controlName[2], pos1, nChar); + controlName[2 + nChar] = '\0'; + } else { + strcpy(controlName, deviceName); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.h b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.h new file mode 100644 index 0000000000..61de607cf7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_alsa_linux.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_MIXER_MANAGER_ALSA_LINUX_H_ +#define AUDIO_DEVICE_AUDIO_MIXER_MANAGER_ALSA_LINUX_H_ + +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/linux/alsasymboltable_linux.h" +#include "rtc_base/criticalsection.h" +#include "typedefs.h" // NOLINT(build/include) + +#include + +namespace webrtc +{ + +class AudioMixerManagerLinuxALSA +{ +public: + int32_t OpenSpeaker(char* deviceName); + int32_t OpenMicrophone(char* deviceName); + int32_t SetSpeakerVolume(uint32_t volume); + int32_t SpeakerVolume(uint32_t& volume) const; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const; + int32_t MinSpeakerVolume(uint32_t& minVolume) const; + int32_t SpeakerVolumeIsAvailable(bool& available); + int32_t SpeakerMuteIsAvailable(bool& available); + int32_t SetSpeakerMute(bool enable); + int32_t SpeakerMute(bool& enabled) const; + int32_t MicrophoneMuteIsAvailable(bool& available); + int32_t SetMicrophoneMute(bool enable); + int32_t MicrophoneMute(bool& enabled) const; + int32_t MicrophoneVolumeIsAvailable(bool& available); + int32_t SetMicrophoneVolume(uint32_t volume); + int32_t MicrophoneVolume(uint32_t& volume) const; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const; + int32_t Close(); + int32_t CloseSpeaker(); + int32_t CloseMicrophone(); + bool SpeakerIsInitialized() const; + bool MicrophoneIsInitialized() const; + +public: + AudioMixerManagerLinuxALSA(); + ~AudioMixerManagerLinuxALSA(); + +private: + int32_t LoadMicMixerElement() const; + int32_t LoadSpeakerMixerElement() const; + void GetControlName(char *controlName, char* deviceName) const; + +private: + rtc::CriticalSection _critSect; + mutable snd_mixer_t* _outputMixerHandle; + char _outputMixerStr[kAdmMaxDeviceNameSize]; + mutable snd_mixer_t* _inputMixerHandle; + char _inputMixerStr[kAdmMaxDeviceNameSize]; + mutable snd_mixer_elem_t* _outputMixerElement; + mutable snd_mixer_elem_t* _inputMixerElement; +}; + +} + +#endif // MODULES_AUDIO_DEVICE_MAIN_SOURCE_LINUX_AUDIO_MIXER_MANAGER_ALSA_LINUX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.cc b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.cc new file mode 100644 index 0000000000..c50d56b799 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.cc @@ -0,0 +1,841 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_device/linux/audio_mixer_manager_pulse_linux.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +extern webrtc::adm_linux_pulse::PulseAudioSymbolTable PaSymbolTable; + +// Accesses Pulse functions through our late-binding symbol table instead of +// directly. This way we don't have to link to libpulse, which means our +// binary will work on systems that don't have it. +#define LATE(sym) \ + LATESYM_GET(webrtc::adm_linux_pulse::PulseAudioSymbolTable, &PaSymbolTable, \ + sym) + +namespace webrtc { + +class AutoPulseLock { + public: + explicit AutoPulseLock(pa_threaded_mainloop* pa_mainloop) + : pa_mainloop_(pa_mainloop) { + LATE(pa_threaded_mainloop_lock)(pa_mainloop_); + } + + ~AutoPulseLock() { LATE(pa_threaded_mainloop_unlock)(pa_mainloop_); } + + private: + pa_threaded_mainloop* const pa_mainloop_; +}; + +AudioMixerManagerLinuxPulse::AudioMixerManagerLinuxPulse() + : _paOutputDeviceIndex(-1), + _paInputDeviceIndex(-1), + _paPlayStream(NULL), + _paRecStream(NULL), + _paMainloop(NULL), + _paContext(NULL), + _paVolume(0), + _paMute(0), + _paVolSteps(0), + _paSpeakerMute(false), + _paSpeakerVolume(PA_VOLUME_NORM), + _paChannels(0), + _paObjectsSet(false) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " created"; +} + +AudioMixerManagerLinuxPulse::~AudioMixerManagerLinuxPulse() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_INFO) << __FUNCTION__ << " destroyed"; + + Close(); +} + +// =========================================================================== +// PUBLIC METHODS +// =========================================================================== + +int32_t AudioMixerManagerLinuxPulse::SetPulseAudioObjects( + pa_threaded_mainloop* mainloop, + pa_context* context) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + if (!mainloop || !context) { + RTC_LOG(LS_ERROR) << "could not set PulseAudio objects for mixer"; + return -1; + } + + _paMainloop = mainloop; + _paContext = context; + _paObjectsSet = true; + + RTC_LOG(LS_VERBOSE) << "the PulseAudio objects for the mixer has been set"; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::Close() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + CloseSpeaker(); + CloseMicrophone(); + + _paMainloop = NULL; + _paContext = NULL; + _paObjectsSet = false; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::CloseSpeaker() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + // Reset the index to -1 + _paOutputDeviceIndex = -1; + _paPlayStream = NULL; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::CloseMicrophone() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + // Reset the index to -1 + _paInputDeviceIndex = -1; + _paRecStream = NULL; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SetPlayStream(pa_stream* playStream) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::SetPlayStream(playStream)"; + + _paPlayStream = playStream; + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SetRecStream(pa_stream* recStream) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxPulse::SetRecStream(recStream)"; + + _paRecStream = recStream; + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::OpenSpeaker(uint16_t deviceIndex) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxPulse::OpenSpeaker(deviceIndex=" + << deviceIndex << ")"; + + // No point in opening the speaker + // if PA objects have not been set + if (!_paObjectsSet) { + RTC_LOG(LS_ERROR) << "PulseAudio objects has not been set"; + return -1; + } + + // Set the index for the PulseAudio + // output device to control + _paOutputDeviceIndex = deviceIndex; + + RTC_LOG(LS_VERBOSE) << "the output mixer device is now open"; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::OpenMicrophone(uint16_t deviceIndex) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::OpenMicrophone(deviceIndex=" + << deviceIndex << ")"; + + // No point in opening the microphone + // if PA objects have not been set + if (!_paObjectsSet) { + RTC_LOG(LS_ERROR) << "PulseAudio objects have not been set"; + return -1; + } + + // Set the index for the PulseAudio + // input device to control + _paInputDeviceIndex = deviceIndex; + + RTC_LOG(LS_VERBOSE) << "the input mixer device is now open"; + + return 0; +} + +bool AudioMixerManagerLinuxPulse::SpeakerIsInitialized() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_INFO) << __FUNCTION__; + + return (_paOutputDeviceIndex != -1); +} + +bool AudioMixerManagerLinuxPulse::MicrophoneIsInitialized() const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_INFO) << __FUNCTION__; + + return (_paInputDeviceIndex != -1); +} + +int32_t AudioMixerManagerLinuxPulse::SetSpeakerVolume(uint32_t volume) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxPulse::SetSpeakerVolume(volume=" + << volume << ")"; + + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + bool setFailed(false); + + if (_paPlayStream && + (LATE(pa_stream_get_state)(_paPlayStream) != PA_STREAM_UNCONNECTED)) { + // We can only really set the volume if we have a connected stream + AutoPulseLock auto_lock(_paMainloop); + + // Get the number of channels from the sample specification + const pa_sample_spec* spec = LATE(pa_stream_get_sample_spec)(_paPlayStream); + if (!spec) { + RTC_LOG(LS_ERROR) << "could not get sample specification"; + return -1; + } + + // Set the same volume for all channels + pa_cvolume cVolumes; + LATE(pa_cvolume_set)(&cVolumes, spec->channels, volume); + + pa_operation* paOperation = NULL; + paOperation = LATE(pa_context_set_sink_input_volume)( + _paContext, LATE(pa_stream_get_index)(_paPlayStream), &cVolumes, + PaSetVolumeCallback, NULL); + if (!paOperation) { + setFailed = true; + } + + // Don't need to wait for the completion + LATE(pa_operation_unref)(paOperation); + } else { + // We have not created a stream or it's not connected to the sink + // Save the volume to be set at connection + _paSpeakerVolume = volume; + } + + if (setFailed) { + RTC_LOG(LS_WARNING) << "could not set speaker volume, error=" + << LATE(pa_context_errno)(_paContext); + + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SpeakerVolume(uint32_t& volume) const { + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + if (_paPlayStream && + (LATE(pa_stream_get_state)(_paPlayStream) != PA_STREAM_UNCONNECTED)) { + // We can only get the volume if we have a connected stream + if (!GetSinkInputInfo()) + return -1; + + AutoPulseLock auto_lock(_paMainloop); + volume = static_cast(_paVolume); + } else { + AutoPulseLock auto_lock(_paMainloop); + volume = _paSpeakerVolume; + } + + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxPulse::SpeakerVolume() => vol=" + << volume; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MaxSpeakerVolume( + uint32_t& maxVolume) const { + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + // PA_VOLUME_NORM corresponds to 100% (0db) + // but PA allows up to 150 db amplification + maxVolume = static_cast(PA_VOLUME_NORM); + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MinSpeakerVolume( + uint32_t& minVolume) const { + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + minVolume = static_cast(PA_VOLUME_MUTED); + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SpeakerVolumeIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + // Always available in Pulse Audio + available = true; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SpeakerMuteIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + // Always available in Pulse Audio + available = true; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SetSpeakerMute(bool enable) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerLinuxPulse::SetSpeakerMute(enable=" + << enable << ")"; + + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + bool setFailed(false); + + if (_paPlayStream && + (LATE(pa_stream_get_state)(_paPlayStream) != PA_STREAM_UNCONNECTED)) { + // We can only really mute if we have a connected stream + AutoPulseLock auto_lock(_paMainloop); + + pa_operation* paOperation = NULL; + paOperation = LATE(pa_context_set_sink_input_mute)( + _paContext, LATE(pa_stream_get_index)(_paPlayStream), (int)enable, + PaSetVolumeCallback, NULL); + if (!paOperation) { + setFailed = true; + } + + // Don't need to wait for the completion + LATE(pa_operation_unref)(paOperation); + } else { + // We have not created a stream or it's not connected to the sink + // Save the mute status to be set at connection + _paSpeakerMute = enable; + } + + if (setFailed) { + RTC_LOG(LS_WARNING) << "could not mute speaker, error=" + << LATE(pa_context_errno)(_paContext); + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SpeakerMute(bool& enabled) const { + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + if (_paPlayStream && + (LATE(pa_stream_get_state)(_paPlayStream) != PA_STREAM_UNCONNECTED)) { + // We can only get the mute status if we have a connected stream + if (!GetSinkInputInfo()) + return -1; + + enabled = static_cast(_paMute); + } else { + enabled = _paSpeakerMute; + } + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::SpeakerMute() => enabled=" << enabled; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::StereoPlayoutIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_paOutputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "output device index has not been set"; + return -1; + } + + uint32_t deviceIndex = (uint32_t)_paOutputDeviceIndex; + + { + AutoPulseLock auto_lock(_paMainloop); + + // Get the actual stream device index if we have a connected stream + // The device used by the stream can be changed + // during the call + if (_paPlayStream && + (LATE(pa_stream_get_state)(_paPlayStream) != PA_STREAM_UNCONNECTED)) { + deviceIndex = LATE(pa_stream_get_device_index)(_paPlayStream); + } + } + + if (!GetSinkInfoByIndex(deviceIndex)) + return -1; + + available = static_cast(_paChannels == 2); + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::StereoRecordingIsAvailable( + bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + uint32_t deviceIndex = (uint32_t)_paInputDeviceIndex; + + AutoPulseLock auto_lock(_paMainloop); + + // Get the actual stream device index if we have a connected stream + // The device used by the stream can be changed + // during the call + if (_paRecStream && + (LATE(pa_stream_get_state)(_paRecStream) != PA_STREAM_UNCONNECTED)) { + deviceIndex = LATE(pa_stream_get_device_index)(_paRecStream); + } + + pa_operation* paOperation = NULL; + + // Get info for this source + // We want to know if the actual device can record in stereo + paOperation = LATE(pa_context_get_source_info_by_index)( + _paContext, deviceIndex, PaSourceInfoCallback, (void*)this); + + WaitForOperationCompletion(paOperation); + + available = static_cast(_paChannels == 2); + + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::StereoRecordingIsAvailable()" + << " => available=" << available; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MicrophoneMuteIsAvailable( + bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + // Always available in Pulse Audio + available = true; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SetMicrophoneMute(bool enable) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::SetMicrophoneMute(enable=" << enable + << ")"; + + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + bool setFailed(false); + pa_operation* paOperation = NULL; + + uint32_t deviceIndex = (uint32_t)_paInputDeviceIndex; + + AutoPulseLock auto_lock(_paMainloop); + + // Get the actual stream device index if we have a connected stream + // The device used by the stream can be changed + // during the call + if (_paRecStream && + (LATE(pa_stream_get_state)(_paRecStream) != PA_STREAM_UNCONNECTED)) { + deviceIndex = LATE(pa_stream_get_device_index)(_paRecStream); + } + + // Set mute switch for the source + paOperation = LATE(pa_context_set_source_mute_by_index)( + _paContext, deviceIndex, enable, PaSetVolumeCallback, NULL); + + if (!paOperation) { + setFailed = true; + } + + // Don't need to wait for this to complete. + LATE(pa_operation_unref)(paOperation); + + if (setFailed) { + RTC_LOG(LS_WARNING) << "could not mute microphone, error=" + << LATE(pa_context_errno)(_paContext); + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MicrophoneMute(bool& enabled) const { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + uint32_t deviceIndex = (uint32_t)_paInputDeviceIndex; + + { + AutoPulseLock auto_lock(_paMainloop); + // Get the actual stream device index if we have a connected stream + // The device used by the stream can be changed + // during the call + if (_paRecStream && + (LATE(pa_stream_get_state)(_paRecStream) != PA_STREAM_UNCONNECTED)) { + deviceIndex = LATE(pa_stream_get_device_index)(_paRecStream); + } + } + + if (!GetSourceInfoByIndex(deviceIndex)) + return -1; + + enabled = static_cast(_paMute); + + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::MicrophoneMute() => enabled=" << enabled; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MicrophoneVolumeIsAvailable( + bool& available) { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + // Always available in Pulse Audio + available = true; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::SetMicrophoneVolume(uint32_t volume) { + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::SetMicrophoneVolume(volume=" << volume + << ")"; + + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + // Unlike output streams, input streams have no concept of a stream + // volume, only a device volume. So we have to change the volume of the + // device itself. + + // The device may have a different number of channels than the stream and + // their mapping may be different, so we don't want to use the channel + // count from our sample spec. We could use PA_CHANNELS_MAX to cover our + // bases, and the server allows that even if the device's channel count + // is lower, but some buggy PA clients don't like that (the pavucontrol + // on Hardy dies in an assert if the channel count is different). So + // instead we look up the actual number of channels that the device has. + AutoPulseLock auto_lock(_paMainloop); + uint32_t deviceIndex = (uint32_t)_paInputDeviceIndex; + + // Get the actual stream device index if we have a connected stream + // The device used by the stream can be changed + // during the call + if (_paRecStream && + (LATE(pa_stream_get_state)(_paRecStream) != PA_STREAM_UNCONNECTED)) { + deviceIndex = LATE(pa_stream_get_device_index)(_paRecStream); + } + + bool setFailed(false); + pa_operation* paOperation = NULL; + + // Get the number of channels for this source + paOperation = LATE(pa_context_get_source_info_by_index)( + _paContext, deviceIndex, PaSourceInfoCallback, (void*)this); + + WaitForOperationCompletion(paOperation); + + uint8_t channels = _paChannels; + pa_cvolume cVolumes; + LATE(pa_cvolume_set)(&cVolumes, channels, volume); + + // Set the volume for the source + paOperation = LATE(pa_context_set_source_volume_by_index)( + _paContext, deviceIndex, &cVolumes, PaSetVolumeCallback, NULL); + + if (!paOperation) { + setFailed = true; + } + + // Don't need to wait for this to complete. + LATE(pa_operation_unref)(paOperation); + + if (setFailed) { + RTC_LOG(LS_WARNING) << "could not set microphone volume, error=" + << LATE(pa_context_errno)(_paContext); + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MicrophoneVolume(uint32_t& volume) const { + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + uint32_t deviceIndex = (uint32_t)_paInputDeviceIndex; + + { + AutoPulseLock auto_lock(_paMainloop); + // Get the actual stream device index if we have a connected stream. + // The device used by the stream can be changed during the call. + if (_paRecStream && + (LATE(pa_stream_get_state)(_paRecStream) != PA_STREAM_UNCONNECTED)) { + deviceIndex = LATE(pa_stream_get_device_index)(_paRecStream); + } + } + + if (!GetSourceInfoByIndex(deviceIndex)) + return -1; + + { + AutoPulseLock auto_lock(_paMainloop); + volume = static_cast(_paVolume); + } + + RTC_LOG(LS_VERBOSE) + << "AudioMixerManagerLinuxPulse::MicrophoneVolume() => vol=" << volume; + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MaxMicrophoneVolume( + uint32_t& maxVolume) const { + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + // PA_VOLUME_NORM corresponds to 100% (0db) + // PA allows up to 150 db amplification (PA_VOLUME_MAX) + // but that doesn't work well for all sound cards + maxVolume = static_cast(PA_VOLUME_NORM); + + return 0; +} + +int32_t AudioMixerManagerLinuxPulse::MinMicrophoneVolume( + uint32_t& minVolume) const { + if (_paInputDeviceIndex == -1) { + RTC_LOG(LS_WARNING) << "input device index has not been set"; + return -1; + } + + minVolume = static_cast(PA_VOLUME_MUTED); + + return 0; +} + +// =========================================================================== +// Private Methods +// =========================================================================== + +void AudioMixerManagerLinuxPulse::PaSinkInfoCallback(pa_context* /*c*/, + const pa_sink_info* i, + int eol, + void* pThis) { + static_cast(pThis)->PaSinkInfoCallbackHandler( + i, eol); +} + +void AudioMixerManagerLinuxPulse::PaSinkInputInfoCallback( + pa_context* /*c*/, + const pa_sink_input_info* i, + int eol, + void* pThis) { + static_cast(pThis) + ->PaSinkInputInfoCallbackHandler(i, eol); +} + +void AudioMixerManagerLinuxPulse::PaSourceInfoCallback(pa_context* /*c*/, + const pa_source_info* i, + int eol, + void* pThis) { + static_cast(pThis)->PaSourceInfoCallbackHandler( + i, eol); +} + +void AudioMixerManagerLinuxPulse::PaSetVolumeCallback(pa_context* c, + int success, + void* /*pThis*/) { + if (!success) { + RTC_LOG(LS_ERROR) << "failed to set volume"; + } +} + +void AudioMixerManagerLinuxPulse::PaSinkInfoCallbackHandler( + const pa_sink_info* i, + int eol) { + if (eol) { + // Signal that we are done + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + return; + } + + _paChannels = i->channel_map.channels; // Get number of channels + pa_volume_t paVolume = PA_VOLUME_MUTED; // Minimum possible value. + for (int j = 0; j < _paChannels; ++j) { + if (paVolume < i->volume.values[j]) { + paVolume = i->volume.values[j]; + } + } + _paVolume = paVolume; // get the max volume for any channel + _paMute = i->mute; // get mute status + + // supported since PA 0.9.15 + //_paVolSteps = i->n_volume_steps; // get the number of volume steps + // default value is PA_VOLUME_NORM+1 + _paVolSteps = PA_VOLUME_NORM + 1; +} + +void AudioMixerManagerLinuxPulse::PaSinkInputInfoCallbackHandler( + const pa_sink_input_info* i, + int eol) { + if (eol) { + // Signal that we are done + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + return; + } + + _paChannels = i->channel_map.channels; // Get number of channels + pa_volume_t paVolume = PA_VOLUME_MUTED; // Minimum possible value. + for (int j = 0; j < _paChannels; ++j) { + if (paVolume < i->volume.values[j]) { + paVolume = i->volume.values[j]; + } + } + _paVolume = paVolume; // Get the max volume for any channel + _paMute = i->mute; // Get mute status +} + +void AudioMixerManagerLinuxPulse::PaSourceInfoCallbackHandler( + const pa_source_info* i, + int eol) { + if (eol) { + // Signal that we are done + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + return; + } + + _paChannels = i->channel_map.channels; // Get number of channels + pa_volume_t paVolume = PA_VOLUME_MUTED; // Minimum possible value. + for (int j = 0; j < _paChannels; ++j) { + if (paVolume < i->volume.values[j]) { + paVolume = i->volume.values[j]; + } + } + _paVolume = paVolume; // Get the max volume for any channel + _paMute = i->mute; // Get mute status + + // supported since PA 0.9.15 + //_paVolSteps = i->n_volume_steps; // Get the number of volume steps + // default value is PA_VOLUME_NORM+1 + _paVolSteps = PA_VOLUME_NORM + 1; +} + +void AudioMixerManagerLinuxPulse::WaitForOperationCompletion( + pa_operation* paOperation) const { + while (LATE(pa_operation_get_state)(paOperation) == PA_OPERATION_RUNNING) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + LATE(pa_operation_unref)(paOperation); +} + +bool AudioMixerManagerLinuxPulse::GetSinkInputInfo() const { + pa_operation* paOperation = NULL; + + AutoPulseLock auto_lock(_paMainloop); + // Get info for this stream (sink input). + paOperation = LATE(pa_context_get_sink_input_info)( + _paContext, LATE(pa_stream_get_index)(_paPlayStream), + PaSinkInputInfoCallback, (void*)this); + + WaitForOperationCompletion(paOperation); + return true; +} + +bool AudioMixerManagerLinuxPulse::GetSinkInfoByIndex(int device_index) const { + pa_operation* paOperation = NULL; + + AutoPulseLock auto_lock(_paMainloop); + paOperation = LATE(pa_context_get_sink_info_by_index)( + _paContext, device_index, PaSinkInfoCallback, (void*)this); + + WaitForOperationCompletion(paOperation); + return true; +} + +bool AudioMixerManagerLinuxPulse::GetSourceInfoByIndex(int device_index) const { + pa_operation* paOperation = NULL; + + AutoPulseLock auto_lock(_paMainloop); + paOperation = LATE(pa_context_get_source_info_by_index)( + _paContext, device_index, PaSourceInfoCallback, (void*)this); + + WaitForOperationCompletion(paOperation); + return true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.h b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.h new file mode 100644 index 0000000000..7846ccbcc0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/audio_mixer_manager_pulse_linux.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_MIXER_MANAGER_PULSE_LINUX_H_ +#define AUDIO_DEVICE_AUDIO_MIXER_MANAGER_PULSE_LINUX_H_ + +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/linux/pulseaudiosymboltable_linux.h" +#include "rtc_base/thread_checker.h" +#include "typedefs.h" // NOLINT(build/include) + +#include +#include + +#ifndef UINT32_MAX +#define UINT32_MAX ((uint32_t)-1) +#endif + +namespace webrtc +{ + +class AudioMixerManagerLinuxPulse +{ +public: + int32_t SetPlayStream(pa_stream* playStream); + int32_t SetRecStream(pa_stream* recStream); + int32_t OpenSpeaker(uint16_t deviceIndex); + int32_t OpenMicrophone(uint16_t deviceIndex); + int32_t SetSpeakerVolume(uint32_t volume); + int32_t SpeakerVolume(uint32_t& volume) const; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const; + int32_t MinSpeakerVolume(uint32_t& minVolume) const; + int32_t SpeakerVolumeIsAvailable(bool& available); + int32_t SpeakerMuteIsAvailable(bool& available); + int32_t SetSpeakerMute(bool enable); + int32_t StereoPlayoutIsAvailable(bool& available); + int32_t StereoRecordingIsAvailable(bool& available); + int32_t SpeakerMute(bool& enabled) const; + int32_t MicrophoneMuteIsAvailable(bool& available); + int32_t SetMicrophoneMute(bool enable); + int32_t MicrophoneMute(bool& enabled) const; + int32_t MicrophoneVolumeIsAvailable(bool& available); + int32_t SetMicrophoneVolume(uint32_t volume); + int32_t MicrophoneVolume(uint32_t& volume) const; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const; + int32_t SetPulseAudioObjects(pa_threaded_mainloop* mainloop, + pa_context* context); + int32_t Close(); + int32_t CloseSpeaker(); + int32_t CloseMicrophone(); + bool SpeakerIsInitialized() const; + bool MicrophoneIsInitialized() const; + +public: + AudioMixerManagerLinuxPulse(); + ~AudioMixerManagerLinuxPulse(); + +private: + static void PaSinkInfoCallback(pa_context *c, const pa_sink_info *i, + int eol, void *pThis); + static void PaSinkInputInfoCallback(pa_context *c, + const pa_sink_input_info *i, int eol, + void *pThis); + static void PaSourceInfoCallback(pa_context *c, const pa_source_info *i, + int eol, void *pThis); + static void + PaSetVolumeCallback(pa_context* /*c*/, int success, void* /*pThis*/); + void PaSinkInfoCallbackHandler(const pa_sink_info *i, int eol); + void PaSinkInputInfoCallbackHandler(const pa_sink_input_info *i, int eol); + void PaSourceInfoCallbackHandler(const pa_source_info *i, int eol); + + void WaitForOperationCompletion(pa_operation* paOperation) const; + + bool GetSinkInputInfo() const; + bool GetSinkInfoByIndex(int device_index)const ; + bool GetSourceInfoByIndex(int device_index) const; + +private: + int16_t _paOutputDeviceIndex; + int16_t _paInputDeviceIndex; + + pa_stream* _paPlayStream; + pa_stream* _paRecStream; + + pa_threaded_mainloop* _paMainloop; + pa_context* _paContext; + + mutable uint32_t _paVolume; + mutable uint32_t _paMute; + mutable uint32_t _paVolSteps; + bool _paSpeakerMute; + mutable uint32_t _paSpeakerVolume; + mutable uint8_t _paChannels; + bool _paObjectsSet; + + // Stores thread ID in constructor. + // We can then use ThreadChecker::CalledOnValidThread() to ensure that + // other methods are called from the same thread. + // Currently only does RTC_DCHECK(thread_checker_.CalledOnValidThread()). + rtc::ThreadChecker thread_checker_; +}; + +} + +#endif // MODULES_AUDIO_DEVICE_MAIN_SOURCE_LINUX_AUDIO_MIXER_MANAGER_PULSE_LINUX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.cc b/third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.cc new file mode 100644 index 0000000000..def4866d5a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/linux/latebindingsymboltable_linux.h" + +#include "rtc_base/logging.h" + +#ifdef WEBRTC_LINUX +#include +#endif + +namespace webrtc { +namespace adm_linux { + +inline static const char* GetDllError() { +#ifdef WEBRTC_LINUX + char* err = dlerror(); + if (err) { + return err; + } else { + return "No error"; + } +#else +#error Not implemented +#endif +} + +DllHandle InternalLoadDll(const char dll_name[]) { +#ifdef WEBRTC_LINUX + DllHandle handle = dlopen(dll_name, RTLD_NOW); +#else +#error Not implemented +#endif + if (handle == kInvalidDllHandle) { + RTC_LOG(LS_WARNING) << "Can't load " << dll_name << " : " << GetDllError(); + } + return handle; +} + +void InternalUnloadDll(DllHandle handle) { +#ifdef WEBRTC_LINUX +// TODO(pbos): Remove this dlclose() exclusion when leaks and suppressions from +// here are gone (or AddressSanitizer can display them properly). +// +// Skip dlclose() on AddressSanitizer as leaks including this module in the +// stack trace gets displayed as instead of the actual library +// -> it can not be suppressed. +// https://code.google.com/p/address-sanitizer/issues/detail?id=89 +#if !defined(ADDRESS_SANITIZER) + if (dlclose(handle) != 0) { + RTC_LOG(LS_ERROR) << GetDllError(); + } +#endif // !defined(ADDRESS_SANITIZER) +#else +#error Not implemented +#endif +} + +static bool LoadSymbol(DllHandle handle, + const char* symbol_name, + void** symbol) { +#ifdef WEBRTC_LINUX + *symbol = dlsym(handle, symbol_name); + char* err = dlerror(); + if (err) { + RTC_LOG(LS_ERROR) << "Error loading symbol " << symbol_name << " : " << err; + return false; + } else if (!*symbol) { + RTC_LOG(LS_ERROR) << "Symbol " << symbol_name << " is NULL"; + return false; + } + return true; +#else +#error Not implemented +#endif +} + +// This routine MUST assign SOME value for every symbol, even if that value is +// NULL, or else some symbols may be left with uninitialized data that the +// caller may later interpret as a valid address. +bool InternalLoadSymbols(DllHandle handle, + int num_symbols, + const char* const symbol_names[], + void* symbols[]) { +#ifdef WEBRTC_LINUX + // Clear any old errors. + dlerror(); +#endif + for (int i = 0; i < num_symbols; ++i) { + if (!LoadSymbol(handle, symbol_names[i], &symbols[i])) { + return false; + } + } + return true; +} + +} // namespace adm_linux +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.h b/third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.h new file mode 100644 index 0000000000..b834f5c76e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/latebindingsymboltable_linux.h @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_LATEBINDINGSYMBOLTABLE_LINUX_H_ +#define AUDIO_DEVICE_LATEBINDINGSYMBOLTABLE_LINUX_H_ + +#include +#include // for NULL +#include + +#include "rtc_base/constructormagic.h" + +// This file provides macros for creating "symbol table" classes to simplify the +// dynamic loading of symbols from DLLs. Currently the implementation only +// supports Linux and pure C symbols. +// See talk/sound/pulseaudiosymboltable.(h|cc) for an example. + +namespace webrtc { +namespace adm_linux { + +#ifdef WEBRTC_LINUX +typedef void *DllHandle; + +const DllHandle kInvalidDllHandle = NULL; +#else +#error Not implemented +#endif + +// These are helpers for use only by the class below. +DllHandle InternalLoadDll(const char dll_name[]); + +void InternalUnloadDll(DllHandle handle); + +bool InternalLoadSymbols(DllHandle handle, + int num_symbols, + const char *const symbol_names[], + void *symbols[]); + +template +class LateBindingSymbolTable { + public: + LateBindingSymbolTable() + : handle_(kInvalidDllHandle), + undefined_symbols_(false) { + memset(symbols_, 0, sizeof(symbols_)); + } + + ~LateBindingSymbolTable() { + Unload(); + } + + static int NumSymbols() { + return SYMBOL_TABLE_SIZE; + } + + // We do not use this, but we offer it for theoretical convenience. + static const char *GetSymbolName(int index) { + assert(index < NumSymbols()); + return kSymbolNames[index]; + } + + bool IsLoaded() const { + return handle_ != kInvalidDllHandle; + } + + // Loads the DLL and the symbol table. Returns true iff the DLL and symbol + // table loaded successfully. + bool Load() { + if (IsLoaded()) { + return true; + } + if (undefined_symbols_) { + // We do not attempt to load again because repeated attempts are not + // likely to succeed and DLL loading is costly. + return false; + } + handle_ = InternalLoadDll(kDllName); + if (!IsLoaded()) { + return false; + } + if (!InternalLoadSymbols(handle_, NumSymbols(), kSymbolNames, symbols_)) { + undefined_symbols_ = true; + Unload(); + return false; + } + return true; + } + + void Unload() { + if (!IsLoaded()) { + return; + } + InternalUnloadDll(handle_); + handle_ = kInvalidDllHandle; + memset(symbols_, 0, sizeof(symbols_)); + } + + // Retrieves the given symbol. NOTE: Recommended to use LATESYM_GET below + // instead of this. + void *GetSymbol(int index) const { + assert(IsLoaded()); + assert(index < NumSymbols()); + return symbols_[index]; + } + + private: + DllHandle handle_; + bool undefined_symbols_; + void *symbols_[SYMBOL_TABLE_SIZE]; + + RTC_DISALLOW_COPY_AND_ASSIGN(LateBindingSymbolTable); +}; + +// This macro must be invoked in a header to declare a symbol table class. +#define LATE_BINDING_SYMBOL_TABLE_DECLARE_BEGIN(ClassName) \ +enum { + +// This macro must be invoked in the header declaration once for each symbol +// (recommended to use an X-Macro to avoid duplication). +// This macro defines an enum with names built from the symbols, which +// essentially creates a hash table in the compiler from symbol names to their +// indices in the symbol table class. +#define LATE_BINDING_SYMBOL_TABLE_DECLARE_ENTRY(ClassName, sym) \ + ClassName##_SYMBOL_TABLE_INDEX_##sym, + +// This macro completes the header declaration. +#define LATE_BINDING_SYMBOL_TABLE_DECLARE_END(ClassName) \ + ClassName##_SYMBOL_TABLE_SIZE \ + } \ + ; \ + \ + extern const char ClassName##_kDllName[]; \ + extern const char* const \ + ClassName##_kSymbolNames[ClassName##_SYMBOL_TABLE_SIZE]; \ + \ + typedef ::webrtc::adm_linux::LateBindingSymbolTable< \ + ClassName##_SYMBOL_TABLE_SIZE, ClassName##_kDllName, \ + ClassName##_kSymbolNames> \ + ClassName; + +// This macro must be invoked in a .cc file to define a previously-declared +// symbol table class. +#define LATE_BINDING_SYMBOL_TABLE_DEFINE_BEGIN(ClassName, dllName) \ +const char ClassName##_kDllName[] = dllName; \ +const char *const ClassName##_kSymbolNames[ClassName##_SYMBOL_TABLE_SIZE] = { + +// This macro must be invoked in the .cc definition once for each symbol +// (recommended to use an X-Macro to avoid duplication). +// This would have to use the mangled name if we were to ever support C++ +// symbols. +#define LATE_BINDING_SYMBOL_TABLE_DEFINE_ENTRY(ClassName, sym) \ + #sym, + +#define LATE_BINDING_SYMBOL_TABLE_DEFINE_END(ClassName) \ +}; + +// Index of a given symbol in the given symbol table class. +#define LATESYM_INDEXOF(ClassName, sym) \ + (ClassName##_SYMBOL_TABLE_INDEX_##sym) + +// Returns a reference to the given late-binded symbol, with the correct type. +#define LATESYM_GET(ClassName, inst, sym) \ + (*reinterpret_cast( \ + (inst)->GetSymbol(LATESYM_INDEXOF(ClassName, sym)))) + +} // namespace adm_linux +} // namespace webrtc + +#endif // ADM_LATEBINDINGSYMBOLTABLE_LINUX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.cc b/third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.cc new file mode 100644 index 0000000000..3d012fe619 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.cc @@ -0,0 +1,41 @@ +/* + * libjingle + * Copyright 2004--2010, Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "modules/audio_device/linux/pulseaudiosymboltable_linux.h" + +namespace webrtc { +namespace adm_linux_pulse { + +LATE_BINDING_SYMBOL_TABLE_DEFINE_BEGIN(PulseAudioSymbolTable, "libpulse.so.0") +#define X(sym) \ + LATE_BINDING_SYMBOL_TABLE_DEFINE_ENTRY(PulseAudioSymbolTable, sym) +PULSE_AUDIO_SYMBOLS_LIST +#undef X +LATE_BINDING_SYMBOL_TABLE_DEFINE_END(PulseAudioSymbolTable) + +} // namespace adm_linux_pulse +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.h b/third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.h new file mode 100644 index 0000000000..d21446ce8a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/linux/pulseaudiosymboltable_linux.h @@ -0,0 +1,106 @@ +/* + * libjingle + * Copyright 2004--2010, Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef AUDIO_DEVICE_PULSEAUDIOSYMBOLTABLE_LINUX_H_ +#define AUDIO_DEVICE_PULSEAUDIOSYMBOLTABLE_LINUX_H_ + +#include "modules/audio_device/linux/latebindingsymboltable_linux.h" + +namespace webrtc { +namespace adm_linux_pulse { + +// The PulseAudio symbols we need, as an X-Macro list. +// This list must contain precisely every libpulse function that is used in +// the ADM LINUX PULSE Device and Mixer classes +#define PULSE_AUDIO_SYMBOLS_LIST \ + X(pa_bytes_per_second) \ + X(pa_context_connect) \ + X(pa_context_disconnect) \ + X(pa_context_errno) \ + X(pa_context_get_protocol_version) \ + X(pa_context_get_server_info) \ + X(pa_context_get_sink_info_list) \ + X(pa_context_get_sink_info_by_index) \ + X(pa_context_get_sink_info_by_name) \ + X(pa_context_get_sink_input_info) \ + X(pa_context_get_source_info_by_index) \ + X(pa_context_get_source_info_by_name) \ + X(pa_context_get_source_info_list) \ + X(pa_context_get_state) \ + X(pa_context_new) \ + X(pa_context_set_sink_input_volume) \ + X(pa_context_set_sink_input_mute) \ + X(pa_context_set_source_volume_by_index) \ + X(pa_context_set_source_mute_by_index) \ + X(pa_context_set_state_callback) \ + X(pa_context_unref) \ + X(pa_cvolume_set) \ + X(pa_operation_get_state) \ + X(pa_operation_unref) \ + X(pa_stream_connect_playback) \ + X(pa_stream_connect_record) \ + X(pa_stream_disconnect) \ + X(pa_stream_drop) \ + X(pa_stream_get_device_index) \ + X(pa_stream_get_index) \ + X(pa_stream_get_latency) \ + X(pa_stream_get_sample_spec) \ + X(pa_stream_get_state) \ + X(pa_stream_new) \ + X(pa_stream_peek) \ + X(pa_stream_readable_size) \ + X(pa_stream_set_buffer_attr) \ + X(pa_stream_set_overflow_callback) \ + X(pa_stream_set_read_callback) \ + X(pa_stream_set_state_callback) \ + X(pa_stream_set_underflow_callback) \ + X(pa_stream_set_write_callback) \ + X(pa_stream_unref) \ + X(pa_stream_writable_size) \ + X(pa_stream_write) \ + X(pa_strerror) \ + X(pa_threaded_mainloop_free) \ + X(pa_threaded_mainloop_get_api) \ + X(pa_threaded_mainloop_lock) \ + X(pa_threaded_mainloop_new) \ + X(pa_threaded_mainloop_signal) \ + X(pa_threaded_mainloop_start) \ + X(pa_threaded_mainloop_stop) \ + X(pa_threaded_mainloop_unlock) \ + X(pa_threaded_mainloop_wait) + +LATE_BINDING_SYMBOL_TABLE_DECLARE_BEGIN(PulseAudioSymbolTable) +#define X(sym) \ + LATE_BINDING_SYMBOL_TABLE_DECLARE_ENTRY(PulseAudioSymbolTable, sym) +PULSE_AUDIO_SYMBOLS_LIST +#undef X +LATE_BINDING_SYMBOL_TABLE_DECLARE_END(PulseAudioSymbolTable) + +} // namespace adm_linux_pulse +} // namespace webrtc + +#endif // AUDIO_DEVICE_PULSEAUDIOSYMBOLTABLE_LINUX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.cc b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.cc new file mode 100644 index 0000000000..110bfcd0d5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.cc @@ -0,0 +1,2574 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/mac/audio_device_mac.h" +#include "modules/audio_device/audio_device_config.h" +#include "modules/audio_device/mac/portaudio/pa_ringbuffer.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/platform_thread.h" +#include "system_wrappers/include/event_wrapper.h" + +#include +#include // OSAtomicCompareAndSwap() +#include // mach_task_self() +#include // sysctlbyname() + +namespace webrtc { + +#define WEBRTC_CA_RETURN_ON_ERR(expr) \ + do { \ + err = expr; \ + if (err != noErr) { \ + logCAMsg(rtc::LS_ERROR, "Error in " #expr, (const char*)&err); \ + return -1; \ + } \ + } while (0) + +#define WEBRTC_CA_LOG_ERR(expr) \ + do { \ + err = expr; \ + if (err != noErr) { \ + logCAMsg(rtc::LS_ERROR, "Error in " #expr, (const char*)&err); \ + } \ + } while (0) + +#define WEBRTC_CA_LOG_WARN(expr) \ + do { \ + err = expr; \ + if (err != noErr) { \ + logCAMsg(rtc::LS_WARNING, "Error in " #expr, (const char*)&err); \ + } \ + } while (0) + +enum { MaxNumberDevices = 64 }; + +void AudioDeviceMac::AtomicSet32(int32_t* theValue, int32_t newValue) { + while (1) { + int32_t oldValue = *theValue; + if (OSAtomicCompareAndSwap32Barrier(oldValue, newValue, theValue) == true) { + return; + } + } +} + +int32_t AudioDeviceMac::AtomicGet32(int32_t* theValue) { + while (1) { + int32_t value = *theValue; + if (OSAtomicCompareAndSwap32Barrier(value, value, theValue) == true) { + return value; + } + } +} + +// CoreAudio errors are best interpreted as four character strings. +void AudioDeviceMac::logCAMsg(const rtc::LoggingSeverity sev, + const char* msg, + const char* err) { + RTC_DCHECK(msg != NULL); + RTC_DCHECK(err != NULL); + +#ifdef WEBRTC_ARCH_BIG_ENDIAN + switch (sev) { + case rtc::LS_ERROR: + RTC_LOG(LS_ERROR) << msg << ": " << err[0] << err[1] << err[2] << err[3]; + break; + case rtc::LS_WARNING: + RTC_LOG(LS_WARNING) << msg << ": " << err[0] << err[1] << err[2] + << err[3]; + break; + case rtc::LS_VERBOSE: + RTC_LOG(LS_VERBOSE) << msg << ": " << err[0] << err[1] << err[2] + << err[3]; + break; + default: + break; + } +#else + // We need to flip the characters in this case. + switch (sev) { + case rtc::LS_ERROR: + RTC_LOG(LS_ERROR) << msg << ": " << err[3] << err[2] << err[1] << err[0]; + break; + case rtc::LS_WARNING: + RTC_LOG(LS_WARNING) << msg << ": " << err[3] << err[2] << err[1] + << err[0]; + break; + case rtc::LS_VERBOSE: + RTC_LOG(LS_VERBOSE) << msg << ": " << err[3] << err[2] << err[1] + << err[0]; + break; + default: + break; + } +#endif +} + +AudioDeviceMac::AudioDeviceMac() + : _ptrAudioBuffer(NULL), + _stopEventRec(*EventWrapper::Create()), + _stopEvent(*EventWrapper::Create()), + _mixerManager(), + _inputDeviceIndex(0), + _outputDeviceIndex(0), + _inputDeviceID(kAudioObjectUnknown), + _outputDeviceID(kAudioObjectUnknown), + _inputDeviceIsSpecified(false), + _outputDeviceIsSpecified(false), + _recChannels(N_REC_CHANNELS), + _playChannels(N_PLAY_CHANNELS), + _captureBufData(NULL), + _renderBufData(NULL), + _initialized(false), + _isShutDown(false), + _recording(false), + _playing(false), + _recIsInitialized(false), + _playIsInitialized(false), + _AGC(false), + _renderDeviceIsAlive(1), + _captureDeviceIsAlive(1), + _twoDevices(true), + _doStop(false), + _doStopRec(false), + _macBookPro(false), + _macBookProPanRight(false), + _captureLatencyUs(0), + _renderLatencyUs(0), + _captureDelayUs(0), + _renderDelayUs(0), + _renderDelayOffsetSamples(0), + _paCaptureBuffer(NULL), + _paRenderBuffer(NULL), + _captureBufSizeSamples(0), + _renderBufSizeSamples(0), + prev_key_state_(), + get_mic_volume_counter_ms_(0) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " created"; + + RTC_DCHECK(&_stopEvent != NULL); + RTC_DCHECK(&_stopEventRec != NULL); + + memset(_renderConvertData, 0, sizeof(_renderConvertData)); + memset(&_outStreamFormat, 0, sizeof(AudioStreamBasicDescription)); + memset(&_outDesiredFormat, 0, sizeof(AudioStreamBasicDescription)); + memset(&_inStreamFormat, 0, sizeof(AudioStreamBasicDescription)); + memset(&_inDesiredFormat, 0, sizeof(AudioStreamBasicDescription)); +} + +AudioDeviceMac::~AudioDeviceMac() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " destroyed"; + + if (!_isShutDown) { + Terminate(); + } + + RTC_DCHECK(!capture_worker_thread_.get()); + RTC_DCHECK(!render_worker_thread_.get()); + + if (_paRenderBuffer) { + delete _paRenderBuffer; + _paRenderBuffer = NULL; + } + + if (_paCaptureBuffer) { + delete _paCaptureBuffer; + _paCaptureBuffer = NULL; + } + + if (_renderBufData) { + delete[] _renderBufData; + _renderBufData = NULL; + } + + if (_captureBufData) { + delete[] _captureBufData; + _captureBufData = NULL; + } + + kern_return_t kernErr = KERN_SUCCESS; + kernErr = semaphore_destroy(mach_task_self(), _renderSemaphore); + if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_destroy() error: " << kernErr; + } + + kernErr = semaphore_destroy(mach_task_self(), _captureSemaphore); + if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_destroy() error: " << kernErr; + } + + delete &_stopEvent; + delete &_stopEventRec; +} + +// ============================================================================ +// API +// ============================================================================ + +void AudioDeviceMac::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + rtc::CritScope lock(&_critSect); + + _ptrAudioBuffer = audioBuffer; + + // inform the AudioBuffer about default settings for this implementation + _ptrAudioBuffer->SetRecordingSampleRate(N_REC_SAMPLES_PER_SEC); + _ptrAudioBuffer->SetPlayoutSampleRate(N_PLAY_SAMPLES_PER_SEC); + _ptrAudioBuffer->SetRecordingChannels(N_REC_CHANNELS); + _ptrAudioBuffer->SetPlayoutChannels(N_PLAY_CHANNELS); +} + +int32_t AudioDeviceMac::ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const { + audioLayer = AudioDeviceModule::kPlatformDefaultAudio; + return 0; +} + +AudioDeviceGeneric::InitStatus AudioDeviceMac::Init() { + rtc::CritScope lock(&_critSect); + + if (_initialized) { + return InitStatus::OK; + } + + OSStatus err = noErr; + + _isShutDown = false; + + // PortAudio ring buffers require an elementCount which is a power of two. + if (_renderBufData == NULL) { + UInt32 powerOfTwo = 1; + while (powerOfTwo < PLAY_BUF_SIZE_IN_SAMPLES) { + powerOfTwo <<= 1; + } + _renderBufSizeSamples = powerOfTwo; + _renderBufData = new SInt16[_renderBufSizeSamples]; + } + + if (_paRenderBuffer == NULL) { + _paRenderBuffer = new PaUtilRingBuffer; + PaRingBufferSize bufSize = -1; + bufSize = PaUtil_InitializeRingBuffer( + _paRenderBuffer, sizeof(SInt16), _renderBufSizeSamples, _renderBufData); + if (bufSize == -1) { + RTC_LOG(LS_ERROR) << "PaUtil_InitializeRingBuffer() error"; + return InitStatus::PLAYOUT_ERROR; + } + } + + if (_captureBufData == NULL) { + UInt32 powerOfTwo = 1; + while (powerOfTwo < REC_BUF_SIZE_IN_SAMPLES) { + powerOfTwo <<= 1; + } + _captureBufSizeSamples = powerOfTwo; + _captureBufData = new Float32[_captureBufSizeSamples]; + } + + if (_paCaptureBuffer == NULL) { + _paCaptureBuffer = new PaUtilRingBuffer; + PaRingBufferSize bufSize = -1; + bufSize = + PaUtil_InitializeRingBuffer(_paCaptureBuffer, sizeof(Float32), + _captureBufSizeSamples, _captureBufData); + if (bufSize == -1) { + RTC_LOG(LS_ERROR) << "PaUtil_InitializeRingBuffer() error"; + return InitStatus::RECORDING_ERROR; + } + } + + kern_return_t kernErr = KERN_SUCCESS; + kernErr = semaphore_create(mach_task_self(), &_renderSemaphore, + SYNC_POLICY_FIFO, 0); + if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_create() error: " << kernErr; + return InitStatus::OTHER_ERROR; + } + + kernErr = semaphore_create(mach_task_self(), &_captureSemaphore, + SYNC_POLICY_FIFO, 0); + if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_create() error: " << kernErr; + return InitStatus::OTHER_ERROR; + } + + // Setting RunLoop to NULL here instructs HAL to manage its own thread for + // notifications. This was the default behaviour on OS X 10.5 and earlier, + // but now must be explicitly specified. HAL would otherwise try to use the + // main thread to issue notifications. + AudioObjectPropertyAddress propertyAddress = { + kAudioHardwarePropertyRunLoop, kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster}; + CFRunLoopRef runLoop = NULL; + UInt32 size = sizeof(CFRunLoopRef); + int aoerr = AudioObjectSetPropertyData( + kAudioObjectSystemObject, &propertyAddress, 0, NULL, size, &runLoop); + if (aoerr != noErr) { + RTC_LOG(LS_ERROR) << "Error in AudioObjectSetPropertyData: " + << (const char*)&aoerr; + return InitStatus::OTHER_ERROR; + } + + // Listen for any device changes. + propertyAddress.mSelector = kAudioHardwarePropertyDevices; + WEBRTC_CA_LOG_ERR(AudioObjectAddPropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + + // Determine if this is a MacBook Pro + _macBookPro = false; + _macBookProPanRight = false; + char buf[128]; + size_t length = sizeof(buf); + memset(buf, 0, length); + + int intErr = sysctlbyname("hw.model", buf, &length, NULL, 0); + if (intErr != 0) { + RTC_LOG(LS_ERROR) << "Error in sysctlbyname(): " << err; + } else { + RTC_LOG(LS_VERBOSE) << "Hardware model: " << buf; + if (strncmp(buf, "MacBookPro", 10) == 0) { + _macBookPro = true; + } + } + + get_mic_volume_counter_ms_ = 0; + + _initialized = true; + + return InitStatus::OK; +} + +int32_t AudioDeviceMac::Terminate() { + if (!_initialized) { + return 0; + } + + if (_recording) { + RTC_LOG(LS_ERROR) << "Recording must be stopped"; + return -1; + } + + if (_playing) { + RTC_LOG(LS_ERROR) << "Playback must be stopped"; + return -1; + } + + _critSect.Enter(); + + _mixerManager.Close(); + + OSStatus err = noErr; + int retVal = 0; + + AudioObjectPropertyAddress propertyAddress = { + kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster}; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + + err = AudioHardwareUnload(); + if (err != noErr) { + logCAMsg(rtc::LS_ERROR, "Error in AudioHardwareUnload()", + (const char*)&err); + retVal = -1; + } + + _isShutDown = true; + _initialized = false; + _outputDeviceIsSpecified = false; + _inputDeviceIsSpecified = false; + + _critSect.Leave(); + + return retVal; +} + +bool AudioDeviceMac::Initialized() const { + return (_initialized); +} + +int32_t AudioDeviceMac::SpeakerIsAvailable(bool& available) { + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + // Make an attempt to open up the + // output mixer corresponding to the currently selected output device. + // + if (!wasInitialized && InitSpeaker() == -1) { + available = false; + return 0; + } + + // Given that InitSpeaker was successful, we know that a valid speaker + // exists. + available = true; + + // Close the initialized output mixer + // + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceMac::InitSpeaker() { + rtc::CritScope lock(&_critSect); + + if (_playing) { + return -1; + } + + if (InitDevice(_outputDeviceIndex, _outputDeviceID, false) == -1) { + return -1; + } + + if (_inputDeviceID == _outputDeviceID) { + _twoDevices = false; + } else { + _twoDevices = true; + } + + if (_mixerManager.OpenSpeaker(_outputDeviceID) == -1) { + return -1; + } + + return 0; +} + +int32_t AudioDeviceMac::MicrophoneIsAvailable(bool& available) { + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected output device. + // + if (!wasInitialized && InitMicrophone() == -1) { + available = false; + return 0; + } + + // Given that InitMicrophone was successful, we know that a valid microphone + // exists. + available = true; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceMac::InitMicrophone() { + rtc::CritScope lock(&_critSect); + + if (_recording) { + return -1; + } + + if (InitDevice(_inputDeviceIndex, _inputDeviceID, true) == -1) { + return -1; + } + + if (_inputDeviceID == _outputDeviceID) { + _twoDevices = false; + } else { + _twoDevices = true; + } + + if (_mixerManager.OpenMicrophone(_inputDeviceID) == -1) { + return -1; + } + + return 0; +} + +bool AudioDeviceMac::SpeakerIsInitialized() const { + return (_mixerManager.SpeakerIsInitialized()); +} + +bool AudioDeviceMac::MicrophoneIsInitialized() const { + return (_mixerManager.MicrophoneIsInitialized()); +} + +int32_t AudioDeviceMac::SpeakerVolumeIsAvailable(bool& available) { + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + // Make an attempt to open up the + // output mixer corresponding to the currently selected output device. + // + if (!wasInitialized && InitSpeaker() == -1) { + // If we end up here it means that the selected speaker has no volume + // control. + available = false; + return 0; + } + + // Given that InitSpeaker was successful, we know that a volume control exists + // + available = true; + + // Close the initialized output mixer + // + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceMac::SetSpeakerVolume(uint32_t volume) { + return (_mixerManager.SetSpeakerVolume(volume)); +} + +int32_t AudioDeviceMac::SpeakerVolume(uint32_t& volume) const { + uint32_t level(0); + + if (_mixerManager.SpeakerVolume(level) == -1) { + return -1; + } + + volume = level; + return 0; +} + +int32_t AudioDeviceMac::MaxSpeakerVolume(uint32_t& maxVolume) const { + uint32_t maxVol(0); + + if (_mixerManager.MaxSpeakerVolume(maxVol) == -1) { + return -1; + } + + maxVolume = maxVol; + return 0; +} + +int32_t AudioDeviceMac::MinSpeakerVolume(uint32_t& minVolume) const { + uint32_t minVol(0); + + if (_mixerManager.MinSpeakerVolume(minVol) == -1) { + return -1; + } + + minVolume = minVol; + return 0; +} + +int32_t AudioDeviceMac::SpeakerMuteIsAvailable(bool& available) { + bool isAvailable(false); + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + // Make an attempt to open up the + // output mixer corresponding to the currently selected output device. + // + if (!wasInitialized && InitSpeaker() == -1) { + // If we end up here it means that the selected speaker has no volume + // control, hence it is safe to state that there is no mute control + // already at this stage. + available = false; + return 0; + } + + // Check if the selected speaker has a mute control + // + _mixerManager.SpeakerMuteIsAvailable(isAvailable); + + available = isAvailable; + + // Close the initialized output mixer + // + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceMac::SetSpeakerMute(bool enable) { + return (_mixerManager.SetSpeakerMute(enable)); +} + +int32_t AudioDeviceMac::SpeakerMute(bool& enabled) const { + bool muted(0); + + if (_mixerManager.SpeakerMute(muted) == -1) { + return -1; + } + + enabled = muted; + return 0; +} + +int32_t AudioDeviceMac::MicrophoneMuteIsAvailable(bool& available) { + bool isAvailable(false); + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected input device. + // + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no volume + // control, hence it is safe to state that there is no boost control + // already at this stage. + available = false; + return 0; + } + + // Check if the selected microphone has a mute control + // + _mixerManager.MicrophoneMuteIsAvailable(isAvailable); + available = isAvailable; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceMac::SetMicrophoneMute(bool enable) { + return (_mixerManager.SetMicrophoneMute(enable)); +} + +int32_t AudioDeviceMac::MicrophoneMute(bool& enabled) const { + bool muted(0); + + if (_mixerManager.MicrophoneMute(muted) == -1) { + return -1; + } + + enabled = muted; + return 0; +} + +int32_t AudioDeviceMac::StereoRecordingIsAvailable(bool& available) { + bool isAvailable(false); + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + if (!wasInitialized && InitMicrophone() == -1) { + // Cannot open the specified device + available = false; + return 0; + } + + // Check if the selected microphone can record stereo + // + _mixerManager.StereoRecordingIsAvailable(isAvailable); + available = isAvailable; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceMac::SetStereoRecording(bool enable) { + if (enable) + _recChannels = 2; + else + _recChannels = 1; + + return 0; +} + +int32_t AudioDeviceMac::StereoRecording(bool& enabled) const { + if (_recChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +int32_t AudioDeviceMac::StereoPlayoutIsAvailable(bool& available) { + bool isAvailable(false); + bool wasInitialized = _mixerManager.SpeakerIsInitialized(); + + if (!wasInitialized && InitSpeaker() == -1) { + // Cannot open the specified device + available = false; + return 0; + } + + // Check if the selected microphone can record stereo + // + _mixerManager.StereoPlayoutIsAvailable(isAvailable); + available = isAvailable; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseSpeaker(); + } + + return 0; +} + +int32_t AudioDeviceMac::SetStereoPlayout(bool enable) { + if (enable) + _playChannels = 2; + else + _playChannels = 1; + + return 0; +} + +int32_t AudioDeviceMac::StereoPlayout(bool& enabled) const { + if (_playChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +int32_t AudioDeviceMac::SetAGC(bool enable) { + _AGC = enable; + + return 0; +} + +bool AudioDeviceMac::AGC() const { + return _AGC; +} + +int32_t AudioDeviceMac::MicrophoneVolumeIsAvailable(bool& available) { + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected output device. + // + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no volume + // control. + available = false; + return 0; + } + + // Given that InitMicrophone was successful, we know that a volume control + // exists + // + available = true; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t AudioDeviceMac::SetMicrophoneVolume(uint32_t volume) { + return (_mixerManager.SetMicrophoneVolume(volume)); +} + +int32_t AudioDeviceMac::MicrophoneVolume(uint32_t& volume) const { + uint32_t level(0); + + if (_mixerManager.MicrophoneVolume(level) == -1) { + RTC_LOG(LS_WARNING) << "failed to retrieve current microphone level"; + return -1; + } + + volume = level; + return 0; +} + +int32_t AudioDeviceMac::MaxMicrophoneVolume(uint32_t& maxVolume) const { + uint32_t maxVol(0); + + if (_mixerManager.MaxMicrophoneVolume(maxVol) == -1) { + return -1; + } + + maxVolume = maxVol; + return 0; +} + +int32_t AudioDeviceMac::MinMicrophoneVolume(uint32_t& minVolume) const { + uint32_t minVol(0); + + if (_mixerManager.MinMicrophoneVolume(minVol) == -1) { + return -1; + } + + minVolume = minVol; + return 0; +} + +int16_t AudioDeviceMac::PlayoutDevices() { + AudioDeviceID playDevices[MaxNumberDevices]; + return GetNumberDevices(kAudioDevicePropertyScopeOutput, playDevices, + MaxNumberDevices); +} + +int32_t AudioDeviceMac::SetPlayoutDevice(uint16_t index) { + rtc::CritScope lock(&_critSect); + + if (_playIsInitialized) { + return -1; + } + + AudioDeviceID playDevices[MaxNumberDevices]; + uint32_t nDevices = GetNumberDevices(kAudioDevicePropertyScopeOutput, + playDevices, MaxNumberDevices); + RTC_LOG(LS_VERBOSE) << "number of available waveform-audio output devices is " + << nDevices; + + if (index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + _outputDeviceIndex = index; + _outputDeviceIsSpecified = true; + + return 0; +} + +int32_t AudioDeviceMac::SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType /*device*/) { + RTC_LOG(LS_ERROR) << "WindowsDeviceType not supported"; + return -1; +} + +int32_t AudioDeviceMac::PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + const uint16_t nDevices(PlayoutDevices()); + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + return GetDeviceName(kAudioDevicePropertyScopeOutput, index, name); +} + +int32_t AudioDeviceMac::RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + const uint16_t nDevices(RecordingDevices()); + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + return GetDeviceName(kAudioDevicePropertyScopeInput, index, name); +} + +int16_t AudioDeviceMac::RecordingDevices() { + AudioDeviceID recDevices[MaxNumberDevices]; + return GetNumberDevices(kAudioDevicePropertyScopeInput, recDevices, + MaxNumberDevices); +} + +int32_t AudioDeviceMac::SetRecordingDevice(uint16_t index) { + if (_recIsInitialized) { + return -1; + } + + AudioDeviceID recDevices[MaxNumberDevices]; + uint32_t nDevices = GetNumberDevices(kAudioDevicePropertyScopeInput, + recDevices, MaxNumberDevices); + RTC_LOG(LS_VERBOSE) << "number of available waveform-audio input devices is " + << nDevices; + + if (index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + _inputDeviceIndex = index; + _inputDeviceIsSpecified = true; + + return 0; +} + +int32_t AudioDeviceMac::SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType /*device*/) { + RTC_LOG(LS_ERROR) << "WindowsDeviceType not supported"; + return -1; +} + +int32_t AudioDeviceMac::PlayoutIsAvailable(bool& available) { + available = true; + + // Try to initialize the playout side + if (InitPlayout() == -1) { + available = false; + } + + // We destroy the IOProc created by InitPlayout() in implDeviceIOProc(). + // We must actually start playout here in order to have the IOProc + // deleted by calling StopPlayout(). + if (StartPlayout() == -1) { + available = false; + } + + // Cancel effect of initialization + if (StopPlayout() == -1) { + available = false; + } + + return 0; +} + +int32_t AudioDeviceMac::RecordingIsAvailable(bool& available) { + available = true; + + // Try to initialize the recording side + if (InitRecording() == -1) { + available = false; + } + + // We destroy the IOProc created by InitRecording() in implInDeviceIOProc(). + // We must actually start recording here in order to have the IOProc + // deleted by calling StopRecording(). + if (StartRecording() == -1) { + available = false; + } + + // Cancel effect of initialization + if (StopRecording() == -1) { + available = false; + } + + return 0; +} + +int32_t AudioDeviceMac::InitPlayout() { + RTC_LOG(LS_INFO) << "InitPlayout"; + rtc::CritScope lock(&_critSect); + + if (_playing) { + return -1; + } + + if (!_outputDeviceIsSpecified) { + return -1; + } + + if (_playIsInitialized) { + return 0; + } + + // Initialize the speaker (devices might have been added or removed) + if (InitSpeaker() == -1) { + RTC_LOG(LS_WARNING) << "InitSpeaker() failed"; + } + + if (!MicrophoneIsInitialized()) { + // Make this call to check if we are using + // one or two devices (_twoDevices) + bool available = false; + if (MicrophoneIsAvailable(available) == -1) { + RTC_LOG(LS_WARNING) << "MicrophoneIsAvailable() failed"; + } + } + + PaUtil_FlushRingBuffer(_paRenderBuffer); + + OSStatus err = noErr; + UInt32 size = 0; + _renderDelayOffsetSamples = 0; + _renderDelayUs = 0; + _renderLatencyUs = 0; + _renderDeviceIsAlive = 1; + _doStop = false; + + // The internal microphone of a MacBook Pro is located under the left speaker + // grille. When the internal speakers are in use, we want to fully stereo + // pan to the right. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyDataSource, kAudioDevicePropertyScopeOutput, 0}; + if (_macBookPro) { + _macBookProPanRight = false; + Boolean hasProperty = + AudioObjectHasProperty(_outputDeviceID, &propertyAddress); + if (hasProperty) { + UInt32 dataSource = 0; + size = sizeof(dataSource); + WEBRTC_CA_LOG_WARN(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &dataSource)); + + if (dataSource == 'ispk') { + _macBookProPanRight = true; + RTC_LOG(LS_VERBOSE) + << "MacBook Pro using internal speakers; stereo panning right"; + } else { + RTC_LOG(LS_VERBOSE) << "MacBook Pro not using internal speakers"; + } + + // Add a listener to determine if the status changes. + WEBRTC_CA_LOG_WARN(AudioObjectAddPropertyListener( + _outputDeviceID, &propertyAddress, &objectListenerProc, this)); + } + } + + // Get current stream description + propertyAddress.mSelector = kAudioDevicePropertyStreamFormat; + memset(&_outStreamFormat, 0, sizeof(_outStreamFormat)); + size = sizeof(_outStreamFormat); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &_outStreamFormat)); + + if (_outStreamFormat.mFormatID != kAudioFormatLinearPCM) { + logCAMsg(rtc::LS_ERROR, "Unacceptable output stream format -> mFormatID", + (const char*)&_outStreamFormat.mFormatID); + return -1; + } + + if (_outStreamFormat.mChannelsPerFrame > N_DEVICE_CHANNELS) { + RTC_LOG(LS_ERROR) + << "Too many channels on output device (mChannelsPerFrame = " + << _outStreamFormat.mChannelsPerFrame << ")"; + return -1; + } + + if (_outStreamFormat.mFormatFlags & kAudioFormatFlagIsNonInterleaved) { + RTC_LOG(LS_ERROR) << "Non-interleaved audio data is not supported." + << "AudioHardware streams should not have this format."; + return -1; + } + + RTC_LOG(LS_VERBOSE) << "Ouput stream format:"; + RTC_LOG(LS_VERBOSE) << "mSampleRate = " << _outStreamFormat.mSampleRate + << ", mChannelsPerFrame = " + << _outStreamFormat.mChannelsPerFrame; + RTC_LOG(LS_VERBOSE) << "mBytesPerPacket = " + << _outStreamFormat.mBytesPerPacket + << ", mFramesPerPacket = " + << _outStreamFormat.mFramesPerPacket; + RTC_LOG(LS_VERBOSE) << "mBytesPerFrame = " << _outStreamFormat.mBytesPerFrame + << ", mBitsPerChannel = " + << _outStreamFormat.mBitsPerChannel; + RTC_LOG(LS_VERBOSE) << "mFormatFlags = " << _outStreamFormat.mFormatFlags; + logCAMsg(rtc::LS_VERBOSE, "mFormatID", + (const char*)&_outStreamFormat.mFormatID); + + // Our preferred format to work with. + if (_outStreamFormat.mChannelsPerFrame < 2) { + // Disable stereo playout when we only have one channel on the device. + _playChannels = 1; + RTC_LOG(LS_VERBOSE) << "Stereo playout unavailable on this device"; + } + WEBRTC_CA_RETURN_ON_ERR(SetDesiredPlayoutFormat()); + + // Listen for format changes. + propertyAddress.mSelector = kAudioDevicePropertyStreamFormat; + WEBRTC_CA_LOG_WARN(AudioObjectAddPropertyListener( + _outputDeviceID, &propertyAddress, &objectListenerProc, this)); + + // Listen for processor overloads. + propertyAddress.mSelector = kAudioDeviceProcessorOverload; + WEBRTC_CA_LOG_WARN(AudioObjectAddPropertyListener( + _outputDeviceID, &propertyAddress, &objectListenerProc, this)); + + if (_twoDevices || !_recIsInitialized) { + WEBRTC_CA_RETURN_ON_ERR(AudioDeviceCreateIOProcID( + _outputDeviceID, deviceIOProc, this, &_deviceIOProcID)); + } + + _playIsInitialized = true; + + return 0; +} + +int32_t AudioDeviceMac::InitRecording() { + RTC_LOG(LS_INFO) << "InitRecording"; + rtc::CritScope lock(&_critSect); + + if (_recording) { + return -1; + } + + if (!_inputDeviceIsSpecified) { + return -1; + } + + if (_recIsInitialized) { + return 0; + } + + // Initialize the microphone (devices might have been added or removed) + if (InitMicrophone() == -1) { + RTC_LOG(LS_WARNING) << "InitMicrophone() failed"; + } + + if (!SpeakerIsInitialized()) { + // Make this call to check if we are using + // one or two devices (_twoDevices) + bool available = false; + if (SpeakerIsAvailable(available) == -1) { + RTC_LOG(LS_WARNING) << "SpeakerIsAvailable() failed"; + } + } + + OSStatus err = noErr; + UInt32 size = 0; + + PaUtil_FlushRingBuffer(_paCaptureBuffer); + + _captureDelayUs = 0; + _captureLatencyUs = 0; + _captureDeviceIsAlive = 1; + _doStopRec = false; + + // Get current stream description + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyStreamFormat, kAudioDevicePropertyScopeInput, 0}; + memset(&_inStreamFormat, 0, sizeof(_inStreamFormat)); + size = sizeof(_inStreamFormat); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &_inStreamFormat)); + + if (_inStreamFormat.mFormatID != kAudioFormatLinearPCM) { + logCAMsg(rtc::LS_ERROR, "Unacceptable input stream format -> mFormatID", + (const char*)&_inStreamFormat.mFormatID); + return -1; + } + + if (_inStreamFormat.mChannelsPerFrame > N_DEVICE_CHANNELS) { + RTC_LOG(LS_ERROR) + << "Too many channels on input device (mChannelsPerFrame = " + << _inStreamFormat.mChannelsPerFrame << ")"; + return -1; + } + + const int io_block_size_samples = _inStreamFormat.mChannelsPerFrame * + _inStreamFormat.mSampleRate / 100 * + N_BLOCKS_IO; + if (io_block_size_samples > _captureBufSizeSamples) { + RTC_LOG(LS_ERROR) << "Input IO block size (" << io_block_size_samples + << ") is larger than ring buffer (" + << _captureBufSizeSamples << ")"; + return -1; + } + + RTC_LOG(LS_VERBOSE) << "Input stream format:"; + RTC_LOG(LS_VERBOSE) << "mSampleRate = " << _inStreamFormat.mSampleRate + << ", mChannelsPerFrame = " + << _inStreamFormat.mChannelsPerFrame; + RTC_LOG(LS_VERBOSE) << "mBytesPerPacket = " << _inStreamFormat.mBytesPerPacket + << ", mFramesPerPacket = " + << _inStreamFormat.mFramesPerPacket; + RTC_LOG(LS_VERBOSE) << "mBytesPerFrame = " << _inStreamFormat.mBytesPerFrame + << ", mBitsPerChannel = " + << _inStreamFormat.mBitsPerChannel; + RTC_LOG(LS_VERBOSE) << "mFormatFlags = " << _inStreamFormat.mFormatFlags; + logCAMsg(rtc::LS_VERBOSE, "mFormatID", + (const char*)&_inStreamFormat.mFormatID); + + // Our preferred format to work with + if (_inStreamFormat.mChannelsPerFrame >= 2 && (_recChannels == 2)) { + _inDesiredFormat.mChannelsPerFrame = 2; + } else { + // Disable stereo recording when we only have one channel on the device. + _inDesiredFormat.mChannelsPerFrame = 1; + _recChannels = 1; + RTC_LOG(LS_VERBOSE) << "Stereo recording unavailable on this device"; + } + + if (_ptrAudioBuffer) { + // Update audio buffer with the selected parameters + _ptrAudioBuffer->SetRecordingSampleRate(N_REC_SAMPLES_PER_SEC); + _ptrAudioBuffer->SetRecordingChannels((uint8_t)_recChannels); + } + + _inDesiredFormat.mSampleRate = N_REC_SAMPLES_PER_SEC; + _inDesiredFormat.mBytesPerPacket = + _inDesiredFormat.mChannelsPerFrame * sizeof(SInt16); + _inDesiredFormat.mFramesPerPacket = 1; + _inDesiredFormat.mBytesPerFrame = + _inDesiredFormat.mChannelsPerFrame * sizeof(SInt16); + _inDesiredFormat.mBitsPerChannel = sizeof(SInt16) * 8; + + _inDesiredFormat.mFormatFlags = + kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; +#ifdef WEBRTC_ARCH_BIG_ENDIAN + _inDesiredFormat.mFormatFlags |= kLinearPCMFormatFlagIsBigEndian; +#endif + _inDesiredFormat.mFormatID = kAudioFormatLinearPCM; + + WEBRTC_CA_RETURN_ON_ERR(AudioConverterNew(&_inStreamFormat, &_inDesiredFormat, + &_captureConverter)); + + // First try to set buffer size to desired value (10 ms * N_BLOCKS_IO) + // TODO(xians): investigate this block. + UInt32 bufByteCount = + (UInt32)((_inStreamFormat.mSampleRate / 1000.0) * 10.0 * N_BLOCKS_IO * + _inStreamFormat.mChannelsPerFrame * sizeof(Float32)); + if (_inStreamFormat.mFramesPerPacket != 0) { + if (bufByteCount % _inStreamFormat.mFramesPerPacket != 0) { + bufByteCount = + ((UInt32)(bufByteCount / _inStreamFormat.mFramesPerPacket) + 1) * + _inStreamFormat.mFramesPerPacket; + } + } + + // Ensure the buffer size is within the acceptable range provided by the + // device. + propertyAddress.mSelector = kAudioDevicePropertyBufferSizeRange; + AudioValueRange range; + size = sizeof(range); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &range)); + if (range.mMinimum > bufByteCount) { + bufByteCount = range.mMinimum; + } else if (range.mMaximum < bufByteCount) { + bufByteCount = range.mMaximum; + } + + propertyAddress.mSelector = kAudioDevicePropertyBufferSize; + size = sizeof(bufByteCount); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, size, &bufByteCount)); + + // Get capture device latency + propertyAddress.mSelector = kAudioDevicePropertyLatency; + UInt32 latency = 0; + size = sizeof(UInt32); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &latency)); + _captureLatencyUs = (UInt32)((1.0e6 * latency) / _inStreamFormat.mSampleRate); + + // Get capture stream latency + propertyAddress.mSelector = kAudioDevicePropertyStreams; + AudioStreamID stream = 0; + size = sizeof(AudioStreamID); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &stream)); + propertyAddress.mSelector = kAudioStreamPropertyLatency; + size = sizeof(UInt32); + latency = 0; + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &latency)); + _captureLatencyUs += + (UInt32)((1.0e6 * latency) / _inStreamFormat.mSampleRate); + + // Listen for format changes + // TODO(xians): should we be using kAudioDevicePropertyDeviceHasChanged? + propertyAddress.mSelector = kAudioDevicePropertyStreamFormat; + WEBRTC_CA_LOG_WARN(AudioObjectAddPropertyListener( + _inputDeviceID, &propertyAddress, &objectListenerProc, this)); + + // Listen for processor overloads + propertyAddress.mSelector = kAudioDeviceProcessorOverload; + WEBRTC_CA_LOG_WARN(AudioObjectAddPropertyListener( + _inputDeviceID, &propertyAddress, &objectListenerProc, this)); + + if (_twoDevices) { + WEBRTC_CA_RETURN_ON_ERR(AudioDeviceCreateIOProcID( + _inputDeviceID, inDeviceIOProc, this, &_inDeviceIOProcID)); + } else if (!_playIsInitialized) { + WEBRTC_CA_RETURN_ON_ERR(AudioDeviceCreateIOProcID( + _inputDeviceID, deviceIOProc, this, &_deviceIOProcID)); + } + + // Mark recording side as initialized + _recIsInitialized = true; + + return 0; +} + +int32_t AudioDeviceMac::StartRecording() { + RTC_LOG(LS_INFO) << "StartRecording"; + rtc::CritScope lock(&_critSect); + + if (!_recIsInitialized) { + return -1; + } + + if (_recording) { + return 0; + } + + if (!_initialized) { + RTC_LOG(LS_ERROR) << "Recording worker thread has not been started"; + return -1; + } + + RTC_DCHECK(!capture_worker_thread_.get()); + capture_worker_thread_.reset( + new rtc::PlatformThread(RunCapture, this, "CaptureWorkerThread")); + RTC_DCHECK(capture_worker_thread_.get()); + capture_worker_thread_->Start(); + capture_worker_thread_->SetPriority(rtc::kRealtimePriority); + + OSStatus err = noErr; + if (_twoDevices) { + WEBRTC_CA_RETURN_ON_ERR( + AudioDeviceStart(_inputDeviceID, _inDeviceIOProcID)); + } else if (!_playing) { + WEBRTC_CA_RETURN_ON_ERR(AudioDeviceStart(_inputDeviceID, _deviceIOProcID)); + } + + _recording = true; + + return 0; +} + +int32_t AudioDeviceMac::StopRecording() { + RTC_LOG(LS_INFO) << "StopRecording"; + rtc::CritScope lock(&_critSect); + + if (!_recIsInitialized) { + return 0; + } + + OSStatus err = noErr; + int32_t captureDeviceIsAlive = AtomicGet32(&_captureDeviceIsAlive); + if (_twoDevices && captureDeviceIsAlive == 1) { + // Recording side uses its own dedicated device and IOProc. + if (_recording) { + _recording = false; + _doStopRec = true; // Signal to io proc to stop audio device + _critSect.Leave(); // Cannot be under lock, risk of deadlock + if (kEventTimeout == _stopEventRec.Wait(2000)) { + rtc::CritScope critScoped(&_critSect); + RTC_LOG(LS_WARNING) << "Timed out stopping the capture IOProc." + << "We may have failed to detect a device removal."; + WEBRTC_CA_LOG_WARN(AudioDeviceStop(_inputDeviceID, _inDeviceIOProcID)); + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_inputDeviceID, _inDeviceIOProcID)); + } + _critSect.Enter(); + _doStopRec = false; + RTC_LOG(LS_INFO) << "Recording stopped (input device)"; + } else if (_recIsInitialized) { + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_inputDeviceID, _inDeviceIOProcID)); + RTC_LOG(LS_INFO) << "Recording uninitialized (input device)"; + } + } else { + // We signal a stop for a shared device even when rendering has + // not yet ended. This is to ensure the IOProc will return early as + // intended (by checking |_recording|) before accessing + // resources we free below (e.g. the capture converter). + // + // In the case of a shared devcie, the IOProc will verify + // rendering has ended before stopping itself. + if (_recording && captureDeviceIsAlive == 1) { + _recording = false; + _doStop = true; // Signal to io proc to stop audio device + _critSect.Leave(); // Cannot be under lock, risk of deadlock + if (kEventTimeout == _stopEvent.Wait(2000)) { + rtc::CritScope critScoped(&_critSect); + RTC_LOG(LS_WARNING) << "Timed out stopping the shared IOProc." + << "We may have failed to detect a device removal."; + // We assume rendering on a shared device has stopped as well if + // the IOProc times out. + WEBRTC_CA_LOG_WARN(AudioDeviceStop(_outputDeviceID, _deviceIOProcID)); + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_outputDeviceID, _deviceIOProcID)); + } + _critSect.Enter(); + _doStop = false; + RTC_LOG(LS_INFO) << "Recording stopped (shared device)"; + } else if (_recIsInitialized && !_playing && !_playIsInitialized) { + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_outputDeviceID, _deviceIOProcID)); + RTC_LOG(LS_INFO) << "Recording uninitialized (shared device)"; + } + } + + // Setting this signal will allow the worker thread to be stopped. + AtomicSet32(&_captureDeviceIsAlive, 0); + + if (capture_worker_thread_.get()) { + _critSect.Leave(); + capture_worker_thread_->Stop(); + capture_worker_thread_.reset(); + _critSect.Enter(); + } + + WEBRTC_CA_LOG_WARN(AudioConverterDispose(_captureConverter)); + + // Remove listeners. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyStreamFormat, kAudioDevicePropertyScopeInput, 0}; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + _inputDeviceID, &propertyAddress, &objectListenerProc, this)); + + propertyAddress.mSelector = kAudioDeviceProcessorOverload; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + _inputDeviceID, &propertyAddress, &objectListenerProc, this)); + + _recIsInitialized = false; + _recording = false; + + return 0; +} + +bool AudioDeviceMac::RecordingIsInitialized() const { + return (_recIsInitialized); +} + +bool AudioDeviceMac::Recording() const { + return (_recording); +} + +bool AudioDeviceMac::PlayoutIsInitialized() const { + return (_playIsInitialized); +} + +int32_t AudioDeviceMac::StartPlayout() { + RTC_LOG(LS_INFO) << "StartPlayout"; + rtc::CritScope lock(&_critSect); + + if (!_playIsInitialized) { + return -1; + } + + if (_playing) { + return 0; + } + + RTC_DCHECK(!render_worker_thread_.get()); + render_worker_thread_.reset( + new rtc::PlatformThread(RunRender, this, "RenderWorkerThread")); + render_worker_thread_->Start(); + render_worker_thread_->SetPriority(rtc::kRealtimePriority); + + if (_twoDevices || !_recording) { + OSStatus err = noErr; + WEBRTC_CA_RETURN_ON_ERR(AudioDeviceStart(_outputDeviceID, _deviceIOProcID)); + } + _playing = true; + + return 0; +} + +int32_t AudioDeviceMac::StopPlayout() { + RTC_LOG(LS_INFO) << "StopPlayout"; + rtc::CritScope lock(&_critSect); + + if (!_playIsInitialized) { + return 0; + } + + OSStatus err = noErr; + int32_t renderDeviceIsAlive = AtomicGet32(&_renderDeviceIsAlive); + if (_playing && renderDeviceIsAlive == 1) { + // We signal a stop for a shared device even when capturing has not + // yet ended. This is to ensure the IOProc will return early as + // intended (by checking |_playing|) before accessing resources we + // free below (e.g. the render converter). + // + // In the case of a shared device, the IOProc will verify capturing + // has ended before stopping itself. + _playing = false; + _doStop = true; // Signal to io proc to stop audio device + _critSect.Leave(); // Cannot be under lock, risk of deadlock + if (kEventTimeout == _stopEvent.Wait(2000)) { + rtc::CritScope critScoped(&_critSect); + RTC_LOG(LS_WARNING) << "Timed out stopping the render IOProc." + << "We may have failed to detect a device removal."; + + // We assume capturing on a shared device has stopped as well if the + // IOProc times out. + WEBRTC_CA_LOG_WARN(AudioDeviceStop(_outputDeviceID, _deviceIOProcID)); + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_outputDeviceID, _deviceIOProcID)); + } + _critSect.Enter(); + _doStop = false; + RTC_LOG(LS_INFO) << "Playout stopped"; + } else if (_twoDevices && _playIsInitialized) { + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_outputDeviceID, _deviceIOProcID)); + RTC_LOG(LS_INFO) << "Playout uninitialized (output device)"; + } else if (!_twoDevices && _playIsInitialized && !_recIsInitialized) { + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_outputDeviceID, _deviceIOProcID)); + RTC_LOG(LS_INFO) << "Playout uninitialized (shared device)"; + } + + // Setting this signal will allow the worker thread to be stopped. + AtomicSet32(&_renderDeviceIsAlive, 0); + if (render_worker_thread_.get()) { + _critSect.Leave(); + render_worker_thread_->Stop(); + render_worker_thread_.reset(); + _critSect.Enter(); + } + + WEBRTC_CA_LOG_WARN(AudioConverterDispose(_renderConverter)); + + // Remove listeners. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyStreamFormat, kAudioDevicePropertyScopeOutput, 0}; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + _outputDeviceID, &propertyAddress, &objectListenerProc, this)); + + propertyAddress.mSelector = kAudioDeviceProcessorOverload; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + _outputDeviceID, &propertyAddress, &objectListenerProc, this)); + + if (_macBookPro) { + Boolean hasProperty = + AudioObjectHasProperty(_outputDeviceID, &propertyAddress); + if (hasProperty) { + propertyAddress.mSelector = kAudioDevicePropertyDataSource; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + _outputDeviceID, &propertyAddress, &objectListenerProc, this)); + } + } + + _playIsInitialized = false; + _playing = false; + + return 0; +} + +int32_t AudioDeviceMac::PlayoutDelay(uint16_t& delayMS) const { + int32_t renderDelayUs = AtomicGet32(&_renderDelayUs); + delayMS = + static_cast(1e-3 * (renderDelayUs + _renderLatencyUs) + 0.5); + return 0; +} + +bool AudioDeviceMac::Playing() const { + return (_playing); +} + +// ============================================================================ +// Private Methods +// ============================================================================ + +int32_t AudioDeviceMac::GetNumberDevices(const AudioObjectPropertyScope scope, + AudioDeviceID scopedDeviceIds[], + const uint32_t deviceListLength) { + OSStatus err = noErr; + + AudioObjectPropertyAddress propertyAddress = { + kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster}; + UInt32 size = 0; + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyDataSize( + kAudioObjectSystemObject, &propertyAddress, 0, NULL, &size)); + if (size == 0) { + RTC_LOG(LS_WARNING) << "No devices"; + return 0; + } + + AudioDeviceID* deviceIds = (AudioDeviceID*)malloc(size); + UInt32 numberDevices = size / sizeof(AudioDeviceID); + AudioBufferList* bufferList = NULL; + UInt32 numberScopedDevices = 0; + + // First check if there is a default device and list it + UInt32 hardwareProperty = 0; + if (scope == kAudioDevicePropertyScopeOutput) { + hardwareProperty = kAudioHardwarePropertyDefaultOutputDevice; + } else { + hardwareProperty = kAudioHardwarePropertyDefaultInputDevice; + } + + AudioObjectPropertyAddress propertyAddressDefault = { + hardwareProperty, kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster}; + + AudioDeviceID usedID; + UInt32 uintSize = sizeof(UInt32); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData(kAudioObjectSystemObject, + &propertyAddressDefault, 0, + NULL, &uintSize, &usedID)); + if (usedID != kAudioDeviceUnknown) { + scopedDeviceIds[numberScopedDevices] = usedID; + numberScopedDevices++; + } else { + RTC_LOG(LS_WARNING) << "GetNumberDevices(): Default device unknown"; + } + + // Then list the rest of the devices + bool listOK = true; + + WEBRTC_CA_LOG_ERR(AudioObjectGetPropertyData( + kAudioObjectSystemObject, &propertyAddress, 0, NULL, &size, deviceIds)); + if (err != noErr) { + listOK = false; + } else { + propertyAddress.mSelector = kAudioDevicePropertyStreamConfiguration; + propertyAddress.mScope = scope; + propertyAddress.mElement = 0; + for (UInt32 i = 0; i < numberDevices; i++) { + // Check for input channels + WEBRTC_CA_LOG_ERR(AudioObjectGetPropertyDataSize( + deviceIds[i], &propertyAddress, 0, NULL, &size)); + if (err == kAudioHardwareBadDeviceError) { + // This device doesn't actually exist; continue iterating. + continue; + } else if (err != noErr) { + listOK = false; + break; + } + + bufferList = (AudioBufferList*)malloc(size); + WEBRTC_CA_LOG_ERR(AudioObjectGetPropertyData( + deviceIds[i], &propertyAddress, 0, NULL, &size, bufferList)); + if (err != noErr) { + listOK = false; + break; + } + + if (bufferList->mNumberBuffers > 0) { + if (numberScopedDevices >= deviceListLength) { + RTC_LOG(LS_ERROR) << "Device list is not long enough"; + listOK = false; + break; + } + + scopedDeviceIds[numberScopedDevices] = deviceIds[i]; + numberScopedDevices++; + } + + free(bufferList); + bufferList = NULL; + } // for + } + + if (!listOK) { + if (deviceIds) { + free(deviceIds); + deviceIds = NULL; + } + + if (bufferList) { + free(bufferList); + bufferList = NULL; + } + + return -1; + } + + // Happy ending + if (deviceIds) { + free(deviceIds); + deviceIds = NULL; + } + + return numberScopedDevices; +} + +int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, + const uint16_t index, + char* name) { + OSStatus err = noErr; + UInt32 len = kAdmMaxDeviceNameSize; + AudioDeviceID deviceIds[MaxNumberDevices]; + + int numberDevices = GetNumberDevices(scope, deviceIds, MaxNumberDevices); + if (numberDevices < 0) { + return -1; + } else if (numberDevices == 0) { + RTC_LOG(LS_ERROR) << "No devices"; + return -1; + } + + // If the number is below the number of devices, assume it's "WEBRTC ID" + // otherwise assume it's a CoreAudio ID + AudioDeviceID usedID; + + // Check if there is a default device + bool isDefaultDevice = false; + if (index == 0) { + UInt32 hardwareProperty = 0; + if (scope == kAudioDevicePropertyScopeOutput) { + hardwareProperty = kAudioHardwarePropertyDefaultOutputDevice; + } else { + hardwareProperty = kAudioHardwarePropertyDefaultInputDevice; + } + AudioObjectPropertyAddress propertyAddress = { + hardwareProperty, kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster}; + UInt32 size = sizeof(UInt32); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + kAudioObjectSystemObject, &propertyAddress, 0, NULL, &size, &usedID)); + if (usedID == kAudioDeviceUnknown) { + RTC_LOG(LS_WARNING) << "GetDeviceName(): Default device unknown"; + } else { + isDefaultDevice = true; + } + } + + AudioObjectPropertyAddress propertyAddress = {kAudioDevicePropertyDeviceName, + scope, 0}; + + if (isDefaultDevice) { + char devName[len]; + + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData(usedID, &propertyAddress, + 0, NULL, &len, devName)); + + sprintf(name, "default (%s)", devName); + } else { + if (index < numberDevices) { + usedID = deviceIds[index]; + } else { + usedID = index; + } + + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData(usedID, &propertyAddress, + 0, NULL, &len, name)); + } + + return 0; +} + +int32_t AudioDeviceMac::InitDevice(const uint16_t userDeviceIndex, + AudioDeviceID& deviceId, + const bool isInput) { + OSStatus err = noErr; + UInt32 size = 0; + AudioObjectPropertyScope deviceScope; + AudioObjectPropertySelector defaultDeviceSelector; + AudioDeviceID deviceIds[MaxNumberDevices]; + + if (isInput) { + deviceScope = kAudioDevicePropertyScopeInput; + defaultDeviceSelector = kAudioHardwarePropertyDefaultInputDevice; + } else { + deviceScope = kAudioDevicePropertyScopeOutput; + defaultDeviceSelector = kAudioHardwarePropertyDefaultOutputDevice; + } + + AudioObjectPropertyAddress propertyAddress = { + defaultDeviceSelector, kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster}; + + // Get the actual device IDs + int numberDevices = + GetNumberDevices(deviceScope, deviceIds, MaxNumberDevices); + if (numberDevices < 0) { + return -1; + } else if (numberDevices == 0) { + RTC_LOG(LS_ERROR) << "InitDevice(): No devices"; + return -1; + } + + bool isDefaultDevice = false; + deviceId = kAudioDeviceUnknown; + if (userDeviceIndex == 0) { + // Try to use default system device + size = sizeof(AudioDeviceID); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + kAudioObjectSystemObject, &propertyAddress, 0, NULL, &size, &deviceId)); + if (deviceId == kAudioDeviceUnknown) { + RTC_LOG(LS_WARNING) << "No default device exists"; + } else { + isDefaultDevice = true; + } + } + + if (!isDefaultDevice) { + deviceId = deviceIds[userDeviceIndex]; + } + + // Obtain device name and manufacturer for logging. + // Also use this as a test to ensure a user-set device ID is valid. + char devName[128]; + char devManf[128]; + memset(devName, 0, sizeof(devName)); + memset(devManf, 0, sizeof(devManf)); + + propertyAddress.mSelector = kAudioDevicePropertyDeviceName; + propertyAddress.mScope = deviceScope; + propertyAddress.mElement = 0; + size = sizeof(devName); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData(deviceId, &propertyAddress, + 0, NULL, &size, devName)); + + propertyAddress.mSelector = kAudioDevicePropertyDeviceManufacturer; + size = sizeof(devManf); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData(deviceId, &propertyAddress, + 0, NULL, &size, devManf)); + + if (isInput) { + RTC_LOG(LS_INFO) << "Input device: " << devManf << " " << devName; + } else { + RTC_LOG(LS_INFO) << "Output device: " << devManf << " " << devName; + } + + return 0; +} + +OSStatus AudioDeviceMac::SetDesiredPlayoutFormat() { + // Our preferred format to work with. + _outDesiredFormat.mSampleRate = N_PLAY_SAMPLES_PER_SEC; + _outDesiredFormat.mChannelsPerFrame = _playChannels; + + if (_ptrAudioBuffer) { + // Update audio buffer with the selected parameters. + _ptrAudioBuffer->SetPlayoutSampleRate(N_PLAY_SAMPLES_PER_SEC); + _ptrAudioBuffer->SetPlayoutChannels((uint8_t)_playChannels); + } + + _renderDelayOffsetSamples = + _renderBufSizeSamples - N_BUFFERS_OUT * ENGINE_PLAY_BUF_SIZE_IN_SAMPLES * + _outDesiredFormat.mChannelsPerFrame; + + _outDesiredFormat.mBytesPerPacket = + _outDesiredFormat.mChannelsPerFrame * sizeof(SInt16); + // In uncompressed audio, a packet is one frame. + _outDesiredFormat.mFramesPerPacket = 1; + _outDesiredFormat.mBytesPerFrame = + _outDesiredFormat.mChannelsPerFrame * sizeof(SInt16); + _outDesiredFormat.mBitsPerChannel = sizeof(SInt16) * 8; + + _outDesiredFormat.mFormatFlags = + kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; +#ifdef WEBRTC_ARCH_BIG_ENDIAN + _outDesiredFormat.mFormatFlags |= kLinearPCMFormatFlagIsBigEndian; +#endif + _outDesiredFormat.mFormatID = kAudioFormatLinearPCM; + + OSStatus err = noErr; + WEBRTC_CA_RETURN_ON_ERR(AudioConverterNew( + &_outDesiredFormat, &_outStreamFormat, &_renderConverter)); + + // Try to set buffer size to desired value set to 20ms. + const uint16_t kPlayBufDelayFixed = 20; + UInt32 bufByteCount = static_cast( + (_outStreamFormat.mSampleRate / 1000.0) * kPlayBufDelayFixed * + _outStreamFormat.mChannelsPerFrame * sizeof(Float32)); + if (_outStreamFormat.mFramesPerPacket != 0) { + if (bufByteCount % _outStreamFormat.mFramesPerPacket != 0) { + bufByteCount = (static_cast(bufByteCount / + _outStreamFormat.mFramesPerPacket) + + 1) * + _outStreamFormat.mFramesPerPacket; + } + } + + // Ensure the buffer size is within the range provided by the device. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyDataSource, kAudioDevicePropertyScopeOutput, 0}; + propertyAddress.mSelector = kAudioDevicePropertyBufferSizeRange; + AudioValueRange range; + UInt32 size = sizeof(range); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &range)); + if (range.mMinimum > bufByteCount) { + bufByteCount = range.mMinimum; + } else if (range.mMaximum < bufByteCount) { + bufByteCount = range.mMaximum; + } + + propertyAddress.mSelector = kAudioDevicePropertyBufferSize; + size = sizeof(bufByteCount); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, size, &bufByteCount)); + + // Get render device latency. + propertyAddress.mSelector = kAudioDevicePropertyLatency; + UInt32 latency = 0; + size = sizeof(UInt32); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &latency)); + _renderLatencyUs = + static_cast((1.0e6 * latency) / _outStreamFormat.mSampleRate); + + // Get render stream latency. + propertyAddress.mSelector = kAudioDevicePropertyStreams; + AudioStreamID stream = 0; + size = sizeof(AudioStreamID); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &stream)); + propertyAddress.mSelector = kAudioStreamPropertyLatency; + size = sizeof(UInt32); + latency = 0; + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &latency)); + _renderLatencyUs += + static_cast((1.0e6 * latency) / _outStreamFormat.mSampleRate); + + RTC_LOG(LS_VERBOSE) << "initial playout status: _renderDelayOffsetSamples=" + << _renderDelayOffsetSamples + << ", _renderDelayUs=" << _renderDelayUs + << ", _renderLatencyUs=" << _renderLatencyUs; + return 0; +} + +OSStatus AudioDeviceMac::objectListenerProc( + AudioObjectID objectId, + UInt32 numberAddresses, + const AudioObjectPropertyAddress addresses[], + void* clientData) { + AudioDeviceMac* ptrThis = (AudioDeviceMac*)clientData; + RTC_DCHECK(ptrThis != NULL); + + ptrThis->implObjectListenerProc(objectId, numberAddresses, addresses); + + // AudioObjectPropertyListenerProc functions are supposed to return 0 + return 0; +} + +OSStatus AudioDeviceMac::implObjectListenerProc( + const AudioObjectID objectId, + const UInt32 numberAddresses, + const AudioObjectPropertyAddress addresses[]) { + RTC_LOG(LS_VERBOSE) << "AudioDeviceMac::implObjectListenerProc()"; + + for (UInt32 i = 0; i < numberAddresses; i++) { + if (addresses[i].mSelector == kAudioHardwarePropertyDevices) { + HandleDeviceChange(); + } else if (addresses[i].mSelector == kAudioDevicePropertyStreamFormat) { + HandleStreamFormatChange(objectId, addresses[i]); + } else if (addresses[i].mSelector == kAudioDevicePropertyDataSource) { + HandleDataSourceChange(objectId, addresses[i]); + } else if (addresses[i].mSelector == kAudioDeviceProcessorOverload) { + HandleProcessorOverload(addresses[i]); + } + } + + return 0; +} + +int32_t AudioDeviceMac::HandleDeviceChange() { + OSStatus err = noErr; + + RTC_LOG(LS_VERBOSE) << "kAudioHardwarePropertyDevices"; + + // A device has changed. Check if our registered devices have been removed. + // Ensure the devices have been initialized, meaning the IDs are valid. + if (MicrophoneIsInitialized()) { + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyDeviceIsAlive, kAudioDevicePropertyScopeInput, 0}; + UInt32 deviceIsAlive = 1; + UInt32 size = sizeof(UInt32); + err = AudioObjectGetPropertyData(_inputDeviceID, &propertyAddress, 0, NULL, + &size, &deviceIsAlive); + + if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { + RTC_LOG(LS_WARNING) << "Capture device is not alive (probably removed)"; + AtomicSet32(&_captureDeviceIsAlive, 0); + _mixerManager.CloseMicrophone(); + } else if (err != noErr) { + logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", + (const char*)&err); + return -1; + } + } + + if (SpeakerIsInitialized()) { + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyDeviceIsAlive, kAudioDevicePropertyScopeOutput, 0}; + UInt32 deviceIsAlive = 1; + UInt32 size = sizeof(UInt32); + err = AudioObjectGetPropertyData(_outputDeviceID, &propertyAddress, 0, NULL, + &size, &deviceIsAlive); + + if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { + RTC_LOG(LS_WARNING) << "Render device is not alive (probably removed)"; + AtomicSet32(&_renderDeviceIsAlive, 0); + _mixerManager.CloseSpeaker(); + } else if (err != noErr) { + logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", + (const char*)&err); + return -1; + } + } + + return 0; +} + +int32_t AudioDeviceMac::HandleStreamFormatChange( + const AudioObjectID objectId, + const AudioObjectPropertyAddress propertyAddress) { + OSStatus err = noErr; + + RTC_LOG(LS_VERBOSE) << "Stream format changed"; + + if (objectId != _inputDeviceID && objectId != _outputDeviceID) { + return 0; + } + + // Get the new device format + AudioStreamBasicDescription streamFormat; + UInt32 size = sizeof(streamFormat); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + objectId, &propertyAddress, 0, NULL, &size, &streamFormat)); + + if (streamFormat.mFormatID != kAudioFormatLinearPCM) { + logCAMsg(rtc::LS_ERROR, "Unacceptable input stream format -> mFormatID", + (const char*)&streamFormat.mFormatID); + return -1; + } + + if (streamFormat.mChannelsPerFrame > N_DEVICE_CHANNELS) { + RTC_LOG(LS_ERROR) << "Too many channels on device (mChannelsPerFrame = " + << streamFormat.mChannelsPerFrame << ")"; + return -1; + } + + if (_ptrAudioBuffer && streamFormat.mChannelsPerFrame != _recChannels) { + RTC_LOG(LS_ERROR) << "Changing channels not supported (mChannelsPerFrame = " + << streamFormat.mChannelsPerFrame << ")"; + return -1; + } + + RTC_LOG(LS_VERBOSE) << "Stream format:"; + RTC_LOG(LS_VERBOSE) << "mSampleRate = " << streamFormat.mSampleRate + << ", mChannelsPerFrame = " + << streamFormat.mChannelsPerFrame; + RTC_LOG(LS_VERBOSE) << "mBytesPerPacket = " << streamFormat.mBytesPerPacket + << ", mFramesPerPacket = " + << streamFormat.mFramesPerPacket; + RTC_LOG(LS_VERBOSE) << "mBytesPerFrame = " << streamFormat.mBytesPerFrame + << ", mBitsPerChannel = " << streamFormat.mBitsPerChannel; + RTC_LOG(LS_VERBOSE) << "mFormatFlags = " << streamFormat.mFormatFlags; + logCAMsg(rtc::LS_VERBOSE, "mFormatID", (const char*)&streamFormat.mFormatID); + + if (propertyAddress.mScope == kAudioDevicePropertyScopeInput) { + const int io_block_size_samples = streamFormat.mChannelsPerFrame * + streamFormat.mSampleRate / 100 * + N_BLOCKS_IO; + if (io_block_size_samples > _captureBufSizeSamples) { + RTC_LOG(LS_ERROR) << "Input IO block size (" << io_block_size_samples + << ") is larger than ring buffer (" + << _captureBufSizeSamples << ")"; + return -1; + } + + memcpy(&_inStreamFormat, &streamFormat, sizeof(streamFormat)); + + if (_inStreamFormat.mChannelsPerFrame >= 2 && (_recChannels == 2)) { + _inDesiredFormat.mChannelsPerFrame = 2; + } else { + // Disable stereo recording when we only have one channel on the device. + _inDesiredFormat.mChannelsPerFrame = 1; + _recChannels = 1; + RTC_LOG(LS_VERBOSE) << "Stereo recording unavailable on this device"; + } + + // Recreate the converter with the new format + // TODO(xians): make this thread safe + WEBRTC_CA_RETURN_ON_ERR(AudioConverterDispose(_captureConverter)); + + WEBRTC_CA_RETURN_ON_ERR(AudioConverterNew(&streamFormat, &_inDesiredFormat, + &_captureConverter)); + } else { + memcpy(&_outStreamFormat, &streamFormat, sizeof(streamFormat)); + + // Our preferred format to work with + if (_outStreamFormat.mChannelsPerFrame < 2) { + _playChannels = 1; + RTC_LOG(LS_VERBOSE) << "Stereo playout unavailable on this device"; + } + WEBRTC_CA_RETURN_ON_ERR(SetDesiredPlayoutFormat()); + } + return 0; +} + +int32_t AudioDeviceMac::HandleDataSourceChange( + const AudioObjectID objectId, + const AudioObjectPropertyAddress propertyAddress) { + OSStatus err = noErr; + + if (_macBookPro && + propertyAddress.mScope == kAudioDevicePropertyScopeOutput) { + RTC_LOG(LS_VERBOSE) << "Data source changed"; + + _macBookProPanRight = false; + UInt32 dataSource = 0; + UInt32 size = sizeof(UInt32); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + objectId, &propertyAddress, 0, NULL, &size, &dataSource)); + if (dataSource == 'ispk') { + _macBookProPanRight = true; + RTC_LOG(LS_VERBOSE) + << "MacBook Pro using internal speakers; stereo panning right"; + } else { + RTC_LOG(LS_VERBOSE) << "MacBook Pro not using internal speakers"; + } + } + + return 0; +} +int32_t AudioDeviceMac::HandleProcessorOverload( + const AudioObjectPropertyAddress propertyAddress) { + // TODO(xians): we probably want to notify the user in some way of the + // overload. However, the Windows interpretations of these errors seem to + // be more severe than what ProcessorOverload is thrown for. + // + // We don't log the notification, as it's sent from the HAL's IO thread. We + // don't want to slow it down even further. + if (propertyAddress.mScope == kAudioDevicePropertyScopeInput) { + // RTC_LOG(LS_WARNING) << "Capture processor // overload"; + //_callback->ProblemIsReported( + // SndCardStreamObserver::ERecordingProblem); + } else { + // RTC_LOG(LS_WARNING) << "Render processor overload"; + //_callback->ProblemIsReported( + // SndCardStreamObserver::EPlaybackProblem); + } + + return 0; +} + +// ============================================================================ +// Thread Methods +// ============================================================================ + +OSStatus AudioDeviceMac::deviceIOProc(AudioDeviceID, + const AudioTimeStamp*, + const AudioBufferList* inputData, + const AudioTimeStamp* inputTime, + AudioBufferList* outputData, + const AudioTimeStamp* outputTime, + void* clientData) { + AudioDeviceMac* ptrThis = (AudioDeviceMac*)clientData; + RTC_DCHECK(ptrThis != NULL); + + ptrThis->implDeviceIOProc(inputData, inputTime, outputData, outputTime); + + // AudioDeviceIOProc functions are supposed to return 0 + return 0; +} + +OSStatus AudioDeviceMac::outConverterProc(AudioConverterRef, + UInt32* numberDataPackets, + AudioBufferList* data, + AudioStreamPacketDescription**, + void* userData) { + AudioDeviceMac* ptrThis = (AudioDeviceMac*)userData; + RTC_DCHECK(ptrThis != NULL); + + return ptrThis->implOutConverterProc(numberDataPackets, data); +} + +OSStatus AudioDeviceMac::inDeviceIOProc(AudioDeviceID, + const AudioTimeStamp*, + const AudioBufferList* inputData, + const AudioTimeStamp* inputTime, + AudioBufferList*, + const AudioTimeStamp*, + void* clientData) { + AudioDeviceMac* ptrThis = (AudioDeviceMac*)clientData; + RTC_DCHECK(ptrThis != NULL); + + ptrThis->implInDeviceIOProc(inputData, inputTime); + + // AudioDeviceIOProc functions are supposed to return 0 + return 0; +} + +OSStatus AudioDeviceMac::inConverterProc( + AudioConverterRef, + UInt32* numberDataPackets, + AudioBufferList* data, + AudioStreamPacketDescription** /*dataPacketDescription*/, + void* userData) { + AudioDeviceMac* ptrThis = static_cast(userData); + RTC_DCHECK(ptrThis != NULL); + + return ptrThis->implInConverterProc(numberDataPackets, data); +} + +OSStatus AudioDeviceMac::implDeviceIOProc(const AudioBufferList* inputData, + const AudioTimeStamp* inputTime, + AudioBufferList* outputData, + const AudioTimeStamp* outputTime) { + OSStatus err = noErr; + UInt64 outputTimeNs = AudioConvertHostTimeToNanos(outputTime->mHostTime); + UInt64 nowNs = AudioConvertHostTimeToNanos(AudioGetCurrentHostTime()); + + if (!_twoDevices && _recording) { + implInDeviceIOProc(inputData, inputTime); + } + + // Check if we should close down audio device + // Double-checked locking optimization to remove locking overhead + if (_doStop) { + _critSect.Enter(); + if (_doStop) { + if (_twoDevices || (!_recording && !_playing)) { + // In the case of a shared device, the single driving ioProc + // is stopped here + WEBRTC_CA_LOG_ERR(AudioDeviceStop(_outputDeviceID, _deviceIOProcID)); + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_outputDeviceID, _deviceIOProcID)); + if (err == noErr) { + RTC_LOG(LS_VERBOSE) << "Playout or shared device stopped"; + } + } + + _doStop = false; + _stopEvent.Set(); + _critSect.Leave(); + return 0; + } + _critSect.Leave(); + } + + if (!_playing) { + // This can be the case when a shared device is capturing but not + // rendering. We allow the checks above before returning to avoid a + // timeout when capturing is stopped. + return 0; + } + + RTC_DCHECK(_outStreamFormat.mBytesPerFrame != 0); + UInt32 size = + outputData->mBuffers->mDataByteSize / _outStreamFormat.mBytesPerFrame; + + // TODO(xians): signal an error somehow? + err = AudioConverterFillComplexBuffer(_renderConverter, outConverterProc, + this, &size, outputData, NULL); + if (err != noErr) { + if (err == 1) { + // This is our own error. + RTC_LOG(LS_ERROR) << "Error in AudioConverterFillComplexBuffer()"; + return 1; + } else { + logCAMsg(rtc::LS_ERROR, "Error in AudioConverterFillComplexBuffer()", + (const char*)&err); + return 1; + } + } + + PaRingBufferSize bufSizeSamples = + PaUtil_GetRingBufferReadAvailable(_paRenderBuffer); + + int32_t renderDelayUs = + static_cast(1e-3 * (outputTimeNs - nowNs) + 0.5); + renderDelayUs += static_cast( + (1.0e6 * bufSizeSamples) / _outDesiredFormat.mChannelsPerFrame / + _outDesiredFormat.mSampleRate + + 0.5); + + AtomicSet32(&_renderDelayUs, renderDelayUs); + + return 0; +} + +OSStatus AudioDeviceMac::implOutConverterProc(UInt32* numberDataPackets, + AudioBufferList* data) { + RTC_DCHECK(data->mNumberBuffers == 1); + PaRingBufferSize numSamples = + *numberDataPackets * _outDesiredFormat.mChannelsPerFrame; + + data->mBuffers->mNumberChannels = _outDesiredFormat.mChannelsPerFrame; + // Always give the converter as much as it wants, zero padding as required. + data->mBuffers->mDataByteSize = + *numberDataPackets * _outDesiredFormat.mBytesPerPacket; + data->mBuffers->mData = _renderConvertData; + memset(_renderConvertData, 0, sizeof(_renderConvertData)); + + PaUtil_ReadRingBuffer(_paRenderBuffer, _renderConvertData, numSamples); + + kern_return_t kernErr = semaphore_signal_all(_renderSemaphore); + if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_signal_all() error: " << kernErr; + return 1; + } + + return 0; +} + +OSStatus AudioDeviceMac::implInDeviceIOProc(const AudioBufferList* inputData, + const AudioTimeStamp* inputTime) { + OSStatus err = noErr; + UInt64 inputTimeNs = AudioConvertHostTimeToNanos(inputTime->mHostTime); + UInt64 nowNs = AudioConvertHostTimeToNanos(AudioGetCurrentHostTime()); + + // Check if we should close down audio device + // Double-checked locking optimization to remove locking overhead + if (_doStopRec) { + _critSect.Enter(); + if (_doStopRec) { + // This will be signalled only when a shared device is not in use. + WEBRTC_CA_LOG_ERR(AudioDeviceStop(_inputDeviceID, _inDeviceIOProcID)); + WEBRTC_CA_LOG_WARN( + AudioDeviceDestroyIOProcID(_inputDeviceID, _inDeviceIOProcID)); + if (err == noErr) { + RTC_LOG(LS_VERBOSE) << "Recording device stopped"; + } + + _doStopRec = false; + _stopEventRec.Set(); + _critSect.Leave(); + return 0; + } + _critSect.Leave(); + } + + if (!_recording) { + // Allow above checks to avoid a timeout on stopping capture. + return 0; + } + + PaRingBufferSize bufSizeSamples = + PaUtil_GetRingBufferReadAvailable(_paCaptureBuffer); + + int32_t captureDelayUs = + static_cast(1e-3 * (nowNs - inputTimeNs) + 0.5); + captureDelayUs += static_cast((1.0e6 * bufSizeSamples) / + _inStreamFormat.mChannelsPerFrame / + _inStreamFormat.mSampleRate + + 0.5); + + AtomicSet32(&_captureDelayUs, captureDelayUs); + + RTC_DCHECK(inputData->mNumberBuffers == 1); + PaRingBufferSize numSamples = inputData->mBuffers->mDataByteSize * + _inStreamFormat.mChannelsPerFrame / + _inStreamFormat.mBytesPerPacket; + PaUtil_WriteRingBuffer(_paCaptureBuffer, inputData->mBuffers->mData, + numSamples); + + kern_return_t kernErr = semaphore_signal_all(_captureSemaphore); + if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_signal_all() error: " << kernErr; + } + + return err; +} + +OSStatus AudioDeviceMac::implInConverterProc(UInt32* numberDataPackets, + AudioBufferList* data) { + RTC_DCHECK(data->mNumberBuffers == 1); + PaRingBufferSize numSamples = + *numberDataPackets * _inStreamFormat.mChannelsPerFrame; + + while (PaUtil_GetRingBufferReadAvailable(_paCaptureBuffer) < numSamples) { + mach_timespec_t timeout; + timeout.tv_sec = 0; + timeout.tv_nsec = TIMER_PERIOD_MS; + + kern_return_t kernErr = semaphore_timedwait(_captureSemaphore, timeout); + if (kernErr == KERN_OPERATION_TIMED_OUT) { + int32_t signal = AtomicGet32(&_captureDeviceIsAlive); + if (signal == 0) { + // The capture device is no longer alive; stop the worker thread. + *numberDataPackets = 0; + return 1; + } + } else if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_wait() error: " << kernErr; + } + } + + // Pass the read pointer directly to the converter to avoid a memcpy. + void* dummyPtr; + PaRingBufferSize dummySize; + PaUtil_GetRingBufferReadRegions(_paCaptureBuffer, numSamples, + &data->mBuffers->mData, &numSamples, + &dummyPtr, &dummySize); + PaUtil_AdvanceRingBufferReadIndex(_paCaptureBuffer, numSamples); + + data->mBuffers->mNumberChannels = _inStreamFormat.mChannelsPerFrame; + *numberDataPackets = numSamples / _inStreamFormat.mChannelsPerFrame; + data->mBuffers->mDataByteSize = + *numberDataPackets * _inStreamFormat.mBytesPerPacket; + + return 0; +} + +bool AudioDeviceMac::RunRender(void* ptrThis) { + return static_cast(ptrThis)->RenderWorkerThread(); +} + +bool AudioDeviceMac::RenderWorkerThread() { + PaRingBufferSize numSamples = + ENGINE_PLAY_BUF_SIZE_IN_SAMPLES * _outDesiredFormat.mChannelsPerFrame; + while (PaUtil_GetRingBufferWriteAvailable(_paRenderBuffer) - + _renderDelayOffsetSamples < + numSamples) { + mach_timespec_t timeout; + timeout.tv_sec = 0; + timeout.tv_nsec = TIMER_PERIOD_MS; + + kern_return_t kernErr = semaphore_timedwait(_renderSemaphore, timeout); + if (kernErr == KERN_OPERATION_TIMED_OUT) { + int32_t signal = AtomicGet32(&_renderDeviceIsAlive); + if (signal == 0) { + // The render device is no longer alive; stop the worker thread. + return false; + } + } else if (kernErr != KERN_SUCCESS) { + RTC_LOG(LS_ERROR) << "semaphore_timedwait() error: " << kernErr; + } + } + + int8_t playBuffer[4 * ENGINE_PLAY_BUF_SIZE_IN_SAMPLES]; + + if (!_ptrAudioBuffer) { + RTC_LOG(LS_ERROR) << "capture AudioBuffer is invalid"; + return false; + } + + // Ask for new PCM data to be played out using the AudioDeviceBuffer. + uint32_t nSamples = + _ptrAudioBuffer->RequestPlayoutData(ENGINE_PLAY_BUF_SIZE_IN_SAMPLES); + + nSamples = _ptrAudioBuffer->GetPlayoutData(playBuffer); + if (nSamples != ENGINE_PLAY_BUF_SIZE_IN_SAMPLES) { + RTC_LOG(LS_ERROR) << "invalid number of output samples(" << nSamples << ")"; + } + + uint32_t nOutSamples = nSamples * _outDesiredFormat.mChannelsPerFrame; + + SInt16* pPlayBuffer = (SInt16*)&playBuffer; + if (_macBookProPanRight && (_playChannels == 2)) { + // Mix entirely into the right channel and zero the left channel. + SInt32 sampleInt32 = 0; + for (uint32_t sampleIdx = 0; sampleIdx < nOutSamples; sampleIdx += 2) { + sampleInt32 = pPlayBuffer[sampleIdx]; + sampleInt32 += pPlayBuffer[sampleIdx + 1]; + sampleInt32 /= 2; + + if (sampleInt32 > 32767) { + sampleInt32 = 32767; + } else if (sampleInt32 < -32768) { + sampleInt32 = -32768; + } + + pPlayBuffer[sampleIdx] = 0; + pPlayBuffer[sampleIdx + 1] = static_cast(sampleInt32); + } + } + + PaUtil_WriteRingBuffer(_paRenderBuffer, pPlayBuffer, nOutSamples); + + return true; +} + +bool AudioDeviceMac::RunCapture(void* ptrThis) { + return static_cast(ptrThis)->CaptureWorkerThread(); +} + +bool AudioDeviceMac::CaptureWorkerThread() { + OSStatus err = noErr; + UInt32 noRecSamples = + ENGINE_REC_BUF_SIZE_IN_SAMPLES * _inDesiredFormat.mChannelsPerFrame; + SInt16 recordBuffer[noRecSamples]; + UInt32 size = ENGINE_REC_BUF_SIZE_IN_SAMPLES; + + AudioBufferList engineBuffer; + engineBuffer.mNumberBuffers = 1; // Interleaved channels. + engineBuffer.mBuffers->mNumberChannels = _inDesiredFormat.mChannelsPerFrame; + engineBuffer.mBuffers->mDataByteSize = + _inDesiredFormat.mBytesPerPacket * noRecSamples; + engineBuffer.mBuffers->mData = recordBuffer; + + err = AudioConverterFillComplexBuffer(_captureConverter, inConverterProc, + this, &size, &engineBuffer, NULL); + if (err != noErr) { + if (err == 1) { + // This is our own error. + return false; + } else { + logCAMsg(rtc::LS_ERROR, "Error in AudioConverterFillComplexBuffer()", + (const char*)&err); + return false; + } + } + + // TODO(xians): what if the returned size is incorrect? + if (size == ENGINE_REC_BUF_SIZE_IN_SAMPLES) { + uint32_t currentMicLevel(0); + uint32_t newMicLevel(0); + int32_t msecOnPlaySide; + int32_t msecOnRecordSide; + + int32_t captureDelayUs = AtomicGet32(&_captureDelayUs); + int32_t renderDelayUs = AtomicGet32(&_renderDelayUs); + + msecOnPlaySide = + static_cast(1e-3 * (renderDelayUs + _renderLatencyUs) + 0.5); + msecOnRecordSide = + static_cast(1e-3 * (captureDelayUs + _captureLatencyUs) + 0.5); + + if (!_ptrAudioBuffer) { + RTC_LOG(LS_ERROR) << "capture AudioBuffer is invalid"; + return false; + } + + // store the recorded buffer (no action will be taken if the + // #recorded samples is not a full buffer) + _ptrAudioBuffer->SetRecordedBuffer((int8_t*)&recordBuffer, (uint32_t)size); + + if (AGC()) { + // Use mod to ensure we check the volume on the first pass. + if (get_mic_volume_counter_ms_ % kGetMicVolumeIntervalMs == 0) { + get_mic_volume_counter_ms_ = 0; + // store current mic level in the audio buffer if AGC is enabled + if (MicrophoneVolume(currentMicLevel) == 0) { + // this call does not affect the actual microphone volume + _ptrAudioBuffer->SetCurrentMicLevel(currentMicLevel); + } + } + get_mic_volume_counter_ms_ += kBufferSizeMs; + } + + _ptrAudioBuffer->SetVQEData(msecOnPlaySide, msecOnRecordSide, 0); + + _ptrAudioBuffer->SetTypingStatus(KeyPressed()); + + // deliver recorded samples at specified sample rate, mic level etc. + // to the observer using callback + _ptrAudioBuffer->DeliverRecordedData(); + + if (AGC()) { + newMicLevel = _ptrAudioBuffer->NewMicLevel(); + if (newMicLevel != 0) { + // The VQE will only deliver non-zero microphone levels when + // a change is needed. + // Set this new mic level (received from the observer as return + // value in the callback). + RTC_LOG(LS_VERBOSE) << "AGC change of volume: old=" << currentMicLevel + << " => new=" << newMicLevel; + if (SetMicrophoneVolume(newMicLevel) == -1) { + RTC_LOG(LS_WARNING) + << "the required modification of the microphone volume failed"; + } + } + } + } + + return true; +} + +bool AudioDeviceMac::KeyPressed() { + bool key_down = false; + // Loop through all Mac virtual key constant values. + for (unsigned int key_index = 0; key_index < arraysize(prev_key_state_); + ++key_index) { + bool keyState = + CGEventSourceKeyState(kCGEventSourceStateHIDSystemState, key_index); + // A false -> true change in keymap means a key is pressed. + key_down |= (keyState && !prev_key_state_[key_index]); + // Save current state. + prev_key_state_[key_index] = keyState; + } + return key_down; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.h b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.h new file mode 100644 index 0000000000..8dece72739 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_device_mac.h @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_DEVICE_MAC_H_ +#define AUDIO_DEVICE_AUDIO_DEVICE_MAC_H_ + +#include + +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/mac/audio_mixer_manager_mac.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/event_wrapper.h" + +#include +#include +#include + +struct PaUtilRingBuffer; + +namespace rtc { +class PlatformThread; +} // namespace rtc + +namespace webrtc { +class EventWrapper; + +const uint32_t N_REC_SAMPLES_PER_SEC = 48000; +const uint32_t N_PLAY_SAMPLES_PER_SEC = 48000; + +const uint32_t N_REC_CHANNELS = 1; // default is mono recording +const uint32_t N_PLAY_CHANNELS = 2; // default is stereo playout +const uint32_t N_DEVICE_CHANNELS = 64; + +const int kBufferSizeMs = 10; + +const uint32_t ENGINE_REC_BUF_SIZE_IN_SAMPLES = + N_REC_SAMPLES_PER_SEC * kBufferSizeMs / 1000; +const uint32_t ENGINE_PLAY_BUF_SIZE_IN_SAMPLES = + N_PLAY_SAMPLES_PER_SEC * kBufferSizeMs / 1000; + +const int N_BLOCKS_IO = 2; +const int N_BUFFERS_IN = 2; // Must be at least N_BLOCKS_IO. +const int N_BUFFERS_OUT = 3; // Must be at least N_BLOCKS_IO. + +const uint32_t TIMER_PERIOD_MS = 2 * 10 * N_BLOCKS_IO * 1000000; + +const uint32_t REC_BUF_SIZE_IN_SAMPLES = + ENGINE_REC_BUF_SIZE_IN_SAMPLES * N_DEVICE_CHANNELS * N_BUFFERS_IN; +const uint32_t PLAY_BUF_SIZE_IN_SAMPLES = + ENGINE_PLAY_BUF_SIZE_IN_SAMPLES * N_PLAY_CHANNELS * N_BUFFERS_OUT; + +const int kGetMicVolumeIntervalMs = 1000; + +class AudioDeviceMac : public AudioDeviceGeneric { + public: + AudioDeviceMac(); + ~AudioDeviceMac(); + + // Retrieve the currently utilized audio layer + virtual int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const; + + // Main initializaton and termination + virtual InitStatus Init(); + virtual int32_t Terminate(); + virtual bool Initialized() const; + + // Device enumeration + virtual int16_t PlayoutDevices(); + virtual int16_t RecordingDevices(); + virtual int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]); + virtual int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]); + + // Device selection + virtual int32_t SetPlayoutDevice(uint16_t index); + virtual int32_t SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType device); + virtual int32_t SetRecordingDevice(uint16_t index); + virtual int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device); + + // Audio transport initialization + virtual int32_t PlayoutIsAvailable(bool& available); + virtual int32_t InitPlayout(); + virtual bool PlayoutIsInitialized() const; + virtual int32_t RecordingIsAvailable(bool& available); + virtual int32_t InitRecording(); + virtual bool RecordingIsInitialized() const; + + // Audio transport control + virtual int32_t StartPlayout(); + virtual int32_t StopPlayout(); + virtual bool Playing() const; + virtual int32_t StartRecording(); + virtual int32_t StopRecording(); + virtual bool Recording() const; + + // Microphone Automatic Gain Control (AGC) + virtual int32_t SetAGC(bool enable); + virtual bool AGC() const; + + // Audio mixer initialization + virtual int32_t InitSpeaker(); + virtual bool SpeakerIsInitialized() const; + virtual int32_t InitMicrophone(); + virtual bool MicrophoneIsInitialized() const; + + // Speaker volume controls + virtual int32_t SpeakerVolumeIsAvailable(bool& available); + virtual int32_t SetSpeakerVolume(uint32_t volume); + virtual int32_t SpeakerVolume(uint32_t& volume) const; + virtual int32_t MaxSpeakerVolume(uint32_t& maxVolume) const; + virtual int32_t MinSpeakerVolume(uint32_t& minVolume) const; + + // Microphone volume controls + virtual int32_t MicrophoneVolumeIsAvailable(bool& available); + virtual int32_t SetMicrophoneVolume(uint32_t volume); + virtual int32_t MicrophoneVolume(uint32_t& volume) const; + virtual int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const; + virtual int32_t MinMicrophoneVolume(uint32_t& minVolume) const; + + // Microphone mute control + virtual int32_t MicrophoneMuteIsAvailable(bool& available); + virtual int32_t SetMicrophoneMute(bool enable); + virtual int32_t MicrophoneMute(bool& enabled) const; + + // Speaker mute control + virtual int32_t SpeakerMuteIsAvailable(bool& available); + virtual int32_t SetSpeakerMute(bool enable); + virtual int32_t SpeakerMute(bool& enabled) const; + + // Stereo support + virtual int32_t StereoPlayoutIsAvailable(bool& available); + virtual int32_t SetStereoPlayout(bool enable); + virtual int32_t StereoPlayout(bool& enabled) const; + virtual int32_t StereoRecordingIsAvailable(bool& available); + virtual int32_t SetStereoRecording(bool enable); + virtual int32_t StereoRecording(bool& enabled) const; + + // Delay information and control + virtual int32_t PlayoutDelay(uint16_t& delayMS) const; + + virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); + + private: + virtual int32_t MicrophoneIsAvailable(bool& available); + virtual int32_t SpeakerIsAvailable(bool& available); + + static void AtomicSet32(int32_t* theValue, int32_t newValue); + static int32_t AtomicGet32(int32_t* theValue); + + static void logCAMsg(const rtc::LoggingSeverity sev, + const char* msg, + const char* err); + + int32_t GetNumberDevices(const AudioObjectPropertyScope scope, + AudioDeviceID scopedDeviceIds[], + const uint32_t deviceListLength); + + int32_t GetDeviceName(const AudioObjectPropertyScope scope, + const uint16_t index, + char* name); + + int32_t InitDevice(uint16_t userDeviceIndex, + AudioDeviceID& deviceId, + bool isInput); + + // Always work with our preferred playout format inside VoE. + // Then convert the output to the OS setting using an AudioConverter. + OSStatus SetDesiredPlayoutFormat(); + + static OSStatus objectListenerProc( + AudioObjectID objectId, + UInt32 numberAddresses, + const AudioObjectPropertyAddress addresses[], + void* clientData); + + OSStatus implObjectListenerProc(AudioObjectID objectId, + UInt32 numberAddresses, + const AudioObjectPropertyAddress addresses[]); + + int32_t HandleDeviceChange(); + + int32_t HandleStreamFormatChange(AudioObjectID objectId, + AudioObjectPropertyAddress propertyAddress); + + int32_t HandleDataSourceChange(AudioObjectID objectId, + AudioObjectPropertyAddress propertyAddress); + + int32_t HandleProcessorOverload(AudioObjectPropertyAddress propertyAddress); + + static OSStatus deviceIOProc(AudioDeviceID device, + const AudioTimeStamp* now, + const AudioBufferList* inputData, + const AudioTimeStamp* inputTime, + AudioBufferList* outputData, + const AudioTimeStamp* outputTime, + void* clientData); + + static OSStatus outConverterProc( + AudioConverterRef audioConverter, + UInt32* numberDataPackets, + AudioBufferList* data, + AudioStreamPacketDescription** dataPacketDescription, + void* userData); + + static OSStatus inDeviceIOProc(AudioDeviceID device, + const AudioTimeStamp* now, + const AudioBufferList* inputData, + const AudioTimeStamp* inputTime, + AudioBufferList* outputData, + const AudioTimeStamp* outputTime, + void* clientData); + + static OSStatus inConverterProc( + AudioConverterRef audioConverter, + UInt32* numberDataPackets, + AudioBufferList* data, + AudioStreamPacketDescription** dataPacketDescription, + void* inUserData); + + OSStatus implDeviceIOProc(const AudioBufferList* inputData, + const AudioTimeStamp* inputTime, + AudioBufferList* outputData, + const AudioTimeStamp* outputTime); + + OSStatus implOutConverterProc(UInt32* numberDataPackets, + AudioBufferList* data); + + OSStatus implInDeviceIOProc(const AudioBufferList* inputData, + const AudioTimeStamp* inputTime); + + OSStatus implInConverterProc(UInt32* numberDataPackets, + AudioBufferList* data); + + static bool RunCapture(void*); + static bool RunRender(void*); + bool CaptureWorkerThread(); + bool RenderWorkerThread(); + + bool KeyPressed(); + + AudioDeviceBuffer* _ptrAudioBuffer; + + rtc::CriticalSection _critSect; + + EventWrapper& _stopEventRec; + EventWrapper& _stopEvent; + + // TODO(pbos): Replace with direct members, just start/stop, no need to + // recreate the thread. + // Only valid/running between calls to StartRecording and StopRecording. + std::unique_ptr capture_worker_thread_; + + // Only valid/running between calls to StartPlayout and StopPlayout. + std::unique_ptr render_worker_thread_; + + AudioMixerManagerMac _mixerManager; + + uint16_t _inputDeviceIndex; + uint16_t _outputDeviceIndex; + AudioDeviceID _inputDeviceID; + AudioDeviceID _outputDeviceID; +#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1050 + AudioDeviceIOProcID _inDeviceIOProcID; + AudioDeviceIOProcID _deviceIOProcID; +#endif + bool _inputDeviceIsSpecified; + bool _outputDeviceIsSpecified; + + uint8_t _recChannels; + uint8_t _playChannels; + + Float32* _captureBufData; + SInt16* _renderBufData; + + SInt16 _renderConvertData[PLAY_BUF_SIZE_IN_SAMPLES]; + + bool _initialized; + bool _isShutDown; + bool _recording; + bool _playing; + bool _recIsInitialized; + bool _playIsInitialized; + bool _AGC; + + // Atomically set varaibles + int32_t _renderDeviceIsAlive; + int32_t _captureDeviceIsAlive; + + bool _twoDevices; + bool _doStop; // For play if not shared device or play+rec if shared device + bool _doStopRec; // For rec if not shared device + bool _macBookPro; + bool _macBookProPanRight; + + AudioConverterRef _captureConverter; + AudioConverterRef _renderConverter; + + AudioStreamBasicDescription _outStreamFormat; + AudioStreamBasicDescription _outDesiredFormat; + AudioStreamBasicDescription _inStreamFormat; + AudioStreamBasicDescription _inDesiredFormat; + + uint32_t _captureLatencyUs; + uint32_t _renderLatencyUs; + + // Atomically set variables + mutable int32_t _captureDelayUs; + mutable int32_t _renderDelayUs; + + int32_t _renderDelayOffsetSamples; + + PaUtilRingBuffer* _paCaptureBuffer; + PaUtilRingBuffer* _paRenderBuffer; + + semaphore_t _renderSemaphore; + semaphore_t _captureSemaphore; + + int _captureBufSizeSamples; + int _renderBufSizeSamples; + + // Typing detection + // 0x5c is key "9", after that comes function keys. + bool prev_key_state_[0x5d]; + + int get_mic_volume_counter_ms_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_MAIN_SOURCE_MAC_AUDIO_DEVICE_MAC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc new file mode 100644 index 0000000000..0932e8b01b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc @@ -0,0 +1,916 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_device/mac/audio_mixer_manager_mac.h" + +#include // getpid() + +namespace webrtc { + +#define WEBRTC_CA_RETURN_ON_ERR(expr) \ + do { \ + err = expr; \ + if (err != noErr) { \ + logCAMsg(rtc::LS_ERROR, "Error in " #expr, (const char*)&err); \ + return -1; \ + } \ + } while (0) + +#define WEBRTC_CA_LOG_ERR(expr) \ + do { \ + err = expr; \ + if (err != noErr) { \ + logCAMsg(rtc::LS_ERROR, "Error in " #expr, (const char*)&err); \ + } \ + } while (0) + +#define WEBRTC_CA_LOG_WARN(expr) \ + do { \ + err = expr; \ + if (err != noErr) { \ + logCAMsg(rtc::LS_WARNING, "Error in " #expr, (const char*)&err); \ + } \ + } while (0) + +AudioMixerManagerMac::AudioMixerManagerMac() + : _inputDeviceID(kAudioObjectUnknown), + _outputDeviceID(kAudioObjectUnknown), + _noInputChannels(0), + _noOutputChannels(0) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " created"; +} + +AudioMixerManagerMac::~AudioMixerManagerMac() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " destroyed"; + Close(); +} + +// ============================================================================ +// PUBLIC METHODS +// ============================================================================ + +int32_t AudioMixerManagerMac::Close() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + rtc::CritScope lock(&_critSect); + + CloseSpeaker(); + CloseMicrophone(); + + return 0; +} + +int32_t AudioMixerManagerMac::CloseSpeaker() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + rtc::CritScope lock(&_critSect); + + _outputDeviceID = kAudioObjectUnknown; + _noOutputChannels = 0; + + return 0; +} + +int32_t AudioMixerManagerMac::CloseMicrophone() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + rtc::CritScope lock(&_critSect); + + _inputDeviceID = kAudioObjectUnknown; + _noInputChannels = 0; + + return 0; +} + +int32_t AudioMixerManagerMac::OpenSpeaker(AudioDeviceID deviceID) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::OpenSpeaker(id=" << deviceID + << ")"; + + rtc::CritScope lock(&_critSect); + + OSStatus err = noErr; + UInt32 size = 0; + pid_t hogPid = -1; + + _outputDeviceID = deviceID; + + // Check which process, if any, has hogged the device. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyHogMode, kAudioDevicePropertyScopeOutput, 0}; + + // First, does it have the property? Aggregate devices don't. + if (AudioObjectHasProperty(_outputDeviceID, &propertyAddress)) { + size = sizeof(hogPid); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &hogPid)); + + if (hogPid == -1) { + RTC_LOG(LS_VERBOSE) << "No process has hogged the output device"; + } + // getpid() is apparently "always successful" + else if (hogPid == getpid()) { + RTC_LOG(LS_VERBOSE) << "Our process has hogged the output device"; + } else { + RTC_LOG(LS_WARNING) << "Another process (pid = " + << static_cast(hogPid) + << ") has hogged the output device"; + + return -1; + } + } + + // get number of channels from stream format + propertyAddress.mSelector = kAudioDevicePropertyStreamFormat; + + // Get the stream format, to be able to read the number of channels. + AudioStreamBasicDescription streamFormat; + size = sizeof(AudioStreamBasicDescription); + memset(&streamFormat, 0, size); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &streamFormat)); + + _noOutputChannels = streamFormat.mChannelsPerFrame; + + return 0; +} + +int32_t AudioMixerManagerMac::OpenMicrophone(AudioDeviceID deviceID) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::OpenMicrophone(id=" << deviceID + << ")"; + + rtc::CritScope lock(&_critSect); + + OSStatus err = noErr; + UInt32 size = 0; + pid_t hogPid = -1; + + _inputDeviceID = deviceID; + + // Check which process, if any, has hogged the device. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyHogMode, kAudioDevicePropertyScopeInput, 0}; + size = sizeof(hogPid); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &hogPid)); + if (hogPid == -1) { + RTC_LOG(LS_VERBOSE) << "No process has hogged the input device"; + } + // getpid() is apparently "always successful" + else if (hogPid == getpid()) { + RTC_LOG(LS_VERBOSE) << "Our process has hogged the input device"; + } else { + RTC_LOG(LS_WARNING) << "Another process (pid = " << static_cast(hogPid) + << ") has hogged the input device"; + + return -1; + } + + // get number of channels from stream format + propertyAddress.mSelector = kAudioDevicePropertyStreamFormat; + + // Get the stream format, to be able to read the number of channels. + AudioStreamBasicDescription streamFormat; + size = sizeof(AudioStreamBasicDescription); + memset(&streamFormat, 0, size); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &streamFormat)); + + _noInputChannels = streamFormat.mChannelsPerFrame; + + return 0; +} + +bool AudioMixerManagerMac::SpeakerIsInitialized() const { + RTC_LOG(LS_INFO) << __FUNCTION__; + + return (_outputDeviceID != kAudioObjectUnknown); +} + +bool AudioMixerManagerMac::MicrophoneIsInitialized() const { + RTC_LOG(LS_INFO) << __FUNCTION__; + + return (_inputDeviceID != kAudioObjectUnknown); +} + +int32_t AudioMixerManagerMac::SetSpeakerVolume(uint32_t volume) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::SetSpeakerVolume(volume=" + << volume << ")"; + + rtc::CritScope lock(&_critSect); + + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + bool success = false; + + // volume range is 0.0 - 1.0, convert from 0 -255 + const Float32 vol = (Float32)(volume / 255.0); + + assert(vol <= 1.0 && vol >= 0.0); + + // Does the capture device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyVolumeScalar, kAudioDevicePropertyScopeOutput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(vol); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, size, &vol)); + + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noOutputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(vol); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, size, &vol)); + } + success = true; + } + + if (!success) { + RTC_LOG(LS_WARNING) << "Unable to set a volume on any output channel"; + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerMac::SpeakerVolume(uint32_t& volume) const { + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + unsigned int channels = 0; + Float32 channelVol = 0; + Float32 vol = 0; + + // Does the device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyVolumeScalar, kAudioDevicePropertyScopeOutput, 0}; + Boolean hasProperty = + AudioObjectHasProperty(_outputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(vol); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &vol)); + + // vol 0.0 to 1.0 -> convert to 0 - 255 + volume = static_cast(vol * 255 + 0.5); + } else { + // Otherwise get the average volume across channels. + vol = 0; + for (UInt32 i = 1; i <= _noOutputChannels; i++) { + channelVol = 0; + propertyAddress.mElement = i; + hasProperty = AudioObjectHasProperty(_outputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(channelVol); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &channelVol)); + + vol += channelVol; + channels++; + } + } + + if (channels == 0) { + RTC_LOG(LS_WARNING) << "Unable to get a volume on any channel"; + return -1; + } + + assert(channels > 0); + // vol 0.0 to 1.0 -> convert to 0 - 255 + volume = static_cast(255 * vol / channels + 0.5); + } + + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::SpeakerVolume() => vol=" << vol; + + return 0; +} + +int32_t AudioMixerManagerMac::MaxSpeakerVolume(uint32_t& maxVolume) const { + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + // volume range is 0.0 to 1.0 + // we convert that to 0 - 255 + maxVolume = 255; + + return 0; +} + +int32_t AudioMixerManagerMac::MinSpeakerVolume(uint32_t& minVolume) const { + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + // volume range is 0.0 to 1.0 + // we convert that to 0 - 255 + minVolume = 0; + + return 0; +} + +int32_t AudioMixerManagerMac::SpeakerVolumeIsAvailable(bool& available) { + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + + // Does the capture device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyVolumeScalar, kAudioDevicePropertyScopeOutput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + available = true; + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noOutputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err != noErr || !isSettable) { + available = false; + RTC_LOG(LS_WARNING) << "Volume cannot be set for output channel " << i + << ", err=" << err; + return -1; + } + } + + available = true; + return 0; +} + +int32_t AudioMixerManagerMac::SpeakerMuteIsAvailable(bool& available) { + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + + // Does the capture device have a master mute control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyMute, kAudioDevicePropertyScopeOutput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + available = true; + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noOutputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err != noErr || !isSettable) { + available = false; + RTC_LOG(LS_WARNING) << "Mute cannot be set for output channel " << i + << ", err=" << err; + return -1; + } + } + + available = true; + return 0; +} + +int32_t AudioMixerManagerMac::SetSpeakerMute(bool enable) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::SetSpeakerMute(enable=" + << enable << ")"; + + rtc::CritScope lock(&_critSect); + + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + UInt32 mute = enable ? 1 : 0; + bool success = false; + + // Does the render device have a master mute control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyMute, kAudioDevicePropertyScopeOutput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(mute); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, size, &mute)); + + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noOutputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_outputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(mute); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, size, &mute)); + } + success = true; + } + + if (!success) { + RTC_LOG(LS_WARNING) << "Unable to set mute on any input channel"; + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerMac::SpeakerMute(bool& enabled) const { + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + unsigned int channels = 0; + UInt32 channelMuted = 0; + UInt32 muted = 0; + + // Does the device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyMute, kAudioDevicePropertyScopeOutput, 0}; + Boolean hasProperty = + AudioObjectHasProperty(_outputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(muted); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &muted)); + + // 1 means muted + enabled = static_cast(muted); + } else { + // Otherwise check if all channels are muted. + for (UInt32 i = 1; i <= _noOutputChannels; i++) { + muted = 0; + propertyAddress.mElement = i; + hasProperty = AudioObjectHasProperty(_outputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(channelMuted); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _outputDeviceID, &propertyAddress, 0, NULL, &size, &channelMuted)); + + muted = (muted && channelMuted); + channels++; + } + } + + if (channels == 0) { + RTC_LOG(LS_WARNING) << "Unable to get mute for any channel"; + return -1; + } + + assert(channels > 0); + // 1 means muted + enabled = static_cast(muted); + } + + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::SpeakerMute() => enabled=" + << enabled; + + return 0; +} + +int32_t AudioMixerManagerMac::StereoPlayoutIsAvailable(bool& available) { + if (_outputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + available = (_noOutputChannels == 2); + return 0; +} + +int32_t AudioMixerManagerMac::StereoRecordingIsAvailable(bool& available) { + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + available = (_noInputChannels == 2); + return 0; +} + +int32_t AudioMixerManagerMac::MicrophoneMuteIsAvailable(bool& available) { + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + + // Does the capture device have a master mute control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyMute, kAudioDevicePropertyScopeInput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + available = true; + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noInputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err != noErr || !isSettable) { + available = false; + RTC_LOG(LS_WARNING) << "Mute cannot be set for output channel " << i + << ", err=" << err; + return -1; + } + } + + available = true; + return 0; +} + +int32_t AudioMixerManagerMac::SetMicrophoneMute(bool enable) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::SetMicrophoneMute(enable=" + << enable << ")"; + + rtc::CritScope lock(&_critSect); + + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + UInt32 mute = enable ? 1 : 0; + bool success = false; + + // Does the capture device have a master mute control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyMute, kAudioDevicePropertyScopeInput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(mute); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, size, &mute)); + + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noInputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(mute); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, size, &mute)); + } + success = true; + } + + if (!success) { + RTC_LOG(LS_WARNING) << "Unable to set mute on any input channel"; + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerMac::MicrophoneMute(bool& enabled) const { + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + unsigned int channels = 0; + UInt32 channelMuted = 0; + UInt32 muted = 0; + + // Does the device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyMute, kAudioDevicePropertyScopeInput, 0}; + Boolean hasProperty = + AudioObjectHasProperty(_inputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(muted); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &muted)); + + // 1 means muted + enabled = static_cast(muted); + } else { + // Otherwise check if all channels are muted. + for (UInt32 i = 1; i <= _noInputChannels; i++) { + muted = 0; + propertyAddress.mElement = i; + hasProperty = AudioObjectHasProperty(_inputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(channelMuted); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &channelMuted)); + + muted = (muted && channelMuted); + channels++; + } + } + + if (channels == 0) { + RTC_LOG(LS_WARNING) << "Unable to get mute for any channel"; + return -1; + } + + assert(channels > 0); + // 1 means muted + enabled = static_cast(muted); + } + + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::MicrophoneMute() => enabled=" + << enabled; + + return 0; +} + +int32_t AudioMixerManagerMac::MicrophoneVolumeIsAvailable(bool& available) { + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + + // Does the capture device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyVolumeScalar, kAudioDevicePropertyScopeInput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + available = true; + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noInputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err != noErr || !isSettable) { + available = false; + RTC_LOG(LS_WARNING) << "Volume cannot be set for input channel " << i + << ", err=" << err; + return -1; + } + } + + available = true; + return 0; +} + +int32_t AudioMixerManagerMac::SetMicrophoneVolume(uint32_t volume) { + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::SetMicrophoneVolume(volume=" + << volume << ")"; + + rtc::CritScope lock(&_critSect); + + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + bool success = false; + + // volume range is 0.0 - 1.0, convert from 0 - 255 + const Float32 vol = (Float32)(volume / 255.0); + + assert(vol <= 1.0 && vol >= 0.0); + + // Does the capture device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyVolumeScalar, kAudioDevicePropertyScopeInput, 0}; + Boolean isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(vol); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, size, &vol)); + + return 0; + } + + // Otherwise try to set each channel. + for (UInt32 i = 1; i <= _noInputChannels; i++) { + propertyAddress.mElement = i; + isSettable = false; + err = AudioObjectIsPropertySettable(_inputDeviceID, &propertyAddress, + &isSettable); + if (err == noErr && isSettable) { + size = sizeof(vol); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectSetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, size, &vol)); + } + success = true; + } + + if (!success) { + RTC_LOG(LS_WARNING) << "Unable to set a level on any input channel"; + return -1; + } + + return 0; +} + +int32_t AudioMixerManagerMac::MicrophoneVolume(uint32_t& volume) const { + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + OSStatus err = noErr; + UInt32 size = 0; + unsigned int channels = 0; + Float32 channelVol = 0; + Float32 volFloat32 = 0; + + // Does the device have a master volume control? + // If so, use it exclusively. + AudioObjectPropertyAddress propertyAddress = { + kAudioDevicePropertyVolumeScalar, kAudioDevicePropertyScopeInput, 0}; + Boolean hasProperty = + AudioObjectHasProperty(_inputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(volFloat32); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &volFloat32)); + + // vol 0.0 to 1.0 -> convert to 0 - 255 + volume = static_cast(volFloat32 * 255 + 0.5); + } else { + // Otherwise get the average volume across channels. + volFloat32 = 0; + for (UInt32 i = 1; i <= _noInputChannels; i++) { + channelVol = 0; + propertyAddress.mElement = i; + hasProperty = AudioObjectHasProperty(_inputDeviceID, &propertyAddress); + if (hasProperty) { + size = sizeof(channelVol); + WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( + _inputDeviceID, &propertyAddress, 0, NULL, &size, &channelVol)); + + volFloat32 += channelVol; + channels++; + } + } + + if (channels == 0) { + RTC_LOG(LS_WARNING) << "Unable to get a level on any channel"; + return -1; + } + + assert(channels > 0); + // vol 0.0 to 1.0 -> convert to 0 - 255 + volume = static_cast(255 * volFloat32 / channels + 0.5); + } + + RTC_LOG(LS_VERBOSE) << "AudioMixerManagerMac::MicrophoneVolume() => vol=" + << volume; + + return 0; +} + +int32_t AudioMixerManagerMac::MaxMicrophoneVolume(uint32_t& maxVolume) const { + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + // volume range is 0.0 to 1.0 + // we convert that to 0 - 255 + maxVolume = 255; + + return 0; +} + +int32_t AudioMixerManagerMac::MinMicrophoneVolume(uint32_t& minVolume) const { + if (_inputDeviceID == kAudioObjectUnknown) { + RTC_LOG(LS_WARNING) << "device ID has not been set"; + return -1; + } + + // volume range is 0.0 to 1.0 + // we convert that to 0 - 10 + minVolume = 0; + + return 0; +} + +// ============================================================================ +// Private Methods +// ============================================================================ + +// CoreAudio errors are best interpreted as four character strings. +void AudioMixerManagerMac::logCAMsg(const rtc::LoggingSeverity sev, + const char* msg, + const char* err) { + RTC_DCHECK(msg != NULL); + RTC_DCHECK(err != NULL); + RTC_DCHECK(sev == rtc::LS_ERROR || sev == rtc::LS_WARNING); + +#ifdef WEBRTC_ARCH_BIG_ENDIAN + switch (sev) { + case rtc::LS_ERROR: + RTC_LOG(LS_ERROR) << msg << ": " << err[0] << err[1] << err[2] << err[3]; + break; + case rtc::LS_WARNING: + RTC_LOG(LS_WARNING) << msg << ": " << err[0] << err[1] << err[2] + << err[3]; + break; + default: + break; + } +#else + // We need to flip the characters in this case. + switch (sev) { + case rtc::LS_ERROR: + RTC_LOG(LS_ERROR) << msg << ": " << err[3] << err[2] << err[1] << err[0]; + break; + case rtc::LS_WARNING: + RTC_LOG(LS_WARNING) << msg << ": " << err[3] << err[2] << err[1] + << err[0]; + break; + default: + break; + } +#endif +} + +} // namespace webrtc +// EOF diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h new file mode 100644 index 0000000000..6681da5913 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_MIXER_MANAGER_MAC_H_ +#define AUDIO_DEVICE_AUDIO_MIXER_MANAGER_MAC_H_ + +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/logging.h" +#include "typedefs.h" // NOLINT(build/include) + +#include + +namespace webrtc { + +class AudioMixerManagerMac { + public: + int32_t OpenSpeaker(AudioDeviceID deviceID); + int32_t OpenMicrophone(AudioDeviceID deviceID); + int32_t SetSpeakerVolume(uint32_t volume); + int32_t SpeakerVolume(uint32_t& volume) const; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const; + int32_t MinSpeakerVolume(uint32_t& minVolume) const; + int32_t SpeakerVolumeIsAvailable(bool& available); + int32_t SpeakerMuteIsAvailable(bool& available); + int32_t SetSpeakerMute(bool enable); + int32_t SpeakerMute(bool& enabled) const; + int32_t StereoPlayoutIsAvailable(bool& available); + int32_t StereoRecordingIsAvailable(bool& available); + int32_t MicrophoneMuteIsAvailable(bool& available); + int32_t SetMicrophoneMute(bool enable); + int32_t MicrophoneMute(bool& enabled) const; + int32_t MicrophoneVolumeIsAvailable(bool& available); + int32_t SetMicrophoneVolume(uint32_t volume); + int32_t MicrophoneVolume(uint32_t& volume) const; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const; + int32_t Close(); + int32_t CloseSpeaker(); + int32_t CloseMicrophone(); + bool SpeakerIsInitialized() const; + bool MicrophoneIsInitialized() const; + + public: + AudioMixerManagerMac(); + ~AudioMixerManagerMac(); + + private: + static void logCAMsg(const rtc::LoggingSeverity sev, + const char* msg, + const char* err); + + private: + rtc::CriticalSection _critSect; + + AudioDeviceID _inputDeviceID; + AudioDeviceID _outputDeviceID; + + uint16_t _noInputChannels; + uint16_t _noOutputChannels; +}; + +} // namespace webrtc + +#endif // AUDIO_MIXER_MAC_H diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_memorybarrier.h b/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_memorybarrier.h new file mode 100644 index 0000000000..f689622201 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_memorybarrier.h @@ -0,0 +1,127 @@ +/* + * $Id: pa_memorybarrier.h 1240 2007-07-17 13:05:07Z bjornroche $ + * Portable Audio I/O Library + * Memory barrier utilities + * + * Author: Bjorn Roche, XO Audio, LLC + * + * This program uses the PortAudio Portable Audio Library. + * For more information see: http://www.portaudio.com + * Copyright (c) 1999-2000 Ross Bencina and Phil Burk + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * The text above constitutes the entire PortAudio license; however, + * the PortAudio community also makes the following non-binding requests: + * + * Any person wishing to distribute modifications to the Software is + * requested to send the modifications to the original developer so that + * they can be incorporated into the canonical version. It is also + * requested that these non-binding requests be included along with the + * license above. + */ + +/** + @file pa_memorybarrier.h + @ingroup common_src +*/ + +/**************** + * Some memory barrier primitives based on the system. + * right now only OS X, FreeBSD, and Linux are supported. In addition to providing + * memory barriers, these functions should ensure that data cached in registers + * is written out to cache where it can be snooped by other CPUs. (ie, the volatile + * keyword should not be required) + * + * the primitives that must be defined are: + * + * PaUtil_FullMemoryBarrier() + * PaUtil_ReadMemoryBarrier() + * PaUtil_WriteMemoryBarrier() + * + ****************/ + +#if defined(__APPLE__) +# include + /* Here are the memory barrier functions. Mac OS X only provides + full memory barriers, so the three types of barriers are the same, + however, these barriers are superior to compiler-based ones. */ +# define PaUtil_FullMemoryBarrier() OSMemoryBarrier() +# define PaUtil_ReadMemoryBarrier() OSMemoryBarrier() +# define PaUtil_WriteMemoryBarrier() OSMemoryBarrier() +#elif defined(__GNUC__) + /* GCC >= 4.1 has built-in intrinsics. We'll use those */ +# if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1) +# define PaUtil_FullMemoryBarrier() __sync_synchronize() +# define PaUtil_ReadMemoryBarrier() __sync_synchronize() +# define PaUtil_WriteMemoryBarrier() __sync_synchronize() + /* as a fallback, GCC understands volatile asm and "memory" to mean it + * should not reorder memory read/writes */ + /* Note that it is not clear that any compiler actually defines __PPC__, + * it can probably removed safely. */ +# elif defined( __ppc__ ) || defined( __powerpc__) || defined( __PPC__ ) +# define PaUtil_FullMemoryBarrier() asm volatile("sync":::"memory") +# define PaUtil_ReadMemoryBarrier() asm volatile("sync":::"memory") +# define PaUtil_WriteMemoryBarrier() asm volatile("sync":::"memory") +# elif defined( __i386__ ) || defined( __i486__ ) || defined( __i586__ ) || \ + defined( __i686__ ) || defined( __x86_64__ ) +# define PaUtil_FullMemoryBarrier() asm volatile("mfence":::"memory") +# define PaUtil_ReadMemoryBarrier() asm volatile("lfence":::"memory") +# define PaUtil_WriteMemoryBarrier() asm volatile("sfence":::"memory") +# else +# ifdef ALLOW_SMP_DANGERS +# warning Memory barriers not defined on this system or system unknown +# warning For SMP safety, you should fix this. +# define PaUtil_FullMemoryBarrier() +# define PaUtil_ReadMemoryBarrier() +# define PaUtil_WriteMemoryBarrier() +# else +# error Memory barriers are not defined on this system. You can still compile by defining ALLOW_SMP_DANGERS, but SMP safety will not be guaranteed. +# endif +# endif +#elif (_MSC_VER >= 1400) && !defined(_WIN32_WCE) +# include +# pragma intrinsic(_ReadWriteBarrier) +# pragma intrinsic(_ReadBarrier) +# pragma intrinsic(_WriteBarrier) +# define PaUtil_FullMemoryBarrier() _ReadWriteBarrier() +# define PaUtil_ReadMemoryBarrier() _ReadBarrier() +# define PaUtil_WriteMemoryBarrier() _WriteBarrier() +#elif defined(_WIN32_WCE) +# define PaUtil_FullMemoryBarrier() +# define PaUtil_ReadMemoryBarrier() +# define PaUtil_WriteMemoryBarrier() +#elif defined(_MSC_VER) || defined(__BORLANDC__) +# define PaUtil_FullMemoryBarrier() _asm { lock add [esp], 0 } +# define PaUtil_ReadMemoryBarrier() _asm { lock add [esp], 0 } +# define PaUtil_WriteMemoryBarrier() _asm { lock add [esp], 0 } +#else +# ifdef ALLOW_SMP_DANGERS +# warning Memory barriers not defined on this system or system unknown +# warning For SMP safety, you should fix this. +# define PaUtil_FullMemoryBarrier() +# define PaUtil_ReadMemoryBarrier() +# define PaUtil_WriteMemoryBarrier() +# else +# error Memory barriers are not defined on this system. You can still compile by defining ALLOW_SMP_DANGERS, but SMP safety will not be guaranteed. +# endif +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.c b/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.c new file mode 100644 index 0000000000..7a95024757 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.c @@ -0,0 +1,237 @@ +/* + * $Id: pa_ringbuffer.c 1421 2009-11-18 16:09:05Z bjornroche $ + * Portable Audio I/O Library + * Ring Buffer utility. + * + * Author: Phil Burk, http://www.softsynth.com + * modified for SMP safety on Mac OS X by Bjorn Roche + * modified for SMP safety on Linux by Leland Lucius + * also, allowed for const where possible + * modified for multiple-byte-sized data elements by Sven Fischer + * + * Note that this is safe only for a single-thread reader and a + * single-thread writer. + * + * This program uses the PortAudio Portable Audio Library. + * For more information see: http://www.portaudio.com + * Copyright (c) 1999-2000 Ross Bencina and Phil Burk + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * The text above constitutes the entire PortAudio license; however, + * the PortAudio community also makes the following non-binding requests: + * + * Any person wishing to distribute modifications to the Software is + * requested to send the modifications to the original developer so that + * they can be incorporated into the canonical version. It is also + * requested that these non-binding requests be included along with the + * license above. + */ + +/** + @file + @ingroup common_src +*/ + +#include +#include +#include +#include +#include "modules/audio_device/mac/portaudio/pa_memorybarrier.h" +#include "modules/audio_device/mac/portaudio/pa_ringbuffer.h" + +/*************************************************************************** + * Initialize FIFO. + * elementCount must be power of 2, returns -1 if not. + */ +PaRingBufferSize PaUtil_InitializeRingBuffer(PaUtilRingBuffer* rbuf, + PaRingBufferSize elementSizeBytes, + PaRingBufferSize elementCount, + void* dataPtr) { + if( ((elementCount-1) & elementCount) != 0) return -1; /* Not Power of two. */ + rbuf->bufferSize = elementCount; + rbuf->buffer = (char *)dataPtr; + PaUtil_FlushRingBuffer( rbuf ); + rbuf->bigMask = (elementCount*2)-1; + rbuf->smallMask = (elementCount)-1; + rbuf->elementSizeBytes = elementSizeBytes; + return 0; +} + +/*************************************************************************** +** Return number of elements available for reading. */ +PaRingBufferSize PaUtil_GetRingBufferReadAvailable( PaUtilRingBuffer *rbuf ) +{ + PaUtil_ReadMemoryBarrier(); + return ( (rbuf->writeIndex - rbuf->readIndex) & rbuf->bigMask ); +} +/*************************************************************************** +** Return number of elements available for writing. */ +PaRingBufferSize PaUtil_GetRingBufferWriteAvailable( PaUtilRingBuffer *rbuf ) +{ + /* Since we are calling PaUtil_GetRingBufferReadAvailable, we don't need an aditional MB */ + return ( rbuf->bufferSize - PaUtil_GetRingBufferReadAvailable(rbuf)); +} + +/*************************************************************************** +** Clear buffer. Should only be called when buffer is NOT being read. */ +void PaUtil_FlushRingBuffer( PaUtilRingBuffer *rbuf ) +{ + rbuf->writeIndex = rbuf->readIndex = 0; +} + +/*************************************************************************** +** Get address of region(s) to which we can write data. +** If the region is contiguous, size2 will be zero. +** If non-contiguous, size2 will be the size of second region. +** Returns room available to be written or elementCount, whichever is smaller. +*/ +PaRingBufferSize PaUtil_GetRingBufferWriteRegions(PaUtilRingBuffer* rbuf, + PaRingBufferSize elementCount, + void** dataPtr1, + PaRingBufferSize* sizePtr1, + void** dataPtr2, + PaRingBufferSize* sizePtr2) { + PaRingBufferSize index; + PaRingBufferSize available = PaUtil_GetRingBufferWriteAvailable( rbuf ); + if( elementCount > available ) elementCount = available; + /* Check to see if write is not contiguous. */ + index = rbuf->writeIndex & rbuf->smallMask; + if( (index + elementCount) > rbuf->bufferSize ) + { + /* Write data in two blocks that wrap the buffer. */ + PaRingBufferSize firstHalf = rbuf->bufferSize - index; + *dataPtr1 = &rbuf->buffer[index*rbuf->elementSizeBytes]; + *sizePtr1 = firstHalf; + *dataPtr2 = &rbuf->buffer[0]; + *sizePtr2 = elementCount - firstHalf; + } + else + { + *dataPtr1 = &rbuf->buffer[index*rbuf->elementSizeBytes]; + *sizePtr1 = elementCount; + *dataPtr2 = NULL; + *sizePtr2 = 0; + } + return elementCount; +} + + +/*************************************************************************** +*/ +PaRingBufferSize PaUtil_AdvanceRingBufferWriteIndex( + PaUtilRingBuffer* rbuf, + PaRingBufferSize elementCount) { + /* we need to ensure that previous writes are seen before we update the write index */ + PaUtil_WriteMemoryBarrier(); + return rbuf->writeIndex = (rbuf->writeIndex + elementCount) & rbuf->bigMask; +} + +/*************************************************************************** +** Get address of region(s) from which we can read data. +** If the region is contiguous, size2 will be zero. +** If non-contiguous, size2 will be the size of second region. +** Returns room available to be written or elementCount, whichever is smaller. +*/ +PaRingBufferSize PaUtil_GetRingBufferReadRegions(PaUtilRingBuffer* rbuf, + PaRingBufferSize elementCount, + void** dataPtr1, + PaRingBufferSize* sizePtr1, + void** dataPtr2, + PaRingBufferSize* sizePtr2) { + PaRingBufferSize index; + PaRingBufferSize available = PaUtil_GetRingBufferReadAvailable( rbuf ); + if( elementCount > available ) elementCount = available; + /* Check to see if read is not contiguous. */ + index = rbuf->readIndex & rbuf->smallMask; + if( (index + elementCount) > rbuf->bufferSize ) + { + /* Write data in two blocks that wrap the buffer. */ + PaRingBufferSize firstHalf = rbuf->bufferSize - index; + *dataPtr1 = &rbuf->buffer[index*rbuf->elementSizeBytes]; + *sizePtr1 = firstHalf; + *dataPtr2 = &rbuf->buffer[0]; + *sizePtr2 = elementCount - firstHalf; + } + else + { + *dataPtr1 = &rbuf->buffer[index*rbuf->elementSizeBytes]; + *sizePtr1 = elementCount; + *dataPtr2 = NULL; + *sizePtr2 = 0; + } + return elementCount; +} +/*************************************************************************** +*/ +PaRingBufferSize PaUtil_AdvanceRingBufferReadIndex( + PaUtilRingBuffer* rbuf, + PaRingBufferSize elementCount) { + /* we need to ensure that previous writes are always seen before updating the index. */ + PaUtil_WriteMemoryBarrier(); + return rbuf->readIndex = (rbuf->readIndex + elementCount) & rbuf->bigMask; +} + +/*************************************************************************** +** Return elements written. */ +PaRingBufferSize PaUtil_WriteRingBuffer(PaUtilRingBuffer* rbuf, + const void* data, + PaRingBufferSize elementCount) { + PaRingBufferSize size1, size2, numWritten; + void *data1, *data2; + numWritten = PaUtil_GetRingBufferWriteRegions( rbuf, elementCount, &data1, &size1, &data2, &size2 ); + if( size2 > 0 ) + { + + memcpy( data1, data, size1*rbuf->elementSizeBytes ); + data = ((char *)data) + size1*rbuf->elementSizeBytes; + memcpy( data2, data, size2*rbuf->elementSizeBytes ); + } + else + { + memcpy( data1, data, size1*rbuf->elementSizeBytes ); + } + PaUtil_AdvanceRingBufferWriteIndex( rbuf, numWritten ); + return numWritten; +} + +/*************************************************************************** +** Return elements read. */ +PaRingBufferSize PaUtil_ReadRingBuffer(PaUtilRingBuffer* rbuf, + void* data, + PaRingBufferSize elementCount) { + PaRingBufferSize size1, size2, numRead; + void *data1, *data2; + numRead = PaUtil_GetRingBufferReadRegions( rbuf, elementCount, &data1, &size1, &data2, &size2 ); + if( size2 > 0 ) + { + memcpy( data, data1, size1*rbuf->elementSizeBytes ); + data = ((char *)data) + size1*rbuf->elementSizeBytes; + memcpy( data, data2, size2*rbuf->elementSizeBytes ); + } + else + { + memcpy( data, data1, size1*rbuf->elementSizeBytes ); + } + PaUtil_AdvanceRingBufferReadIndex( rbuf, numRead ); + return numRead; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.h b/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.h new file mode 100644 index 0000000000..210a65e4d8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mac/portaudio/pa_ringbuffer.h @@ -0,0 +1,233 @@ +#ifndef AUDIO_DEVICE_PA_RINGBUFFER_H_ +#define AUDIO_DEVICE_PA_RINGBUFFER_H_ +/* + * $Id: pa_ringbuffer.h 1421 2009-11-18 16:09:05Z bjornroche $ + * Portable Audio I/O Library + * Ring Buffer utility. + * + * Author: Phil Burk, http://www.softsynth.com + * modified for SMP safety on OS X by Bjorn Roche. + * also allowed for const where possible. + * modified for multiple-byte-sized data elements by Sven Fischer + * + * Note that this is safe only for a single-thread reader + * and a single-thread writer. + * + * This program is distributed with the PortAudio Portable Audio Library. + * For more information see: http://www.portaudio.com + * Copyright (c) 1999-2000 Ross Bencina and Phil Burk + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * The text above constitutes the entire PortAudio license; however, + * the PortAudio community also makes the following non-binding requests: + * + * Any person wishing to distribute modifications to the Software is + * requested to send the modifications to the original developer so that + * they can be incorporated into the canonical version. It is also + * requested that these non-binding requests be included along with the + * license above. + */ + +/** @file + @ingroup common_src + @brief Single-reader single-writer lock-free ring buffer + + PaUtilRingBuffer is a ring buffer used to transport samples between + different execution contexts (threads, OS callbacks, interrupt handlers) + without requiring the use of any locks. This only works when there is + a single reader and a single writer (ie. one thread or callback writes + to the ring buffer, another thread or callback reads from it). + + The PaUtilRingBuffer structure manages a ring buffer containing N + elements, where N must be a power of two. An element may be any size + (specified in bytes). + + The memory area used to store the buffer elements must be allocated by + the client prior to calling PaUtil_InitializeRingBuffer() and must outlive + the use of the ring buffer. +*/ + +#if defined(__APPLE__) +#include +typedef int32_t PaRingBufferSize; +#elif defined( __GNUC__ ) +typedef long PaRingBufferSize; +#elif (_MSC_VER >= 1400) +typedef long PaRingBufferSize; +#elif defined(_MSC_VER) || defined(__BORLANDC__) +typedef long PaRingBufferSize; +#else +typedef long PaRingBufferSize; +#endif + + + +#ifdef __cplusplus +extern "C" +{ +#endif /* __cplusplus */ + +typedef struct PaUtilRingBuffer +{ + PaRingBufferSize bufferSize; /**< Number of elements in FIFO. Power of 2. Set by PaUtil_InitRingBuffer. */ + PaRingBufferSize writeIndex; /**< Index of next writable element. Set by PaUtil_AdvanceRingBufferWriteIndex. */ + PaRingBufferSize readIndex; /**< Index of next readable element. Set by PaUtil_AdvanceRingBufferReadIndex. */ + PaRingBufferSize bigMask; /**< Used for wrapping indices with extra bit to distinguish full/empty. */ + PaRingBufferSize smallMask; /**< Used for fitting indices to buffer. */ + PaRingBufferSize elementSizeBytes; /**< Number of bytes per element. */ + char *buffer; /**< Pointer to the buffer containing the actual data. */ +}PaUtilRingBuffer; + +/** Initialize Ring Buffer. + + @param rbuf The ring buffer. + + @param elementSizeBytes The size of a single data element in bytes. + + @param elementCount The number of elements in the buffer (must be power of 2). + + @param dataPtr A pointer to a previously allocated area where the data + will be maintained. It must be elementCount*elementSizeBytes long. + + @return -1 if elementCount is not a power of 2, otherwise 0. +*/ +PaRingBufferSize PaUtil_InitializeRingBuffer( PaUtilRingBuffer *rbuf, PaRingBufferSize elementSizeBytes, PaRingBufferSize elementCount, void *dataPtr ); + +/** Clear buffer. Should only be called when buffer is NOT being read. + + @param rbuf The ring buffer. +*/ +void PaUtil_FlushRingBuffer( PaUtilRingBuffer *rbuf ); + +/** Retrieve the number of elements available in the ring buffer for writing. + + @param rbuf The ring buffer. + + @return The number of elements available for writing. +*/ +PaRingBufferSize PaUtil_GetRingBufferWriteAvailable( PaUtilRingBuffer *rbuf ); + +/** Retrieve the number of elements available in the ring buffer for reading. + + @param rbuf The ring buffer. + + @return The number of elements available for reading. +*/ +PaRingBufferSize PaUtil_GetRingBufferReadAvailable( PaUtilRingBuffer *rbuf ); + +/** Write data to the ring buffer. + + @param rbuf The ring buffer. + + @param data The address of new data to write to the buffer. + + @param elementCount The number of elements to be written. + + @return The number of elements written. +*/ +PaRingBufferSize PaUtil_WriteRingBuffer( PaUtilRingBuffer *rbuf, const void *data, PaRingBufferSize elementCount ); + +/** Read data from the ring buffer. + + @param rbuf The ring buffer. + + @param data The address where the data should be stored. + + @param elementCount The number of elements to be read. + + @return The number of elements read. +*/ +PaRingBufferSize PaUtil_ReadRingBuffer( PaUtilRingBuffer *rbuf, void *data, PaRingBufferSize elementCount ); + +/** Get address of region(s) to which we can write data. + + @param rbuf The ring buffer. + + @param elementCount The number of elements desired. + + @param dataPtr1 The address where the first (or only) region pointer will be + stored. + + @param sizePtr1 The address where the first (or only) region length will be + stored. + + @param dataPtr2 The address where the second region pointer will be stored if + the first region is too small to satisfy elementCount. + + @param sizePtr2 The address where the second region length will be stored if + the first region is too small to satisfy elementCount. + + @return The room available to be written or elementCount, whichever is smaller. +*/ +PaRingBufferSize PaUtil_GetRingBufferWriteRegions( PaUtilRingBuffer *rbuf, PaRingBufferSize elementCount, + void **dataPtr1, PaRingBufferSize *sizePtr1, + void **dataPtr2, PaRingBufferSize *sizePtr2 ); + +/** Advance the write index to the next location to be written. + + @param rbuf The ring buffer. + + @param elementCount The number of elements to advance. + + @return The new position. +*/ +PaRingBufferSize PaUtil_AdvanceRingBufferWriteIndex( PaUtilRingBuffer *rbuf, PaRingBufferSize elementCount ); + +/** Get address of region(s) from which we can write data. + + @param rbuf The ring buffer. + + @param elementCount The number of elements desired. + + @param dataPtr1 The address where the first (or only) region pointer will be + stored. + + @param sizePtr1 The address where the first (or only) region length will be + stored. + + @param dataPtr2 The address where the second region pointer will be stored if + the first region is too small to satisfy elementCount. + + @param sizePtr2 The address where the second region length will be stored if + the first region is too small to satisfy elementCount. + + @return The number of elements available for reading. +*/ +PaRingBufferSize PaUtil_GetRingBufferReadRegions( PaUtilRingBuffer *rbuf, PaRingBufferSize elementCount, + void **dataPtr1, PaRingBufferSize *sizePtr1, + void **dataPtr2, PaRingBufferSize *sizePtr2 ); + +/** Advance the read index to the next location to be read. + + @param rbuf The ring buffer. + + @param elementCount The number of elements to advance. + + @return The new position. +*/ +PaRingBufferSize PaUtil_AdvanceRingBufferReadIndex( PaUtilRingBuffer *rbuf, PaRingBufferSize elementCount ); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* MODULES_AUDIO_DEVICE_MAIN_SOURCE_MAC_PORTAUDIO_PA_RINGBUFFER_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/mock_audio_device_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_device/mock_audio_device_buffer.h new file mode 100644 index 0000000000..f172c17eec --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/mock_audio_device_buffer.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_DEVICE_MOCK_AUDIO_DEVICE_BUFFER_H_ +#define MODULES_AUDIO_DEVICE_MOCK_AUDIO_DEVICE_BUFFER_H_ + +#include "modules/audio_device/audio_device_buffer.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockAudioDeviceBuffer : public AudioDeviceBuffer { + public: + MockAudioDeviceBuffer() {} + virtual ~MockAudioDeviceBuffer() {} + MOCK_METHOD1(RequestPlayoutData, int32_t(size_t nSamples)); + MOCK_METHOD1(GetPlayoutData, int32_t(void* audioBuffer)); + MOCK_METHOD2(SetRecordedBuffer, + int32_t(const void* audioBuffer, size_t nSamples)); + MOCK_METHOD3(SetVQEData, + void(int playDelayMS, int recDelayMS, int clockDrift)); + MOCK_METHOD0(DeliverRecordedData, int32_t()); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_DEVICE_MOCK_AUDIO_DEVICE_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.cc b/third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.cc new file mode 100644 index 0000000000..a7bad3af6e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if defined(_MSC_VER) +#include +#endif + +#include "single_rw_fifo.h" + +#include + +static int UpdatePos(int pos, int capacity) { + return (pos + 1) % capacity; +} + +namespace webrtc { + +namespace subtle { + +// Start with compiler support, then processor-specific hacks +#if defined(__GNUC__) || defined(__clang__) +// Available on GCC and clang - others? +inline void MemoryBarrier() { + __sync_synchronize(); +} + +#elif defined(_MSC_VER) +inline void MemoryBarrier() { + ::MemoryBarrier(); +} + +#elif defined(__aarch64__) +// From http://http://src.chromium.org/viewvc/chrome/trunk/src/base/atomicops_internals_arm64_gcc.h +inline void MemoryBarrier() { + __asm__ __volatile__ ("dmb ish" ::: "memory"); +} + +#elif defined(__ARMEL__) +// From http://src.chromium.org/viewvc/chrome/trunk/src/base/atomicops_internals_arm_gcc.h +inline void MemoryBarrier() { + // Note: This is a function call, which is also an implicit compiler barrier. + typedef void (*KernelMemoryBarrierFunc)(); + ((KernelMemoryBarrierFunc)0xffff0fa0)(); +} + +#elif defined(__x86_64__) || defined (__i386__) +// From http://src.chromium.org/viewvc/chrome/trunk/src/base/atomicops_internals_x86_gcc.h +// mfence exists on x64 and x86 platforms containing SSE2. +// x86 platforms that don't have SSE2 will crash with SIGILL. +// If this code needs to run on such platforms in the future, +// add runtime CPU detection here. +inline void MemoryBarrier() { + __asm__ __volatile__("mfence" : : : "memory"); +} + +#elif defined(__MIPSEL__) +// From http://src.chromium.org/viewvc/chrome/trunk/src/base/atomicops_internals_mips_gcc.h +inline void MemoryBarrier() { + __asm__ __volatile__("sync" : : : "memory"); +} + +#else +#error Add an implementation of MemoryBarrier() for this platform! +#endif + +} // namespace subtle + +SingleRwFifo::SingleRwFifo(int capacity) + : queue_(new int8_t*[capacity]), + capacity_(capacity), + size_(0), + read_pos_(0), + write_pos_(0) +{ +} + +SingleRwFifo::~SingleRwFifo() { +} + +void SingleRwFifo::Push(int8_t* mem) { + assert(mem); + + // Ensure that there is space for the new data in the FIFO. + // Note there is only one writer meaning that the other thread is guaranteed + // only to decrease the size. + const int free_slots = capacity() - size(); + if (free_slots <= 0) { + // Size can be queried outside of the Push function. The caller is assumed + // to ensure that Push will be successful before calling it. + assert(false); + return; + } + queue_.get()[write_pos_] = mem; + // Memory barrier ensures that |size_| is updated after the size has changed. + subtle::MemoryBarrier(); + ++size_; + write_pos_ = UpdatePos(write_pos_, capacity()); +} + +int8_t* SingleRwFifo::Pop() { + int8_t* ret_val = NULL; + if (size() <= 0) { + // Size can be queried outside of the Pop function. The caller is assumed + // to ensure that Pop will be successfull before calling it. + assert(false); + return ret_val; + } + ret_val = queue_.get()[read_pos_]; + // Memory barrier ensures that |size_| is updated after the size has changed. + subtle::MemoryBarrier(); + --size_; + read_pos_ = UpdatePos(read_pos_, capacity()); + return ret_val; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.h b/third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.h new file mode 100644 index 0000000000..ad0c677c82 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/opensl/single_rw_fifo.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_SINGLE_RW_FIFO_H_ +#define WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_SINGLE_RW_FIFO_H_ + +#include "system_wrappers/include/atomic32.h" +#include "typedefs.h" + +namespace webrtc { + +// Implements a lock-free FIFO losely based on +// http://src.chromium.org/viewvc/chrome/trunk/src/media/base/audio_fifo.cc +// Note that this class assumes there is one producer (writer) and one +// consumer (reader) thread. +class SingleRwFifo { + public: + explicit SingleRwFifo(int capacity); + ~SingleRwFifo(); + + void Push(int8_t* mem); + int8_t* Pop(); + + void Clear(); + + int size() { return size_.Value(); } + int capacity() const { return capacity_; } + + private: + std::unique_ptr queue_; + int capacity_; + + Atomic32 size_; + + int read_pos_; + int write_pos_; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_SINGLE_RW_FIFO_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.cc b/third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.cc new file mode 100644 index 0000000000..cf7df4b5de --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.cc @@ -0,0 +1,4360 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#pragma warning(disable : 4995) // name was marked as #pragma deprecated + +#if (_MSC_VER >= 1310) && (_MSC_VER < 1400) +// Reports the major and minor versions of the compiler. +// For example, 1310 for Microsoft Visual C++ .NET 2003. 1310 represents version +// 13 and a 1.0 point release. The Visual C++ 2005 compiler version is 1400. +// Type cl /? at the command line to see the major and minor versions of your +// compiler along with the build number. +#pragma message(">> INFO: Windows Core Audio is not supported in VS 2003") +#endif + +#include "modules/audio_device/audio_device_config.h" + +#ifdef WEBRTC_WINDOWS_CORE_AUDIO_BUILD + +#include "modules/audio_device/win/audio_device_core_win.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "rtc_base/logging.h" +#include "rtc_base/platform_thread.h" +#include "system_wrappers/include/sleep.h" + +// Macro that calls a COM method returning HRESULT value. +#define EXIT_ON_ERROR(hres) \ + do { \ + if (FAILED(hres)) \ + goto Exit; \ + } while (0) + +// Macro that continues to a COM error. +#define CONTINUE_ON_ERROR(hres) \ + do { \ + if (FAILED(hres)) \ + goto Next; \ + } while (0) + +// Macro that releases a COM object if not NULL. +#define SAFE_RELEASE(p) \ + do { \ + if ((p)) { \ + (p)->Release(); \ + (p) = NULL; \ + } \ + } while (0) + +#define ROUND(x) ((x) >= 0 ? (int)((x) + 0.5) : (int)((x)-0.5)) + +// REFERENCE_TIME time units per millisecond +#define REFTIMES_PER_MILLISEC 10000 + +typedef struct tagTHREADNAME_INFO { + DWORD dwType; // must be 0x1000 + LPCSTR szName; // pointer to name (in user addr space) + DWORD dwThreadID; // thread ID (-1=caller thread) + DWORD dwFlags; // reserved for future use, must be zero +} THREADNAME_INFO; + +namespace webrtc { +namespace { + +enum { COM_THREADING_MODEL = COINIT_MULTITHREADED }; + +enum { kAecCaptureStreamIndex = 0, kAecRenderStreamIndex = 1 }; + +// An implementation of IMediaBuffer, as required for +// IMediaObject::ProcessOutput(). After consuming data provided by +// ProcessOutput(), call SetLength() to update the buffer availability. +// +// Example implementation: +// http://msdn.microsoft.com/en-us/library/dd376684(v=vs.85).aspx +class MediaBufferImpl : public IMediaBuffer { + public: + explicit MediaBufferImpl(DWORD maxLength) + : _data(new BYTE[maxLength]), + _length(0), + _maxLength(maxLength), + _refCount(0) {} + + // IMediaBuffer methods. + STDMETHOD(GetBufferAndLength(BYTE** ppBuffer, DWORD* pcbLength)) { + if (!ppBuffer || !pcbLength) { + return E_POINTER; + } + + *ppBuffer = _data; + *pcbLength = _length; + + return S_OK; + } + + STDMETHOD(GetMaxLength(DWORD* pcbMaxLength)) { + if (!pcbMaxLength) { + return E_POINTER; + } + + *pcbMaxLength = _maxLength; + return S_OK; + } + + STDMETHOD(SetLength(DWORD cbLength)) { + if (cbLength > _maxLength) { + return E_INVALIDARG; + } + + _length = cbLength; + return S_OK; + } + + // IUnknown methods. + STDMETHOD_(ULONG, AddRef()) { return InterlockedIncrement(&_refCount); } + + STDMETHOD(QueryInterface(REFIID riid, void** ppv)) { + if (!ppv) { + return E_POINTER; + } else if (riid != IID_IMediaBuffer && riid != IID_IUnknown) { + return E_NOINTERFACE; + } + + *ppv = static_cast(this); + AddRef(); + return S_OK; + } + + STDMETHOD_(ULONG, Release()) { + LONG refCount = InterlockedDecrement(&_refCount); + if (refCount == 0) { + delete this; + } + + return refCount; + } + + private: + ~MediaBufferImpl() { delete[] _data; } + + BYTE* _data; + DWORD _length; + const DWORD _maxLength; + LONG _refCount; +}; +} // namespace + +// ============================================================================ +// Static Methods +// ============================================================================ + +// ---------------------------------------------------------------------------- +// CoreAudioIsSupported +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::CoreAudioIsSupported() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + bool MMDeviceIsAvailable(false); + bool coreAudioIsSupported(false); + + HRESULT hr(S_OK); + TCHAR buf[MAXERRORLENGTH]; + TCHAR errorText[MAXERRORLENGTH]; + + // 1) Check if Windows version is Vista SP1 or later. + // + // CoreAudio is only available on Vista SP1 and later. + // + OSVERSIONINFOEX osvi; + DWORDLONG dwlConditionMask = 0; + int op = VER_LESS_EQUAL; + + // Initialize the OSVERSIONINFOEX structure. + ZeroMemory(&osvi, sizeof(OSVERSIONINFOEX)); + osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX); + osvi.dwMajorVersion = 6; + osvi.dwMinorVersion = 0; + osvi.wServicePackMajor = 0; + osvi.wServicePackMinor = 0; + osvi.wProductType = VER_NT_WORKSTATION; + + // Initialize the condition mask. + VER_SET_CONDITION(dwlConditionMask, VER_MAJORVERSION, op); + VER_SET_CONDITION(dwlConditionMask, VER_MINORVERSION, op); + VER_SET_CONDITION(dwlConditionMask, VER_SERVICEPACKMAJOR, op); + VER_SET_CONDITION(dwlConditionMask, VER_SERVICEPACKMINOR, op); + VER_SET_CONDITION(dwlConditionMask, VER_PRODUCT_TYPE, VER_EQUAL); + + DWORD dwTypeMask = VER_MAJORVERSION | VER_MINORVERSION | + VER_SERVICEPACKMAJOR | VER_SERVICEPACKMINOR | + VER_PRODUCT_TYPE; + + // Perform the test. + BOOL isVistaRTMorXP = VerifyVersionInfo(&osvi, dwTypeMask, dwlConditionMask); + if (isVistaRTMorXP != 0) { + RTC_LOG(LS_VERBOSE) + << "*** Windows Core Audio is only supported on Vista SP1 or later" + << " => will revert to the Wave API ***"; + return false; + } + + // 2) Initializes the COM library for use by the calling thread. + + // The COM init wrapper sets the thread's concurrency model to MTA, + // and creates a new apartment for the thread if one is required. The + // wrapper also ensures that each call to CoInitializeEx is balanced + // by a corresponding call to CoUninitialize. + // + ScopedCOMInitializer comInit(ScopedCOMInitializer::kMTA); + if (!comInit.succeeded()) { + // Things will work even if an STA thread is calling this method but we + // want to ensure that MTA is used and therefore return false here. + return false; + } + + // 3) Check if the MMDevice API is available. + // + // The Windows Multimedia Device (MMDevice) API enables audio clients to + // discover audio endpoint devices, determine their capabilities, and create + // driver instances for those devices. + // Header file Mmdeviceapi.h defines the interfaces in the MMDevice API. + // The MMDevice API consists of several interfaces. The first of these is the + // IMMDeviceEnumerator interface. To access the interfaces in the MMDevice + // API, a client obtains a reference to the IMMDeviceEnumerator interface of a + // device-enumerator object by calling the CoCreateInstance function. + // + // Through the IMMDeviceEnumerator interface, the client can obtain references + // to the other interfaces in the MMDevice API. The MMDevice API implements + // the following interfaces: + // + // IMMDevice Represents an audio device. + // IMMDeviceCollection Represents a collection of audio devices. + // IMMDeviceEnumerator Provides methods for enumerating audio devices. + // IMMEndpoint Represents an audio endpoint device. + // + IMMDeviceEnumerator* pIMMD(NULL); + const CLSID CLSID_MMDeviceEnumerator = __uuidof(MMDeviceEnumerator); + const IID IID_IMMDeviceEnumerator = __uuidof(IMMDeviceEnumerator); + + hr = CoCreateInstance( + CLSID_MMDeviceEnumerator, // GUID value of MMDeviceEnumerator coclass + NULL, CLSCTX_ALL, + IID_IMMDeviceEnumerator, // GUID value of the IMMDeviceEnumerator + // interface + (void**)&pIMMD); + + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "AudioDeviceWindowsCore::CoreAudioIsSupported()" + << " Failed to create the required COM object (hr=" << hr + << ")"; + RTC_LOG(LS_VERBOSE) << "AudioDeviceWindowsCore::CoreAudioIsSupported()" + << " CoCreateInstance(MMDeviceEnumerator) failed (hr=" + << hr << ")"; + + const DWORD dwFlags = + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD dwLangID = MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US); + + // Gets the system's human readable message string for this HRESULT. + // All error message in English by default. + DWORD messageLength = ::FormatMessageW(dwFlags, 0, hr, dwLangID, errorText, + MAXERRORLENGTH, NULL); + + assert(messageLength <= MAXERRORLENGTH); + + // Trims tailing white space (FormatMessage() leaves a trailing cr-lf.). + for (; messageLength && ::isspace(errorText[messageLength - 1]); + --messageLength) { + errorText[messageLength - 1] = '\0'; + } + + StringCchPrintf(buf, MAXERRORLENGTH, TEXT("Error details: ")); + StringCchCat(buf, MAXERRORLENGTH, errorText); + RTC_LOG(LS_VERBOSE) << buf; + } else { + MMDeviceIsAvailable = true; + RTC_LOG(LS_VERBOSE) + << "AudioDeviceWindowsCore::CoreAudioIsSupported()" + << " CoCreateInstance(MMDeviceEnumerator) succeeded (hr=" << hr << ")"; + SAFE_RELEASE(pIMMD); + } + + // 4) Verify that we can create and initialize our Core Audio class. + // + // Also, perform a limited "API test" to ensure that Core Audio is supported + // for all devices. + // + if (MMDeviceIsAvailable) { + coreAudioIsSupported = false; + + AudioDeviceWindowsCore* p = new AudioDeviceWindowsCore(); + if (p == NULL) { + return false; + } + + int ok(0); + int temp_ok(0); + bool available(false); + + if (p->Init() != InitStatus::OK) { + ok |= -1; + } + + int16_t numDevsRec = p->RecordingDevices(); + for (uint16_t i = 0; i < numDevsRec; i++) { + ok |= p->SetRecordingDevice(i); + temp_ok = p->RecordingIsAvailable(available); + ok |= temp_ok; + ok |= (available == false); + if (available) { + ok |= p->InitMicrophone(); + } + if (ok) { + RTC_LOG(LS_WARNING) + << "AudioDeviceWindowsCore::CoreAudioIsSupported()" + << " Failed to use Core Audio Recording for device id=" << i; + } + } + + int16_t numDevsPlay = p->PlayoutDevices(); + for (uint16_t i = 0; i < numDevsPlay; i++) { + ok |= p->SetPlayoutDevice(i); + temp_ok = p->PlayoutIsAvailable(available); + ok |= temp_ok; + ok |= (available == false); + if (available) { + ok |= p->InitSpeaker(); + } + if (ok) { + RTC_LOG(LS_WARNING) + << "AudioDeviceWindowsCore::CoreAudioIsSupported()" + << " Failed to use Core Audio Playout for device id=" << i; + } + } + + ok |= p->Terminate(); + + if (ok == 0) { + coreAudioIsSupported = true; + } + + delete p; + } + + if (coreAudioIsSupported) { + RTC_LOG(LS_VERBOSE) << "*** Windows Core Audio is supported ***"; + } else { + RTC_LOG(LS_VERBOSE) << "*** Windows Core Audio is NOT supported" + << " => will revert to the Wave API ***"; + } + + return (coreAudioIsSupported); +} + +// ============================================================================ +// Construction & Destruction +// ============================================================================ + +// ---------------------------------------------------------------------------- +// AudioDeviceWindowsCore() - ctor +// ---------------------------------------------------------------------------- + +AudioDeviceWindowsCore::AudioDeviceWindowsCore() + : _comInit(ScopedCOMInitializer::kMTA), + _ptrAudioBuffer(NULL), + _ptrEnumerator(NULL), + _ptrRenderCollection(NULL), + _ptrCaptureCollection(NULL), + _ptrDeviceOut(NULL), + _ptrDeviceIn(NULL), + _ptrClientOut(NULL), + _ptrClientIn(NULL), + _ptrRenderClient(NULL), + _ptrCaptureClient(NULL), + _ptrCaptureVolume(NULL), + _ptrRenderSimpleVolume(NULL), + _dmo(NULL), + _mediaBuffer(NULL), + _builtInAecEnabled(false), + _playAudioFrameSize(0), + _playSampleRate(0), + _playBlockSize(0), + _playChannels(2), + _sndCardPlayDelay(0), + _sndCardRecDelay(0), + _writtenSamples(0), + _readSamples(0), + _recAudioFrameSize(0), + _recSampleRate(0), + _recBlockSize(0), + _recChannels(2), + _avrtLibrary(NULL), + _winSupportAvrt(false), + _hRenderSamplesReadyEvent(NULL), + _hPlayThread(NULL), + _hCaptureSamplesReadyEvent(NULL), + _hRecThread(NULL), + _hShutdownRenderEvent(NULL), + _hShutdownCaptureEvent(NULL), + _hRenderStartedEvent(NULL), + _hCaptureStartedEvent(NULL), + _hGetCaptureVolumeThread(NULL), + _hSetCaptureVolumeThread(NULL), + _hSetCaptureVolumeEvent(NULL), + _hMmTask(NULL), + _initialized(false), + _recording(false), + _playing(false), + _recIsInitialized(false), + _playIsInitialized(false), + _speakerIsInitialized(false), + _microphoneIsInitialized(false), + _AGC(false), + _playBufDelay(80), + _usingInputDeviceIndex(false), + _usingOutputDeviceIndex(false), + _inputDevice(AudioDeviceModule::kDefaultCommunicationDevice), + _outputDevice(AudioDeviceModule::kDefaultCommunicationDevice), + _inputDeviceIndex(0), + _outputDeviceIndex(0), + _newMicLevel(0) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " created"; + assert(_comInit.succeeded()); + + // Try to load the Avrt DLL + if (!_avrtLibrary) { + // Get handle to the Avrt DLL module. + _avrtLibrary = LoadLibrary(TEXT("Avrt.dll")); + if (_avrtLibrary) { + // Handle is valid (should only happen if OS larger than vista & win7). + // Try to get the function addresses. + RTC_LOG(LS_VERBOSE) << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + << " The Avrt DLL module is now loaded"; + + _PAvRevertMmThreadCharacteristics = + (PAvRevertMmThreadCharacteristics)GetProcAddress( + _avrtLibrary, "AvRevertMmThreadCharacteristics"); + _PAvSetMmThreadCharacteristicsA = + (PAvSetMmThreadCharacteristicsA)GetProcAddress( + _avrtLibrary, "AvSetMmThreadCharacteristicsA"); + _PAvSetMmThreadPriority = (PAvSetMmThreadPriority)GetProcAddress( + _avrtLibrary, "AvSetMmThreadPriority"); + + if (_PAvRevertMmThreadCharacteristics && + _PAvSetMmThreadCharacteristicsA && _PAvSetMmThreadPriority) { + RTC_LOG(LS_VERBOSE) + << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + << " AvRevertMmThreadCharacteristics() is OK"; + RTC_LOG(LS_VERBOSE) + << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + << " AvSetMmThreadCharacteristicsA() is OK"; + RTC_LOG(LS_VERBOSE) + << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + << " AvSetMmThreadPriority() is OK"; + _winSupportAvrt = true; + } + } + } + + // Create our samples ready events - we want auto reset events that start in + // the not-signaled state. The state of an auto-reset event object remains + // signaled until a single waiting thread is released, at which time the + // system automatically sets the state to nonsignaled. If no threads are + // waiting, the event object's state remains signaled. (Except for + // _hShutdownCaptureEvent, which is used to shutdown multiple threads). + _hRenderSamplesReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hCaptureSamplesReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hShutdownRenderEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hShutdownCaptureEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + _hRenderStartedEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hCaptureStartedEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hSetCaptureVolumeEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + + _perfCounterFreq.QuadPart = 1; + _perfCounterFactor = 0.0; + + // list of number of channels to use on recording side + _recChannelsPrioList[0] = 2; // stereo is prio 1 + _recChannelsPrioList[1] = 1; // mono is prio 2 + _recChannelsPrioList[2] = 4; // quad is prio 3 + + // list of number of channels to use on playout side + _playChannelsPrioList[0] = 2; // stereo is prio 1 + _playChannelsPrioList[1] = 1; // mono is prio 2 + + HRESULT hr; + + // We know that this API will work since it has already been verified in + // CoreAudioIsSupported, hence no need to check for errors here as well. + + // Retrive the IMMDeviceEnumerator API (should load the MMDevAPI.dll) + // TODO(henrika): we should probably move this allocation to Init() instead + // and deallocate in Terminate() to make the implementation more symmetric. + CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, + __uuidof(IMMDeviceEnumerator), + reinterpret_cast(&_ptrEnumerator)); + assert(NULL != _ptrEnumerator); + + // DMO initialization for built-in WASAPI AEC. + { + IMediaObject* ptrDMO = NULL; + hr = CoCreateInstance(CLSID_CWMAudioAEC, NULL, CLSCTX_INPROC_SERVER, + IID_IMediaObject, reinterpret_cast(&ptrDMO)); + if (FAILED(hr) || ptrDMO == NULL) { + // Since we check that _dmo is non-NULL in EnableBuiltInAEC(), the + // feature is prevented from being enabled. + _builtInAecEnabled = false; + _TraceCOMError(hr); + } + _dmo = ptrDMO; + SAFE_RELEASE(ptrDMO); + } +} + +// ---------------------------------------------------------------------------- +// AudioDeviceWindowsCore() - dtor +// ---------------------------------------------------------------------------- + +AudioDeviceWindowsCore::~AudioDeviceWindowsCore() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " destroyed"; + + Terminate(); + + // The IMMDeviceEnumerator is created during construction. Must release + // it here and not in Terminate() since we don't recreate it in Init(). + SAFE_RELEASE(_ptrEnumerator); + + _ptrAudioBuffer = NULL; + + if (NULL != _hRenderSamplesReadyEvent) { + CloseHandle(_hRenderSamplesReadyEvent); + _hRenderSamplesReadyEvent = NULL; + } + + if (NULL != _hCaptureSamplesReadyEvent) { + CloseHandle(_hCaptureSamplesReadyEvent); + _hCaptureSamplesReadyEvent = NULL; + } + + if (NULL != _hRenderStartedEvent) { + CloseHandle(_hRenderStartedEvent); + _hRenderStartedEvent = NULL; + } + + if (NULL != _hCaptureStartedEvent) { + CloseHandle(_hCaptureStartedEvent); + _hCaptureStartedEvent = NULL; + } + + if (NULL != _hShutdownRenderEvent) { + CloseHandle(_hShutdownRenderEvent); + _hShutdownRenderEvent = NULL; + } + + if (NULL != _hShutdownCaptureEvent) { + CloseHandle(_hShutdownCaptureEvent); + _hShutdownCaptureEvent = NULL; + } + + if (NULL != _hSetCaptureVolumeEvent) { + CloseHandle(_hSetCaptureVolumeEvent); + _hSetCaptureVolumeEvent = NULL; + } + + if (_avrtLibrary) { + BOOL freeOK = FreeLibrary(_avrtLibrary); + if (!freeOK) { + RTC_LOG(LS_WARNING) + << "AudioDeviceWindowsCore::~AudioDeviceWindowsCore()" + << " failed to free the loaded Avrt DLL module correctly"; + } else { + RTC_LOG(LS_WARNING) << "AudioDeviceWindowsCore::~AudioDeviceWindowsCore()" + << " the Avrt DLL module is now unloaded"; + } + } +} + +// ============================================================================ +// API +// ============================================================================ + +// ---------------------------------------------------------------------------- +// AttachAudioBuffer +// ---------------------------------------------------------------------------- + +void AudioDeviceWindowsCore::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + _ptrAudioBuffer = audioBuffer; + + // Inform the AudioBuffer about default settings for this implementation. + // Set all values to zero here since the actual settings will be done by + // InitPlayout and InitRecording later. + _ptrAudioBuffer->SetRecordingSampleRate(0); + _ptrAudioBuffer->SetPlayoutSampleRate(0); + _ptrAudioBuffer->SetRecordingChannels(0); + _ptrAudioBuffer->SetPlayoutChannels(0); +} + +// ---------------------------------------------------------------------------- +// ActiveAudioLayer +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const { + audioLayer = AudioDeviceModule::kWindowsCoreAudio; + return 0; +} + +// ---------------------------------------------------------------------------- +// Init +// ---------------------------------------------------------------------------- + +AudioDeviceGeneric::InitStatus AudioDeviceWindowsCore::Init() { + rtc::CritScope lock(&_critSect); + + if (_initialized) { + return InitStatus::OK; + } + + // Enumerate all audio rendering and capturing endpoint devices. + // Note that, some of these will not be able to select by the user. + // The complete collection is for internal use only. + _EnumerateEndpointDevicesAll(eRender); + _EnumerateEndpointDevicesAll(eCapture); + + _initialized = true; + + return InitStatus::OK; +} + +// ---------------------------------------------------------------------------- +// Terminate +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::Terminate() { + rtc::CritScope lock(&_critSect); + + if (!_initialized) { + return 0; + } + + _initialized = false; + _speakerIsInitialized = false; + _microphoneIsInitialized = false; + _playing = false; + _recording = false; + + SAFE_RELEASE(_ptrRenderCollection); + SAFE_RELEASE(_ptrCaptureCollection); + SAFE_RELEASE(_ptrDeviceOut); + SAFE_RELEASE(_ptrDeviceIn); + SAFE_RELEASE(_ptrClientOut); + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrRenderClient); + SAFE_RELEASE(_ptrCaptureClient); + SAFE_RELEASE(_ptrCaptureVolume); + SAFE_RELEASE(_ptrRenderSimpleVolume); + + return 0; +} + +// ---------------------------------------------------------------------------- +// Initialized +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::Initialized() const { + return (_initialized); +} + +// ---------------------------------------------------------------------------- +// InitSpeaker +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::InitSpeaker() { + rtc::CritScope lock(&_critSect); + + if (_playing) { + return -1; + } + + if (_ptrDeviceOut == NULL) { + return -1; + } + + if (_usingOutputDeviceIndex) { + int16_t nDevices = PlayoutDevices(); + if (_outputDeviceIndex > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "current device selection is invalid => unable to" + << " initialize"; + return -1; + } + } + + int32_t ret(0); + + SAFE_RELEASE(_ptrDeviceOut); + if (_usingOutputDeviceIndex) { + // Refresh the selected rendering endpoint device using current index + ret = _GetListDevice(eRender, _outputDeviceIndex, &_ptrDeviceOut); + } else { + ERole role; + (_outputDevice == AudioDeviceModule::kDefaultDevice) + ? role = eConsole + : role = eCommunications; + // Refresh the selected rendering endpoint device using role + ret = _GetDefaultDevice(eRender, role, &_ptrDeviceOut); + } + + if (ret != 0 || (_ptrDeviceOut == NULL)) { + RTC_LOG(LS_ERROR) << "failed to initialize the rendering enpoint device"; + SAFE_RELEASE(_ptrDeviceOut); + return -1; + } + + IAudioSessionManager* pManager = NULL; + ret = _ptrDeviceOut->Activate(__uuidof(IAudioSessionManager), CLSCTX_ALL, + NULL, (void**)&pManager); + if (ret != 0 || pManager == NULL) { + RTC_LOG(LS_ERROR) << "failed to initialize the render manager"; + SAFE_RELEASE(pManager); + return -1; + } + + SAFE_RELEASE(_ptrRenderSimpleVolume); + ret = pManager->GetSimpleAudioVolume(NULL, FALSE, &_ptrRenderSimpleVolume); + if (ret != 0 || _ptrRenderSimpleVolume == NULL) { + RTC_LOG(LS_ERROR) << "failed to initialize the render simple volume"; + SAFE_RELEASE(pManager); + SAFE_RELEASE(_ptrRenderSimpleVolume); + return -1; + } + SAFE_RELEASE(pManager); + + _speakerIsInitialized = true; + + return 0; +} + +// ---------------------------------------------------------------------------- +// InitMicrophone +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::InitMicrophone() { + rtc::CritScope lock(&_critSect); + + if (_recording) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + + if (_usingInputDeviceIndex) { + int16_t nDevices = RecordingDevices(); + if (_inputDeviceIndex > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "current device selection is invalid => unable to" + << " initialize"; + return -1; + } + } + + int32_t ret(0); + + SAFE_RELEASE(_ptrDeviceIn); + if (_usingInputDeviceIndex) { + // Refresh the selected capture endpoint device using current index + ret = _GetListDevice(eCapture, _inputDeviceIndex, &_ptrDeviceIn); + } else { + ERole role; + (_inputDevice == AudioDeviceModule::kDefaultDevice) + ? role = eConsole + : role = eCommunications; + // Refresh the selected capture endpoint device using role + ret = _GetDefaultDevice(eCapture, role, &_ptrDeviceIn); + } + + if (ret != 0 || (_ptrDeviceIn == NULL)) { + RTC_LOG(LS_ERROR) << "failed to initialize the capturing enpoint device"; + SAFE_RELEASE(_ptrDeviceIn); + return -1; + } + + SAFE_RELEASE(_ptrCaptureVolume); + ret = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&_ptrCaptureVolume)); + if (ret != 0 || _ptrCaptureVolume == NULL) { + RTC_LOG(LS_ERROR) << "failed to initialize the capture volume"; + SAFE_RELEASE(_ptrCaptureVolume); + return -1; + } + + _microphoneIsInitialized = true; + + return 0; +} + +// ---------------------------------------------------------------------------- +// SpeakerIsInitialized +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::SpeakerIsInitialized() const { + return (_speakerIsInitialized); +} + +// ---------------------------------------------------------------------------- +// MicrophoneIsInitialized +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::MicrophoneIsInitialized() const { + return (_microphoneIsInitialized); +} + +// ---------------------------------------------------------------------------- +// SpeakerVolumeIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SpeakerVolumeIsAvailable(bool& available) { + rtc::CritScope lock(&_critSect); + + if (_ptrDeviceOut == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioSessionManager* pManager = NULL; + ISimpleAudioVolume* pVolume = NULL; + + hr = _ptrDeviceOut->Activate(__uuidof(IAudioSessionManager), CLSCTX_ALL, NULL, + (void**)&pManager); + EXIT_ON_ERROR(hr); + + hr = pManager->GetSimpleAudioVolume(NULL, FALSE, &pVolume); + EXIT_ON_ERROR(hr); + + float volume(0.0f); + hr = pVolume->GetMasterVolume(&volume); + if (FAILED(hr)) { + available = false; + } + available = true; + + SAFE_RELEASE(pManager); + SAFE_RELEASE(pVolume); + + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pManager); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// SetSpeakerVolume +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetSpeakerVolume(uint32_t volume) { + { + rtc::CritScope lock(&_critSect); + + if (!_speakerIsInitialized) { + return -1; + } + + if (_ptrDeviceOut == NULL) { + return -1; + } + } + + if (volume < (uint32_t)MIN_CORE_SPEAKER_VOLUME || + volume > (uint32_t)MAX_CORE_SPEAKER_VOLUME) { + return -1; + } + + HRESULT hr = S_OK; + + // scale input volume to valid range (0.0 to 1.0) + const float fLevel = (float)volume / MAX_CORE_SPEAKER_VOLUME; + _volumeMutex.Enter(); + hr = _ptrRenderSimpleVolume->SetMasterVolume(fLevel, NULL); + _volumeMutex.Leave(); + EXIT_ON_ERROR(hr); + + return 0; + +Exit: + _TraceCOMError(hr); + return -1; +} + +// ---------------------------------------------------------------------------- +// SpeakerVolume +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SpeakerVolume(uint32_t& volume) const { + { + rtc::CritScope lock(&_critSect); + + if (!_speakerIsInitialized) { + return -1; + } + + if (_ptrDeviceOut == NULL) { + return -1; + } + } + + HRESULT hr = S_OK; + float fLevel(0.0f); + + _volumeMutex.Enter(); + hr = _ptrRenderSimpleVolume->GetMasterVolume(&fLevel); + _volumeMutex.Leave(); + EXIT_ON_ERROR(hr); + + // scale input volume range [0.0,1.0] to valid output range + volume = static_cast(fLevel * MAX_CORE_SPEAKER_VOLUME); + + return 0; + +Exit: + _TraceCOMError(hr); + return -1; +} + +// ---------------------------------------------------------------------------- +// MaxSpeakerVolume +// +// The internal range for Core Audio is 0.0 to 1.0, where 0.0 indicates +// silence and 1.0 indicates full volume (no attenuation). +// We add our (webrtc-internal) own max level to match the Wave API and +// how it is used today in VoE. +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MaxSpeakerVolume(uint32_t& maxVolume) const { + if (!_speakerIsInitialized) { + return -1; + } + + maxVolume = static_cast(MAX_CORE_SPEAKER_VOLUME); + + return 0; +} + +// ---------------------------------------------------------------------------- +// MinSpeakerVolume +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MinSpeakerVolume(uint32_t& minVolume) const { + if (!_speakerIsInitialized) { + return -1; + } + + minVolume = static_cast(MIN_CORE_SPEAKER_VOLUME); + + return 0; +} + +// ---------------------------------------------------------------------------- +// SpeakerMuteIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SpeakerMuteIsAvailable(bool& available) { + rtc::CritScope lock(&_critSect); + + if (_ptrDeviceOut == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Query the speaker system mute state. + hr = _ptrDeviceOut->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + BOOL mute; + hr = pVolume->GetMute(&mute); + if (FAILED(hr)) + available = false; + else + available = true; + + SAFE_RELEASE(pVolume); + + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// SetSpeakerMute +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetSpeakerMute(bool enable) { + rtc::CritScope lock(&_critSect); + + if (!_speakerIsInitialized) { + return -1; + } + + if (_ptrDeviceOut == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Set the speaker system mute state. + hr = _ptrDeviceOut->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + const BOOL mute(enable); + hr = pVolume->SetMute(mute, NULL); + EXIT_ON_ERROR(hr); + + SAFE_RELEASE(pVolume); + + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// SpeakerMute +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SpeakerMute(bool& enabled) const { + if (!_speakerIsInitialized) { + return -1; + } + + if (_ptrDeviceOut == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Query the speaker system mute state. + hr = _ptrDeviceOut->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + BOOL mute; + hr = pVolume->GetMute(&mute); + EXIT_ON_ERROR(hr); + + enabled = (mute == TRUE) ? true : false; + + SAFE_RELEASE(pVolume); + + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// MicrophoneMuteIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MicrophoneMuteIsAvailable(bool& available) { + rtc::CritScope lock(&_critSect); + + if (_ptrDeviceIn == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Query the microphone system mute state. + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + BOOL mute; + hr = pVolume->GetMute(&mute); + if (FAILED(hr)) + available = false; + else + available = true; + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// SetMicrophoneMute +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetMicrophoneMute(bool enable) { + if (!_microphoneIsInitialized) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Set the microphone system mute state. + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + const BOOL mute(enable); + hr = pVolume->SetMute(mute, NULL); + EXIT_ON_ERROR(hr); + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// MicrophoneMute +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MicrophoneMute(bool& enabled) const { + if (!_microphoneIsInitialized) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Query the microphone system mute state. + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + BOOL mute; + hr = pVolume->GetMute(&mute); + EXIT_ON_ERROR(hr); + + enabled = (mute == TRUE) ? true : false; + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// StereoRecordingIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StereoRecordingIsAvailable(bool& available) { + available = true; + return 0; +} + +// ---------------------------------------------------------------------------- +// SetStereoRecording +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetStereoRecording(bool enable) { + rtc::CritScope lock(&_critSect); + + if (enable) { + _recChannelsPrioList[0] = 2; // try stereo first + _recChannelsPrioList[1] = 1; + _recChannels = 2; + } else { + _recChannelsPrioList[0] = 1; // try mono first + _recChannelsPrioList[1] = 2; + _recChannels = 1; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// StereoRecording +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StereoRecording(bool& enabled) const { + if (_recChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +// ---------------------------------------------------------------------------- +// StereoPlayoutIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StereoPlayoutIsAvailable(bool& available) { + available = true; + return 0; +} + +// ---------------------------------------------------------------------------- +// SetStereoPlayout +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetStereoPlayout(bool enable) { + rtc::CritScope lock(&_critSect); + + if (enable) { + _playChannelsPrioList[0] = 2; // try stereo first + _playChannelsPrioList[1] = 1; + _playChannels = 2; + } else { + _playChannelsPrioList[0] = 1; // try mono first + _playChannelsPrioList[1] = 2; + _playChannels = 1; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// StereoPlayout +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StereoPlayout(bool& enabled) const { + if (_playChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + +// ---------------------------------------------------------------------------- +// SetAGC +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetAGC(bool enable) { + rtc::CritScope lock(&_critSect); + _AGC = enable; + return 0; +} + +// ---------------------------------------------------------------------------- +// AGC +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::AGC() const { + rtc::CritScope lock(&_critSect); + return _AGC; +} + +// ---------------------------------------------------------------------------- +// MicrophoneVolumeIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MicrophoneVolumeIsAvailable(bool& available) { + rtc::CritScope lock(&_critSect); + + if (_ptrDeviceIn == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + float volume(0.0f); + hr = pVolume->GetMasterVolumeLevelScalar(&volume); + if (FAILED(hr)) { + available = false; + } + available = true; + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// SetMicrophoneVolume +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetMicrophoneVolume(uint32_t volume) { + RTC_LOG(LS_VERBOSE) << "AudioDeviceWindowsCore::SetMicrophoneVolume(volume=" + << volume << ")"; + + { + rtc::CritScope lock(&_critSect); + + if (!_microphoneIsInitialized) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + } + + if (volume < static_cast(MIN_CORE_MICROPHONE_VOLUME) || + volume > static_cast(MAX_CORE_MICROPHONE_VOLUME)) { + return -1; + } + + HRESULT hr = S_OK; + // scale input volume to valid range (0.0 to 1.0) + const float fLevel = static_cast(volume) / MAX_CORE_MICROPHONE_VOLUME; + _volumeMutex.Enter(); + _ptrCaptureVolume->SetMasterVolumeLevelScalar(fLevel, NULL); + _volumeMutex.Leave(); + EXIT_ON_ERROR(hr); + + return 0; + +Exit: + _TraceCOMError(hr); + return -1; +} + +// ---------------------------------------------------------------------------- +// MicrophoneVolume +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MicrophoneVolume(uint32_t& volume) const { + { + rtc::CritScope lock(&_critSect); + + if (!_microphoneIsInitialized) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + } + + HRESULT hr = S_OK; + float fLevel(0.0f); + volume = 0; + _volumeMutex.Enter(); + hr = _ptrCaptureVolume->GetMasterVolumeLevelScalar(&fLevel); + _volumeMutex.Leave(); + EXIT_ON_ERROR(hr); + + // scale input volume range [0.0,1.0] to valid output range + volume = static_cast(fLevel * MAX_CORE_MICROPHONE_VOLUME); + + return 0; + +Exit: + _TraceCOMError(hr); + return -1; +} + +// ---------------------------------------------------------------------------- +// MaxMicrophoneVolume +// +// The internal range for Core Audio is 0.0 to 1.0, where 0.0 indicates +// silence and 1.0 indicates full volume (no attenuation). +// We add our (webrtc-internal) own max level to match the Wave API and +// how it is used today in VoE. +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MaxMicrophoneVolume(uint32_t& maxVolume) const { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + if (!_microphoneIsInitialized) { + return -1; + } + + maxVolume = static_cast(MAX_CORE_MICROPHONE_VOLUME); + + return 0; +} + +// ---------------------------------------------------------------------------- +// MinMicrophoneVolume +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::MinMicrophoneVolume(uint32_t& minVolume) const { + if (!_microphoneIsInitialized) { + return -1; + } + + minVolume = static_cast(MIN_CORE_MICROPHONE_VOLUME); + + return 0; +} + +// ---------------------------------------------------------------------------- +// PlayoutDevices +// ---------------------------------------------------------------------------- + +int16_t AudioDeviceWindowsCore::PlayoutDevices() { + rtc::CritScope lock(&_critSect); + + if (_RefreshDeviceList(eRender) != -1) { + return (_DeviceListCount(eRender)); + } + + return -1; +} + +// ---------------------------------------------------------------------------- +// SetPlayoutDevice I (II) +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetPlayoutDevice(uint16_t index) { + if (_playIsInitialized) { + return -1; + } + + // Get current number of available rendering endpoint devices and refresh the + // rendering collection. + UINT nDevices = PlayoutDevices(); + + if (index < 0 || index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + rtc::CritScope lock(&_critSect); + + HRESULT hr(S_OK); + + assert(_ptrRenderCollection != NULL); + + // Select an endpoint rendering device given the specified index + SAFE_RELEASE(_ptrDeviceOut); + hr = _ptrRenderCollection->Item(index, &_ptrDeviceOut); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(_ptrDeviceOut); + return -1; + } + + WCHAR szDeviceName[MAX_PATH]; + const int bufferLen = sizeof(szDeviceName) / sizeof(szDeviceName)[0]; + + // Get the endpoint device's friendly-name + if (_GetDeviceName(_ptrDeviceOut, szDeviceName, bufferLen) == 0) { + RTC_LOG(LS_VERBOSE) << "friendly name: \"" << szDeviceName << "\""; + } + + _usingOutputDeviceIndex = true; + _outputDeviceIndex = index; + + return 0; +} + +// ---------------------------------------------------------------------------- +// SetPlayoutDevice II (II) +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) { + if (_playIsInitialized) { + return -1; + } + + ERole role(eCommunications); + + if (device == AudioDeviceModule::kDefaultDevice) { + role = eConsole; + } else if (device == AudioDeviceModule::kDefaultCommunicationDevice) { + role = eCommunications; + } + + rtc::CritScope lock(&_critSect); + + // Refresh the list of rendering endpoint devices + _RefreshDeviceList(eRender); + + HRESULT hr(S_OK); + + assert(_ptrEnumerator != NULL); + + // Select an endpoint rendering device given the specified role + SAFE_RELEASE(_ptrDeviceOut); + hr = _ptrEnumerator->GetDefaultAudioEndpoint(eRender, role, &_ptrDeviceOut); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(_ptrDeviceOut); + return -1; + } + + WCHAR szDeviceName[MAX_PATH]; + const int bufferLen = sizeof(szDeviceName) / sizeof(szDeviceName)[0]; + + // Get the endpoint device's friendly-name + if (_GetDeviceName(_ptrDeviceOut, szDeviceName, bufferLen) == 0) { + RTC_LOG(LS_VERBOSE) << "friendly name: \"" << szDeviceName << "\""; + } + + _usingOutputDeviceIndex = false; + _outputDevice = device; + + return 0; +} + +// ---------------------------------------------------------------------------- +// PlayoutDeviceName +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::PlayoutDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + bool defaultCommunicationDevice(false); + const int16_t nDevices(PlayoutDevices()); // also updates the list of devices + + // Special fix for the case when the user selects '-1' as index (<=> Default + // Communication Device) + if (index == (uint16_t)(-1)) { + defaultCommunicationDevice = true; + index = 0; + RTC_LOG(LS_VERBOSE) << "Default Communication endpoint device will be used"; + } + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + rtc::CritScope lock(&_critSect); + + int32_t ret(-1); + WCHAR szDeviceName[MAX_PATH]; + const int bufferLen = sizeof(szDeviceName) / sizeof(szDeviceName)[0]; + + // Get the endpoint device's friendly-name + if (defaultCommunicationDevice) { + ret = _GetDefaultDeviceName(eRender, eCommunications, szDeviceName, + bufferLen); + } else { + ret = _GetListDeviceName(eRender, index, szDeviceName, bufferLen); + } + + if (ret == 0) { + // Convert the endpoint device's friendly-name to UTF-8 + if (WideCharToMultiByte(CP_UTF8, 0, szDeviceName, -1, name, + kAdmMaxDeviceNameSize, NULL, NULL) == 0) { + RTC_LOG(LS_ERROR) + << "WideCharToMultiByte(CP_UTF8) failed with error code " + << GetLastError(); + } + } + + // Get the endpoint ID string (uniquely identifies the device among all audio + // endpoint devices) + if (defaultCommunicationDevice) { + ret = + _GetDefaultDeviceID(eRender, eCommunications, szDeviceName, bufferLen); + } else { + ret = _GetListDeviceID(eRender, index, szDeviceName, bufferLen); + } + + if (guid != NULL && ret == 0) { + // Convert the endpoint device's ID string to UTF-8 + if (WideCharToMultiByte(CP_UTF8, 0, szDeviceName, -1, guid, kAdmMaxGuidSize, + NULL, NULL) == 0) { + RTC_LOG(LS_ERROR) + << "WideCharToMultiByte(CP_UTF8) failed with error code " + << GetLastError(); + } + } + + return ret; +} + +// ---------------------------------------------------------------------------- +// RecordingDeviceName +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::RecordingDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + bool defaultCommunicationDevice(false); + const int16_t nDevices( + RecordingDevices()); // also updates the list of devices + + // Special fix for the case when the user selects '-1' as index (<=> Default + // Communication Device) + if (index == (uint16_t)(-1)) { + defaultCommunicationDevice = true; + index = 0; + RTC_LOG(LS_VERBOSE) << "Default Communication endpoint device will be used"; + } + + if ((index > (nDevices - 1)) || (name == NULL)) { + return -1; + } + + memset(name, 0, kAdmMaxDeviceNameSize); + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); + } + + rtc::CritScope lock(&_critSect); + + int32_t ret(-1); + WCHAR szDeviceName[MAX_PATH]; + const int bufferLen = sizeof(szDeviceName) / sizeof(szDeviceName)[0]; + + // Get the endpoint device's friendly-name + if (defaultCommunicationDevice) { + ret = _GetDefaultDeviceName(eCapture, eCommunications, szDeviceName, + bufferLen); + } else { + ret = _GetListDeviceName(eCapture, index, szDeviceName, bufferLen); + } + + if (ret == 0) { + // Convert the endpoint device's friendly-name to UTF-8 + if (WideCharToMultiByte(CP_UTF8, 0, szDeviceName, -1, name, + kAdmMaxDeviceNameSize, NULL, NULL) == 0) { + RTC_LOG(LS_ERROR) + << "WideCharToMultiByte(CP_UTF8) failed with error code " + << GetLastError(); + } + } + + // Get the endpoint ID string (uniquely identifies the device among all audio + // endpoint devices) + if (defaultCommunicationDevice) { + ret = + _GetDefaultDeviceID(eCapture, eCommunications, szDeviceName, bufferLen); + } else { + ret = _GetListDeviceID(eCapture, index, szDeviceName, bufferLen); + } + + if (guid != NULL && ret == 0) { + // Convert the endpoint device's ID string to UTF-8 + if (WideCharToMultiByte(CP_UTF8, 0, szDeviceName, -1, guid, kAdmMaxGuidSize, + NULL, NULL) == 0) { + RTC_LOG(LS_ERROR) + << "WideCharToMultiByte(CP_UTF8) failed with error code " + << GetLastError(); + } + } + + return ret; +} + +// ---------------------------------------------------------------------------- +// RecordingDevices +// ---------------------------------------------------------------------------- + +int16_t AudioDeviceWindowsCore::RecordingDevices() { + rtc::CritScope lock(&_critSect); + + if (_RefreshDeviceList(eCapture) != -1) { + return (_DeviceListCount(eCapture)); + } + + return -1; +} + +// ---------------------------------------------------------------------------- +// SetRecordingDevice I (II) +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetRecordingDevice(uint16_t index) { + if (_recIsInitialized) { + return -1; + } + + // Get current number of available capture endpoint devices and refresh the + // capture collection. + UINT nDevices = RecordingDevices(); + + if (index < 0 || index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + rtc::CritScope lock(&_critSect); + + HRESULT hr(S_OK); + + assert(_ptrCaptureCollection != NULL); + + // Select an endpoint capture device given the specified index + SAFE_RELEASE(_ptrDeviceIn); + hr = _ptrCaptureCollection->Item(index, &_ptrDeviceIn); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(_ptrDeviceIn); + return -1; + } + + WCHAR szDeviceName[MAX_PATH]; + const int bufferLen = sizeof(szDeviceName) / sizeof(szDeviceName)[0]; + + // Get the endpoint device's friendly-name + if (_GetDeviceName(_ptrDeviceIn, szDeviceName, bufferLen) == 0) { + RTC_LOG(LS_VERBOSE) << "friendly name: \"" << szDeviceName << "\""; + } + + _usingInputDeviceIndex = true; + _inputDeviceIndex = index; + + return 0; +} + +// ---------------------------------------------------------------------------- +// SetRecordingDevice II (II) +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) { + if (_recIsInitialized) { + return -1; + } + + ERole role(eCommunications); + + if (device == AudioDeviceModule::kDefaultDevice) { + role = eConsole; + } else if (device == AudioDeviceModule::kDefaultCommunicationDevice) { + role = eCommunications; + } + + rtc::CritScope lock(&_critSect); + + // Refresh the list of capture endpoint devices + _RefreshDeviceList(eCapture); + + HRESULT hr(S_OK); + + assert(_ptrEnumerator != NULL); + + // Select an endpoint capture device given the specified role + SAFE_RELEASE(_ptrDeviceIn); + hr = _ptrEnumerator->GetDefaultAudioEndpoint(eCapture, role, &_ptrDeviceIn); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(_ptrDeviceIn); + return -1; + } + + WCHAR szDeviceName[MAX_PATH]; + const int bufferLen = sizeof(szDeviceName) / sizeof(szDeviceName)[0]; + + // Get the endpoint device's friendly-name + if (_GetDeviceName(_ptrDeviceIn, szDeviceName, bufferLen) == 0) { + RTC_LOG(LS_VERBOSE) << "friendly name: \"" << szDeviceName << "\""; + } + + _usingInputDeviceIndex = false; + _inputDevice = device; + + return 0; +} + +// ---------------------------------------------------------------------------- +// PlayoutIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::PlayoutIsAvailable(bool& available) { + available = false; + + // Try to initialize the playout side + int32_t res = InitPlayout(); + + // Cancel effect of initialization + StopPlayout(); + + if (res != -1) { + available = true; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// RecordingIsAvailable +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::RecordingIsAvailable(bool& available) { + available = false; + + // Try to initialize the recording side + int32_t res = InitRecording(); + + // Cancel effect of initialization + StopRecording(); + + if (res != -1) { + available = true; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// InitPlayout +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::InitPlayout() { + rtc::CritScope lock(&_critSect); + + if (_playing) { + return -1; + } + + if (_playIsInitialized) { + return 0; + } + + if (_ptrDeviceOut == NULL) { + return -1; + } + + // Initialize the speaker (devices might have been added or removed) + if (InitSpeaker() == -1) { + RTC_LOG(LS_WARNING) << "InitSpeaker() failed"; + } + + // Ensure that the updated rendering endpoint device is valid + if (_ptrDeviceOut == NULL) { + return -1; + } + + if (_builtInAecEnabled && _recIsInitialized) { + // Ensure the correct render device is configured in case + // InitRecording() was called before InitPlayout(). + if (SetDMOProperties() == -1) { + return -1; + } + } + + HRESULT hr = S_OK; + WAVEFORMATEX* pWfxOut = NULL; + WAVEFORMATEX Wfx = WAVEFORMATEX(); + WAVEFORMATEX* pWfxClosestMatch = NULL; + + // Create COM object with IAudioClient interface. + SAFE_RELEASE(_ptrClientOut); + hr = _ptrDeviceOut->Activate(__uuidof(IAudioClient), CLSCTX_ALL, NULL, + (void**)&_ptrClientOut); + EXIT_ON_ERROR(hr); + + // Retrieve the stream format that the audio engine uses for its internal + // processing (mixing) of shared-mode streams. + hr = _ptrClientOut->GetMixFormat(&pWfxOut); + if (SUCCEEDED(hr)) { + RTC_LOG(LS_VERBOSE) << "Audio Engine's current rendering mix format:"; + // format type + RTC_LOG(LS_VERBOSE) << "wFormatTag : 0x" << std::hex + << pWfxOut->wFormatTag << std::dec << " (" + << pWfxOut->wFormatTag << ")"; + // number of channels (i.e. mono, stereo...) + RTC_LOG(LS_VERBOSE) << "nChannels : " << pWfxOut->nChannels; + // sample rate + RTC_LOG(LS_VERBOSE) << "nSamplesPerSec : " << pWfxOut->nSamplesPerSec; + // for buffer estimation + RTC_LOG(LS_VERBOSE) << "nAvgBytesPerSec: " << pWfxOut->nAvgBytesPerSec; + // block size of data + RTC_LOG(LS_VERBOSE) << "nBlockAlign : " << pWfxOut->nBlockAlign; + // number of bits per sample of mono data + RTC_LOG(LS_VERBOSE) << "wBitsPerSample : " << pWfxOut->wBitsPerSample; + RTC_LOG(LS_VERBOSE) << "cbSize : " << pWfxOut->cbSize; + } + + // Set wave format + Wfx.wFormatTag = WAVE_FORMAT_PCM; + Wfx.wBitsPerSample = 16; + Wfx.cbSize = 0; + + const int freqs[] = {48000, 44100, 16000, 96000, 32000, 8000}; + hr = S_FALSE; + + // Iterate over frequencies and channels, in order of priority + for (unsigned int freq = 0; freq < sizeof(freqs) / sizeof(freqs[0]); freq++) { + for (unsigned int chan = 0; chan < sizeof(_playChannelsPrioList) / + sizeof(_playChannelsPrioList[0]); + chan++) { + Wfx.nChannels = _playChannelsPrioList[chan]; + Wfx.nSamplesPerSec = freqs[freq]; + Wfx.nBlockAlign = Wfx.nChannels * Wfx.wBitsPerSample / 8; + Wfx.nAvgBytesPerSec = Wfx.nSamplesPerSec * Wfx.nBlockAlign; + // If the method succeeds and the audio endpoint device supports the + // specified stream format, it returns S_OK. If the method succeeds and + // provides a closest match to the specified format, it returns S_FALSE. + hr = _ptrClientOut->IsFormatSupported(AUDCLNT_SHAREMODE_SHARED, &Wfx, + &pWfxClosestMatch); + if (hr == S_OK) { + break; + } else { + if (pWfxClosestMatch) { + RTC_LOG(INFO) << "nChannels=" << Wfx.nChannels + << ", nSamplesPerSec=" << Wfx.nSamplesPerSec + << " is not supported. Closest match: " + << "nChannels=" << pWfxClosestMatch->nChannels + << ", nSamplesPerSec=" + << pWfxClosestMatch->nSamplesPerSec; + CoTaskMemFree(pWfxClosestMatch); + pWfxClosestMatch = NULL; + } else { + RTC_LOG(INFO) << "nChannels=" << Wfx.nChannels + << ", nSamplesPerSec=" << Wfx.nSamplesPerSec + << " is not supported. No closest match."; + } + } + } + if (hr == S_OK) + break; + } + + // TODO(andrew): what happens in the event of failure in the above loop? + // Is _ptrClientOut->Initialize expected to fail? + // Same in InitRecording(). + if (hr == S_OK) { + _playAudioFrameSize = Wfx.nBlockAlign; + // Block size is the number of samples each channel in 10ms. + _playBlockSize = Wfx.nSamplesPerSec / 100; + _playSampleRate = Wfx.nSamplesPerSec; + _devicePlaySampleRate = + Wfx.nSamplesPerSec; // The device itself continues to run at 44.1 kHz. + _devicePlayBlockSize = Wfx.nSamplesPerSec / 100; + _playChannels = Wfx.nChannels; + + RTC_LOG(LS_VERBOSE) << "VoE selected this rendering format:"; + RTC_LOG(LS_VERBOSE) << "wFormatTag : 0x" << std::hex + << Wfx.wFormatTag << std::dec << " (" << Wfx.wFormatTag + << ")"; + RTC_LOG(LS_VERBOSE) << "nChannels : " << Wfx.nChannels; + RTC_LOG(LS_VERBOSE) << "nSamplesPerSec : " << Wfx.nSamplesPerSec; + RTC_LOG(LS_VERBOSE) << "nAvgBytesPerSec : " << Wfx.nAvgBytesPerSec; + RTC_LOG(LS_VERBOSE) << "nBlockAlign : " << Wfx.nBlockAlign; + RTC_LOG(LS_VERBOSE) << "wBitsPerSample : " << Wfx.wBitsPerSample; + RTC_LOG(LS_VERBOSE) << "cbSize : " << Wfx.cbSize; + RTC_LOG(LS_VERBOSE) << "Additional settings:"; + RTC_LOG(LS_VERBOSE) << "_playAudioFrameSize: " << _playAudioFrameSize; + RTC_LOG(LS_VERBOSE) << "_playBlockSize : " + << _playBlockSize; + RTC_LOG(LS_VERBOSE) << "_playChannels : " << _playChannels; + } + + // Create a rendering stream. + // + // **************************************************************************** + // For a shared-mode stream that uses event-driven buffering, the caller must + // set both hnsPeriodicity and hnsBufferDuration to 0. The Initialize method + // determines how large a buffer to allocate based on the scheduling period + // of the audio engine. Although the client's buffer processing thread is + // event driven, the basic buffer management process, as described previously, + // is unaltered. + // Each time the thread awakens, it should call + // IAudioClient::GetCurrentPadding to determine how much data to write to a + // rendering buffer or read from a capture buffer. In contrast to the two + // buffers that the Initialize method allocates for an exclusive-mode stream + // that uses event-driven buffering, a shared-mode stream requires a single + // buffer. + // **************************************************************************** + // + REFERENCE_TIME hnsBufferDuration = + 0; // ask for minimum buffer size (default) + if (_devicePlaySampleRate == 44100) { + // Ask for a larger buffer size (30ms) when using 44.1kHz as render rate. + // There seems to be a larger risk of underruns for 44.1 compared + // with the default rate (48kHz). When using default, we set the requested + // buffer duration to 0, which sets the buffer to the minimum size + // required by the engine thread. The actual buffer size can then be + // read by GetBufferSize() and it is 20ms on most machines. + hnsBufferDuration = 30 * 10000; + } + hr = _ptrClientOut->Initialize( + AUDCLNT_SHAREMODE_SHARED, // share Audio Engine with other applications + AUDCLNT_STREAMFLAGS_EVENTCALLBACK, // processing of the audio buffer by + // the client will be event driven + hnsBufferDuration, // requested buffer capacity as a time value (in + // 100-nanosecond units) + 0, // periodicity + &Wfx, // selected wave format + NULL); // session GUID + + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "IAudioClient::Initialize() failed:"; + } + EXIT_ON_ERROR(hr); + + if (_ptrAudioBuffer) { + // Update the audio buffer with the selected parameters + _ptrAudioBuffer->SetPlayoutSampleRate(_playSampleRate); + _ptrAudioBuffer->SetPlayoutChannels((uint8_t)_playChannels); + } else { + // We can enter this state during CoreAudioIsSupported() when no + // AudioDeviceImplementation has been created, hence the AudioDeviceBuffer + // does not exist. It is OK to end up here since we don't initiate any media + // in CoreAudioIsSupported(). + RTC_LOG(LS_VERBOSE) + << "AudioDeviceBuffer must be attached before streaming can start"; + } + + // Get the actual size of the shared (endpoint buffer). + // Typical value is 960 audio frames <=> 20ms @ 48kHz sample rate. + UINT bufferFrameCount(0); + hr = _ptrClientOut->GetBufferSize(&bufferFrameCount); + if (SUCCEEDED(hr)) { + RTC_LOG(LS_VERBOSE) << "IAudioClient::GetBufferSize() => " + << bufferFrameCount << " (<=> " + << bufferFrameCount * _playAudioFrameSize << " bytes)"; + } + + // Set the event handle that the system signals when an audio buffer is ready + // to be processed by the client. + hr = _ptrClientOut->SetEventHandle(_hRenderSamplesReadyEvent); + EXIT_ON_ERROR(hr); + + // Get an IAudioRenderClient interface. + SAFE_RELEASE(_ptrRenderClient); + hr = _ptrClientOut->GetService(__uuidof(IAudioRenderClient), + (void**)&_ptrRenderClient); + EXIT_ON_ERROR(hr); + + // Mark playout side as initialized + _playIsInitialized = true; + + CoTaskMemFree(pWfxOut); + CoTaskMemFree(pWfxClosestMatch); + + RTC_LOG(LS_VERBOSE) << "render side is now initialized"; + return 0; + +Exit: + _TraceCOMError(hr); + CoTaskMemFree(pWfxOut); + CoTaskMemFree(pWfxClosestMatch); + SAFE_RELEASE(_ptrClientOut); + SAFE_RELEASE(_ptrRenderClient); + return -1; +} + +// Capture initialization when the built-in AEC DirectX Media Object (DMO) is +// used. Called from InitRecording(), most of which is skipped over. The DMO +// handles device initialization itself. +// Reference: http://msdn.microsoft.com/en-us/library/ff819492(v=vs.85).aspx +int32_t AudioDeviceWindowsCore::InitRecordingDMO() { + assert(_builtInAecEnabled); + assert(_dmo != NULL); + + if (SetDMOProperties() == -1) { + return -1; + } + + DMO_MEDIA_TYPE mt = {0}; + HRESULT hr = MoInitMediaType(&mt, sizeof(WAVEFORMATEX)); + if (FAILED(hr)) { + MoFreeMediaType(&mt); + _TraceCOMError(hr); + return -1; + } + mt.majortype = MEDIATYPE_Audio; + mt.subtype = MEDIASUBTYPE_PCM; + mt.formattype = FORMAT_WaveFormatEx; + + // Supported formats + // nChannels: 1 (in AEC-only mode) + // nSamplesPerSec: 8000, 11025, 16000, 22050 + // wBitsPerSample: 16 + WAVEFORMATEX* ptrWav = reinterpret_cast(mt.pbFormat); + ptrWav->wFormatTag = WAVE_FORMAT_PCM; + ptrWav->nChannels = 1; + // 16000 is the highest we can support with our resampler. + ptrWav->nSamplesPerSec = 16000; + ptrWav->nAvgBytesPerSec = 32000; + ptrWav->nBlockAlign = 2; + ptrWav->wBitsPerSample = 16; + ptrWav->cbSize = 0; + + // Set the VoE format equal to the AEC output format. + _recAudioFrameSize = ptrWav->nBlockAlign; + _recSampleRate = ptrWav->nSamplesPerSec; + _recBlockSize = ptrWav->nSamplesPerSec / 100; + _recChannels = ptrWav->nChannels; + + // Set the DMO output format parameters. + hr = _dmo->SetOutputType(kAecCaptureStreamIndex, &mt, 0); + MoFreeMediaType(&mt); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + if (_ptrAudioBuffer) { + _ptrAudioBuffer->SetRecordingSampleRate(_recSampleRate); + _ptrAudioBuffer->SetRecordingChannels(_recChannels); + } else { + // Refer to InitRecording() for comments. + RTC_LOG(LS_VERBOSE) + << "AudioDeviceBuffer must be attached before streaming can start"; + } + + _mediaBuffer = new MediaBufferImpl(_recBlockSize * _recAudioFrameSize); + + // Optional, but if called, must be after media types are set. + hr = _dmo->AllocateStreamingResources(); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + _recIsInitialized = true; + RTC_LOG(LS_VERBOSE) << "Capture side is now initialized"; + + return 0; +} + +// ---------------------------------------------------------------------------- +// InitRecording +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::InitRecording() { + rtc::CritScope lock(&_critSect); + + if (_recording) { + return -1; + } + + if (_recIsInitialized) { + return 0; + } + + if (QueryPerformanceFrequency(&_perfCounterFreq) == 0) { + return -1; + } + _perfCounterFactor = 10000000.0 / (double)_perfCounterFreq.QuadPart; + + if (_ptrDeviceIn == NULL) { + return -1; + } + + // Initialize the microphone (devices might have been added or removed) + if (InitMicrophone() == -1) { + RTC_LOG(LS_WARNING) << "InitMicrophone() failed"; + } + + // Ensure that the updated capturing endpoint device is valid + if (_ptrDeviceIn == NULL) { + return -1; + } + + if (_builtInAecEnabled) { + // The DMO will configure the capture device. + return InitRecordingDMO(); + } + + HRESULT hr = S_OK; + WAVEFORMATEX* pWfxIn = NULL; + WAVEFORMATEXTENSIBLE Wfx = WAVEFORMATEXTENSIBLE(); + WAVEFORMATEX* pWfxClosestMatch = NULL; + + // Create COM object with IAudioClient interface. + SAFE_RELEASE(_ptrClientIn); + hr = _ptrDeviceIn->Activate(__uuidof(IAudioClient), CLSCTX_ALL, NULL, + (void**)&_ptrClientIn); + EXIT_ON_ERROR(hr); + + // Retrieve the stream format that the audio engine uses for its internal + // processing (mixing) of shared-mode streams. + hr = _ptrClientIn->GetMixFormat(&pWfxIn); + if (SUCCEEDED(hr)) { + RTC_LOG(LS_VERBOSE) << "Audio Engine's current capturing mix format:"; + // format type + RTC_LOG(LS_VERBOSE) << "wFormatTag : 0x" << std::hex + << pWfxIn->wFormatTag << std::dec << " (" + << pWfxIn->wFormatTag << ")"; + // number of channels (i.e. mono, stereo...) + RTC_LOG(LS_VERBOSE) << "nChannels : " << pWfxIn->nChannels; + // sample rate + RTC_LOG(LS_VERBOSE) << "nSamplesPerSec : " << pWfxIn->nSamplesPerSec; + // for buffer estimation + RTC_LOG(LS_VERBOSE) << "nAvgBytesPerSec: " << pWfxIn->nAvgBytesPerSec; + // block size of data + RTC_LOG(LS_VERBOSE) << "nBlockAlign : " << pWfxIn->nBlockAlign; + // number of bits per sample of mono data + RTC_LOG(LS_VERBOSE) << "wBitsPerSample : " << pWfxIn->wBitsPerSample; + RTC_LOG(LS_VERBOSE) << "cbSize : " << pWfxIn->cbSize; + } + + // Set wave format + Wfx.Format.wFormatTag = WAVE_FORMAT_EXTENSIBLE; + Wfx.Format.wBitsPerSample = 16; + Wfx.Format.cbSize = 22; + Wfx.dwChannelMask = 0; + Wfx.Samples.wValidBitsPerSample = Wfx.Format.wBitsPerSample; + Wfx.SubFormat = KSDATAFORMAT_SUBTYPE_PCM; + + const int freqs[6] = {48000, 44100, 16000, 96000, 32000, 8000}; + hr = S_FALSE; + + // Iterate over frequencies and channels, in order of priority + for (unsigned int freq = 0; freq < sizeof(freqs) / sizeof(freqs[0]); freq++) { + for (unsigned int chan = 0; + chan < sizeof(_recChannelsPrioList) / sizeof(_recChannelsPrioList[0]); + chan++) { + Wfx.Format.nChannels = _recChannelsPrioList[chan]; + Wfx.Format.nSamplesPerSec = freqs[freq]; + Wfx.Format.nBlockAlign = + Wfx.Format.nChannels * Wfx.Format.wBitsPerSample / 8; + Wfx.Format.nAvgBytesPerSec = + Wfx.Format.nSamplesPerSec * Wfx.Format.nBlockAlign; + // If the method succeeds and the audio endpoint device supports the + // specified stream format, it returns S_OK. If the method succeeds and + // provides a closest match to the specified format, it returns S_FALSE. + hr = _ptrClientIn->IsFormatSupported( + AUDCLNT_SHAREMODE_SHARED, (WAVEFORMATEX*)&Wfx, &pWfxClosestMatch); + if (hr == S_OK) { + break; + } else { + if (pWfxClosestMatch) { + RTC_LOG(INFO) << "nChannels=" << Wfx.Format.nChannels + << ", nSamplesPerSec=" << Wfx.Format.nSamplesPerSec + << " is not supported. Closest match: " + << "nChannels=" << pWfxClosestMatch->nChannels + << ", nSamplesPerSec=" + << pWfxClosestMatch->nSamplesPerSec; + CoTaskMemFree(pWfxClosestMatch); + pWfxClosestMatch = NULL; + } else { + RTC_LOG(INFO) << "nChannels=" << Wfx.Format.nChannels + << ", nSamplesPerSec=" << Wfx.Format.nSamplesPerSec + << " is not supported. No closest match."; + } + } + } + if (hr == S_OK) + break; + } + + if (hr == S_OK) { + _recAudioFrameSize = Wfx.Format.nBlockAlign; + _recSampleRate = Wfx.Format.nSamplesPerSec; + _recBlockSize = Wfx.Format.nSamplesPerSec / 100; + _recChannels = Wfx.Format.nChannels; + + RTC_LOG(LS_VERBOSE) << "VoE selected this capturing format:"; + RTC_LOG(LS_VERBOSE) << "wFormatTag : 0x" << std::hex + << Wfx.Format.wFormatTag << std::dec << " (" + << Wfx.Format.wFormatTag << ")"; + RTC_LOG(LS_VERBOSE) << "nChannels : " << Wfx.Format.nChannels; + RTC_LOG(LS_VERBOSE) << "nSamplesPerSec : " << Wfx.Format.nSamplesPerSec; + RTC_LOG(LS_VERBOSE) << "nAvgBytesPerSec : " << Wfx.Format.nAvgBytesPerSec; + RTC_LOG(LS_VERBOSE) << "nBlockAlign : " << Wfx.Format.nBlockAlign; + RTC_LOG(LS_VERBOSE) << "wBitsPerSample : " << Wfx.Format.wBitsPerSample; + RTC_LOG(LS_VERBOSE) << "cbSize : " << Wfx.Format.cbSize; + RTC_LOG(LS_VERBOSE) << "Additional settings:"; + RTC_LOG(LS_VERBOSE) << "_recAudioFrameSize: " << _recAudioFrameSize; + RTC_LOG(LS_VERBOSE) << "_recBlockSize : " << _recBlockSize; + RTC_LOG(LS_VERBOSE) << "_recChannels : " << _recChannels; + } + + // Create a capturing stream. + hr = _ptrClientIn->Initialize( + AUDCLNT_SHAREMODE_SHARED, // share Audio Engine with other applications + AUDCLNT_STREAMFLAGS_EVENTCALLBACK | // processing of the audio buffer by + // the client will be event driven + AUDCLNT_STREAMFLAGS_NOPERSIST, // volume and mute settings for an + // audio session will not persist + // across system restarts + 0, // required for event-driven shared mode + 0, // periodicity + (WAVEFORMATEX*)&Wfx, // selected wave format + NULL); // session GUID + + if (hr != S_OK) { + RTC_LOG(LS_ERROR) << "IAudioClient::Initialize() failed:"; + } + EXIT_ON_ERROR(hr); + + if (_ptrAudioBuffer) { + // Update the audio buffer with the selected parameters + _ptrAudioBuffer->SetRecordingSampleRate(_recSampleRate); + _ptrAudioBuffer->SetRecordingChannels((uint8_t)_recChannels); + } else { + // We can enter this state during CoreAudioIsSupported() when no + // AudioDeviceImplementation has been created, hence the AudioDeviceBuffer + // does not exist. It is OK to end up here since we don't initiate any media + // in CoreAudioIsSupported(). + RTC_LOG(LS_VERBOSE) + << "AudioDeviceBuffer must be attached before streaming can start"; + } + + // Get the actual size of the shared (endpoint buffer). + // Typical value is 960 audio frames <=> 20ms @ 48kHz sample rate. + UINT bufferFrameCount(0); + hr = _ptrClientIn->GetBufferSize(&bufferFrameCount); + if (SUCCEEDED(hr)) { + RTC_LOG(LS_VERBOSE) << "IAudioClient::GetBufferSize() => " + << bufferFrameCount << " (<=> " + << bufferFrameCount * _recAudioFrameSize << " bytes)"; + } + + // Set the event handle that the system signals when an audio buffer is ready + // to be processed by the client. + hr = _ptrClientIn->SetEventHandle(_hCaptureSamplesReadyEvent); + EXIT_ON_ERROR(hr); + + // Get an IAudioCaptureClient interface. + SAFE_RELEASE(_ptrCaptureClient); + hr = _ptrClientIn->GetService(__uuidof(IAudioCaptureClient), + (void**)&_ptrCaptureClient); + EXIT_ON_ERROR(hr); + + // Mark capture side as initialized + _recIsInitialized = true; + + CoTaskMemFree(pWfxIn); + CoTaskMemFree(pWfxClosestMatch); + + RTC_LOG(LS_VERBOSE) << "capture side is now initialized"; + return 0; + +Exit: + _TraceCOMError(hr); + CoTaskMemFree(pWfxIn); + CoTaskMemFree(pWfxClosestMatch); + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrCaptureClient); + return -1; +} + +// ---------------------------------------------------------------------------- +// StartRecording +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StartRecording() { + if (!_recIsInitialized) { + return -1; + } + + if (_hRecThread != NULL) { + return 0; + } + + if (_recording) { + return 0; + } + + { + rtc::CritScope critScoped(&_critSect); + + // Create thread which will drive the capturing + LPTHREAD_START_ROUTINE lpStartAddress = WSAPICaptureThread; + if (_builtInAecEnabled) { + // Redirect to the DMO polling method. + lpStartAddress = WSAPICaptureThreadPollDMO; + + if (!_playing) { + // The DMO won't provide us captured output data unless we + // give it render data to process. + RTC_LOG(LS_ERROR) + << "Playout must be started before recording when using" + << " the built-in AEC"; + return -1; + } + } + + assert(_hRecThread == NULL); + _hRecThread = CreateThread(NULL, 0, lpStartAddress, this, 0, NULL); + if (_hRecThread == NULL) { + RTC_LOG(LS_ERROR) << "failed to create the recording thread"; + return -1; + } + + // Set thread priority to highest possible + SetThreadPriority(_hRecThread, THREAD_PRIORITY_TIME_CRITICAL); + + assert(_hGetCaptureVolumeThread == NULL); + _hGetCaptureVolumeThread = + CreateThread(NULL, 0, GetCaptureVolumeThread, this, 0, NULL); + if (_hGetCaptureVolumeThread == NULL) { + RTC_LOG(LS_ERROR) << "failed to create the volume getter thread"; + return -1; + } + + assert(_hSetCaptureVolumeThread == NULL); + _hSetCaptureVolumeThread = + CreateThread(NULL, 0, SetCaptureVolumeThread, this, 0, NULL); + if (_hSetCaptureVolumeThread == NULL) { + RTC_LOG(LS_ERROR) << "failed to create the volume setter thread"; + return -1; + } + } // critScoped + + DWORD ret = WaitForSingleObject(_hCaptureStartedEvent, 1000); + if (ret != WAIT_OBJECT_0) { + RTC_LOG(LS_VERBOSE) << "capturing did not start up properly"; + return -1; + } + RTC_LOG(LS_VERBOSE) << "capture audio stream has now started..."; + + _recording = true; + + return 0; +} + +// ---------------------------------------------------------------------------- +// StopRecording +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StopRecording() { + int32_t err = 0; + + if (!_recIsInitialized) { + return 0; + } + + _Lock(); + + if (_hRecThread == NULL) { + RTC_LOG(LS_VERBOSE) + << "no capturing stream is active => close down WASAPI only"; + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrCaptureClient); + _recIsInitialized = false; + _recording = false; + _UnLock(); + return 0; + } + + // Stop the driving thread... + RTC_LOG(LS_VERBOSE) << "closing down the webrtc_core_audio_capture_thread..."; + // Manual-reset event; it will remain signalled to stop all capture threads. + SetEvent(_hShutdownCaptureEvent); + + _UnLock(); + DWORD ret = WaitForSingleObject(_hRecThread, 2000); + if (ret != WAIT_OBJECT_0) { + RTC_LOG(LS_ERROR) + << "failed to close down webrtc_core_audio_capture_thread"; + err = -1; + } else { + RTC_LOG(LS_VERBOSE) << "webrtc_core_audio_capture_thread is now closed"; + } + + ret = WaitForSingleObject(_hGetCaptureVolumeThread, 2000); + if (ret != WAIT_OBJECT_0) { + // the thread did not stop as it should + RTC_LOG(LS_ERROR) << "failed to close down volume getter thread"; + err = -1; + } else { + RTC_LOG(LS_VERBOSE) << "volume getter thread is now closed"; + } + + ret = WaitForSingleObject(_hSetCaptureVolumeThread, 2000); + if (ret != WAIT_OBJECT_0) { + // the thread did not stop as it should + RTC_LOG(LS_ERROR) << "failed to close down volume setter thread"; + err = -1; + } else { + RTC_LOG(LS_VERBOSE) << "volume setter thread is now closed"; + } + _Lock(); + + ResetEvent(_hShutdownCaptureEvent); // Must be manually reset. + // Ensure that the thread has released these interfaces properly. + assert(err == -1 || _ptrClientIn == NULL); + assert(err == -1 || _ptrCaptureClient == NULL); + + _recIsInitialized = false; + _recording = false; + + // These will create thread leaks in the result of an error, + // but we can at least resume the call. + CloseHandle(_hRecThread); + _hRecThread = NULL; + + CloseHandle(_hGetCaptureVolumeThread); + _hGetCaptureVolumeThread = NULL; + + CloseHandle(_hSetCaptureVolumeThread); + _hSetCaptureVolumeThread = NULL; + + if (_builtInAecEnabled) { + assert(_dmo != NULL); + // This is necessary. Otherwise the DMO can generate garbage render + // audio even after rendering has stopped. + HRESULT hr = _dmo->FreeStreamingResources(); + if (FAILED(hr)) { + _TraceCOMError(hr); + err = -1; + } + } + + // Reset the recording delay value. + _sndCardRecDelay = 0; + + _UnLock(); + + return err; +} + +// ---------------------------------------------------------------------------- +// RecordingIsInitialized +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::RecordingIsInitialized() const { + return (_recIsInitialized); +} + +// ---------------------------------------------------------------------------- +// Recording +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::Recording() const { + return (_recording); +} + +// ---------------------------------------------------------------------------- +// PlayoutIsInitialized +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::PlayoutIsInitialized() const { + return (_playIsInitialized); +} + +// ---------------------------------------------------------------------------- +// StartPlayout +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StartPlayout() { + if (!_playIsInitialized) { + return -1; + } + + if (_hPlayThread != NULL) { + return 0; + } + + if (_playing) { + return 0; + } + + { + rtc::CritScope critScoped(&_critSect); + + // Create thread which will drive the rendering. + assert(_hPlayThread == NULL); + _hPlayThread = CreateThread(NULL, 0, WSAPIRenderThread, this, 0, NULL); + if (_hPlayThread == NULL) { + RTC_LOG(LS_ERROR) << "failed to create the playout thread"; + return -1; + } + + // Set thread priority to highest possible. + SetThreadPriority(_hPlayThread, THREAD_PRIORITY_TIME_CRITICAL); + } // critScoped + + DWORD ret = WaitForSingleObject(_hRenderStartedEvent, 1000); + if (ret != WAIT_OBJECT_0) { + RTC_LOG(LS_VERBOSE) << "rendering did not start up properly"; + return -1; + } + + _playing = true; + RTC_LOG(LS_VERBOSE) << "rendering audio stream has now started..."; + + return 0; +} + +// ---------------------------------------------------------------------------- +// StopPlayout +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::StopPlayout() { + if (!_playIsInitialized) { + return 0; + } + + { + rtc::CritScope critScoped(&_critSect); + + if (_hPlayThread == NULL) { + RTC_LOG(LS_VERBOSE) + << "no rendering stream is active => close down WASAPI only"; + SAFE_RELEASE(_ptrClientOut); + SAFE_RELEASE(_ptrRenderClient); + _playIsInitialized = false; + _playing = false; + return 0; + } + + // stop the driving thread... + RTC_LOG(LS_VERBOSE) + << "closing down the webrtc_core_audio_render_thread..."; + SetEvent(_hShutdownRenderEvent); + } // critScoped + + DWORD ret = WaitForSingleObject(_hPlayThread, 2000); + if (ret != WAIT_OBJECT_0) { + // the thread did not stop as it should + RTC_LOG(LS_ERROR) << "failed to close down webrtc_core_audio_render_thread"; + CloseHandle(_hPlayThread); + _hPlayThread = NULL; + _playIsInitialized = false; + _playing = false; + return -1; + } + + { + rtc::CritScope critScoped(&_critSect); + RTC_LOG(LS_VERBOSE) << "webrtc_core_audio_render_thread is now closed"; + + // to reset this event manually at each time we finish with it, + // in case that the render thread has exited before StopPlayout(), + // this event might be caught by the new render thread within same VoE + // instance. + ResetEvent(_hShutdownRenderEvent); + + SAFE_RELEASE(_ptrClientOut); + SAFE_RELEASE(_ptrRenderClient); + + _playIsInitialized = false; + _playing = false; + + CloseHandle(_hPlayThread); + _hPlayThread = NULL; + + if (_builtInAecEnabled && _recording) { + // The DMO won't provide us captured output data unless we + // give it render data to process. + // + // We still permit the playout to shutdown, and trace a warning. + // Otherwise, VoE can get into a state which will never permit + // playout to stop properly. + RTC_LOG(LS_WARNING) + << "Recording should be stopped before playout when using the" + << " built-in AEC"; + } + + // Reset the playout delay value. + _sndCardPlayDelay = 0; + } // critScoped + + return 0; +} + +// ---------------------------------------------------------------------------- +// PlayoutDelay +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::PlayoutDelay(uint16_t& delayMS) const { + rtc::CritScope critScoped(&_critSect); + delayMS = static_cast(_sndCardPlayDelay); + return 0; +} + +// ---------------------------------------------------------------------------- +// Playing +// ---------------------------------------------------------------------------- + +bool AudioDeviceWindowsCore::Playing() const { + return (_playing); +} + +// ============================================================================ +// Private Methods +// ============================================================================ + +// ---------------------------------------------------------------------------- +// [static] WSAPIRenderThread +// ---------------------------------------------------------------------------- + +DWORD WINAPI AudioDeviceWindowsCore::WSAPIRenderThread(LPVOID context) { + return reinterpret_cast(context)->DoRenderThread(); +} + +// ---------------------------------------------------------------------------- +// [static] WSAPICaptureThread +// ---------------------------------------------------------------------------- + +DWORD WINAPI AudioDeviceWindowsCore::WSAPICaptureThread(LPVOID context) { + return reinterpret_cast(context)->DoCaptureThread(); +} + +DWORD WINAPI AudioDeviceWindowsCore::WSAPICaptureThreadPollDMO(LPVOID context) { + return reinterpret_cast(context) + ->DoCaptureThreadPollDMO(); +} + +DWORD WINAPI AudioDeviceWindowsCore::GetCaptureVolumeThread(LPVOID context) { + return reinterpret_cast(context) + ->DoGetCaptureVolumeThread(); +} + +DWORD WINAPI AudioDeviceWindowsCore::SetCaptureVolumeThread(LPVOID context) { + return reinterpret_cast(context) + ->DoSetCaptureVolumeThread(); +} + +DWORD AudioDeviceWindowsCore::DoGetCaptureVolumeThread() { + HANDLE waitObject = _hShutdownCaptureEvent; + + while (1) { + if (AGC()) { + uint32_t currentMicLevel = 0; + if (MicrophoneVolume(currentMicLevel) == 0) { + // This doesn't set the system volume, just stores it. + _Lock(); + if (_ptrAudioBuffer) { + _ptrAudioBuffer->SetCurrentMicLevel(currentMicLevel); + } + _UnLock(); + } + } + + DWORD waitResult = + WaitForSingleObject(waitObject, GET_MIC_VOLUME_INTERVAL_MS); + switch (waitResult) { + case WAIT_OBJECT_0: // _hShutdownCaptureEvent + return 0; + case WAIT_TIMEOUT: // timeout notification + break; + default: // unexpected error + RTC_LOG(LS_WARNING) << "unknown wait termination on get volume thread"; + return 1; + } + } +} + +DWORD AudioDeviceWindowsCore::DoSetCaptureVolumeThread() { + HANDLE waitArray[2] = {_hShutdownCaptureEvent, _hSetCaptureVolumeEvent}; + + while (1) { + DWORD waitResult = WaitForMultipleObjects(2, waitArray, FALSE, INFINITE); + switch (waitResult) { + case WAIT_OBJECT_0: // _hShutdownCaptureEvent + return 0; + case WAIT_OBJECT_0 + 1: // _hSetCaptureVolumeEvent + break; + default: // unexpected error + RTC_LOG(LS_WARNING) << "unknown wait termination on set volume thread"; + return 1; + } + + _Lock(); + uint32_t newMicLevel = _newMicLevel; + _UnLock(); + + if (SetMicrophoneVolume(newMicLevel) == -1) { + RTC_LOG(LS_WARNING) + << "the required modification of the microphone volume failed"; + } + } +} + +// ---------------------------------------------------------------------------- +// DoRenderThread +// ---------------------------------------------------------------------------- + +DWORD AudioDeviceWindowsCore::DoRenderThread() { + bool keepPlaying = true; + HANDLE waitArray[2] = {_hShutdownRenderEvent, _hRenderSamplesReadyEvent}; + HRESULT hr = S_OK; + HANDLE hMmTask = NULL; + + // Initialize COM as MTA in this thread. + ScopedCOMInitializer comInit(ScopedCOMInitializer::kMTA); + if (!comInit.succeeded()) { + RTC_LOG(LS_ERROR) << "failed to initialize COM in render thread"; + return 1; + } + + rtc::SetCurrentThreadName("webrtc_core_audio_render_thread"); + + // Use Multimedia Class Scheduler Service (MMCSS) to boost the thread + // priority. + // + if (_winSupportAvrt) { + DWORD taskIndex(0); + hMmTask = _PAvSetMmThreadCharacteristicsA("Pro Audio", &taskIndex); + if (hMmTask) { + if (FALSE == _PAvSetMmThreadPriority(hMmTask, AVRT_PRIORITY_CRITICAL)) { + RTC_LOG(LS_WARNING) << "failed to boost play-thread using MMCSS"; + } + RTC_LOG(LS_VERBOSE) + << "render thread is now registered with MMCSS (taskIndex=" + << taskIndex << ")"; + } else { + RTC_LOG(LS_WARNING) << "failed to enable MMCSS on render thread (err=" + << GetLastError() << ")"; + _TraceCOMError(GetLastError()); + } + } + + _Lock(); + + IAudioClock* clock = NULL; + + // Get size of rendering buffer (length is expressed as the number of audio + // frames the buffer can hold). This value is fixed during the rendering + // session. + // + UINT32 bufferLength = 0; + hr = _ptrClientOut->GetBufferSize(&bufferLength); + EXIT_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "[REND] size of buffer : " << bufferLength; + + // Get maximum latency for the current stream (will not change for the + // lifetime of the IAudioClient object). + // + REFERENCE_TIME latency; + _ptrClientOut->GetStreamLatency(&latency); + RTC_LOG(LS_VERBOSE) << "[REND] max stream latency : " << (DWORD)latency + << " (" << (double)(latency / 10000.0) << " ms)"; + + // Get the length of the periodic interval separating successive processing + // passes by the audio engine on the data in the endpoint buffer. + // + // The period between processing passes by the audio engine is fixed for a + // particular audio endpoint device and represents the smallest processing + // quantum for the audio engine. This period plus the stream latency between + // the buffer and endpoint device represents the minimum possible latency that + // an audio application can achieve. Typical value: 100000 <=> 0.01 sec = + // 10ms. + // + REFERENCE_TIME devPeriod = 0; + REFERENCE_TIME devPeriodMin = 0; + _ptrClientOut->GetDevicePeriod(&devPeriod, &devPeriodMin); + RTC_LOG(LS_VERBOSE) << "[REND] device period : " << (DWORD)devPeriod + << " (" << (double)(devPeriod / 10000.0) << " ms)"; + + // Derive initial rendering delay. + // Example: 10*(960/480) + 15 = 20 + 15 = 35ms + // + int playout_delay = 10 * (bufferLength / _playBlockSize) + + (int)((latency + devPeriod) / 10000); + _sndCardPlayDelay = playout_delay; + _writtenSamples = 0; + RTC_LOG(LS_VERBOSE) << "[REND] initial delay : " << playout_delay; + + double endpointBufferSizeMS = + 10.0 * ((double)bufferLength / (double)_devicePlayBlockSize); + RTC_LOG(LS_VERBOSE) << "[REND] endpointBufferSizeMS : " + << endpointBufferSizeMS; + + // Before starting the stream, fill the rendering buffer with silence. + // + BYTE* pData = NULL; + hr = _ptrRenderClient->GetBuffer(bufferLength, &pData); + EXIT_ON_ERROR(hr); + + hr = + _ptrRenderClient->ReleaseBuffer(bufferLength, AUDCLNT_BUFFERFLAGS_SILENT); + EXIT_ON_ERROR(hr); + + _writtenSamples += bufferLength; + + hr = _ptrClientOut->GetService(__uuidof(IAudioClock), (void**)&clock); + if (FAILED(hr)) { + RTC_LOG(LS_WARNING) + << "failed to get IAudioClock interface from the IAudioClient"; + } + + // Start up the rendering audio stream. + hr = _ptrClientOut->Start(); + EXIT_ON_ERROR(hr); + + _UnLock(); + + // Set event which will ensure that the calling thread modifies the playing + // state to true. + // + SetEvent(_hRenderStartedEvent); + + // >> ------------------ THREAD LOOP ------------------ + + while (keepPlaying) { + // Wait for a render notification event or a shutdown event + DWORD waitResult = WaitForMultipleObjects(2, waitArray, FALSE, 500); + switch (waitResult) { + case WAIT_OBJECT_0 + 0: // _hShutdownRenderEvent + keepPlaying = false; + break; + case WAIT_OBJECT_0 + 1: // _hRenderSamplesReadyEvent + break; + case WAIT_TIMEOUT: // timeout notification + RTC_LOG(LS_WARNING) << "render event timed out after 0.5 seconds"; + goto Exit; + default: // unexpected error + RTC_LOG(LS_WARNING) << "unknown wait termination on render side"; + goto Exit; + } + + while (keepPlaying) { + _Lock(); + + // Sanity check to ensure that essential states are not modified + // during the unlocked period. + if (_ptrRenderClient == NULL || _ptrClientOut == NULL) { + _UnLock(); + RTC_LOG(LS_ERROR) + << "output state has been modified during unlocked period"; + goto Exit; + } + + // Get the number of frames of padding (queued up to play) in the endpoint + // buffer. + UINT32 padding = 0; + hr = _ptrClientOut->GetCurrentPadding(&padding); + EXIT_ON_ERROR(hr); + + // Derive the amount of available space in the output buffer + uint32_t framesAvailable = bufferLength - padding; + + // Do we have 10 ms available in the render buffer? + if (framesAvailable < _playBlockSize) { + // Not enough space in render buffer to store next render packet. + _UnLock(); + break; + } + + // Write n*10ms buffers to the render buffer + const uint32_t n10msBuffers = (framesAvailable / _playBlockSize); + for (uint32_t n = 0; n < n10msBuffers; n++) { + // Get pointer (i.e., grab the buffer) to next space in the shared + // render buffer. + hr = _ptrRenderClient->GetBuffer(_playBlockSize, &pData); + EXIT_ON_ERROR(hr); + + if (_ptrAudioBuffer) { + // Request data to be played out (#bytes = + // _playBlockSize*_audioFrameSize) + _UnLock(); + int32_t nSamples = + _ptrAudioBuffer->RequestPlayoutData(_playBlockSize); + _Lock(); + + if (nSamples == -1) { + _UnLock(); + RTC_LOG(LS_ERROR) << "failed to read data from render client"; + goto Exit; + } + + // Sanity check to ensure that essential states are not modified + // during the unlocked period + if (_ptrRenderClient == NULL || _ptrClientOut == NULL) { + _UnLock(); + RTC_LOG(LS_ERROR) + << "output state has been modified during unlocked" + << " period"; + goto Exit; + } + if (nSamples != static_cast(_playBlockSize)) { + RTC_LOG(LS_WARNING) + << "nSamples(" << nSamples << ") != _playBlockSize" + << _playBlockSize << ")"; + } + + // Get the actual (stored) data + nSamples = _ptrAudioBuffer->GetPlayoutData((int8_t*)pData); + } + + DWORD dwFlags(0); + hr = _ptrRenderClient->ReleaseBuffer(_playBlockSize, dwFlags); + // See http://msdn.microsoft.com/en-us/library/dd316605(VS.85).aspx + // for more details regarding AUDCLNT_E_DEVICE_INVALIDATED. + EXIT_ON_ERROR(hr); + + _writtenSamples += _playBlockSize; + } + + // Check the current delay on the playout side. + if (clock) { + UINT64 pos = 0; + UINT64 freq = 1; + clock->GetPosition(&pos, NULL); + clock->GetFrequency(&freq); + playout_delay = ROUND((double(_writtenSamples) / _devicePlaySampleRate - + double(pos) / freq) * + 1000.0); + _sndCardPlayDelay = playout_delay; + } + + _UnLock(); + } + } + + // ------------------ THREAD LOOP ------------------ << + + SleepMs(static_cast(endpointBufferSizeMS + 0.5)); + hr = _ptrClientOut->Stop(); + +Exit: + SAFE_RELEASE(clock); + + if (FAILED(hr)) { + _ptrClientOut->Stop(); + _UnLock(); + _TraceCOMError(hr); + } + + if (_winSupportAvrt) { + if (NULL != hMmTask) { + _PAvRevertMmThreadCharacteristics(hMmTask); + } + } + + _Lock(); + + if (keepPlaying) { + if (_ptrClientOut != NULL) { + hr = _ptrClientOut->Stop(); + if (FAILED(hr)) { + _TraceCOMError(hr); + } + hr = _ptrClientOut->Reset(); + if (FAILED(hr)) { + _TraceCOMError(hr); + } + } + RTC_LOG(LS_ERROR) + << "Playout error: rendering thread has ended pre-maturely"; + } else { + RTC_LOG(LS_VERBOSE) << "_Rendering thread is now terminated properly"; + } + + _UnLock(); + + return (DWORD)hr; +} + +DWORD AudioDeviceWindowsCore::InitCaptureThreadPriority() { + _hMmTask = NULL; + + rtc::SetCurrentThreadName("webrtc_core_audio_capture_thread"); + + // Use Multimedia Class Scheduler Service (MMCSS) to boost the thread + // priority. + if (_winSupportAvrt) { + DWORD taskIndex(0); + _hMmTask = _PAvSetMmThreadCharacteristicsA("Pro Audio", &taskIndex); + if (_hMmTask) { + if (!_PAvSetMmThreadPriority(_hMmTask, AVRT_PRIORITY_CRITICAL)) { + RTC_LOG(LS_WARNING) << "failed to boost rec-thread using MMCSS"; + } + RTC_LOG(LS_VERBOSE) + << "capture thread is now registered with MMCSS (taskIndex=" + << taskIndex << ")"; + } else { + RTC_LOG(LS_WARNING) << "failed to enable MMCSS on capture thread (err=" + << GetLastError() << ")"; + _TraceCOMError(GetLastError()); + } + } + + return S_OK; +} + +void AudioDeviceWindowsCore::RevertCaptureThreadPriority() { + if (_winSupportAvrt) { + if (NULL != _hMmTask) { + _PAvRevertMmThreadCharacteristics(_hMmTask); + } + } + + _hMmTask = NULL; +} + +DWORD AudioDeviceWindowsCore::DoCaptureThreadPollDMO() { + assert(_mediaBuffer != NULL); + bool keepRecording = true; + + // Initialize COM as MTA in this thread. + ScopedCOMInitializer comInit(ScopedCOMInitializer::kMTA); + if (!comInit.succeeded()) { + RTC_LOG(LS_ERROR) << "failed to initialize COM in polling DMO thread"; + return 1; + } + + HRESULT hr = InitCaptureThreadPriority(); + if (FAILED(hr)) { + return hr; + } + + // Set event which will ensure that the calling thread modifies the + // recording state to true. + SetEvent(_hCaptureStartedEvent); + + // >> ---------------------------- THREAD LOOP ---------------------------- + while (keepRecording) { + // Poll the DMO every 5 ms. + // (The same interval used in the Wave implementation.) + DWORD waitResult = WaitForSingleObject(_hShutdownCaptureEvent, 5); + switch (waitResult) { + case WAIT_OBJECT_0: // _hShutdownCaptureEvent + keepRecording = false; + break; + case WAIT_TIMEOUT: // timeout notification + break; + default: // unexpected error + RTC_LOG(LS_WARNING) << "Unknown wait termination on capture side"; + hr = -1; // To signal an error callback. + keepRecording = false; + break; + } + + while (keepRecording) { + rtc::CritScope critScoped(&_critSect); + + DWORD dwStatus = 0; + { + DMO_OUTPUT_DATA_BUFFER dmoBuffer = {0}; + dmoBuffer.pBuffer = _mediaBuffer; + dmoBuffer.pBuffer->AddRef(); + + // Poll the DMO for AEC processed capture data. The DMO will + // copy available data to |dmoBuffer|, and should only return + // 10 ms frames. The value of |dwStatus| should be ignored. + hr = _dmo->ProcessOutput(0, 1, &dmoBuffer, &dwStatus); + SAFE_RELEASE(dmoBuffer.pBuffer); + dwStatus = dmoBuffer.dwStatus; + } + if (FAILED(hr)) { + _TraceCOMError(hr); + keepRecording = false; + assert(false); + break; + } + + ULONG bytesProduced = 0; + BYTE* data; + // Get a pointer to the data buffer. This should be valid until + // the next call to ProcessOutput. + hr = _mediaBuffer->GetBufferAndLength(&data, &bytesProduced); + if (FAILED(hr)) { + _TraceCOMError(hr); + keepRecording = false; + assert(false); + break; + } + + // TODO(andrew): handle AGC. + + if (bytesProduced > 0) { + const int kSamplesProduced = bytesProduced / _recAudioFrameSize; + // TODO(andrew): verify that this is always satisfied. It might + // be that ProcessOutput will try to return more than 10 ms if + // we fail to call it frequently enough. + assert(kSamplesProduced == static_cast(_recBlockSize)); + assert(sizeof(BYTE) == sizeof(int8_t)); + _ptrAudioBuffer->SetRecordedBuffer(reinterpret_cast(data), + kSamplesProduced); + _ptrAudioBuffer->SetVQEData(0, 0, 0); + + _UnLock(); // Release lock while making the callback. + _ptrAudioBuffer->DeliverRecordedData(); + _Lock(); + } + + // Reset length to indicate buffer availability. + hr = _mediaBuffer->SetLength(0); + if (FAILED(hr)) { + _TraceCOMError(hr); + keepRecording = false; + assert(false); + break; + } + + if (!(dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)) { + // The DMO cannot currently produce more data. This is the + // normal case; otherwise it means the DMO had more than 10 ms + // of data available and ProcessOutput should be called again. + break; + } + } + } + // ---------------------------- THREAD LOOP ---------------------------- << + + RevertCaptureThreadPriority(); + + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) + << "Recording error: capturing thread has ended prematurely"; + } else { + RTC_LOG(LS_VERBOSE) << "Capturing thread is now terminated properly"; + } + + return hr; +} + +// ---------------------------------------------------------------------------- +// DoCaptureThread +// ---------------------------------------------------------------------------- + +DWORD AudioDeviceWindowsCore::DoCaptureThread() { + bool keepRecording = true; + HANDLE waitArray[2] = {_hShutdownCaptureEvent, _hCaptureSamplesReadyEvent}; + HRESULT hr = S_OK; + + LARGE_INTEGER t1; + + BYTE* syncBuffer = NULL; + UINT32 syncBufIndex = 0; + + _readSamples = 0; + + // Initialize COM as MTA in this thread. + ScopedCOMInitializer comInit(ScopedCOMInitializer::kMTA); + if (!comInit.succeeded()) { + RTC_LOG(LS_ERROR) << "failed to initialize COM in capture thread"; + return 1; + } + + hr = InitCaptureThreadPriority(); + if (FAILED(hr)) { + return hr; + } + + _Lock(); + + // Get size of capturing buffer (length is expressed as the number of audio + // frames the buffer can hold). This value is fixed during the capturing + // session. + // + UINT32 bufferLength = 0; + if (_ptrClientIn == NULL) { + RTC_LOG(LS_ERROR) + << "input state has been modified before capture loop starts."; + return 1; + } + hr = _ptrClientIn->GetBufferSize(&bufferLength); + EXIT_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "[CAPT] size of buffer : " << bufferLength; + + // Allocate memory for sync buffer. + // It is used for compensation between native 44.1 and internal 44.0 and + // for cases when the capture buffer is larger than 10ms. + // + const UINT32 syncBufferSize = 2 * (bufferLength * _recAudioFrameSize); + syncBuffer = new BYTE[syncBufferSize]; + if (syncBuffer == NULL) { + return (DWORD)E_POINTER; + } + RTC_LOG(LS_VERBOSE) << "[CAPT] size of sync buffer : " << syncBufferSize + << " [bytes]"; + + // Get maximum latency for the current stream (will not change for the + // lifetime of the IAudioClient object). + // + REFERENCE_TIME latency; + _ptrClientIn->GetStreamLatency(&latency); + RTC_LOG(LS_VERBOSE) << "[CAPT] max stream latency : " << (DWORD)latency + << " (" << (double)(latency / 10000.0) << " ms)"; + + // Get the length of the periodic interval separating successive processing + // passes by the audio engine on the data in the endpoint buffer. + // + REFERENCE_TIME devPeriod = 0; + REFERENCE_TIME devPeriodMin = 0; + _ptrClientIn->GetDevicePeriod(&devPeriod, &devPeriodMin); + RTC_LOG(LS_VERBOSE) << "[CAPT] device period : " << (DWORD)devPeriod + << " (" << (double)(devPeriod / 10000.0) << " ms)"; + + double extraDelayMS = (double)((latency + devPeriod) / 10000.0); + RTC_LOG(LS_VERBOSE) << "[CAPT] extraDelayMS : " << extraDelayMS; + + double endpointBufferSizeMS = + 10.0 * ((double)bufferLength / (double)_recBlockSize); + RTC_LOG(LS_VERBOSE) << "[CAPT] endpointBufferSizeMS : " + << endpointBufferSizeMS; + + // Start up the capturing stream. + // + hr = _ptrClientIn->Start(); + EXIT_ON_ERROR(hr); + + _UnLock(); + + // Set event which will ensure that the calling thread modifies the recording + // state to true. + // + SetEvent(_hCaptureStartedEvent); + + // >> ---------------------------- THREAD LOOP ---------------------------- + + while (keepRecording) { + // Wait for a capture notification event or a shutdown event + DWORD waitResult = WaitForMultipleObjects(2, waitArray, FALSE, 500); + switch (waitResult) { + case WAIT_OBJECT_0 + 0: // _hShutdownCaptureEvent + keepRecording = false; + break; + case WAIT_OBJECT_0 + 1: // _hCaptureSamplesReadyEvent + break; + case WAIT_TIMEOUT: // timeout notification + RTC_LOG(LS_WARNING) << "capture event timed out after 0.5 seconds"; + goto Exit; + default: // unexpected error + RTC_LOG(LS_WARNING) << "unknown wait termination on capture side"; + goto Exit; + } + + while (keepRecording) { + BYTE* pData = 0; + UINT32 framesAvailable = 0; + DWORD flags = 0; + UINT64 recTime = 0; + UINT64 recPos = 0; + + _Lock(); + + // Sanity check to ensure that essential states are not modified + // during the unlocked period. + if (_ptrCaptureClient == NULL || _ptrClientIn == NULL) { + _UnLock(); + RTC_LOG(LS_ERROR) + << "input state has been modified during unlocked period"; + goto Exit; + } + + // Find out how much capture data is available + // + hr = _ptrCaptureClient->GetBuffer( + &pData, // packet which is ready to be read by used + &framesAvailable, // #frames in the captured packet (can be zero) + &flags, // support flags (check) + &recPos, // device position of first audio frame in data packet + &recTime); // value of performance counter at the time of recording + // the first audio frame + + if (SUCCEEDED(hr)) { + if (AUDCLNT_S_BUFFER_EMPTY == hr) { + // Buffer was empty => start waiting for a new capture notification + // event + _UnLock(); + break; + } + + if (flags & AUDCLNT_BUFFERFLAGS_SILENT) { + // Treat all of the data in the packet as silence and ignore the + // actual data values. + RTC_LOG(LS_WARNING) << "AUDCLNT_BUFFERFLAGS_SILENT"; + pData = NULL; + } + + assert(framesAvailable != 0); + + if (pData) { + CopyMemory(&syncBuffer[syncBufIndex * _recAudioFrameSize], pData, + framesAvailable * _recAudioFrameSize); + } else { + ZeroMemory(&syncBuffer[syncBufIndex * _recAudioFrameSize], + framesAvailable * _recAudioFrameSize); + } + assert(syncBufferSize >= (syncBufIndex * _recAudioFrameSize) + + framesAvailable * _recAudioFrameSize); + + // Release the capture buffer + // + hr = _ptrCaptureClient->ReleaseBuffer(framesAvailable); + EXIT_ON_ERROR(hr); + + _readSamples += framesAvailable; + syncBufIndex += framesAvailable; + + QueryPerformanceCounter(&t1); + + // Get the current recording and playout delay. + uint32_t sndCardRecDelay = (uint32_t)( + ((((UINT64)t1.QuadPart * _perfCounterFactor) - recTime) / 10000) + + (10 * syncBufIndex) / _recBlockSize - 10); + uint32_t sndCardPlayDelay = static_cast(_sndCardPlayDelay); + + _sndCardRecDelay = sndCardRecDelay; + + while (syncBufIndex >= _recBlockSize) { + if (_ptrAudioBuffer) { + _ptrAudioBuffer->SetRecordedBuffer((const int8_t*)syncBuffer, + _recBlockSize); + _ptrAudioBuffer->SetVQEData(sndCardPlayDelay, sndCardRecDelay, 0); + + _ptrAudioBuffer->SetTypingStatus(KeyPressed()); + + _UnLock(); // release lock while making the callback + _ptrAudioBuffer->DeliverRecordedData(); + _Lock(); // restore the lock + + // Sanity check to ensure that essential states are not modified + // during the unlocked period + if (_ptrCaptureClient == NULL || _ptrClientIn == NULL) { + _UnLock(); + RTC_LOG(LS_ERROR) << "input state has been modified during" + << " unlocked period"; + goto Exit; + } + } + + // store remaining data which was not able to deliver as 10ms segment + MoveMemory(&syncBuffer[0], + &syncBuffer[_recBlockSize * _recAudioFrameSize], + (syncBufIndex - _recBlockSize) * _recAudioFrameSize); + syncBufIndex -= _recBlockSize; + sndCardRecDelay -= 10; + } + + if (_AGC) { + uint32_t newMicLevel = _ptrAudioBuffer->NewMicLevel(); + if (newMicLevel != 0) { + // The VQE will only deliver non-zero microphone levels when a + // change is needed. Set this new mic level (received from the + // observer as return value in the callback). + RTC_LOG(LS_VERBOSE) << "AGC change of volume: new=" << newMicLevel; + // We store this outside of the audio buffer to avoid + // having it overwritten by the getter thread. + _newMicLevel = newMicLevel; + SetEvent(_hSetCaptureVolumeEvent); + } + } + } else { + // If GetBuffer returns AUDCLNT_E_BUFFER_ERROR, the thread consuming the + // audio samples must wait for the next processing pass. The client + // might benefit from keeping a count of the failed GetBuffer calls. If + // GetBuffer returns this error repeatedly, the client can start a new + // processing loop after shutting down the current client by calling + // IAudioClient::Stop, IAudioClient::Reset, and releasing the audio + // client. + RTC_LOG(LS_ERROR) << "IAudioCaptureClient::GetBuffer returned" + << " AUDCLNT_E_BUFFER_ERROR, hr = 0x" << std::hex + << hr << std::dec; + goto Exit; + } + + _UnLock(); + } + } + + // ---------------------------- THREAD LOOP ---------------------------- << + + if (_ptrClientIn) { + hr = _ptrClientIn->Stop(); + } + +Exit: + if (FAILED(hr)) { + _ptrClientIn->Stop(); + _UnLock(); + _TraceCOMError(hr); + } + + RevertCaptureThreadPriority(); + + _Lock(); + + if (keepRecording) { + if (_ptrClientIn != NULL) { + hr = _ptrClientIn->Stop(); + if (FAILED(hr)) { + _TraceCOMError(hr); + } + hr = _ptrClientIn->Reset(); + if (FAILED(hr)) { + _TraceCOMError(hr); + } + } + + RTC_LOG(LS_ERROR) + << "Recording error: capturing thread has ended pre-maturely"; + } else { + RTC_LOG(LS_VERBOSE) << "_Capturing thread is now terminated properly"; + } + + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrCaptureClient); + + _UnLock(); + + if (syncBuffer) { + delete[] syncBuffer; + } + + return (DWORD)hr; +} + +int32_t AudioDeviceWindowsCore::EnableBuiltInAEC(bool enable) { + if (_recIsInitialized) { + RTC_LOG(LS_ERROR) + << "Attempt to set Windows AEC with recording already initialized"; + return -1; + } + + if (_dmo == NULL) { + RTC_LOG(LS_ERROR) + << "Built-in AEC DMO was not initialized properly at create time"; + return -1; + } + + _builtInAecEnabled = enable; + return 0; +} + +int AudioDeviceWindowsCore::SetDMOProperties() { + HRESULT hr = S_OK; + assert(_dmo != NULL); + + rtc::scoped_refptr ps; + { + IPropertyStore* ptrPS = NULL; + hr = _dmo->QueryInterface(IID_IPropertyStore, + reinterpret_cast(&ptrPS)); + if (FAILED(hr) || ptrPS == NULL) { + _TraceCOMError(hr); + return -1; + } + ps = ptrPS; + SAFE_RELEASE(ptrPS); + } + + // Set the AEC system mode. + // SINGLE_CHANNEL_AEC - AEC processing only. + if (SetVtI4Property(ps, MFPKEY_WMAAECMA_SYSTEM_MODE, SINGLE_CHANNEL_AEC)) { + return -1; + } + + // Set the AEC source mode. + // VARIANT_TRUE - Source mode (we poll the AEC for captured data). + if (SetBoolProperty(ps, MFPKEY_WMAAECMA_DMO_SOURCE_MODE, VARIANT_TRUE) == + -1) { + return -1; + } + + // Enable the feature mode. + // This lets us override all the default processing settings below. + if (SetBoolProperty(ps, MFPKEY_WMAAECMA_FEATURE_MODE, VARIANT_TRUE) == -1) { + return -1; + } + + // Disable analog AGC (default enabled). + if (SetBoolProperty(ps, MFPKEY_WMAAECMA_MIC_GAIN_BOUNDER, VARIANT_FALSE) == + -1) { + return -1; + } + + // Disable noise suppression (default enabled). + // 0 - Disabled, 1 - Enabled + if (SetVtI4Property(ps, MFPKEY_WMAAECMA_FEATR_NS, 0) == -1) { + return -1; + } + + // Relevant parameters to leave at default settings: + // MFPKEY_WMAAECMA_FEATR_AGC - Digital AGC (disabled). + // MFPKEY_WMAAECMA_FEATR_CENTER_CLIP - AEC center clipping (enabled). + // MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH - Filter length (256 ms). + // TODO(andrew): investigate decresing the length to 128 ms. + // MFPKEY_WMAAECMA_FEATR_FRAME_SIZE - Frame size (0). + // 0 is automatic; defaults to 160 samples (or 10 ms frames at the + // selected 16 kHz) as long as mic array processing is disabled. + // MFPKEY_WMAAECMA_FEATR_NOISE_FILL - Comfort noise (enabled). + // MFPKEY_WMAAECMA_FEATR_VAD - VAD (disabled). + + // Set the devices selected by VoE. If using a default device, we need to + // search for the device index. + int inDevIndex = _inputDeviceIndex; + int outDevIndex = _outputDeviceIndex; + if (!_usingInputDeviceIndex) { + ERole role = eCommunications; + if (_inputDevice == AudioDeviceModule::kDefaultDevice) { + role = eConsole; + } + + if (_GetDefaultDeviceIndex(eCapture, role, &inDevIndex) == -1) { + return -1; + } + } + + if (!_usingOutputDeviceIndex) { + ERole role = eCommunications; + if (_outputDevice == AudioDeviceModule::kDefaultDevice) { + role = eConsole; + } + + if (_GetDefaultDeviceIndex(eRender, role, &outDevIndex) == -1) { + return -1; + } + } + + DWORD devIndex = static_cast(outDevIndex << 16) + + static_cast(0x0000ffff & inDevIndex); + RTC_LOG(LS_VERBOSE) << "Capture device index: " << inDevIndex + << ", render device index: " << outDevIndex; + if (SetVtI4Property(ps, MFPKEY_WMAAECMA_DEVICE_INDEXES, devIndex) == -1) { + return -1; + } + + return 0; +} + +int AudioDeviceWindowsCore::SetBoolProperty(IPropertyStore* ptrPS, + REFPROPERTYKEY key, + VARIANT_BOOL value) { + PROPVARIANT pv; + PropVariantInit(&pv); + pv.vt = VT_BOOL; + pv.boolVal = value; + HRESULT hr = ptrPS->SetValue(key, pv); + PropVariantClear(&pv); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + return 0; +} + +int AudioDeviceWindowsCore::SetVtI4Property(IPropertyStore* ptrPS, + REFPROPERTYKEY key, + LONG value) { + PROPVARIANT pv; + PropVariantInit(&pv); + pv.vt = VT_I4; + pv.lVal = value; + HRESULT hr = ptrPS->SetValue(key, pv); + PropVariantClear(&pv); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + return 0; +} + +// ---------------------------------------------------------------------------- +// _RefreshDeviceList +// +// Creates a new list of endpoint rendering or capture devices after +// deleting any previously created (and possibly out-of-date) list of +// such devices. +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_RefreshDeviceList(EDataFlow dir) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + IMMDeviceCollection* pCollection = NULL; + + assert(dir == eRender || dir == eCapture); + assert(_ptrEnumerator != NULL); + + // Create a fresh list of devices using the specified direction + hr = _ptrEnumerator->EnumAudioEndpoints(dir, DEVICE_STATE_ACTIVE, + &pCollection); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pCollection); + return -1; + } + + if (dir == eRender) { + SAFE_RELEASE(_ptrRenderCollection); + _ptrRenderCollection = pCollection; + } else { + SAFE_RELEASE(_ptrCaptureCollection); + _ptrCaptureCollection = pCollection; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// _DeviceListCount +// +// Gets a count of the endpoint rendering or capture devices in the +// current list of such devices. +// ---------------------------------------------------------------------------- + +int16_t AudioDeviceWindowsCore::_DeviceListCount(EDataFlow dir) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + UINT count = 0; + + assert(eRender == dir || eCapture == dir); + + if (eRender == dir && NULL != _ptrRenderCollection) { + hr = _ptrRenderCollection->GetCount(&count); + } else if (NULL != _ptrCaptureCollection) { + hr = _ptrCaptureCollection->GetCount(&count); + } + + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + return static_cast(count); +} + +// ---------------------------------------------------------------------------- +// _GetListDeviceName +// +// Gets the friendly name of an endpoint rendering or capture device +// from the current list of such devices. The caller uses an index +// into the list to identify the device. +// +// Uses: _ptrRenderCollection or _ptrCaptureCollection which is updated +// in _RefreshDeviceList(). +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetListDeviceName(EDataFlow dir, + int index, + LPWSTR szBuffer, + int bufferLen) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + IMMDevice* pDevice = NULL; + + assert(dir == eRender || dir == eCapture); + + if (eRender == dir && NULL != _ptrRenderCollection) { + hr = _ptrRenderCollection->Item(index, &pDevice); + } else if (NULL != _ptrCaptureCollection) { + hr = _ptrCaptureCollection->Item(index, &pDevice); + } + + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pDevice); + return -1; + } + + int32_t res = _GetDeviceName(pDevice, szBuffer, bufferLen); + SAFE_RELEASE(pDevice); + return res; +} + +// ---------------------------------------------------------------------------- +// _GetDefaultDeviceName +// +// Gets the friendly name of an endpoint rendering or capture device +// given a specified device role. +// +// Uses: _ptrEnumerator +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetDefaultDeviceName(EDataFlow dir, + ERole role, + LPWSTR szBuffer, + int bufferLen) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + IMMDevice* pDevice = NULL; + + assert(dir == eRender || dir == eCapture); + assert(role == eConsole || role == eCommunications); + assert(_ptrEnumerator != NULL); + + hr = _ptrEnumerator->GetDefaultAudioEndpoint(dir, role, &pDevice); + + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pDevice); + return -1; + } + + int32_t res = _GetDeviceName(pDevice, szBuffer, bufferLen); + SAFE_RELEASE(pDevice); + return res; +} + +// ---------------------------------------------------------------------------- +// _GetListDeviceID +// +// Gets the unique ID string of an endpoint rendering or capture device +// from the current list of such devices. The caller uses an index +// into the list to identify the device. +// +// Uses: _ptrRenderCollection or _ptrCaptureCollection which is updated +// in _RefreshDeviceList(). +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetListDeviceID(EDataFlow dir, + int index, + LPWSTR szBuffer, + int bufferLen) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + IMMDevice* pDevice = NULL; + + assert(dir == eRender || dir == eCapture); + + if (eRender == dir && NULL != _ptrRenderCollection) { + hr = _ptrRenderCollection->Item(index, &pDevice); + } else if (NULL != _ptrCaptureCollection) { + hr = _ptrCaptureCollection->Item(index, &pDevice); + } + + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pDevice); + return -1; + } + + int32_t res = _GetDeviceID(pDevice, szBuffer, bufferLen); + SAFE_RELEASE(pDevice); + return res; +} + +// ---------------------------------------------------------------------------- +// _GetDefaultDeviceID +// +// Gets the uniqe device ID of an endpoint rendering or capture device +// given a specified device role. +// +// Uses: _ptrEnumerator +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetDefaultDeviceID(EDataFlow dir, + ERole role, + LPWSTR szBuffer, + int bufferLen) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + IMMDevice* pDevice = NULL; + + assert(dir == eRender || dir == eCapture); + assert(role == eConsole || role == eCommunications); + assert(_ptrEnumerator != NULL); + + hr = _ptrEnumerator->GetDefaultAudioEndpoint(dir, role, &pDevice); + + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pDevice); + return -1; + } + + int32_t res = _GetDeviceID(pDevice, szBuffer, bufferLen); + SAFE_RELEASE(pDevice); + return res; +} + +int32_t AudioDeviceWindowsCore::_GetDefaultDeviceIndex(EDataFlow dir, + ERole role, + int* index) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + WCHAR szDefaultDeviceID[MAX_PATH] = {0}; + WCHAR szDeviceID[MAX_PATH] = {0}; + + const size_t kDeviceIDLength = sizeof(szDeviceID) / sizeof(szDeviceID[0]); + assert(kDeviceIDLength == + sizeof(szDefaultDeviceID) / sizeof(szDefaultDeviceID[0])); + + if (_GetDefaultDeviceID(dir, role, szDefaultDeviceID, kDeviceIDLength) == + -1) { + return -1; + } + + IMMDeviceCollection* collection = _ptrCaptureCollection; + if (dir == eRender) { + collection = _ptrRenderCollection; + } + + if (!collection) { + RTC_LOG(LS_ERROR) << "Device collection not valid"; + return -1; + } + + UINT count = 0; + hr = collection->GetCount(&count); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + *index = -1; + for (UINT i = 0; i < count; i++) { + memset(szDeviceID, 0, sizeof(szDeviceID)); + rtc::scoped_refptr device; + { + IMMDevice* ptrDevice = NULL; + hr = collection->Item(i, &ptrDevice); + if (FAILED(hr) || ptrDevice == NULL) { + _TraceCOMError(hr); + return -1; + } + device = ptrDevice; + SAFE_RELEASE(ptrDevice); + } + + if (_GetDeviceID(device, szDeviceID, kDeviceIDLength) == -1) { + return -1; + } + + if (wcsncmp(szDefaultDeviceID, szDeviceID, kDeviceIDLength) == 0) { + // Found a match. + *index = i; + break; + } + } + + if (*index == -1) { + RTC_LOG(LS_ERROR) << "Unable to find collection index for default device"; + return -1; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// _GetDeviceName +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetDeviceName(IMMDevice* pDevice, + LPWSTR pszBuffer, + int bufferLen) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + static const WCHAR szDefault[] = L""; + + HRESULT hr = E_FAIL; + IPropertyStore* pProps = NULL; + PROPVARIANT varName; + + assert(pszBuffer != NULL); + assert(bufferLen > 0); + + if (pDevice != NULL) { + hr = pDevice->OpenPropertyStore(STGM_READ, &pProps); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "IMMDevice::OpenPropertyStore failed, hr = 0x" + << std::hex << hr << std::dec; + } + } + + // Initialize container for property value. + PropVariantInit(&varName); + + if (SUCCEEDED(hr)) { + // Get the endpoint device's friendly-name property. + hr = pProps->GetValue(PKEY_Device_FriendlyName, &varName); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "IPropertyStore::GetValue failed, hr = 0x" + << std::hex << hr << std::dec; + } + } + + if ((SUCCEEDED(hr)) && (VT_EMPTY == varName.vt)) { + hr = E_FAIL; + RTC_LOG(LS_ERROR) << "IPropertyStore::GetValue returned no value," + << " hr = 0x" << std::hex << hr << std::dec; + } + + if ((SUCCEEDED(hr)) && (VT_LPWSTR != varName.vt)) { + // The returned value is not a wide null terminated string. + hr = E_UNEXPECTED; + RTC_LOG(LS_ERROR) << "IPropertyStore::GetValue returned unexpected" + << " type, hr = 0x" << std::hex << hr << std::dec; + } + + if (SUCCEEDED(hr) && (varName.pwszVal != NULL)) { + // Copy the valid device name to the provided ouput buffer. + wcsncpy_s(pszBuffer, bufferLen, varName.pwszVal, _TRUNCATE); + } else { + // Failed to find the device name. + wcsncpy_s(pszBuffer, bufferLen, szDefault, _TRUNCATE); + } + + PropVariantClear(&varName); + SAFE_RELEASE(pProps); + + return 0; +} + +// ---------------------------------------------------------------------------- +// _GetDeviceID +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetDeviceID(IMMDevice* pDevice, + LPWSTR pszBuffer, + int bufferLen) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + static const WCHAR szDefault[] = L""; + + HRESULT hr = E_FAIL; + LPWSTR pwszID = NULL; + + assert(pszBuffer != NULL); + assert(bufferLen > 0); + + if (pDevice != NULL) { + hr = pDevice->GetId(&pwszID); + } + + if (hr == S_OK) { + // Found the device ID. + wcsncpy_s(pszBuffer, bufferLen, pwszID, _TRUNCATE); + } else { + // Failed to find the device ID. + wcsncpy_s(pszBuffer, bufferLen, szDefault, _TRUNCATE); + } + + CoTaskMemFree(pwszID); + return 0; +} + +// ---------------------------------------------------------------------------- +// _GetDefaultDevice +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetDefaultDevice(EDataFlow dir, + ERole role, + IMMDevice** ppDevice) { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr(S_OK); + + assert(_ptrEnumerator != NULL); + + hr = _ptrEnumerator->GetDefaultAudioEndpoint(dir, role, ppDevice); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// _GetListDevice +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_GetListDevice(EDataFlow dir, + int index, + IMMDevice** ppDevice) { + HRESULT hr(S_OK); + + assert(_ptrEnumerator != NULL); + + IMMDeviceCollection* pCollection = NULL; + + hr = _ptrEnumerator->EnumAudioEndpoints( + dir, + DEVICE_STATE_ACTIVE, // only active endpoints are OK + &pCollection); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pCollection); + return -1; + } + + hr = pCollection->Item(index, ppDevice); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pCollection); + return -1; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// _EnumerateEndpointDevicesAll +// ---------------------------------------------------------------------------- + +int32_t AudioDeviceWindowsCore::_EnumerateEndpointDevicesAll( + EDataFlow dataFlow) const { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; + + assert(_ptrEnumerator != NULL); + + HRESULT hr = S_OK; + IMMDeviceCollection* pCollection = NULL; + IMMDevice* pEndpoint = NULL; + IPropertyStore* pProps = NULL; + IAudioEndpointVolume* pEndpointVolume = NULL; + LPWSTR pwszID = NULL; + + // Generate a collection of audio endpoint devices in the system. + // Get states for *all* endpoint devices. + // Output: IMMDeviceCollection interface. + hr = _ptrEnumerator->EnumAudioEndpoints( + dataFlow, // data-flow direction (input parameter) + DEVICE_STATE_ACTIVE | DEVICE_STATE_DISABLED | DEVICE_STATE_UNPLUGGED, + &pCollection); // release interface when done + + EXIT_ON_ERROR(hr); + + // use the IMMDeviceCollection interface... + + UINT count = 0; + + // Retrieve a count of the devices in the device collection. + hr = pCollection->GetCount(&count); + EXIT_ON_ERROR(hr); + if (dataFlow == eRender) + RTC_LOG(LS_VERBOSE) << "#rendering endpoint devices (counting all): " + << count; + else if (dataFlow == eCapture) + RTC_LOG(LS_VERBOSE) << "#capturing endpoint devices (counting all): " + << count; + + if (count == 0) { + return 0; + } + + // Each loop prints the name of an endpoint device. + for (ULONG i = 0; i < count; i++) { + RTC_LOG(LS_VERBOSE) << "Endpoint " << i << ":"; + + // Get pointer to endpoint number i. + // Output: IMMDevice interface. + hr = pCollection->Item(i, &pEndpoint); + CONTINUE_ON_ERROR(hr); + + // use the IMMDevice interface of the specified endpoint device... + + // Get the endpoint ID string (uniquely identifies the device among all + // audio endpoint devices) + hr = pEndpoint->GetId(&pwszID); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "ID string : " << pwszID; + + // Retrieve an interface to the device's property store. + // Output: IPropertyStore interface. + hr = pEndpoint->OpenPropertyStore(STGM_READ, &pProps); + CONTINUE_ON_ERROR(hr); + + // use the IPropertyStore interface... + + PROPVARIANT varName; + // Initialize container for property value. + PropVariantInit(&varName); + + // Get the endpoint's friendly-name property. + // Example: "Speakers (Realtek High Definition Audio)" + hr = pProps->GetValue(PKEY_Device_FriendlyName, &varName); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "friendly name: \"" << varName.pwszVal << "\""; + + // Get the endpoint's current device state + DWORD dwState; + hr = pEndpoint->GetState(&dwState); + CONTINUE_ON_ERROR(hr); + if (dwState & DEVICE_STATE_ACTIVE) + RTC_LOG(LS_VERBOSE) << "state (0x" << std::hex << dwState << std::dec + << ") : *ACTIVE*"; + if (dwState & DEVICE_STATE_DISABLED) + RTC_LOG(LS_VERBOSE) << "state (0x" << std::hex << dwState << std::dec + << ") : DISABLED"; + if (dwState & DEVICE_STATE_NOTPRESENT) + RTC_LOG(LS_VERBOSE) << "state (0x" << std::hex << dwState << std::dec + << ") : NOTPRESENT"; + if (dwState & DEVICE_STATE_UNPLUGGED) + RTC_LOG(LS_VERBOSE) << "state (0x" << std::hex << dwState << std::dec + << ") : UNPLUGGED"; + + // Check the hardware volume capabilities. + DWORD dwHwSupportMask = 0; + hr = pEndpoint->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + (void**)&pEndpointVolume); + CONTINUE_ON_ERROR(hr); + hr = pEndpointVolume->QueryHardwareSupport(&dwHwSupportMask); + CONTINUE_ON_ERROR(hr); + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_VOLUME) + // The audio endpoint device supports a hardware volume control + RTC_LOG(LS_VERBOSE) << "hwmask (0x" << std::hex << dwHwSupportMask + << std::dec << ") : HARDWARE_SUPPORT_VOLUME"; + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_MUTE) + // The audio endpoint device supports a hardware mute control + RTC_LOG(LS_VERBOSE) << "hwmask (0x" << std::hex << dwHwSupportMask + << std::dec << ") : HARDWARE_SUPPORT_MUTE"; + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_METER) + // The audio endpoint device supports a hardware peak meter + RTC_LOG(LS_VERBOSE) << "hwmask (0x" << std::hex << dwHwSupportMask + << std::dec << ") : HARDWARE_SUPPORT_METER"; + + // Check the channel count (#channels in the audio stream that enters or + // leaves the audio endpoint device) + UINT nChannelCount(0); + hr = pEndpointVolume->GetChannelCount(&nChannelCount); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "#channels : " << nChannelCount; + + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_VOLUME) { + // Get the volume range. + float fLevelMinDB(0.0); + float fLevelMaxDB(0.0); + float fVolumeIncrementDB(0.0); + hr = pEndpointVolume->GetVolumeRange(&fLevelMinDB, &fLevelMaxDB, + &fVolumeIncrementDB); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "volume range : " << fLevelMinDB << " (min), " + << fLevelMaxDB << " (max), " << fVolumeIncrementDB + << " (inc) [dB]"; + + // The volume range from vmin = fLevelMinDB to vmax = fLevelMaxDB is + // divided into n uniform intervals of size vinc = fVolumeIncrementDB, + // where n = (vmax ?vmin) / vinc. The values vmin, vmax, and vinc are + // measured in decibels. The client can set the volume level to one of n + + // 1 discrete values in the range from vmin to vmax. + int n = (int)((fLevelMaxDB - fLevelMinDB) / fVolumeIncrementDB); + RTC_LOG(LS_VERBOSE) << "#intervals : " << n; + + // Get information about the current step in the volume range. + // This method represents the volume level of the audio stream that enters + // or leaves the audio endpoint device as an index or "step" in a range of + // discrete volume levels. Output value nStepCount is the number of steps + // in the range. Output value nStep is the step index of the current + // volume level. If the number of steps is n = nStepCount, then step index + // nStep can assume values from 0 (minimum volume) to n ?1 (maximum + // volume). + UINT nStep(0); + UINT nStepCount(0); + hr = pEndpointVolume->GetVolumeStepInfo(&nStep, &nStepCount); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "volume steps : " << nStep << " (nStep), " + << nStepCount << " (nStepCount)"; + } + Next: + if (FAILED(hr)) { + RTC_LOG(LS_VERBOSE) << "Error when logging device information"; + } + CoTaskMemFree(pwszID); + pwszID = NULL; + PropVariantClear(&varName); + SAFE_RELEASE(pProps); + SAFE_RELEASE(pEndpoint); + SAFE_RELEASE(pEndpointVolume); + } + SAFE_RELEASE(pCollection); + return 0; + +Exit: + _TraceCOMError(hr); + CoTaskMemFree(pwszID); + pwszID = NULL; + SAFE_RELEASE(pCollection); + SAFE_RELEASE(pEndpoint); + SAFE_RELEASE(pEndpointVolume); + SAFE_RELEASE(pProps); + return -1; +} + +// ---------------------------------------------------------------------------- +// _TraceCOMError +// ---------------------------------------------------------------------------- + +void AudioDeviceWindowsCore::_TraceCOMError(HRESULT hr) const { + TCHAR buf[MAXERRORLENGTH]; + TCHAR errorText[MAXERRORLENGTH]; + + const DWORD dwFlags = + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD dwLangID = MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US); + + // Gets the system's human readable message string for this HRESULT. + // All error message in English by default. + DWORD messageLength = ::FormatMessageW(dwFlags, 0, hr, dwLangID, errorText, + MAXERRORLENGTH, NULL); + + assert(messageLength <= MAXERRORLENGTH); + + // Trims tailing white space (FormatMessage() leaves a trailing cr-lf.). + for (; messageLength && ::isspace(errorText[messageLength - 1]); + --messageLength) { + errorText[messageLength - 1] = '\0'; + } + + RTC_LOG(LS_ERROR) << "Core Audio method failed (hr=" << hr << ")"; + StringCchPrintf(buf, MAXERRORLENGTH, TEXT("Error details: ")); + StringCchCat(buf, MAXERRORLENGTH, errorText); + RTC_LOG(LS_ERROR) << WideToUTF8(buf); +} + +// ---------------------------------------------------------------------------- +// WideToUTF8 +// ---------------------------------------------------------------------------- + +char* AudioDeviceWindowsCore::WideToUTF8(const TCHAR* src) const { +#ifdef UNICODE + const size_t kStrLen = sizeof(_str); + memset(_str, 0, kStrLen); + // Get required size (in bytes) to be able to complete the conversion. + unsigned int required_size = + (unsigned int)WideCharToMultiByte(CP_UTF8, 0, src, -1, _str, 0, 0, 0); + if (required_size <= kStrLen) { + // Process the entire input string, including the terminating null char. + if (WideCharToMultiByte(CP_UTF8, 0, src, -1, _str, kStrLen, 0, 0) == 0) + memset(_str, 0, kStrLen); + } + return _str; +#else + return const_cast(src); +#endif +} + +bool AudioDeviceWindowsCore::KeyPressed() const { + int key_down = 0; + for (int key = VK_SPACE; key < VK_NUMLOCK; key++) { + short res = GetAsyncKeyState(key); + key_down |= res & 0x1; // Get the LSB + } + return (key_down > 0); +} +} // namespace webrtc + +#endif // WEBRTC_WINDOWS_CORE_AUDIO_BUILD diff --git a/third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.h b/third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.h new file mode 100644 index 0000000000..30e767b2f2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_device/win/audio_device_core_win.h @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_DEVICE_AUDIO_DEVICE_CORE_WIN_H_ +#define AUDIO_DEVICE_AUDIO_DEVICE_CORE_WIN_H_ + +#if (_MSC_VER >= 1400) // only include for VS 2005 and higher + +#include "modules/audio_device/audio_device_generic.h" + +#include // CLSID_CWMAudioAEC + // (must be before audioclient.h) +#include // WASAPI +#include +#include // MMDevice +#include // Avrt +#include +#include // IMediaObject + +#include "rtc_base/criticalsection.h" +#include "rtc_base/scoped_ref_ptr.h" + +// Use Multimedia Class Scheduler Service (MMCSS) to boost the thread priority +#pragma comment( lib, "avrt.lib" ) +// AVRT function pointers +typedef BOOL (WINAPI *PAvRevertMmThreadCharacteristics)(HANDLE); +typedef HANDLE (WINAPI *PAvSetMmThreadCharacteristicsA)(LPCSTR, LPDWORD); +typedef BOOL (WINAPI *PAvSetMmThreadPriority)(HANDLE, AVRT_PRIORITY); + +namespace webrtc { + +const float MAX_CORE_SPEAKER_VOLUME = 255.0f; +const float MIN_CORE_SPEAKER_VOLUME = 0.0f; +const float MAX_CORE_MICROPHONE_VOLUME = 255.0f; +const float MIN_CORE_MICROPHONE_VOLUME = 0.0f; +const uint16_t CORE_SPEAKER_VOLUME_STEP_SIZE = 1; +const uint16_t CORE_MICROPHONE_VOLUME_STEP_SIZE = 1; + +// Utility class which initializes COM in the constructor (STA or MTA), +// and uninitializes COM in the destructor. +class ScopedCOMInitializer { + public: + // Enum value provided to initialize the thread as an MTA instead of STA. + enum SelectMTA { kMTA }; + + // Constructor for STA initialization. + ScopedCOMInitializer() { + Initialize(COINIT_APARTMENTTHREADED); + } + + // Constructor for MTA initialization. + explicit ScopedCOMInitializer(SelectMTA mta) { + Initialize(COINIT_MULTITHREADED); + } + + ScopedCOMInitializer::~ScopedCOMInitializer() { + if (SUCCEEDED(hr_)) + CoUninitialize(); + } + + bool succeeded() const { return SUCCEEDED(hr_); } + + private: + void Initialize(COINIT init) { + hr_ = CoInitializeEx(NULL, init); + } + + HRESULT hr_; + + ScopedCOMInitializer(const ScopedCOMInitializer&); + void operator=(const ScopedCOMInitializer&); +}; + + +class AudioDeviceWindowsCore : public AudioDeviceGeneric +{ +public: + AudioDeviceWindowsCore(); + ~AudioDeviceWindowsCore(); + + static bool CoreAudioIsSupported(); + + // Retrieve the currently utilized audio layer + virtual int32_t ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const; + + // Main initializaton and termination + virtual InitStatus Init(); + virtual int32_t Terminate(); + virtual bool Initialized() const; + + // Device enumeration + virtual int16_t PlayoutDevices(); + virtual int16_t RecordingDevices(); + virtual int32_t PlayoutDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]); + virtual int32_t RecordingDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]); + + // Device selection + virtual int32_t SetPlayoutDevice(uint16_t index); + virtual int32_t SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType device); + virtual int32_t SetRecordingDevice(uint16_t index); + virtual int32_t SetRecordingDevice(AudioDeviceModule::WindowsDeviceType device); + + // Audio transport initialization + virtual int32_t PlayoutIsAvailable(bool& available); + virtual int32_t InitPlayout(); + virtual bool PlayoutIsInitialized() const; + virtual int32_t RecordingIsAvailable(bool& available); + virtual int32_t InitRecording(); + virtual bool RecordingIsInitialized() const; + + // Audio transport control + virtual int32_t StartPlayout(); + virtual int32_t StopPlayout(); + virtual bool Playing() const; + virtual int32_t StartRecording(); + virtual int32_t StopRecording(); + virtual bool Recording() const; + + // Microphone Automatic Gain Control (AGC) + virtual int32_t SetAGC(bool enable); + virtual bool AGC() const; + + // Audio mixer initialization + virtual int32_t InitSpeaker(); + virtual bool SpeakerIsInitialized() const; + virtual int32_t InitMicrophone(); + virtual bool MicrophoneIsInitialized() const; + + // Speaker volume controls + virtual int32_t SpeakerVolumeIsAvailable(bool& available); + virtual int32_t SetSpeakerVolume(uint32_t volume); + virtual int32_t SpeakerVolume(uint32_t& volume) const; + virtual int32_t MaxSpeakerVolume(uint32_t& maxVolume) const; + virtual int32_t MinSpeakerVolume(uint32_t& minVolume) const; + + // Microphone volume controls + virtual int32_t MicrophoneVolumeIsAvailable(bool& available); + virtual int32_t SetMicrophoneVolume(uint32_t volume); + virtual int32_t MicrophoneVolume(uint32_t& volume) const; + virtual int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const; + virtual int32_t MinMicrophoneVolume(uint32_t& minVolume) const; + + // Speaker mute control + virtual int32_t SpeakerMuteIsAvailable(bool& available); + virtual int32_t SetSpeakerMute(bool enable); + virtual int32_t SpeakerMute(bool& enabled) const; + + // Microphone mute control + virtual int32_t MicrophoneMuteIsAvailable(bool& available); + virtual int32_t SetMicrophoneMute(bool enable); + virtual int32_t MicrophoneMute(bool& enabled) const; + + // Stereo support + virtual int32_t StereoPlayoutIsAvailable(bool& available); + virtual int32_t SetStereoPlayout(bool enable); + virtual int32_t StereoPlayout(bool& enabled) const; + virtual int32_t StereoRecordingIsAvailable(bool& available); + virtual int32_t SetStereoRecording(bool enable); + virtual int32_t StereoRecording(bool& enabled) const; + + // Delay information and control + virtual int32_t PlayoutDelay(uint16_t& delayMS) const; + + virtual int32_t EnableBuiltInAEC(bool enable); + +public: + virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); + +private: + bool KeyPressed() const; + +private: // avrt function pointers + PAvRevertMmThreadCharacteristics _PAvRevertMmThreadCharacteristics; + PAvSetMmThreadCharacteristicsA _PAvSetMmThreadCharacteristicsA; + PAvSetMmThreadPriority _PAvSetMmThreadPriority; + HMODULE _avrtLibrary; + bool _winSupportAvrt; + +private: // thread functions + DWORD InitCaptureThreadPriority(); + void RevertCaptureThreadPriority(); + static DWORD WINAPI WSAPICaptureThread(LPVOID context); + DWORD DoCaptureThread(); + + static DWORD WINAPI WSAPICaptureThreadPollDMO(LPVOID context); + DWORD DoCaptureThreadPollDMO(); + + static DWORD WINAPI WSAPIRenderThread(LPVOID context); + DWORD DoRenderThread(); + + static DWORD WINAPI GetCaptureVolumeThread(LPVOID context); + DWORD DoGetCaptureVolumeThread(); + + static DWORD WINAPI SetCaptureVolumeThread(LPVOID context); + DWORD DoSetCaptureVolumeThread(); + + void _Lock() { _critSect.Enter(); }; + void _UnLock() { _critSect.Leave(); }; + + int SetDMOProperties(); + + int SetBoolProperty(IPropertyStore* ptrPS, + REFPROPERTYKEY key, + VARIANT_BOOL value); + + int SetVtI4Property(IPropertyStore* ptrPS, + REFPROPERTYKEY key, + LONG value); + + int32_t _EnumerateEndpointDevicesAll(EDataFlow dataFlow) const; + void _TraceCOMError(HRESULT hr) const; + + int32_t _RefreshDeviceList(EDataFlow dir); + int16_t _DeviceListCount(EDataFlow dir); + int32_t _GetDefaultDeviceName(EDataFlow dir, ERole role, LPWSTR szBuffer, int bufferLen); + int32_t _GetListDeviceName(EDataFlow dir, int index, LPWSTR szBuffer, int bufferLen); + int32_t _GetDeviceName(IMMDevice* pDevice, LPWSTR pszBuffer, int bufferLen); + int32_t _GetListDeviceID(EDataFlow dir, int index, LPWSTR szBuffer, int bufferLen); + int32_t _GetDefaultDeviceID(EDataFlow dir, ERole role, LPWSTR szBuffer, int bufferLen); + int32_t _GetDefaultDeviceIndex(EDataFlow dir, ERole role, int* index); + int32_t _GetDeviceID(IMMDevice* pDevice, LPWSTR pszBuffer, int bufferLen); + int32_t _GetDefaultDevice(EDataFlow dir, ERole role, IMMDevice** ppDevice); + int32_t _GetListDevice(EDataFlow dir, int index, IMMDevice** ppDevice); + + // Converts from wide-char to UTF-8 if UNICODE is defined. + // Does nothing if UNICODE is undefined. + char* WideToUTF8(const TCHAR* src) const; + + int32_t InitRecordingDMO(); + + ScopedCOMInitializer _comInit; + AudioDeviceBuffer* _ptrAudioBuffer; + rtc::CriticalSection _critSect; + rtc::CriticalSection _volumeMutex; + + IMMDeviceEnumerator* _ptrEnumerator; + IMMDeviceCollection* _ptrRenderCollection; + IMMDeviceCollection* _ptrCaptureCollection; + IMMDevice* _ptrDeviceOut; + IMMDevice* _ptrDeviceIn; + + IAudioClient* _ptrClientOut; + IAudioClient* _ptrClientIn; + IAudioRenderClient* _ptrRenderClient; + IAudioCaptureClient* _ptrCaptureClient; + IAudioEndpointVolume* _ptrCaptureVolume; + ISimpleAudioVolume* _ptrRenderSimpleVolume; + + // DirectX Media Object (DMO) for the built-in AEC. + rtc::scoped_refptr _dmo; + rtc::scoped_refptr _mediaBuffer; + bool _builtInAecEnabled; + + HANDLE _hRenderSamplesReadyEvent; + HANDLE _hPlayThread; + HANDLE _hRenderStartedEvent; + HANDLE _hShutdownRenderEvent; + + HANDLE _hCaptureSamplesReadyEvent; + HANDLE _hRecThread; + HANDLE _hCaptureStartedEvent; + HANDLE _hShutdownCaptureEvent; + + HANDLE _hGetCaptureVolumeThread; + HANDLE _hSetCaptureVolumeThread; + HANDLE _hSetCaptureVolumeEvent; + + HANDLE _hMmTask; + + UINT _playAudioFrameSize; + uint32_t _playSampleRate; + uint32_t _devicePlaySampleRate; + uint32_t _playBlockSize; + uint32_t _devicePlayBlockSize; + uint32_t _playChannels; + uint32_t _sndCardPlayDelay; + UINT64 _writtenSamples; + + UINT _recAudioFrameSize; + uint32_t _recSampleRate; + uint32_t _recBlockSize; + uint32_t _recChannels; + UINT64 _readSamples; + uint32_t _sndCardRecDelay; + + uint16_t _recChannelsPrioList[3]; + uint16_t _playChannelsPrioList[2]; + + LARGE_INTEGER _perfCounterFreq; + double _perfCounterFactor; + +private: + bool _initialized; + bool _recording; + bool _playing; + bool _recIsInitialized; + bool _playIsInitialized; + bool _speakerIsInitialized; + bool _microphoneIsInitialized; + + bool _usingInputDeviceIndex; + bool _usingOutputDeviceIndex; + AudioDeviceModule::WindowsDeviceType _inputDevice; + AudioDeviceModule::WindowsDeviceType _outputDevice; + uint16_t _inputDeviceIndex; + uint16_t _outputDeviceIndex; + + bool _AGC; + + uint16_t _playBufDelay; + + uint16_t _newMicLevel; + + mutable char _str[512]; +}; + +#endif // #if (_MSC_VER >= 1400) + +} // namespace webrtc + +#endif // AUDIO_DEVICE_AUDIO_DEVICE_CORE_WIN_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/BUILD.gn b/third_party/libwebrtc/webrtc/modules/audio_mixer/BUILD.gn new file mode 100644 index 0000000000..00e3e609b0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/BUILD.gn @@ -0,0 +1,95 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +group("audio_mixer") { + public_deps = [ + ":audio_frame_manipulator", + ":audio_mixer_impl", + ] +} + +rtc_static_library("audio_mixer_impl") { + sources = [ + "audio_mixer_impl.cc", + "audio_mixer_impl.h", + "default_output_rate_calculator.cc", + "default_output_rate_calculator.h", + "frame_combiner.cc", + "frame_combiner.h", + "output_rate_calculator.h", + ] + + public = [ + "audio_mixer_impl.h", + "default_output_rate_calculator.h", # For creating a mixer with limiter disabled. + "frame_combiner.h", + ] + + public_deps = [ + "../../api:audio_mixer_api", + ] + + deps = [ + ":audio_frame_manipulator", + "..:module_api", + "../..:webrtc_common", + "../../api:array_view", + "../../audio/utility:audio_frame_operations", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../audio_processing", + ] +} + +rtc_static_library("audio_frame_manipulator") { + visibility = [ + ":*", + "../../modules:*", + ] + + sources = [ + "audio_frame_manipulator.cc", + "audio_frame_manipulator.h", + ] + + deps = [ + "..:module_api", + "../../audio/utility", + "../../rtc_base:rtc_base_approved", + ] +} + +if (rtc_include_tests) { + rtc_source_set("audio_mixer_unittests") { + testonly = true + + sources = [ + "audio_frame_manipulator_unittest.cc", + "audio_mixer_impl_unittest.cc", + "frame_combiner_unittest.cc", + "gain_change_calculator.cc", + "gain_change_calculator.h", + "sine_wave_generator.cc", + "sine_wave_generator.h", + ] + deps = [ + ":audio_frame_manipulator", + ":audio_mixer_impl", + "..:module_api", + "../../api:array_view", + "../../api:audio_mixer_api", + "../../audio/utility:audio_frame_operations", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_task_queue", + "../../test:test_support", + "//testing/gmock", + ] + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/DEPS b/third_party/libwebrtc/webrtc/modules/audio_mixer/DEPS new file mode 100644 index 0000000000..51608ad47d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/DEPS @@ -0,0 +1,14 @@ +include_rules = [ + "+audio/utility/audio_frame_operations.h", + "+call", + "+common_audio", + "+modules/audio_coding", + "+modules/audio_device", + "+modules/audio_processing", + "+modules/media_file", + "+modules/pacing", + "+modules/rtp_rtcp", + "+modules/utility", + "+system_wrappers", + "+voice_engine", +] diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_mixer/OWNERS new file mode 100644 index 0000000000..633cd5621a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/OWNERS @@ -0,0 +1,7 @@ +aleloi@webrtc.org +henrik.lundin@webrtc.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.cc new file mode 100644 index 0000000000..0f5a83fed7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/audio_frame_manipulator.h" +#include "audio/utility/audio_frame_operations.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +uint32_t AudioMixerCalculateEnergy(const AudioFrame& audio_frame) { + if (audio_frame.muted()) { + return 0; + } + + uint32_t energy = 0; + const int16_t* frame_data = audio_frame.data(); + for (size_t position = 0; position < audio_frame.samples_per_channel_; + position++) { + // TODO(aleloi): This can overflow. Convert to floats. + energy += frame_data[position] * frame_data[position]; + } + return energy; +} + +void Ramp(float start_gain, float target_gain, AudioFrame* audio_frame) { + RTC_DCHECK(audio_frame); + RTC_DCHECK_GE(start_gain, 0.0f); + RTC_DCHECK_GE(target_gain, 0.0f); + if (start_gain == target_gain || audio_frame->muted()) { + return; + } + + size_t samples = audio_frame->samples_per_channel_; + RTC_DCHECK_LT(0, samples); + float increment = (target_gain - start_gain) / samples; + float gain = start_gain; + int16_t* frame_data = audio_frame->mutable_data(); + for (size_t i = 0; i < samples; ++i) { + // If the audio is interleaved of several channels, we want to + // apply the same gain change to the ith sample of every channel. + for (size_t ch = 0; ch < audio_frame->num_channels_; ++ch) { + frame_data[audio_frame->num_channels_ * i + ch] *= gain; + } + gain += increment; + } +} + +void RemixFrame(size_t target_number_of_channels, AudioFrame* frame) { + RTC_DCHECK_GE(target_number_of_channels, 1); + RTC_DCHECK_LE(target_number_of_channels, 2); + if (frame->num_channels_ == 1 && target_number_of_channels == 2) { + AudioFrameOperations::MonoToStereo(frame); + } else if (frame->num_channels_ == 2 && target_number_of_channels == 1) { + AudioFrameOperations::StereoToMono(frame); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.h b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.h new file mode 100644 index 0000000000..fe87169676 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_AUDIO_FRAME_MANIPULATOR_H_ +#define MODULES_AUDIO_MIXER_AUDIO_FRAME_MANIPULATOR_H_ + +#include "modules/include/module_common_types.h" + +namespace webrtc { + +// Updates the audioFrame's energy (based on its samples). +uint32_t AudioMixerCalculateEnergy(const AudioFrame& audio_frame); + +// Ramps up or down the provided audio frame. Ramp(0, 1, frame) will +// linearly increase the samples in the frame from 0 to full volume. +void Ramp(float start_gain, float target_gain, AudioFrame* audio_frame); + +// Downmixes or upmixes a frame between stereo and mono. +void RemixFrame(size_t target_number_of_channels, AudioFrame* frame); + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_AUDIO_FRAME_MANIPULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_gn/moz.build new file mode 100644 index 0000000000..4b44f4f38c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_gn/moz.build @@ -0,0 +1,225 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_frame_manipulator_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc new file mode 100644 index 0000000000..b3be883ce0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_mixer/audio_frame_manipulator.h" +#include "modules/include/module_common_types.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +void FillFrameWithConstants(size_t samples_per_channel, + size_t number_of_channels, + int16_t value, + AudioFrame* frame) { + frame->num_channels_ = number_of_channels; + frame->samples_per_channel_ = samples_per_channel; + int16_t* frame_data = frame->mutable_data(); + std::fill(frame_data, + frame_data + samples_per_channel * number_of_channels, value); +} +} // namespace + +TEST(AudioFrameManipulator, CompareForwardRampWithExpectedResultStereo) { + constexpr int kSamplesPerChannel = 5; + constexpr int kNumberOfChannels = 2; + + // Create a frame with values 5, 5, 5, ... and channels & samples as above. + AudioFrame frame; + FillFrameWithConstants(kSamplesPerChannel, kNumberOfChannels, 5, &frame); + + Ramp(0.0f, 1.0f, &frame); + + const int total_samples = kSamplesPerChannel * kNumberOfChannels; + const int16_t expected_result[total_samples] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4}; + const int16_t* frame_data = frame.data(); + EXPECT_TRUE( + std::equal(frame_data, frame_data + total_samples, expected_result)); +} + +TEST(AudioFrameManipulator, CompareBackwardRampWithExpectedResultMono) { + constexpr int kSamplesPerChannel = 5; + constexpr int kNumberOfChannels = 1; + + // Create a frame with values 5, 5, 5, ... and channels & samples as above. + AudioFrame frame; + FillFrameWithConstants(kSamplesPerChannel, kNumberOfChannels, 5, &frame); + + Ramp(1.0f, 0.0f, &frame); + + const int total_samples = kSamplesPerChannel * kNumberOfChannels; + const int16_t expected_result[total_samples] = {5, 4, 3, 2, 1}; + const int16_t* frame_data = frame.data(); + EXPECT_TRUE( + std::equal(frame_data, frame_data + total_samples, expected_result)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.cc new file mode 100644 index 0000000000..34b4ae54fe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.cc @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/audio_mixer_impl.h" + +#include +#include +#include +#include + +#include "modules/audio_mixer/audio_frame_manipulator.h" +#include "modules/audio_mixer/default_output_rate_calculator.h" +#include "rtc_base/logging.h" +#include "rtc_base/refcountedobject.h" + +namespace webrtc { +namespace { + +struct SourceFrame { + SourceFrame(AudioMixerImpl::SourceStatus* source_status, + AudioFrame* audio_frame, + bool muted) + : source_status(source_status), audio_frame(audio_frame), muted(muted) { + RTC_DCHECK(source_status); + RTC_DCHECK(audio_frame); + if (!muted) { + energy = AudioMixerCalculateEnergy(*audio_frame); + } + } + + SourceFrame(AudioMixerImpl::SourceStatus* source_status, + AudioFrame* audio_frame, + bool muted, + uint32_t energy) + : source_status(source_status), + audio_frame(audio_frame), + muted(muted), + energy(energy) { + RTC_DCHECK(source_status); + RTC_DCHECK(audio_frame); + } + + AudioMixerImpl::SourceStatus* source_status = nullptr; + AudioFrame* audio_frame = nullptr; + bool muted = true; + uint32_t energy = 0; +}; + +// ShouldMixBefore(a, b) is used to select mixer sources. +bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) { + if (a.muted != b.muted) { + return b.muted; + } + + const auto a_activity = a.audio_frame->vad_activity_; + const auto b_activity = b.audio_frame->vad_activity_; + + if (a_activity != b_activity) { + return a_activity == AudioFrame::kVadActive; + } + + return a.energy > b.energy; +} + +void RampAndUpdateGain( + const std::vector& mixed_sources_and_frames) { + for (const auto& source_frame : mixed_sources_and_frames) { + float target_gain = source_frame.source_status->is_mixed ? 1.0f : 0.0f; + Ramp(source_frame.source_status->gain, target_gain, + source_frame.audio_frame); + source_frame.source_status->gain = target_gain; + } +} + +AudioMixerImpl::SourceStatusList::const_iterator FindSourceInList( + AudioMixerImpl::Source const* audio_source, + AudioMixerImpl::SourceStatusList const* audio_source_list) { + return std::find_if( + audio_source_list->begin(), audio_source_list->end(), + [audio_source](const std::unique_ptr& p) { + return p->audio_source == audio_source; + }); +} + +// TODO(aleloi): remove non-const version when WEBRTC only supports modern STL. +AudioMixerImpl::SourceStatusList::iterator FindSourceInList( + AudioMixerImpl::Source const* audio_source, + AudioMixerImpl::SourceStatusList* audio_source_list) { + return std::find_if( + audio_source_list->begin(), audio_source_list->end(), + [audio_source](const std::unique_ptr& p) { + return p->audio_source == audio_source; + }); +} + +} // namespace + +AudioMixerImpl::AudioMixerImpl( + std::unique_ptr output_rate_calculator, + bool use_limiter) + : output_rate_calculator_(std::move(output_rate_calculator)), + output_frequency_(0), + sample_size_(0), + audio_source_list_(), + frame_combiner_(use_limiter) {} + +AudioMixerImpl::~AudioMixerImpl() {} + +rtc::scoped_refptr AudioMixerImpl::Create() { + return Create(std::unique_ptr( + new DefaultOutputRateCalculator()), + true); +} + +rtc::scoped_refptr AudioMixerImpl::Create( + std::unique_ptr output_rate_calculator, + bool use_limiter) { + return rtc::scoped_refptr( + new rtc::RefCountedObject( + std::move(output_rate_calculator), use_limiter)); +} + +void AudioMixerImpl::Mix(size_t number_of_channels, + AudioFrame* audio_frame_for_mixing) { + RTC_DCHECK(number_of_channels == 1 || number_of_channels == 2); + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + + CalculateOutputFrequency(); + + { + rtc::CritScope lock(&crit_); + const size_t number_of_streams = audio_source_list_.size(); + frame_combiner_.Combine(GetAudioFromSources(), number_of_channels, + OutputFrequency(), number_of_streams, + audio_frame_for_mixing); + } + + return; +} + +void AudioMixerImpl::CalculateOutputFrequency() { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + rtc::CritScope lock(&crit_); + + std::vector preferred_rates; + std::transform(audio_source_list_.begin(), audio_source_list_.end(), + std::back_inserter(preferred_rates), + [&](std::unique_ptr& a) { + return a->audio_source->PreferredSampleRate(); + }); + + output_frequency_ = + output_rate_calculator_->CalculateOutputRate(preferred_rates); + sample_size_ = (output_frequency_ * kFrameDurationInMs) / 1000; +} + +int AudioMixerImpl::OutputFrequency() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + return output_frequency_; +} + +bool AudioMixerImpl::AddSource(Source* audio_source) { + RTC_DCHECK(audio_source); + rtc::CritScope lock(&crit_); + RTC_DCHECK(FindSourceInList(audio_source, &audio_source_list_) == + audio_source_list_.end()) + << "Source already added to mixer"; + audio_source_list_.emplace_back(new SourceStatus(audio_source, false, 0)); + return true; +} + +void AudioMixerImpl::RemoveSource(Source* audio_source) { + RTC_DCHECK(audio_source); + rtc::CritScope lock(&crit_); + const auto iter = FindSourceInList(audio_source, &audio_source_list_); + RTC_DCHECK(iter != audio_source_list_.end()) << "Source not present in mixer"; + audio_source_list_.erase(iter); +} + +AudioFrameList AudioMixerImpl::GetAudioFromSources() { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + AudioFrameList result; + std::vector audio_source_mixing_data_list; + std::vector ramp_list; + + // Get audio from the audio sources and put it in the SourceFrame vector. + for (auto& source_and_status : audio_source_list_) { + const auto audio_frame_info = + source_and_status->audio_source->GetAudioFrameWithInfo( + OutputFrequency(), &source_and_status->audio_frame); + + if (audio_frame_info == Source::AudioFrameInfo::kError) { + RTC_LOG_F(LS_WARNING) << "failed to GetAudioFrameWithInfo() from source"; + continue; + } + audio_source_mixing_data_list.emplace_back( + source_and_status.get(), &source_and_status->audio_frame, + audio_frame_info == Source::AudioFrameInfo::kMuted); + } + + // Sort frames by sorting function. + std::sort(audio_source_mixing_data_list.begin(), + audio_source_mixing_data_list.end(), ShouldMixBefore); + + int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources; + + // Go through list in order and put unmuted frames in result list. + for (const auto& p : audio_source_mixing_data_list) { + // Filter muted. + if (p.muted) { + p.source_status->is_mixed = false; + continue; + } + + // Add frame to result vector for mixing. + bool is_mixed = false; + if (max_audio_frame_counter > 0) { + --max_audio_frame_counter; + result.push_back(p.audio_frame); + ramp_list.emplace_back(p.source_status, p.audio_frame, false, -1); + is_mixed = true; + } + p.source_status->is_mixed = is_mixed; + } + RampAndUpdateGain(ramp_list); + return result; +} + +bool AudioMixerImpl::GetAudioSourceMixabilityStatusForTest( + AudioMixerImpl::Source* audio_source) const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + rtc::CritScope lock(&crit_); + + const auto iter = FindSourceInList(audio_source, &audio_source_list_); + if (iter != audio_source_list_.end()) { + return (*iter)->is_mixed; + } + + RTC_LOG(LS_ERROR) << "Audio source unknown"; + return false; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.h b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.h new file mode 100644 index 0000000000..8d49e913e4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_AUDIO_MIXER_IMPL_H_ +#define MODULES_AUDIO_MIXER_AUDIO_MIXER_IMPL_H_ + +#include +#include + +#include "api/audio/audio_mixer.h" +#include "modules/audio_mixer/frame_combiner.h" +#include "modules/audio_mixer/output_rate_calculator.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "rtc_base/thread_annotations.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +typedef std::vector AudioFrameList; + +class AudioMixerImpl : public AudioMixer { + public: + struct SourceStatus { + SourceStatus(Source* audio_source, bool is_mixed, float gain) + : audio_source(audio_source), is_mixed(is_mixed), gain(gain) {} + Source* audio_source = nullptr; + bool is_mixed = false; + float gain = 0.0f; + + // A frame that will be passed to audio_source->GetAudioFrameWithInfo. + AudioFrame audio_frame; + }; + + using SourceStatusList = std::vector>; + + // AudioProcessing only accepts 10 ms frames. + static const int kFrameDurationInMs = 10; + static const int kMaximumAmountOfMixedAudioSources = 3; + + static rtc::scoped_refptr Create(); + + static rtc::scoped_refptr Create( + std::unique_ptr output_rate_calculator, + bool use_limiter); + + ~AudioMixerImpl() override; + + // AudioMixer functions + bool AddSource(Source* audio_source) override; + void RemoveSource(Source* audio_source) override; + + void Mix(size_t number_of_channels, + AudioFrame* audio_frame_for_mixing) override + RTC_LOCKS_EXCLUDED(crit_); + + // Returns true if the source was mixed last round. Returns + // false and logs an error if the source was never added to the + // mixer. + bool GetAudioSourceMixabilityStatusForTest(Source* audio_source) const; + + protected: + AudioMixerImpl(std::unique_ptr output_rate_calculator, + bool use_limiter); + + private: + // Set mixing frequency through OutputFrequencyCalculator. + void CalculateOutputFrequency(); + // Get mixing frequency. + int OutputFrequency() const; + + // Compute what audio sources to mix from audio_source_list_. Ramp + // in and out. Update mixed status. Mixes up to + // kMaximumAmountOfMixedAudioSources audio sources. + AudioFrameList GetAudioFromSources() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + + // Add/remove the MixerAudioSource to the specified + // MixerAudioSource list. + bool AddAudioSourceToList(Source* audio_source, + SourceStatusList* audio_source_list) const; + bool RemoveAudioSourceFromList(Source* remove_audio_source, + SourceStatusList* audio_source_list) const; + + // The critical section lock guards audio source insertion and + // removal, which can be done from any thread. The race checker + // checks that mixing is done sequentially. + rtc::CriticalSection crit_; + rtc::RaceChecker race_checker_; + + std::unique_ptr output_rate_calculator_; + // The current sample frequency and sample size when mixing. + int output_frequency_ RTC_GUARDED_BY(race_checker_); + size_t sample_size_ RTC_GUARDED_BY(race_checker_); + + // List of all audio sources. Note all lists are disjunct + SourceStatusList audio_source_list_ RTC_GUARDED_BY(crit_); // May be mixed. + + // Component that handles actual adding of audio frames. + FrameCombiner frame_combiner_ RTC_GUARDED_BY(race_checker_); + + RTC_DISALLOW_COPY_AND_ASSIGN(AudioMixerImpl); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_AUDIO_MIXER_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_gn/moz.build new file mode 100644 index 0000000000..e75d8fa6b2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_gn/moz.build @@ -0,0 +1,228 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "m", + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_mixer_impl_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc new file mode 100644 index 0000000000..4461e6e038 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc @@ -0,0 +1,543 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include +#include +#include +#include +#include + +#include "api/audio/audio_mixer.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "modules/audio_mixer/default_output_rate_calculator.h" +#include "rtc_base/bind.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/task_queue.h" +#include "test/gmock.h" + +using testing::_; +using testing::Exactly; +using testing::Invoke; +using testing::Return; + +namespace webrtc { + +namespace { + +constexpr int kDefaultSampleRateHz = 48000; + +// Utility function that resets the frame member variables with +// sensible defaults. +void ResetFrame(AudioFrame* frame) { + frame->sample_rate_hz_ = kDefaultSampleRateHz; + frame->num_channels_ = 1; + + // Frame duration 10ms. + frame->samples_per_channel_ = kDefaultSampleRateHz / 100; + frame->vad_activity_ = AudioFrame::kVadActive; + frame->speech_type_ = AudioFrame::kNormalSpeech; +} + +std::string ProduceDebugText(int sample_rate_hz, + int number_of_channels, + int number_of_sources) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz << " "; + ss << "Number of channels: " << number_of_channels << " "; + ss << "Number of sources: " << number_of_sources; + return ss.str(); +} + +AudioFrame frame_for_mixing; + +} // namespace + +class MockMixerAudioSource : public AudioMixer::Source { + public: + MockMixerAudioSource() + : fake_audio_frame_info_(AudioMixer::Source::AudioFrameInfo::kNormal) { + ON_CALL(*this, GetAudioFrameWithInfo(_, _)) + .WillByDefault( + Invoke(this, &MockMixerAudioSource::FakeAudioFrameWithInfo)); + ON_CALL(*this, PreferredSampleRate()) + .WillByDefault(Return(kDefaultSampleRateHz)); + } + + MOCK_METHOD2(GetAudioFrameWithInfo, + AudioFrameInfo(int sample_rate_hz, AudioFrame* audio_frame)); + + MOCK_CONST_METHOD0(PreferredSampleRate, int()); + MOCK_CONST_METHOD0(Ssrc, int()); + + AudioFrame* fake_frame() { return &fake_frame_; } + AudioFrameInfo fake_info() { return fake_audio_frame_info_; } + void set_fake_info(const AudioFrameInfo audio_frame_info) { + fake_audio_frame_info_ = audio_frame_info; + } + + private: + AudioFrameInfo FakeAudioFrameWithInfo(int sample_rate_hz, + AudioFrame* audio_frame) { + audio_frame->CopyFrom(fake_frame_); + audio_frame->sample_rate_hz_ = sample_rate_hz; + audio_frame->samples_per_channel_ = + rtc::CheckedDivExact(sample_rate_hz, 100); + return fake_info(); + } + + AudioFrame fake_frame_; + AudioFrameInfo fake_audio_frame_info_; +}; + +class CustomRateCalculator : public OutputRateCalculator { + public: + explicit CustomRateCalculator(int rate) : rate_(rate) {} + int CalculateOutputRate(const std::vector& preferred_rates) override { + return rate_; + } + + private: + const int rate_; +}; + +// Creates participants from |frames| and |frame_info| and adds them +// to the mixer. Compares mixed status with |expected_status| +void MixAndCompare( + const std::vector& frames, + const std::vector& frame_info, + const std::vector& expected_status) { + const size_t num_audio_sources = frames.size(); + RTC_DCHECK(frames.size() == frame_info.size()); + RTC_DCHECK(frame_info.size() == expected_status.size()); + + const auto mixer = AudioMixerImpl::Create(); + std::vector participants(num_audio_sources); + + for (size_t i = 0; i < num_audio_sources; ++i) { + participants[i].fake_frame()->CopyFrom(frames[i]); + participants[i].set_fake_info(frame_info[i]); + } + + for (size_t i = 0; i < num_audio_sources; ++i) { + EXPECT_TRUE(mixer->AddSource(&participants[i])); + EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + } + + mixer->Mix(1, &frame_for_mixing); + + for (size_t i = 0; i < num_audio_sources; ++i) { + EXPECT_EQ(expected_status[i], + mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) + << "Mixed status of AudioSource #" << i << " wrong."; + } +} + +void MixMonoAtGivenNativeRate(int native_sample_rate, + AudioFrame* mix_frame, + rtc::scoped_refptr mixer, + MockMixerAudioSource* audio_source) { + ON_CALL(*audio_source, PreferredSampleRate()) + .WillByDefault(Return(native_sample_rate)); + audio_source->fake_frame()->sample_rate_hz_ = native_sample_rate; + audio_source->fake_frame()->samples_per_channel_ = native_sample_rate / 100; + + mixer->Mix(1, mix_frame); +} + +TEST(AudioMixer, LargestEnergyVadActiveMixed) { + constexpr int kAudioSources = + AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 3; + + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participants[kAudioSources]; + + for (int i = 0; i < kAudioSources; ++i) { + ResetFrame(participants[i].fake_frame()); + + // We set the 80-th sample value since the first 80 samples may be + // modified by a ramped-in window. + participants[i].fake_frame()->mutable_data()[80] = i; + + EXPECT_TRUE(mixer->AddSource(&participants[i])); + EXPECT_CALL(participants[i], GetAudioFrameWithInfo(_, _)).Times(Exactly(1)); + } + + // Last participant gives audio frame with passive VAD, although it has the + // largest energy. + participants[kAudioSources - 1].fake_frame()->vad_activity_ = + AudioFrame::kVadPassive; + + AudioFrame audio_frame; + mixer->Mix(1, // number of channels + &audio_frame); + + for (int i = 0; i < kAudioSources; ++i) { + bool is_mixed = + mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]); + if (i == kAudioSources - 1 || + i < kAudioSources - 1 - + AudioMixerImpl::kMaximumAmountOfMixedAudioSources) { + EXPECT_FALSE(is_mixed) << "Mixing status of AudioSource #" << i + << " wrong."; + } else { + EXPECT_TRUE(is_mixed) << "Mixing status of AudioSource #" << i + << " wrong."; + } + } +} + +TEST(AudioMixer, FrameNotModifiedForSingleParticipant) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participant; + + ResetFrame(participant.fake_frame()); + const size_t n_samples = participant.fake_frame()->samples_per_channel_; + + // Modify the frame so that it's not zero. + int16_t* fake_frame_data = participant.fake_frame()->mutable_data(); + for (size_t j = 0; j < n_samples; ++j) { + fake_frame_data[j] = static_cast(j); + } + + EXPECT_TRUE(mixer->AddSource(&participant)); + EXPECT_CALL(participant, GetAudioFrameWithInfo(_, _)).Times(Exactly(2)); + + AudioFrame audio_frame; + // Two mix iteration to compare after the ramp-up step. + for (int i = 0; i < 2; ++i) { + mixer->Mix(1, // number of channels + &audio_frame); + } + + EXPECT_EQ( + 0, + memcmp(participant.fake_frame()->data(), audio_frame.data(), n_samples)); +} + +TEST(AudioMixer, SourceAtNativeRateShouldNeverResample) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource audio_source; + ResetFrame(audio_source.fake_frame()); + + mixer->AddSource(&audio_source); + + for (auto frequency : {8000, 16000, 32000, 48000}) { + EXPECT_CALL(audio_source, GetAudioFrameWithInfo(frequency, _)) + .Times(Exactly(1)); + + MixMonoAtGivenNativeRate(frequency, &frame_for_mixing, mixer, + &audio_source); + } +} + +TEST(AudioMixer, MixerShouldMixAtNativeSourceRate) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource audio_source; + ResetFrame(audio_source.fake_frame()); + + mixer->AddSource(&audio_source); + + for (auto frequency : {8000, 16000, 32000, 48000}) { + MixMonoAtGivenNativeRate(frequency, &frame_for_mixing, mixer, + &audio_source); + + EXPECT_EQ(frequency, frame_for_mixing.sample_rate_hz_); + } +} + +TEST(AudioMixer, MixerShouldAlwaysMixAtNativeRate) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participant; + ResetFrame(participant.fake_frame()); + mixer->AddSource(&participant); + + const int needed_frequency = 44100; + ON_CALL(participant, PreferredSampleRate()) + .WillByDefault(Return(needed_frequency)); + + // We expect mixing frequency to be native and >= needed_frequency. + const int expected_mix_frequency = 48000; + EXPECT_CALL(participant, GetAudioFrameWithInfo(expected_mix_frequency, _)) + .Times(Exactly(1)); + participant.fake_frame()->sample_rate_hz_ = expected_mix_frequency; + participant.fake_frame()->samples_per_channel_ = expected_mix_frequency / 100; + + mixer->Mix(1, &frame_for_mixing); + + EXPECT_EQ(48000, frame_for_mixing.sample_rate_hz_); +} + +// Check that the mixing rate is always >= participants preferred rate. +TEST(AudioMixer, ShouldNotCauseQualityLossForMultipleSources) { + const auto mixer = AudioMixerImpl::Create(); + + std::vector audio_sources(2); + const std::vector source_sample_rates = {8000, 16000}; + for (int i = 0; i < 2; ++i) { + auto& source = audio_sources[i]; + ResetFrame(source.fake_frame()); + mixer->AddSource(&source); + const auto sample_rate = source_sample_rates[i]; + EXPECT_CALL(source, PreferredSampleRate()).WillOnce(Return(sample_rate)); + + EXPECT_CALL(source, GetAudioFrameWithInfo(testing::Ge(sample_rate), _)); + } + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, ParticipantNumberOfChannels) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participant; + ResetFrame(participant.fake_frame()); + + EXPECT_TRUE(mixer->AddSource(&participant)); + for (size_t number_of_channels : {1, 2}) { + EXPECT_CALL(participant, GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + mixer->Mix(number_of_channels, &frame_for_mixing); + EXPECT_EQ(number_of_channels, frame_for_mixing.num_channels_); + } +} + +// Maximal amount of participants are mixed one iteration, then +// another participant with higher energy is added. +TEST(AudioMixer, RampedOutSourcesShouldNotBeMarkedMixed) { + constexpr int kAudioSources = + AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + + const auto mixer = AudioMixerImpl::Create(); + MockMixerAudioSource participants[kAudioSources]; + + for (int i = 0; i < kAudioSources; ++i) { + ResetFrame(participants[i].fake_frame()); + // Set the participant audio energy to increase with the index + // |i|. + participants[i].fake_frame()->mutable_data()[0] = 100 * i; + } + + // Add all participants but the loudest for mixing. + for (int i = 0; i < kAudioSources - 1; ++i) { + EXPECT_TRUE(mixer->AddSource(&participants[i])); + EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + } + + // First mixer iteration + mixer->Mix(1, &frame_for_mixing); + + // All participants but the loudest should have been mixed. + for (int i = 0; i < kAudioSources - 1; ++i) { + EXPECT_TRUE(mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) + << "Mixed status of AudioSource #" << i << " wrong."; + } + + // Add new participant with higher energy. + EXPECT_TRUE(mixer->AddSource(&participants[kAudioSources - 1])); + for (int i = 0; i < kAudioSources; ++i) { + EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + } + + mixer->Mix(1, &frame_for_mixing); + + // The most quiet participant should not have been mixed. + EXPECT_FALSE(mixer->GetAudioSourceMixabilityStatusForTest(&participants[0])) + << "Mixed status of AudioSource #0 wrong."; + + // The loudest participants should have been mixed. + for (int i = 1; i < kAudioSources; ++i) { + EXPECT_EQ(true, + mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) + << "Mixed status of AudioSource #" << i << " wrong."; + } +} + +// This test checks that the initialization and participant addition +// can be done on a different thread. +TEST(AudioMixer, ConstructFromOtherThread) { + rtc::TaskQueue init_queue("init"); + rtc::scoped_refptr mixer; + rtc::Event event(false, false); + init_queue.PostTask([&mixer, &event]() { + mixer = AudioMixerImpl::Create(); + event.Set(); + }); + event.Wait(rtc::Event::kForever); + + MockMixerAudioSource participant; + EXPECT_CALL(participant, PreferredSampleRate()) + .WillRepeatedly(Return(kDefaultSampleRateHz)); + + ResetFrame(participant.fake_frame()); + + rtc::TaskQueue participant_queue("participant"); + participant_queue.PostTask([&mixer, &event, &participant]() { + mixer->AddSource(&participant); + event.Set(); + }); + event.Wait(rtc::Event::kForever); + + EXPECT_CALL(participant, GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + + // Do one mixer iteration + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, MutedShouldMixAfterUnmuted) { + constexpr int kAudioSources = + AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + + std::vector frames(kAudioSources); + for (auto& frame : frames) { + ResetFrame(&frame); + } + + std::vector frame_info( + kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); + frame_info[0] = AudioMixer::Source::AudioFrameInfo::kMuted; + std::vector expected_status(kAudioSources, true); + expected_status[0] = false; + + MixAndCompare(frames, frame_info, expected_status); +} + +TEST(AudioMixer, PassiveShouldMixAfterNormal) { + constexpr int kAudioSources = + AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + + std::vector frames(kAudioSources); + for (auto& frame : frames) { + ResetFrame(&frame); + } + + std::vector frame_info( + kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); + frames[0].vad_activity_ = AudioFrame::kVadPassive; + std::vector expected_status(kAudioSources, true); + expected_status[0] = false; + + MixAndCompare(frames, frame_info, expected_status); +} + +TEST(AudioMixer, ActiveShouldMixBeforeLoud) { + constexpr int kAudioSources = + AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + + std::vector frames(kAudioSources); + for (auto& frame : frames) { + ResetFrame(&frame); + } + + std::vector frame_info( + kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); + frames[0].vad_activity_ = AudioFrame::kVadPassive; + int16_t* frame_data = frames[0].mutable_data(); + std::fill(frame_data, frame_data + kDefaultSampleRateHz / 100, + std::numeric_limits::max()); + std::vector expected_status(kAudioSources, true); + expected_status[0] = false; + + MixAndCompare(frames, frame_info, expected_status); +} + +TEST(AudioMixer, UnmutedShouldMixBeforeLoud) { + constexpr int kAudioSources = + AudioMixerImpl::kMaximumAmountOfMixedAudioSources + 1; + + std::vector frames(kAudioSources); + for (auto& frame : frames) { + ResetFrame(&frame); + } + + std::vector frame_info( + kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); + frame_info[0] = AudioMixer::Source::AudioFrameInfo::kMuted; + int16_t* frame_data = frames[0].mutable_data(); + std::fill(frame_data, frame_data + kDefaultSampleRateHz / 100, + std::numeric_limits::max()); + std::vector expected_status(kAudioSources, true); + expected_status[0] = false; + + MixAndCompare(frames, frame_info, expected_status); +} + +TEST(AudioMixer, MixingRateShouldBeDecidedByRateCalculator) { + constexpr int kOutputRate = 22000; + const auto mixer = + AudioMixerImpl::Create(std::unique_ptr( + new CustomRateCalculator(kOutputRate)), + true); + MockMixerAudioSource audio_source; + mixer->AddSource(&audio_source); + ResetFrame(audio_source.fake_frame()); + + EXPECT_CALL(audio_source, GetAudioFrameWithInfo(kOutputRate, _)) + .Times(Exactly(1)); + + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, ZeroSourceRateShouldBeDecidedByRateCalculator) { + constexpr int kOutputRate = 8000; + const auto mixer = + AudioMixerImpl::Create(std::unique_ptr( + new CustomRateCalculator(kOutputRate)), + true); + + mixer->Mix(1, &frame_for_mixing); + + EXPECT_EQ(kOutputRate, frame_for_mixing.sample_rate_hz_); +} + +TEST(AudioMixer, NoLimiterBasicApiCalls) { + const auto mixer = AudioMixerImpl::Create( + std::unique_ptr(new DefaultOutputRateCalculator()), + false); + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, AnyRateIsPossibleWithNoLimiter) { + // No APM limiter means no AudioProcessing::NativeRate restriction + // on mixing rate. The rate has to be divisible by 100 since we use + // 10 ms frames, though. + for (const auto rate : {8000, 20000, 24000, 32000, 44100}) { + for (const size_t number_of_channels : {1, 2}) { + for (const auto number_of_sources : {0, 1, 2, 3, 4}) { + SCOPED_TRACE( + ProduceDebugText(rate, number_of_sources, number_of_sources)); + const auto mixer = + AudioMixerImpl::Create(std::unique_ptr( + new CustomRateCalculator(rate)), + false); + + std::vector sources(number_of_sources); + for (auto& source : sources) { + mixer->AddSource(&source); + } + + mixer->Mix(number_of_channels, &frame_for_mixing); + EXPECT_EQ(rate, frame_for_mixing.sample_rate_hz_); + EXPECT_EQ(number_of_channels, frame_for_mixing.num_channels_); + } + } + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.cc new file mode 100644 index 0000000000..05c7bc3366 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/default_output_rate_calculator.h" + +#include + +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { + +int DefaultOutputRateCalculator::CalculateOutputRate( + const std::vector& preferred_sample_rates) { + if (preferred_sample_rates.empty()) { + return DefaultOutputRateCalculator::kDefaultFrequency; + } + using NativeRate = AudioProcessing::NativeRate; + const int maximal_frequency = *std::max_element( + preferred_sample_rates.begin(), preferred_sample_rates.end()); + + RTC_DCHECK_LE(NativeRate::kSampleRate8kHz, maximal_frequency); + RTC_DCHECK_GE(NativeRate::kSampleRate48kHz, maximal_frequency); + + static constexpr NativeRate native_rates[] = { + NativeRate::kSampleRate8kHz, NativeRate::kSampleRate16kHz, + NativeRate::kSampleRate32kHz, NativeRate::kSampleRate48kHz}; + const auto* rounded_up_index = std::lower_bound( + std::begin(native_rates), std::end(native_rates), maximal_frequency); + RTC_DCHECK(rounded_up_index != std::end(native_rates)); + return *rounded_up_index; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.h b/third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.h new file mode 100644 index 0000000000..2d34f3489c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/default_output_rate_calculator.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_DEFAULT_OUTPUT_RATE_CALCULATOR_H_ +#define MODULES_AUDIO_MIXER_DEFAULT_OUTPUT_RATE_CALCULATOR_H_ + +#include + +#include "modules/audio_mixer/output_rate_calculator.h" + +namespace webrtc { + +class DefaultOutputRateCalculator : public OutputRateCalculator { + public: + static const int kDefaultFrequency = 48000; + + // Produces the least native rate greater or equal to the preferred + // sample rates. A native rate is one in + // AudioProcessing::NativeRate. If |preferred_sample_rates| is + // empty, returns |kDefaultFrequency|. + int CalculateOutputRate( + const std::vector& preferred_sample_rates) override; + ~DefaultOutputRateCalculator() override {} +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_DEFAULT_OUTPUT_RATE_CALCULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.cc new file mode 100644 index 0000000000..09faf56874 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.cc @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/frame_combiner.h" + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "audio/utility/audio_frame_operations.h" +#include "modules/audio_mixer/audio_frame_manipulator.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +// Stereo, 48 kHz, 10 ms. +constexpr int kMaximalFrameSize = 2 * 48 * 10; + +void CombineZeroFrames(bool use_limiter, + AudioProcessing* limiter, + AudioFrame* audio_frame_for_mixing) { + audio_frame_for_mixing->elapsed_time_ms_ = -1; + AudioFrameOperations::Mute(audio_frame_for_mixing); + // The limiter should still process a zero frame to avoid jumps in + // its gain curve. + if (use_limiter) { + RTC_DCHECK(limiter); + // The limiter smoothly increases frames with half gain to full + // volume. Here there's no need to apply half gain, since the frame + // is zero anyway. + limiter->ProcessStream(audio_frame_for_mixing); + } +} + +void CombineOneFrame(const AudioFrame* input_frame, + bool use_limiter, + AudioProcessing* limiter, + AudioFrame* audio_frame_for_mixing) { + audio_frame_for_mixing->timestamp_ = input_frame->timestamp_; + audio_frame_for_mixing->elapsed_time_ms_ = input_frame->elapsed_time_ms_; + // TODO(yujo): can we optimize muted frames? + std::copy(input_frame->data(), + input_frame->data() + + input_frame->num_channels_ * input_frame->samples_per_channel_, + audio_frame_for_mixing->mutable_data()); + if (use_limiter) { + AudioFrameOperations::ApplyHalfGain(audio_frame_for_mixing); + RTC_DCHECK(limiter); + limiter->ProcessStream(audio_frame_for_mixing); + AudioFrameOperations::Add(*audio_frame_for_mixing, audio_frame_for_mixing); + } +} + +// Lower-level helper function called from Combine(...) when there +// are several input frames. +// +// TODO(aleloi): change interface to ArrayView output_frame +// once we have gotten rid of the APM limiter. +// +// Only the 'data' field of output_frame should be modified. The +// rest are used for potentially sending the output to the APM +// limiter. +void CombineMultipleFrames( + const std::vector>& input_frames, + bool use_limiter, + AudioProcessing* limiter, + AudioFrame* audio_frame_for_mixing) { + RTC_DCHECK(!input_frames.empty()); + RTC_DCHECK(audio_frame_for_mixing); + + const size_t frame_length = input_frames.front().size(); + for (const auto& frame : input_frames) { + RTC_DCHECK_EQ(frame_length, frame.size()); + } + + // Algorithm: int16 frames are added to a sufficiently large + // statically allocated int32 buffer. For > 2 participants this is + // more efficient than addition in place in the int16 audio + // frame. The audio quality loss due to halving the samples is + // smaller than 16-bit addition in place. + RTC_DCHECK_GE(kMaximalFrameSize, frame_length); + std::array add_buffer; + + add_buffer.fill(0); + + for (const auto& frame : input_frames) { + // TODO(yujo): skip this for muted frames. + std::transform(frame.begin(), frame.end(), add_buffer.begin(), + add_buffer.begin(), std::plus()); + } + + if (use_limiter) { + // Halve all samples to avoid saturation before limiting. + std::transform(add_buffer.begin(), add_buffer.begin() + frame_length, + audio_frame_for_mixing->mutable_data(), [](int32_t a) { + return rtc::saturated_cast(a / 2); + }); + + // Smoothly limit the audio. + RTC_DCHECK(limiter); + const int error = limiter->ProcessStream(audio_frame_for_mixing); + if (error != limiter->kNoError) { + RTC_LOG_F(LS_ERROR) << "Error from AudioProcessing: " << error; + RTC_NOTREACHED(); + } + + // And now we can safely restore the level. This procedure results in + // some loss of resolution, deemed acceptable. + // + // It's possible to apply the gain in the AGC (with a target level of 0 dbFS + // and compression gain of 6 dB). However, in the transition frame when this + // is enabled (moving from one to two audio sources) it has the potential to + // create discontinuities in the mixed frame. + // + // Instead we double the frame (with addition since left-shifting a + // negative value is undefined). + AudioFrameOperations::Add(*audio_frame_for_mixing, audio_frame_for_mixing); + } else { + std::transform(add_buffer.begin(), add_buffer.begin() + frame_length, + audio_frame_for_mixing->mutable_data(), + [](int32_t a) { return rtc::saturated_cast(a); }); + } +} + +std::unique_ptr CreateLimiter() { + Config config; + config.Set(new ExperimentalAgc(false)); + + std::unique_ptr limiter(AudioProcessing::Create(config)); + RTC_DCHECK(limiter); + + webrtc::AudioProcessing::Config apm_config; + apm_config.residual_echo_detector.enabled = false; + limiter->ApplyConfig(apm_config); + + const auto check_no_error = [](int x) { + RTC_DCHECK_EQ(x, AudioProcessing::kNoError); + }; + auto* const gain_control = limiter->gain_control(); + check_no_error(gain_control->set_mode(GainControl::kFixedDigital)); + + // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the + // divide-by-2 but -7 is used instead to give a bit of headroom since the + // AGC is not a hard limiter. + check_no_error(gain_control->set_target_level_dbfs(7)); + + check_no_error(gain_control->set_compression_gain_db(0)); + check_no_error(gain_control->enable_limiter(true)); + check_no_error(gain_control->Enable(true)); + return limiter; +} +} // namespace + +FrameCombiner::FrameCombiner(bool use_apm_limiter) + : use_apm_limiter_(use_apm_limiter), + limiter_(use_apm_limiter ? CreateLimiter() : nullptr) {} + +FrameCombiner::~FrameCombiner() = default; + +void FrameCombiner::Combine(const std::vector& mix_list, + size_t number_of_channels, + int sample_rate, + size_t number_of_streams, + AudioFrame* audio_frame_for_mixing) const { + RTC_DCHECK(audio_frame_for_mixing); + const size_t samples_per_channel = static_cast( + (sample_rate * webrtc::AudioMixerImpl::kFrameDurationInMs) / 1000); + + for (const auto* frame : mix_list) { + RTC_DCHECK_EQ(samples_per_channel, frame->samples_per_channel_); + RTC_DCHECK_EQ(sample_rate, frame->sample_rate_hz_); + } + + // Frames could be both stereo and mono. + for (auto* frame : mix_list) { + RemixFrame(number_of_channels, frame); + } + + // TODO(aleloi): Issue bugs.webrtc.org/3390. + // Audio frame timestamp. The 'timestamp_' field is set to dummy + // value '0', because it is only supported in the one channel case and + // is then updated in the helper functions. + audio_frame_for_mixing->UpdateFrame( + 0, nullptr, samples_per_channel, sample_rate, AudioFrame::kUndefined, + AudioFrame::kVadUnknown, number_of_channels); + + const bool use_limiter_this_round = use_apm_limiter_ && number_of_streams > 1; + + if (mix_list.empty()) { + CombineZeroFrames(use_limiter_this_round, limiter_.get(), + audio_frame_for_mixing); + } else if (mix_list.size() == 1) { + CombineOneFrame(mix_list.front(), use_limiter_this_round, limiter_.get(), + audio_frame_for_mixing); + } else { + std::vector> input_frames; + for (size_t i = 0; i < mix_list.size(); ++i) { + input_frames.push_back(rtc::ArrayView( + mix_list[i]->data(), samples_per_channel * number_of_channels)); + } + CombineMultipleFrames(input_frames, use_limiter_this_round, limiter_.get(), + audio_frame_for_mixing); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.h b/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.h new file mode 100644 index 0000000000..88ab0d7f9a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_FRAME_COMBINER_H_ +#define MODULES_AUDIO_MIXER_FRAME_COMBINER_H_ + +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/include/module_common_types.h" + +namespace webrtc { + +class FrameCombiner { + public: + explicit FrameCombiner(bool use_apm_limiter); + ~FrameCombiner(); + + // Combine several frames into one. Assumes sample_rate, + // samples_per_channel of the input frames match the parameters. The + // parameters 'number_of_channels' and 'sample_rate' are needed + // because 'mix_list' can be empty. The parameter + // 'number_of_streams' is used for determining whether to pass the + // data through a limiter. + void Combine(const std::vector& mix_list, + size_t number_of_channels, + int sample_rate, + size_t number_of_streams, + AudioFrame* audio_frame_for_mixing) const; + + private: + const bool use_apm_limiter_; + std::unique_ptr limiter_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_FRAME_COMBINER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner_unittest.cc new file mode 100644 index 0000000000..490e99e495 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/frame_combiner_unittest.cc @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/frame_combiner.h" + +#include +#include +#include + +#include "audio/utility/audio_frame_operations.h" +#include "modules/audio_mixer/gain_change_calculator.h" +#include "modules/audio_mixer/sine_wave_generator.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +std::string ProduceDebugText(int sample_rate_hz, + int number_of_channels, + int number_of_sources) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz << " ,"; + ss << "number of channels: " << number_of_channels << " ,"; + ss << "number of sources: " << number_of_sources; + return ss.str(); +} + +std::string ProduceDebugText(int sample_rate_hz, + int number_of_channels, + int number_of_sources, + bool limiter_active, + float wave_frequency) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz << " ,"; + ss << "number of channels: " << number_of_channels << " ,"; + ss << "number of sources: " << number_of_sources << " ,"; + ss << "limiter active: " << (limiter_active ? "true" : "false") << " ,"; + ss << "wave frequency: " << wave_frequency << " ,"; + return ss.str(); +} + +AudioFrame frame1; +AudioFrame frame2; +AudioFrame audio_frame_for_mixing; + +void SetUpFrames(int sample_rate_hz, int number_of_channels) { + for (auto* frame : {&frame1, &frame2}) { + frame->UpdateFrame(0, nullptr, rtc::CheckedDivExact(sample_rate_hz, 100), + sample_rate_hz, AudioFrame::kNormalSpeech, + AudioFrame::kVadActive, number_of_channels); + } +} +} // namespace + +TEST(FrameCombiner, BasicApiCallsLimiter) { + FrameCombiner combiner(true); + for (const int rate : {8000, 16000, 32000, 48000}) { + for (const int number_of_channels : {1, 2}) { + const std::vector all_frames = {&frame1, &frame2}; + SetUpFrames(rate, number_of_channels); + + for (const int number_of_frames : {0, 1, 2}) { + SCOPED_TRACE( + ProduceDebugText(rate, number_of_channels, number_of_frames)); + const std::vector frames_to_combine( + all_frames.begin(), all_frames.begin() + number_of_frames); + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + } + } + } +} + +// No APM limiter means no AudioProcessing::NativeRate restriction +// on rate. The rate has to be divisible by 100 since we use +// 10 ms frames, though. +TEST(FrameCombiner, BasicApiCallsNoLimiter) { + FrameCombiner combiner(false); + for (const int rate : {8000, 10000, 11000, 32000, 44100}) { + for (const int number_of_channels : {1, 2}) { + const std::vector all_frames = {&frame1, &frame2}; + SetUpFrames(rate, number_of_channels); + + for (const int number_of_frames : {0, 1, 2}) { + SCOPED_TRACE( + ProduceDebugText(rate, number_of_channels, number_of_frames)); + const std::vector frames_to_combine( + all_frames.begin(), all_frames.begin() + number_of_frames); + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + } + } + } +} + +TEST(FrameCombiner, CombiningZeroFramesShouldProduceSilence) { + FrameCombiner combiner(false); + for (const int rate : {8000, 10000, 11000, 32000, 44100}) { + for (const int number_of_channels : {1, 2}) { + SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 0)); + + const std::vector frames_to_combine; + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + + const int16_t* audio_frame_for_mixing_data = + audio_frame_for_mixing.data(); + const std::vector mixed_data( + audio_frame_for_mixing_data, + audio_frame_for_mixing_data + number_of_channels * rate / 100); + + const std::vector expected(number_of_channels * rate / 100, 0); + EXPECT_EQ(mixed_data, expected); + } + } +} + +TEST(FrameCombiner, CombiningOneFrameShouldNotChangeFrame) { + FrameCombiner combiner(false); + for (const int rate : {8000, 10000, 11000, 32000, 44100}) { + for (const int number_of_channels : {1, 2}) { + SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 1)); + + SetUpFrames(rate, number_of_channels); + int16_t* frame1_data = frame1.mutable_data(); + std::iota(frame1_data, frame1_data + number_of_channels * rate / 100, 0); + const std::vector frames_to_combine = {&frame1}; + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + + const int16_t* audio_frame_for_mixing_data = + audio_frame_for_mixing.data(); + const std::vector mixed_data( + audio_frame_for_mixing_data, + audio_frame_for_mixing_data + number_of_channels * rate / 100); + + std::vector expected(number_of_channels * rate / 100); + std::iota(expected.begin(), expected.end(), 0); + EXPECT_EQ(mixed_data, expected); + } + } +} + +// Send a sine wave through the FrameCombiner, and check that the +// difference between input and output varies smoothly. This is to +// catch issues like chromium:695993. +TEST(FrameCombiner, GainCurveIsSmoothForAlternatingNumberOfStreams) { + // Test doesn't work with rates requiring a band split, because it + // introduces a small delay measured in single samples, and this + // test cannot handle it. + // + // TODO(aleloi): Add more rates when APM limiter doesn't use band + // split. + for (const bool use_limiter : {true, false}) { + for (const int rate : {8000, 16000}) { + constexpr int number_of_channels = 2; + for (const float wave_frequency : {50, 400, 3200}) { + SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 1, use_limiter, + wave_frequency)); + + FrameCombiner combiner(use_limiter); + + constexpr int16_t wave_amplitude = 30000; + SineWaveGenerator wave_generator(wave_frequency, wave_amplitude); + + GainChangeCalculator change_calculator; + float cumulative_change = 0.f; + + constexpr size_t iterations = 100; + + for (size_t i = 0; i < iterations; ++i) { + SetUpFrames(rate, number_of_channels); + wave_generator.GenerateNextFrame(&frame1); + AudioFrameOperations::Mute(&frame2); + + std::vector frames_to_combine = {&frame1}; + if (i % 2 == 0) { + frames_to_combine.push_back(&frame2); + } + const size_t number_of_samples = + frame1.samples_per_channel_ * number_of_channels; + + // Ensures limiter is on if 'use_limiter'. + constexpr size_t number_of_streams = 2; + combiner.Combine(frames_to_combine, number_of_channels, rate, + number_of_streams, &audio_frame_for_mixing); + cumulative_change += change_calculator.CalculateGainChange( + rtc::ArrayView(frame1.data(), number_of_samples), + rtc::ArrayView(audio_frame_for_mixing.data(), + number_of_samples)); + } + RTC_DCHECK_LT(cumulative_change, 10); + } + } + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.cc new file mode 100644 index 0000000000..ae7c6d59e7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/gain_change_calculator.h" + +#include +#include + +namespace webrtc { + +namespace { +constexpr int16_t kReliabilityThreshold = 100; +} // namespace + +float GainChangeCalculator::CalculateGainChange( + rtc::ArrayView in, + rtc::ArrayView out) { + RTC_DCHECK_EQ(in.size(), out.size()); + + std::vector gain(in.size()); + CalculateGain(in, out, gain); + return CalculateDifferences(gain); +} + +void GainChangeCalculator::CalculateGain(rtc::ArrayView in, + rtc::ArrayView out, + rtc::ArrayView gain) { + RTC_DCHECK_EQ(in.size(), out.size()); + RTC_DCHECK_EQ(in.size(), gain.size()); + + for (size_t i = 0; i < in.size(); ++i) { + if (std::abs(in[i]) >= kReliabilityThreshold) { + last_reliable_gain_ = out[i] / static_cast(in[i]); + } + gain[i] = last_reliable_gain_; + } +} + +float GainChangeCalculator::CalculateDifferences( + rtc::ArrayView values) { + float res = 0; + for (float f : values) { + res += fabs(f - last_value_); + last_value_ = f; + } + return res; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.h b/third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.h new file mode 100644 index 0000000000..fe7957cdd5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/gain_change_calculator.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_GAIN_CHANGE_CALCULATOR_H_ +#define MODULES_AUDIO_MIXER_GAIN_CHANGE_CALCULATOR_H_ + +#include "api/array_view.h" + +namespace webrtc { + +class GainChangeCalculator { + public: + // The 'out' signal is assumed to be produced from 'in' by applying + // a smoothly varying gain. This method computes variations of the + // gain and handles special cases when the samples are small. + float CalculateGainChange(rtc::ArrayView in, + rtc::ArrayView out); + + private: + void CalculateGain(rtc::ArrayView in, + rtc::ArrayView out, + rtc::ArrayView gain); + + float CalculateDifferences(rtc::ArrayView values); + float last_value_ = 0.f; + float last_reliable_gain_ = 1.0f; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_GAIN_CHANGE_CALCULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/output_rate_calculator.h b/third_party/libwebrtc/webrtc/modules/audio_mixer/output_rate_calculator.h new file mode 100644 index 0000000000..cb3ca96b2d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/output_rate_calculator.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_OUTPUT_RATE_CALCULATOR_H_ +#define MODULES_AUDIO_MIXER_OUTPUT_RATE_CALCULATOR_H_ + +#include + +namespace webrtc { + +// Decides the sample rate of a mixing iteration given the preferred +// sample rates of the sources. +class OutputRateCalculator { + public: + virtual int CalculateOutputRate( + const std::vector& preferred_sample_rates) = 0; + virtual ~OutputRateCalculator() {} +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_OUTPUT_RATE_CALCULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.cc b/third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.cc new file mode 100644 index 0000000000..505d637284 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.cc @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/sine_wave_generator.h" + +#include + +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { +constexpr float kPi = 3.14159265f; +} // namespace + +void SineWaveGenerator::GenerateNextFrame(AudioFrame* frame) { + RTC_DCHECK(frame); + int16_t* frame_data = frame->mutable_data(); + for (size_t i = 0; i < frame->samples_per_channel_; ++i) { + for (size_t ch = 0; ch < frame->num_channels_; ++ch) { + frame_data[frame->num_channels_ * i + ch] = + rtc::saturated_cast(amplitude_ * sinf(phase_)); + } + phase_ += wave_frequency_hz_ * 2 * kPi / frame->sample_rate_hz_; + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.h b/third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.h new file mode 100644 index 0000000000..b5c4674e20 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_mixer/sine_wave_generator.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_SINE_WAVE_GENERATOR_H_ +#define MODULES_AUDIO_MIXER_SINE_WAVE_GENERATOR_H_ + +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +class SineWaveGenerator { + public: + SineWaveGenerator(float wave_frequency_hz, int16_t amplitude) + : wave_frequency_hz_(wave_frequency_hz), amplitude_(amplitude) { + RTC_DCHECK_GT(wave_frequency_hz, 0); + } + + // Produces appropriate output based on frame->num_channels_, + // frame->sample_rate_hz_. + void GenerateNextFrame(AudioFrame* frame); + + private: + float phase_ = 0.f; + const float wave_frequency_hz_; + const int16_t amplitude_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_SINE_WAVE_GENERATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/BUILD.gn b/third_party/libwebrtc/webrtc/modules/audio_processing/BUILD.gn new file mode 100644 index 0000000000..23eefc22c7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/BUILD.gn @@ -0,0 +1,905 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("//build/config/arm.gni") +import("../../webrtc.gni") +if (!build_with_mozilla) { + import("//third_party/protobuf/proto_library.gni") +} + +declare_args() { + # Disables the usual mode where we trust the reported system delay + # values the AEC receives. The corresponding define is set appropriately + # in the code, but it can be force-enabled here for testing. + aec_untrusted_delay_for_testing = false +} + +rtc_static_library("audio_processing") { + sources = [ + "aec/aec_core.cc", + "aec/aec_core.h", + "aec/aec_core_optimized_methods.h", + "aec/aec_resampler.cc", + "aec/aec_resampler.h", + "aec/echo_cancellation.cc", + "aec/echo_cancellation.h", + "aec3/adaptive_fir_filter.cc", + "aec3/adaptive_fir_filter.h", + "aec3/aec3_common.cc", + "aec3/aec3_common.h", + "aec3/aec3_fft.cc", + "aec3/aec3_fft.h", + "aec3/aec_state.cc", + "aec3/aec_state.h", + "aec3/block_framer.cc", + "aec3/block_framer.h", + "aec3/block_processor.cc", + "aec3/block_processor.h", + "aec3/block_processor_metrics.cc", + "aec3/block_processor_metrics.h", + "aec3/cascaded_biquad_filter.cc", + "aec3/cascaded_biquad_filter.h", + "aec3/comfort_noise_generator.cc", + "aec3/comfort_noise_generator.h", + "aec3/decimator.cc", + "aec3/decimator.h", + "aec3/downsampled_render_buffer.cc", + "aec3/downsampled_render_buffer.h", + "aec3/echo_canceller3.cc", + "aec3/echo_canceller3.h", + "aec3/echo_path_delay_estimator.cc", + "aec3/echo_path_delay_estimator.h", + "aec3/echo_path_variability.cc", + "aec3/echo_path_variability.h", + "aec3/echo_remover.cc", + "aec3/echo_remover.h", + "aec3/echo_remover_metrics.cc", + "aec3/echo_remover_metrics.h", + "aec3/erl_estimator.cc", + "aec3/erl_estimator.h", + "aec3/erle_estimator.cc", + "aec3/erle_estimator.h", + "aec3/fft_data.h", + "aec3/frame_blocker.cc", + "aec3/frame_blocker.h", + "aec3/main_filter_update_gain.cc", + "aec3/main_filter_update_gain.h", + "aec3/matched_filter.cc", + "aec3/matched_filter.h", + "aec3/matched_filter_lag_aggregator.cc", + "aec3/matched_filter_lag_aggregator.h", + "aec3/output_selector.cc", + "aec3/output_selector.h", + "aec3/render_buffer.cc", + "aec3/render_buffer.h", + "aec3/render_delay_buffer.cc", + "aec3/render_delay_buffer.h", + "aec3/render_delay_controller.cc", + "aec3/render_delay_controller.h", + "aec3/render_delay_controller_metrics.cc", + "aec3/render_delay_controller_metrics.h", + "aec3/render_signal_analyzer.cc", + "aec3/render_signal_analyzer.h", + "aec3/residual_echo_estimator.cc", + "aec3/residual_echo_estimator.h", + "aec3/shadow_filter_update_gain.cc", + "aec3/shadow_filter_update_gain.h", + "aec3/subtractor.cc", + "aec3/subtractor.h", + "aec3/subtractor_output.h", + "aec3/suppression_filter.cc", + "aec3/suppression_filter.h", + "aec3/suppression_gain.cc", + "aec3/suppression_gain.h", + "aec3/vector_math.h", + "aecm/aecm_core.cc", + "aecm/aecm_core.h", + "aecm/echo_control_mobile.cc", + "aecm/echo_control_mobile.h", + "agc/agc.cc", + "agc/agc.h", + "agc/agc_manager_direct.cc", + "agc/agc_manager_direct.h", + "agc/gain_map_internal.h", + "agc/loudness_histogram.cc", + "agc/loudness_histogram.h", + "agc/utility.cc", + "agc/utility.h", + "agc2/gain_controller2.cc", + "agc2/gain_controller2.h", + "audio_buffer.cc", + "audio_buffer.h", + "audio_processing_impl.cc", + "audio_processing_impl.h", + "beamformer/array_util.cc", + "beamformer/array_util.h", + "beamformer/complex_matrix.h", + "beamformer/covariance_matrix_generator.cc", + "beamformer/covariance_matrix_generator.h", + "beamformer/matrix.h", + "beamformer/nonlinear_beamformer.cc", + "beamformer/nonlinear_beamformer.h", + "common.h", + "echo_cancellation_impl.cc", + "echo_cancellation_impl.h", + "echo_control_mobile_impl.cc", + "echo_control_mobile_impl.h", + "echo_detector/circular_buffer.cc", + "echo_detector/circular_buffer.h", + "echo_detector/mean_variance_estimator.cc", + "echo_detector/mean_variance_estimator.h", + "echo_detector/moving_max.cc", + "echo_detector/moving_max.h", + "echo_detector/normalized_covariance_estimator.cc", + "echo_detector/normalized_covariance_estimator.h", + "gain_control_for_experimental_agc.cc", + "gain_control_for_experimental_agc.h", + "gain_control_impl.cc", + "gain_control_impl.h", + "include/audio_processing.cc", + "include/audio_processing.h", + "include/config.cc", + "include/config.h", + "level_controller/biquad_filter.cc", + "level_controller/biquad_filter.h", + "level_controller/down_sampler.cc", + "level_controller/down_sampler.h", + "level_controller/gain_applier.cc", + "level_controller/gain_applier.h", + "level_controller/gain_selector.cc", + "level_controller/gain_selector.h", + "level_controller/level_controller.cc", + "level_controller/level_controller.h", + "level_controller/level_controller_constants.h", + "level_controller/noise_level_estimator.cc", + "level_controller/noise_level_estimator.h", + "level_controller/noise_spectrum_estimator.cc", + "level_controller/noise_spectrum_estimator.h", + "level_controller/peak_level_estimator.cc", + "level_controller/peak_level_estimator.h", + "level_controller/saturating_gain_estimator.cc", + "level_controller/saturating_gain_estimator.h", + "level_controller/signal_classifier.cc", + "level_controller/signal_classifier.h", + "level_estimator_impl.cc", + "level_estimator_impl.h", + "logging/apm_data_dumper.cc", + "logging/apm_data_dumper.h", + "low_cut_filter.cc", + "low_cut_filter.h", + "noise_suppression_impl.cc", + "noise_suppression_impl.h", + "render_queue_item_verifier.h", + "residual_echo_detector.cc", + "residual_echo_detector.h", + "rms_level.cc", + "rms_level.h", + "splitting_filter.cc", + "splitting_filter.h", + "three_band_filter_bank.cc", + "three_band_filter_bank.h", + "transient/common.h", + "transient/daubechies_8_wavelet_coeffs.h", + "transient/dyadic_decimator.h", + "transient/moving_moments.cc", + "transient/moving_moments.h", + "transient/transient_detector.cc", + "transient/transient_detector.h", + "transient/transient_suppressor.cc", + "transient/transient_suppressor.h", + "transient/wpd_node.cc", + "transient/wpd_node.h", + "transient/wpd_tree.cc", + "transient/wpd_tree.h", + "typing_detection.cc", + "typing_detection.h", + "utility/block_mean_calculator.cc", + "utility/block_mean_calculator.h", + "utility/delay_estimator.cc", + "utility/delay_estimator.h", + "utility/delay_estimator_internal.h", + "utility/delay_estimator_wrapper.cc", + "utility/delay_estimator_wrapper.h", + "utility/ooura_fft.cc", + "utility/ooura_fft.h", + "utility/ooura_fft_tables_common.h", + "vad/common.h", + "vad/gmm.cc", + "vad/gmm.h", + "vad/noise_gmm_tables.h", + "vad/pitch_based_vad.cc", + "vad/pitch_based_vad.h", + "vad/pitch_internal.cc", + "vad/pitch_internal.h", + "vad/pole_zero_filter.cc", + "vad/pole_zero_filter.h", + "vad/standalone_vad.cc", + "vad/standalone_vad.h", + "vad/vad_audio_proc.cc", + "vad/vad_audio_proc.h", + "vad/vad_audio_proc_internal.h", + "vad/vad_circular_buffer.cc", + "vad/vad_circular_buffer.h", + "vad/voice_activity_detector.cc", + "vad/voice_activity_detector.h", + "vad/voice_gmm_tables.h", + "voice_detection_impl.cc", + "voice_detection_impl.h", + ] + + defines = [] + deps = [ + ":aec_dump_interface", + ":audio_processing_statistics", + "..:module_api", + "../..:webrtc_common", + "../../api:array_view", + "../../api:optional", + "../../audio/utility:audio_frame_operations", + "../../rtc_base:gtest_prod", + "../../rtc_base:protobuf_utils", + "../../system_wrappers:field_trial_api", + "../audio_coding:isac", + ] + public_deps = [ + ":audio_processing_c", + ] + + if (apm_debug_dump) { + defines += [ "WEBRTC_APM_DEBUG_DUMP=1" ] + } else { + defines += [ "WEBRTC_APM_DEBUG_DUMP=0" ] + } + + if (aec_untrusted_delay_for_testing) { + defines += [ "WEBRTC_UNTRUSTED_DELAY" ] + } + + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_AUDIOPROC_DEBUG_DUMP" ] + deps += [ ":audioproc_debug_proto" ] + } + + if (rtc_enable_intelligibility_enhancer) { + defines += [ "WEBRTC_INTELLIGIBILITY_ENHANCER=1" ] + sources += [ + "intelligibility/intelligibility_enhancer.cc", + "intelligibility/intelligibility_enhancer.h", + "intelligibility/intelligibility_utils.cc", + "intelligibility/intelligibility_utils.h", + ] + } else { + defines += [ "WEBRTC_INTELLIGIBILITY_ENHANCER=0" ] + } + + if (rtc_prefer_fixed_point) { + defines += [ "WEBRTC_NS_FIXED" ] + } else { + defines += [ "WEBRTC_NS_FLOAT" ] + } + + if (current_cpu == "x86" || current_cpu == "x64") { + deps += [ ":audio_processing_sse2" ] + } + + if (rtc_build_with_neon) { + deps += [ ":audio_processing_neon" ] + } + + if (current_cpu == "mipsel") { + sources += [ "aecm/aecm_core_mips.cc" ] + if (mips_float_abi == "hard") { + sources += [ + "aec/aec_core_mips.cc", + "utility/ooura_fft_mips.cc", + ] + } + } else { + sources += [ "aecm/aecm_core_c.cc" ] + } + + # TODO(jschuh): Bug 1348: fix this warning. + configs += [ "//build/config/compiler:no_size_t_to_int_warning" ] + + deps += [ + "../../common_audio", + "../../common_audio:fir_filter", + "../../common_audio:fir_filter_factory", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] +} + +rtc_source_set("audio_processing_statistics") { + sources = [ + "include/audio_processing_statistics.cc", + "include/audio_processing_statistics.h", + ] + deps = [ + "../../api:optional", + ] +} + +rtc_source_set("aec_dump_interface") { + sources = [ + "include/aec_dump.cc", + "include/aec_dump.h", + ] + + deps = [ + "../../api:array_view", + "../../rtc_base:rtc_base_approved", + ] +} + +rtc_source_set("audio_processing_c") { + visibility = [ ":*" ] # Only targets in this file can depend on this. + sources = [ + "agc/legacy/analog_agc.c", + "agc/legacy/analog_agc.h", + "agc/legacy/digital_agc.c", + "agc/legacy/digital_agc.h", + "agc/legacy/gain_control.h", + ] + + if (rtc_prefer_fixed_point) { + sources += [ + "ns/noise_suppression_x.c", + "ns/noise_suppression_x.h", + "ns/nsx_core.c", + "ns/nsx_core.h", + "ns/nsx_defines.h", + ] + if (current_cpu == "mipsel") { + sources += [ "ns/nsx_core_mips.c" ] + } else { + sources += [ "ns/nsx_core_c.c" ] + } + } else { + sources += [ + "ns/defines.h", + "ns/noise_suppression.c", + "ns/noise_suppression.h", + "ns/ns_core.c", + "ns/ns_core.h", + "ns/windows_private.h", + ] + } + + deps = [ + "../..:webrtc_common", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] + + if (rtc_build_with_neon) { + deps += [ ":audio_processing_neon_c" ] + } +} + +if (rtc_enable_protobuf) { + proto_library("audioproc_debug_proto") { + sources = [ + "debug.proto", + ] + + proto_out_dir = "modules/audio_processing" + } +} + +if (current_cpu == "x86" || current_cpu == "x64") { + rtc_static_library("audio_processing_sse2") { + # TODO(ehmaldonado): Remove (bugs.webrtc.org/6828) + # Errors on cyclic dependency with :audio_processing if enabled. + check_includes = false + + sources = [ + "aec/aec_core_sse2.cc", + "utility/ooura_fft_sse2.cc", + "utility/ooura_fft_tables_neon_sse2.h", + ] + + if (is_posix) { + cflags = [ "-msse2" ] + } + + if (apm_debug_dump) { + defines = [ "WEBRTC_APM_DEBUG_DUMP=1" ] + } else { + defines = [ "WEBRTC_APM_DEBUG_DUMP=0" ] + } + } +} + +if (rtc_build_with_neon) { + rtc_static_library("audio_processing_neon") { + # TODO(ehmaldonado): Remove (bugs.webrtc.org/6828) + # Errors on cyclic dependency with :audio_processing if enabled. + check_includes = false + + sources = [ + "aec/aec_core_neon.cc", + "aecm/aecm_core_neon.cc", + "utility/ooura_fft_neon.cc", + "utility/ooura_fft_tables_neon_sse2.h", + ] + + if (current_cpu != "arm64") { + # Enable compilation for the NEON instruction set. This is needed + # since //build/config/arm.gni only enables NEON for iOS, not Android. + # This provides the same functionality as webrtc/build/arm_neon.gypi. + suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] + cflags = [ "-mfpu=neon" ] + } + + # Disable LTO on NEON targets due to compiler bug. + # TODO(fdegans): Enable this. See crbug.com/408997. + if (rtc_use_lto) { + cflags -= [ + "-flto", + "-ffat-lto-objects", + ] + } + + deps = [ + "../../common_audio", + ] + public_deps = [ + ":audio_processing_neon_c", + ] + + if (apm_debug_dump) { + defines = [ "WEBRTC_APM_DEBUG_DUMP=1" ] + } else { + defines = [ "WEBRTC_APM_DEBUG_DUMP=0" ] + } + } + + rtc_static_library("audio_processing_neon_c") { + # TODO(mbonadei): Remove (bugs.webrtc.org/6828) + # Errors on cyclic dependency with :audio_processing_c if enabled. + check_includes = false + + sources = [ + "ns/nsx_core_neon.c", + ] + + if (current_cpu != "arm64") { + # Enable compilation for the NEON instruction set. This is needed + # since //build/config/arm.gni only enables NEON for iOS, not Android. + # This provides the same functionality as webrtc/build/arm_neon.gypi. + suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] + cflags = [ "-mfpu=neon" ] + } + + # Disable LTO on NEON targets due to compiler bug. + # TODO(fdegans): Enable this. See crbug.com/408997. + if (rtc_use_lto) { + cflags -= [ + "-flto", + "-ffat-lto-objects", + ] + } + deps = [ + "../../rtc_base:rtc_base_approved", + ] + } +} + +if (rtc_include_tests) { + group("audio_processing_tests") { + testonly = true + public_deps = [ + ":audioproc_test_utils", + ":click_annotate", + ":nonlinear_beamformer_test", + ":transient_suppression_test", + ] + + if (rtc_enable_intelligibility_enhancer) { + public_deps += [ ":intelligibility_proc" ] + } + + if (rtc_enable_protobuf) { + public_deps += [ + ":audioproc_f", + ":audioproc_unittest_proto", + ":unpack_aecdump", + "aec_dump:aec_dump_unittests", + "test/conversational_speech", + "test/py_quality_assessment", + ] + } + } + + rtc_source_set("audio_processing_unittests") { + testonly = true + + sources = [ + "aec/echo_cancellation_unittest.cc", + "aec/system_delay_unittest.cc", + "agc/agc_manager_direct_unittest.cc", + "agc/loudness_histogram_unittest.cc", + "agc/mock_agc.h", + "audio_buffer_unittest.cc", + "beamformer/array_util_unittest.cc", + "beamformer/complex_matrix_unittest.cc", + "beamformer/covariance_matrix_generator_unittest.cc", + "beamformer/matrix_unittest.cc", + "beamformer/mock_nonlinear_beamformer.h", + "config_unittest.cc", + "echo_cancellation_impl_unittest.cc", + "splitting_filter_unittest.cc", + "test/fake_recording_device_unittest.cc", + "transient/dyadic_decimator_unittest.cc", + "transient/file_utils.cc", + "transient/file_utils.h", + "transient/file_utils_unittest.cc", + "transient/moving_moments_unittest.cc", + "transient/transient_detector_unittest.cc", + "transient/transient_suppressor_unittest.cc", + "transient/wpd_node_unittest.cc", + "transient/wpd_tree_unittest.cc", + "utility/block_mean_calculator_unittest.cc", + "utility/delay_estimator_unittest.cc", + "vad/gmm_unittest.cc", + "vad/pitch_based_vad_unittest.cc", + "vad/pitch_internal_unittest.cc", + "vad/pole_zero_filter_unittest.cc", + "vad/standalone_vad_unittest.cc", + "vad/vad_audio_proc_unittest.cc", + "vad/vad_circular_buffer_unittest.cc", + "vad/voice_activity_detector_unittest.cc", + ] + + deps = [ + ":analog_mic_simulation", + ":audio_processing", + ":audioproc_test_utils", + "..:module_api", + "../..:webrtc_common", + "../../api:array_view", + "../../api:optional", + "../../common_audio:common_audio", + "../../rtc_base:gtest_prod", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + "../../test:test_support", + "../audio_coding:neteq_input_audio_tools", + "aec_dump:mock_aec_dump_unittests", + "test/conversational_speech:unittest", + "//testing/gmock", + "//testing/gtest", + ] + + defines = [] + + if (apm_debug_dump) { + defines += [ "WEBRTC_APM_DEBUG_DUMP=1" ] + } else { + defines += [ "WEBRTC_APM_DEBUG_DUMP=0" ] + } + + if (rtc_enable_intelligibility_enhancer) { + defines += [ "WEBRTC_INTELLIGIBILITY_ENHANCER=1" ] + sources += [ + "intelligibility/intelligibility_enhancer_unittest.cc", + "intelligibility/intelligibility_utils_unittest.cc", + ] + } else { + defines += [ "WEBRTC_INTELLIGIBILITY_ENHANCER=0" ] + } + + if (rtc_prefer_fixed_point) { + defines += [ "WEBRTC_AUDIOPROC_FIXED_PROFILE" ] + } else { + defines += [ "WEBRTC_AUDIOPROC_FLOAT_PROFILE" ] + } + + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_AUDIOPROC_DEBUG_DUMP" ] + deps += [ + ":audioproc_debug_proto", + ":audioproc_protobuf_utils", + ":audioproc_test_utils", + ":audioproc_unittest_proto", + "../../rtc_base:rtc_task_queue", + "aec_dump", + "aec_dump:aec_dump_unittests", + ] + sources += [ + "aec3/adaptive_fir_filter_unittest.cc", + "aec3/aec3_fft_unittest.cc", + "aec3/aec_state_unittest.cc", + "aec3/block_framer_unittest.cc", + "aec3/block_processor_metrics_unittest.cc", + "aec3/block_processor_unittest.cc", + "aec3/cascaded_biquad_filter_unittest.cc", + "aec3/comfort_noise_generator_unittest.cc", + "aec3/decimator_unittest.cc", + "aec3/echo_canceller3_unittest.cc", + "aec3/echo_path_delay_estimator_unittest.cc", + "aec3/echo_path_variability_unittest.cc", + "aec3/echo_remover_metrics_unittest.cc", + "aec3/echo_remover_unittest.cc", + "aec3/erl_estimator_unittest.cc", + "aec3/erle_estimator_unittest.cc", + "aec3/fft_data_unittest.cc", + "aec3/frame_blocker_unittest.cc", + "aec3/main_filter_update_gain_unittest.cc", + "aec3/matched_filter_lag_aggregator_unittest.cc", + "aec3/matched_filter_unittest.cc", + "aec3/output_selector_unittest.cc", + "aec3/render_buffer_unittest.cc", + "aec3/render_delay_buffer_unittest.cc", + "aec3/render_delay_controller_metrics_unittest.cc", + "aec3/render_delay_controller_unittest.cc", + "aec3/render_signal_analyzer_unittest.cc", + "aec3/residual_echo_estimator_unittest.cc", + "aec3/shadow_filter_update_gain_unittest.cc", + "aec3/subtractor_unittest.cc", + "aec3/suppression_filter_unittest.cc", + "aec3/suppression_gain_unittest.cc", + "aec3/vector_math_unittest.cc", + "agc2/gain_controller2_unittest.cc", + "audio_processing_impl_locking_unittest.cc", + "audio_processing_impl_unittest.cc", + "audio_processing_unittest.cc", + "beamformer/nonlinear_beamformer_unittest.cc", + "echo_cancellation_bit_exact_unittest.cc", + "echo_control_mobile_unittest.cc", + "echo_detector/circular_buffer_unittest.cc", + "echo_detector/mean_variance_estimator_unittest.cc", + "echo_detector/moving_max_unittest.cc", + "echo_detector/normalized_covariance_estimator_unittest.cc", + "gain_control_unittest.cc", + "level_controller/level_controller_unittest.cc", + "level_estimator_unittest.cc", + "low_cut_filter_unittest.cc", + "noise_suppression_unittest.cc", + "residual_echo_detector_unittest.cc", + "rms_level_unittest.cc", + "test/debug_dump_replayer.cc", + "test/debug_dump_replayer.h", + "test/debug_dump_test.cc", + "test/echo_canceller_test_tools.cc", + "test/echo_canceller_test_tools.h", + "test/echo_canceller_test_tools_unittest.cc", + "test/test_utils.h", + "voice_detection_unittest.cc", + ] + } + + if ((!build_with_chromium || is_win) && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + + rtc_source_set("audio_processing_perf_tests") { + # Has problems with autogenerated targets on Android and iOS + # Dependency chain (there may also be others): + # :audio_processing_perf_tests --> + # ..:modules_unittests --[private]--> + # ..:modules_unittests_apk --> + # ..:modules_unittests_apk__create --> + # ..:modules_unittests_apk__create__finalize --> + # ..:modules_unittests_apk__create__package --[private]--> + # ..:_modules_unittests__library + check_includes = false + testonly = true + + sources = [ + "audio_processing_performance_unittest.cc", + "level_controller/level_controller_complexity_unittest.cc", + ] + deps = [ + ":audio_processing", + ":audioproc_test_utils", + "../../api:array_view", + "../../modules:module_api", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + "../../test:test_support", + ] + + if (rtc_enable_intelligibility_enhancer) { + defines = [ "WEBRTC_INTELLIGIBILITY_ENHANCER=1" ] + } else { + defines = [ "WEBRTC_INTELLIGIBILITY_ENHANCER=0" ] + } + } + + rtc_source_set("analog_mic_simulation") { + sources = [ + "test/fake_recording_device.cc", + "test/fake_recording_device.h", + ] + deps = [ + "../../api:array_view", + "../../common_audio:common_audio", + "../../modules:module_api", + "../../rtc_base:rtc_base_approved", + ] + } + + if (rtc_enable_protobuf) { + rtc_executable("unpack_aecdump") { + testonly = true + sources = [ + "test/unpack.cc", + ] + + deps = [ + ":audio_processing", + ":audioproc_debug_proto", + ":audioproc_protobuf_utils", + ":audioproc_test_utils", + "../..:webrtc_common", + "../../common_audio", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers_default", + ] + } # unpack_aecdump + + rtc_executable("audioproc_f") { + testonly = true + sources = [ + "test/aec_dump_based_simulator.cc", + "test/aec_dump_based_simulator.h", + "test/audio_processing_simulator.cc", + "test/audio_processing_simulator.h", + "test/audioproc_float.cc", + "test/wav_based_simulator.cc", + "test/wav_based_simulator.h", + ] + + deps = [ + ":analog_mic_simulation", + ":audio_processing", + ":audioproc_debug_proto", + ":audioproc_protobuf_utils", + ":audioproc_test_utils", + "../../api:optional", + "../../common_audio:common_audio", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_task_queue", + "../../system_wrappers", + "../../system_wrappers:system_wrappers_default", + "../../test:test_support", + "aec_dump", + "aec_dump:aec_dump_impl", + "//testing/gtest", + ] + } # audioproc_f + } + + rtc_source_set("audioproc_test_utils") { + testonly = true + sources = [ + "test/audio_buffer_tools.cc", + "test/audio_buffer_tools.h", + "test/bitexactness_tools.cc", + "test/bitexactness_tools.h", + "test/performance_timer.cc", + "test/performance_timer.h", + "test/simulator_buffers.cc", + "test/simulator_buffers.h", + "test/test_utils.cc", + "test/test_utils.h", + ] + + deps = [ + ":audio_processing", + "..:module_api", + "../../api:array_view", + "../../api:optional", + "../../common_audio", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + "../../test:test_support", + "../audio_coding:neteq_input_audio_tools", + "//testing/gtest", + ] + } + + rtc_executable("transient_suppression_test") { + testonly = true + sources = [ + "transient/file_utils.cc", + "transient/file_utils.h", + "transient/transient_suppression_test.cc", + ] + deps = [ + ":audio_processing", + "..:module_api", + "../..:webrtc_common", + "../../common_audio:common_audio", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:metrics_default", + "../../system_wrappers:system_wrappers", + "../../test:test_support", + "//testing/gtest", + ] + } + + rtc_executable("click_annotate") { + testonly = true + sources = [ + "transient/click_annotate.cc", + "transient/file_utils.cc", + "transient/file_utils.h", + ] + deps = [ + ":audio_processing", + "../..:webrtc_common", + "../../system_wrappers:metrics_default", + "../../system_wrappers:system_wrappers", + ] + } + + rtc_executable("nonlinear_beamformer_test") { + testonly = true + sources = [ + "beamformer/nonlinear_beamformer_test.cc", + ] + deps = [ + ":audio_processing", + ":audioproc_test_utils", + "../../common_audio:common_audio", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:metrics_default", + ] + } + + if (rtc_enable_intelligibility_enhancer) { + rtc_executable("intelligibility_proc") { + testonly = true + sources = [ + "intelligibility/test/intelligibility_proc.cc", + ] + deps = [ + ":audio_processing", + ":audioproc_test_utils", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:metrics_default", + "../../test:test_support", + "//testing/gtest", + ] + } + } + + if (rtc_enable_protobuf) { + proto_library("audioproc_unittest_proto") { + sources = [ + "test/unittest.proto", + ] + proto_out_dir = "modules/audio_processing/test" + } + + rtc_static_library("audioproc_protobuf_utils") { + sources = [ + "test/protobuf_utils.cc", + "test/protobuf_utils.h", + ] + + deps = [ + ":audioproc_debug_proto", + "../..:webrtc_common", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_base_approved", + ] + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/DEPS b/third_party/libwebrtc/webrtc/modules/audio_processing/DEPS new file mode 100644 index 0000000000..79fd071785 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/DEPS @@ -0,0 +1,14 @@ +include_rules = [ + "+audio/utility/audio_frame_operations.h", + "+common_audio", + "+system_wrappers", +] + +specific_include_rules = { + ".*test\.cc": [ + "+rtc_tools", + # Android platform build has different paths. + "+gtest", + "+external/webrtc", + ], +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_processing/OWNERS new file mode 100644 index 0000000000..9c521d230a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/OWNERS @@ -0,0 +1,9 @@ +aleloi@webrtc.org +aluebs@webrtc.org +henrik.lundin@webrtc.org +peah@webrtc.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_common.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_common.h new file mode 100644 index 0000000000..80c5c14813 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_common.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_AEC_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_AEC_AEC_COMMON_H_ + +#include "typedefs.h" // NOLINT(build/include) + +#ifdef _MSC_VER /* visual c++ */ +#define ALIGN16_BEG __declspec(align(16)) +#define ALIGN16_END +#else /* gcc or icc */ +#define ALIGN16_BEG +#define ALIGN16_END __attribute__((aligned(16))) +#endif + +#ifdef __cplusplus +namespace webrtc { +#endif + +extern ALIGN16_BEG const float ALIGN16_END WebRtcAec_sqrtHanning[65]; +extern ALIGN16_BEG const float ALIGN16_END WebRtcAec_weightCurve[65]; +extern ALIGN16_BEG const float ALIGN16_END WebRtcAec_overDriveCurve[65]; +extern const float WebRtcAec_kExtendedSmoothingCoefficients[2][2]; +extern const float WebRtcAec_kNormalSmoothingCoefficients[2][2]; +extern const float WebRtcAec_kMinFarendPSD; + +#ifdef __cplusplus +} // namespace webrtc +#endif + +#endif // MODULES_AUDIO_PROCESSING_AEC_AEC_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.cc new file mode 100644 index 0000000000..8f4fda1a1b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.cc @@ -0,0 +1,2057 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * The core AEC algorithm, which is presented with time-aligned signals. + */ + +#include "modules/audio_processing/aec/aec_core.h" + +#include +#include +#include // size_t +#include +#include + +#include "rtc_base/checks.h" +extern "C" { +#include "common_audio/ring_buffer.h" +} +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/aec/aec_common.h" +#include "modules/audio_processing/aec/aec_core_optimized_methods.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/utility/delay_estimator_wrapper.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "system_wrappers/include/metrics.h" +#include "typedefs.h" // NOLINT(build/include) +#include "rtc_base/criticalsection.h" + +namespace webrtc { +namespace { +enum class DelaySource { + kSystemDelay, // The delay values come from the OS. + kDelayAgnostic, // The delay values come from the DA-AEC. +}; + +constexpr int kMinDelayLogValue = -200; +constexpr int kMaxDelayLogValue = 200; +constexpr int kNumDelayLogBuckets = 100; + +void MaybeLogDelayAdjustment(int moved_ms, DelaySource source) { + if (moved_ms == 0) + return; + switch (source) { + case DelaySource::kSystemDelay: + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecDelayAdjustmentMsSystemValue", + moved_ms, kMinDelayLogValue, kMaxDelayLogValue, + kNumDelayLogBuckets); + return; + case DelaySource::kDelayAgnostic: + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecDelayAdjustmentMsAgnosticValue", + moved_ms, kMinDelayLogValue, kMaxDelayLogValue, + kNumDelayLogBuckets); + return; + } +} +} // namespace + +// Buffer size (samples) +static const size_t kBufferSizeBlocks = 250; // 1 second of audio in 16 kHz. + +// Metrics +static const size_t kSubCountLen = 4; +static const size_t kCountLen = 50; +static const int kDelayMetricsAggregationWindow = 1250; // 5 seconds at 16 kHz. + +// Divergence metric is based on audio level, which gets updated every +// |kSubCountLen + 1| * PART_LEN samples. Divergence metric takes the statistics +// of |kDivergentFilterFractionAggregationWindowSize| audio levels. The +// following value corresponds to 1 second at 16 kHz. +static const int kDivergentFilterFractionAggregationWindowSize = 50; + +// Quantities to control H band scaling for SWB input +static const float cnScaleHband = 0.4f; // scale for comfort noise in H band. +// Initial bin for averaging nlp gain in low band +static const int freqAvgIc = PART_LEN / 2; + +// Matlab code to produce table: +// win = sqrt(hanning(63)); win = [0 ; win(1:32)]; +// fprintf(1, '\t%.14f, %.14f, %.14f,\n', win); +ALIGN16_BEG const float ALIGN16_END WebRtcAec_sqrtHanning[65] = { + 0.00000000000000f, 0.02454122852291f, 0.04906767432742f, 0.07356456359967f, + 0.09801714032956f, 0.12241067519922f, 0.14673047445536f, 0.17096188876030f, + 0.19509032201613f, 0.21910124015687f, 0.24298017990326f, 0.26671275747490f, + 0.29028467725446f, 0.31368174039889f, 0.33688985339222f, 0.35989503653499f, + 0.38268343236509f, 0.40524131400499f, 0.42755509343028f, 0.44961132965461f, + 0.47139673682600f, 0.49289819222978f, 0.51410274419322f, 0.53499761988710f, + 0.55557023301960f, 0.57580819141785f, 0.59569930449243f, 0.61523159058063f, + 0.63439328416365f, 0.65317284295378f, 0.67155895484702f, 0.68954054473707f, + 0.70710678118655f, 0.72424708295147f, 0.74095112535496f, 0.75720884650648f, + 0.77301045336274f, 0.78834642762661f, 0.80320753148064f, 0.81758481315158f, + 0.83146961230255f, 0.84485356524971f, 0.85772861000027f, 0.87008699110871f, + 0.88192126434835f, 0.89322430119552f, 0.90398929312344f, 0.91420975570353f, + 0.92387953251129f, 0.93299279883474f, 0.94154406518302f, 0.94952818059304f, + 0.95694033573221f, 0.96377606579544f, 0.97003125319454f, 0.97570213003853f, + 0.98078528040323f, 0.98527764238894f, 0.98917650996478f, 0.99247953459871f, + 0.99518472667220f, 0.99729045667869f, 0.99879545620517f, 0.99969881869620f, + 1.00000000000000f}; + +// Matlab code to produce table: +// weightCurve = [0 ; 0.3 * sqrt(linspace(0,1,64))' + 0.1]; +// fprintf(1, '\t%.4f, %.4f, %.4f, %.4f, %.4f, %.4f,\n', weightCurve); +ALIGN16_BEG const float ALIGN16_END WebRtcAec_weightCurve[65] = { + 0.0000f, 0.1000f, 0.1378f, 0.1535f, 0.1655f, 0.1756f, 0.1845f, 0.1926f, + 0.2000f, 0.2069f, 0.2134f, 0.2195f, 0.2254f, 0.2309f, 0.2363f, 0.2414f, + 0.2464f, 0.2512f, 0.2558f, 0.2604f, 0.2648f, 0.2690f, 0.2732f, 0.2773f, + 0.2813f, 0.2852f, 0.2890f, 0.2927f, 0.2964f, 0.3000f, 0.3035f, 0.3070f, + 0.3104f, 0.3138f, 0.3171f, 0.3204f, 0.3236f, 0.3268f, 0.3299f, 0.3330f, + 0.3360f, 0.3390f, 0.3420f, 0.3449f, 0.3478f, 0.3507f, 0.3535f, 0.3563f, + 0.3591f, 0.3619f, 0.3646f, 0.3673f, 0.3699f, 0.3726f, 0.3752f, 0.3777f, + 0.3803f, 0.3828f, 0.3854f, 0.3878f, 0.3903f, 0.3928f, 0.3952f, 0.3976f, + 0.4000f}; + +// Matlab code to produce table: +// overDriveCurve = [sqrt(linspace(0,1,65))' + 1]; +// fprintf(1, '\t%.4f, %.4f, %.4f, %.4f, %.4f, %.4f,\n', overDriveCurve); +ALIGN16_BEG const float ALIGN16_END WebRtcAec_overDriveCurve[65] = { + 1.0000f, 1.1250f, 1.1768f, 1.2165f, 1.2500f, 1.2795f, 1.3062f, 1.3307f, + 1.3536f, 1.3750f, 1.3953f, 1.4146f, 1.4330f, 1.4507f, 1.4677f, 1.4841f, + 1.5000f, 1.5154f, 1.5303f, 1.5449f, 1.5590f, 1.5728f, 1.5863f, 1.5995f, + 1.6124f, 1.6250f, 1.6374f, 1.6495f, 1.6614f, 1.6731f, 1.6847f, 1.6960f, + 1.7071f, 1.7181f, 1.7289f, 1.7395f, 1.7500f, 1.7603f, 1.7706f, 1.7806f, + 1.7906f, 1.8004f, 1.8101f, 1.8197f, 1.8292f, 1.8385f, 1.8478f, 1.8570f, + 1.8660f, 1.8750f, 1.8839f, 1.8927f, 1.9014f, 1.9100f, 1.9186f, 1.9270f, + 1.9354f, 1.9437f, 1.9520f, 1.9601f, 1.9682f, 1.9763f, 1.9843f, 1.9922f, + 2.0000f}; + +// Delay Agnostic AEC parameters, still under development and may change. +static const float kDelayQualityThresholdMax = 0.07f; +static const float kDelayQualityThresholdMin = 0.01f; +static const int kInitialShiftOffset = 5; +#if !defined(WEBRTC_ANDROID) +static const int kDelayCorrectionStart = 1500; // 10 ms chunks +#endif + +// Target suppression levels for nlp modes. +// log{0.001, 0.00001, 0.00000001} +static const float kTargetSupp[3] = {-6.9f, -11.5f, -18.4f}; + +// Two sets of parameters, one for the extended filter mode. +static const float kExtendedMinOverDrive[3] = {3.0f, 6.0f, 15.0f}; +static const float kNormalMinOverDrive[3] = {1.0f, 2.0f, 5.0f}; +const float WebRtcAec_kExtendedSmoothingCoefficients[2][2] = {{0.9f, 0.1f}, + {0.92f, 0.08f}}; +const float WebRtcAec_kNormalSmoothingCoefficients[2][2] = {{0.9f, 0.1f}, + {0.93f, 0.07f}}; + +// Number of partitions forming the NLP's "preferred" bands. +enum { kPrefBandSize = 24 }; + +rtc::CriticalSection WebRtcAec_CriticalSection; +WebRtcAecFilterFar WebRtcAec_FilterFar; +WebRtcAecScaleErrorSignal WebRtcAec_ScaleErrorSignal; +WebRtcAecFilterAdaptation WebRtcAec_FilterAdaptation; +WebRtcAecOverdrive WebRtcAec_Overdrive; +WebRtcAecSuppress WebRtcAec_Suppress; +WebRtcAecComputeCoherence WebRtcAec_ComputeCoherence; +WebRtcAecUpdateCoherenceSpectra WebRtcAec_UpdateCoherenceSpectra; +WebRtcAecStoreAsComplex WebRtcAec_StoreAsComplex; +WebRtcAecPartitionDelay WebRtcAec_PartitionDelay; +WebRtcAecWindowData WebRtcAec_WindowData; + +__inline static float MulRe(float aRe, float aIm, float bRe, float bIm) { + return aRe * bRe - aIm * bIm; +} + +__inline static float MulIm(float aRe, float aIm, float bRe, float bIm) { + return aRe * bIm + aIm * bRe; +} + +// TODO(minyue): Due to a legacy bug, |framelevel| and |averagelevel| use a +// window, of which the length is 1 unit longer than indicated. Remove "+1" when +// the code is refactored. +PowerLevel::PowerLevel() + : framelevel(kSubCountLen + 1), + averagelevel(kCountLen + 1) { +} + +BlockBuffer::BlockBuffer() { + buffer_ = WebRtc_CreateBuffer(kBufferSizeBlocks, sizeof(float) * PART_LEN); + RTC_CHECK(buffer_); + ReInit(); +} + +BlockBuffer::~BlockBuffer() { + WebRtc_FreeBuffer(buffer_); +} + +void BlockBuffer::ReInit() { + WebRtc_InitBuffer(buffer_); +} + +void BlockBuffer::Insert(const float block[PART_LEN]) { + WebRtc_WriteBuffer(buffer_, block, 1); +} + +void BlockBuffer::ExtractExtendedBlock(float extended_block[PART_LEN2]) { + float* block_ptr = NULL; + RTC_DCHECK_LT(0, AvaliableSpace()); + + // Extract the previous block. + WebRtc_MoveReadPtr(buffer_, -1); + size_t read_elements = WebRtc_ReadBuffer( + buffer_, reinterpret_cast(&block_ptr), &extended_block[0], 1); + if (read_elements == 0u) { + std::fill_n(&extended_block[0], PART_LEN, 0.0f); + } else if (block_ptr != &extended_block[0]) { + memcpy(&extended_block[0], block_ptr, PART_LEN * sizeof(float)); + } + + // Extract the current block. + read_elements = + WebRtc_ReadBuffer(buffer_, reinterpret_cast(&block_ptr), + &extended_block[PART_LEN], 1); + if (read_elements == 0u) { + std::fill_n(&extended_block[PART_LEN], PART_LEN, 0.0f); + } else if (block_ptr != &extended_block[PART_LEN]) { + memcpy(&extended_block[PART_LEN], block_ptr, PART_LEN * sizeof(float)); + } +} + +int BlockBuffer::AdjustSize(int buffer_size_decrease) { + return WebRtc_MoveReadPtr(buffer_, buffer_size_decrease); +} + +size_t BlockBuffer::Size() { + return static_cast(WebRtc_available_read(buffer_)); +} + +size_t BlockBuffer::AvaliableSpace() { + return WebRtc_available_write(buffer_); +} + +DivergentFilterFraction::DivergentFilterFraction() + : count_(0), + occurrence_(0), + fraction_(-1.0) { +} + +void DivergentFilterFraction::Reset() { + Clear(); + fraction_ = -1.0; +} + +void DivergentFilterFraction::AddObservation(const PowerLevel& nearlevel, + const PowerLevel& linoutlevel, + const PowerLevel& nlpoutlevel) { + const float near_level = nearlevel.framelevel.GetLatestMean(); + const float level_increase = + linoutlevel.framelevel.GetLatestMean() - near_level; + const bool output_signal_active = nlpoutlevel.framelevel.GetLatestMean() > + 40.0 * nlpoutlevel.minlevel; + // Level increase should be, in principle, negative, when the filter + // does not diverge. Here we allow some margin (0.01 * near end level) and + // numerical error (1.0). We count divergence only when the AEC output + // signal is active. + if (output_signal_active && + level_increase > std::max(0.01 * near_level, 1.0)) + occurrence_++; + ++count_; + if (count_ == kDivergentFilterFractionAggregationWindowSize) { + fraction_ = static_cast(occurrence_) / + kDivergentFilterFractionAggregationWindowSize; + Clear(); + } +} + +float DivergentFilterFraction::GetLatestFraction() const { + return fraction_; +} + +void DivergentFilterFraction::Clear() { + count_ = 0; + occurrence_ = 0; +} + +// TODO(minyue): Moving some initialization from WebRtcAec_CreateAec() to ctor. +AecCore::AecCore(int instance_index) + : data_dumper(new ApmDataDumper(instance_index)) {} + +AecCore::~AecCore() {} + +static int CmpFloat(const void* a, const void* b) { + const float* da = (const float*)a; + const float* db = (const float*)b; + + return (*da > *db) - (*da < *db); +} + +static void FilterFar(int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float y_fft[2][PART_LEN1]) { + int i; + for (i = 0; i < num_partitions; i++) { + int j; + int xPos = (i + x_fft_buf_block_pos) * PART_LEN1; + int pos = i * PART_LEN1; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * (PART_LEN1); + } + + for (j = 0; j < PART_LEN1; j++) { + y_fft[0][j] += MulRe(x_fft_buf[0][xPos + j], x_fft_buf[1][xPos + j], + h_fft_buf[0][pos + j], h_fft_buf[1][pos + j]); + y_fft[1][j] += MulIm(x_fft_buf[0][xPos + j], x_fft_buf[1][xPos + j], + h_fft_buf[0][pos + j], h_fft_buf[1][pos + j]); + } + } +} + +static void ScaleErrorSignal(float mu, + float error_threshold, + float x_pow[PART_LEN1], + float ef[2][PART_LEN1]) { + int i; + float abs_ef; + for (i = 0; i < (PART_LEN1); i++) { + ef[0][i] /= (x_pow[i] + 1e-10f); + ef[1][i] /= (x_pow[i] + 1e-10f); + abs_ef = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]); + + if (abs_ef > error_threshold) { + abs_ef = error_threshold / (abs_ef + 1e-10f); + ef[0][i] *= abs_ef; + ef[1][i] *= abs_ef; + } + + // Stepsize factor + ef[0][i] *= mu; + ef[1][i] *= mu; + } +} + +static void FilterAdaptation( + const OouraFft& ooura_fft, + int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float e_fft[2][PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]) { + int i, j; + float fft[PART_LEN2]; + for (i = 0; i < num_partitions; i++) { + int xPos = (i + x_fft_buf_block_pos) * (PART_LEN1); + int pos; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * PART_LEN1; + } + + pos = i * PART_LEN1; + + for (j = 0; j < PART_LEN; j++) { + fft[2 * j] = MulRe(x_fft_buf[0][xPos + j], -x_fft_buf[1][xPos + j], + e_fft[0][j], e_fft[1][j]); + fft[2 * j + 1] = MulIm(x_fft_buf[0][xPos + j], -x_fft_buf[1][xPos + j], + e_fft[0][j], e_fft[1][j]); + } + fft[1] = + MulRe(x_fft_buf[0][xPos + PART_LEN], -x_fft_buf[1][xPos + PART_LEN], + e_fft[0][PART_LEN], e_fft[1][PART_LEN]); + + ooura_fft.InverseFft(fft); + memset(fft + PART_LEN, 0, sizeof(float) * PART_LEN); + + // fft scaling + { + float scale = 2.0f / PART_LEN2; + for (j = 0; j < PART_LEN; j++) { + fft[j] *= scale; + } + } + ooura_fft.Fft(fft); + + h_fft_buf[0][pos] += fft[0]; + h_fft_buf[0][pos + PART_LEN] += fft[1]; + + for (j = 1; j < PART_LEN; j++) { + h_fft_buf[0][pos + j] += fft[2 * j]; + h_fft_buf[1][pos + j] += fft[2 * j + 1]; + } + } +} + +static void Overdrive(float overdrive_scaling, + const float hNlFb, + float hNl[PART_LEN1]) { + for (int i = 0; i < PART_LEN1; ++i) { + // Weight subbands + if (hNl[i] > hNlFb) { + hNl[i] = WebRtcAec_weightCurve[i] * hNlFb + + (1 - WebRtcAec_weightCurve[i]) * hNl[i]; + } + hNl[i] = powf(hNl[i], overdrive_scaling * WebRtcAec_overDriveCurve[i]); + } +} + +static void Suppress(const float hNl[PART_LEN1], float efw[2][PART_LEN1]) { + for (int i = 0; i < PART_LEN1; ++i) { + // Suppress error signal + efw[0][i] *= hNl[i]; + efw[1][i] *= hNl[i]; + + // Ooura fft returns incorrect sign on imaginary component. It matters here + // because we are making an additive change with comfort noise. + efw[1][i] *= -1; + } +} + +static int PartitionDelay(int num_partitions, + float h_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1]) { + // Measures the energy in each filter partition and returns the partition with + // highest energy. + // TODO(bjornv): Spread computational cost by computing one partition per + // block? + float wfEnMax = 0; + int i; + int delay = 0; + + for (i = 0; i < num_partitions; i++) { + int j; + int pos = i * PART_LEN1; + float wfEn = 0; + for (j = 0; j < PART_LEN1; j++) { + wfEn += h_fft_buf[0][pos + j] * h_fft_buf[0][pos + j] + + h_fft_buf[1][pos + j] * h_fft_buf[1][pos + j]; + } + + if (wfEn > wfEnMax) { + wfEnMax = wfEn; + delay = i; + } + } + return delay; +} + +// Update metric with 10 * log10(numerator / denominator). +static void UpdateLogRatioMetric(Stats* metric, float numerator, + float denominator) { + RTC_DCHECK(metric); + RTC_CHECK(numerator >= 0); + RTC_CHECK(denominator >= 0); + + const float log_numerator = log10(numerator + 1e-10f); + const float log_denominator = log10(denominator + 1e-10f); + metric->instant = 10.0f * (log_numerator - log_denominator); + + // Max. + if (metric->instant > metric->max) + metric->max = metric->instant; + + // Min. + if (metric->instant < metric->min) + metric->min = metric->instant; + + // Average. + metric->counter++; + // This is to protect overflow, which should almost never happen. + RTC_CHECK_NE(0, metric->counter); + metric->sum += metric->instant; + metric->average = metric->sum / metric->counter; + + // Upper mean. + if (metric->instant > metric->average) { + metric->hicounter++; + // This is to protect overflow, which should almost never happen. + RTC_CHECK_NE(0, metric->hicounter); + metric->hisum += metric->instant; + metric->himean = metric->hisum / metric->hicounter; + } +} + +// Threshold to protect against the ill-effects of a zero far-end. +const float WebRtcAec_kMinFarendPSD = 15; + +// Updates the following smoothed Power Spectral Densities (PSD): +// - sd : near-end +// - se : residual echo +// - sx : far-end +// - sde : cross-PSD of near-end and residual echo +// - sxd : cross-PSD of near-end and far-end +// +// In addition to updating the PSDs, also the filter diverge state is +// determined. +static void UpdateCoherenceSpectra(int mult, + bool extended_filter_enabled, + float efw[2][PART_LEN1], + float dfw[2][PART_LEN1], + float xfw[2][PART_LEN1], + CoherenceState* coherence_state, + short* filter_divergence_state, + int* extreme_filter_divergence) { + // Power estimate smoothing coefficients. + const float* ptrGCoh = + extended_filter_enabled + ? WebRtcAec_kExtendedSmoothingCoefficients[mult - 1] + : WebRtcAec_kNormalSmoothingCoefficients[mult - 1]; + int i; + float sdSum = 0, seSum = 0; + + for (i = 0; i < PART_LEN1; i++) { + coherence_state->sd[i] = + ptrGCoh[0] * coherence_state->sd[i] + + ptrGCoh[1] * (dfw[0][i] * dfw[0][i] + dfw[1][i] * dfw[1][i]); + coherence_state->se[i] = + ptrGCoh[0] * coherence_state->se[i] + + ptrGCoh[1] * (efw[0][i] * efw[0][i] + efw[1][i] * efw[1][i]); + // We threshold here to protect against the ill-effects of a zero farend. + // The threshold is not arbitrarily chosen, but balances protection and + // adverse interaction with the algorithm's tuning. + // TODO(bjornv): investigate further why this is so sensitive. + coherence_state->sx[i] = + ptrGCoh[0] * coherence_state->sx[i] + + ptrGCoh[1] * + WEBRTC_SPL_MAX(xfw[0][i] * xfw[0][i] + xfw[1][i] * xfw[1][i], + WebRtcAec_kMinFarendPSD); + + coherence_state->sde[i][0] = + ptrGCoh[0] * coherence_state->sde[i][0] + + ptrGCoh[1] * (dfw[0][i] * efw[0][i] + dfw[1][i] * efw[1][i]); + coherence_state->sde[i][1] = + ptrGCoh[0] * coherence_state->sde[i][1] + + ptrGCoh[1] * (dfw[0][i] * efw[1][i] - dfw[1][i] * efw[0][i]); + + coherence_state->sxd[i][0] = + ptrGCoh[0] * coherence_state->sxd[i][0] + + ptrGCoh[1] * (dfw[0][i] * xfw[0][i] + dfw[1][i] * xfw[1][i]); + coherence_state->sxd[i][1] = + ptrGCoh[0] * coherence_state->sxd[i][1] + + ptrGCoh[1] * (dfw[0][i] * xfw[1][i] - dfw[1][i] * xfw[0][i]); + + sdSum += coherence_state->sd[i]; + seSum += coherence_state->se[i]; + } + + // Divergent filter safeguard update. + *filter_divergence_state = + (*filter_divergence_state ? 1.05f : 1.0f) * seSum > sdSum; + + // Signal extreme filter divergence if the error is significantly larger + // than the nearend (13 dB). + *extreme_filter_divergence = (seSum > (19.95f * sdSum)); +} + +// Window time domain data to be used by the fft. +__inline static void WindowData(float* x_windowed, const float* x) { + int i; + for (i = 0; i < PART_LEN; i++) { + x_windowed[i] = x[i] * WebRtcAec_sqrtHanning[i]; + x_windowed[PART_LEN + i] = + x[PART_LEN + i] * WebRtcAec_sqrtHanning[PART_LEN - i]; + } +} + +// Puts fft output data into a complex valued array. +__inline static void StoreAsComplex(const float* data, + float data_complex[2][PART_LEN1]) { + int i; + data_complex[0][0] = data[0]; + data_complex[1][0] = 0; + for (i = 1; i < PART_LEN; i++) { + data_complex[0][i] = data[2 * i]; + data_complex[1][i] = data[2 * i + 1]; + } + data_complex[0][PART_LEN] = data[1]; + data_complex[1][PART_LEN] = 0; +} + +static void ComputeCoherence(const CoherenceState* coherence_state, + float* cohde, + float* cohxd) { + // Subband coherence + for (int i = 0; i < PART_LEN1; i++) { + cohde[i] = (coherence_state->sde[i][0] * coherence_state->sde[i][0] + + coherence_state->sde[i][1] * coherence_state->sde[i][1]) / + (coherence_state->sd[i] * coherence_state->se[i] + 1e-10f); + cohxd[i] = (coherence_state->sxd[i][0] * coherence_state->sxd[i][0] + + coherence_state->sxd[i][1] * coherence_state->sxd[i][1]) / + (coherence_state->sx[i] * coherence_state->sd[i] + 1e-10f); + } +} + +static void GetHighbandGain(const float* lambda, float* nlpGainHband) { + int i; + + *nlpGainHband = 0.0f; + for (i = freqAvgIc; i < PART_LEN1 - 1; i++) { + *nlpGainHband += lambda[i]; + } + *nlpGainHband /= static_cast(PART_LEN1 - 1 - freqAvgIc); +} + +static void GenerateComplexNoise(uint32_t* seed, float noise[2][PART_LEN1]) { + const float kPi2 = 6.28318530717959f; + int16_t randW16[PART_LEN]; + WebRtcSpl_RandUArray(randW16, PART_LEN, seed); + + noise[0][0] = 0; + noise[1][0] = 0; + for (size_t i = 1; i < PART_LEN1; i++) { + float tmp = kPi2 * randW16[i - 1] / 32768.f; + noise[0][i] = cosf(tmp); + noise[1][i] = -sinf(tmp); + } + noise[1][PART_LEN] = 0; +} + +static void ComfortNoise(bool generate_high_frequency_noise, + uint32_t* seed, + float e_fft[2][PART_LEN1], + float high_frequency_comfort_noise[2][PART_LEN1], + const float* noise_spectrum, + const float* suppressor_gain) { + float complex_noise[2][PART_LEN1]; + + GenerateComplexNoise(seed, complex_noise); + + // Shape, scale and add comfort noise. + for (int i = 1; i < PART_LEN1; ++i) { + float noise_scaling = + sqrtf(WEBRTC_SPL_MAX(1 - suppressor_gain[i] * suppressor_gain[i], 0)) * + sqrtf(noise_spectrum[i]); + e_fft[0][i] += noise_scaling * complex_noise[0][i]; + e_fft[1][i] += noise_scaling * complex_noise[1][i]; + } + + // Form comfort noise for higher frequencies. + if (generate_high_frequency_noise) { + // Compute average noise power and nlp gain over the second half of freq + // spectrum (i.e., 4->8khz). + int start_avg_band = PART_LEN1 / 2; + float upper_bands_noise_power = 0.f; + float upper_bands_suppressor_gain = 0.f; + for (int i = start_avg_band; i < PART_LEN1; ++i) { + upper_bands_noise_power += sqrtf(noise_spectrum[i]); + upper_bands_suppressor_gain += + sqrtf(WEBRTC_SPL_MAX(1 - suppressor_gain[i] * suppressor_gain[i], 0)); + } + upper_bands_noise_power /= (PART_LEN1 - start_avg_band); + upper_bands_suppressor_gain /= (PART_LEN1 - start_avg_band); + + // Shape, scale and add comfort noise. + float noise_scaling = upper_bands_suppressor_gain * upper_bands_noise_power; + high_frequency_comfort_noise[0][0] = 0; + high_frequency_comfort_noise[1][0] = 0; + for (int i = 1; i < PART_LEN1; ++i) { + high_frequency_comfort_noise[0][i] = noise_scaling * complex_noise[0][i]; + high_frequency_comfort_noise[1][i] = noise_scaling * complex_noise[1][i]; + } + high_frequency_comfort_noise[1][PART_LEN] = 0; + } else { + memset(high_frequency_comfort_noise, 0, + 2 * PART_LEN1 * sizeof(high_frequency_comfort_noise[0][0])); + } +} + +static void InitLevel(PowerLevel* level) { + const float kBigFloat = 1E17f; + level->averagelevel.Reset(); + level->framelevel.Reset(); + level->minlevel = kBigFloat; +} + +static void InitStats(Stats* stats) { + stats->instant = kOffsetLevel; + stats->average = kOffsetLevel; + stats->max = kOffsetLevel; + stats->min = kOffsetLevel * (-1); + stats->sum = 0; + stats->hisum = 0; + stats->himean = kOffsetLevel; + stats->counter = 0; + stats->hicounter = 0; +} + +static void InitMetrics(AecCore* self) { + self->stateCounter = 0; + InitLevel(&self->farlevel); + InitLevel(&self->nearlevel); + InitLevel(&self->linoutlevel); + InitLevel(&self->nlpoutlevel); + + InitStats(&self->erl); + InitStats(&self->erle); + InitStats(&self->aNlp); + InitStats(&self->rerl); + + self->divergent_filter_fraction.Reset(); +} + +static float CalculatePower(const float* in, size_t num_samples) { + size_t k; + float energy = 0.0f; + + for (k = 0; k < num_samples; ++k) { + energy += in[k] * in[k]; + } + return energy / num_samples; +} + +static void UpdateLevel(PowerLevel* level, float power) { + level->framelevel.AddValue(power); + if (level->framelevel.EndOfBlock()) { + const float new_frame_level = level->framelevel.GetLatestMean(); + if (new_frame_level > 0) { + if (new_frame_level < level->minlevel) { + level->minlevel = new_frame_level; // New minimum. + } else { + level->minlevel *= (1 + 0.001f); // Small increase. + } + } + level->averagelevel.AddValue(new_frame_level); + } +} + +static void UpdateMetrics(AecCore* aec) { + const float actThresholdNoisy = 8.0f; + const float actThresholdClean = 40.0f; + + const float noisyPower = 300000.0f; + + float actThreshold; + + if (aec->echoState) { // Check if echo is likely present + aec->stateCounter++; + } + + if (aec->linoutlevel.framelevel.EndOfBlock()) { + aec->divergent_filter_fraction.AddObservation(aec->nearlevel, + aec->linoutlevel, + aec->nlpoutlevel); + } + + if (aec->farlevel.averagelevel.EndOfBlock()) { + if (aec->farlevel.minlevel < noisyPower) { + actThreshold = actThresholdClean; + } else { + actThreshold = actThresholdNoisy; + } + + const float far_average_level = aec->farlevel.averagelevel.GetLatestMean(); + + // The last condition is to let estimation be made in active far-end + // segments only. + if ((aec->stateCounter > (0.5f * kCountLen * kSubCountLen)) && + (aec->farlevel.framelevel.EndOfBlock()) && + (far_average_level > (actThreshold * aec->farlevel.minlevel))) { + + // ERL: error return loss. + const float near_average_level = + aec->nearlevel.averagelevel.GetLatestMean(); + UpdateLogRatioMetric(&aec->erl, far_average_level, near_average_level); + + // A_NLP: error return loss enhanced before the nonlinear suppression. + const float linout_average_level = + aec->linoutlevel.averagelevel.GetLatestMean(); + UpdateLogRatioMetric(&aec->aNlp, near_average_level, + linout_average_level); + + // ERLE: error return loss enhanced. + const float nlpout_average_level = + aec->nlpoutlevel.averagelevel.GetLatestMean(); + UpdateLogRatioMetric(&aec->erle, near_average_level, + nlpout_average_level); + } + + aec->stateCounter = 0; + } +} + +static void UpdateDelayMetrics(AecCore* self) { + int i = 0; + int delay_values = 0; + int median = 0; + int lookahead = WebRtc_lookahead(self->delay_estimator); + const int kMsPerBlock = PART_LEN / (self->mult * 8); + int64_t l1_norm = 0; + + if (self->num_delay_values == 0) { + // We have no new delay value data. Even though -1 is a valid |median| in + // the sense that we allow negative values, it will practically never be + // used since multiples of |kMsPerBlock| will always be returned. + // We therefore use -1 to indicate in the logs that the delay estimator was + // not able to estimate the delay. + self->delay_median = -1; + self->delay_std = -1; + self->fraction_poor_delays = -1; + return; + } + + // Start value for median count down. + delay_values = self->num_delay_values >> 1; + // Get median of delay values since last update. + for (i = 0; i < kHistorySizeBlocks; i++) { + delay_values -= self->delay_histogram[i]; + if (delay_values < 0) { + median = i; + break; + } + } + // Account for lookahead. + self->delay_median = (median - lookahead) * kMsPerBlock; + + // Calculate the L1 norm, with median value as central moment. + for (i = 0; i < kHistorySizeBlocks; i++) { + l1_norm += abs(i - median) * self->delay_histogram[i]; + } + self->delay_std = + static_cast((l1_norm + self->num_delay_values / 2) / + self->num_delay_values) * kMsPerBlock; + + // Determine fraction of delays that are out of bounds, that is, either + // negative (anti-causal system) or larger than the AEC filter length. + { + int num_delays_out_of_bounds = self->num_delay_values; + const int histogram_length = + sizeof(self->delay_histogram) / sizeof(self->delay_histogram[0]); + for (i = lookahead; i < lookahead + self->num_partitions; ++i) { + if (i < histogram_length) + num_delays_out_of_bounds -= self->delay_histogram[i]; + } + self->fraction_poor_delays = + static_cast(num_delays_out_of_bounds) / self->num_delay_values; + } + + // Reset histogram. + memset(self->delay_histogram, 0, sizeof(self->delay_histogram)); + self->num_delay_values = 0; +} + +static void ScaledInverseFft(const OouraFft& ooura_fft, + float freq_data[2][PART_LEN1], + float time_data[PART_LEN2], + float scale, + int conjugate) { + int i; + const float normalization = scale / static_cast(PART_LEN2); + const float sign = (conjugate ? -1 : 1); + time_data[0] = freq_data[0][0] * normalization; + time_data[1] = freq_data[0][PART_LEN] * normalization; + for (i = 1; i < PART_LEN; i++) { + time_data[2 * i] = freq_data[0][i] * normalization; + time_data[2 * i + 1] = sign * freq_data[1][i] * normalization; + } + ooura_fft.InverseFft(time_data); +} + +static void Fft(const OouraFft& ooura_fft, + float time_data[PART_LEN2], + float freq_data[2][PART_LEN1]) { + int i; + ooura_fft.Fft(time_data); + + // Reorder fft output data. + freq_data[1][0] = 0; + freq_data[1][PART_LEN] = 0; + freq_data[0][0] = time_data[0]; + freq_data[0][PART_LEN] = time_data[1]; + for (i = 1; i < PART_LEN; i++) { + freq_data[0][i] = time_data[2 * i]; + freq_data[1][i] = time_data[2 * i + 1]; + } +} + +static int SignalBasedDelayCorrection(AecCore* self) { + int delay_correction = 0; + int last_delay = -2; + RTC_DCHECK(self); +#if !defined(WEBRTC_ANDROID) + // On desktops, turn on correction after |kDelayCorrectionStart| frames. This + // is to let the delay estimation get a chance to converge. Also, if the + // playout audio volume is low (or even muted) the delay estimation can return + // a very large delay, which will break the AEC if it is applied. + if (self->frame_count < kDelayCorrectionStart) { + self->data_dumper->DumpRaw("aec_da_reported_delay", 1, &last_delay); + return 0; + } +#endif + + // 1. Check for non-negative delay estimate. Note that the estimates we get + // from the delay estimation are not compensated for lookahead. Hence, a + // negative |last_delay| is an invalid one. + // 2. Verify that there is a delay change. In addition, only allow a change + // if the delay is outside a certain region taking the AEC filter length + // into account. + // TODO(bjornv): Investigate if we can remove the non-zero delay change check. + // 3. Only allow delay correction if the delay estimation quality exceeds + // |delay_quality_threshold|. + // 4. Finally, verify that the proposed |delay_correction| is feasible by + // comparing with the size of the far-end buffer. + last_delay = WebRtc_last_delay(self->delay_estimator); + self->data_dumper->DumpRaw("aec_da_reported_delay", 1, &last_delay); + if ((last_delay >= 0) && (last_delay != self->previous_delay) && + (WebRtc_last_delay_quality(self->delay_estimator) > + self->delay_quality_threshold)) { + int delay = last_delay - WebRtc_lookahead(self->delay_estimator); + // Allow for a slack in the actual delay, defined by a |lower_bound| and an + // |upper_bound|. The adaptive echo cancellation filter is currently + // |num_partitions| (of 64 samples) long. If the delay estimate is negative + // or at least 3/4 of the filter length we open up for correction. + const int lower_bound = 0; + const int upper_bound = self->num_partitions * 3 / 4; + const int do_correction = delay <= lower_bound || delay > upper_bound; + if (do_correction == 1) { + int available_read = self->farend_block_buffer_.Size(); + // With |shift_offset| we gradually rely on the delay estimates. For + // positive delays we reduce the correction by |shift_offset| to lower the + // risk of pushing the AEC into a non causal state. For negative delays + // we rely on the values up to a rounding error, hence compensate by 1 + // element to make sure to push the delay into the causal region. + delay_correction = -delay; + delay_correction += delay > self->shift_offset ? self->shift_offset : 1; + self->shift_offset--; + self->shift_offset = (self->shift_offset <= 1 ? 1 : self->shift_offset); + if (delay_correction > available_read - self->mult - 1) { + // There is not enough data in the buffer to perform this shift. Hence, + // we do not rely on the delay estimate and do nothing. + delay_correction = 0; + } else { + self->previous_delay = last_delay; + ++self->delay_correction_count; + } + } + } + // Update the |delay_quality_threshold| once we have our first delay + // correction. + if (self->delay_correction_count > 0) { + float delay_quality = WebRtc_last_delay_quality(self->delay_estimator); + delay_quality = + (delay_quality > kDelayQualityThresholdMax ? kDelayQualityThresholdMax + : delay_quality); + self->delay_quality_threshold = + (delay_quality > self->delay_quality_threshold + ? delay_quality + : self->delay_quality_threshold); + } + self->data_dumper->DumpRaw("aec_da_delay_correction", 1, &delay_correction); + + return delay_correction; +} + +static void RegressorPower(int num_partitions, + int latest_added_partition, + float x_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1], + float x_pow[PART_LEN1]) { + RTC_DCHECK_LT(latest_added_partition, num_partitions); + memset(x_pow, 0, PART_LEN1 * sizeof(x_pow[0])); + + int partition = latest_added_partition; + int x_fft_buf_position = partition * PART_LEN1; + for (int i = 0; i < num_partitions; ++i) { + for (int bin = 0; bin < PART_LEN1; ++bin) { + float re = x_fft_buf[0][x_fft_buf_position]; + float im = x_fft_buf[1][x_fft_buf_position]; + x_pow[bin] += re * re + im * im; + ++x_fft_buf_position; + } + + ++partition; + if (partition == num_partitions) { + partition = 0; + RTC_DCHECK_EQ(num_partitions * PART_LEN1, x_fft_buf_position); + x_fft_buf_position = 0; + } + } +} + +static void EchoSubtraction(const OouraFft& ooura_fft, + int num_partitions, + int extended_filter_enabled, + int* extreme_filter_divergence, + float filter_step_size, + float error_threshold, + float* x_fft, + int* x_fft_buf_block_pos, + float x_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1], + float* const y, + float x_pow[PART_LEN1], + float h_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1], + float echo_subtractor_output[PART_LEN]) { + float s_fft[2][PART_LEN1]; + float e_extended[PART_LEN2]; + float s_extended[PART_LEN2]; + float* s; + float e[PART_LEN]; + float e_fft[2][PART_LEN1]; + int i; + + // Update the x_fft_buf block position. + (*x_fft_buf_block_pos)--; + if ((*x_fft_buf_block_pos) == -1) { + *x_fft_buf_block_pos = num_partitions - 1; + } + + // Buffer x_fft. + memcpy(x_fft_buf[0] + (*x_fft_buf_block_pos) * PART_LEN1, x_fft, + sizeof(float) * PART_LEN1); + memcpy(x_fft_buf[1] + (*x_fft_buf_block_pos) * PART_LEN1, &x_fft[PART_LEN1], + sizeof(float) * PART_LEN1); + + memset(s_fft, 0, sizeof(s_fft)); + + // Conditionally reset the echo subtraction filter if the filter has diverged + // significantly. + if (!extended_filter_enabled && *extreme_filter_divergence) { + memset(h_fft_buf, 0, + 2 * kExtendedNumPartitions * PART_LEN1 * sizeof(h_fft_buf[0][0])); + *extreme_filter_divergence = 0; + } + + // Produce echo estimate s_fft. + WebRtcAec_FilterFar(num_partitions, *x_fft_buf_block_pos, x_fft_buf, + h_fft_buf, s_fft); + + // Compute the time-domain echo estimate s. + ScaledInverseFft(ooura_fft, s_fft, s_extended, 2.0f, 0); + s = &s_extended[PART_LEN]; + + // Compute the time-domain echo prediction error. + for (i = 0; i < PART_LEN; ++i) { + e[i] = y[i] - s[i]; + } + + // Compute the frequency domain echo prediction error. + memset(e_extended, 0, sizeof(float) * PART_LEN); + memcpy(e_extended + PART_LEN, e, sizeof(float) * PART_LEN); + Fft(ooura_fft, e_extended, e_fft); + + // Scale error signal inversely with far power. + WebRtcAec_ScaleErrorSignal(filter_step_size, error_threshold, x_pow, e_fft); + WebRtcAec_FilterAdaptation(ooura_fft, num_partitions, *x_fft_buf_block_pos, + x_fft_buf, e_fft, h_fft_buf); + memcpy(echo_subtractor_output, e, sizeof(float) * PART_LEN); +} + +static void FormSuppressionGain(AecCore* aec, + float cohde[PART_LEN1], + float cohxd[PART_LEN1], + float hNl[PART_LEN1]) { + float hNlDeAvg, hNlXdAvg; + float hNlPref[kPrefBandSize]; + float hNlFb = 0, hNlFbLow = 0; + const int prefBandSize = kPrefBandSize / aec->mult; + const float prefBandQuant = 0.75f, prefBandQuantLow = 0.5f; + const int minPrefBand = 4 / aec->mult; + // Power estimate smoothing coefficients. + const float* min_overdrive = aec->extended_filter_enabled + ? kExtendedMinOverDrive + : kNormalMinOverDrive; + + hNlXdAvg = 0; + for (int i = minPrefBand; i < prefBandSize + minPrefBand; ++i) { + hNlXdAvg += cohxd[i]; + } + hNlXdAvg /= prefBandSize; + hNlXdAvg = 1 - hNlXdAvg; + + hNlDeAvg = 0; + for (int i = minPrefBand; i < prefBandSize + minPrefBand; ++i) { + hNlDeAvg += cohde[i]; + } + hNlDeAvg /= prefBandSize; + + if (hNlXdAvg < 0.75f && hNlXdAvg < aec->hNlXdAvgMin) { + aec->hNlXdAvgMin = hNlXdAvg; + } + + if (hNlDeAvg > 0.98f && hNlXdAvg > 0.9f) { + aec->stNearState = 1; + } else if (hNlDeAvg < 0.95f || hNlXdAvg < 0.8f) { + aec->stNearState = 0; + } + + if (aec->hNlXdAvgMin == 1) { + aec->echoState = 0; + aec->overDrive = min_overdrive[aec->nlp_mode]; + + if (aec->stNearState == 1) { + memcpy(hNl, cohde, sizeof(hNl[0]) * PART_LEN1); + hNlFb = hNlDeAvg; + hNlFbLow = hNlDeAvg; + } else { + for (int i = 0; i < PART_LEN1; ++i) { + hNl[i] = 1 - cohxd[i]; + } + hNlFb = hNlXdAvg; + hNlFbLow = hNlXdAvg; + } + } else { + if (aec->stNearState == 1) { + aec->echoState = 0; + memcpy(hNl, cohde, sizeof(hNl[0]) * PART_LEN1); + hNlFb = hNlDeAvg; + hNlFbLow = hNlDeAvg; + } else { + aec->echoState = 1; + for (int i = 0; i < PART_LEN1; ++i) { + hNl[i] = WEBRTC_SPL_MIN(cohde[i], 1 - cohxd[i]); + } + + // Select an order statistic from the preferred bands. + // TODO(peah): Using quicksort now, but a selection algorithm may be + // preferred. + memcpy(hNlPref, &hNl[minPrefBand], sizeof(float) * prefBandSize); + qsort(hNlPref, prefBandSize, sizeof(float), CmpFloat); + hNlFb = hNlPref[static_cast(floor(prefBandQuant * + (prefBandSize - 1)))]; + hNlFbLow = hNlPref[static_cast(floor(prefBandQuantLow * + (prefBandSize - 1)))]; + } + } + + // Track the local filter minimum to determine suppression overdrive. + if (hNlFbLow < 0.6f && hNlFbLow < aec->hNlFbLocalMin) { + aec->hNlFbLocalMin = hNlFbLow; + aec->hNlFbMin = hNlFbLow; + aec->hNlNewMin = 1; + aec->hNlMinCtr = 0; + } + aec->hNlFbLocalMin = + WEBRTC_SPL_MIN(aec->hNlFbLocalMin + 0.0008f / aec->mult, 1); + aec->hNlXdAvgMin = WEBRTC_SPL_MIN(aec->hNlXdAvgMin + 0.0006f / aec->mult, 1); + + if (aec->hNlNewMin == 1) { + aec->hNlMinCtr++; + } + if (aec->hNlMinCtr == 2) { + aec->hNlNewMin = 0; + aec->hNlMinCtr = 0; + aec->overDrive = + WEBRTC_SPL_MAX(kTargetSupp[aec->nlp_mode] / + static_cast(log(aec->hNlFbMin + 1e-10f) + 1e-10f), + min_overdrive[aec->nlp_mode]); + } + + // Smooth the overdrive. + if (aec->overDrive < aec->overdrive_scaling) { + aec->overdrive_scaling = + 0.99f * aec->overdrive_scaling + 0.01f * aec->overDrive; + } else { + aec->overdrive_scaling = + 0.9f * aec->overdrive_scaling + 0.1f * aec->overDrive; + } + + // Apply the overdrive. + WebRtcAec_Overdrive(aec->overdrive_scaling, hNlFb, hNl); +} + +static void EchoSuppression(const OouraFft& ooura_fft, + AecCore* aec, + float* nearend_extended_block_lowest_band, + float farend_extended_block[PART_LEN2], + float* echo_subtractor_output, + float output[NUM_HIGH_BANDS_MAX + 1][PART_LEN]) { + float efw[2][PART_LEN1]; + float xfw[2][PART_LEN1]; + float dfw[2][PART_LEN1]; + float comfortNoiseHband[2][PART_LEN1]; + float fft[PART_LEN2]; + float nlpGainHband; + int i; + size_t j; + + // Coherence and non-linear filter + float cohde[PART_LEN1], cohxd[PART_LEN1]; + float hNl[PART_LEN1]; + + // Filter energy + const int delayEstInterval = 10 * aec->mult; + + float* xfw_ptr = NULL; + + // Update eBuf with echo subtractor output. + memcpy(aec->eBuf + PART_LEN, echo_subtractor_output, + sizeof(float) * PART_LEN); + + // Analysis filter banks for the echo suppressor. + // Windowed near-end ffts. + WindowData(fft, nearend_extended_block_lowest_band); + ooura_fft.Fft(fft); + StoreAsComplex(fft, dfw); + + // Windowed echo suppressor output ffts. + WindowData(fft, aec->eBuf); + ooura_fft.Fft(fft); + StoreAsComplex(fft, efw); + + // NLP + + // Convert far-end partition to the frequency domain with windowing. + WindowData(fft, farend_extended_block); + Fft(ooura_fft, fft, xfw); + xfw_ptr = &xfw[0][0]; + + // Buffer far. + memcpy(aec->xfwBuf, xfw_ptr, sizeof(float) * 2 * PART_LEN1); + + aec->delayEstCtr++; + if (aec->delayEstCtr == delayEstInterval) { + aec->delayEstCtr = 0; + aec->delayIdx = WebRtcAec_PartitionDelay(aec->num_partitions, aec->wfBuf); + } + + aec->data_dumper->DumpRaw("aec_nlp_delay", 1, &aec->delayIdx); + + // Use delayed far. + memcpy(xfw, aec->xfwBuf + aec->delayIdx * PART_LEN1, + sizeof(xfw[0][0]) * 2 * PART_LEN1); + + WebRtcAec_UpdateCoherenceSpectra(aec->mult, aec->extended_filter_enabled == 1, + efw, dfw, xfw, &aec->coherence_state, + &aec->divergeState, + &aec->extreme_filter_divergence); + + WebRtcAec_ComputeCoherence(&aec->coherence_state, cohde, cohxd); + + // Select the microphone signal as output if the filter is deemed to have + // diverged. + if (aec->divergeState) { + memcpy(efw, dfw, sizeof(efw[0][0]) * 2 * PART_LEN1); + } + + FormSuppressionGain(aec, cohde, cohxd, hNl); + + aec->data_dumper->DumpRaw("aec_nlp_gain", PART_LEN1, hNl); + + WebRtcAec_Suppress(hNl, efw); + + // Add comfort noise. + ComfortNoise(aec->num_bands > 1, &aec->seed, efw, comfortNoiseHband, + aec->noisePow, hNl); + + // Inverse error fft. + ScaledInverseFft(ooura_fft, efw, fft, 2.0f, 1); + + // Overlap and add to obtain output. + for (i = 0; i < PART_LEN; i++) { + output[0][i] = (fft[i] * WebRtcAec_sqrtHanning[i] + + aec->outBuf[i] * WebRtcAec_sqrtHanning[PART_LEN - i]); + + // Saturate output to keep it in the allowed range. + output[0][i] = WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, output[0][i], + WEBRTC_SPL_WORD16_MIN); + } + memcpy(aec->outBuf, &fft[PART_LEN], PART_LEN * sizeof(aec->outBuf[0])); + + // For H band + if (aec->num_bands > 1) { + // H band gain + // average nlp over low band: average over second half of freq spectrum + // (4->8khz) + GetHighbandGain(hNl, &nlpGainHband); + + // Inverse comfort_noise + ScaledInverseFft(ooura_fft, comfortNoiseHband, fft, 2.0f, 0); + + // compute gain factor + for (j = 1; j < aec->num_bands; ++j) { + for (i = 0; i < PART_LEN; i++) { + output[j][i] = aec->previous_nearend_block[j][i] * nlpGainHband; + } + } + + // Add some comfort noise where Hband is attenuated. + for (i = 0; i < PART_LEN; i++) { + output[1][i] += cnScaleHband * fft[i]; + } + + // Saturate output to keep it in the allowed range. + for (j = 1; j < aec->num_bands; ++j) { + for (i = 0; i < PART_LEN; i++) { + output[j][i] = WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, output[j][i], + WEBRTC_SPL_WORD16_MIN); + } + } + } + + // Copy the current block to the old position. + memcpy(aec->eBuf, aec->eBuf + PART_LEN, sizeof(float) * PART_LEN); + + memmove(aec->xfwBuf + PART_LEN1, aec->xfwBuf, + sizeof(aec->xfwBuf) - sizeof(complex_t) * PART_LEN1); +} + +static void ProcessNearendBlock( + AecCore* aec, + float farend_extended_block_lowest_band[PART_LEN2], + float nearend_block[NUM_HIGH_BANDS_MAX + 1][PART_LEN], + float output_block[NUM_HIGH_BANDS_MAX + 1][PART_LEN]) { + size_t i; + + float fft[PART_LEN2]; + float nearend_extended_block_lowest_band[PART_LEN2]; + float farend_fft[2][PART_LEN1]; + float nearend_fft[2][PART_LEN1]; + float far_spectrum = 0.0f; + float near_spectrum = 0.0f; + float abs_far_spectrum[PART_LEN1]; + float abs_near_spectrum[PART_LEN1]; + + const float gPow[2] = {0.9f, 0.1f}; + + // Noise estimate constants. + const int noiseInitBlocks = 500 * aec->mult; + const float step = 0.1f; + const float ramp = 1.0002f; + const float gInitNoise[2] = {0.999f, 0.001f}; + + float echo_subtractor_output[PART_LEN]; + + aec->data_dumper->DumpWav("aec_far", PART_LEN, + &farend_extended_block_lowest_band[PART_LEN], + std::min(aec->sampFreq, 16000), 1); + aec->data_dumper->DumpWav("aec_near", PART_LEN, &nearend_block[0][0], + std::min(aec->sampFreq, 16000), 1); + + if (aec->metricsMode == 1) { + // Update power levels + UpdateLevel( + &aec->farlevel, + CalculatePower(&farend_extended_block_lowest_band[PART_LEN], PART_LEN)); + UpdateLevel(&aec->nearlevel, + CalculatePower(&nearend_block[0][0], PART_LEN)); + } + + // Convert far-end signal to the frequency domain. + memcpy(fft, farend_extended_block_lowest_band, sizeof(float) * PART_LEN2); + Fft(aec->ooura_fft, fft, farend_fft); + + // Form extended nearend frame. + memcpy(&nearend_extended_block_lowest_band[0], + &aec->previous_nearend_block[0][0], sizeof(float) * PART_LEN); + memcpy(&nearend_extended_block_lowest_band[PART_LEN], &nearend_block[0][0], + sizeof(float) * PART_LEN); + + // Convert near-end signal to the frequency domain. + memcpy(fft, nearend_extended_block_lowest_band, sizeof(float) * PART_LEN2); + Fft(aec->ooura_fft, fft, nearend_fft); + + // Power smoothing. + if (aec->refined_adaptive_filter_enabled) { + for (i = 0; i < PART_LEN1; ++i) { + far_spectrum = farend_fft[0][i] * farend_fft[0][i] + + farend_fft[1][i] * farend_fft[1][i]; + // Calculate the magnitude spectrum. + abs_far_spectrum[i] = sqrtf(far_spectrum); + } + RegressorPower(aec->num_partitions, aec->xfBufBlockPos, aec->xfBuf, + aec->xPow); + } else { + for (i = 0; i < PART_LEN1; ++i) { + far_spectrum = farend_fft[0][i] * farend_fft[0][i] + + farend_fft[1][i] * farend_fft[1][i]; + aec->xPow[i] = + gPow[0] * aec->xPow[i] + gPow[1] * aec->num_partitions * far_spectrum; + // Calculate the magnitude spectrum. + abs_far_spectrum[i] = sqrtf(far_spectrum); + } + } + + for (i = 0; i < PART_LEN1; ++i) { + near_spectrum = nearend_fft[0][i] * nearend_fft[0][i] + + nearend_fft[1][i] * nearend_fft[1][i]; + aec->dPow[i] = gPow[0] * aec->dPow[i] + gPow[1] * near_spectrum; + // Calculate the magnitude spectrum. + abs_near_spectrum[i] = sqrtf(near_spectrum); + } + + // Estimate noise power. Wait until dPow is more stable. + if (aec->noiseEstCtr > 50) { + for (i = 0; i < PART_LEN1; i++) { + if (aec->dPow[i] < aec->dMinPow[i]) { + aec->dMinPow[i] = + (aec->dPow[i] + step * (aec->dMinPow[i] - aec->dPow[i])) * ramp; + } else { + aec->dMinPow[i] *= ramp; + } + } + } + + // Smooth increasing noise power from zero at the start, + // to avoid a sudden burst of comfort noise. + if (aec->noiseEstCtr < noiseInitBlocks) { + aec->noiseEstCtr++; + for (i = 0; i < PART_LEN1; i++) { + if (aec->dMinPow[i] > aec->dInitMinPow[i]) { + aec->dInitMinPow[i] = gInitNoise[0] * aec->dInitMinPow[i] + + gInitNoise[1] * aec->dMinPow[i]; + } else { + aec->dInitMinPow[i] = aec->dMinPow[i]; + } + } + aec->noisePow = aec->dInitMinPow; + } else { + aec->noisePow = aec->dMinPow; + } + + // Block wise delay estimation used for logging + if (aec->delay_logging_enabled) { + if (WebRtc_AddFarSpectrumFloat(aec->delay_estimator_farend, + abs_far_spectrum, PART_LEN1) == 0) { + int delay_estimate = WebRtc_DelayEstimatorProcessFloat( + aec->delay_estimator, abs_near_spectrum, PART_LEN1); + if (delay_estimate >= 0) { + // Update delay estimate buffer. + aec->delay_histogram[delay_estimate]++; + aec->num_delay_values++; + } + if (aec->delay_metrics_delivered == 1 && + aec->num_delay_values >= kDelayMetricsAggregationWindow) { + UpdateDelayMetrics(aec); + } + } + } + + // Perform echo subtraction. + EchoSubtraction( + aec->ooura_fft, aec->num_partitions, aec->extended_filter_enabled, + &aec->extreme_filter_divergence, aec->filter_step_size, + aec->error_threshold, &farend_fft[0][0], &aec->xfBufBlockPos, aec->xfBuf, + &nearend_block[0][0], aec->xPow, aec->wfBuf, echo_subtractor_output); + aec->data_dumper->DumpRaw("aec_h_fft", PART_LEN1 * aec->num_partitions, + &aec->wfBuf[0][0]); + aec->data_dumper->DumpRaw("aec_h_fft", PART_LEN1 * aec->num_partitions, + &aec->wfBuf[1][0]); + + aec->data_dumper->DumpWav("aec_out_linear", PART_LEN, echo_subtractor_output, + std::min(aec->sampFreq, 16000), 1); + + if (aec->metricsMode == 1) { + UpdateLevel(&aec->linoutlevel, + CalculatePower(echo_subtractor_output, PART_LEN)); + } + + // Perform echo suppression. + EchoSuppression(aec->ooura_fft, aec, nearend_extended_block_lowest_band, + farend_extended_block_lowest_band, echo_subtractor_output, + output_block); + + if (aec->metricsMode == 1) { + UpdateLevel(&aec->nlpoutlevel, + CalculatePower(&output_block[0][0], PART_LEN)); + UpdateMetrics(aec); + } + + // Store the nearend signal until the next frame. + for (i = 0; i < aec->num_bands; ++i) { + memcpy(&aec->previous_nearend_block[i][0], &nearend_block[i][0], + sizeof(float) * PART_LEN); + } + + aec->data_dumper->DumpWav("aec_out", PART_LEN, &output_block[0][0], + std::min(aec->sampFreq, 16000), 1); +} + +AecCore* WebRtcAec_CreateAec(int instance_count) { + AecCore* aec = new AecCore(instance_count); + + if (!aec) { + return NULL; + } + aec->nearend_buffer_size = 0; + memset(&aec->nearend_buffer[0], 0, sizeof(aec->nearend_buffer)); + // Start the output buffer with zeros to be able to produce + // a full output frame in the first frame. + aec->output_buffer_size = PART_LEN - (FRAME_LEN - PART_LEN); + memset(&aec->output_buffer[0], 0, sizeof(aec->output_buffer)); + + aec->delay_estimator_farend = + WebRtc_CreateDelayEstimatorFarend(PART_LEN1, kHistorySizeBlocks); + if (aec->delay_estimator_farend == NULL) { + WebRtcAec_FreeAec(aec); + return NULL; + } + // We create the delay_estimator with the same amount of maximum lookahead as + // the delay history size (kHistorySizeBlocks) for symmetry reasons. + aec->delay_estimator = WebRtc_CreateDelayEstimator( + aec->delay_estimator_farend, kHistorySizeBlocks); + if (aec->delay_estimator == NULL) { + WebRtcAec_FreeAec(aec); + return NULL; + } +#ifdef WEBRTC_ANDROID + aec->delay_agnostic_enabled = 1; // DA-AEC enabled by default. + // DA-AEC assumes the system is causal from the beginning and will self adjust + // the lookahead when shifting is required. + WebRtc_set_lookahead(aec->delay_estimator, 0); +#else + aec->delay_agnostic_enabled = 0; + WebRtc_set_lookahead(aec->delay_estimator, kLookaheadBlocks); +#endif + aec->extended_filter_enabled = 0; + aec->refined_adaptive_filter_enabled = false; + + rtc::CritScope cs_init(&WebRtcAec_CriticalSection); + static bool initted = false; + if (!initted) { + // Assembly optimization + WebRtcAec_FilterFar = FilterFar; + WebRtcAec_ScaleErrorSignal = ScaleErrorSignal; + WebRtcAec_FilterAdaptation = FilterAdaptation; + WebRtcAec_Overdrive = Overdrive; + WebRtcAec_Suppress = Suppress; + WebRtcAec_ComputeCoherence = ComputeCoherence; + WebRtcAec_UpdateCoherenceSpectra = UpdateCoherenceSpectra; + WebRtcAec_StoreAsComplex = StoreAsComplex; + WebRtcAec_PartitionDelay = PartitionDelay; + WebRtcAec_WindowData = WindowData; + +#if defined(WEBRTC_ARCH_X86_FAMILY) + if (WebRtc_GetCPUInfo(kSSE2)) { + WebRtcAec_InitAec_SSE2(); + } +#endif + +#if defined(MIPS_FPU_LE) + WebRtcAec_InitAec_mips(); +#endif + +#if defined(WEBRTC_HAS_NEON) + WebRtcAec_InitAec_neon(); +#endif + initted = true; + } + + return aec; +} + +void WebRtcAec_FreeAec(AecCore* aec) { + if (aec == NULL) { + return; + } + + WebRtc_FreeDelayEstimator(aec->delay_estimator); + WebRtc_FreeDelayEstimatorFarend(aec->delay_estimator_farend); + + delete aec; +} + +static void SetAdaptiveFilterStepSize(AecCore* aec) { + // Extended filter adaptation parameter. + // TODO(ajm): No narrowband tuning yet. + const float kExtendedMu = 0.4f; + + if (aec->refined_adaptive_filter_enabled) { + aec->filter_step_size = 0.05f; + } else { + if (aec->extended_filter_enabled) { + aec->filter_step_size = kExtendedMu; + } else { + if (aec->sampFreq == 8000) { + aec->filter_step_size = 0.6f; + } else { + aec->filter_step_size = 0.5f; + } + } + } +} + +static void SetErrorThreshold(AecCore* aec) { + // Extended filter adaptation parameter. + // TODO(ajm): No narrowband tuning yet. + static const float kExtendedErrorThreshold = 1.0e-6f; + + if (aec->extended_filter_enabled) { + aec->error_threshold = kExtendedErrorThreshold; + } else { + if (aec->sampFreq == 8000) { + aec->error_threshold = 2e-6f; + } else { + aec->error_threshold = 1.5e-6f; + } + } +} + +int WebRtcAec_InitAec(AecCore* aec, int sampFreq) { + int i; + aec->data_dumper->InitiateNewSetOfRecordings(); + + aec->sampFreq = sampFreq; + + SetAdaptiveFilterStepSize(aec); + SetErrorThreshold(aec); + + if (sampFreq == 8000) { + aec->num_bands = 1; + } else { + aec->num_bands = (size_t)(sampFreq / 16000); + } + + // Start the output buffer with zeros to be able to produce + // a full output frame in the first frame. + aec->output_buffer_size = PART_LEN - (FRAME_LEN - PART_LEN); + memset(&aec->output_buffer[0], 0, sizeof(aec->output_buffer)); + aec->nearend_buffer_size = 0; + memset(&aec->nearend_buffer[0], 0, sizeof(aec->nearend_buffer)); + + // Initialize far-end buffer. + aec->farend_block_buffer_.ReInit(); + + aec->system_delay = 0; + + if (WebRtc_InitDelayEstimatorFarend(aec->delay_estimator_farend) != 0) { + return -1; + } + if (WebRtc_InitDelayEstimator(aec->delay_estimator) != 0) { + return -1; + } + aec->delay_logging_enabled = 0; + aec->delay_metrics_delivered = 0; + memset(aec->delay_histogram, 0, sizeof(aec->delay_histogram)); + aec->num_delay_values = 0; + aec->delay_median = -1; + aec->delay_std = -1; + aec->fraction_poor_delays = -1.0f; + + aec->previous_delay = -2; // (-2): Uninitialized. + aec->delay_correction_count = 0; + aec->shift_offset = kInitialShiftOffset; + aec->delay_quality_threshold = kDelayQualityThresholdMin; + + aec->num_partitions = kNormalNumPartitions; + + // Update the delay estimator with filter length. We use half the + // |num_partitions| to take the echo path into account. In practice we say + // that the echo has a duration of maximum half |num_partitions|, which is not + // true, but serves as a crude measure. + WebRtc_set_allowed_offset(aec->delay_estimator, aec->num_partitions / 2); + // TODO(bjornv): I currently hard coded the enable. Once we've established + // that AECM has no performance regression, robust_validation will be enabled + // all the time and the APIs to turn it on/off will be removed. Hence, remove + // this line then. + WebRtc_enable_robust_validation(aec->delay_estimator, 1); + aec->frame_count = 0; + + // Default target suppression mode. + aec->nlp_mode = 1; + + // Sampling frequency multiplier w.r.t. 8 kHz. + // In case of multiple bands we process the lower band in 16 kHz, hence the + // multiplier is always 2. + if (aec->num_bands > 1) { + aec->mult = 2; + } else { + aec->mult = static_cast(aec->sampFreq) / 8000; + } + + aec->farBufWritePos = 0; + aec->farBufReadPos = 0; + + aec->inSamples = 0; + aec->outSamples = 0; + aec->knownDelay = 0; + + // Initialize buffers + memset(aec->previous_nearend_block, 0, sizeof(aec->previous_nearend_block)); + memset(aec->eBuf, 0, sizeof(aec->eBuf)); + + memset(aec->xPow, 0, sizeof(aec->xPow)); + memset(aec->dPow, 0, sizeof(aec->dPow)); + memset(aec->dInitMinPow, 0, sizeof(aec->dInitMinPow)); + aec->noisePow = aec->dInitMinPow; + aec->noiseEstCtr = 0; + + // Initial comfort noise power + for (i = 0; i < PART_LEN1; i++) { + aec->dMinPow[i] = 1.0e6f; + } + + // Holds the last block written to + aec->xfBufBlockPos = 0; + // TODO(peah): Investigate need for these initializations. Deleting them + // doesn't change the output at all and yields 0.4% overall speedup. + memset(aec->xfBuf, 0, sizeof(complex_t) * kExtendedNumPartitions * PART_LEN1); + memset(aec->wfBuf, 0, sizeof(complex_t) * kExtendedNumPartitions * PART_LEN1); + memset(aec->coherence_state.sde, 0, sizeof(complex_t) * PART_LEN1); + memset(aec->coherence_state.sxd, 0, sizeof(complex_t) * PART_LEN1); + memset(aec->xfwBuf, 0, + sizeof(complex_t) * kExtendedNumPartitions * PART_LEN1); + memset(aec->coherence_state.se, 0, sizeof(float) * PART_LEN1); + + // To prevent numerical instability in the first block. + for (i = 0; i < PART_LEN1; i++) { + aec->coherence_state.sd[i] = 1; + } + for (i = 0; i < PART_LEN1; i++) { + aec->coherence_state.sx[i] = 1; + } + + memset(aec->hNs, 0, sizeof(aec->hNs)); + memset(aec->outBuf, 0, sizeof(float) * PART_LEN); + + aec->hNlFbMin = 1; + aec->hNlFbLocalMin = 1; + aec->hNlXdAvgMin = 1; + aec->hNlNewMin = 0; + aec->hNlMinCtr = 0; + aec->overDrive = 2; + aec->overdrive_scaling = 2; + aec->delayIdx = 0; + aec->stNearState = 0; + aec->echoState = 0; + aec->divergeState = 0; + + aec->seed = 777; + aec->delayEstCtr = 0; + + aec->extreme_filter_divergence = 0; + + // Metrics disabled by default + aec->metricsMode = 0; + InitMetrics(aec); + + return 0; +} + +void WebRtcAec_BufferFarendBlock(AecCore* aec, const float* farend) { + // Check if the buffer is full, and in that case flush the oldest data. + if (aec->farend_block_buffer_.AvaliableSpace() < 1) { + aec->farend_block_buffer_.AdjustSize(1); + } + aec->farend_block_buffer_.Insert(farend); +} + +int WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(AecCore* aec, + int buffer_size_decrease) { + int achieved_buffer_size_decrease = + aec->farend_block_buffer_.AdjustSize(buffer_size_decrease); + aec->system_delay -= achieved_buffer_size_decrease * PART_LEN; + return achieved_buffer_size_decrease; +} + +void FormNearendBlock( + size_t nearend_start_index, + size_t num_bands, + const float* const* nearend_frame, + size_t num_samples_from_nearend_frame, + const float nearend_buffer[NUM_HIGH_BANDS_MAX + 1] + [PART_LEN - (FRAME_LEN - PART_LEN)], + float nearend_block[NUM_HIGH_BANDS_MAX + 1][PART_LEN]) { + RTC_DCHECK_LE(num_samples_from_nearend_frame, PART_LEN); + const int num_samples_from_buffer = PART_LEN - num_samples_from_nearend_frame; + + if (num_samples_from_buffer > 0) { + for (size_t i = 0; i < num_bands; ++i) { + memcpy(&nearend_block[i][0], &nearend_buffer[i][0], + num_samples_from_buffer * sizeof(float)); + } + } + + for (size_t i = 0; i < num_bands; ++i) { + memcpy(&nearend_block[i][num_samples_from_buffer], + &nearend_frame[i][nearend_start_index], + num_samples_from_nearend_frame * sizeof(float)); + } +} + +void BufferNearendFrame( + size_t nearend_start_index, + size_t num_bands, + const float* const* nearend_frame, + size_t num_samples_to_buffer, + float nearend_buffer[NUM_HIGH_BANDS_MAX + 1] + [PART_LEN - (FRAME_LEN - PART_LEN)]) { + for (size_t i = 0; i < num_bands; ++i) { + memcpy( + &nearend_buffer[i][0], + &nearend_frame[i] + [nearend_start_index + FRAME_LEN - num_samples_to_buffer], + num_samples_to_buffer * sizeof(float)); + } +} + +void BufferOutputBlock(size_t num_bands, + const float output_block[NUM_HIGH_BANDS_MAX + 1] + [PART_LEN], + size_t* output_buffer_size, + float output_buffer[NUM_HIGH_BANDS_MAX + 1] + [2 * PART_LEN]) { + for (size_t i = 0; i < num_bands; ++i) { + memcpy(&output_buffer[i][*output_buffer_size], &output_block[i][0], + PART_LEN * sizeof(float)); + } + (*output_buffer_size) += PART_LEN; +} + +void FormOutputFrame(size_t output_start_index, + size_t num_bands, + size_t* output_buffer_size, + float output_buffer[NUM_HIGH_BANDS_MAX + 1][2 * PART_LEN], + float* const* output_frame) { + RTC_DCHECK_LE(FRAME_LEN, *output_buffer_size); + for (size_t i = 0; i < num_bands; ++i) { + memcpy(&output_frame[i][output_start_index], &output_buffer[i][0], + FRAME_LEN * sizeof(float)); + } + (*output_buffer_size) -= FRAME_LEN; + if (*output_buffer_size > 0) { + RTC_DCHECK_GE(2 * PART_LEN - FRAME_LEN, (*output_buffer_size)); + for (size_t i = 0; i < num_bands; ++i) { + memcpy(&output_buffer[i][0], &output_buffer[i][FRAME_LEN], + (*output_buffer_size) * sizeof(float)); + } + } +} + +void WebRtcAec_ProcessFrames(AecCore* aec, + const float* const* nearend, + size_t num_bands, + size_t num_samples, + int knownDelay, + float* const* out) { + RTC_DCHECK(num_samples == 80 || num_samples == 160); + + aec->frame_count++; + // For each frame the process is as follows: + // 1) If the system_delay indicates on being too small for processing a + // frame we stuff the buffer with enough data for 10 ms. + // 2 a) Adjust the buffer to the system delay, by moving the read pointer. + // b) Apply signal based delay correction, if we have detected poor AEC + // performance. + // 3) TODO(bjornv): Investigate if we need to add this: + // If we can't move read pointer due to buffer size limitations we + // flush/stuff the buffer. + // 4) Process as many partitions as possible. + // 5) Update the |system_delay| with respect to a full frame of FRAME_LEN + // samples. Even though we will have data left to process (we work with + // partitions) we consider updating a whole frame, since that's the + // amount of data we input and output in audio_processing. + // 6) Update the outputs. + + // The AEC has two different delay estimation algorithms built in. The + // first relies on delay input values from the user and the amount of + // shifted buffer elements is controlled by |knownDelay|. This delay will + // give a guess on how much we need to shift far-end buffers to align with + // the near-end signal. The other delay estimation algorithm uses the + // far- and near-end signals to find the offset between them. This one + // (called "signal delay") is then used to fine tune the alignment, or + // simply compensate for errors in the system based one. + // Note that the two algorithms operate independently. Currently, we only + // allow one algorithm to be turned on. + + RTC_DCHECK_EQ(aec->num_bands, num_bands); + + for (size_t j = 0; j < num_samples; j += FRAME_LEN) { + // 1) At most we process |aec->mult|+1 partitions in 10 ms. Make sure we + // have enough far-end data for that by stuffing the buffer if the + // |system_delay| indicates others. + if (aec->system_delay < FRAME_LEN) { + // We don't have enough data so we rewind 10 ms. + WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(aec, -(aec->mult + 1)); + } + + if (!aec->delay_agnostic_enabled) { + // 2 a) Compensate for a possible change in the system delay. + + // TODO(bjornv): Investigate how we should round the delay difference; + // right now we know that incoming |knownDelay| is underestimated when + // it's less than |aec->knownDelay|. We therefore, round (-32) in that + // direction. In the other direction, we don't have this situation, but + // might flush one partition too little. This can cause non-causality, + // which should be investigated. Maybe, allow for a non-symmetric + // rounding, like -16. + int move_elements = (aec->knownDelay - knownDelay - 32) / PART_LEN; + int moved_elements = aec->farend_block_buffer_.AdjustSize(move_elements); + MaybeLogDelayAdjustment(moved_elements * (aec->sampFreq == 8000 ? 8 : 4), + DelaySource::kSystemDelay); + aec->knownDelay -= moved_elements * PART_LEN; + } else { + // 2 b) Apply signal based delay correction. + int move_elements = SignalBasedDelayCorrection(aec); + int moved_elements = aec->farend_block_buffer_.AdjustSize(move_elements); + MaybeLogDelayAdjustment(moved_elements * (aec->sampFreq == 8000 ? 8 : 4), + DelaySource::kDelayAgnostic); + int far_near_buffer_diff = + aec->farend_block_buffer_.Size() - + (aec->nearend_buffer_size + FRAME_LEN) / PART_LEN; + WebRtc_SoftResetDelayEstimator(aec->delay_estimator, moved_elements); + WebRtc_SoftResetDelayEstimatorFarend(aec->delay_estimator_farend, + moved_elements); + // If we rely on reported system delay values only, a buffer underrun here + // can never occur since we've taken care of that in 1) above. Here, we + // apply signal based delay correction and can therefore end up with + // buffer underruns since the delay estimation can be wrong. We therefore + // stuff the buffer with enough elements if needed. + if (far_near_buffer_diff < 0) { + WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(aec, + far_near_buffer_diff); + } + } + + static_assert( + 16 == (FRAME_LEN - PART_LEN), + "These constants need to be properly related for this code to work"); + float output_block[NUM_HIGH_BANDS_MAX + 1][PART_LEN]; + float nearend_block[NUM_HIGH_BANDS_MAX + 1][PART_LEN]; + float farend_extended_block_lowest_band[PART_LEN2]; + + // Form and process a block of nearend samples, buffer the output block of + // samples. + aec->farend_block_buffer_.ExtractExtendedBlock( + farend_extended_block_lowest_band); + FormNearendBlock(j, num_bands, nearend, PART_LEN - aec->nearend_buffer_size, + aec->nearend_buffer, nearend_block); + ProcessNearendBlock(aec, farend_extended_block_lowest_band, nearend_block, + output_block); + BufferOutputBlock(num_bands, output_block, &aec->output_buffer_size, + aec->output_buffer); + + if ((FRAME_LEN - PART_LEN + aec->nearend_buffer_size) == PART_LEN) { + // When possible (every fourth frame) form and process a second block of + // nearend samples, buffer the output block of samples. + aec->farend_block_buffer_.ExtractExtendedBlock( + farend_extended_block_lowest_band); + FormNearendBlock(j + FRAME_LEN - PART_LEN, num_bands, nearend, PART_LEN, + aec->nearend_buffer, nearend_block); + ProcessNearendBlock(aec, farend_extended_block_lowest_band, nearend_block, + output_block); + BufferOutputBlock(num_bands, output_block, &aec->output_buffer_size, + aec->output_buffer); + + // Reset the buffer size as there are no samples left in the nearend input + // to buffer. + aec->nearend_buffer_size = 0; + } else { + // Buffer the remaining samples in the nearend input. + aec->nearend_buffer_size += FRAME_LEN - PART_LEN; + BufferNearendFrame(j, num_bands, nearend, aec->nearend_buffer_size, + aec->nearend_buffer); + } + + // 5) Update system delay with respect to the entire frame. + aec->system_delay -= FRAME_LEN; + + // 6) Form the output frame. + FormOutputFrame(j, num_bands, &aec->output_buffer_size, aec->output_buffer, + out); + } +} + +int WebRtcAec_GetDelayMetricsCore(AecCore* self, + int* median, + int* std, + float* fraction_poor_delays) { + RTC_DCHECK(self); + RTC_DCHECK(median); + RTC_DCHECK(std); + + if (self->delay_logging_enabled == 0) { + // Logging disabled. + return -1; + } + + if (self->delay_metrics_delivered == 0) { + UpdateDelayMetrics(self); + self->delay_metrics_delivered = 1; + } + *median = self->delay_median; + *std = self->delay_std; + *fraction_poor_delays = self->fraction_poor_delays; + + return 0; +} + +int WebRtcAec_echo_state(AecCore* self) { + return self->echoState; +} + +void WebRtcAec_GetEchoStats(AecCore* self, + Stats* erl, + Stats* erle, + Stats* a_nlp, + float* divergent_filter_fraction) { + RTC_DCHECK(erl); + RTC_DCHECK(erle); + RTC_DCHECK(a_nlp); + *erl = self->erl; + *erle = self->erle; + *a_nlp = self->aNlp; + *divergent_filter_fraction = + self->divergent_filter_fraction.GetLatestFraction(); +} + +void WebRtcAec_SetConfigCore(AecCore* self, + int nlp_mode, + int metrics_mode, + int delay_logging) { + RTC_DCHECK_GE(nlp_mode, 0); + RTC_DCHECK_LT(nlp_mode, 3); + self->nlp_mode = nlp_mode; + self->metricsMode = metrics_mode; + if (self->metricsMode) { + InitMetrics(self); + } + // Turn on delay logging if it is either set explicitly or if delay agnostic + // AEC is enabled (which requires delay estimates). + self->delay_logging_enabled = delay_logging || self->delay_agnostic_enabled; + if (self->delay_logging_enabled) { + memset(self->delay_histogram, 0, sizeof(self->delay_histogram)); + } +} + +void WebRtcAec_enable_delay_agnostic(AecCore* self, int enable) { + self->delay_agnostic_enabled = enable; +} + +int WebRtcAec_delay_agnostic_enabled(AecCore* self) { + return self->delay_agnostic_enabled; +} + +void WebRtcAec_enable_refined_adaptive_filter(AecCore* self, bool enable) { + self->refined_adaptive_filter_enabled = enable; + SetAdaptiveFilterStepSize(self); + SetErrorThreshold(self); +} + +bool WebRtcAec_refined_adaptive_filter_enabled(const AecCore* self) { + return self->refined_adaptive_filter_enabled; +} + +void WebRtcAec_enable_extended_filter(AecCore* self, int enable) { + self->extended_filter_enabled = enable; + SetAdaptiveFilterStepSize(self); + SetErrorThreshold(self); + self->num_partitions = enable ? kExtendedNumPartitions : kNormalNumPartitions; + // Update the delay estimator with filter length. See InitAEC() for details. + WebRtc_set_allowed_offset(self->delay_estimator, self->num_partitions / 2); +} + +int WebRtcAec_extended_filter_enabled(AecCore* self) { + return self->extended_filter_enabled; +} + +int WebRtcAec_system_delay(AecCore* self) { + return self->system_delay; +} + +void WebRtcAec_SetSystemDelay(AecCore* self, int delay) { + RTC_DCHECK_GE(delay, 0); + self->system_delay = delay; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.h new file mode 100644 index 0000000000..78596ec4b4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.h @@ -0,0 +1,335 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * Specifies the interface for the AEC core. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_H_ +#define MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_H_ + +#include + +#include + +extern "C" { +#include "common_audio/ring_buffer.h" +} +#include "common_audio/wav_file.h" +#include "modules/audio_processing/aec/aec_common.h" +#include "modules/audio_processing/utility/block_mean_calculator.h" +#include "modules/audio_processing/utility/ooura_fft.h" +#include "rtc_base/constructormagic.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#define FRAME_LEN 80 +#define PART_LEN 64 // Length of partition +#define PART_LEN1 (PART_LEN + 1) // Unique fft coefficients +#define PART_LEN2 (PART_LEN * 2) // Length of partition * 2 +#define NUM_HIGH_BANDS_MAX 2 // Max number of high bands + +class ApmDataDumper; + +typedef float complex_t[2]; +// For performance reasons, some arrays of complex numbers are replaced by twice +// as long arrays of float, all the real parts followed by all the imaginary +// ones (complex_t[SIZE] -> float[2][SIZE]). This allows SIMD optimizations and +// is better than two arrays (one for the real parts and one for the imaginary +// parts) as this other way would require two pointers instead of one and cause +// extra register spilling. This also allows the offsets to be calculated at +// compile time. + +// Metrics +enum { kOffsetLevel = -100 }; + +typedef struct Stats { + float instant; + float average; + float min; + float max; + float sum; + float hisum; + float himean; + size_t counter; + size_t hicounter; +} Stats; + +// Number of partitions for the extended filter mode. The first one is an enum +// to be used in array declarations, as it represents the maximum filter length. +enum { kExtendedNumPartitions = 32 }; +static const int kNormalNumPartitions = 12; + +// Delay estimator constants, used for logging and delay compensation if +// if reported delays are disabled. +enum { kLookaheadBlocks = 15 }; +enum { + // 500 ms for 16 kHz which is equivalent with the limit of reported delays. + kHistorySizeBlocks = 125 +}; + +typedef struct PowerLevel { + PowerLevel(); + + BlockMeanCalculator framelevel; + BlockMeanCalculator averagelevel; + float minlevel; +} PowerLevel; + +class BlockBuffer { + public: + BlockBuffer(); + ~BlockBuffer(); + void ReInit(); + void Insert(const float block[PART_LEN]); + void ExtractExtendedBlock(float extended_block[PART_LEN]); + int AdjustSize(int buffer_size_decrease); + size_t Size(); + size_t AvaliableSpace(); + + private: + RingBuffer* buffer_; +}; + +class DivergentFilterFraction { + public: + DivergentFilterFraction(); + + // Reset. + void Reset(); + + void AddObservation(const PowerLevel& nearlevel, + const PowerLevel& linoutlevel, + const PowerLevel& nlpoutlevel); + + // Return the latest fraction. + float GetLatestFraction() const; + + private: + // Clear all values added. + void Clear(); + + size_t count_; + size_t occurrence_; + float fraction_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DivergentFilterFraction); +}; + +typedef struct CoherenceState { + complex_t sde[PART_LEN1]; // cross-psd of nearend and error + complex_t sxd[PART_LEN1]; // cross-psd of farend and nearend + float sx[PART_LEN1], sd[PART_LEN1], se[PART_LEN1]; // far, near, error psd +} CoherenceState; + +struct AecCore { + explicit AecCore(int instance_index); + ~AecCore(); + + std::unique_ptr data_dumper; + const OouraFft ooura_fft; + + CoherenceState coherence_state; + + int farBufWritePos, farBufReadPos; + + int knownDelay; + int inSamples, outSamples; + int delayEstCtr; + + // Nearend buffer used for changing from FRAME_LEN to PART_LEN sample block + // sizes. The buffer stores all the incoming bands and for each band a maximum + // of PART_LEN - (FRAME_LEN - PART_LEN) values need to be buffered in order to + // change the block size from FRAME_LEN to PART_LEN. + float nearend_buffer[NUM_HIGH_BANDS_MAX + 1] + [PART_LEN - (FRAME_LEN - PART_LEN)]; + size_t nearend_buffer_size; + float output_buffer[NUM_HIGH_BANDS_MAX + 1][2 * PART_LEN]; + size_t output_buffer_size; + + float eBuf[PART_LEN2]; // error + + float previous_nearend_block[NUM_HIGH_BANDS_MAX + 1][PART_LEN]; + + float xPow[PART_LEN1]; + float dPow[PART_LEN1]; + float dMinPow[PART_LEN1]; + float dInitMinPow[PART_LEN1]; + float* noisePow; + + float xfBuf[2][kExtendedNumPartitions * PART_LEN1]; // farend fft buffer + float wfBuf[2][kExtendedNumPartitions * PART_LEN1]; // filter fft + // Farend windowed fft buffer. + complex_t xfwBuf[kExtendedNumPartitions * PART_LEN1]; + + float hNs[PART_LEN1]; + float hNlFbMin, hNlFbLocalMin; + float hNlXdAvgMin; + int hNlNewMin, hNlMinCtr; + float overDrive; + float overdrive_scaling; + int nlp_mode; + float outBuf[PART_LEN]; + int delayIdx; + + short stNearState, echoState; + short divergeState; + + int xfBufBlockPos; + + BlockBuffer farend_block_buffer_; + + int system_delay; // Current system delay buffered in AEC. + + int mult; // sampling frequency multiple + int sampFreq = 16000; + size_t num_bands; + uint32_t seed; + + float filter_step_size; // stepsize + float error_threshold; // error threshold + + int noiseEstCtr; + + PowerLevel farlevel; + PowerLevel nearlevel; + PowerLevel linoutlevel; + PowerLevel nlpoutlevel; + + int metricsMode; + int stateCounter; + Stats erl; + Stats erle; + Stats aNlp; + Stats rerl; + DivergentFilterFraction divergent_filter_fraction; + + // Quantities to control H band scaling for SWB input + int freq_avg_ic; // initial bin for averaging nlp gain + int flag_Hband_cn; // for comfort noise + float cn_scale_Hband; // scale for comfort noise in H band + + int delay_metrics_delivered; + int delay_histogram[kHistorySizeBlocks]; + int num_delay_values; + int delay_median; + int delay_std; + float fraction_poor_delays; + int delay_logging_enabled; + void* delay_estimator_farend; + void* delay_estimator; + // Variables associated with delay correction through signal based delay + // estimation feedback. + int previous_delay; + int delay_correction_count; + int shift_offset; + float delay_quality_threshold; + int frame_count; + + // 0 = delay agnostic mode (signal based delay correction) disabled. + // Otherwise enabled. + int delay_agnostic_enabled; + // 1 = extended filter mode enabled, 0 = disabled. + int extended_filter_enabled; + // 1 = refined filter adaptation aec mode enabled, 0 = disabled. + bool refined_adaptive_filter_enabled; + + // Runtime selection of number of filter partitions. + int num_partitions; + + // Flag that extreme filter divergence has been detected by the Echo + // Suppressor. + int extreme_filter_divergence; +}; + +AecCore* WebRtcAec_CreateAec(int instance_count); // Returns NULL on error. +void WebRtcAec_FreeAec(AecCore* aec); +int WebRtcAec_InitAec(AecCore* aec, int sampFreq); +void WebRtcAec_InitAec_SSE2(void); +#if defined(MIPS_FPU_LE) +void WebRtcAec_InitAec_mips(void); +#endif +#if defined(WEBRTC_HAS_NEON) +void WebRtcAec_InitAec_neon(void); +#endif + +void WebRtcAec_BufferFarendBlock(AecCore* aec, const float* farend); +void WebRtcAec_ProcessFrames(AecCore* aec, + const float* const* nearend, + size_t num_bands, + size_t num_samples, + int knownDelay, + float* const* out); + +// A helper function to call adjust the farend buffer size. +// Returns the number of elements the size was decreased with, and adjusts +// |system_delay| by the corresponding amount in ms. +int WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(AecCore* aec, + int size_decrease); + +// Calculates the median, standard deviation and amount of poor values among the +// delay estimates aggregated up to the first call to the function. After that +// first call the metrics are aggregated and updated every second. With poor +// values we mean values that most likely will cause the AEC to perform poorly. +// TODO(bjornv): Consider changing tests and tools to handle constant +// constant aggregation window throughout the session instead. +int WebRtcAec_GetDelayMetricsCore(AecCore* self, + int* median, + int* std, + float* fraction_poor_delays); + +// Returns the echo state (1: echo, 0: no echo). +int WebRtcAec_echo_state(AecCore* self); + +// Gets statistics of the echo metrics ERL, ERLE, A_NLP. +void WebRtcAec_GetEchoStats(AecCore* self, + Stats* erl, + Stats* erle, + Stats* a_nlp, + float* divergent_filter_fraction); + +// Sets local configuration modes. +void WebRtcAec_SetConfigCore(AecCore* self, + int nlp_mode, + int metrics_mode, + int delay_logging); + +// Non-zero enables, zero disables. +void WebRtcAec_enable_delay_agnostic(AecCore* self, int enable); + +// Returns non-zero if delay agnostic (i.e., signal based delay estimation) is +// enabled and zero if disabled. +int WebRtcAec_delay_agnostic_enabled(AecCore* self); + +// Turns on/off the refined adaptive filter feature. +void WebRtcAec_enable_refined_adaptive_filter(AecCore* self, bool enable); + +// Returns whether the refined adaptive filter is enabled. +bool WebRtcAec_refined_adaptive_filter(const AecCore* self); + +// Enables or disables extended filter mode. Non-zero enables, zero disables. +void WebRtcAec_enable_extended_filter(AecCore* self, int enable); + +// Returns non-zero if extended filter mode is enabled and zero if disabled. +int WebRtcAec_extended_filter_enabled(AecCore* self); + +// Returns the current |system_delay|, i.e., the buffered difference between +// far-end and near-end. +int WebRtcAec_system_delay(AecCore* self); + +// Sets the |system_delay| to |value|. Note that if the value is changed +// improperly, there can be a performance regression. So it should be used with +// care. +void WebRtcAec_SetSystemDelay(AecCore* self, int delay); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_mips.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_mips.cc new file mode 100644 index 0000000000..ebe6349e93 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_mips.cc @@ -0,0 +1,490 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * The core AEC algorithm, which is presented with time-aligned signals. + */ + +#include "modules/audio_processing/aec/aec_core.h" + +#include + +extern "C" { +#include "common_audio/signal_processing/include/signal_processing_library.h" +} +#include "modules/audio_processing/aec/aec_core_optimized_methods.h" +#include "modules/audio_processing/utility/ooura_fft.h" + +namespace webrtc { + +extern const float WebRtcAec_weightCurve[65]; +extern const float WebRtcAec_overDriveCurve[65]; + +void WebRtcAec_FilterFar_mips( + int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float y_fft[2][PART_LEN1]) { + int i; + for (i = 0; i < num_partitions; i++) { + int xPos = (i + x_fft_buf_block_pos) * PART_LEN1; + int pos = i * PART_LEN1; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * (PART_LEN1); + } + float* yf0 = y_fft[0]; + float* yf1 = y_fft[1]; + float* aRe = x_fft_buf[0] + xPos; + float* aIm = x_fft_buf[1] + xPos; + float* bRe = h_fft_buf[0] + pos; + float* bIm = h_fft_buf[1] + pos; + float f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13; + int len = PART_LEN1 >> 1; + + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "lwc1 %[f0], 0(%[aRe]) \n\t" + "lwc1 %[f1], 0(%[bRe]) \n\t" + "lwc1 %[f2], 0(%[bIm]) \n\t" + "lwc1 %[f3], 0(%[aIm]) \n\t" + "lwc1 %[f4], 4(%[aRe]) \n\t" + "lwc1 %[f5], 4(%[bRe]) \n\t" + "lwc1 %[f6], 4(%[bIm]) \n\t" + "mul.s %[f8], %[f0], %[f1] \n\t" + "mul.s %[f0], %[f0], %[f2] \n\t" + "mul.s %[f9], %[f4], %[f5] \n\t" + "mul.s %[f4], %[f4], %[f6] \n\t" + "lwc1 %[f7], 4(%[aIm]) \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f12], %[f2], %[f3] \n\t" + "mul.s %[f1], %[f3], %[f1] \n\t" + "mul.s %[f11], %[f6], %[f7] \n\t" + "addiu %[aRe], %[aRe], 8 \n\t" + "addiu %[aIm], %[aIm], 8 \n\t" + "addiu %[len], %[len], -1 \n\t" + "sub.s %[f8], %[f8], %[f12] \n\t" + "mul.s %[f12], %[f7], %[f5] \n\t" + "lwc1 %[f2], 0(%[yf0]) \n\t" + "add.s %[f1], %[f0], %[f1] \n\t" + "lwc1 %[f3], 0(%[yf1]) \n\t" + "sub.s %[f9], %[f9], %[f11] \n\t" + "lwc1 %[f6], 4(%[yf0]) \n\t" + "add.s %[f4], %[f4], %[f12] \n\t" +#else // #if !defined(MIPS32_R2_LE) + "addiu %[aRe], %[aRe], 8 \n\t" + "addiu %[aIm], %[aIm], 8 \n\t" + "addiu %[len], %[len], -1 \n\t" + "nmsub.s %[f8], %[f8], %[f2], %[f3] \n\t" + "lwc1 %[f2], 0(%[yf0]) \n\t" + "madd.s %[f1], %[f0], %[f3], %[f1] \n\t" + "lwc1 %[f3], 0(%[yf1]) \n\t" + "nmsub.s %[f9], %[f9], %[f6], %[f7] \n\t" + "lwc1 %[f6], 4(%[yf0]) \n\t" + "madd.s %[f4], %[f4], %[f7], %[f5] \n\t" +#endif // #if !defined(MIPS32_R2_LE) + "lwc1 %[f5], 4(%[yf1]) \n\t" + "add.s %[f2], %[f2], %[f8] \n\t" + "addiu %[bRe], %[bRe], 8 \n\t" + "addiu %[bIm], %[bIm], 8 \n\t" + "add.s %[f3], %[f3], %[f1] \n\t" + "add.s %[f6], %[f6], %[f9] \n\t" + "add.s %[f5], %[f5], %[f4] \n\t" + "swc1 %[f2], 0(%[yf0]) \n\t" + "swc1 %[f3], 0(%[yf1]) \n\t" + "swc1 %[f6], 4(%[yf0]) \n\t" + "swc1 %[f5], 4(%[yf1]) \n\t" + "addiu %[yf0], %[yf0], 8 \n\t" + "bgtz %[len], 1b \n\t" + " addiu %[yf1], %[yf1], 8 \n\t" + "lwc1 %[f0], 0(%[aRe]) \n\t" + "lwc1 %[f1], 0(%[bRe]) \n\t" + "lwc1 %[f2], 0(%[bIm]) \n\t" + "lwc1 %[f3], 0(%[aIm]) \n\t" + "mul.s %[f8], %[f0], %[f1] \n\t" + "mul.s %[f0], %[f0], %[f2] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f12], %[f2], %[f3] \n\t" + "mul.s %[f1], %[f3], %[f1] \n\t" + "sub.s %[f8], %[f8], %[f12] \n\t" + "lwc1 %[f2], 0(%[yf0]) \n\t" + "add.s %[f1], %[f0], %[f1] \n\t" + "lwc1 %[f3], 0(%[yf1]) \n\t" +#else // #if !defined(MIPS32_R2_LE) + "nmsub.s %[f8], %[f8], %[f2], %[f3] \n\t" + "lwc1 %[f2], 0(%[yf0]) \n\t" + "madd.s %[f1], %[f0], %[f3], %[f1] \n\t" + "lwc1 %[f3], 0(%[yf1]) \n\t" +#endif // #if !defined(MIPS32_R2_LE) + "add.s %[f2], %[f2], %[f8] \n\t" + "add.s %[f3], %[f3], %[f1] \n\t" + "swc1 %[f2], 0(%[yf0]) \n\t" + "swc1 %[f3], 0(%[yf1]) \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), + [f3] "=&f" (f3), [f4] "=&f" (f4), [f5] "=&f" (f5), + [f6] "=&f" (f6), [f7] "=&f" (f7), [f8] "=&f" (f8), + [f9] "=&f" (f9), [f10] "=&f" (f10), [f11] "=&f" (f11), + [f12] "=&f" (f12), [f13] "=&f" (f13), [aRe] "+r" (aRe), + [aIm] "+r" (aIm), [bRe] "+r" (bRe), [bIm] "+r" (bIm), + [yf0] "+r" (yf0), [yf1] "+r" (yf1), [len] "+r" (len) + : + : "memory"); + } +} + +void WebRtcAec_FilterAdaptation_mips( + const OouraFft& ooura_fft, + int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float e_fft[2][PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]) { + float fft[PART_LEN2]; + int i; + for (i = 0; i < num_partitions; i++) { + int xPos = (i + x_fft_buf_block_pos) * (PART_LEN1); + int pos; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * PART_LEN1; + } + + pos = i * PART_LEN1; + float* aRe = x_fft_buf[0] + xPos; + float* aIm = x_fft_buf[1] + xPos; + float* bRe = e_fft[0]; + float* bIm = e_fft[1]; + float* fft_tmp; + + float f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12; + int len = PART_LEN >> 1; + + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[fft_tmp], %[fft], 0 \n\t" + "1: \n\t" + "lwc1 %[f0], 0(%[aRe]) \n\t" + "lwc1 %[f1], 0(%[bRe]) \n\t" + "lwc1 %[f2], 0(%[bIm]) \n\t" + "lwc1 %[f4], 4(%[aRe]) \n\t" + "lwc1 %[f5], 4(%[bRe]) \n\t" + "lwc1 %[f6], 4(%[bIm]) \n\t" + "addiu %[aRe], %[aRe], 8 \n\t" + "addiu %[bRe], %[bRe], 8 \n\t" + "mul.s %[f8], %[f0], %[f1] \n\t" + "mul.s %[f0], %[f0], %[f2] \n\t" + "lwc1 %[f3], 0(%[aIm]) \n\t" + "mul.s %[f9], %[f4], %[f5] \n\t" + "lwc1 %[f7], 4(%[aIm]) \n\t" + "mul.s %[f4], %[f4], %[f6] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f10], %[f3], %[f2] \n\t" + "mul.s %[f1], %[f3], %[f1] \n\t" + "mul.s %[f11], %[f7], %[f6] \n\t" + "mul.s %[f5], %[f7], %[f5] \n\t" + "addiu %[aIm], %[aIm], 8 \n\t" + "addiu %[bIm], %[bIm], 8 \n\t" + "addiu %[len], %[len], -1 \n\t" + "add.s %[f8], %[f8], %[f10] \n\t" + "sub.s %[f1], %[f0], %[f1] \n\t" + "add.s %[f9], %[f9], %[f11] \n\t" + "sub.s %[f5], %[f4], %[f5] \n\t" +#else // #if !defined(MIPS32_R2_LE) + "addiu %[aIm], %[aIm], 8 \n\t" + "addiu %[bIm], %[bIm], 8 \n\t" + "addiu %[len], %[len], -1 \n\t" + "madd.s %[f8], %[f8], %[f3], %[f2] \n\t" + "nmsub.s %[f1], %[f0], %[f3], %[f1] \n\t" + "madd.s %[f9], %[f9], %[f7], %[f6] \n\t" + "nmsub.s %[f5], %[f4], %[f7], %[f5] \n\t" +#endif // #if !defined(MIPS32_R2_LE) + "swc1 %[f8], 0(%[fft_tmp]) \n\t" + "swc1 %[f1], 4(%[fft_tmp]) \n\t" + "swc1 %[f9], 8(%[fft_tmp]) \n\t" + "swc1 %[f5], 12(%[fft_tmp]) \n\t" + "bgtz %[len], 1b \n\t" + " addiu %[fft_tmp], %[fft_tmp], 16 \n\t" + "lwc1 %[f0], 0(%[aRe]) \n\t" + "lwc1 %[f1], 0(%[bRe]) \n\t" + "lwc1 %[f2], 0(%[bIm]) \n\t" + "lwc1 %[f3], 0(%[aIm]) \n\t" + "mul.s %[f8], %[f0], %[f1] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f10], %[f3], %[f2] \n\t" + "add.s %[f8], %[f8], %[f10] \n\t" +#else // #if !defined(MIPS32_R2_LE) + "madd.s %[f8], %[f8], %[f3], %[f2] \n\t" +#endif // #if !defined(MIPS32_R2_LE) + "swc1 %[f8], 4(%[fft]) \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), + [f3] "=&f" (f3), [f4] "=&f" (f4), [f5] "=&f" (f5), + [f6] "=&f" (f6), [f7] "=&f" (f7), [f8] "=&f" (f8), + [f9] "=&f" (f9), [f10] "=&f" (f10), [f11] "=&f" (f11), + [f12] "=&f" (f12), [aRe] "+r" (aRe), [aIm] "+r" (aIm), + [bRe] "+r" (bRe), [bIm] "+r" (bIm), [fft_tmp] "=&r" (fft_tmp), + [len] "+r" (len) + : [fft] "r" (fft) + : "memory"); + + ooura_fft.InverseFft(fft); + memset(fft + PART_LEN, 0, sizeof(float) * PART_LEN); + + // fft scaling + { + float scale = 2.0f / PART_LEN2; + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[fft_tmp], %[fft], 0 \n\t" + "addiu %[len], $zero, 8 \n\t" + "1: \n\t" + "addiu %[len], %[len], -1 \n\t" + "lwc1 %[f0], 0(%[fft_tmp]) \n\t" + "lwc1 %[f1], 4(%[fft_tmp]) \n\t" + "lwc1 %[f2], 8(%[fft_tmp]) \n\t" + "lwc1 %[f3], 12(%[fft_tmp]) \n\t" + "mul.s %[f0], %[f0], %[scale] \n\t" + "mul.s %[f1], %[f1], %[scale] \n\t" + "mul.s %[f2], %[f2], %[scale] \n\t" + "mul.s %[f3], %[f3], %[scale] \n\t" + "lwc1 %[f4], 16(%[fft_tmp]) \n\t" + "lwc1 %[f5], 20(%[fft_tmp]) \n\t" + "lwc1 %[f6], 24(%[fft_tmp]) \n\t" + "lwc1 %[f7], 28(%[fft_tmp]) \n\t" + "mul.s %[f4], %[f4], %[scale] \n\t" + "mul.s %[f5], %[f5], %[scale] \n\t" + "mul.s %[f6], %[f6], %[scale] \n\t" + "mul.s %[f7], %[f7], %[scale] \n\t" + "swc1 %[f0], 0(%[fft_tmp]) \n\t" + "swc1 %[f1], 4(%[fft_tmp]) \n\t" + "swc1 %[f2], 8(%[fft_tmp]) \n\t" + "swc1 %[f3], 12(%[fft_tmp]) \n\t" + "swc1 %[f4], 16(%[fft_tmp]) \n\t" + "swc1 %[f5], 20(%[fft_tmp]) \n\t" + "swc1 %[f6], 24(%[fft_tmp]) \n\t" + "swc1 %[f7], 28(%[fft_tmp]) \n\t" + "bgtz %[len], 1b \n\t" + " addiu %[fft_tmp], %[fft_tmp], 32 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), + [f3] "=&f" (f3), [f4] "=&f" (f4), [f5] "=&f" (f5), + [f6] "=&f" (f6), [f7] "=&f" (f7), [len] "=&r" (len), + [fft_tmp] "=&r" (fft_tmp) + : [scale] "f" (scale), [fft] "r" (fft) + : "memory"); + } + ooura_fft.Fft(fft); + aRe = h_fft_buf[0] + pos; + aIm = h_fft_buf[1] + pos; + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[fft_tmp], %[fft], 0 \n\t" + "addiu %[len], $zero, 31 \n\t" + "lwc1 %[f0], 0(%[aRe]) \n\t" + "lwc1 %[f1], 0(%[fft_tmp]) \n\t" + "lwc1 %[f2], 256(%[aRe]) \n\t" + "lwc1 %[f3], 4(%[fft_tmp]) \n\t" + "lwc1 %[f4], 4(%[aRe]) \n\t" + "lwc1 %[f5], 8(%[fft_tmp]) \n\t" + "lwc1 %[f6], 4(%[aIm]) \n\t" + "lwc1 %[f7], 12(%[fft_tmp]) \n\t" + "add.s %[f0], %[f0], %[f1] \n\t" + "add.s %[f2], %[f2], %[f3] \n\t" + "add.s %[f4], %[f4], %[f5] \n\t" + "add.s %[f6], %[f6], %[f7] \n\t" + "addiu %[fft_tmp], %[fft_tmp], 16 \n\t" + "swc1 %[f0], 0(%[aRe]) \n\t" + "swc1 %[f2], 256(%[aRe]) \n\t" + "swc1 %[f4], 4(%[aRe]) \n\t" + "addiu %[aRe], %[aRe], 8 \n\t" + "swc1 %[f6], 4(%[aIm]) \n\t" + "addiu %[aIm], %[aIm], 8 \n\t" + "1: \n\t" + "lwc1 %[f0], 0(%[aRe]) \n\t" + "lwc1 %[f1], 0(%[fft_tmp]) \n\t" + "lwc1 %[f2], 0(%[aIm]) \n\t" + "lwc1 %[f3], 4(%[fft_tmp]) \n\t" + "lwc1 %[f4], 4(%[aRe]) \n\t" + "lwc1 %[f5], 8(%[fft_tmp]) \n\t" + "lwc1 %[f6], 4(%[aIm]) \n\t" + "lwc1 %[f7], 12(%[fft_tmp]) \n\t" + "add.s %[f0], %[f0], %[f1] \n\t" + "add.s %[f2], %[f2], %[f3] \n\t" + "add.s %[f4], %[f4], %[f5] \n\t" + "add.s %[f6], %[f6], %[f7] \n\t" + "addiu %[len], %[len], -1 \n\t" + "addiu %[fft_tmp], %[fft_tmp], 16 \n\t" + "swc1 %[f0], 0(%[aRe]) \n\t" + "swc1 %[f2], 0(%[aIm]) \n\t" + "swc1 %[f4], 4(%[aRe]) \n\t" + "addiu %[aRe], %[aRe], 8 \n\t" + "swc1 %[f6], 4(%[aIm]) \n\t" + "bgtz %[len], 1b \n\t" + " addiu %[aIm], %[aIm], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), + [f3] "=&f" (f3), [f4] "=&f" (f4), [f5] "=&f" (f5), + [f6] "=&f" (f6), [f7] "=&f" (f7), [len] "=&r" (len), + [fft_tmp] "=&r" (fft_tmp), [aRe] "+r" (aRe), [aIm] "+r" (aIm) + : [fft] "r" (fft) + : "memory"); + } +} + +void WebRtcAec_Overdrive_mips(float overdrive_scaling, + float hNlFb, + float hNl[PART_LEN1]) { + const float one = 1.0; + float* p_hNl; + const float* p_WebRtcAec_wC; + float temp1, temp2, temp3, temp4; + + p_hNl = &hNl[0]; + p_WebRtcAec_wC = &WebRtcAec_weightCurve[0]; + + for (int i = 0; i < PART_LEN1; ++i) { + // Weight subbands + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "lwc1 %[temp1], 0(%[p_hNl]) \n\t" + "lwc1 %[temp2], 0(%[p_wC]) \n\t" + "c.lt.s %[hNlFb], %[temp1] \n\t" + "bc1f 1f \n\t" + " mul.s %[temp3], %[temp2], %[hNlFb] \n\t" + "sub.s %[temp4], %[one], %[temp2] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[temp1], %[temp1], %[temp4] \n\t" + "add.s %[temp1], %[temp3], %[temp1] \n\t" +#else // #if !defined(MIPS32_R2_LE) + "madd.s %[temp1], %[temp3], %[temp1], %[temp4] \n\t" +#endif // #if !defined(MIPS32_R2_LE) + "swc1 %[temp1], 0(%[p_hNl]) \n\t" + "1: \n\t" + "addiu %[p_wC], %[p_wC], 4 \n\t" + ".set pop \n\t" + : [temp1] "=&f" (temp1), [temp2] "=&f" (temp2), [temp3] "=&f" (temp3), + [temp4] "=&f" (temp4), [p_wC] "+r" (p_WebRtcAec_wC) + : [hNlFb] "f" (hNlFb), [one] "f" (one), [p_hNl] "r" (p_hNl) + : "memory"); + + hNl[i] = powf(hNl[i], overdrive_scaling * WebRtcAec_overDriveCurve[i]); + } +} + +void WebRtcAec_Suppress_mips(const float hNl[PART_LEN1], + float efw[2][PART_LEN1]) { + const float* p_hNl; + float* p_efw0; + float* p_efw1; + float temp1, temp2, temp3, temp4; + + p_hNl = &hNl[0]; + p_efw0 = &efw[0][0]; + p_efw1 = &efw[1][0]; + + for (int i = 0; i < PART_LEN1; ++i) { + __asm __volatile( + "lwc1 %[temp1], 0(%[p_hNl]) \n\t" + "lwc1 %[temp3], 0(%[p_efw1]) \n\t" + "lwc1 %[temp2], 0(%[p_efw0]) \n\t" + "addiu %[p_hNl], %[p_hNl], 4 \n\t" + "mul.s %[temp3], %[temp3], %[temp1] \n\t" + "mul.s %[temp2], %[temp2], %[temp1] \n\t" + "addiu %[p_efw0], %[p_efw0], 4 \n\t" + "addiu %[p_efw1], %[p_efw1], 4 \n\t" + "neg.s %[temp4], %[temp3] \n\t" + "swc1 %[temp2], -4(%[p_efw0]) \n\t" + "swc1 %[temp4], -4(%[p_efw1]) \n\t" + : [temp1] "=&f" (temp1), [temp2] "=&f" (temp2), [temp3] "=&f" (temp3), + [temp4] "=&f" (temp4), [p_efw0] "+r" (p_efw0), [p_efw1] "+r" (p_efw1), + [p_hNl] "+r" (p_hNl) + : + : "memory"); + } +} + +void WebRtcAec_ScaleErrorSignal_mips(float mu, + float error_threshold, + float x_pow[PART_LEN1], + float ef[2][PART_LEN1]) { + int len = (PART_LEN1); + float* ef0 = ef[0]; + float* ef1 = ef[1]; + float fac1 = 1e-10f; + float err_th2 = error_threshold * error_threshold; + float f0, f1, f2; +#if !defined(MIPS32_R2_LE) + float f3; +#endif + + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "lwc1 %[f0], 0(%[x_pow]) \n\t" + "lwc1 %[f1], 0(%[ef0]) \n\t" + "lwc1 %[f2], 0(%[ef1]) \n\t" + "add.s %[f0], %[f0], %[fac1] \n\t" + "div.s %[f1], %[f1], %[f0] \n\t" + "div.s %[f2], %[f2], %[f0] \n\t" + "mul.s %[f0], %[f1], %[f1] \n\t" +#if defined(MIPS32_R2_LE) + "madd.s %[f0], %[f0], %[f2], %[f2] \n\t" +#else + "mul.s %[f3], %[f2], %[f2] \n\t" + "add.s %[f0], %[f0], %[f3] \n\t" +#endif + "c.le.s %[f0], %[err_th2] \n\t" + "nop \n\t" + "bc1t 2f \n\t" + " nop \n\t" + "sqrt.s %[f0], %[f0] \n\t" + "add.s %[f0], %[f0], %[fac1] \n\t" + "div.s %[f0], %[err_th], %[f0] \n\t" + "mul.s %[f1], %[f1], %[f0] \n\t" + "mul.s %[f2], %[f2], %[f0] \n\t" + "2: \n\t" + "mul.s %[f1], %[f1], %[mu] \n\t" + "mul.s %[f2], %[f2], %[mu] \n\t" + "swc1 %[f1], 0(%[ef0]) \n\t" + "swc1 %[f2], 0(%[ef1]) \n\t" + "addiu %[len], %[len], -1 \n\t" + "addiu %[x_pow], %[x_pow], 4 \n\t" + "addiu %[ef0], %[ef0], 4 \n\t" + "bgtz %[len], 1b \n\t" + " addiu %[ef1], %[ef1], 4 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), +#if !defined(MIPS32_R2_LE) + [f3] "=&f" (f3), +#endif + [x_pow] "+r" (x_pow), [ef0] "+r" (ef0), [ef1] "+r" (ef1), + [len] "+r" (len) + : [fac1] "f" (fac1), [err_th2] "f" (err_th2), [mu] "f" (mu), + [err_th] "f" (error_threshold) + : "memory"); +} + +void WebRtcAec_InitAec_mips(void) { + WebRtcAec_FilterFar = WebRtcAec_FilterFar_mips; + WebRtcAec_FilterAdaptation = WebRtcAec_FilterAdaptation_mips; + WebRtcAec_ScaleErrorSignal = WebRtcAec_ScaleErrorSignal_mips; + WebRtcAec_Overdrive = WebRtcAec_Overdrive_mips; + WebRtcAec_Suppress = WebRtcAec_Suppress_mips; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_neon.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_neon.cc new file mode 100644 index 0000000000..1fbf56b8b6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_neon.cc @@ -0,0 +1,737 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * The core AEC algorithm, neon version of speed-critical functions. + * + * Based on aec_core_sse2.c. + */ + +#include +#include +#include // memset + +extern "C" { +#include "common_audio/signal_processing/include/signal_processing_library.h" +} +#include "modules/audio_processing/aec/aec_common.h" +#include "modules/audio_processing/aec/aec_core_optimized_methods.h" +#include "modules/audio_processing/utility/ooura_fft.h" + +namespace webrtc { + +enum { kShiftExponentIntoTopMantissa = 8 }; +enum { kFloatExponentShift = 23 }; + +__inline static float MulRe(float aRe, float aIm, float bRe, float bIm) { + return aRe * bRe - aIm * bIm; +} + +__inline static float MulIm(float aRe, float aIm, float bRe, float bIm) { + return aRe * bIm + aIm * bRe; +} + +static void FilterFarNEON(int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1], + float h_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1], + float y_fft[2][PART_LEN1]) { + int i; + for (i = 0; i < num_partitions; i++) { + int j; + int xPos = (i + x_fft_buf_block_pos) * PART_LEN1; + int pos = i * PART_LEN1; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * PART_LEN1; + } + + // vectorized code (four at once) + for (j = 0; j + 3 < PART_LEN1; j += 4) { + const float32x4_t x_fft_buf_re = vld1q_f32(&x_fft_buf[0][xPos + j]); + const float32x4_t x_fft_buf_im = vld1q_f32(&x_fft_buf[1][xPos + j]); + const float32x4_t h_fft_buf_re = vld1q_f32(&h_fft_buf[0][pos + j]); + const float32x4_t h_fft_buf_im = vld1q_f32(&h_fft_buf[1][pos + j]); + const float32x4_t y_fft_re = vld1q_f32(&y_fft[0][j]); + const float32x4_t y_fft_im = vld1q_f32(&y_fft[1][j]); + const float32x4_t a = vmulq_f32(x_fft_buf_re, h_fft_buf_re); + const float32x4_t e = vmlsq_f32(a, x_fft_buf_im, h_fft_buf_im); + const float32x4_t c = vmulq_f32(x_fft_buf_re, h_fft_buf_im); + const float32x4_t f = vmlaq_f32(c, x_fft_buf_im, h_fft_buf_re); + const float32x4_t g = vaddq_f32(y_fft_re, e); + const float32x4_t h = vaddq_f32(y_fft_im, f); + vst1q_f32(&y_fft[0][j], g); + vst1q_f32(&y_fft[1][j], h); + } + // scalar code for the remaining items. + for (; j < PART_LEN1; j++) { + y_fft[0][j] += MulRe(x_fft_buf[0][xPos + j], x_fft_buf[1][xPos + j], + h_fft_buf[0][pos + j], h_fft_buf[1][pos + j]); + y_fft[1][j] += MulIm(x_fft_buf[0][xPos + j], x_fft_buf[1][xPos + j], + h_fft_buf[0][pos + j], h_fft_buf[1][pos + j]); + } + } +} + +// ARM64's arm_neon.h has already defined vdivq_f32 vsqrtq_f32. +#if !defined(WEBRTC_ARCH_ARM64) +static float32x4_t vdivq_f32(float32x4_t a, float32x4_t b) { + int i; + float32x4_t x = vrecpeq_f32(b); + // from arm documentation + // The Newton-Raphson iteration: + // x[n+1] = x[n] * (2 - d * x[n]) + // converges to (1/d) if x0 is the result of VRECPE applied to d. + // + // Note: The precision did not improve after 2 iterations. + for (i = 0; i < 2; i++) { + x = vmulq_f32(vrecpsq_f32(b, x), x); + } + // a/b = a*(1/b) + return vmulq_f32(a, x); +} + +static float32x4_t vsqrtq_f32(float32x4_t s) { + int i; + float32x4_t x = vrsqrteq_f32(s); + + // Code to handle sqrt(0). + // If the input to sqrtf() is zero, a zero will be returned. + // If the input to vrsqrteq_f32() is zero, positive infinity is returned. + const uint32x4_t vec_p_inf = vdupq_n_u32(0x7F800000); + // check for divide by zero + const uint32x4_t div_by_zero = vceqq_u32(vec_p_inf, vreinterpretq_u32_f32(x)); + // zero out the positive infinity results + x = vreinterpretq_f32_u32( + vandq_u32(vmvnq_u32(div_by_zero), vreinterpretq_u32_f32(x))); + // from arm documentation + // The Newton-Raphson iteration: + // x[n+1] = x[n] * (3 - d * (x[n] * x[n])) / 2) + // converges to (1/√d) if x0 is the result of VRSQRTE applied to d. + // + // Note: The precision did not improve after 2 iterations. + for (i = 0; i < 2; i++) { + x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x, x), s), x); + } + // sqrt(s) = s * 1/sqrt(s) + return vmulq_f32(s, x); +} +#endif // WEBRTC_ARCH_ARM64 + +static void ScaleErrorSignalNEON(float mu, + float error_threshold, + float x_pow[PART_LEN1], + float ef[2][PART_LEN1]) { + const float32x4_t k1e_10f = vdupq_n_f32(1e-10f); + const float32x4_t kMu = vmovq_n_f32(mu); + const float32x4_t kThresh = vmovq_n_f32(error_threshold); + int i; + // vectorized code (four at once) + for (i = 0; i + 3 < PART_LEN1; i += 4) { + const float32x4_t x_pow_local = vld1q_f32(&x_pow[i]); + const float32x4_t ef_re_base = vld1q_f32(&ef[0][i]); + const float32x4_t ef_im_base = vld1q_f32(&ef[1][i]); + const float32x4_t xPowPlus = vaddq_f32(x_pow_local, k1e_10f); + float32x4_t ef_re = vdivq_f32(ef_re_base, xPowPlus); + float32x4_t ef_im = vdivq_f32(ef_im_base, xPowPlus); + const float32x4_t ef_re2 = vmulq_f32(ef_re, ef_re); + const float32x4_t ef_sum2 = vmlaq_f32(ef_re2, ef_im, ef_im); + const float32x4_t absEf = vsqrtq_f32(ef_sum2); + const uint32x4_t bigger = vcgtq_f32(absEf, kThresh); + const float32x4_t absEfPlus = vaddq_f32(absEf, k1e_10f); + const float32x4_t absEfInv = vdivq_f32(kThresh, absEfPlus); + uint32x4_t ef_re_if = vreinterpretq_u32_f32(vmulq_f32(ef_re, absEfInv)); + uint32x4_t ef_im_if = vreinterpretq_u32_f32(vmulq_f32(ef_im, absEfInv)); + uint32x4_t ef_re_u32 = + vandq_u32(vmvnq_u32(bigger), vreinterpretq_u32_f32(ef_re)); + uint32x4_t ef_im_u32 = + vandq_u32(vmvnq_u32(bigger), vreinterpretq_u32_f32(ef_im)); + ef_re_if = vandq_u32(bigger, ef_re_if); + ef_im_if = vandq_u32(bigger, ef_im_if); + ef_re_u32 = vorrq_u32(ef_re_u32, ef_re_if); + ef_im_u32 = vorrq_u32(ef_im_u32, ef_im_if); + ef_re = vmulq_f32(vreinterpretq_f32_u32(ef_re_u32), kMu); + ef_im = vmulq_f32(vreinterpretq_f32_u32(ef_im_u32), kMu); + vst1q_f32(&ef[0][i], ef_re); + vst1q_f32(&ef[1][i], ef_im); + } + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + float abs_ef; + ef[0][i] /= (x_pow[i] + 1e-10f); + ef[1][i] /= (x_pow[i] + 1e-10f); + abs_ef = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]); + + if (abs_ef > error_threshold) { + abs_ef = error_threshold / (abs_ef + 1e-10f); + ef[0][i] *= abs_ef; + ef[1][i] *= abs_ef; + } + + // Stepsize factor + ef[0][i] *= mu; + ef[1][i] *= mu; + } +} + +static void FilterAdaptationNEON( + const OouraFft& ooura_fft, + int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float e_fft[2][PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]) { + float fft[PART_LEN2]; + int i; + for (i = 0; i < num_partitions; i++) { + int xPos = (i + x_fft_buf_block_pos) * PART_LEN1; + int pos = i * PART_LEN1; + int j; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * PART_LEN1; + } + + // Process the whole array... + for (j = 0; j < PART_LEN; j += 4) { + // Load x_fft_buf and e_fft. + const float32x4_t x_fft_buf_re = vld1q_f32(&x_fft_buf[0][xPos + j]); + const float32x4_t x_fft_buf_im = vld1q_f32(&x_fft_buf[1][xPos + j]); + const float32x4_t e_fft_re = vld1q_f32(&e_fft[0][j]); + const float32x4_t e_fft_im = vld1q_f32(&e_fft[1][j]); + // Calculate the product of conjugate(x_fft_buf) by e_fft. + // re(conjugate(a) * b) = aRe * bRe + aIm * bIm + // im(conjugate(a) * b)= aRe * bIm - aIm * bRe + const float32x4_t a = vmulq_f32(x_fft_buf_re, e_fft_re); + const float32x4_t e = vmlaq_f32(a, x_fft_buf_im, e_fft_im); + const float32x4_t c = vmulq_f32(x_fft_buf_re, e_fft_im); + const float32x4_t f = vmlsq_f32(c, x_fft_buf_im, e_fft_re); + // Interleave real and imaginary parts. + const float32x4x2_t g_n_h = vzipq_f32(e, f); + // Store + vst1q_f32(&fft[2 * j + 0], g_n_h.val[0]); + vst1q_f32(&fft[2 * j + 4], g_n_h.val[1]); + } + // ... and fixup the first imaginary entry. + fft[1] = + MulRe(x_fft_buf[0][xPos + PART_LEN], -x_fft_buf[1][xPos + PART_LEN], + e_fft[0][PART_LEN], e_fft[1][PART_LEN]); + + ooura_fft.InverseFft(fft); + memset(fft + PART_LEN, 0, sizeof(float) * PART_LEN); + + // fft scaling + { + const float scale = 2.0f / PART_LEN2; + const float32x4_t scale_ps = vmovq_n_f32(scale); + for (j = 0; j < PART_LEN; j += 4) { + const float32x4_t fft_ps = vld1q_f32(&fft[j]); + const float32x4_t fft_scale = vmulq_f32(fft_ps, scale_ps); + vst1q_f32(&fft[j], fft_scale); + } + } + ooura_fft.Fft(fft); + + { + const float wt1 = h_fft_buf[1][pos]; + h_fft_buf[0][pos + PART_LEN] += fft[1]; + for (j = 0; j < PART_LEN; j += 4) { + float32x4_t wtBuf_re = vld1q_f32(&h_fft_buf[0][pos + j]); + float32x4_t wtBuf_im = vld1q_f32(&h_fft_buf[1][pos + j]); + const float32x4_t fft0 = vld1q_f32(&fft[2 * j + 0]); + const float32x4_t fft4 = vld1q_f32(&fft[2 * j + 4]); + const float32x4x2_t fft_re_im = vuzpq_f32(fft0, fft4); + wtBuf_re = vaddq_f32(wtBuf_re, fft_re_im.val[0]); + wtBuf_im = vaddq_f32(wtBuf_im, fft_re_im.val[1]); + + vst1q_f32(&h_fft_buf[0][pos + j], wtBuf_re); + vst1q_f32(&h_fft_buf[1][pos + j], wtBuf_im); + } + h_fft_buf[1][pos] = wt1; + } + } +} + +static float32x4_t vpowq_f32(float32x4_t a, float32x4_t b) { + // a^b = exp2(b * log2(a)) + // exp2(x) and log2(x) are calculated using polynomial approximations. + float32x4_t log2_a, b_log2_a, a_exp_b; + + // Calculate log2(x), x = a. + { + // To calculate log2(x), we decompose x like this: + // x = y * 2^n + // n is an integer + // y is in the [1.0, 2.0) range + // + // log2(x) = log2(y) + n + // n can be evaluated by playing with float representation. + // log2(y) in a small range can be approximated, this code uses an order + // five polynomial approximation. The coefficients have been + // estimated with the Remez algorithm and the resulting + // polynomial has a maximum relative error of 0.00086%. + + // Compute n. + // This is done by masking the exponent, shifting it into the top bit of + // the mantissa, putting eight into the biased exponent (to shift/ + // compensate the fact that the exponent has been shifted in the top/ + // fractional part and finally getting rid of the implicit leading one + // from the mantissa by substracting it out. + const uint32x4_t vec_float_exponent_mask = vdupq_n_u32(0x7F800000); + const uint32x4_t vec_eight_biased_exponent = vdupq_n_u32(0x43800000); + const uint32x4_t vec_implicit_leading_one = vdupq_n_u32(0x43BF8000); + const uint32x4_t two_n = + vandq_u32(vreinterpretq_u32_f32(a), vec_float_exponent_mask); + const uint32x4_t n_1 = vshrq_n_u32(two_n, kShiftExponentIntoTopMantissa); + const uint32x4_t n_0 = vorrq_u32(n_1, vec_eight_biased_exponent); + const float32x4_t n = + vsubq_f32(vreinterpretq_f32_u32(n_0), + vreinterpretq_f32_u32(vec_implicit_leading_one)); + // Compute y. + const uint32x4_t vec_mantissa_mask = vdupq_n_u32(0x007FFFFF); + const uint32x4_t vec_zero_biased_exponent_is_one = vdupq_n_u32(0x3F800000); + const uint32x4_t mantissa = + vandq_u32(vreinterpretq_u32_f32(a), vec_mantissa_mask); + const float32x4_t y = vreinterpretq_f32_u32( + vorrq_u32(mantissa, vec_zero_biased_exponent_is_one)); + // Approximate log2(y) ~= (y - 1) * pol5(y). + // pol5(y) = C5 * y^5 + C4 * y^4 + C3 * y^3 + C2 * y^2 + C1 * y + C0 + const float32x4_t C5 = vdupq_n_f32(-3.4436006e-2f); + const float32x4_t C4 = vdupq_n_f32(3.1821337e-1f); + const float32x4_t C3 = vdupq_n_f32(-1.2315303f); + const float32x4_t C2 = vdupq_n_f32(2.5988452f); + const float32x4_t C1 = vdupq_n_f32(-3.3241990f); + const float32x4_t C0 = vdupq_n_f32(3.1157899f); + float32x4_t pol5_y = C5; + pol5_y = vmlaq_f32(C4, y, pol5_y); + pol5_y = vmlaq_f32(C3, y, pol5_y); + pol5_y = vmlaq_f32(C2, y, pol5_y); + pol5_y = vmlaq_f32(C1, y, pol5_y); + pol5_y = vmlaq_f32(C0, y, pol5_y); + const float32x4_t y_minus_one = + vsubq_f32(y, vreinterpretq_f32_u32(vec_zero_biased_exponent_is_one)); + const float32x4_t log2_y = vmulq_f32(y_minus_one, pol5_y); + + // Combine parts. + log2_a = vaddq_f32(n, log2_y); + } + + // b * log2(a) + b_log2_a = vmulq_f32(b, log2_a); + + // Calculate exp2(x), x = b * log2(a). + { + // To calculate 2^x, we decompose x like this: + // x = n + y + // n is an integer, the value of x - 0.5 rounded down, therefore + // y is in the [0.5, 1.5) range + // + // 2^x = 2^n * 2^y + // 2^n can be evaluated by playing with float representation. + // 2^y in a small range can be approximated, this code uses an order two + // polynomial approximation. The coefficients have been estimated + // with the Remez algorithm and the resulting polynomial has a + // maximum relative error of 0.17%. + // To avoid over/underflow, we reduce the range of input to ]-127, 129]. + const float32x4_t max_input = vdupq_n_f32(129.f); + const float32x4_t min_input = vdupq_n_f32(-126.99999f); + const float32x4_t x_min = vminq_f32(b_log2_a, max_input); + const float32x4_t x_max = vmaxq_f32(x_min, min_input); + // Compute n. + const float32x4_t half = vdupq_n_f32(0.5f); + const float32x4_t x_minus_half = vsubq_f32(x_max, half); + const int32x4_t x_minus_half_floor = vcvtq_s32_f32(x_minus_half); + + // Compute 2^n. + const int32x4_t float_exponent_bias = vdupq_n_s32(127); + const int32x4_t two_n_exponent = + vaddq_s32(x_minus_half_floor, float_exponent_bias); + const float32x4_t two_n = + vreinterpretq_f32_s32(vshlq_n_s32(two_n_exponent, kFloatExponentShift)); + // Compute y. + const float32x4_t y = vsubq_f32(x_max, vcvtq_f32_s32(x_minus_half_floor)); + + // Approximate 2^y ~= C2 * y^2 + C1 * y + C0. + const float32x4_t C2 = vdupq_n_f32(3.3718944e-1f); + const float32x4_t C1 = vdupq_n_f32(6.5763628e-1f); + const float32x4_t C0 = vdupq_n_f32(1.0017247f); + float32x4_t exp2_y = C2; + exp2_y = vmlaq_f32(C1, y, exp2_y); + exp2_y = vmlaq_f32(C0, y, exp2_y); + + // Combine parts. + a_exp_b = vmulq_f32(exp2_y, two_n); + } + + return a_exp_b; +} + +static void OverdriveNEON(float overdrive_scaling, + float hNlFb, + float hNl[PART_LEN1]) { + int i; + const float32x4_t vec_hNlFb = vmovq_n_f32(hNlFb); + const float32x4_t vec_one = vdupq_n_f32(1.0f); + const float32x4_t vec_overdrive_scaling = vmovq_n_f32(overdrive_scaling); + + // vectorized code (four at once) + for (i = 0; i + 3 < PART_LEN1; i += 4) { + // Weight subbands + float32x4_t vec_hNl = vld1q_f32(&hNl[i]); + const float32x4_t vec_weightCurve = vld1q_f32(&WebRtcAec_weightCurve[i]); + const uint32x4_t bigger = vcgtq_f32(vec_hNl, vec_hNlFb); + const float32x4_t vec_weightCurve_hNlFb = + vmulq_f32(vec_weightCurve, vec_hNlFb); + const float32x4_t vec_one_weightCurve = vsubq_f32(vec_one, vec_weightCurve); + const float32x4_t vec_one_weightCurve_hNl = + vmulq_f32(vec_one_weightCurve, vec_hNl); + const uint32x4_t vec_if0 = + vandq_u32(vmvnq_u32(bigger), vreinterpretq_u32_f32(vec_hNl)); + const float32x4_t vec_one_weightCurve_add = + vaddq_f32(vec_weightCurve_hNlFb, vec_one_weightCurve_hNl); + const uint32x4_t vec_if1 = + vandq_u32(bigger, vreinterpretq_u32_f32(vec_one_weightCurve_add)); + + vec_hNl = vreinterpretq_f32_u32(vorrq_u32(vec_if0, vec_if1)); + + const float32x4_t vec_overDriveCurve = + vld1q_f32(&WebRtcAec_overDriveCurve[i]); + const float32x4_t vec_overDriveSm_overDriveCurve = + vmulq_f32(vec_overdrive_scaling, vec_overDriveCurve); + vec_hNl = vpowq_f32(vec_hNl, vec_overDriveSm_overDriveCurve); + vst1q_f32(&hNl[i], vec_hNl); + } + + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + // Weight subbands + if (hNl[i] > hNlFb) { + hNl[i] = WebRtcAec_weightCurve[i] * hNlFb + + (1 - WebRtcAec_weightCurve[i]) * hNl[i]; + } + + hNl[i] = powf(hNl[i], overdrive_scaling * WebRtcAec_overDriveCurve[i]); + } +} + +static void SuppressNEON(const float hNl[PART_LEN1], float efw[2][PART_LEN1]) { + int i; + const float32x4_t vec_minus_one = vdupq_n_f32(-1.0f); + // vectorized code (four at once) + for (i = 0; i + 3 < PART_LEN1; i += 4) { + float32x4_t vec_hNl = vld1q_f32(&hNl[i]); + float32x4_t vec_efw_re = vld1q_f32(&efw[0][i]); + float32x4_t vec_efw_im = vld1q_f32(&efw[1][i]); + vec_efw_re = vmulq_f32(vec_efw_re, vec_hNl); + vec_efw_im = vmulq_f32(vec_efw_im, vec_hNl); + + // Ooura fft returns incorrect sign on imaginary component. It matters + // here because we are making an additive change with comfort noise. + vec_efw_im = vmulq_f32(vec_efw_im, vec_minus_one); + vst1q_f32(&efw[0][i], vec_efw_re); + vst1q_f32(&efw[1][i], vec_efw_im); + } + + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + efw[0][i] *= hNl[i]; + efw[1][i] *= hNl[i]; + + // Ooura fft returns incorrect sign on imaginary component. It matters + // here because we are making an additive change with comfort noise. + efw[1][i] *= -1; + } +} + +static int PartitionDelayNEON( + int num_partitions, + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]) { + // Measures the energy in each filter partition and returns the partition with + // highest energy. + // TODO(bjornv): Spread computational cost by computing one partition per + // block? + float wfEnMax = 0; + int i; + int delay = 0; + + for (i = 0; i < num_partitions; i++) { + int j; + int pos = i * PART_LEN1; + float wfEn = 0; + float32x4_t vec_wfEn = vdupq_n_f32(0.0f); + // vectorized code (four at once) + for (j = 0; j + 3 < PART_LEN1; j += 4) { + const float32x4_t vec_wfBuf0 = vld1q_f32(&h_fft_buf[0][pos + j]); + const float32x4_t vec_wfBuf1 = vld1q_f32(&h_fft_buf[1][pos + j]); + vec_wfEn = vmlaq_f32(vec_wfEn, vec_wfBuf0, vec_wfBuf0); + vec_wfEn = vmlaq_f32(vec_wfEn, vec_wfBuf1, vec_wfBuf1); + } + { + float32x2_t vec_total; + // A B C D + vec_total = vpadd_f32(vget_low_f32(vec_wfEn), vget_high_f32(vec_wfEn)); + // A+B C+D + vec_total = vpadd_f32(vec_total, vec_total); + // A+B+C+D A+B+C+D + wfEn = vget_lane_f32(vec_total, 0); + } + + // scalar code for the remaining items. + for (; j < PART_LEN1; j++) { + wfEn += h_fft_buf[0][pos + j] * h_fft_buf[0][pos + j] + + h_fft_buf[1][pos + j] * h_fft_buf[1][pos + j]; + } + + if (wfEn > wfEnMax) { + wfEnMax = wfEn; + delay = i; + } + } + return delay; +} + +// Updates the following smoothed Power Spectral Densities (PSD): +// - sd : near-end +// - se : residual echo +// - sx : far-end +// - sde : cross-PSD of near-end and residual echo +// - sxd : cross-PSD of near-end and far-end +// +// In addition to updating the PSDs, also the filter diverge state is determined +// upon actions are taken. +static void UpdateCoherenceSpectraNEON(int mult, + bool extended_filter_enabled, + float efw[2][PART_LEN1], + float dfw[2][PART_LEN1], + float xfw[2][PART_LEN1], + CoherenceState* coherence_state, + short* filter_divergence_state, + int* extreme_filter_divergence) { + // Power estimate smoothing coefficients. + const float* ptrGCoh = + extended_filter_enabled + ? WebRtcAec_kExtendedSmoothingCoefficients[mult - 1] + : WebRtcAec_kNormalSmoothingCoefficients[mult - 1]; + int i; + float sdSum = 0, seSum = 0; + const float32x4_t vec_15 = vdupq_n_f32(WebRtcAec_kMinFarendPSD); + float32x4_t vec_sdSum = vdupq_n_f32(0.0f); + float32x4_t vec_seSum = vdupq_n_f32(0.0f); + + for (i = 0; i + 3 < PART_LEN1; i += 4) { + const float32x4_t vec_dfw0 = vld1q_f32(&dfw[0][i]); + const float32x4_t vec_dfw1 = vld1q_f32(&dfw[1][i]); + const float32x4_t vec_efw0 = vld1q_f32(&efw[0][i]); + const float32x4_t vec_efw1 = vld1q_f32(&efw[1][i]); + const float32x4_t vec_xfw0 = vld1q_f32(&xfw[0][i]); + const float32x4_t vec_xfw1 = vld1q_f32(&xfw[1][i]); + float32x4_t vec_sd = + vmulq_n_f32(vld1q_f32(&coherence_state->sd[i]), ptrGCoh[0]); + float32x4_t vec_se = + vmulq_n_f32(vld1q_f32(&coherence_state->se[i]), ptrGCoh[0]); + float32x4_t vec_sx = + vmulq_n_f32(vld1q_f32(&coherence_state->sx[i]), ptrGCoh[0]); + float32x4_t vec_dfw_sumsq = vmulq_f32(vec_dfw0, vec_dfw0); + float32x4_t vec_efw_sumsq = vmulq_f32(vec_efw0, vec_efw0); + float32x4_t vec_xfw_sumsq = vmulq_f32(vec_xfw0, vec_xfw0); + + vec_dfw_sumsq = vmlaq_f32(vec_dfw_sumsq, vec_dfw1, vec_dfw1); + vec_efw_sumsq = vmlaq_f32(vec_efw_sumsq, vec_efw1, vec_efw1); + vec_xfw_sumsq = vmlaq_f32(vec_xfw_sumsq, vec_xfw1, vec_xfw1); + vec_xfw_sumsq = vmaxq_f32(vec_xfw_sumsq, vec_15); + vec_sd = vmlaq_n_f32(vec_sd, vec_dfw_sumsq, ptrGCoh[1]); + vec_se = vmlaq_n_f32(vec_se, vec_efw_sumsq, ptrGCoh[1]); + vec_sx = vmlaq_n_f32(vec_sx, vec_xfw_sumsq, ptrGCoh[1]); + + vst1q_f32(&coherence_state->sd[i], vec_sd); + vst1q_f32(&coherence_state->se[i], vec_se); + vst1q_f32(&coherence_state->sx[i], vec_sx); + + { + float32x4x2_t vec_sde = vld2q_f32(&coherence_state->sde[i][0]); + float32x4_t vec_dfwefw0011 = vmulq_f32(vec_dfw0, vec_efw0); + float32x4_t vec_dfwefw0110 = vmulq_f32(vec_dfw0, vec_efw1); + vec_sde.val[0] = vmulq_n_f32(vec_sde.val[0], ptrGCoh[0]); + vec_sde.val[1] = vmulq_n_f32(vec_sde.val[1], ptrGCoh[0]); + vec_dfwefw0011 = vmlaq_f32(vec_dfwefw0011, vec_dfw1, vec_efw1); + vec_dfwefw0110 = vmlsq_f32(vec_dfwefw0110, vec_dfw1, vec_efw0); + vec_sde.val[0] = vmlaq_n_f32(vec_sde.val[0], vec_dfwefw0011, ptrGCoh[1]); + vec_sde.val[1] = vmlaq_n_f32(vec_sde.val[1], vec_dfwefw0110, ptrGCoh[1]); + vst2q_f32(&coherence_state->sde[i][0], vec_sde); + } + + { + float32x4x2_t vec_sxd = vld2q_f32(&coherence_state->sxd[i][0]); + float32x4_t vec_dfwxfw0011 = vmulq_f32(vec_dfw0, vec_xfw0); + float32x4_t vec_dfwxfw0110 = vmulq_f32(vec_dfw0, vec_xfw1); + vec_sxd.val[0] = vmulq_n_f32(vec_sxd.val[0], ptrGCoh[0]); + vec_sxd.val[1] = vmulq_n_f32(vec_sxd.val[1], ptrGCoh[0]); + vec_dfwxfw0011 = vmlaq_f32(vec_dfwxfw0011, vec_dfw1, vec_xfw1); + vec_dfwxfw0110 = vmlsq_f32(vec_dfwxfw0110, vec_dfw1, vec_xfw0); + vec_sxd.val[0] = vmlaq_n_f32(vec_sxd.val[0], vec_dfwxfw0011, ptrGCoh[1]); + vec_sxd.val[1] = vmlaq_n_f32(vec_sxd.val[1], vec_dfwxfw0110, ptrGCoh[1]); + vst2q_f32(&coherence_state->sxd[i][0], vec_sxd); + } + + vec_sdSum = vaddq_f32(vec_sdSum, vec_sd); + vec_seSum = vaddq_f32(vec_seSum, vec_se); + } + { + float32x2_t vec_sdSum_total; + float32x2_t vec_seSum_total; + // A B C D + vec_sdSum_total = + vpadd_f32(vget_low_f32(vec_sdSum), vget_high_f32(vec_sdSum)); + vec_seSum_total = + vpadd_f32(vget_low_f32(vec_seSum), vget_high_f32(vec_seSum)); + // A+B C+D + vec_sdSum_total = vpadd_f32(vec_sdSum_total, vec_sdSum_total); + vec_seSum_total = vpadd_f32(vec_seSum_total, vec_seSum_total); + // A+B+C+D A+B+C+D + sdSum = vget_lane_f32(vec_sdSum_total, 0); + seSum = vget_lane_f32(vec_seSum_total, 0); + } + + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + coherence_state->sd[i] = + ptrGCoh[0] * coherence_state->sd[i] + + ptrGCoh[1] * (dfw[0][i] * dfw[0][i] + dfw[1][i] * dfw[1][i]); + coherence_state->se[i] = + ptrGCoh[0] * coherence_state->se[i] + + ptrGCoh[1] * (efw[0][i] * efw[0][i] + efw[1][i] * efw[1][i]); + // We threshold here to protect against the ill-effects of a zero farend. + // The threshold is not arbitrarily chosen, but balances protection and + // adverse interaction with the algorithm's tuning. + // TODO(bjornv): investigate further why this is so sensitive. + coherence_state->sx[i] = + ptrGCoh[0] * coherence_state->sx[i] + + ptrGCoh[1] * + WEBRTC_SPL_MAX(xfw[0][i] * xfw[0][i] + xfw[1][i] * xfw[1][i], + WebRtcAec_kMinFarendPSD); + + coherence_state->sde[i][0] = + ptrGCoh[0] * coherence_state->sde[i][0] + + ptrGCoh[1] * (dfw[0][i] * efw[0][i] + dfw[1][i] * efw[1][i]); + coherence_state->sde[i][1] = + ptrGCoh[0] * coherence_state->sde[i][1] + + ptrGCoh[1] * (dfw[0][i] * efw[1][i] - dfw[1][i] * efw[0][i]); + + coherence_state->sxd[i][0] = + ptrGCoh[0] * coherence_state->sxd[i][0] + + ptrGCoh[1] * (dfw[0][i] * xfw[0][i] + dfw[1][i] * xfw[1][i]); + coherence_state->sxd[i][1] = + ptrGCoh[0] * coherence_state->sxd[i][1] + + ptrGCoh[1] * (dfw[0][i] * xfw[1][i] - dfw[1][i] * xfw[0][i]); + + sdSum += coherence_state->sd[i]; + seSum += coherence_state->se[i]; + } + + // Divergent filter safeguard update. + *filter_divergence_state = + (*filter_divergence_state ? 1.05f : 1.0f) * seSum > sdSum; + + // Signal extreme filter divergence if the error is significantly larger + // than the nearend (13 dB). + *extreme_filter_divergence = (seSum > (19.95f * sdSum)); +} + +// Window time domain data to be used by the fft. +static void WindowDataNEON(float* x_windowed, const float* x) { + int i; + for (i = 0; i < PART_LEN; i += 4) { + const float32x4_t vec_Buf1 = vld1q_f32(&x[i]); + const float32x4_t vec_Buf2 = vld1q_f32(&x[PART_LEN + i]); + const float32x4_t vec_sqrtHanning = vld1q_f32(&WebRtcAec_sqrtHanning[i]); + // A B C D + float32x4_t vec_sqrtHanning_rev = + vld1q_f32(&WebRtcAec_sqrtHanning[PART_LEN - i - 3]); + // B A D C + vec_sqrtHanning_rev = vrev64q_f32(vec_sqrtHanning_rev); + // D C B A + vec_sqrtHanning_rev = vcombine_f32(vget_high_f32(vec_sqrtHanning_rev), + vget_low_f32(vec_sqrtHanning_rev)); + vst1q_f32(&x_windowed[i], vmulq_f32(vec_Buf1, vec_sqrtHanning)); + vst1q_f32(&x_windowed[PART_LEN + i], + vmulq_f32(vec_Buf2, vec_sqrtHanning_rev)); + } +} + +// Puts fft output data into a complex valued array. +static void StoreAsComplexNEON(const float* data, + float data_complex[2][PART_LEN1]) { + int i; + for (i = 0; i < PART_LEN; i += 4) { + const float32x4x2_t vec_data = vld2q_f32(&data[2 * i]); + vst1q_f32(&data_complex[0][i], vec_data.val[0]); + vst1q_f32(&data_complex[1][i], vec_data.val[1]); + } + // fix beginning/end values + data_complex[1][0] = 0; + data_complex[1][PART_LEN] = 0; + data_complex[0][0] = data[0]; + data_complex[0][PART_LEN] = data[1]; +} + +static void ComputeCoherenceNEON(const CoherenceState* coherence_state, + float* cohde, + float* cohxd) { + int i; + + { + const float32x4_t vec_1eminus10 = vdupq_n_f32(1e-10f); + + // Subband coherence + for (i = 0; i + 3 < PART_LEN1; i += 4) { + const float32x4_t vec_sd = vld1q_f32(&coherence_state->sd[i]); + const float32x4_t vec_se = vld1q_f32(&coherence_state->se[i]); + const float32x4_t vec_sx = vld1q_f32(&coherence_state->sx[i]); + const float32x4_t vec_sdse = vmlaq_f32(vec_1eminus10, vec_sd, vec_se); + const float32x4_t vec_sdsx = vmlaq_f32(vec_1eminus10, vec_sd, vec_sx); + float32x4x2_t vec_sde = vld2q_f32(&coherence_state->sde[i][0]); + float32x4x2_t vec_sxd = vld2q_f32(&coherence_state->sxd[i][0]); + float32x4_t vec_cohde = vmulq_f32(vec_sde.val[0], vec_sde.val[0]); + float32x4_t vec_cohxd = vmulq_f32(vec_sxd.val[0], vec_sxd.val[0]); + vec_cohde = vmlaq_f32(vec_cohde, vec_sde.val[1], vec_sde.val[1]); + vec_cohde = vdivq_f32(vec_cohde, vec_sdse); + vec_cohxd = vmlaq_f32(vec_cohxd, vec_sxd.val[1], vec_sxd.val[1]); + vec_cohxd = vdivq_f32(vec_cohxd, vec_sdsx); + + vst1q_f32(&cohde[i], vec_cohde); + vst1q_f32(&cohxd[i], vec_cohxd); + } + } + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + cohde[i] = (coherence_state->sde[i][0] * coherence_state->sde[i][0] + + coherence_state->sde[i][1] * coherence_state->sde[i][1]) / + (coherence_state->sd[i] * coherence_state->se[i] + 1e-10f); + cohxd[i] = (coherence_state->sxd[i][0] * coherence_state->sxd[i][0] + + coherence_state->sxd[i][1] * coherence_state->sxd[i][1]) / + (coherence_state->sx[i] * coherence_state->sd[i] + 1e-10f); + } +} + +void WebRtcAec_InitAec_neon(void) { + WebRtcAec_FilterFar = FilterFarNEON; + WebRtcAec_ScaleErrorSignal = ScaleErrorSignalNEON; + WebRtcAec_FilterAdaptation = FilterAdaptationNEON; + WebRtcAec_Overdrive = OverdriveNEON; + WebRtcAec_Suppress = SuppressNEON; + WebRtcAec_ComputeCoherence = ComputeCoherenceNEON; + WebRtcAec_UpdateCoherenceSpectra = UpdateCoherenceSpectraNEON; + WebRtcAec_StoreAsComplex = StoreAsComplexNEON; + WebRtcAec_PartitionDelay = PartitionDelayNEON; + WebRtcAec_WindowData = WindowDataNEON; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_optimized_methods.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_optimized_methods.h new file mode 100644 index 0000000000..a8a20e94d8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_optimized_methods.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_OPTIMIZED_METHODS_H_ +#define MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_OPTIMIZED_METHODS_H_ + +#include + +#include "modules/audio_processing/aec/aec_core.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +typedef void (*WebRtcAecFilterFar)( + int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float y_fft[2][PART_LEN1]); +extern WebRtcAecFilterFar WebRtcAec_FilterFar; +typedef void (*WebRtcAecScaleErrorSignal)(float mu, + float error_threshold, + float x_pow[PART_LEN1], + float ef[2][PART_LEN1]); +extern WebRtcAecScaleErrorSignal WebRtcAec_ScaleErrorSignal; +typedef void (*WebRtcAecFilterAdaptation)( + const OouraFft& ooura_fft, + int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float e_fft[2][PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]); +extern WebRtcAecFilterAdaptation WebRtcAec_FilterAdaptation; + +typedef void (*WebRtcAecOverdrive)(float overdrive_scaling, + const float hNlFb, + float hNl[PART_LEN1]); +extern WebRtcAecOverdrive WebRtcAec_Overdrive; + +typedef void (*WebRtcAecSuppress)(const float hNl[PART_LEN1], + float efw[2][PART_LEN1]); +extern WebRtcAecSuppress WebRtcAec_Suppress; + +typedef void (*WebRtcAecComputeCoherence)(const CoherenceState* coherence_state, + float* cohde, + float* cohxd); +extern WebRtcAecComputeCoherence WebRtcAec_ComputeCoherence; + +typedef void (*WebRtcAecUpdateCoherenceSpectra)(int mult, + bool extended_filter_enabled, + float efw[2][PART_LEN1], + float dfw[2][PART_LEN1], + float xfw[2][PART_LEN1], + CoherenceState* coherence_state, + short* filter_divergence_state, + int* extreme_filter_divergence); +extern WebRtcAecUpdateCoherenceSpectra WebRtcAec_UpdateCoherenceSpectra; + +typedef int (*WebRtcAecPartitionDelay)( + int num_partitions, + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]); +extern WebRtcAecPartitionDelay WebRtcAec_PartitionDelay; + +typedef void (*WebRtcAecStoreAsComplex)(const float* data, + float data_complex[2][PART_LEN1]); +extern WebRtcAecStoreAsComplex WebRtcAec_StoreAsComplex; + +typedef void (*WebRtcAecWindowData)(float* x_windowed, const float* x); +extern WebRtcAecWindowData WebRtcAec_WindowData; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_AEC_CORE_OPTIMIZED_METHODS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_sse2.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_sse2.cc new file mode 100644 index 0000000000..0532662bed --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_sse2.cc @@ -0,0 +1,751 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * The core AEC algorithm, SSE2 version of speed-critical functions. + */ + +#include +#include +#include // memset + +extern "C" { +#include "common_audio/signal_processing/include/signal_processing_library.h" +} +#include "modules/audio_processing/aec/aec_common.h" +#include "modules/audio_processing/aec/aec_core_optimized_methods.h" +#include "modules/audio_processing/utility/ooura_fft.h" + +namespace webrtc { + +__inline static float MulRe(float aRe, float aIm, float bRe, float bIm) { + return aRe * bRe - aIm * bIm; +} + +__inline static float MulIm(float aRe, float aIm, float bRe, float bIm) { + return aRe * bIm + aIm * bRe; +} + +static void FilterFarSSE2(int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1], + float h_fft_buf[2] + [kExtendedNumPartitions * PART_LEN1], + float y_fft[2][PART_LEN1]) { + int i; + for (i = 0; i < num_partitions; i++) { + int j; + int xPos = (i + x_fft_buf_block_pos) * PART_LEN1; + int pos = i * PART_LEN1; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * (PART_LEN1); + } + + // vectorized code (four at once) + for (j = 0; j + 3 < PART_LEN1; j += 4) { + const __m128 x_fft_buf_re = _mm_loadu_ps(&x_fft_buf[0][xPos + j]); + const __m128 x_fft_buf_im = _mm_loadu_ps(&x_fft_buf[1][xPos + j]); + const __m128 h_fft_buf_re = _mm_loadu_ps(&h_fft_buf[0][pos + j]); + const __m128 h_fft_buf_im = _mm_loadu_ps(&h_fft_buf[1][pos + j]); + const __m128 y_fft_re = _mm_loadu_ps(&y_fft[0][j]); + const __m128 y_fft_im = _mm_loadu_ps(&y_fft[1][j]); + const __m128 a = _mm_mul_ps(x_fft_buf_re, h_fft_buf_re); + const __m128 b = _mm_mul_ps(x_fft_buf_im, h_fft_buf_im); + const __m128 c = _mm_mul_ps(x_fft_buf_re, h_fft_buf_im); + const __m128 d = _mm_mul_ps(x_fft_buf_im, h_fft_buf_re); + const __m128 e = _mm_sub_ps(a, b); + const __m128 f = _mm_add_ps(c, d); + const __m128 g = _mm_add_ps(y_fft_re, e); + const __m128 h = _mm_add_ps(y_fft_im, f); + _mm_storeu_ps(&y_fft[0][j], g); + _mm_storeu_ps(&y_fft[1][j], h); + } + // scalar code for the remaining items. + for (; j < PART_LEN1; j++) { + y_fft[0][j] += MulRe(x_fft_buf[0][xPos + j], x_fft_buf[1][xPos + j], + h_fft_buf[0][pos + j], h_fft_buf[1][pos + j]); + y_fft[1][j] += MulIm(x_fft_buf[0][xPos + j], x_fft_buf[1][xPos + j], + h_fft_buf[0][pos + j], h_fft_buf[1][pos + j]); + } + } +} + +static void ScaleErrorSignalSSE2(float mu, + float error_threshold, + float x_pow[PART_LEN1], + float ef[2][PART_LEN1]) { + const __m128 k1e_10f = _mm_set1_ps(1e-10f); + const __m128 kMu = _mm_set1_ps(mu); + const __m128 kThresh = _mm_set1_ps(error_threshold); + + int i; + // vectorized code (four at once) + for (i = 0; i + 3 < PART_LEN1; i += 4) { + const __m128 x_pow_local = _mm_loadu_ps(&x_pow[i]); + const __m128 ef_re_base = _mm_loadu_ps(&ef[0][i]); + const __m128 ef_im_base = _mm_loadu_ps(&ef[1][i]); + + const __m128 xPowPlus = _mm_add_ps(x_pow_local, k1e_10f); + __m128 ef_re = _mm_div_ps(ef_re_base, xPowPlus); + __m128 ef_im = _mm_div_ps(ef_im_base, xPowPlus); + const __m128 ef_re2 = _mm_mul_ps(ef_re, ef_re); + const __m128 ef_im2 = _mm_mul_ps(ef_im, ef_im); + const __m128 ef_sum2 = _mm_add_ps(ef_re2, ef_im2); + const __m128 absEf = _mm_sqrt_ps(ef_sum2); + const __m128 bigger = _mm_cmpgt_ps(absEf, kThresh); + __m128 absEfPlus = _mm_add_ps(absEf, k1e_10f); + const __m128 absEfInv = _mm_div_ps(kThresh, absEfPlus); + __m128 ef_re_if = _mm_mul_ps(ef_re, absEfInv); + __m128 ef_im_if = _mm_mul_ps(ef_im, absEfInv); + ef_re_if = _mm_and_ps(bigger, ef_re_if); + ef_im_if = _mm_and_ps(bigger, ef_im_if); + ef_re = _mm_andnot_ps(bigger, ef_re); + ef_im = _mm_andnot_ps(bigger, ef_im); + ef_re = _mm_or_ps(ef_re, ef_re_if); + ef_im = _mm_or_ps(ef_im, ef_im_if); + ef_re = _mm_mul_ps(ef_re, kMu); + ef_im = _mm_mul_ps(ef_im, kMu); + + _mm_storeu_ps(&ef[0][i], ef_re); + _mm_storeu_ps(&ef[1][i], ef_im); + } + // scalar code for the remaining items. + { + for (; i < (PART_LEN1); i++) { + float abs_ef; + ef[0][i] /= (x_pow[i] + 1e-10f); + ef[1][i] /= (x_pow[i] + 1e-10f); + abs_ef = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]); + + if (abs_ef > error_threshold) { + abs_ef = error_threshold / (abs_ef + 1e-10f); + ef[0][i] *= abs_ef; + ef[1][i] *= abs_ef; + } + + // Stepsize factor + ef[0][i] *= mu; + ef[1][i] *= mu; + } + } +} + +static void FilterAdaptationSSE2( + const OouraFft& ooura_fft, + int num_partitions, + int x_fft_buf_block_pos, + float x_fft_buf[2][kExtendedNumPartitions * PART_LEN1], + float e_fft[2][PART_LEN1], + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]) { + float fft[PART_LEN2]; + int i, j; + for (i = 0; i < num_partitions; i++) { + int xPos = (i + x_fft_buf_block_pos) * (PART_LEN1); + int pos = i * PART_LEN1; + // Check for wrap + if (i + x_fft_buf_block_pos >= num_partitions) { + xPos -= num_partitions * PART_LEN1; + } + + // Process the whole array... + for (j = 0; j < PART_LEN; j += 4) { + // Load x_fft_buf and e_fft. + const __m128 x_fft_buf_re = _mm_loadu_ps(&x_fft_buf[0][xPos + j]); + const __m128 x_fft_buf_im = _mm_loadu_ps(&x_fft_buf[1][xPos + j]); + const __m128 e_fft_re = _mm_loadu_ps(&e_fft[0][j]); + const __m128 e_fft_im = _mm_loadu_ps(&e_fft[1][j]); + // Calculate the product of conjugate(x_fft_buf) by e_fft. + // re(conjugate(a) * b) = aRe * bRe + aIm * bIm + // im(conjugate(a) * b)= aRe * bIm - aIm * bRe + const __m128 a = _mm_mul_ps(x_fft_buf_re, e_fft_re); + const __m128 b = _mm_mul_ps(x_fft_buf_im, e_fft_im); + const __m128 c = _mm_mul_ps(x_fft_buf_re, e_fft_im); + const __m128 d = _mm_mul_ps(x_fft_buf_im, e_fft_re); + const __m128 e = _mm_add_ps(a, b); + const __m128 f = _mm_sub_ps(c, d); + // Interleave real and imaginary parts. + const __m128 g = _mm_unpacklo_ps(e, f); + const __m128 h = _mm_unpackhi_ps(e, f); + // Store + _mm_storeu_ps(&fft[2 * j + 0], g); + _mm_storeu_ps(&fft[2 * j + 4], h); + } + // ... and fixup the first imaginary entry. + fft[1] = + MulRe(x_fft_buf[0][xPos + PART_LEN], -x_fft_buf[1][xPos + PART_LEN], + e_fft[0][PART_LEN], e_fft[1][PART_LEN]); + + ooura_fft.InverseFft(fft); + memset(fft + PART_LEN, 0, sizeof(float) * PART_LEN); + + // fft scaling + { + float scale = 2.0f / PART_LEN2; + const __m128 scale_ps = _mm_load_ps1(&scale); + for (j = 0; j < PART_LEN; j += 4) { + const __m128 fft_ps = _mm_loadu_ps(&fft[j]); + const __m128 fft_scale = _mm_mul_ps(fft_ps, scale_ps); + _mm_storeu_ps(&fft[j], fft_scale); + } + } + ooura_fft.Fft(fft); + + { + float wt1 = h_fft_buf[1][pos]; + h_fft_buf[0][pos + PART_LEN] += fft[1]; + for (j = 0; j < PART_LEN; j += 4) { + __m128 wtBuf_re = _mm_loadu_ps(&h_fft_buf[0][pos + j]); + __m128 wtBuf_im = _mm_loadu_ps(&h_fft_buf[1][pos + j]); + const __m128 fft0 = _mm_loadu_ps(&fft[2 * j + 0]); + const __m128 fft4 = _mm_loadu_ps(&fft[2 * j + 4]); + const __m128 fft_re = + _mm_shuffle_ps(fft0, fft4, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128 fft_im = + _mm_shuffle_ps(fft0, fft4, _MM_SHUFFLE(3, 1, 3, 1)); + wtBuf_re = _mm_add_ps(wtBuf_re, fft_re); + wtBuf_im = _mm_add_ps(wtBuf_im, fft_im); + _mm_storeu_ps(&h_fft_buf[0][pos + j], wtBuf_re); + _mm_storeu_ps(&h_fft_buf[1][pos + j], wtBuf_im); + } + h_fft_buf[1][pos] = wt1; + } + } +} + +static __m128 mm_pow_ps(__m128 a, __m128 b) { + // a^b = exp2(b * log2(a)) + // exp2(x) and log2(x) are calculated using polynomial approximations. + __m128 log2_a, b_log2_a, a_exp_b; + + // Calculate log2(x), x = a. + { + // To calculate log2(x), we decompose x like this: + // x = y * 2^n + // n is an integer + // y is in the [1.0, 2.0) range + // + // log2(x) = log2(y) + n + // n can be evaluated by playing with float representation. + // log2(y) in a small range can be approximated, this code uses an order + // five polynomial approximation. The coefficients have been + // estimated with the Remez algorithm and the resulting + // polynomial has a maximum relative error of 0.00086%. + + // Compute n. + // This is done by masking the exponent, shifting it into the top bit of + // the mantissa, putting eight into the biased exponent (to shift/ + // compensate the fact that the exponent has been shifted in the top/ + // fractional part and finally getting rid of the implicit leading one + // from the mantissa by substracting it out. + static const ALIGN16_BEG int float_exponent_mask[4] ALIGN16_END = { + 0x7F800000, 0x7F800000, 0x7F800000, 0x7F800000}; + static const ALIGN16_BEG int eight_biased_exponent[4] ALIGN16_END = { + 0x43800000, 0x43800000, 0x43800000, 0x43800000}; + static const ALIGN16_BEG int implicit_leading_one[4] ALIGN16_END = { + 0x43BF8000, 0x43BF8000, 0x43BF8000, 0x43BF8000}; + static const int shift_exponent_into_top_mantissa = 8; + const __m128 two_n = + _mm_and_ps(a, *(reinterpret_cast(float_exponent_mask))); + const __m128 n_1 = _mm_castsi128_ps(_mm_srli_epi32( + _mm_castps_si128(two_n), shift_exponent_into_top_mantissa)); + const __m128 n_0 = + _mm_or_ps(n_1, *(reinterpret_cast(eight_biased_exponent))); + const __m128 n = + _mm_sub_ps(n_0, *(reinterpret_cast(implicit_leading_one))); + + // Compute y. + static const ALIGN16_BEG int mantissa_mask[4] ALIGN16_END = { + 0x007FFFFF, 0x007FFFFF, 0x007FFFFF, 0x007FFFFF}; + static const ALIGN16_BEG int zero_biased_exponent_is_one[4] ALIGN16_END = { + 0x3F800000, 0x3F800000, 0x3F800000, 0x3F800000}; + const __m128 mantissa = + _mm_and_ps(a, *(reinterpret_cast(mantissa_mask))); + const __m128 y = + _mm_or_ps(mantissa, + *(reinterpret_cast(zero_biased_exponent_is_one))); + + // Approximate log2(y) ~= (y - 1) * pol5(y). + // pol5(y) = C5 * y^5 + C4 * y^4 + C3 * y^3 + C2 * y^2 + C1 * y + C0 + static const ALIGN16_BEG float ALIGN16_END C5[4] = { + -3.4436006e-2f, -3.4436006e-2f, -3.4436006e-2f, -3.4436006e-2f}; + static const ALIGN16_BEG float ALIGN16_END C4[4] = { + 3.1821337e-1f, 3.1821337e-1f, 3.1821337e-1f, 3.1821337e-1f}; + static const ALIGN16_BEG float ALIGN16_END C3[4] = { + -1.2315303f, -1.2315303f, -1.2315303f, -1.2315303f}; + static const ALIGN16_BEG float ALIGN16_END C2[4] = {2.5988452f, 2.5988452f, + 2.5988452f, 2.5988452f}; + static const ALIGN16_BEG float ALIGN16_END C1[4] = { + -3.3241990f, -3.3241990f, -3.3241990f, -3.3241990f}; + static const ALIGN16_BEG float ALIGN16_END C0[4] = {3.1157899f, 3.1157899f, + 3.1157899f, 3.1157899f}; + const __m128 pol5_y_0 = + _mm_mul_ps(y, *(reinterpret_cast(C5))); + const __m128 pol5_y_1 = + _mm_add_ps(pol5_y_0, *(reinterpret_cast(C4))); + const __m128 pol5_y_2 = _mm_mul_ps(pol5_y_1, y); + const __m128 pol5_y_3 = + _mm_add_ps(pol5_y_2, *(reinterpret_cast(C3))); + const __m128 pol5_y_4 = _mm_mul_ps(pol5_y_3, y); + const __m128 pol5_y_5 = + _mm_add_ps(pol5_y_4, *(reinterpret_cast(C2))); + const __m128 pol5_y_6 = _mm_mul_ps(pol5_y_5, y); + const __m128 pol5_y_7 = + _mm_add_ps(pol5_y_6, *(reinterpret_cast(C1))); + const __m128 pol5_y_8 = _mm_mul_ps(pol5_y_7, y); + const __m128 pol5_y = + _mm_add_ps(pol5_y_8, *(reinterpret_cast(C0))); + const __m128 y_minus_one = + _mm_sub_ps(y, + *(reinterpret_cast(zero_biased_exponent_is_one))); + const __m128 log2_y = _mm_mul_ps(y_minus_one, pol5_y); + + // Combine parts. + log2_a = _mm_add_ps(n, log2_y); + } + + // b * log2(a) + b_log2_a = _mm_mul_ps(b, log2_a); + + // Calculate exp2(x), x = b * log2(a). + { + // To calculate 2^x, we decompose x like this: + // x = n + y + // n is an integer, the value of x - 0.5 rounded down, therefore + // y is in the [0.5, 1.5) range + // + // 2^x = 2^n * 2^y + // 2^n can be evaluated by playing with float representation. + // 2^y in a small range can be approximated, this code uses an order two + // polynomial approximation. The coefficients have been estimated + // with the Remez algorithm and the resulting polynomial has a + // maximum relative error of 0.17%. + + // To avoid over/underflow, we reduce the range of input to ]-127, 129]. + static const ALIGN16_BEG float max_input[4] ALIGN16_END = {129.f, 129.f, + 129.f, 129.f}; + static const ALIGN16_BEG float min_input[4] ALIGN16_END = { + -126.99999f, -126.99999f, -126.99999f, -126.99999f}; + const __m128 x_min = + _mm_min_ps(b_log2_a, *(reinterpret_cast(max_input))); + const __m128 x_max = + _mm_max_ps(x_min, *(reinterpret_cast(min_input))); + // Compute n. + static const ALIGN16_BEG float half[4] ALIGN16_END = {0.5f, 0.5f, 0.5f, + 0.5f}; + const __m128 x_minus_half = + _mm_sub_ps(x_max, *(reinterpret_cast(half))); + const __m128i x_minus_half_floor = _mm_cvtps_epi32(x_minus_half); + // Compute 2^n. + static const ALIGN16_BEG int float_exponent_bias[4] ALIGN16_END = { + 127, 127, 127, 127}; + static const int float_exponent_shift = 23; + const __m128i two_n_exponent = + _mm_add_epi32(x_minus_half_floor, + *(reinterpret_cast(float_exponent_bias))); + const __m128 two_n = + _mm_castsi128_ps(_mm_slli_epi32(two_n_exponent, float_exponent_shift)); + // Compute y. + const __m128 y = _mm_sub_ps(x_max, _mm_cvtepi32_ps(x_minus_half_floor)); + // Approximate 2^y ~= C2 * y^2 + C1 * y + C0. + static const ALIGN16_BEG float C2[4] ALIGN16_END = { + 3.3718944e-1f, 3.3718944e-1f, 3.3718944e-1f, 3.3718944e-1f}; + static const ALIGN16_BEG float C1[4] ALIGN16_END = { + 6.5763628e-1f, 6.5763628e-1f, 6.5763628e-1f, 6.5763628e-1f}; + static const ALIGN16_BEG float C0[4] ALIGN16_END = {1.0017247f, 1.0017247f, + 1.0017247f, 1.0017247f}; + const __m128 exp2_y_0 = + _mm_mul_ps(y, *(reinterpret_cast(C2))); + const __m128 exp2_y_1 = + _mm_add_ps(exp2_y_0, *(reinterpret_cast(C1))); + const __m128 exp2_y_2 = _mm_mul_ps(exp2_y_1, y); + const __m128 exp2_y = + _mm_add_ps(exp2_y_2, *(reinterpret_cast(C0))); + + // Combine parts. + a_exp_b = _mm_mul_ps(exp2_y, two_n); + } + return a_exp_b; +} + +static void OverdriveSSE2(float overdrive_scaling, + float hNlFb, + float hNl[PART_LEN1]) { + int i; + const __m128 vec_hNlFb = _mm_set1_ps(hNlFb); + const __m128 vec_one = _mm_set1_ps(1.0f); + const __m128 vec_overdrive_scaling = _mm_set1_ps(overdrive_scaling); + // vectorized code (four at once) + for (i = 0; i + 3 < PART_LEN1; i += 4) { + // Weight subbands + __m128 vec_hNl = _mm_loadu_ps(&hNl[i]); + const __m128 vec_weightCurve = _mm_loadu_ps(&WebRtcAec_weightCurve[i]); + const __m128 bigger = _mm_cmpgt_ps(vec_hNl, vec_hNlFb); + const __m128 vec_weightCurve_hNlFb = _mm_mul_ps(vec_weightCurve, vec_hNlFb); + const __m128 vec_one_weightCurve = _mm_sub_ps(vec_one, vec_weightCurve); + const __m128 vec_one_weightCurve_hNl = + _mm_mul_ps(vec_one_weightCurve, vec_hNl); + const __m128 vec_if0 = _mm_andnot_ps(bigger, vec_hNl); + const __m128 vec_if1 = _mm_and_ps( + bigger, _mm_add_ps(vec_weightCurve_hNlFb, vec_one_weightCurve_hNl)); + vec_hNl = _mm_or_ps(vec_if0, vec_if1); + + const __m128 vec_overDriveCurve = + _mm_loadu_ps(&WebRtcAec_overDriveCurve[i]); + const __m128 vec_overDriveSm_overDriveCurve = + _mm_mul_ps(vec_overdrive_scaling, vec_overDriveCurve); + vec_hNl = mm_pow_ps(vec_hNl, vec_overDriveSm_overDriveCurve); + _mm_storeu_ps(&hNl[i], vec_hNl); + } + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + // Weight subbands + if (hNl[i] > hNlFb) { + hNl[i] = WebRtcAec_weightCurve[i] * hNlFb + + (1 - WebRtcAec_weightCurve[i]) * hNl[i]; + } + hNl[i] = powf(hNl[i], overdrive_scaling * WebRtcAec_overDriveCurve[i]); + } +} + +static void SuppressSSE2(const float hNl[PART_LEN1], float efw[2][PART_LEN1]) { + int i; + const __m128 vec_minus_one = _mm_set1_ps(-1.0f); + // vectorized code (four at once) + for (i = 0; i + 3 < PART_LEN1; i += 4) { + // Suppress error signal + __m128 vec_hNl = _mm_loadu_ps(&hNl[i]); + __m128 vec_efw_re = _mm_loadu_ps(&efw[0][i]); + __m128 vec_efw_im = _mm_loadu_ps(&efw[1][i]); + vec_efw_re = _mm_mul_ps(vec_efw_re, vec_hNl); + vec_efw_im = _mm_mul_ps(vec_efw_im, vec_hNl); + + // Ooura fft returns incorrect sign on imaginary component. It matters + // here because we are making an additive change with comfort noise. + vec_efw_im = _mm_mul_ps(vec_efw_im, vec_minus_one); + _mm_storeu_ps(&efw[0][i], vec_efw_re); + _mm_storeu_ps(&efw[1][i], vec_efw_im); + } + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + // Suppress error signal + efw[0][i] *= hNl[i]; + efw[1][i] *= hNl[i]; + + // Ooura fft returns incorrect sign on imaginary component. It matters + // here because we are making an additive change with comfort noise. + efw[1][i] *= -1; + } +} + +__inline static void _mm_add_ps_4x1(__m128 sum, float* dst) { + // A+B C+D + sum = _mm_add_ps(sum, _mm_shuffle_ps(sum, sum, _MM_SHUFFLE(0, 0, 3, 2))); + // A+B+C+D A+B+C+D + sum = _mm_add_ps(sum, _mm_shuffle_ps(sum, sum, _MM_SHUFFLE(1, 1, 1, 1))); + _mm_store_ss(dst, sum); +} + +static int PartitionDelaySSE2( + int num_partitions, + float h_fft_buf[2][kExtendedNumPartitions * PART_LEN1]) { + // Measures the energy in each filter partition and returns the partition with + // highest energy. + // TODO(bjornv): Spread computational cost by computing one partition per + // block? + float wfEnMax = 0; + int i; + int delay = 0; + + for (i = 0; i < num_partitions; i++) { + int j; + int pos = i * PART_LEN1; + float wfEn = 0; + __m128 vec_wfEn = _mm_set1_ps(0.0f); + // vectorized code (four at once) + for (j = 0; j + 3 < PART_LEN1; j += 4) { + const __m128 vec_wfBuf0 = _mm_loadu_ps(&h_fft_buf[0][pos + j]); + const __m128 vec_wfBuf1 = _mm_loadu_ps(&h_fft_buf[1][pos + j]); + vec_wfEn = _mm_add_ps(vec_wfEn, _mm_mul_ps(vec_wfBuf0, vec_wfBuf0)); + vec_wfEn = _mm_add_ps(vec_wfEn, _mm_mul_ps(vec_wfBuf1, vec_wfBuf1)); + } + _mm_add_ps_4x1(vec_wfEn, &wfEn); + + // scalar code for the remaining items. + for (; j < PART_LEN1; j++) { + wfEn += h_fft_buf[0][pos + j] * h_fft_buf[0][pos + j] + + h_fft_buf[1][pos + j] * h_fft_buf[1][pos + j]; + } + + if (wfEn > wfEnMax) { + wfEnMax = wfEn; + delay = i; + } + } + return delay; +} + +// Updates the following smoothed Power Spectral Densities (PSD): +// - sd : near-end +// - se : residual echo +// - sx : far-end +// - sde : cross-PSD of near-end and residual echo +// - sxd : cross-PSD of near-end and far-end +// +// In addition to updating the PSDs, also the filter diverge state is determined +// upon actions are taken. +static void UpdateCoherenceSpectraSSE2(int mult, + bool extended_filter_enabled, + float efw[2][PART_LEN1], + float dfw[2][PART_LEN1], + float xfw[2][PART_LEN1], + CoherenceState* coherence_state, + short* filter_divergence_state, + int* extreme_filter_divergence) { + // Power estimate smoothing coefficients. + const float* ptrGCoh = + extended_filter_enabled + ? WebRtcAec_kExtendedSmoothingCoefficients[mult - 1] + : WebRtcAec_kNormalSmoothingCoefficients[mult - 1]; + int i; + float sdSum = 0, seSum = 0; + const __m128 vec_15 = _mm_set1_ps(WebRtcAec_kMinFarendPSD); + const __m128 vec_GCoh0 = _mm_set1_ps(ptrGCoh[0]); + const __m128 vec_GCoh1 = _mm_set1_ps(ptrGCoh[1]); + __m128 vec_sdSum = _mm_set1_ps(0.0f); + __m128 vec_seSum = _mm_set1_ps(0.0f); + + for (i = 0; i + 3 < PART_LEN1; i += 4) { + const __m128 vec_dfw0 = _mm_loadu_ps(&dfw[0][i]); + const __m128 vec_dfw1 = _mm_loadu_ps(&dfw[1][i]); + const __m128 vec_efw0 = _mm_loadu_ps(&efw[0][i]); + const __m128 vec_efw1 = _mm_loadu_ps(&efw[1][i]); + const __m128 vec_xfw0 = _mm_loadu_ps(&xfw[0][i]); + const __m128 vec_xfw1 = _mm_loadu_ps(&xfw[1][i]); + __m128 vec_sd = + _mm_mul_ps(_mm_loadu_ps(&coherence_state->sd[i]), vec_GCoh0); + __m128 vec_se = + _mm_mul_ps(_mm_loadu_ps(&coherence_state->se[i]), vec_GCoh0); + __m128 vec_sx = + _mm_mul_ps(_mm_loadu_ps(&coherence_state->sx[i]), vec_GCoh0); + __m128 vec_dfw_sumsq = _mm_mul_ps(vec_dfw0, vec_dfw0); + __m128 vec_efw_sumsq = _mm_mul_ps(vec_efw0, vec_efw0); + __m128 vec_xfw_sumsq = _mm_mul_ps(vec_xfw0, vec_xfw0); + vec_dfw_sumsq = _mm_add_ps(vec_dfw_sumsq, _mm_mul_ps(vec_dfw1, vec_dfw1)); + vec_efw_sumsq = _mm_add_ps(vec_efw_sumsq, _mm_mul_ps(vec_efw1, vec_efw1)); + vec_xfw_sumsq = _mm_add_ps(vec_xfw_sumsq, _mm_mul_ps(vec_xfw1, vec_xfw1)); + vec_xfw_sumsq = _mm_max_ps(vec_xfw_sumsq, vec_15); + vec_sd = _mm_add_ps(vec_sd, _mm_mul_ps(vec_dfw_sumsq, vec_GCoh1)); + vec_se = _mm_add_ps(vec_se, _mm_mul_ps(vec_efw_sumsq, vec_GCoh1)); + vec_sx = _mm_add_ps(vec_sx, _mm_mul_ps(vec_xfw_sumsq, vec_GCoh1)); + _mm_storeu_ps(&coherence_state->sd[i], vec_sd); + _mm_storeu_ps(&coherence_state->se[i], vec_se); + _mm_storeu_ps(&coherence_state->sx[i], vec_sx); + + { + const __m128 vec_3210 = _mm_loadu_ps(&coherence_state->sde[i][0]); + const __m128 vec_7654 = _mm_loadu_ps(&coherence_state->sde[i + 2][0]); + __m128 vec_a = + _mm_shuffle_ps(vec_3210, vec_7654, _MM_SHUFFLE(2, 0, 2, 0)); + __m128 vec_b = + _mm_shuffle_ps(vec_3210, vec_7654, _MM_SHUFFLE(3, 1, 3, 1)); + __m128 vec_dfwefw0011 = _mm_mul_ps(vec_dfw0, vec_efw0); + __m128 vec_dfwefw0110 = _mm_mul_ps(vec_dfw0, vec_efw1); + vec_a = _mm_mul_ps(vec_a, vec_GCoh0); + vec_b = _mm_mul_ps(vec_b, vec_GCoh0); + vec_dfwefw0011 = + _mm_add_ps(vec_dfwefw0011, _mm_mul_ps(vec_dfw1, vec_efw1)); + vec_dfwefw0110 = + _mm_sub_ps(vec_dfwefw0110, _mm_mul_ps(vec_dfw1, vec_efw0)); + vec_a = _mm_add_ps(vec_a, _mm_mul_ps(vec_dfwefw0011, vec_GCoh1)); + vec_b = _mm_add_ps(vec_b, _mm_mul_ps(vec_dfwefw0110, vec_GCoh1)); + _mm_storeu_ps(&coherence_state->sde[i][0], _mm_unpacklo_ps(vec_a, vec_b)); + _mm_storeu_ps(&coherence_state->sde[i + 2][0], + _mm_unpackhi_ps(vec_a, vec_b)); + } + + { + const __m128 vec_3210 = _mm_loadu_ps(&coherence_state->sxd[i][0]); + const __m128 vec_7654 = _mm_loadu_ps(&coherence_state->sxd[i + 2][0]); + __m128 vec_a = + _mm_shuffle_ps(vec_3210, vec_7654, _MM_SHUFFLE(2, 0, 2, 0)); + __m128 vec_b = + _mm_shuffle_ps(vec_3210, vec_7654, _MM_SHUFFLE(3, 1, 3, 1)); + __m128 vec_dfwxfw0011 = _mm_mul_ps(vec_dfw0, vec_xfw0); + __m128 vec_dfwxfw0110 = _mm_mul_ps(vec_dfw0, vec_xfw1); + vec_a = _mm_mul_ps(vec_a, vec_GCoh0); + vec_b = _mm_mul_ps(vec_b, vec_GCoh0); + vec_dfwxfw0011 = + _mm_add_ps(vec_dfwxfw0011, _mm_mul_ps(vec_dfw1, vec_xfw1)); + vec_dfwxfw0110 = + _mm_sub_ps(vec_dfwxfw0110, _mm_mul_ps(vec_dfw1, vec_xfw0)); + vec_a = _mm_add_ps(vec_a, _mm_mul_ps(vec_dfwxfw0011, vec_GCoh1)); + vec_b = _mm_add_ps(vec_b, _mm_mul_ps(vec_dfwxfw0110, vec_GCoh1)); + _mm_storeu_ps(&coherence_state->sxd[i][0], _mm_unpacklo_ps(vec_a, vec_b)); + _mm_storeu_ps(&coherence_state->sxd[i + 2][0], + _mm_unpackhi_ps(vec_a, vec_b)); + } + + vec_sdSum = _mm_add_ps(vec_sdSum, vec_sd); + vec_seSum = _mm_add_ps(vec_seSum, vec_se); + } + + _mm_add_ps_4x1(vec_sdSum, &sdSum); + _mm_add_ps_4x1(vec_seSum, &seSum); + + for (; i < PART_LEN1; i++) { + coherence_state->sd[i] = + ptrGCoh[0] * coherence_state->sd[i] + + ptrGCoh[1] * (dfw[0][i] * dfw[0][i] + dfw[1][i] * dfw[1][i]); + coherence_state->se[i] = + ptrGCoh[0] * coherence_state->se[i] + + ptrGCoh[1] * (efw[0][i] * efw[0][i] + efw[1][i] * efw[1][i]); + // We threshold here to protect against the ill-effects of a zero farend. + // The threshold is not arbitrarily chosen, but balances protection and + // adverse interaction with the algorithm's tuning. + // TODO(bjornv): investigate further why this is so sensitive. + coherence_state->sx[i] = + ptrGCoh[0] * coherence_state->sx[i] + + ptrGCoh[1] * + WEBRTC_SPL_MAX(xfw[0][i] * xfw[0][i] + xfw[1][i] * xfw[1][i], + WebRtcAec_kMinFarendPSD); + + coherence_state->sde[i][0] = + ptrGCoh[0] * coherence_state->sde[i][0] + + ptrGCoh[1] * (dfw[0][i] * efw[0][i] + dfw[1][i] * efw[1][i]); + coherence_state->sde[i][1] = + ptrGCoh[0] * coherence_state->sde[i][1] + + ptrGCoh[1] * (dfw[0][i] * efw[1][i] - dfw[1][i] * efw[0][i]); + + coherence_state->sxd[i][0] = + ptrGCoh[0] * coherence_state->sxd[i][0] + + ptrGCoh[1] * (dfw[0][i] * xfw[0][i] + dfw[1][i] * xfw[1][i]); + coherence_state->sxd[i][1] = + ptrGCoh[0] * coherence_state->sxd[i][1] + + ptrGCoh[1] * (dfw[0][i] * xfw[1][i] - dfw[1][i] * xfw[0][i]); + + sdSum += coherence_state->sd[i]; + seSum += coherence_state->se[i]; + } + + // Divergent filter safeguard update. + *filter_divergence_state = + (*filter_divergence_state ? 1.05f : 1.0f) * seSum > sdSum; + + // Signal extreme filter divergence if the error is significantly larger + // than the nearend (13 dB). + *extreme_filter_divergence = (seSum > (19.95f * sdSum)); +} + +// Window time domain data to be used by the fft. +static void WindowDataSSE2(float* x_windowed, const float* x) { + int i; + for (i = 0; i < PART_LEN; i += 4) { + const __m128 vec_Buf1 = _mm_loadu_ps(&x[i]); + const __m128 vec_Buf2 = _mm_loadu_ps(&x[PART_LEN + i]); + const __m128 vec_sqrtHanning = _mm_load_ps(&WebRtcAec_sqrtHanning[i]); + // A B C D + __m128 vec_sqrtHanning_rev = + _mm_loadu_ps(&WebRtcAec_sqrtHanning[PART_LEN - i - 3]); + // D C B A + vec_sqrtHanning_rev = _mm_shuffle_ps( + vec_sqrtHanning_rev, vec_sqrtHanning_rev, _MM_SHUFFLE(0, 1, 2, 3)); + _mm_storeu_ps(&x_windowed[i], _mm_mul_ps(vec_Buf1, vec_sqrtHanning)); + _mm_storeu_ps(&x_windowed[PART_LEN + i], + _mm_mul_ps(vec_Buf2, vec_sqrtHanning_rev)); + } +} + +// Puts fft output data into a complex valued array. +static void StoreAsComplexSSE2(const float* data, + float data_complex[2][PART_LEN1]) { + int i; + for (i = 0; i < PART_LEN; i += 4) { + const __m128 vec_fft0 = _mm_loadu_ps(&data[2 * i]); + const __m128 vec_fft4 = _mm_loadu_ps(&data[2 * i + 4]); + const __m128 vec_a = + _mm_shuffle_ps(vec_fft0, vec_fft4, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128 vec_b = + _mm_shuffle_ps(vec_fft0, vec_fft4, _MM_SHUFFLE(3, 1, 3, 1)); + _mm_storeu_ps(&data_complex[0][i], vec_a); + _mm_storeu_ps(&data_complex[1][i], vec_b); + } + // fix beginning/end values + data_complex[1][0] = 0; + data_complex[1][PART_LEN] = 0; + data_complex[0][0] = data[0]; + data_complex[0][PART_LEN] = data[1]; +} + +static void ComputeCoherenceSSE2(const CoherenceState* coherence_state, + float* cohde, + float* cohxd) { + int i; + + { + const __m128 vec_1eminus10 = _mm_set1_ps(1e-10f); + + // Subband coherence + for (i = 0; i + 3 < PART_LEN1; i += 4) { + const __m128 vec_sd = _mm_loadu_ps(&coherence_state->sd[i]); + const __m128 vec_se = _mm_loadu_ps(&coherence_state->se[i]); + const __m128 vec_sx = _mm_loadu_ps(&coherence_state->sx[i]); + const __m128 vec_sdse = + _mm_add_ps(vec_1eminus10, _mm_mul_ps(vec_sd, vec_se)); + const __m128 vec_sdsx = + _mm_add_ps(vec_1eminus10, _mm_mul_ps(vec_sd, vec_sx)); + const __m128 vec_sde_3210 = _mm_loadu_ps(&coherence_state->sde[i][0]); + const __m128 vec_sde_7654 = _mm_loadu_ps(&coherence_state->sde[i + 2][0]); + const __m128 vec_sxd_3210 = _mm_loadu_ps(&coherence_state->sxd[i][0]); + const __m128 vec_sxd_7654 = _mm_loadu_ps(&coherence_state->sxd[i + 2][0]); + const __m128 vec_sde_0 = + _mm_shuffle_ps(vec_sde_3210, vec_sde_7654, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128 vec_sde_1 = + _mm_shuffle_ps(vec_sde_3210, vec_sde_7654, _MM_SHUFFLE(3, 1, 3, 1)); + const __m128 vec_sxd_0 = + _mm_shuffle_ps(vec_sxd_3210, vec_sxd_7654, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128 vec_sxd_1 = + _mm_shuffle_ps(vec_sxd_3210, vec_sxd_7654, _MM_SHUFFLE(3, 1, 3, 1)); + __m128 vec_cohde = _mm_mul_ps(vec_sde_0, vec_sde_0); + __m128 vec_cohxd = _mm_mul_ps(vec_sxd_0, vec_sxd_0); + vec_cohde = _mm_add_ps(vec_cohde, _mm_mul_ps(vec_sde_1, vec_sde_1)); + vec_cohde = _mm_div_ps(vec_cohde, vec_sdse); + vec_cohxd = _mm_add_ps(vec_cohxd, _mm_mul_ps(vec_sxd_1, vec_sxd_1)); + vec_cohxd = _mm_div_ps(vec_cohxd, vec_sdsx); + _mm_storeu_ps(&cohde[i], vec_cohde); + _mm_storeu_ps(&cohxd[i], vec_cohxd); + } + + // scalar code for the remaining items. + for (; i < PART_LEN1; i++) { + cohde[i] = (coherence_state->sde[i][0] * coherence_state->sde[i][0] + + coherence_state->sde[i][1] * coherence_state->sde[i][1]) / + (coherence_state->sd[i] * coherence_state->se[i] + 1e-10f); + cohxd[i] = (coherence_state->sxd[i][0] * coherence_state->sxd[i][0] + + coherence_state->sxd[i][1] * coherence_state->sxd[i][1]) / + (coherence_state->sx[i] * coherence_state->sd[i] + 1e-10f); + } + } +} + +void WebRtcAec_InitAec_SSE2(void) { + WebRtcAec_FilterFar = FilterFarSSE2; + WebRtcAec_ScaleErrorSignal = ScaleErrorSignalSSE2; + WebRtcAec_FilterAdaptation = FilterAdaptationSSE2; + WebRtcAec_Overdrive = OverdriveSSE2; + WebRtcAec_Suppress = SuppressSSE2; + WebRtcAec_ComputeCoherence = ComputeCoherenceSSE2; + WebRtcAec_UpdateCoherenceSpectra = UpdateCoherenceSpectraSSE2; + WebRtcAec_StoreAsComplex = StoreAsComplexSSE2; + WebRtcAec_PartitionDelay = PartitionDelaySSE2; + WebRtcAec_WindowData = WindowDataSSE2; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.cc new file mode 100644 index 0000000000..2851c0b6dd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.cc @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* Resamples a signal to an arbitrary rate. Used by the AEC to compensate for + * clock skew by resampling the farend signal. + */ + +#include "modules/audio_processing/aec/aec_resampler.h" + +#include +#include +#include + +#include "modules/audio_processing/aec/aec_core.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +enum { kEstimateLengthFrames = 400 }; + +typedef struct { + float buffer[kResamplerBufferSize]; + float position; + + int deviceSampleRateHz; + int skewData[kEstimateLengthFrames]; + int skewDataIndex; + float skewEstimate; +} AecResampler; + +static int EstimateSkew(const int* rawSkew, + int size, + int deviceSampleRateHz, + float* skewEst); + +void* WebRtcAec_CreateResampler() { + return malloc(sizeof(AecResampler)); +} + +int WebRtcAec_InitResampler(void* resampInst, int deviceSampleRateHz) { + AecResampler* obj = static_cast(resampInst); + memset(obj->buffer, 0, sizeof(obj->buffer)); + obj->position = 0.0; + + obj->deviceSampleRateHz = deviceSampleRateHz; + memset(obj->skewData, 0, sizeof(obj->skewData)); + obj->skewDataIndex = 0; + obj->skewEstimate = 0.0; + + return 0; +} + +void WebRtcAec_FreeResampler(void* resampInst) { + AecResampler* obj = static_cast(resampInst); + free(obj); +} + +void WebRtcAec_ResampleLinear(void* resampInst, + const float* inspeech, + size_t size, + float skew, + float* outspeech, + size_t* size_out) { + AecResampler* obj = static_cast(resampInst); + + float* y; + float be, tnew; + size_t tn, mm; + + RTC_DCHECK_LE(size, 2 * FRAME_LEN); + RTC_DCHECK(resampInst); + RTC_DCHECK(inspeech); + RTC_DCHECK(outspeech); + RTC_DCHECK(size_out); + + // Add new frame data in lookahead + memcpy(&obj->buffer[FRAME_LEN + kResamplingDelay], inspeech, + size * sizeof(inspeech[0])); + + // Sample rate ratio + be = 1 + skew; + + // Loop over input frame + mm = 0; + y = &obj->buffer[FRAME_LEN]; // Point at current frame + + tnew = be * mm + obj->position; + tn = (size_t)tnew; + + while (tn < size) { + // Interpolation + outspeech[mm] = y[tn] + (tnew - tn) * (y[tn + 1] - y[tn]); + mm++; + + tnew = be * mm + obj->position; + tn = static_cast(tnew); + } + + *size_out = mm; + obj->position += (*size_out) * be - size; + + // Shift buffer + memmove(obj->buffer, &obj->buffer[size], + (kResamplerBufferSize - size) * sizeof(obj->buffer[0])); +} + +int WebRtcAec_GetSkew(void* resampInst, int rawSkew, float* skewEst) { + AecResampler* obj = static_cast(resampInst); + int err = 0; + + if (obj->skewDataIndex < kEstimateLengthFrames) { + obj->skewData[obj->skewDataIndex] = rawSkew; + obj->skewDataIndex++; + } else if (obj->skewDataIndex == kEstimateLengthFrames) { + err = EstimateSkew(obj->skewData, kEstimateLengthFrames, + obj->deviceSampleRateHz, skewEst); + obj->skewEstimate = *skewEst; + obj->skewDataIndex++; + } else { + *skewEst = obj->skewEstimate; + } + + return err; +} + +int EstimateSkew(const int* rawSkew, + int size, + int deviceSampleRateHz, + float* skewEst) { + const int absLimitOuter = static_cast(0.04f * deviceSampleRateHz); + const int absLimitInner = static_cast(0.0025f * deviceSampleRateHz); + int i = 0; + int n = 0; + float rawAvg = 0; + float err = 0; + float rawAbsDev = 0; + int upperLimit = 0; + int lowerLimit = 0; + float cumSum = 0; + float x = 0; + float x2 = 0; + float y = 0; + float xy = 0; + float xAvg = 0; + float denom = 0; + float skew = 0; + + *skewEst = 0; // Set in case of error below. + for (i = 0; i < size; i++) { + if ((rawSkew[i] < absLimitOuter && rawSkew[i] > -absLimitOuter)) { + n++; + rawAvg += rawSkew[i]; + } + } + + if (n == 0) { + return -1; + } + RTC_DCHECK_GT(n, 0); + rawAvg /= n; + + for (i = 0; i < size; i++) { + if ((rawSkew[i] < absLimitOuter && rawSkew[i] > -absLimitOuter)) { + err = rawSkew[i] - rawAvg; + rawAbsDev += err >= 0 ? err : -err; + } + } + RTC_DCHECK_GT(n, 0); + rawAbsDev /= n; + upperLimit = static_cast(rawAvg + 5 * rawAbsDev + 1); // +1 for ceiling. + lowerLimit = static_cast(rawAvg - 5 * rawAbsDev - 1); // -1 for floor. + + n = 0; + for (i = 0; i < size; i++) { + if ((rawSkew[i] < absLimitInner && rawSkew[i] > -absLimitInner) || + (rawSkew[i] < upperLimit && rawSkew[i] > lowerLimit)) { + n++; + cumSum += rawSkew[i]; + x += n; + x2 += n * n; + y += cumSum; + xy += n * cumSum; + } + } + + if (n == 0) { + return -1; + } + RTC_DCHECK_GT(n, 0); + xAvg = x / n; + denom = x2 - xAvg * x; + + if (denom != 0) { + skew = (xy - xAvg * y) / denom; + } + + *skewEst = skew; + return 0; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.h new file mode 100644 index 0000000000..130f7ec7c7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_AEC_RESAMPLER_H_ +#define MODULES_AUDIO_PROCESSING_AEC_AEC_RESAMPLER_H_ + +#include "modules/audio_processing/aec/aec_core.h" + +namespace webrtc { + +enum { kResamplingDelay = 1 }; +enum { kResamplerBufferSize = FRAME_LEN * 4 }; + +// Unless otherwise specified, functions return 0 on success and -1 on error. +void* WebRtcAec_CreateResampler(); // Returns NULL on error. +int WebRtcAec_InitResampler(void* resampInst, int deviceSampleRateHz); +void WebRtcAec_FreeResampler(void* resampInst); + +// Estimates skew from raw measurement. +int WebRtcAec_GetSkew(void* resampInst, int rawSkew, float* skewEst); + +// Resamples input using linear interpolation. +void WebRtcAec_ResampleLinear(void* resampInst, + const float* inspeech, + size_t size, + float skew, + float* outspeech, + size_t* size_out); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_AEC_RESAMPLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.cc new file mode 100644 index 0000000000..eeaa909a17 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.cc @@ -0,0 +1,868 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * Contains the API functions for the AEC. + */ +#include "modules/audio_processing/aec/echo_cancellation.h" + +#include +#include +#include + +extern "C" { +#include "common_audio/ring_buffer.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +} +#include "modules/audio_processing/aec/aec_core.h" +#include "modules/audio_processing/aec/aec_resampler.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +Aec::Aec() = default; +Aec::~Aec() = default; + +// Measured delays [ms] +// Device Chrome GTP +// MacBook Air 10 +// MacBook Retina 10 100 +// MacPro 30? +// +// Win7 Desktop 70 80? +// Win7 T430s 110 +// Win8 T420s 70 +// +// Daisy 50 +// Pixel (w/ preproc?) 240 +// Pixel (w/o preproc?) 110 110 + +// The extended filter mode gives us the flexibility to ignore the system's +// reported delays. We do this for platforms which we believe provide results +// which are incompatible with the AEC's expectations. Based on measurements +// (some provided above) we set a conservative (i.e. lower than measured) +// fixed delay. +// +// WEBRTC_UNTRUSTED_DELAY will only have an impact when |extended_filter_mode| +// is enabled. See the note along with |DelayCorrection| in +// echo_cancellation_impl.h for more details on the mode. +// +// Justification: +// Chromium/Mac: Here, the true latency is so low (~10-20 ms), that it plays +// havoc with the AEC's buffering. To avoid this, we set a fixed delay of 20 ms +// and then compensate by rewinding by 10 ms (in wideband) through +// kDelayDiffOffsetSamples. This trick does not seem to work for larger rewind +// values, but fortunately this is sufficient. +// +// Chromium/Linux(ChromeOS): The values we get on this platform don't correspond +// well to reality. The variance doesn't match the AEC's buffer changes, and the +// bulk values tend to be too low. However, the range across different hardware +// appears to be too large to choose a single value. +// +// GTP/Linux(ChromeOS): TBD, but for the moment we will trust the values. +#if defined(WEBRTC_CHROMIUM_BUILD) && defined(WEBRTC_MAC) +#define WEBRTC_UNTRUSTED_DELAY +#endif + +#if defined(WEBRTC_UNTRUSTED_DELAY) && defined(WEBRTC_MAC) +static const int kDelayDiffOffsetSamples = -160; +#else +// Not enabled for now. +static const int kDelayDiffOffsetSamples = 0; +#endif + +#if defined(WEBRTC_MAC) +static const int kFixedDelayMs = 20; +#else +static const int kFixedDelayMs = 50; +#endif +#if !defined(WEBRTC_UNTRUSTED_DELAY) +static const int kMinTrustedDelayMs = 20; +#endif +static const int kMaxTrustedDelayMs = 500; + +// Maximum length of resampled signal. Must be an integer multiple of frames +// (ceil(1/(1 + MIN_SKEW)*2) + 1)*FRAME_LEN +// The factor of 2 handles wb, and the + 1 is as a safety margin +// TODO(bjornv): Replace with kResamplerBufferSize +#define MAX_RESAMP_LEN (5 * FRAME_LEN) + +static const int kMaxBufSizeStart = 62; // In partitions +static const int sampMsNb = 8; // samples per ms in nb +static const int initCheck = 42; + +std::atomic Aec::instance_count = 0; + +// Estimates delay to set the position of the far-end buffer read pointer +// (controlled by knownDelay) +static void EstBufDelayNormal(Aec* aecInst); +static void EstBufDelayExtended(Aec* aecInst); +static int ProcessNormal(Aec* aecInst, + const float* const* nearend, + size_t num_bands, + float* const* out, + size_t num_samples, + int16_t reported_delay_ms, + int32_t skew); +static void ProcessExtended(Aec* aecInst, + const float* const* nearend, + size_t num_bands, + float* const* out, + size_t num_samples, + int16_t reported_delay_ms, + int32_t skew); + +void* WebRtcAec_Create() { + Aec* aecpc = new Aec(); + + if (!aecpc) { + return NULL; + } + aecpc->data_dumper.reset(new ApmDataDumper(aecpc->instance_count)); + + aecpc->aec = WebRtcAec_CreateAec(aecpc->instance_count); + if (!aecpc->aec) { + WebRtcAec_Free(aecpc); + return NULL; + } + aecpc->resampler = WebRtcAec_CreateResampler(); + if (!aecpc->resampler) { + WebRtcAec_Free(aecpc); + return NULL; + } + // Create far-end pre-buffer. The buffer size has to be large enough for + // largest possible drift compensation (kResamplerBufferSize) + "almost" an + // FFT buffer (PART_LEN2 - 1). + aecpc->far_pre_buf = + WebRtc_CreateBuffer(PART_LEN2 + kResamplerBufferSize, sizeof(float)); + if (!aecpc->far_pre_buf) { + WebRtcAec_Free(aecpc); + return NULL; + } + + aecpc->initFlag = 0; + + aecpc->instance_count++; + return aecpc; +} + +void WebRtcAec_Free(void* aecInst) { + Aec* aecpc = reinterpret_cast(aecInst); + + if (aecpc == NULL) { + return; + } + + WebRtc_FreeBuffer(aecpc->far_pre_buf); + + WebRtcAec_FreeAec(aecpc->aec); + WebRtcAec_FreeResampler(aecpc->resampler); + delete aecpc; +} + +int32_t WebRtcAec_Init(void* aecInst, int32_t sampFreq, int32_t scSampFreq) { + Aec* aecpc = reinterpret_cast(aecInst); + aecpc->data_dumper->InitiateNewSetOfRecordings(); + AecConfig aecConfig; + + if (sampFreq != 8000 && sampFreq != 16000 && sampFreq != 32000 && + sampFreq != 48000) { + return AEC_BAD_PARAMETER_ERROR; + } + aecpc->sampFreq = sampFreq; + + if (scSampFreq < 1 || scSampFreq > 96000) { + return AEC_BAD_PARAMETER_ERROR; + } + aecpc->scSampFreq = scSampFreq; + + // Initialize echo canceller core + if (WebRtcAec_InitAec(aecpc->aec, aecpc->sampFreq) == -1) { + return AEC_UNSPECIFIED_ERROR; + } + + if (WebRtcAec_InitResampler(aecpc->resampler, aecpc->scSampFreq) == -1) { + return AEC_UNSPECIFIED_ERROR; + } + + WebRtc_InitBuffer(aecpc->far_pre_buf); + WebRtc_MoveReadPtr(aecpc->far_pre_buf, -PART_LEN); // Start overlap. + + aecpc->initFlag = initCheck; // indicates that initialization has been done + + if (aecpc->sampFreq == 32000 || aecpc->sampFreq == 48000) { + aecpc->splitSampFreq = 16000; + } else { + aecpc->splitSampFreq = sampFreq; + } + + aecpc->delayCtr = 0; + aecpc->sampFactor = (aecpc->scSampFreq * 1.0f) / aecpc->splitSampFreq; + // Sampling frequency multiplier (SWB is processed as 160 frame size). + aecpc->rate_factor = aecpc->splitSampFreq / 8000; + + aecpc->sum = 0; + aecpc->counter = 0; + aecpc->checkBuffSize = 1; + aecpc->firstVal = 0; + + // We skip the startup_phase completely (setting to 0) if DA-AEC is enabled, + // but not extended_filter mode. + aecpc->startup_phase = WebRtcAec_extended_filter_enabled(aecpc->aec) || + !WebRtcAec_delay_agnostic_enabled(aecpc->aec); + aecpc->bufSizeStart = 0; + aecpc->checkBufSizeCtr = 0; + aecpc->msInSndCardBuf = 0; + aecpc->filtDelay = -1; // -1 indicates an initialized state. + aecpc->timeForDelayChange = 0; + aecpc->knownDelay = 0; + aecpc->lastDelayDiff = 0; + + aecpc->skewFrCtr = 0; + aecpc->resample = kAecFalse; + aecpc->highSkewCtr = 0; + aecpc->skew = 0; + + aecpc->farend_started = 0; + + // Default settings. + aecConfig.nlpMode = kAecNlpModerate; + aecConfig.skewMode = kAecFalse; + aecConfig.metricsMode = kAecFalse; + aecConfig.delay_logging = kAecFalse; + + if (WebRtcAec_set_config(aecpc, aecConfig) == -1) { + return AEC_UNSPECIFIED_ERROR; + } + + return 0; +} + +// Returns any error that is caused when buffering the +// far-end signal. +int32_t WebRtcAec_GetBufferFarendError(void* aecInst, + const float* farend, + size_t nrOfSamples) { + Aec* aecpc = reinterpret_cast(aecInst); + + if (!farend) + return AEC_NULL_POINTER_ERROR; + + if (aecpc->initFlag != initCheck) + return AEC_UNINITIALIZED_ERROR; + + // number of samples == 160 for SWB input + if (nrOfSamples != 80 && nrOfSamples != 160) + return AEC_BAD_PARAMETER_ERROR; + + return 0; +} + +// only buffer L band for farend +int32_t WebRtcAec_BufferFarend(void* aecInst, + const float* farend, + size_t nrOfSamples) { + Aec* aecpc = reinterpret_cast(aecInst); + size_t newNrOfSamples = nrOfSamples; + float new_farend[MAX_RESAMP_LEN]; + const float* farend_ptr = farend; + + // Get any error caused by buffering the farend signal. + int32_t error_code = + WebRtcAec_GetBufferFarendError(aecInst, farend, nrOfSamples); + + if (error_code != 0) + return error_code; + + if (aecpc->skewMode == kAecTrue && aecpc->resample == kAecTrue) { + // Resample and get a new number of samples + WebRtcAec_ResampleLinear(aecpc->resampler, farend, nrOfSamples, aecpc->skew, + new_farend, &newNrOfSamples); + farend_ptr = new_farend; + } + + aecpc->farend_started = 1; + WebRtcAec_SetSystemDelay(aecpc->aec, WebRtcAec_system_delay(aecpc->aec) + + static_cast(newNrOfSamples)); + + // Write the time-domain data to |far_pre_buf|. + WebRtc_WriteBuffer(aecpc->far_pre_buf, farend_ptr, newNrOfSamples); + + // TODO(minyue): reduce to |PART_LEN| samples for each buffering. + while (WebRtc_available_read(aecpc->far_pre_buf) >= PART_LEN2) { + // We have enough data to pass to the FFT, hence read PART_LEN2 samples. + { + float* ptmp = NULL; + float tmp[PART_LEN2]; + WebRtc_ReadBuffer(aecpc->far_pre_buf, + reinterpret_cast(&ptmp), tmp, PART_LEN2); + WebRtcAec_BufferFarendBlock(aecpc->aec, &ptmp[PART_LEN]); + } + + // Rewind |far_pre_buf| PART_LEN samples for overlap before continuing. + WebRtc_MoveReadPtr(aecpc->far_pre_buf, -PART_LEN); + } + + return 0; +} + +int32_t WebRtcAec_Process(void* aecInst, + const float* const* nearend, + size_t num_bands, + float* const* out, + size_t nrOfSamples, + int16_t msInSndCardBuf, + int32_t skew) { + Aec* aecpc = reinterpret_cast(aecInst); + int32_t retVal = 0; + + if (out == NULL) { + return AEC_NULL_POINTER_ERROR; + } + + if (aecpc->initFlag != initCheck) { + return AEC_UNINITIALIZED_ERROR; + } + + // number of samples == 160 for SWB input + if (nrOfSamples != 80 && nrOfSamples != 160) { + return AEC_BAD_PARAMETER_ERROR; + } + + if (msInSndCardBuf < 0) { + msInSndCardBuf = 0; + retVal = AEC_BAD_PARAMETER_WARNING; + } else if (msInSndCardBuf > kMaxTrustedDelayMs) { + // The clamping is now done in ProcessExtended/Normal(). + retVal = AEC_BAD_PARAMETER_WARNING; + } + + // This returns the value of aec->extended_filter_enabled. + if (WebRtcAec_extended_filter_enabled(aecpc->aec)) { + ProcessExtended(aecpc, nearend, num_bands, out, nrOfSamples, msInSndCardBuf, + skew); + } else { + retVal = ProcessNormal(aecpc, nearend, num_bands, out, nrOfSamples, + msInSndCardBuf, skew); + } + + int far_buf_size_samples = WebRtcAec_system_delay(aecpc->aec); + aecpc->data_dumper->DumpRaw("aec_system_delay", 1, &far_buf_size_samples); + aecpc->data_dumper->DumpRaw("aec_known_delay", 1, &aecpc->knownDelay); + + return retVal; +} + +int WebRtcAec_set_config(void* handle, AecConfig config) { + Aec* self = reinterpret_cast(handle); + if (self->initFlag != initCheck) { + return AEC_UNINITIALIZED_ERROR; + } + + if (config.skewMode != kAecFalse && config.skewMode != kAecTrue) { + return AEC_BAD_PARAMETER_ERROR; + } + self->skewMode = config.skewMode; + + if (config.nlpMode != kAecNlpConservative && + config.nlpMode != kAecNlpModerate && + config.nlpMode != kAecNlpAggressive) { + return AEC_BAD_PARAMETER_ERROR; + } + + if (config.metricsMode != kAecFalse && config.metricsMode != kAecTrue) { + return AEC_BAD_PARAMETER_ERROR; + } + + if (config.delay_logging != kAecFalse && config.delay_logging != kAecTrue) { + return AEC_BAD_PARAMETER_ERROR; + } + + WebRtcAec_SetConfigCore(self->aec, config.nlpMode, config.metricsMode, + config.delay_logging); + return 0; +} + +int WebRtcAec_get_echo_status(void* handle, int* status) { + Aec* self = reinterpret_cast(handle); + if (status == NULL) { + return AEC_NULL_POINTER_ERROR; + } + if (self->initFlag != initCheck) { + return AEC_UNINITIALIZED_ERROR; + } + + *status = WebRtcAec_echo_state(self->aec); + + return 0; +} + +int WebRtcAec_GetMetrics(void* handle, AecMetrics* metrics) { + const float kUpWeight = 0.7f; + float dtmp; + int stmp; + Aec* self = reinterpret_cast(handle); + Stats erl; + Stats erle; + Stats a_nlp; + + if (handle == NULL) { + return -1; + } + if (metrics == NULL) { + return AEC_NULL_POINTER_ERROR; + } + if (self->initFlag != initCheck) { + return AEC_UNINITIALIZED_ERROR; + } + + WebRtcAec_GetEchoStats(self->aec, &erl, &erle, &a_nlp, + &metrics->divergent_filter_fraction); + + // ERL + metrics->erl.instant = static_cast(erl.instant); + + if ((erl.himean > kOffsetLevel) && (erl.average > kOffsetLevel)) { + // Use a mix between regular average and upper part average. + dtmp = kUpWeight * erl.himean + (1 - kUpWeight) * erl.average; + metrics->erl.average = static_cast(dtmp); + } else { + metrics->erl.average = kOffsetLevel; + } + + metrics->erl.max = static_cast(erl.max); + + if (erl.min < (kOffsetLevel * (-1))) { + metrics->erl.min = static_cast(erl.min); + } else { + metrics->erl.min = kOffsetLevel; + } + + // ERLE + metrics->erle.instant = static_cast(erle.instant); + + if ((erle.himean > kOffsetLevel) && (erle.average > kOffsetLevel)) { + // Use a mix between regular average and upper part average. + dtmp = kUpWeight * erle.himean + (1 - kUpWeight) * erle.average; + metrics->erle.average = static_cast(dtmp); + } else { + metrics->erle.average = kOffsetLevel; + } + + metrics->erle.max = static_cast(erle.max); + + if (erle.min < (kOffsetLevel * (-1))) { + metrics->erle.min = static_cast(erle.min); + } else { + metrics->erle.min = kOffsetLevel; + } + + // RERL + if ((metrics->erl.average > kOffsetLevel) && + (metrics->erle.average > kOffsetLevel)) { + stmp = metrics->erl.average + metrics->erle.average; + } else { + stmp = kOffsetLevel; + } + metrics->rerl.average = stmp; + + // No other statistics needed, but returned for completeness. + metrics->rerl.instant = stmp; + metrics->rerl.max = stmp; + metrics->rerl.min = stmp; + + // A_NLP + metrics->aNlp.instant = static_cast(a_nlp.instant); + + if ((a_nlp.himean > kOffsetLevel) && (a_nlp.average > kOffsetLevel)) { + // Use a mix between regular average and upper part average. + dtmp = kUpWeight * a_nlp.himean + (1 - kUpWeight) * a_nlp.average; + metrics->aNlp.average = static_cast(dtmp); + } else { + metrics->aNlp.average = kOffsetLevel; + } + + metrics->aNlp.max = static_cast(a_nlp.max); + + if (a_nlp.min < (kOffsetLevel * (-1))) { + metrics->aNlp.min = static_cast(a_nlp.min); + } else { + metrics->aNlp.min = kOffsetLevel; + } + + return 0; +} + +int WebRtcAec_GetDelayMetrics(void* handle, + int* median, + int* std, + float* fraction_poor_delays) { + Aec* self = reinterpret_cast(handle); + if (median == NULL) { + return AEC_NULL_POINTER_ERROR; + } + if (std == NULL) { + return AEC_NULL_POINTER_ERROR; + } + if (self->initFlag != initCheck) { + return AEC_UNINITIALIZED_ERROR; + } + if (WebRtcAec_GetDelayMetricsCore(self->aec, median, std, + fraction_poor_delays) == -1) { + // Logging disabled. + return AEC_UNSUPPORTED_FUNCTION_ERROR; + } + + return 0; +} + +AecCore* WebRtcAec_aec_core(void* handle) { + if (!handle) { + return NULL; + } + return reinterpret_cast(handle)->aec; +} + +static int ProcessNormal(Aec* aecInst, + const float* const* nearend, + size_t num_bands, + float* const* out, + size_t num_samples, + int16_t reported_delay_ms, + int32_t skew) { + int retVal = 0; + size_t i; + size_t nBlocks10ms; + // Limit resampling to doubling/halving of signal + const float minSkewEst = -0.5f; + const float maxSkewEst = 1.0f; + + reported_delay_ms = + reported_delay_ms > kMaxTrustedDelayMs ? kMaxTrustedDelayMs : + reported_delay_ms; + // TODO(andrew): we need to investigate if this +10 is really wanted. + reported_delay_ms += 10; + aecInst->msInSndCardBuf = reported_delay_ms; + + if (aecInst->skewMode == kAecTrue) { + if (aecInst->skewFrCtr < 25) { + aecInst->skewFrCtr++; + } else { + retVal = WebRtcAec_GetSkew(aecInst->resampler, skew, &aecInst->skew); + if (retVal == -1) { + aecInst->skew = 0; + retVal = AEC_BAD_PARAMETER_WARNING; + } + + aecInst->skew /= aecInst->sampFactor * num_samples; + + if (aecInst->skew < 1.0e-3 && aecInst->skew > -1.0e-3) { + aecInst->resample = kAecFalse; + } else { + aecInst->resample = kAecTrue; + } + + if (aecInst->skew < minSkewEst) { + aecInst->skew = minSkewEst; + } else if (aecInst->skew > maxSkewEst) { + aecInst->skew = maxSkewEst; + } + + aecInst->data_dumper->DumpRaw("aec_skew", 1, &aecInst->skew); + } + } + + nBlocks10ms = num_samples / (FRAME_LEN * aecInst->rate_factor); + + if (aecInst->startup_phase) { + for (i = 0; i < num_bands; ++i) { + // Only needed if they don't already point to the same place. + if (nearend[i] != out[i]) { + memcpy(out[i], nearend[i], sizeof(nearend[i][0]) * num_samples); + } + } + + // The AEC is in the start up mode + // AEC is disabled until the system delay is OK + + // Mechanism to ensure that the system delay is reasonably stable. + if (aecInst->checkBuffSize) { + aecInst->checkBufSizeCtr++; + // Before we fill up the far-end buffer we require the system delay + // to be stable (+/-8 ms) compared to the first value. This + // comparison is made during the following 6 consecutive 10 ms + // blocks. If it seems to be stable then we start to fill up the + // far-end buffer. + if (aecInst->counter == 0) { + aecInst->firstVal = aecInst->msInSndCardBuf; + aecInst->sum = 0; + } + + if (abs(aecInst->firstVal - aecInst->msInSndCardBuf) < + WEBRTC_SPL_MAX(0.2 * aecInst->msInSndCardBuf, sampMsNb)) { + aecInst->sum += aecInst->msInSndCardBuf; + aecInst->counter++; + } else { + aecInst->counter = 0; + } + + if (aecInst->counter * nBlocks10ms >= 6) { + // The far-end buffer size is determined in partitions of + // PART_LEN samples. Use 75% of the average value of the system + // delay as buffer size to start with. + aecInst->bufSizeStart = + WEBRTC_SPL_MIN((3 * aecInst->sum * aecInst->rate_factor * 8) / + (4 * aecInst->counter * PART_LEN), + kMaxBufSizeStart); + // Buffer size has now been determined. + aecInst->checkBuffSize = 0; + } + + if (aecInst->checkBufSizeCtr * nBlocks10ms > 50) { + // For really bad systems, don't disable the echo canceller for + // more than 0.5 sec. + aecInst->bufSizeStart = WEBRTC_SPL_MIN( + (aecInst->msInSndCardBuf * aecInst->rate_factor * 3) / 40, + kMaxBufSizeStart); + aecInst->checkBuffSize = 0; + } + } + + // If |checkBuffSize| changed in the if-statement above. + if (!aecInst->checkBuffSize) { + // The system delay is now reasonably stable (or has been unstable + // for too long). When the far-end buffer is filled with + // approximately the same amount of data as reported by the system + // we end the startup phase. + int overhead_elements = + WebRtcAec_system_delay(aecInst->aec) / PART_LEN - + aecInst->bufSizeStart; + if (overhead_elements == 0) { + // Enable the AEC + aecInst->startup_phase = 0; + } else if (overhead_elements > 0) { + // TODO(bjornv): Do we need a check on how much we actually + // moved the read pointer? It should always be possible to move + // the pointer |overhead_elements| since we have only added data + // to the buffer and no delay compensation nor AEC processing + // has been done. + WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(aecInst->aec, + overhead_elements); + + // Enable the AEC + aecInst->startup_phase = 0; + } + } + } else { + // AEC is enabled. + EstBufDelayNormal(aecInst); + + // Call the AEC. + // TODO(bjornv): Re-structure such that we don't have to pass + // |aecInst->knownDelay| as input. Change name to something like + // |system_buffer_diff|. + WebRtcAec_ProcessFrames(aecInst->aec, nearend, num_bands, num_samples, + aecInst->knownDelay, out); + } + + return retVal; +} + +static void ProcessExtended(Aec* self, + const float* const* near, + size_t num_bands, + float* const* out, + size_t num_samples, + int16_t reported_delay_ms, + int32_t skew) { + size_t i; + const int delay_diff_offset = kDelayDiffOffsetSamples; + RTC_DCHECK(num_samples == 80 || num_samples == 160); +#if defined(WEBRTC_UNTRUSTED_DELAY) + reported_delay_ms = kFixedDelayMs; +#else + // This is the usual mode where we trust the reported system delay values. + // Due to the longer filter, we no longer add 10 ms to the reported delay + // to reduce chance of non-causality. Instead we apply a minimum here to avoid + // issues with the read pointer jumping around needlessly. + reported_delay_ms = reported_delay_ms < kMinTrustedDelayMs + ? kMinTrustedDelayMs + : reported_delay_ms; + // If the reported delay appears to be bogus, we attempt to recover by using + // the measured fixed delay values. We use >= here because higher layers + // may already clamp to this maximum value, and we would otherwise not + // detect it here. + reported_delay_ms = reported_delay_ms >= kMaxTrustedDelayMs + ? kFixedDelayMs + : reported_delay_ms; +#endif + self->msInSndCardBuf = reported_delay_ms; + + if (!self->farend_started) { + for (i = 0; i < num_bands; ++i) { + // Only needed if they don't already point to the same place. + if (near[i] != out[i]) { + memcpy(out[i], near[i], sizeof(near[i][0]) * num_samples); + } + } + return; + } + if (self->startup_phase) { + // In the extended mode, there isn't a startup "phase", just a special + // action on the first frame. In the trusted delay case, we'll take the + // current reported delay, unless it's less then our conservative + // measurement. + int startup_size_ms = + reported_delay_ms < kFixedDelayMs ? kFixedDelayMs : reported_delay_ms; +#if defined(WEBRTC_ANDROID) + int target_delay = startup_size_ms * self->rate_factor * 8; +#else + // To avoid putting the AEC in a non-causal state we're being slightly + // conservative and scale by 2. On Android we use a fixed delay and + // therefore there is no need to scale the target_delay. + int target_delay = startup_size_ms * self->rate_factor * 8 / 2; +#endif + int overhead_elements = + (WebRtcAec_system_delay(self->aec) - target_delay) / PART_LEN; + WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(self->aec, + overhead_elements); + self->startup_phase = 0; + } + + EstBufDelayExtended(self); + + { + // |delay_diff_offset| gives us the option to manually rewind the delay on + // very low delay platforms which can't be expressed purely through + // |reported_delay_ms|. + const int adjusted_known_delay = + WEBRTC_SPL_MAX(0, self->knownDelay + delay_diff_offset); + + WebRtcAec_ProcessFrames(self->aec, near, num_bands, num_samples, + adjusted_known_delay, out); + } +} + +static void EstBufDelayNormal(Aec* aecInst) { + int nSampSndCard = aecInst->msInSndCardBuf * sampMsNb * aecInst->rate_factor; + int current_delay = nSampSndCard - WebRtcAec_system_delay(aecInst->aec); + int delay_difference = 0; + + // Before we proceed with the delay estimate filtering we: + // 1) Compensate for the frame that will be read. + // 2) Compensate for drift resampling. + // 3) Compensate for non-causality if needed, since the estimated delay can't + // be negative. + + // 1) Compensating for the frame(s) that will be read/processed. + current_delay += FRAME_LEN * aecInst->rate_factor; + + // 2) Account for resampling frame delay. + if (aecInst->skewMode == kAecTrue && aecInst->resample == kAecTrue) { + current_delay -= kResamplingDelay; + } + + // 3) Compensate for non-causality, if needed, by flushing one block. + if (current_delay < PART_LEN) { + current_delay += + WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(aecInst->aec, 1) * + PART_LEN; + } + + // We use -1 to signal an initialized state in the "extended" implementation; + // compensate for that. + aecInst->filtDelay = aecInst->filtDelay < 0 ? 0 : aecInst->filtDelay; + aecInst->filtDelay = + WEBRTC_SPL_MAX(0, static_cast(0.8 * + aecInst->filtDelay + + 0.2 * current_delay)); + + delay_difference = aecInst->filtDelay - aecInst->knownDelay; + if (delay_difference > 224) { + if (aecInst->lastDelayDiff < 96) { + aecInst->timeForDelayChange = 0; + } else { + aecInst->timeForDelayChange++; + } + } else if (delay_difference < 96 && aecInst->knownDelay > 0) { + if (aecInst->lastDelayDiff > 224) { + aecInst->timeForDelayChange = 0; + } else { + aecInst->timeForDelayChange++; + } + } else { + aecInst->timeForDelayChange = 0; + } + aecInst->lastDelayDiff = delay_difference; + + if (aecInst->timeForDelayChange > 25) { + aecInst->knownDelay = WEBRTC_SPL_MAX((int)aecInst->filtDelay - 160, 0); + } +} + +static void EstBufDelayExtended(Aec* aecInst) { + int reported_delay = aecInst->msInSndCardBuf * sampMsNb * + aecInst->rate_factor; + int current_delay = reported_delay - WebRtcAec_system_delay(aecInst->aec); + int delay_difference = 0; + + // Before we proceed with the delay estimate filtering we: + // 1) Compensate for the frame that will be read. + // 2) Compensate for drift resampling. + // 3) Compensate for non-causality if needed, since the estimated delay can't + // be negative. + + // 1) Compensating for the frame(s) that will be read/processed. + current_delay += FRAME_LEN * aecInst->rate_factor; + + // 2) Account for resampling frame delay. + if (aecInst->skewMode == kAecTrue && aecInst->resample == kAecTrue) { + current_delay -= kResamplingDelay; + } + + // 3) Compensate for non-causality, if needed, by flushing two blocks. + if (current_delay < PART_LEN) { + current_delay += + WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(aecInst->aec, 2) * + PART_LEN; + } + + if (aecInst->filtDelay == -1) { + aecInst->filtDelay = WEBRTC_SPL_MAX(0, 0.5 * current_delay); + } else { + aecInst->filtDelay = WEBRTC_SPL_MAX( + 0, static_cast(0.95 * aecInst->filtDelay + 0.05 * + current_delay)); + } + + delay_difference = aecInst->filtDelay - aecInst->knownDelay; + if (delay_difference > 384) { + if (aecInst->lastDelayDiff < 128) { + aecInst->timeForDelayChange = 0; + } else { + aecInst->timeForDelayChange++; + } + } else if (delay_difference < 128 && aecInst->knownDelay > 0) { + if (aecInst->lastDelayDiff > 384) { + aecInst->timeForDelayChange = 0; + } else { + aecInst->timeForDelayChange++; + } + } else { + aecInst->timeForDelayChange = 0; + } + aecInst->lastDelayDiff = delay_difference; + + if (aecInst->timeForDelayChange > 25) { + aecInst->knownDelay = WEBRTC_SPL_MAX((int)aecInst->filtDelay - 256, 0); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.h new file mode 100644 index 0000000000..96132ee721 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.h @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_ECHO_CANCELLATION_H_ +#define MODULES_AUDIO_PROCESSING_AEC_ECHO_CANCELLATION_H_ + +#include +#include + +#include + +extern "C" { +#include "common_audio/ring_buffer.h" +} +#include "modules/audio_processing/aec/aec_core.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Errors +#define AEC_UNSPECIFIED_ERROR 12000 +#define AEC_UNSUPPORTED_FUNCTION_ERROR 12001 +#define AEC_UNINITIALIZED_ERROR 12002 +#define AEC_NULL_POINTER_ERROR 12003 +#define AEC_BAD_PARAMETER_ERROR 12004 + +// Warnings +#define AEC_BAD_PARAMETER_WARNING 12050 + +enum { kAecNlpConservative = 0, kAecNlpModerate, kAecNlpAggressive }; + +enum { kAecFalse = 0, kAecTrue }; + +typedef struct { + int16_t nlpMode; // default kAecNlpModerate + int16_t skewMode; // default kAecFalse + int16_t metricsMode; // default kAecFalse + int delay_logging; // default kAecFalse + // float realSkew; +} AecConfig; + +typedef struct { + int instant; + int average; + int max; + int min; +} AecLevel; + +typedef struct { + AecLevel rerl; + AecLevel erl; + AecLevel erle; + AecLevel aNlp; + float divergent_filter_fraction; +} AecMetrics; + +struct AecCore; + +class ApmDataDumper; + +typedef struct Aec { + Aec(); + ~Aec(); + + std::unique_ptr data_dumper; + + int delayCtr; + int sampFreq; + int splitSampFreq; + int scSampFreq; + float sampFactor; // scSampRate / sampFreq + short skewMode; + int bufSizeStart; + int knownDelay; + int rate_factor; + + short initFlag; // indicates if AEC has been initialized + + // Variables used for averaging far end buffer size + short counter; + int sum; + short firstVal; + short checkBufSizeCtr; + + // Variables used for delay shifts + short msInSndCardBuf; + short filtDelay; // Filtered delay estimate. + int timeForDelayChange; + int startup_phase; + int checkBuffSize; + short lastDelayDiff; + + // Structures + void* resampler; + + int skewFrCtr; + int resample; // if the skew is small enough we don't resample + int highSkewCtr; + float skew; + + RingBuffer* far_pre_buf; // Time domain far-end pre-buffer. + + int farend_started; + + // Aec instance counter. + static std::atomic instance_count; + AecCore* aec; +} Aec; + +/* + * Allocates the memory needed by the AEC. The memory needs to be initialized + * separately using the WebRtcAec_Init() function. Returns a pointer to the + * object or NULL on error. + */ +void* WebRtcAec_Create(); + +/* + * This function releases the memory allocated by WebRtcAec_Create(). + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecInst Pointer to the AEC instance + */ +void WebRtcAec_Free(void* aecInst); + +/* + * Initializes an AEC instance. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecInst Pointer to the AEC instance + * int32_t sampFreq Sampling frequency of data + * int32_t scSampFreq Soundcard sampling frequency + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * -1: error + */ +int32_t WebRtcAec_Init(void* aecInst, int32_t sampFreq, int32_t scSampFreq); + +/* + * Inserts an 80 or 160 sample block of data into the farend buffer. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecInst Pointer to the AEC instance + * const float* farend In buffer containing one frame of + * farend signal for L band + * int16_t nrOfSamples Number of samples in farend buffer + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 12000-12050: error code + */ +int32_t WebRtcAec_BufferFarend(void* aecInst, + const float* farend, + size_t nrOfSamples); + +/* + * Reports any errors that would arise if buffering a farend buffer + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecInst Pointer to the AEC instance + * const float* farend In buffer containing one frame of + * farend signal for L band + * int16_t nrOfSamples Number of samples in farend buffer + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 12000-12050: error code + */ +int32_t WebRtcAec_GetBufferFarendError(void* aecInst, + const float* farend, + size_t nrOfSamples); + +/* + * Runs the echo canceller on an 80 or 160 sample blocks of data. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecInst Pointer to the AEC instance + * float* const* nearend In buffer containing one frame of + * nearend+echo signal for each band + * int num_bands Number of bands in nearend buffer + * int16_t nrOfSamples Number of samples in nearend buffer + * int16_t msInSndCardBuf Delay estimate for sound card and + * system buffers + * int16_t skew Difference between number of samples played + * and recorded at the soundcard (for clock skew + * compensation) + * + * Outputs Description + * ------------------------------------------------------------------- + * float* const* out Out buffer, one frame of processed nearend + * for each band + * int32_t return 0: OK + * 12000-12050: error code + */ +int32_t WebRtcAec_Process(void* aecInst, + const float* const* nearend, + size_t num_bands, + float* const* out, + size_t nrOfSamples, + int16_t msInSndCardBuf, + int32_t skew); + +/* + * This function enables the user to set certain parameters on-the-fly. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* handle Pointer to the AEC instance + * AecConfig config Config instance that contains all + * properties to be set + * + * Outputs Description + * ------------------------------------------------------------------- + * int return 0: OK + * 12000-12050: error code + */ +int WebRtcAec_set_config(void* handle, AecConfig config); + +/* + * Gets the current echo status of the nearend signal. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* handle Pointer to the AEC instance + * + * Outputs Description + * ------------------------------------------------------------------- + * int* status 0: Almost certainly nearend single-talk + * 1: Might not be neared single-talk + * int return 0: OK + * 12000-12050: error code + */ +int WebRtcAec_get_echo_status(void* handle, int* status); + +/* + * Gets the current echo metrics for the session. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* handle Pointer to the AEC instance + * + * Outputs Description + * ------------------------------------------------------------------- + * AecMetrics* metrics Struct which will be filled out with the + * current echo metrics. + * int return 0: OK + * 12000-12050: error code + */ +int WebRtcAec_GetMetrics(void* handle, AecMetrics* metrics); + +/* + * Gets the current delay metrics for the session. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* handle Pointer to the AEC instance + * + * Outputs Description + * ------------------------------------------------------------------- + * int* median Delay median value. + * int* std Delay standard deviation. + * float* fraction_poor_delays Fraction of the delay estimates that may + * cause the AEC to perform poorly. + * + * int return 0: OK + * 12000-12050: error code + */ +int WebRtcAec_GetDelayMetrics(void* handle, + int* median, + int* std, + float* fraction_poor_delays); + +// Returns a pointer to the low level AEC handle. +// +// Input: +// - handle : Pointer to the AEC instance. +// +// Return value: +// - AecCore pointer : NULL for error. +// +struct AecCore* WebRtcAec_aec_core(void* handle); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_ECHO_CANCELLATION_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation_unittest.cc new file mode 100644 index 0000000000..b9c89fd9e9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation_unittest.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// TODO(bjornv): Make this a comprehensive test. + +#include "modules/audio_processing/aec/echo_cancellation.h" + +#include +#include + +#include "modules/audio_processing/aec/aec_core.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(EchoCancellationTest, CreateAndFreeHasExpectedBehavior) { + void* handle = WebRtcAec_Create(); + ASSERT_TRUE(handle); + WebRtcAec_Free(nullptr); + WebRtcAec_Free(handle); +} + +TEST(EchoCancellationTest, ApplyAecCoreHandle) { + void* handle = WebRtcAec_Create(); + ASSERT_TRUE(handle); + EXPECT_TRUE(WebRtcAec_aec_core(NULL) == NULL); + AecCore* aec_core = WebRtcAec_aec_core(handle); + EXPECT_TRUE(aec_core != NULL); + // A simple test to verify that we can set and get a value from the lower + // level |aec_core| handle. + int delay = 111; + WebRtcAec_SetSystemDelay(aec_core, delay); + EXPECT_EQ(delay, WebRtcAec_system_delay(aec_core)); + WebRtcAec_Free(handle); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec/system_delay_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/system_delay_unittest.cc new file mode 100644 index 0000000000..fc57af8b87 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec/system_delay_unittest.cc @@ -0,0 +1,601 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec/aec_core.h" +#include "modules/audio_processing/aec/echo_cancellation.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) +namespace webrtc { +namespace { + +class SystemDelayTest : public ::testing::Test { + protected: + SystemDelayTest(); + virtual void SetUp(); + virtual void TearDown(); + + // Initialization of AEC handle with respect to |sample_rate_hz|. Since the + // device sample rate is unimportant we set that value to 48000 Hz. + void Init(int sample_rate_hz); + + // Makes one render call and one capture call in that specific order. + void RenderAndCapture(int device_buffer_ms); + + // Fills up the far-end buffer with respect to the default device buffer size. + size_t BufferFillUp(); + + // Runs and verifies the behavior in a stable startup procedure. + void RunStableStartup(); + + // Maps buffer size in ms into samples, taking the unprocessed frame into + // account. + int MapBufferSizeToSamples(int size_in_ms, bool extended_filter); + + void* handle_; + Aec* self_; + size_t samples_per_frame_; + // Dummy input/output speech data. + static const int kSamplesPerChunk = 160; + float far_[kSamplesPerChunk]; + float near_[kSamplesPerChunk]; + float out_[kSamplesPerChunk]; + const float* near_ptr_; + float* out_ptr_; +}; + +SystemDelayTest::SystemDelayTest() + : handle_(NULL), self_(NULL), samples_per_frame_(0) { + // Dummy input data are set with more or less arbitrary non-zero values. + for (int i = 0; i < kSamplesPerChunk; i++) { + far_[i] = 257.0; + near_[i] = 514.0; + } + memset(out_, 0, sizeof(out_)); + near_ptr_ = near_; + out_ptr_ = out_; +} + +void SystemDelayTest::SetUp() { + handle_ = WebRtcAec_Create(); + ASSERT_TRUE(handle_); + self_ = reinterpret_cast(handle_); +} + +void SystemDelayTest::TearDown() { + // Free AEC + WebRtcAec_Free(handle_); + handle_ = NULL; +} + +// In SWB mode nothing is added to the buffer handling with respect to +// functionality compared to WB. We therefore only verify behavior in NB and WB. +static const int kSampleRateHz[] = {8000, 16000}; +static const size_t kNumSampleRates = + sizeof(kSampleRateHz) / sizeof(*kSampleRateHz); + +// Default audio device buffer size used. +static const int kDeviceBufMs = 100; + +// Requirement for a stable device convergence time in ms. Should converge in +// less than |kStableConvergenceMs|. +static const int kStableConvergenceMs = 100; + +// Maximum convergence time in ms. This means that we should leave the startup +// phase after |kMaxConvergenceMs| independent of device buffer stability +// conditions. +static const int kMaxConvergenceMs = 500; + +void SystemDelayTest::Init(int sample_rate_hz) { + // Initialize AEC + EXPECT_EQ(0, WebRtcAec_Init(handle_, sample_rate_hz, 48000)); + EXPECT_EQ(0, WebRtcAec_system_delay(self_->aec)); + + // One frame equals 10 ms of data. + samples_per_frame_ = static_cast(sample_rate_hz / 100); +} + +void SystemDelayTest::RenderAndCapture(int device_buffer_ms) { + EXPECT_EQ(0, WebRtcAec_BufferFarend(handle_, far_, samples_per_frame_)); + EXPECT_EQ(0, + WebRtcAec_Process(handle_, + &near_ptr_, + 1, + &out_ptr_, + samples_per_frame_, + device_buffer_ms, + 0)); +} + +size_t SystemDelayTest::BufferFillUp() { + // To make sure we have a full buffer when we verify stability we first fill + // up the far-end buffer with the same amount as we will report in through + // Process(). + size_t buffer_size = 0; + for (int i = 0; i < kDeviceBufMs / 10; i++) { + EXPECT_EQ(0, WebRtcAec_BufferFarend(handle_, far_, samples_per_frame_)); + buffer_size += samples_per_frame_; + EXPECT_EQ(static_cast(buffer_size), + WebRtcAec_system_delay(self_->aec)); + } + return buffer_size; +} + +void SystemDelayTest::RunStableStartup() { + // To make sure we have a full buffer when we verify stability we first fill + // up the far-end buffer with the same amount as we will report in through + // Process(). + size_t buffer_size = BufferFillUp(); + + if (WebRtcAec_delay_agnostic_enabled(self_->aec) == 1) { + // In extended_filter mode we set the buffer size after the first processed + // 10 ms chunk. Hence, we don't need to wait for the reported system delay + // values to become stable. + RenderAndCapture(kDeviceBufMs); + buffer_size += samples_per_frame_; + EXPECT_EQ(0, self_->startup_phase); + } else { + // A stable device should be accepted and put in a regular process mode + // within |kStableConvergenceMs|. + int process_time_ms = 0; + for (; process_time_ms < kStableConvergenceMs; process_time_ms += 10) { + RenderAndCapture(kDeviceBufMs); + buffer_size += samples_per_frame_; + if (self_->startup_phase == 0) { + // We have left the startup phase. + break; + } + } + // Verify convergence time. + EXPECT_GT(kStableConvergenceMs, process_time_ms); + } + // Verify that the buffer has been flushed. + EXPECT_GE(static_cast(buffer_size), + WebRtcAec_system_delay(self_->aec)); +} + + int SystemDelayTest::MapBufferSizeToSamples(int size_in_ms, + bool extended_filter) { + // If extended_filter is disabled we add an extra 10 ms for the unprocessed + // frame. That is simply how the algorithm is constructed. + return static_cast( + (size_in_ms + (extended_filter ? 0 : 10)) * samples_per_frame_ / 10); +} + +// The tests should meet basic requirements and not be adjusted to what is +// actually implemented. If we don't get good code coverage this way we either +// lack in tests or have unnecessary code. +// General requirements: +// 1) If we add far-end data the system delay should be increased with the same +// amount we add. +// 2) If the far-end buffer is full we should flush the oldest data to make room +// for the new. In this case the system delay is unaffected. +// 3) There should exist a startup phase in which the buffer size is to be +// determined. In this phase no cancellation should be performed. +// 4) Under stable conditions (small variations in device buffer sizes) the AEC +// should determine an appropriate local buffer size within +// |kStableConvergenceMs| ms. +// 5) Under unstable conditions the AEC should make a decision within +// |kMaxConvergenceMs| ms. +// 6) If the local buffer runs out of data we should stuff the buffer with older +// frames. +// 7) The system delay should within |kMaxConvergenceMs| ms heal from +// disturbances like drift, data glitches, toggling events and outliers. +// 8) The system delay should never become negative. + +TEST_F(SystemDelayTest, CorrectIncreaseWhenBufferFarend) { + // When we add data to the AEC buffer the internal system delay should be + // incremented with the same amount as the size of data. + // This process should be independent of DA-AEC and extended_filter mode. + for (int extended_filter = 0; extended_filter <= 1; ++extended_filter) { + WebRtcAec_enable_extended_filter(self_->aec, extended_filter); + EXPECT_EQ(extended_filter, WebRtcAec_extended_filter_enabled(self_->aec)); + for (int da_aec = 0; da_aec <= 1; ++da_aec) { + WebRtcAec_enable_delay_agnostic(self_->aec, da_aec); + EXPECT_EQ(da_aec, WebRtcAec_delay_agnostic_enabled(self_->aec)); + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + // Loop through a couple of calls to make sure the system delay + // increments correctly. + for (int j = 1; j <= 5; j++) { + EXPECT_EQ(0, + WebRtcAec_BufferFarend(handle_, far_, samples_per_frame_)); + EXPECT_EQ(static_cast(j * samples_per_frame_), + WebRtcAec_system_delay(self_->aec)); + } + } + } + } +} + +// TODO(bjornv): Add a test to verify behavior if the far-end buffer is full +// when adding new data. + +TEST_F(SystemDelayTest, CorrectDelayAfterStableStartup) { + // We run the system in a stable startup. After that we verify that the system + // delay meets the requirements. + // This process should be independent of DA-AEC and extended_filter mode. + for (int extended_filter = 0; extended_filter <= 1; ++extended_filter) { + WebRtcAec_enable_extended_filter(self_->aec, extended_filter); + EXPECT_EQ(extended_filter, WebRtcAec_extended_filter_enabled(self_->aec)); + for (int da_aec = 0; da_aec <= 1; ++da_aec) { + WebRtcAec_enable_delay_agnostic(self_->aec, da_aec); + EXPECT_EQ(da_aec, WebRtcAec_delay_agnostic_enabled(self_->aec)); + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + RunStableStartup(); + + // Verify system delay with respect to requirements, i.e., the + // |system_delay| is in the interval [75%, 100%] of what's reported on + // the average. + // In extended_filter mode we target 50% and measure after one processed + // 10 ms chunk. + int average_reported_delay = + static_cast(kDeviceBufMs * samples_per_frame_ / 10); + EXPECT_GE(average_reported_delay, WebRtcAec_system_delay(self_->aec)); + int lower_bound = WebRtcAec_extended_filter_enabled(self_->aec) + ? (average_reported_delay / 2 - + rtc::checked_cast(samples_per_frame_)) + : average_reported_delay * 3 / 4; + EXPECT_LE(lower_bound, WebRtcAec_system_delay(self_->aec)); + } + } + } +} + +TEST_F(SystemDelayTest, CorrectDelayAfterUnstableStartup) { + // This test does not apply in extended_filter mode, since we only use the + // the first 10 ms chunk to determine a reasonable buffer size. Neither does + // it apply if DA-AEC is on because that overrides the startup procedure. + WebRtcAec_enable_extended_filter(self_->aec, 0); + EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(self_->aec)); + WebRtcAec_enable_delay_agnostic(self_->aec, 0); + EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(self_->aec)); + + // In an unstable system we would start processing after |kMaxConvergenceMs|. + // On the last frame the AEC buffer is adjusted to 60% of the last reported + // device buffer size. + // We construct an unstable system by altering the device buffer size between + // two values |kDeviceBufMs| +- 25 ms. + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + + // To make sure we have a full buffer when we verify stability we first fill + // up the far-end buffer with the same amount as we will report in on the + // average through Process(). + size_t buffer_size = BufferFillUp(); + + int buffer_offset_ms = 25; + int reported_delay_ms = 0; + int process_time_ms = 0; + for (; process_time_ms <= kMaxConvergenceMs; process_time_ms += 10) { + reported_delay_ms = kDeviceBufMs + buffer_offset_ms; + RenderAndCapture(reported_delay_ms); + buffer_size += samples_per_frame_; + buffer_offset_ms = -buffer_offset_ms; + if (self_->startup_phase == 0) { + // We have left the startup phase. + break; + } + } + // Verify convergence time. + EXPECT_GE(kMaxConvergenceMs, process_time_ms); + // Verify that the buffer has been flushed. + EXPECT_GE(static_cast(buffer_size), + WebRtcAec_system_delay(self_->aec)); + + // Verify system delay with respect to requirements, i.e., the + // |system_delay| is in the interval [60%, 100%] of what's last reported. + EXPECT_GE(static_cast(reported_delay_ms * samples_per_frame_ / 10), + WebRtcAec_system_delay(self_->aec)); + EXPECT_LE( + static_cast(reported_delay_ms * samples_per_frame_ / 10 * 3 / 5), + WebRtcAec_system_delay(self_->aec)); + } +} + +TEST_F(SystemDelayTest, CorrectDelayAfterStableBufferBuildUp) { + // This test does not apply in extended_filter mode, since we only use the + // the first 10 ms chunk to determine a reasonable buffer size. Neither does + // it apply if DA-AEC is on because that overrides the startup procedure. + WebRtcAec_enable_extended_filter(self_->aec, 0); + EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(self_->aec)); + WebRtcAec_enable_delay_agnostic(self_->aec, 0); + EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(self_->aec)); + + // In this test we start by establishing the device buffer size during stable + // conditions, but with an empty internal far-end buffer. Once that is done we + // verify that the system delay is increased correctly until we have reach an + // internal buffer size of 75% of what's been reported. + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + + // We assume that running |kStableConvergenceMs| calls will put the + // algorithm in a state where the device buffer size has been determined. We + // can make that assumption since we have a separate stability test. + int process_time_ms = 0; + for (; process_time_ms < kStableConvergenceMs; process_time_ms += 10) { + EXPECT_EQ(0, + WebRtcAec_Process(handle_, + &near_ptr_, + 1, + &out_ptr_, + samples_per_frame_, + kDeviceBufMs, + 0)); + } + // Verify that a buffer size has been established. + EXPECT_EQ(0, self_->checkBuffSize); + + // We now have established the required buffer size. Let us verify that we + // fill up before leaving the startup phase for normal processing. + size_t buffer_size = 0; + size_t target_buffer_size = kDeviceBufMs * samples_per_frame_ / 10 * 3 / 4; + process_time_ms = 0; + for (; process_time_ms <= kMaxConvergenceMs; process_time_ms += 10) { + RenderAndCapture(kDeviceBufMs); + buffer_size += samples_per_frame_; + if (self_->startup_phase == 0) { + // We have left the startup phase. + break; + } + } + // Verify convergence time. + EXPECT_GT(kMaxConvergenceMs, process_time_ms); + // Verify that the buffer has reached the desired size. + EXPECT_LE(static_cast(target_buffer_size), + WebRtcAec_system_delay(self_->aec)); + + // Verify normal behavior (system delay is kept constant) after startup by + // running a couple of calls to BufferFarend() and Process(). + for (int j = 0; j < 6; j++) { + int system_delay_before_calls = WebRtcAec_system_delay(self_->aec); + RenderAndCapture(kDeviceBufMs); + EXPECT_EQ(system_delay_before_calls, WebRtcAec_system_delay(self_->aec)); + } + } +} + +TEST_F(SystemDelayTest, CorrectDelayWhenBufferUnderrun) { + // Here we test a buffer under run scenario. If we keep on calling + // WebRtcAec_Process() we will finally run out of data, but should + // automatically stuff the buffer. We verify this behavior by checking if the + // system delay goes negative. + // This process should be independent of DA-AEC and extended_filter mode. + for (int extended_filter = 0; extended_filter <= 1; ++extended_filter) { + WebRtcAec_enable_extended_filter(self_->aec, extended_filter); + EXPECT_EQ(extended_filter, WebRtcAec_extended_filter_enabled(self_->aec)); + for (int da_aec = 0; da_aec <= 1; ++da_aec) { + WebRtcAec_enable_delay_agnostic(self_->aec, da_aec); + EXPECT_EQ(da_aec, WebRtcAec_delay_agnostic_enabled(self_->aec)); + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + RunStableStartup(); + + // The AEC has now left the Startup phase. We now have at most + // |kStableConvergenceMs| in the buffer. Keep on calling Process() until + // we run out of data and verify that the system delay is non-negative. + for (int j = 0; j <= kStableConvergenceMs; j += 10) { + EXPECT_EQ(0, WebRtcAec_Process(handle_, &near_ptr_, 1, &out_ptr_, + samples_per_frame_, kDeviceBufMs, 0)); + EXPECT_LE(0, WebRtcAec_system_delay(self_->aec)); + } + } + } + } +} + +TEST_F(SystemDelayTest, CorrectDelayDuringDrift) { + // This drift test should verify that the system delay is never exceeding the + // device buffer. The drift is simulated by decreasing the reported device + // buffer size by 1 ms every 100 ms. If the device buffer size goes below 30 + // ms we jump (add) 10 ms to give a repeated pattern. + + // This process should be independent of DA-AEC and extended_filter mode. + for (int extended_filter = 0; extended_filter <= 1; ++extended_filter) { + WebRtcAec_enable_extended_filter(self_->aec, extended_filter); + EXPECT_EQ(extended_filter, WebRtcAec_extended_filter_enabled(self_->aec)); + for (int da_aec = 0; da_aec <= 1; ++da_aec) { + WebRtcAec_enable_delay_agnostic(self_->aec, da_aec); + EXPECT_EQ(da_aec, WebRtcAec_delay_agnostic_enabled(self_->aec)); + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + RunStableStartup(); + + // We have left the startup phase and proceed with normal processing. + int jump = 0; + for (int j = 0; j < 1000; j++) { + // Drift = -1 ms per 100 ms of data. + int device_buf_ms = kDeviceBufMs - (j / 10) + jump; + int device_buf = MapBufferSizeToSamples(device_buf_ms, + extended_filter == 1); + + if (device_buf_ms < 30) { + // Add 10 ms data, taking affect next frame. + jump += 10; + } + RenderAndCapture(device_buf_ms); + + // Verify that the system delay does not exceed the device buffer. + EXPECT_GE(device_buf, WebRtcAec_system_delay(self_->aec)); + + // Verify that the system delay is non-negative. + EXPECT_LE(0, WebRtcAec_system_delay(self_->aec)); + } + } + } + } +} + +TEST_F(SystemDelayTest, ShouldRecoverAfterGlitch) { + // This glitch test should verify that the system delay recovers if there is + // a glitch in data. The data glitch is constructed as 200 ms of buffering + // after which the stable procedure continues. The glitch is never reported by + // the device. + // The system is said to be in a non-causal state if the difference between + // the device buffer and system delay is less than a block (64 samples). + + // This process should be independent of DA-AEC and extended_filter mode. + for (int extended_filter = 0; extended_filter <= 1; ++extended_filter) { + WebRtcAec_enable_extended_filter(self_->aec, extended_filter); + EXPECT_EQ(extended_filter, WebRtcAec_extended_filter_enabled(self_->aec)); + for (int da_aec = 0; da_aec <= 1; ++da_aec) { + WebRtcAec_enable_delay_agnostic(self_->aec, da_aec); + EXPECT_EQ(da_aec, WebRtcAec_delay_agnostic_enabled(self_->aec)); + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + RunStableStartup(); + int device_buf = MapBufferSizeToSamples(kDeviceBufMs, + extended_filter == 1); + // Glitch state. + for (int j = 0; j < 20; j++) { + EXPECT_EQ(0, + WebRtcAec_BufferFarend(handle_, far_, samples_per_frame_)); + // No need to verify system delay, since that is done in a separate + // test. + } + // Verify that we are in a non-causal state, i.e., + // |system_delay| > |device_buf|. + EXPECT_LT(device_buf, WebRtcAec_system_delay(self_->aec)); + + // Recover state. Should recover at least 4 ms of data per 10 ms, hence + // a glitch of 200 ms will take at most 200 * 10 / 4 = 500 ms to recover + // from. + bool non_causal = true; // We are currently in a non-causal state. + for (int j = 0; j < 50; j++) { + int system_delay_before = WebRtcAec_system_delay(self_->aec); + RenderAndCapture(kDeviceBufMs); + int system_delay_after = WebRtcAec_system_delay(self_->aec); + // We have recovered if + // |device_buf| - |system_delay_after| >= PART_LEN (1 block). + // During recovery, |system_delay_after| < |system_delay_before|, + // otherwise they are equal. + if (non_causal) { + EXPECT_LT(system_delay_after, system_delay_before); + if (device_buf - system_delay_after >= PART_LEN) { + non_causal = false; + } + } else { + EXPECT_EQ(system_delay_before, system_delay_after); + } + // Verify that the system delay is non-negative. + EXPECT_LE(0, WebRtcAec_system_delay(self_->aec)); + } + // Check that we have recovered. + EXPECT_FALSE(non_causal); + } + } + } +} + +TEST_F(SystemDelayTest, UnaffectedWhenSpuriousDeviceBufferValues) { + // This test does not apply in extended_filter mode, since we only use the + // the first 10 ms chunk to determine a reasonable buffer size. + const int extended_filter = 0; + WebRtcAec_enable_extended_filter(self_->aec, extended_filter); + EXPECT_EQ(extended_filter, WebRtcAec_extended_filter_enabled(self_->aec)); + + // Should be DA-AEC independent. + for (int da_aec = 0; da_aec <= 1; ++da_aec) { + WebRtcAec_enable_delay_agnostic(self_->aec, da_aec); + EXPECT_EQ(da_aec, WebRtcAec_delay_agnostic_enabled(self_->aec)); + // This spurious device buffer data test aims at verifying that the system + // delay is unaffected by large outliers. + // The system is said to be in a non-causal state if the difference between + // the device buffer and system delay is less than a block (64 samples). + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + RunStableStartup(); + int device_buf = MapBufferSizeToSamples(kDeviceBufMs, + extended_filter == 1); + + // Normal state. We are currently not in a non-causal state. + bool non_causal = false; + + // Run 1 s and replace device buffer size with 500 ms every 100 ms. + for (int j = 0; j < 100; j++) { + int system_delay_before_calls = WebRtcAec_system_delay(self_->aec); + int device_buf_ms = j % 10 == 0 ? 500 : kDeviceBufMs; + RenderAndCapture(device_buf_ms); + + // Check for non-causality. + if (device_buf - WebRtcAec_system_delay(self_->aec) < PART_LEN) { + non_causal = true; + } + EXPECT_FALSE(non_causal); + EXPECT_EQ(system_delay_before_calls, + WebRtcAec_system_delay(self_->aec)); + + // Verify that the system delay is non-negative. + EXPECT_LE(0, WebRtcAec_system_delay(self_->aec)); + } + } + } +} + +TEST_F(SystemDelayTest, CorrectImpactWhenTogglingDeviceBufferValues) { + // This test aims at verifying that the system delay is "unaffected" by + // toggling values reported by the device. + // The test is constructed such that every other device buffer value is zero + // and then 2 * |kDeviceBufMs|, hence the size is constant on the average. The + // zero values will force us into a non-causal state and thereby lowering the + // system delay until we basically run out of data. Once that happens the + // buffer will be stuffed. + // TODO(bjornv): This test will have a better impact if we verified that the + // delay estimate goes up when the system delay goes down to meet the average + // device buffer size. + + // This test does not apply if DA-AEC is enabled and extended_filter mode + // disabled. + for (int extended_filter = 0; extended_filter <= 1; ++extended_filter) { + WebRtcAec_enable_extended_filter(self_->aec, extended_filter); + EXPECT_EQ(extended_filter, WebRtcAec_extended_filter_enabled(self_->aec)); + for (int da_aec = 0; da_aec <= 1; ++da_aec) { + WebRtcAec_enable_delay_agnostic(self_->aec, da_aec); + EXPECT_EQ(da_aec, WebRtcAec_delay_agnostic_enabled(self_->aec)); + if (extended_filter == 0 && da_aec == 1) { + continue; + } + for (size_t i = 0; i < kNumSampleRates; i++) { + Init(kSampleRateHz[i]); + RunStableStartup(); + const int device_buf = MapBufferSizeToSamples(kDeviceBufMs, + extended_filter == 1); + + // Normal state. We are currently not in a non-causal state. + bool non_causal = false; + + // Loop through 100 frames (both render and capture), which equals 1 s + // of data. Every odd frame we set the device buffer size to + // 2 * |kDeviceBufMs| and even frames we set the device buffer size to + // zero. + for (int j = 0; j < 100; j++) { + int system_delay_before_calls = WebRtcAec_system_delay(self_->aec); + int device_buf_ms = 2 * (j % 2) * kDeviceBufMs; + RenderAndCapture(device_buf_ms); + + // Check for non-causality, compared with the average device buffer + // size. + non_causal |= (device_buf - WebRtcAec_system_delay(self_->aec) < 64); + EXPECT_GE(system_delay_before_calls, + WebRtcAec_system_delay(self_->aec)); + + // Verify that the system delay is non-negative. + EXPECT_LE(0, WebRtcAec_system_delay(self_->aec)); + } + // Verify we are not in a non-causal state. + EXPECT_FALSE(non_causal); + } + } + } +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc new file mode 100644 index 0000000000..fb3f48a918 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc @@ -0,0 +1,525 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/adaptive_fir_filter.h" + +#if defined(WEBRTC_HAS_NEON) +#include +#endif +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include +#include + +#include "modules/audio_processing/aec3/fft_data.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace aec3 { + +// Computes and stores the frequency response of the filter. +void UpdateFrequencyResponse( + rtc::ArrayView H, + std::vector>* H2) { + RTC_DCHECK_EQ(H.size(), H2->size()); + for (size_t k = 0; k < H.size(); ++k) { + std::transform(H[k].re.begin(), H[k].re.end(), H[k].im.begin(), + (*H2)[k].begin(), + [](float a, float b) { return a * a + b * b; }); + } +} + +#if defined(WEBRTC_HAS_NEON) +// Computes and stores the frequency response of the filter. +void UpdateFrequencyResponse_NEON( + rtc::ArrayView H, + std::vector>* H2) { + RTC_DCHECK_EQ(H.size(), H2->size()); + for (size_t k = 0; k < H.size(); ++k) { + for (size_t j = 0; j < kFftLengthBy2; j += 4) { + const float32x4_t re = vld1q_f32(&H[k].re[j]); + const float32x4_t im = vld1q_f32(&H[k].im[j]); + float32x4_t H2_k_j = vmulq_f32(re, re); + H2_k_j = vmlaq_f32(H2_k_j, im, im); + vst1q_f32(&(*H2)[k][j], H2_k_j); + } + (*H2)[k][kFftLengthBy2] = H[k].re[kFftLengthBy2] * H[k].re[kFftLengthBy2] + + H[k].im[kFftLengthBy2] * H[k].im[kFftLengthBy2]; + } +} +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Computes and stores the frequency response of the filter. +void UpdateFrequencyResponse_SSE2( + rtc::ArrayView H, + std::vector>* H2) { + RTC_DCHECK_EQ(H.size(), H2->size()); + for (size_t k = 0; k < H.size(); ++k) { + for (size_t j = 0; j < kFftLengthBy2; j += 4) { + const __m128 re = _mm_loadu_ps(&H[k].re[j]); + const __m128 re2 = _mm_mul_ps(re, re); + const __m128 im = _mm_loadu_ps(&H[k].im[j]); + const __m128 im2 = _mm_mul_ps(im, im); + const __m128 H2_k_j = _mm_add_ps(re2, im2); + _mm_storeu_ps(&(*H2)[k][j], H2_k_j); + } + (*H2)[k][kFftLengthBy2] = H[k].re[kFftLengthBy2] * H[k].re[kFftLengthBy2] + + H[k].im[kFftLengthBy2] * H[k].im[kFftLengthBy2]; + } +} +#endif + +// Computes and stores the echo return loss estimate of the filter, which is the +// sum of the partition frequency responses. +void UpdateErlEstimator( + const std::vector>& H2, + std::array* erl) { + erl->fill(0.f); + for (auto& H2_j : H2) { + std::transform(H2_j.begin(), H2_j.end(), erl->begin(), erl->begin(), + std::plus()); + } +} + +#if defined(WEBRTC_HAS_NEON) +// Computes and stores the echo return loss estimate of the filter, which is the +// sum of the partition frequency responses. +void UpdateErlEstimator_NEON( + const std::vector>& H2, + std::array* erl) { + erl->fill(0.f); + for (auto& H2_j : H2) { + for (size_t k = 0; k < kFftLengthBy2; k += 4) { + const float32x4_t H2_j_k = vld1q_f32(&H2_j[k]); + float32x4_t erl_k = vld1q_f32(&(*erl)[k]); + erl_k = vaddq_f32(erl_k, H2_j_k); + vst1q_f32(&(*erl)[k], erl_k); + } + (*erl)[kFftLengthBy2] += H2_j[kFftLengthBy2]; + } +} +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Computes and stores the echo return loss estimate of the filter, which is the +// sum of the partition frequency responses. +void UpdateErlEstimator_SSE2( + const std::vector>& H2, + std::array* erl) { + erl->fill(0.f); + for (auto& H2_j : H2) { + for (size_t k = 0; k < kFftLengthBy2; k += 4) { + const __m128 H2_j_k = _mm_loadu_ps(&H2_j[k]); + __m128 erl_k = _mm_loadu_ps(&(*erl)[k]); + erl_k = _mm_add_ps(erl_k, H2_j_k); + _mm_storeu_ps(&(*erl)[k], erl_k); + } + (*erl)[kFftLengthBy2] += H2_j[kFftLengthBy2]; + } +} +#endif + +// Adapts the filter partitions as H(t+1)=H(t)+G(t)*conj(X(t)). +void AdaptPartitions(const RenderBuffer& render_buffer, + const FftData& G, + rtc::ArrayView H) { + rtc::ArrayView render_buffer_data = render_buffer.Buffer(); + size_t index = render_buffer.Position(); + for (auto& H_j : H) { + const FftData& X = render_buffer_data[index]; + for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { + H_j.re[k] += X.re[k] * G.re[k] + X.im[k] * G.im[k]; + H_j.im[k] += X.re[k] * G.im[k] - X.im[k] * G.re[k]; + } + + index = index < (render_buffer_data.size() - 1) ? index + 1 : 0; + } +} + +#if defined(WEBRTC_HAS_NEON) +// Adapts the filter partitions. (NEON variant) +void AdaptPartitions_NEON(const RenderBuffer& render_buffer, + const FftData& G, + rtc::ArrayView H) { + rtc::ArrayView render_buffer_data = render_buffer.Buffer(); + const int lim1 = + std::min(render_buffer_data.size() - render_buffer.Position(), H.size()); + const int lim2 = H.size(); + constexpr int kNumFourBinBands = kFftLengthBy2 / 4; + FftData* H_j = &H[0]; + const FftData* X = &render_buffer_data[render_buffer.Position()]; + int limit = lim1; + int j = 0; + do { + for (; j < limit; ++j, ++H_j, ++X) { + for (int k = 0, n = 0; n < kNumFourBinBands; ++n, k += 4) { + const float32x4_t G_re = vld1q_f32(&G.re[k]); + const float32x4_t G_im = vld1q_f32(&G.im[k]); + const float32x4_t X_re = vld1q_f32(&X->re[k]); + const float32x4_t X_im = vld1q_f32(&X->im[k]); + const float32x4_t H_re = vld1q_f32(&H_j->re[k]); + const float32x4_t H_im = vld1q_f32(&H_j->im[k]); + const float32x4_t a = vmulq_f32(X_re, G_re); + const float32x4_t e = vmlaq_f32(a, X_im, G_im); + const float32x4_t c = vmulq_f32(X_re, G_im); + const float32x4_t f = vmlsq_f32(c, X_im, G_re); + const float32x4_t g = vaddq_f32(H_re, e); + const float32x4_t h = vaddq_f32(H_im, f); + + vst1q_f32(&H_j->re[k], g); + vst1q_f32(&H_j->im[k], h); + } + } + + X = &render_buffer_data[0]; + limit = lim2; + } while (j < lim2); + + H_j = &H[0]; + X = &render_buffer_data[render_buffer.Position()]; + limit = lim1; + j = 0; + do { + for (; j < limit; ++j, ++H_j, ++X) { + H_j->re[kFftLengthBy2] += X->re[kFftLengthBy2] * G.re[kFftLengthBy2] + + X->im[kFftLengthBy2] * G.im[kFftLengthBy2]; + H_j->im[kFftLengthBy2] += X->re[kFftLengthBy2] * G.im[kFftLengthBy2] - + X->im[kFftLengthBy2] * G.re[kFftLengthBy2]; + } + + X = &render_buffer_data[0]; + limit = lim2; + } while (j < lim2); +} +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Adapts the filter partitions. (SSE2 variant) +void AdaptPartitions_SSE2(const RenderBuffer& render_buffer, + const FftData& G, + rtc::ArrayView H) { + rtc::ArrayView render_buffer_data = render_buffer.Buffer(); + const int lim1 = + std::min(render_buffer_data.size() - render_buffer.Position(), H.size()); + const int lim2 = H.size(); + constexpr int kNumFourBinBands = kFftLengthBy2 / 4; + FftData* H_j; + const FftData* X; + int limit; + int j; + for (int k = 0, n = 0; n < kNumFourBinBands; ++n, k += 4) { + const __m128 G_re = _mm_loadu_ps(&G.re[k]); + const __m128 G_im = _mm_loadu_ps(&G.im[k]); + + H_j = &H[0]; + X = &render_buffer_data[render_buffer.Position()]; + limit = lim1; + j = 0; + do { + for (; j < limit; ++j, ++H_j, ++X) { + const __m128 X_re = _mm_loadu_ps(&X->re[k]); + const __m128 X_im = _mm_loadu_ps(&X->im[k]); + const __m128 H_re = _mm_loadu_ps(&H_j->re[k]); + const __m128 H_im = _mm_loadu_ps(&H_j->im[k]); + const __m128 a = _mm_mul_ps(X_re, G_re); + const __m128 b = _mm_mul_ps(X_im, G_im); + const __m128 c = _mm_mul_ps(X_re, G_im); + const __m128 d = _mm_mul_ps(X_im, G_re); + const __m128 e = _mm_add_ps(a, b); + const __m128 f = _mm_sub_ps(c, d); + const __m128 g = _mm_add_ps(H_re, e); + const __m128 h = _mm_add_ps(H_im, f); + _mm_storeu_ps(&H_j->re[k], g); + _mm_storeu_ps(&H_j->im[k], h); + } + + X = &render_buffer_data[0]; + limit = lim2; + } while (j < lim2); + } + + H_j = &H[0]; + X = &render_buffer_data[render_buffer.Position()]; + limit = lim1; + j = 0; + do { + for (; j < limit; ++j, ++H_j, ++X) { + H_j->re[kFftLengthBy2] += X->re[kFftLengthBy2] * G.re[kFftLengthBy2] + + X->im[kFftLengthBy2] * G.im[kFftLengthBy2]; + H_j->im[kFftLengthBy2] += X->re[kFftLengthBy2] * G.im[kFftLengthBy2] - + X->im[kFftLengthBy2] * G.re[kFftLengthBy2]; + } + + X = &render_buffer_data[0]; + limit = lim2; + } while (j < lim2); +} +#endif + +// Produces the filter output. +void ApplyFilter(const RenderBuffer& render_buffer, + rtc::ArrayView H, + FftData* S) { + S->re.fill(0.f); + S->im.fill(0.f); + + rtc::ArrayView render_buffer_data = render_buffer.Buffer(); + size_t index = render_buffer.Position(); + for (auto& H_j : H) { + const FftData& X = render_buffer_data[index]; + for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { + S->re[k] += X.re[k] * H_j.re[k] - X.im[k] * H_j.im[k]; + S->im[k] += X.re[k] * H_j.im[k] + X.im[k] * H_j.re[k]; + } + index = index < (render_buffer_data.size() - 1) ? index + 1 : 0; + } +} + +#if defined(WEBRTC_HAS_NEON) +// Produces the filter output (NEON variant). +void ApplyFilter_NEON(const RenderBuffer& render_buffer, + rtc::ArrayView H, + FftData* S) { + RTC_DCHECK_GE(H.size(), H.size() - 1); + S->re.fill(0.f); + S->im.fill(0.f); + + rtc::ArrayView render_buffer_data = render_buffer.Buffer(); + const int lim1 = + std::min(render_buffer_data.size() - render_buffer.Position(), H.size()); + const int lim2 = H.size(); + constexpr int kNumFourBinBands = kFftLengthBy2 / 4; + const FftData* H_j = &H[0]; + const FftData* X = &render_buffer_data[render_buffer.Position()]; + + int j = 0; + int limit = lim1; + do { + for (; j < limit; ++j, ++H_j, ++X) { + for (int k = 0, n = 0; n < kNumFourBinBands; ++n, k += 4) { + const float32x4_t X_re = vld1q_f32(&X->re[k]); + const float32x4_t X_im = vld1q_f32(&X->im[k]); + const float32x4_t H_re = vld1q_f32(&H_j->re[k]); + const float32x4_t H_im = vld1q_f32(&H_j->im[k]); + const float32x4_t S_re = vld1q_f32(&S->re[k]); + const float32x4_t S_im = vld1q_f32(&S->im[k]); + const float32x4_t a = vmulq_f32(X_re, H_re); + const float32x4_t e = vmlsq_f32(a, X_im, H_im); + const float32x4_t c = vmulq_f32(X_re, H_im); + const float32x4_t f = vmlaq_f32(c, X_im, H_re); + const float32x4_t g = vaddq_f32(S_re, e); + const float32x4_t h = vaddq_f32(S_im, f); + vst1q_f32(&S->re[k], g); + vst1q_f32(&S->im[k], h); + } + } + limit = lim2; + X = &render_buffer_data[0]; + } while (j < lim2); + + H_j = &H[0]; + X = &render_buffer_data[render_buffer.Position()]; + j = 0; + limit = lim1; + do { + for (; j < limit; ++j, ++H_j, ++X) { + S->re[kFftLengthBy2] += X->re[kFftLengthBy2] * H_j->re[kFftLengthBy2] - + X->im[kFftLengthBy2] * H_j->im[kFftLengthBy2]; + S->im[kFftLengthBy2] += X->re[kFftLengthBy2] * H_j->im[kFftLengthBy2] + + X->im[kFftLengthBy2] * H_j->re[kFftLengthBy2]; + } + limit = lim2; + X = &render_buffer_data[0]; + } while (j < lim2); +} +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Produces the filter output (SSE2 variant). +void ApplyFilter_SSE2(const RenderBuffer& render_buffer, + rtc::ArrayView H, + FftData* S) { + RTC_DCHECK_GE(H.size(), H.size() - 1); + S->re.fill(0.f); + S->im.fill(0.f); + + rtc::ArrayView render_buffer_data = render_buffer.Buffer(); + const int lim1 = + std::min(render_buffer_data.size() - render_buffer.Position(), H.size()); + const int lim2 = H.size(); + constexpr int kNumFourBinBands = kFftLengthBy2 / 4; + const FftData* H_j = &H[0]; + const FftData* X = &render_buffer_data[render_buffer.Position()]; + + int j = 0; + int limit = lim1; + do { + for (; j < limit; ++j, ++H_j, ++X) { + for (int k = 0, n = 0; n < kNumFourBinBands; ++n, k += 4) { + const __m128 X_re = _mm_loadu_ps(&X->re[k]); + const __m128 X_im = _mm_loadu_ps(&X->im[k]); + const __m128 H_re = _mm_loadu_ps(&H_j->re[k]); + const __m128 H_im = _mm_loadu_ps(&H_j->im[k]); + const __m128 S_re = _mm_loadu_ps(&S->re[k]); + const __m128 S_im = _mm_loadu_ps(&S->im[k]); + const __m128 a = _mm_mul_ps(X_re, H_re); + const __m128 b = _mm_mul_ps(X_im, H_im); + const __m128 c = _mm_mul_ps(X_re, H_im); + const __m128 d = _mm_mul_ps(X_im, H_re); + const __m128 e = _mm_sub_ps(a, b); + const __m128 f = _mm_add_ps(c, d); + const __m128 g = _mm_add_ps(S_re, e); + const __m128 h = _mm_add_ps(S_im, f); + _mm_storeu_ps(&S->re[k], g); + _mm_storeu_ps(&S->im[k], h); + } + } + limit = lim2; + X = &render_buffer_data[0]; + } while (j < lim2); + + H_j = &H[0]; + X = &render_buffer_data[render_buffer.Position()]; + j = 0; + limit = lim1; + do { + for (; j < limit; ++j, ++H_j, ++X) { + S->re[kFftLengthBy2] += X->re[kFftLengthBy2] * H_j->re[kFftLengthBy2] - + X->im[kFftLengthBy2] * H_j->im[kFftLengthBy2]; + S->im[kFftLengthBy2] += X->re[kFftLengthBy2] * H_j->im[kFftLengthBy2] + + X->im[kFftLengthBy2] * H_j->re[kFftLengthBy2]; + } + limit = lim2; + X = &render_buffer_data[0]; + } while (j < lim2); +} +#endif + +} // namespace aec3 + +AdaptiveFirFilter::AdaptiveFirFilter(size_t size_partitions, + Aec3Optimization optimization, + ApmDataDumper* data_dumper) + : data_dumper_(data_dumper), + fft_(), + optimization_(optimization), + H_(size_partitions), + H2_(size_partitions, std::array()) { + RTC_DCHECK(data_dumper_); + + h_.fill(0.f); + for (auto& H_j : H_) { + H_j.Clear(); + } + for (auto& H2_k : H2_) { + H2_k.fill(0.f); + } + erl_.fill(0.f); +} + +AdaptiveFirFilter::~AdaptiveFirFilter() = default; + +void AdaptiveFirFilter::HandleEchoPathChange() { + h_.fill(0.f); + for (auto& H_j : H_) { + H_j.Clear(); + } + for (auto& H2_k : H2_) { + H2_k.fill(0.f); + } + erl_.fill(0.f); +} + +void AdaptiveFirFilter::Filter(const RenderBuffer& render_buffer, + FftData* S) const { + RTC_DCHECK(S); + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: + aec3::ApplyFilter_SSE2(render_buffer, H_, S); + break; +#endif +#if defined(WEBRTC_HAS_NEON) + case Aec3Optimization::kNeon: + aec3::ApplyFilter_NEON(render_buffer, H_, S); + break; +#endif + default: + aec3::ApplyFilter(render_buffer, H_, S); + } +} + +void AdaptiveFirFilter::Adapt(const RenderBuffer& render_buffer, + const FftData& G) { + // Adapt the filter. + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: + aec3::AdaptPartitions_SSE2(render_buffer, G, H_); + break; +#endif +#if defined(WEBRTC_HAS_NEON) + case Aec3Optimization::kNeon: + aec3::AdaptPartitions_NEON(render_buffer, G, H_); + break; +#endif + default: + aec3::AdaptPartitions(render_buffer, G, H_); + } + + // Constrain the filter partitions in a cyclic manner. + Constrain(); + + // Update the frequency response and echo return loss for the filter. + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: + aec3::UpdateFrequencyResponse_SSE2(H_, &H2_); + aec3::UpdateErlEstimator_SSE2(H2_, &erl_); + break; +#endif +#if defined(WEBRTC_HAS_NEON) + case Aec3Optimization::kNeon: + aec3::UpdateFrequencyResponse_NEON(H_, &H2_); + aec3::UpdateErlEstimator_NEON(H2_, &erl_); + break; +#endif + default: + aec3::UpdateFrequencyResponse(H_, &H2_); + aec3::UpdateErlEstimator(H2_, &erl_); + } +} + +// Constrains the a partiton of the frequency domain filter to be limited in +// time via setting the relevant time-domain coefficients to zero. +void AdaptiveFirFilter::Constrain() { + std::array h; + fft_.Ifft(H_[partition_to_constrain_], &h); + + static constexpr float kScale = 1.0f / kFftLengthBy2; + std::for_each(h.begin(), h.begin() + kFftLengthBy2, + [](float& a) { a *= kScale; }); + std::fill(h.begin() + kFftLengthBy2, h.end(), 0.f); + + std::copy(h.begin(), h.begin() + kFftLengthBy2, + h_.begin() + partition_to_constrain_ * kFftLengthBy2); + + fft_.Fft(&h, &H_[partition_to_constrain_]); + + partition_to_constrain_ = partition_to_constrain_ < (H_.size() - 1) + ? partition_to_constrain_ + 1 + : 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h new file mode 100644 index 0000000000..7872869351 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ADAPTIVE_FIR_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ADAPTIVE_FIR_FILTER_H_ + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace aec3 { +// Computes and stores the frequency response of the filter. +void UpdateFrequencyResponse( + rtc::ArrayView H, + std::vector>* H2); +#if defined(WEBRTC_HAS_NEON) +void UpdateFrequencyResponse_NEON( + rtc::ArrayView H, + std::vector>* H2); +#endif +#if defined(WEBRTC_ARCH_X86_FAMILY) +void UpdateFrequencyResponse_SSE2( + rtc::ArrayView H, + std::vector>* H2); +#endif + +// Computes and stores the echo return loss estimate of the filter, which is the +// sum of the partition frequency responses. +void UpdateErlEstimator( + const std::vector>& H2, + std::array* erl); +#if defined(WEBRTC_HAS_NEON) +void UpdateErlEstimator_NEON( + const std::vector>& H2, + std::array* erl); +#endif +#if defined(WEBRTC_ARCH_X86_FAMILY) +void UpdateErlEstimator_SSE2( + const std::vector>& H2, + std::array* erl); +#endif + +// Adapts the filter partitions. +void AdaptPartitions(const RenderBuffer& render_buffer, + const FftData& G, + rtc::ArrayView H); +#if defined(WEBRTC_HAS_NEON) +void AdaptPartitions_NEON(const RenderBuffer& render_buffer, + const FftData& G, + rtc::ArrayView H); +#endif +#if defined(WEBRTC_ARCH_X86_FAMILY) +void AdaptPartitions_SSE2(const RenderBuffer& render_buffer, + const FftData& G, + rtc::ArrayView H); +#endif + +// Produces the filter output. +void ApplyFilter(const RenderBuffer& render_buffer, + rtc::ArrayView H, + FftData* S); +#if defined(WEBRTC_HAS_NEON) +void ApplyFilter_NEON(const RenderBuffer& render_buffer, + rtc::ArrayView H, + FftData* S); +#endif +#if defined(WEBRTC_ARCH_X86_FAMILY) +void ApplyFilter_SSE2(const RenderBuffer& render_buffer, + rtc::ArrayView H, + FftData* S); +#endif + +} // namespace aec3 + +// Provides a frequency domain adaptive filter functionality. +class AdaptiveFirFilter { + public: + AdaptiveFirFilter(size_t size_partitions, + Aec3Optimization optimization, + ApmDataDumper* data_dumper); + + ~AdaptiveFirFilter(); + + // Produces the output of the filter. + void Filter(const RenderBuffer& render_buffer, FftData* S) const; + + // Adapts the filter. + void Adapt(const RenderBuffer& render_buffer, const FftData& G); + + // Receives reports that known echo path changes have occured and adjusts + // the filter adaptation accordingly. + void HandleEchoPathChange(); + + // Returns the filter size. + size_t SizePartitions() const { return H_.size(); } + + // Returns the filter based echo return loss. + const std::array& Erl() const { return erl_; } + + // Returns the frequency responses for the filter partitions. + const std::vector>& + FilterFrequencyResponse() const { + return H2_; + } + + // Returns the estimate of the impulse response. + const std::array& + FilterImpulseResponse() const { + return h_; + } + + void DumpFilter(const char* name) { + for (auto& H : H_) { + data_dumper_->DumpRaw(name, H.re); + data_dumper_->DumpRaw(name, H.im); + } + } + + private: + // Constrain the filter partitions in a cyclic manner. + void Constrain(); + + ApmDataDumper* const data_dumper_; + const Aec3Fft fft_; + const Aec3Optimization optimization_; + std::vector H_; + std::vector> H2_; + std::array h_; + std::array erl_; + size_t partition_to_constrain_ = 0; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AdaptiveFirFilter); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ADAPTIVE_FIR_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc new file mode 100644 index 0000000000..11d7e02b41 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/adaptive_fir_filter.h" + +#include +#include +#include +#include +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/cascaded_biquad_filter.h" +#include "modules/audio_processing/aec3/render_signal_analyzer.h" +#include "modules/audio_processing/aec3/shadow_filter_update_gain.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/random.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" + +namespace webrtc { +namespace aec3 { +namespace { + +std::string ProduceDebugText(size_t delay) { + std::ostringstream ss; + ss << ", Delay: " << delay; + return ss.str(); +} + +} // namespace + +#if defined(WEBRTC_HAS_NEON) +// Verifies that the optimized methods for filter adaptation are similar to +// their reference counterparts. +TEST(AdaptiveFirFilter, FilterAdaptationNeonOptimizations) { + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 12, + std::vector(1, 12)); + Random random_generator(42U); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + FftData S_C; + FftData S_NEON; + FftData G; + Aec3Fft fft; + std::vector H_C(10); + std::vector H_NEON(10); + for (auto& H_j : H_C) { + H_j.Clear(); + } + for (auto& H_j : H_NEON) { + H_j.Clear(); + } + + for (size_t k = 0; k < 30; ++k) { + RandomizeSampleVector(&random_generator, x[0]); + render_buffer.Insert(x); + } + + for (size_t j = 0; j < G.re.size(); ++j) { + G.re[j] = j / 10001.f; + } + for (size_t j = 1; j < G.im.size() - 1; ++j) { + G.im[j] = j / 20001.f; + } + G.im[0] = 0.f; + G.im[G.im.size() - 1] = 0.f; + + AdaptPartitions_NEON(render_buffer, G, H_NEON); + AdaptPartitions(render_buffer, G, H_C); + AdaptPartitions_NEON(render_buffer, G, H_NEON); + AdaptPartitions(render_buffer, G, H_C); + + for (size_t l = 0; l < H_C.size(); ++l) { + for (size_t j = 0; j < H_C[l].im.size(); ++j) { + EXPECT_NEAR(H_C[l].re[j], H_NEON[l].re[j], fabs(H_C[l].re[j] * 0.00001f)); + EXPECT_NEAR(H_C[l].im[j], H_NEON[l].im[j], fabs(H_C[l].im[j] * 0.00001f)); + } + } + + ApplyFilter_NEON(render_buffer, H_NEON, &S_NEON); + ApplyFilter(render_buffer, H_C, &S_C); + for (size_t j = 0; j < S_C.re.size(); ++j) { + EXPECT_NEAR(S_C.re[j], S_NEON.re[j], fabs(S_C.re[j] * 0.00001f)); + EXPECT_NEAR(S_C.im[j], S_NEON.im[j], fabs(S_C.re[j] * 0.00001f)); + } +} + +// Verifies that the optimized method for frequency response computation is +// bitexact to the reference counterpart. +TEST(AdaptiveFirFilter, UpdateFrequencyResponseNeonOptimization) { + const size_t kNumPartitions = 12; + std::vector H(kNumPartitions); + std::vector> H2(kNumPartitions); + std::vector> H2_NEON(kNumPartitions); + + for (size_t j = 0; j < H.size(); ++j) { + for (size_t k = 0; k < H[j].re.size(); ++k) { + H[j].re[k] = k + j / 3.f; + H[j].im[k] = j + k / 7.f; + } + } + + UpdateFrequencyResponse(H, &H2); + UpdateFrequencyResponse_NEON(H, &H2_NEON); + + for (size_t j = 0; j < H2.size(); ++j) { + for (size_t k = 0; k < H[j].re.size(); ++k) { + EXPECT_FLOAT_EQ(H2[j][k], H2_NEON[j][k]); + } + } +} + +// Verifies that the optimized method for echo return loss computation is +// bitexact to the reference counterpart. +TEST(AdaptiveFirFilter, UpdateErlNeonOptimization) { + const size_t kNumPartitions = 12; + std::vector> H2(kNumPartitions); + std::array erl; + std::array erl_NEON; + + for (size_t j = 0; j < H2.size(); ++j) { + for (size_t k = 0; k < H2[j].size(); ++k) { + H2[j][k] = k + j / 3.f; + } + } + + UpdateErlEstimator(H2, &erl); + UpdateErlEstimator_NEON(H2, &erl_NEON); + + for (size_t j = 0; j < erl.size(); ++j) { + EXPECT_FLOAT_EQ(erl[j], erl_NEON[j]); + } +} + +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Verifies that the optimized methods for filter adaptation are bitexact to +// their reference counterparts. +TEST(AdaptiveFirFilter, FilterAdaptationSse2Optimizations) { + bool use_sse2 = (WebRtc_GetCPUInfo(kSSE2) != 0); + if (use_sse2) { + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 12, + std::vector(1, 12)); + Random random_generator(42U); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + FftData S_C; + FftData S_SSE2; + FftData G; + Aec3Fft fft; + std::vector H_C(10); + std::vector H_SSE2(10); + for (auto& H_j : H_C) { + H_j.Clear(); + } + for (auto& H_j : H_SSE2) { + H_j.Clear(); + } + + for (size_t k = 0; k < 500; ++k) { + RandomizeSampleVector(&random_generator, x[0]); + render_buffer.Insert(x); + + ApplyFilter_SSE2(render_buffer, H_SSE2, &S_SSE2); + ApplyFilter(render_buffer, H_C, &S_C); + for (size_t j = 0; j < S_C.re.size(); ++j) { + EXPECT_FLOAT_EQ(S_C.re[j], S_SSE2.re[j]); + EXPECT_FLOAT_EQ(S_C.im[j], S_SSE2.im[j]); + } + + std::for_each(G.re.begin(), G.re.end(), + [&](float& a) { a = random_generator.Rand(); }); + std::for_each(G.im.begin(), G.im.end(), + [&](float& a) { a = random_generator.Rand(); }); + + AdaptPartitions_SSE2(render_buffer, G, H_SSE2); + AdaptPartitions(render_buffer, G, H_C); + + for (size_t k = 0; k < H_C.size(); ++k) { + for (size_t j = 0; j < H_C[k].re.size(); ++j) { + EXPECT_FLOAT_EQ(H_C[k].re[j], H_SSE2[k].re[j]); + EXPECT_FLOAT_EQ(H_C[k].im[j], H_SSE2[k].im[j]); + } + } + } + } +} + +// Verifies that the optimized method for frequency response computation is +// bitexact to the reference counterpart. +TEST(AdaptiveFirFilter, UpdateFrequencyResponseSse2Optimization) { + bool use_sse2 = (WebRtc_GetCPUInfo(kSSE2) != 0); + if (use_sse2) { + const size_t kNumPartitions = 12; + std::vector H(kNumPartitions); + std::vector> H2(kNumPartitions); + std::vector> H2_SSE2(kNumPartitions); + + for (size_t j = 0; j < H.size(); ++j) { + for (size_t k = 0; k < H[j].re.size(); ++k) { + H[j].re[k] = k + j / 3.f; + H[j].im[k] = j + k / 7.f; + } + } + + UpdateFrequencyResponse(H, &H2); + UpdateFrequencyResponse_SSE2(H, &H2_SSE2); + + for (size_t j = 0; j < H2.size(); ++j) { + for (size_t k = 0; k < H[j].re.size(); ++k) { + EXPECT_FLOAT_EQ(H2[j][k], H2_SSE2[j][k]); + } + } + } +} + +// Verifies that the optimized method for echo return loss computation is +// bitexact to the reference counterpart. +TEST(AdaptiveFirFilter, UpdateErlSse2Optimization) { + bool use_sse2 = (WebRtc_GetCPUInfo(kSSE2) != 0); + if (use_sse2) { + const size_t kNumPartitions = 12; + std::vector> H2(kNumPartitions); + std::array erl; + std::array erl_SSE2; + + for (size_t j = 0; j < H2.size(); ++j) { + for (size_t k = 0; k < H2[j].size(); ++k) { + H2[j][k] = k + j / 3.f; + } + } + + UpdateErlEstimator(H2, &erl); + UpdateErlEstimator_SSE2(H2, &erl_SSE2); + + for (size_t j = 0; j < erl.size(); ++j) { + EXPECT_FLOAT_EQ(erl[j], erl_SSE2[j]); + } + } +} + +#endif + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Verifies that the check for non-null data dumper works. +TEST(AdaptiveFirFilter, NullDataDumper) { + EXPECT_DEATH(AdaptiveFirFilter(9, DetectOptimization(), nullptr), ""); +} + +// Verifies that the check for non-null filter output works. +TEST(AdaptiveFirFilter, NullFilterOutput) { + ApmDataDumper data_dumper(42); + AdaptiveFirFilter filter(9, DetectOptimization(), &data_dumper); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, + filter.SizePartitions(), + std::vector(1, filter.SizePartitions())); + EXPECT_DEATH(filter.Filter(render_buffer, nullptr), ""); +} + +#endif + +// Verifies that the filter statistics can be accessed when filter statistics +// are turned on. +TEST(AdaptiveFirFilter, FilterStatisticsAccess) { + ApmDataDumper data_dumper(42); + AdaptiveFirFilter filter(9, DetectOptimization(), &data_dumper); + filter.Erl(); + filter.FilterFrequencyResponse(); +} + +// Verifies that the filter size if correctly repported. +TEST(AdaptiveFirFilter, FilterSize) { + ApmDataDumper data_dumper(42); + for (size_t filter_size = 1; filter_size < 5; ++filter_size) { + AdaptiveFirFilter filter(filter_size, DetectOptimization(), &data_dumper); + EXPECT_EQ(filter_size, filter.SizePartitions()); + } +} + +// Verifies that the filter is being able to properly filter a signal and to +// adapt its coefficients. +TEST(AdaptiveFirFilter, FilterAndAdapt) { + constexpr size_t kNumBlocksToProcess = 500; + ApmDataDumper data_dumper(42); + AdaptiveFirFilter filter(9, DetectOptimization(), &data_dumper); + Aec3Fft fft; + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, + filter.SizePartitions(), + std::vector(1, filter.SizePartitions())); + ShadowFilterUpdateGain gain; + Random random_generator(42U); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector n(kBlockSize, 0.f); + std::vector y(kBlockSize, 0.f); + AecState aec_state(EchoCanceller3Config{}); + RenderSignalAnalyzer render_signal_analyzer; + std::vector e(kBlockSize, 0.f); + std::array s_scratch; + std::array s; + FftData S; + FftData G; + FftData E; + std::array Y2; + std::array E2_main; + std::array E2_shadow; + // [B,A] = butter(2,100/8000,'high') + constexpr CascadedBiQuadFilter::BiQuadCoefficients + kHighPassFilterCoefficients = {{0.97261f, -1.94523f, 0.97261f}, + {-1.94448f, 0.94598f}}; + Y2.fill(0.f); + E2_main.fill(0.f); + E2_shadow.fill(0.f); + + constexpr float kScale = 1.0f / kFftLengthBy2; + + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + DelayBuffer delay_buffer(delay_samples); + CascadedBiQuadFilter x_hp_filter(kHighPassFilterCoefficients, 1); + CascadedBiQuadFilter y_hp_filter(kHighPassFilterCoefficients, 1); + + SCOPED_TRACE(ProduceDebugText(delay_samples)); + for (size_t k = 0; k < kNumBlocksToProcess; ++k) { + RandomizeSampleVector(&random_generator, x[0]); + delay_buffer.Delay(x[0], y); + + RandomizeSampleVector(&random_generator, n); + static constexpr float kNoiseScaling = 1.f / 100.f; + std::transform( + y.begin(), y.end(), n.begin(), y.begin(), + [](float a, float b) { return a + b * kNoiseScaling; }); + + x_hp_filter.Process(x[0]); + y_hp_filter.Process(y); + + render_buffer.Insert(x); + render_signal_analyzer.Update(render_buffer, aec_state.FilterDelay()); + + filter.Filter(render_buffer, &S); + fft.Ifft(S, &s_scratch); + std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, + e.begin(), + [&](float a, float b) { return a - b * kScale; }); + std::for_each(e.begin(), e.end(), + [](float& a) { a = rtc::SafeClamp(a, -32768.f, 32767.f); }); + fft.ZeroPaddedFft(e, &E); + for (size_t k = 0; k < kBlockSize; ++k) { + s[k] = kScale * s_scratch[k + kFftLengthBy2]; + } + + gain.Compute(render_buffer, render_signal_analyzer, E, + filter.SizePartitions(), false, &G); + filter.Adapt(render_buffer, G); + aec_state.HandleEchoPathChange(EchoPathVariability(false, false)); + aec_state.Update(filter.FilterFrequencyResponse(), + filter.FilterImpulseResponse(), true, rtc::nullopt, + render_buffer, E2_main, Y2, x[0], s, false); + } + // Verify that the filter is able to perform well. + EXPECT_LT(1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f), + std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); + ASSERT_TRUE(aec_state.FilterDelay()); + EXPECT_EQ(delay_samples / kBlockSize, *aec_state.FilterDelay()); + } +} +} // namespace aec3 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.cc new file mode 100644 index 0000000000..7becce49f1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.cc @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/aec3_common.h" + +#include "typedefs.h" // NOLINT(build/include) +#include "system_wrappers/include/cpu_features_wrapper.h" + +namespace webrtc { + +Aec3Optimization DetectOptimization() { +#if defined(WEBRTC_ARCH_X86_FAMILY) + if (WebRtc_GetCPUInfo(kSSE2) != 0) { + return Aec3Optimization::kSse2; + } +#endif + +#if defined(WEBRTC_HAS_NEON) + return Aec3Optimization::kNeon; +#endif + + return Aec3Optimization::kNone; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.h new file mode 100644 index 0000000000..a85f3ace21 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_AEC3_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_AEC3_COMMON_H_ + +#include +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#ifdef _MSC_VER /* visual c++ */ +#define ALIGN16_BEG __declspec(align(16)) +#define ALIGN16_END +#else /* gcc or icc */ +#define ALIGN16_BEG +#define ALIGN16_END __attribute__((aligned(16))) +#endif + +enum class Aec3Optimization { kNone, kSse2, kNeon }; + +constexpr int kNumBlocksPerSecond = 250; + +constexpr int kMetricsReportingIntervalBlocks = 10 * kNumBlocksPerSecond; +constexpr int kMetricsComputationBlocks = 9; +constexpr int kMetricsCollectionBlocks = + kMetricsReportingIntervalBlocks - kMetricsComputationBlocks; + +constexpr size_t kFftLengthBy2 = 64; +constexpr size_t kFftLengthBy2Plus1 = kFftLengthBy2 + 1; +constexpr size_t kFftLengthBy2Minus1 = kFftLengthBy2 - 1; +constexpr size_t kFftLength = 2 * kFftLengthBy2; + +constexpr int kAdaptiveFilterLength = 12; +constexpr int kUnknownDelayRenderWindowSize = 12; +constexpr int kAdaptiveFilterTimeDomainLength = + kAdaptiveFilterLength * kFftLengthBy2; + +constexpr size_t kMaxNumBands = 3; +constexpr size_t kSubFrameLength = 80; + +constexpr size_t kBlockSize = kFftLengthBy2; +constexpr size_t kExtendedBlockSize = 2 * kFftLengthBy2; +constexpr size_t kMatchedFilterWindowSizeSubBlocks = 32; +constexpr size_t kMatchedFilterAlignmentShiftSizeSubBlocks = + kMatchedFilterWindowSizeSubBlocks * 3 / 4; + +constexpr size_t kMinEchoPathDelayBlocks = 5; +constexpr size_t kMaxApiCallsJitterBlocks = 26; +constexpr size_t kRenderTransferQueueSize = kMaxApiCallsJitterBlocks / 2; +static_assert(2 * kRenderTransferQueueSize >= kMaxApiCallsJitterBlocks, + "Requirement to ensure buffer overflow detection"); + +constexpr size_t kEchoPathChangeConvergenceBlocks = 2 * kNumBlocksPerSecond; + +// TODO(peah): Integrate this with how it is done inside audio_processing_impl. +constexpr size_t NumBandsForRate(int sample_rate_hz) { + return static_cast(sample_rate_hz == 8000 ? 1 + : sample_rate_hz / 16000); +} +constexpr int LowestBandRate(int sample_rate_hz) { + return sample_rate_hz == 8000 ? sample_rate_hz : 16000; +} + +constexpr bool ValidFullBandRate(int sample_rate_hz) { + return sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000; +} + +constexpr size_t GetDownSampledBufferSize(size_t down_sampling_factor, + size_t num_matched_filters) { + return kBlockSize / down_sampling_factor * + (kMatchedFilterAlignmentShiftSizeSubBlocks * num_matched_filters + + kMatchedFilterWindowSizeSubBlocks + 1); +} + +constexpr size_t GetRenderDelayBufferSize(size_t down_sampling_factor, + size_t num_matched_filters) { + return (3 * + GetDownSampledBufferSize(down_sampling_factor, num_matched_filters)) / + (4 * kBlockSize / down_sampling_factor); +} + +// Detects what kind of optimizations to use for the code. +Aec3Optimization DetectOptimization(); + +static_assert(1 == NumBandsForRate(8000), "Number of bands for 8 kHz"); +static_assert(1 == NumBandsForRate(16000), "Number of bands for 16 kHz"); +static_assert(2 == NumBandsForRate(32000), "Number of bands for 32 kHz"); +static_assert(3 == NumBandsForRate(48000), "Number of bands for 48 kHz"); + +static_assert(8000 == LowestBandRate(8000), "Sample rate of band 0 for 8 kHz"); +static_assert(16000 == LowestBandRate(16000), + "Sample rate of band 0 for 16 kHz"); +static_assert(16000 == LowestBandRate(32000), + "Sample rate of band 0 for 32 kHz"); +static_assert(16000 == LowestBandRate(48000), + "Sample rate of band 0 for 48 kHz"); + +static_assert(ValidFullBandRate(8000), + "Test that 8 kHz is a valid sample rate"); +static_assert(ValidFullBandRate(16000), + "Test that 16 kHz is a valid sample rate"); +static_assert(ValidFullBandRate(32000), + "Test that 32 kHz is a valid sample rate"); +static_assert(ValidFullBandRate(48000), + "Test that 48 kHz is a valid sample rate"); +static_assert(!ValidFullBandRate(8001), + "Test that 8001 Hz is not a valid sample rate"); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_AEC3_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.cc new file mode 100644 index 0000000000..c8120cb864 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.cc @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/aec3_fft.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +// TODO(peah): Change x to be std::array once the rest of the code allows this. +void Aec3Fft::ZeroPaddedFft(rtc::ArrayView x, FftData* X) const { + RTC_DCHECK(X); + RTC_DCHECK_EQ(kFftLengthBy2, x.size()); + std::array fft; + std::fill(fft.begin(), fft.begin() + kFftLengthBy2, 0.f); + std::copy(x.begin(), x.end(), fft.begin() + kFftLengthBy2); + Fft(&fft, X); +} + +void Aec3Fft::PaddedFft(rtc::ArrayView x, + rtc::ArrayView x_old, + FftData* X) const { + RTC_DCHECK(X); + RTC_DCHECK_EQ(kFftLengthBy2, x.size()); + RTC_DCHECK_EQ(kFftLengthBy2, x_old.size()); + std::array fft; + std::copy(x_old.begin(), x_old.end(), fft.begin()); + std::copy(x.begin(), x.end(), fft.begin() + x_old.size()); + std::copy(x.begin(), x.end(), x_old.begin()); + Fft(&fft, X); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.h new file mode 100644 index 0000000000..2a5dfef47f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_AEC3_FFT_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_AEC3_FFT_H_ + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "modules/audio_processing/utility/ooura_fft.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Wrapper class that provides 128 point real valued FFT functionality with the +// FftData type. +class Aec3Fft { + public: + Aec3Fft() = default; + // Computes the FFT. Note that both the input and output are modified. + void Fft(std::array* x, FftData* X) const { + RTC_DCHECK(x); + RTC_DCHECK(X); + ooura_fft_.Fft(x->data()); + X->CopyFromPackedArray(*x); + } + // Computes the inverse Fft. + void Ifft(const FftData& X, std::array* x) const { + RTC_DCHECK(x); + X.CopyToPackedArray(x); + ooura_fft_.InverseFft(x->data()); + } + + // Pads the input with kFftLengthBy2 initial zeros before computing the Fft. + void ZeroPaddedFft(rtc::ArrayView x, FftData* X) const; + + // Concatenates the kFftLengthBy2 values long x and x_old before computing the + // Fft. After that, x is copied to x_old. + void PaddedFft(rtc::ArrayView x, + rtc::ArrayView x_old, + FftData* X) const; + + private: + const OouraFft ooura_fft_; + + RTC_DISALLOW_COPY_AND_ASSIGN(Aec3Fft); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_AEC3_FFT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc new file mode 100644 index 0000000000..46831e021a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/aec3_fft.h" + +#include + +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies that the check for non-null input in Fft works. +TEST(Aec3Fft, NullFftInput) { + Aec3Fft fft; + FftData X; + EXPECT_DEATH(fft.Fft(nullptr, &X), ""); +} + +// Verifies that the check for non-null input in Fft works. +TEST(Aec3Fft, NullFftOutput) { + Aec3Fft fft; + std::array x; + EXPECT_DEATH(fft.Fft(&x, nullptr), ""); +} + +// Verifies that the check for non-null output in Ifft works. +TEST(Aec3Fft, NullIfftOutput) { + Aec3Fft fft; + FftData X; + EXPECT_DEATH(fft.Ifft(X, nullptr), ""); +} + +// Verifies that the check for non-null output in ZeroPaddedFft works. +TEST(Aec3Fft, NullZeroPaddedFftOutput) { + Aec3Fft fft; + std::array x; + EXPECT_DEATH(fft.ZeroPaddedFft(x, nullptr), ""); +} + +// Verifies that the check for input length in ZeroPaddedFft works. +TEST(Aec3Fft, ZeroPaddedFftWrongInputLength) { + Aec3Fft fft; + FftData X; + std::array x; + EXPECT_DEATH(fft.ZeroPaddedFft(x, &X), ""); +} + +// Verifies that the check for non-null output in PaddedFft works. +TEST(Aec3Fft, NullPaddedFftOutput) { + Aec3Fft fft; + std::array x; + std::array x_old; + EXPECT_DEATH(fft.PaddedFft(x, x_old, nullptr), ""); +} + +// Verifies that the check for input length in PaddedFft works. +TEST(Aec3Fft, PaddedFftWrongInputLength) { + Aec3Fft fft; + FftData X; + std::array x; + std::array x_old; + EXPECT_DEATH(fft.PaddedFft(x, x_old, &X), ""); +} + +// Verifies that the check for length in the old value in PaddedFft works. +TEST(Aec3Fft, PaddedFftWrongOldValuesLength) { + Aec3Fft fft; + FftData X; + std::array x; + std::array x_old; + EXPECT_DEATH(fft.PaddedFft(x, x_old, &X), ""); +} + +#endif + +// Verifies that Fft works as intended. +TEST(Aec3Fft, Fft) { + Aec3Fft fft; + FftData X; + std::array x; + x.fill(0.f); + fft.Fft(&x, &X); + EXPECT_THAT(X.re, ::testing::Each(0.f)); + EXPECT_THAT(X.im, ::testing::Each(0.f)); + + x.fill(0.f); + x[0] = 1.f; + fft.Fft(&x, &X); + EXPECT_THAT(X.re, ::testing::Each(1.f)); + EXPECT_THAT(X.im, ::testing::Each(0.f)); + + x.fill(1.f); + fft.Fft(&x, &X); + EXPECT_EQ(128.f, X.re[0]); + std::for_each(X.re.begin() + 1, X.re.end(), + [](float a) { EXPECT_EQ(0.f, a); }); + EXPECT_THAT(X.im, ::testing::Each(0.f)); +} + +// Verifies that InverseFft works as intended. +TEST(Aec3Fft, Ifft) { + Aec3Fft fft; + FftData X; + std::array x; + + X.re.fill(0.f); + X.im.fill(0.f); + fft.Ifft(X, &x); + EXPECT_THAT(x, ::testing::Each(0.f)); + + X.re.fill(1.f); + X.im.fill(0.f); + fft.Ifft(X, &x); + EXPECT_EQ(64.f, x[0]); + std::for_each(x.begin() + 1, x.end(), [](float a) { EXPECT_EQ(0.f, a); }); + + X.re.fill(0.f); + X.re[0] = 128; + X.im.fill(0.f); + fft.Ifft(X, &x); + EXPECT_THAT(x, ::testing::Each(64.f)); +} + +// Verifies that InverseFft and Fft work as intended. +TEST(Aec3Fft, FftAndIfft) { + Aec3Fft fft; + FftData X; + std::array x; + std::array x_ref; + + int v = 0; + for (int k = 0; k < 20; ++k) { + for (size_t j = 0; j < x.size(); ++j) { + x[j] = v++; + x_ref[j] = x[j] * 64.f; + } + fft.Fft(&x, &X); + fft.Ifft(X, &x); + for (size_t j = 0; j < x.size(); ++j) { + EXPECT_NEAR(x_ref[j], x[j], 0.001f); + } + } +} + +// Verifies that ZeroPaddedFft work as intended. +TEST(Aec3Fft, ZeroPaddedFft) { + Aec3Fft fft; + FftData X; + std::array x_in; + std::array x_ref; + std::array x_out; + + int v = 0; + x_ref.fill(0.f); + for (int k = 0; k < 20; ++k) { + for (size_t j = 0; j < x_in.size(); ++j) { + x_in[j] = v++; + x_ref[j + kFftLengthBy2] = x_in[j] * 64.f; + } + fft.ZeroPaddedFft(x_in, &X); + fft.Ifft(X, &x_out); + for (size_t j = 0; j < x_out.size(); ++j) { + EXPECT_NEAR(x_ref[j], x_out[j], 0.1f); + } + } +} + +// Verifies that ZeroPaddedFft work as intended. +TEST(Aec3Fft, PaddedFft) { + Aec3Fft fft; + FftData X; + std::array x_in; + std::array x_out; + std::array x_old; + std::array x_old_ref; + std::array x_ref; + + int v = 0; + x_old.fill(0.f); + for (int k = 0; k < 20; ++k) { + for (size_t j = 0; j < x_in.size(); ++j) { + x_in[j] = v++; + } + + std::copy(x_old.begin(), x_old.end(), x_ref.begin()); + std::copy(x_in.begin(), x_in.end(), x_ref.begin() + kFftLengthBy2); + std::copy(x_in.begin(), x_in.end(), x_old_ref.begin()); + std::for_each(x_ref.begin(), x_ref.end(), [](float& a) { a *= 64.f; }); + + fft.PaddedFft(x_in, x_old, &X); + fft.Ifft(X, &x_out); + + for (size_t j = 0; j < x_out.size(); ++j) { + EXPECT_NEAR(x_ref[j], x_out[j], 0.1f); + } + + EXPECT_EQ(x_old_ref, x_old); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.cc new file mode 100644 index 0000000000..0318b04ef4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.cc @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/aec_state.h" + +#include + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Computes delay of the adaptive filter. +int EstimateFilterDelay( + const std::vector>& + adaptive_filter_frequency_response) { + const auto& H2 = adaptive_filter_frequency_response; + constexpr size_t kUpperBin = kFftLengthBy2 - 5; + RTC_DCHECK_GE(kAdaptiveFilterLength, H2.size()); + std::array delays; + delays.fill(0); + for (size_t k = 1; k < kUpperBin; ++k) { + // Find the maximum of H2[j]. + size_t peak = 0; + for (size_t j = 0; j < H2.size(); ++j) { + if (H2[j][k] > H2[peak][k]) { + peak = j; + } + } + ++delays[peak]; + } + + return std::distance(delays.begin(), + std::max_element(delays.begin(), delays.end())); +} + +} // namespace + +int AecState::instance_count_ = 0; + +AecState::AecState(const EchoCanceller3Config& config) + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + erle_estimator_(config.erle.min, config.erle.max_l, config.erle.max_h), + config_(config), + reverb_decay_(config_.ep_strength.default_len) { + max_render_.fill(0.f); +} + +AecState::~AecState() = default; + +void AecState::HandleEchoPathChange( + const EchoPathVariability& echo_path_variability) { + if (echo_path_variability.AudioPathChanged()) { + blocks_since_last_saturation_ = 0; + usable_linear_estimate_ = false; + echo_leakage_detected_ = false; + capture_signal_saturation_ = false; + echo_saturation_ = false; + previous_max_sample_ = 0.f; + max_render_.fill(0.f); + + if (echo_path_variability.delay_change) { + force_zero_gain_counter_ = 0; + blocks_with_filter_adaptation_ = 0; + blocks_with_strong_render_ = 0; + initial_state_ = true; + linear_echo_estimate_ = false; + sufficient_filter_updates_ = false; + render_received_ = false; + force_zero_gain_ = true; + capture_block_counter_ = 0; + } + if (echo_path_variability.gain_change) { + capture_block_counter_ = kNumBlocksPerSecond; + } + } +} + +void AecState::Update(const std::vector>& + adaptive_filter_frequency_response, + const std::array& + adaptive_filter_impulse_response, + bool converged_filter, + const rtc::Optional& external_delay_samples, + const RenderBuffer& render_buffer, + const std::array& E2_main, + const std::array& Y2, + rtc::ArrayView x, + const std::array& s, + bool echo_leakage_detected) { + // Store input parameters. + echo_leakage_detected_ = echo_leakage_detected; + + // Update counters. + ++capture_block_counter_; + + // Force zero echo suppression gain after an echo path change to allow at + // least some render data to be collected in order to avoid an initial echo + // burst. + force_zero_gain_ = (++force_zero_gain_counter_) < kNumBlocksPerSecond / 5; + + // Estimate delays. + filter_delay_ = EstimateFilterDelay(adaptive_filter_frequency_response); + external_delay_ = + external_delay_samples + ? rtc::Optional(*external_delay_samples / kBlockSize) + : rtc::nullopt; + + // Update the ERL and ERLE measures. + if (converged_filter && capture_block_counter_ >= 2 * kNumBlocksPerSecond) { + const auto& X2 = render_buffer.Spectrum(*filter_delay_); + erle_estimator_.Update(X2, Y2, E2_main); + erl_estimator_.Update(X2, Y2); + } + + // Update the echo audibility evaluator. + echo_audibility_.Update(x, s, converged_filter); + + // Detect and flag echo saturation. + // TODO(peah): Add the delay in this computation to ensure that the render and + // capture signals are properly aligned. + RTC_DCHECK_LT(0, x.size()); + const float max_sample = fabs(*std::max_element( + x.begin(), x.end(), [](float a, float b) { return a * a < b * b; })); + + if (config_.ep_strength.echo_can_saturate) { + const bool saturated_echo = + (previous_max_sample_ > 200.f) && SaturatedCapture(); + + // Counts the blocks since saturation. + constexpr size_t kSaturationLeakageBlocks = 20; + + // Set flag for potential presence of saturated echo + blocks_since_last_saturation_ = + saturated_echo ? 0 : blocks_since_last_saturation_ + 1; + + echo_saturation_ = blocks_since_last_saturation_ < kSaturationLeakageBlocks; + } else { + echo_saturation_ = false; + } + previous_max_sample_ = max_sample; + + // TODO(peah): Move? + sufficient_filter_updates_ = + blocks_with_filter_adaptation_ >= kEchoPathChangeConvergenceBlocks; + initial_state_ = capture_block_counter_ < 3 * kNumBlocksPerSecond; + + // Flag whether the linear filter estimate is usable. + usable_linear_estimate_ = + (!echo_saturation_) && (converged_filter || SufficientFilterUpdates()) && + capture_block_counter_ >= 2 * kNumBlocksPerSecond && external_delay_; + + linear_echo_estimate_ = UsableLinearEstimate() && !TransparentMode(); + + // After an amount of active render samples for which an echo should have been + // detected in the capture signal if the ERL was not infinite, flag that a + // transparent mode should be entered. + const float x_energy = std::inner_product(x.begin(), x.end(), x.begin(), 0.f); + const bool active_render_block = + x_energy > (config_.render_levels.active_render_limit * + config_.render_levels.active_render_limit) * + kFftLengthBy2; + + if (active_render_block) { + render_received_ = true; + } + + // Update counters. + blocks_with_filter_adaptation_ += + (active_render_block && (!SaturatedCapture()) ? 1 : 0); + + transparent_mode_ = !converged_filter && + (!render_received_ || blocks_with_filter_adaptation_ >= + 5 * kNumBlocksPerSecond); + + // Update the room reverb estimate. + UpdateReverb(adaptive_filter_impulse_response); +} + +void AecState::UpdateReverb( + const std::array& + impulse_response) { + if ((!(filter_delay_ && usable_linear_estimate_)) || + (*filter_delay_ > kAdaptiveFilterLength - 4)) { + return; + } + + // Form the data to match against by squaring the impulse response + // coefficients. + std::array matching_data; + std::transform(impulse_response.begin(), impulse_response.end(), + matching_data.begin(), [](float a) { return a * a; }); + + // Avoid matching against noise in the model by subtracting an estimate of the + // model noise power. + constexpr size_t kTailLength = 64; + constexpr size_t tail_index = kAdaptiveFilterTimeDomainLength - kTailLength; + const float tail_power = *std::max_element(matching_data.begin() + tail_index, + matching_data.end()); + std::for_each(matching_data.begin(), matching_data.begin() + tail_index, + [tail_power](float& a) { a = std::max(0.f, a - tail_power); }); + + // Identify the peak index of the impulse response. + const size_t peak_index = *std::max_element( + matching_data.begin(), matching_data.begin() + tail_index); + + if (peak_index + 128 < tail_index) { + size_t start_index = peak_index + 64; + // Compute the matching residual error for the current candidate to match. + float residual_sqr_sum = 0.f; + float d_k = reverb_decay_to_test_; + for (size_t k = start_index; k < tail_index; ++k) { + if (matching_data[start_index + 1] == 0.f) { + break; + } + + float residual = matching_data[k] - matching_data[peak_index] * d_k; + residual_sqr_sum += residual * residual; + d_k *= reverb_decay_to_test_; + } + + // If needed, update the best candidate for the reverb decay. + if (reverb_decay_candidate_residual_ < 0.f || + residual_sqr_sum < reverb_decay_candidate_residual_) { + reverb_decay_candidate_residual_ = residual_sqr_sum; + reverb_decay_candidate_ = reverb_decay_to_test_; + } + } + + // Compute the next reverb candidate to evaluate such that all candidates will + // be evaluated within one second. + reverb_decay_to_test_ += (0.9965f - 0.9f) / (5 * kNumBlocksPerSecond); + + // If all reverb candidates have been evaluated, choose the best one as the + // reverb decay. + if (reverb_decay_to_test_ >= 0.9965f) { + if (reverb_decay_candidate_residual_ < 0.f) { + // Transform the decay to be in the unit of blocks. + reverb_decay_ = powf(reverb_decay_candidate_, kFftLengthBy2); + + // Limit the estimated reverb_decay_ to the maximum one needed in practice + // to minimize the impact of incorrect estimates. + reverb_decay_ = std::min(config_.ep_strength.default_len, reverb_decay_); + } + reverb_decay_to_test_ = 0.9f; + reverb_decay_candidate_residual_ = -1.f; + } + + // For noisy impulse responses, assume a fixed tail length. + if (tail_power > 0.0005f) { + reverb_decay_ = config_.ep_strength.default_len; + } + data_dumper_->DumpRaw("aec3_reverb_decay", reverb_decay_); + data_dumper_->DumpRaw("aec3_tail_power", tail_power); +} + +void AecState::EchoAudibility::Update(rtc::ArrayView x, + const std::array& s, + bool converged_filter) { + auto result_x = std::minmax_element(x.begin(), x.end()); + auto result_s = std::minmax_element(s.begin(), s.end()); + const float x_abs = + std::max(fabsf(*result_x.first), fabsf(*result_x.second)); + const float s_abs = + std::max(fabsf(*result_s.first), fabsf(*result_s.second)); + + if (converged_filter) { + if (x_abs < 20.f) { + ++low_farend_counter_; + } else { + low_farend_counter_ = 0; + } + } else { + if (x_abs < 100.f) { + ++low_farend_counter_; + } else { + low_farend_counter_ = 0; + } + } + + // The echo is deemed as not audible if the echo estimate is on the level of + // the quantization noise in the FFTs and the nearend level is sufficiently + // strong to mask that by ensuring that the playout and AGC gains do not boost + // any residual echo that is below the quantization noise level. Furthermore, + // cases where the render signal is very close to zero are also identified as + // not producing audible echo. + inaudible_echo_ = (max_nearend_ > 500 && s_abs < 30.f) || + (!converged_filter && x_abs < 500); + inaudible_echo_ = inaudible_echo_ || low_farend_counter_ > 20; +} + +void AecState::EchoAudibility::UpdateWithOutput(rtc::ArrayView e) { + const float e_max = *std::max_element(e.begin(), e.end()); + const float e_min = *std::min_element(e.begin(), e.end()); + const float e_abs = std::max(fabsf(e_max), fabsf(e_min)); + + if (max_nearend_ < e_abs) { + max_nearend_ = e_abs; + max_nearend_counter_ = 0; + } else { + if (++max_nearend_counter_ > 5 * kNumBlocksPerSecond) { + max_nearend_ *= 0.995f; + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.h new file mode 100644 index 0000000000..b8c1523a00 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_ + +#include +#include +#include + +#include "api/array_view.h" +#include "api/optional.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/echo_path_variability.h" +#include "modules/audio_processing/aec3/erl_estimator.h" +#include "modules/audio_processing/aec3/erle_estimator.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; + +// Handles the state and the conditions for the echo removal functionality. +class AecState { + public: + explicit AecState(const EchoCanceller3Config& config); + ~AecState(); + + // Returns whether the linear filter estimate is usable. + bool UsableLinearEstimate() const { return usable_linear_estimate_; } + + // Returns whether there has been echo leakage detected. + bool EchoLeakageDetected() const { return echo_leakage_detected_; } + + // Returns whether the render signal is currently active. + // TODO(peah): Deprecate this in an upcoming CL. + bool ActiveRender() const { return blocks_with_filter_adaptation_ > 200; } + + // Returns the ERLE. + const std::array& Erle() const { + return erle_estimator_.Erle(); + } + + // Returns the time-domain ERLE. + float ErleTimeDomain() const { return erle_estimator_.ErleTimeDomain(); } + + // Returns the ERL. + const std::array& Erl() const { + return erl_estimator_.Erl(); + } + + // Returns the time-domain ERL. + float ErlTimeDomain() const { return erl_estimator_.ErlTimeDomain(); } + + // Returns the delay estimate based on the linear filter. + rtc::Optional FilterDelay() const { return filter_delay_; } + + // Returns the externally provided delay. + rtc::Optional ExternalDelay() const { return external_delay_; } + + // Returns whether the capture signal is saturated. + bool SaturatedCapture() const { return capture_signal_saturation_; } + + // Returns whether the echo signal is saturated. + bool SaturatedEcho() const { return echo_saturation_; } + + // Returns whether the echo path can saturate. + bool SaturatingEchoPath() const { return saturating_echo_path_; } + + // Updates the capture signal saturation. + void UpdateCaptureSaturation(bool capture_signal_saturation) { + capture_signal_saturation_ = capture_signal_saturation; + } + + // Returns whether the transparent mode is active + bool TransparentMode() const { return transparent_mode_; } + + // Takes appropriate action at an echo path change. + void HandleEchoPathChange(const EchoPathVariability& echo_path_variability); + + // Returns the decay factor for the echo reverberation. + float ReverbDecay() const { return reverb_decay_; } + + // Returns whether the echo suppression gain should be forced to zero. + bool ForcedZeroGain() const { return force_zero_gain_; } + + // Returns whether the echo in the capture signal is audible. + bool InaudibleEcho() const { return echo_audibility_.InaudibleEcho(); } + + // Updates the aec state with the AEC output signal. + void UpdateWithOutput(rtc::ArrayView e) { + echo_audibility_.UpdateWithOutput(e); + } + + // Returns whether the linear filter should have been able to adapt properly. + bool SufficientFilterUpdates() const { return sufficient_filter_updates_; } + + // Returns whether the echo subtractor can be used to determine the residual + // echo. + bool LinearEchoEstimate() const { return linear_echo_estimate_; } + + // Returns whether the AEC is in an initial state. + bool InitialState() const { return initial_state_; } + + // Updates the aec state. + void Update(const std::vector>& + adaptive_filter_frequency_response, + const std::array& + adaptive_filter_impulse_response, + bool converged_filter, + const rtc::Optional& external_delay_samples, + const RenderBuffer& render_buffer, + const std::array& E2_main, + const std::array& Y2, + rtc::ArrayView x, + const std::array& s_main, + bool echo_leakage_detected); + + private: + class EchoAudibility { + public: + void Update(rtc::ArrayView x, + const std::array& s, + bool converged_filter); + void UpdateWithOutput(rtc::ArrayView e); + bool InaudibleEcho() const { return inaudible_echo_; } + + private: + float max_nearend_ = 0.f; + size_t max_nearend_counter_ = 0; + size_t low_farend_counter_ = 0; + bool inaudible_echo_ = false; + }; + + void UpdateReverb(const std::array& + impulse_response); + + static int instance_count_; + std::unique_ptr data_dumper_; + ErlEstimator erl_estimator_; + ErleEstimator erle_estimator_; + size_t capture_block_counter_ = 0; + size_t blocks_with_filter_adaptation_ = 0; + size_t blocks_with_strong_render_ = 0; + bool usable_linear_estimate_ = false; + bool echo_leakage_detected_ = false; + bool capture_signal_saturation_ = false; + bool echo_saturation_ = false; + bool transparent_mode_ = false; + float previous_max_sample_ = 0.f; + std::array max_render_; + bool force_zero_gain_ = false; + bool render_received_ = false; + size_t force_zero_gain_counter_ = 0; + rtc::Optional filter_delay_; + rtc::Optional external_delay_; + size_t blocks_since_last_saturation_ = 1000; + float reverb_decay_to_test_ = 0.9f; + float reverb_decay_candidate_ = 0.f; + float reverb_decay_candidate_residual_ = -1.f; + EchoAudibility echo_audibility_; + const EchoCanceller3Config config_; + float reverb_decay_; + bool saturating_echo_path_ = false; + bool initial_state_ = true; + bool linear_echo_estimate_ = false; + bool sufficient_filter_updates_ = false; + + RTC_DISALLOW_COPY_AND_ASSIGN(AecState); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state_unittest.cc new file mode 100644 index 0000000000..34b877b4f9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state_unittest.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/aec_state.h" + +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "test/gtest.h" + +namespace webrtc { + +// Verify the general functionality of AecState +TEST(AecState, NormalUsage) { + ApmDataDumper data_dumper(42); + AecState state(EchoCanceller3Config{}); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 30, + std::vector(1, 30)); + std::array E2_main = {}; + std::array Y2 = {}; + std::vector> x(3, std::vector(kBlockSize, 0.f)); + EchoPathVariability echo_path_variability(false, false); + std::array s; + s.fill(100.f); + + std::vector> + converged_filter_frequency_response(10); + for (auto& v : converged_filter_frequency_response) { + v.fill(0.01f); + } + std::vector> + diverged_filter_frequency_response = converged_filter_frequency_response; + converged_filter_frequency_response[2].fill(100.f); + converged_filter_frequency_response[2][0] = 1.f; + + std::array impulse_response; + impulse_response.fill(0.f); + + // Verify that linear AEC usability is false when the filter is diverged and + // there is no external delay reported. + state.Update(diverged_filter_frequency_response, impulse_response, true, + rtc::nullopt, render_buffer, E2_main, Y2, x[0], s, false); + EXPECT_FALSE(state.UsableLinearEstimate()); + + // Verify that linear AEC usability is true when the filter is converged + std::fill(x[0].begin(), x[0].end(), 101.f); + for (int k = 0; k < 3000; ++k) { + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + } + EXPECT_TRUE(state.UsableLinearEstimate()); + + // Verify that linear AEC usability becomes false after an echo path change is + // reported + state.HandleEchoPathChange(EchoPathVariability(true, false)); + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + EXPECT_FALSE(state.UsableLinearEstimate()); + + // Verify that the active render detection works as intended. + std::fill(x[0].begin(), x[0].end(), 101.f); + state.HandleEchoPathChange(EchoPathVariability(true, true)); + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + EXPECT_FALSE(state.ActiveRender()); + + for (int k = 0; k < 1000; ++k) { + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + } + EXPECT_TRUE(state.ActiveRender()); + + // Verify that echo leakage is properly reported. + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + EXPECT_FALSE(state.EchoLeakageDetected()); + + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, true); + EXPECT_TRUE(state.EchoLeakageDetected()); + + // Verify that the ERL is properly estimated + for (auto& x_k : x) { + x_k = std::vector(kBlockSize, 0.f); + } + + x[0][0] = 5000.f; + for (size_t k = 0; k < render_buffer.Buffer().size(); ++k) { + render_buffer.Insert(x); + } + + Y2.fill(10.f * 10000.f * 10000.f); + for (size_t k = 0; k < 1000; ++k) { + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + } + + ASSERT_TRUE(state.UsableLinearEstimate()); + const std::array& erl = state.Erl(); + EXPECT_EQ(erl[0], erl[1]); + for (size_t k = 1; k < erl.size() - 1; ++k) { + EXPECT_NEAR(k % 2 == 0 ? 10.f : 1000.f, erl[k], 0.1); + } + EXPECT_EQ(erl[erl.size() - 2], erl[erl.size() - 1]); + + // Verify that the ERLE is properly estimated + E2_main.fill(1.f * 10000.f * 10000.f); + Y2.fill(10.f * E2_main[0]); + for (size_t k = 0; k < 1000; ++k) { + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + } + ASSERT_TRUE(state.UsableLinearEstimate()); + { + const auto& erle = state.Erle(); + EXPECT_EQ(erle[0], erle[1]); + constexpr size_t kLowFrequencyLimit = 32; + for (size_t k = 1; k < kLowFrequencyLimit; ++k) { + EXPECT_NEAR(k % 2 == 0 ? 8.f : 1.f, erle[k], 0.1); + } + for (size_t k = kLowFrequencyLimit; k < erle.size() - 1; ++k) { + EXPECT_NEAR(k % 2 == 0 ? 1.5f : 1.f, erle[k], 0.1); + } + EXPECT_EQ(erle[erle.size() - 2], erle[erle.size() - 1]); + } + + E2_main.fill(1.f * 10000.f * 10000.f); + Y2.fill(5.f * E2_main[0]); + for (size_t k = 0; k < 1000; ++k) { + state.Update(converged_filter_frequency_response, impulse_response, true, 2, + render_buffer, E2_main, Y2, x[0], s, false); + } + + ASSERT_TRUE(state.UsableLinearEstimate()); + { + const auto& erle = state.Erle(); + EXPECT_EQ(erle[0], erle[1]); + constexpr size_t kLowFrequencyLimit = 32; + for (size_t k = 1; k < kLowFrequencyLimit; ++k) { + EXPECT_NEAR(k % 2 == 0 ? 5.f : 1.f, erle[k], 0.1); + } + for (size_t k = kLowFrequencyLimit; k < erle.size() - 1; ++k) { + EXPECT_NEAR(k % 2 == 0 ? 1.5f : 1.f, erle[k], 0.1); + } + EXPECT_EQ(erle[erle.size() - 2], erle[erle.size() - 1]); + } +} + +// Verifies the delay for a converged filter is correctly identified. +TEST(AecState, ConvergedFilterDelay) { + constexpr int kFilterLength = 10; + AecState state(EchoCanceller3Config{}); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 30, + std::vector(1, 30)); + std::array E2_main; + std::array Y2; + std::array x; + EchoPathVariability echo_path_variability(false, false); + std::array s; + s.fill(100.f); + x.fill(0.f); + + std::vector> frequency_response( + kFilterLength); + + std::array impulse_response; + impulse_response.fill(0.f); + + // Verify that the filter delay for a converged filter is properly identified. + for (int k = 0; k < kFilterLength; ++k) { + for (auto& v : frequency_response) { + v.fill(0.01f); + } + frequency_response[k].fill(100.f); + frequency_response[k][0] = 0.f; + state.HandleEchoPathChange(echo_path_variability); + state.Update(frequency_response, impulse_response, true, rtc::nullopt, + render_buffer, E2_main, Y2, x, s, false); + EXPECT_TRUE(k == (kFilterLength - 1) || state.FilterDelay()); + if (k != (kFilterLength - 1)) { + EXPECT_EQ(k, state.FilterDelay()); + } + } +} + +// Verify that the externally reported delay is properly reported and converted. +TEST(AecState, ExternalDelay) { + AecState state(EchoCanceller3Config{}); + std::array E2_main; + std::array E2_shadow; + std::array Y2; + std::array x; + std::array s; + s.fill(100.f); + E2_main.fill(0.f); + E2_shadow.fill(0.f); + Y2.fill(0.f); + x.fill(0.f); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 30, + std::vector(1, 30)); + std::vector> frequency_response( + kAdaptiveFilterLength); + for (auto& v : frequency_response) { + v.fill(0.01f); + } + + std::array impulse_response; + impulse_response.fill(0.f); + + for (size_t k = 0; k < frequency_response.size() - 1; ++k) { + state.HandleEchoPathChange(EchoPathVariability(false, false)); + state.Update(frequency_response, impulse_response, true, k * kBlockSize + 5, + render_buffer, E2_main, Y2, x, s, false); + EXPECT_TRUE(state.ExternalDelay()); + EXPECT_EQ(k, state.ExternalDelay()); + } + + // Verify that the externally reported delay is properly unset when it is no + // longer present. + state.HandleEchoPathChange(EchoPathVariability(false, false)); + state.Update(frequency_response, impulse_response, true, rtc::nullopt, + render_buffer, E2_main, Y2, x, s, false); + EXPECT_FALSE(state.ExternalDelay()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.cc new file mode 100644 index 0000000000..3160624515 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/block_framer.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +BlockFramer::BlockFramer(size_t num_bands) + : num_bands_(num_bands), + buffer_(num_bands_, std::vector(kBlockSize, 0.f)) {} + +BlockFramer::~BlockFramer() = default; + +// All the constants are chosen so that the buffer is either empty or has enough +// samples for InsertBlockAndExtractSubFrame to produce a frame. In order to +// achieve this, the InsertBlockAndExtractSubFrame and InsertBlock methods need +// to be called in the correct order. +void BlockFramer::InsertBlock(const std::vector>& block) { + RTC_DCHECK_EQ(num_bands_, block.size()); + for (size_t i = 0; i < num_bands_; ++i) { + RTC_DCHECK_EQ(kBlockSize, block[i].size()); + RTC_DCHECK_EQ(0, buffer_[i].size()); + buffer_[i].insert(buffer_[i].begin(), block[i].begin(), block[i].end()); + } +} + +void BlockFramer::InsertBlockAndExtractSubFrame( + const std::vector>& block, + std::vector>* sub_frame) { + RTC_DCHECK(sub_frame); + RTC_DCHECK_EQ(num_bands_, block.size()); + RTC_DCHECK_EQ(num_bands_, sub_frame->size()); + for (size_t i = 0; i < num_bands_; ++i) { + RTC_DCHECK_LE(kSubFrameLength, buffer_[i].size() + kBlockSize); + RTC_DCHECK_EQ(kBlockSize, block[i].size()); + RTC_DCHECK_GE(kBlockSize, buffer_[i].size()); + RTC_DCHECK_EQ(kSubFrameLength, (*sub_frame)[i].size()); + const int samples_to_frame = kSubFrameLength - buffer_[i].size(); + std::copy(buffer_[i].begin(), buffer_[i].end(), (*sub_frame)[i].begin()); + std::copy(block[i].begin(), block[i].begin() + samples_to_frame, + (*sub_frame)[i].begin() + buffer_[i].size()); + buffer_[i].clear(); + buffer_[i].insert(buffer_[i].begin(), block[i].begin() + samples_to_frame, + block[i].end()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.h new file mode 100644 index 0000000000..923e4cfb53 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_BLOCK_FRAMER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_BLOCK_FRAMER_H_ + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Class for producing frames consisting of 1 or 2 subframes of 80 samples each +// from 64 sample blocks. The class is designed to work together with the +// FrameBlocker class which performs the reverse conversion. Used together with +// that, this class produces output frames are the same rate as frames are +// received by the FrameBlocker class. Note that the internal buffers will +// overrun if any other rate of packets insertion is used. +class BlockFramer { + public: + explicit BlockFramer(size_t num_bands); + ~BlockFramer(); + // Adds a 64 sample block into the data that will form the next output frame. + void InsertBlock(const std::vector>& block); + // Adds a 64 sample block and extracts an 80 sample subframe. + void InsertBlockAndExtractSubFrame( + const std::vector>& block, + std::vector>* sub_frame); + + private: + const size_t num_bands_; + std::vector> buffer_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockFramer); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_BLOCK_FRAMER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc new file mode 100644 index 0000000000..16d3944e13 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/block_framer.h" + +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +void SetupSubFrameView(std::vector>* sub_frame, + std::vector>* sub_frame_view) { + for (size_t k = 0; k < sub_frame_view->size(); ++k) { + (*sub_frame_view)[k] = + rtc::ArrayView((*sub_frame)[k].data(), (*sub_frame)[k].size()); + } +} + +float ComputeSampleValue(size_t chunk_counter, + size_t chunk_size, + size_t band, + size_t sample_index, + int offset) { + float value = + static_cast(chunk_counter * chunk_size + sample_index) + offset; + return value > 0 ? 5000 * band + value : 0; +} + +bool VerifySubFrame(size_t sub_frame_counter, + int offset, + const std::vector>& sub_frame_view) { + for (size_t k = 0; k < sub_frame_view.size(); ++k) { + for (size_t i = 0; i < sub_frame_view[k].size(); ++i) { + const float reference_value = + ComputeSampleValue(sub_frame_counter, kSubFrameLength, k, i, offset); + if (reference_value != sub_frame_view[k][i]) { + return false; + } + } + } + return true; +} + +void FillBlock(size_t block_counter, std::vector>* block) { + for (size_t k = 0; k < block->size(); ++k) { + for (size_t i = 0; i < (*block)[0].size(); ++i) { + (*block)[k][i] = ComputeSampleValue(block_counter, kBlockSize, k, i, 0); + } + } +} + +// Verifies that the BlockFramer is able to produce the expected frame content. +void RunFramerTest(int sample_rate_hz) { + constexpr size_t kNumSubFramesToProcess = 2; + const size_t num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> block(num_bands, + std::vector(kBlockSize, 0.f)); + std::vector> output_sub_frame( + num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> output_sub_frame_view(num_bands); + SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); + BlockFramer framer(num_bands); + + size_t block_index = 0; + for (size_t sub_frame_index = 0; sub_frame_index < kNumSubFramesToProcess; + ++sub_frame_index) { + FillBlock(block_index++, &block); + framer.InsertBlockAndExtractSubFrame(block, &output_sub_frame_view); + EXPECT_TRUE(VerifySubFrame(sub_frame_index, -64, output_sub_frame_view)); + + if ((sub_frame_index + 1) % 4 == 0) { + FillBlock(block_index++, &block); + framer.InsertBlock(block); + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Verifies that the BlockFramer crashes if the InsertBlockAndExtractSubFrame +// method is called for inputs with the wrong number of bands or band lengths. +void RunWronglySizedInsertAndExtractParametersTest(int sample_rate_hz, + size_t num_block_bands, + size_t block_length, + size_t num_sub_frame_bands, + size_t sub_frame_length) { + const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> block(num_block_bands, + std::vector(block_length, 0.f)); + std::vector> output_sub_frame( + num_sub_frame_bands, std::vector(sub_frame_length, 0.f)); + std::vector> output_sub_frame_view( + output_sub_frame.size()); + SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); + BlockFramer framer(correct_num_bands); + EXPECT_DEATH( + framer.InsertBlockAndExtractSubFrame(block, &output_sub_frame_view), ""); +} + +// Verifies that the BlockFramer crashes if the InsertBlock method is called for +// inputs with the wrong number of bands or band lengths. +void RunWronglySizedInsertParameterTest(int sample_rate_hz, + size_t num_block_bands, + size_t block_length) { + const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> correct_block( + correct_num_bands, std::vector(kBlockSize, 0.f)); + std::vector> wrong_block( + num_block_bands, std::vector(block_length, 0.f)); + std::vector> output_sub_frame( + correct_num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> output_sub_frame_view( + output_sub_frame.size()); + SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); + BlockFramer framer(correct_num_bands); + framer.InsertBlockAndExtractSubFrame(correct_block, &output_sub_frame_view); + framer.InsertBlockAndExtractSubFrame(correct_block, &output_sub_frame_view); + framer.InsertBlockAndExtractSubFrame(correct_block, &output_sub_frame_view); + framer.InsertBlockAndExtractSubFrame(correct_block, &output_sub_frame_view); + + EXPECT_DEATH(framer.InsertBlock(wrong_block), ""); +} + +// Verifies that the BlockFramer crashes if the InsertBlock method is called +// after a wrong number of previous InsertBlockAndExtractSubFrame method calls +// have been made. +void RunWronglyInsertOrderTest(int sample_rate_hz, + size_t num_preceeding_api_calls) { + const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> block(correct_num_bands, + std::vector(kBlockSize, 0.f)); + std::vector> output_sub_frame( + correct_num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> output_sub_frame_view( + output_sub_frame.size()); + SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); + BlockFramer framer(correct_num_bands); + for (size_t k = 0; k < num_preceeding_api_calls; ++k) { + framer.InsertBlockAndExtractSubFrame(block, &output_sub_frame_view); + } + + EXPECT_DEATH(framer.InsertBlock(block), ""); +} +#endif + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlockAndExtractSubFrame) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, wrong_num_bands, kBlockSize, correct_num_bands, kSubFrameLength); + } +} + +TEST(BlockFramer, + WrongNumberOfBandsInSubFrameForInsertBlockAndExtractSubFrame) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_bands, kBlockSize, wrong_num_bands, kSubFrameLength); + } +} + +TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlockAndExtractSubFrame) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_bands, kBlockSize - 1, correct_num_bands, + kSubFrameLength); + } +} + +TEST(BlockFramer, + WrongNumberOfSamplesInSubFrameForInsertBlockAndExtractSubFrame) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertAndExtractParametersTest(rate, correct_num_bands, + kBlockSize, correct_num_bands, + kSubFrameLength - 1); + } +} + +TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertParameterTest(rate, wrong_num_bands, kBlockSize); + } +} + +TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertParameterTest(rate, correct_num_bands, kBlockSize - 1); + } +} + +TEST(BlockFramer, WrongNumberOfPreceedingApiCallsForInsertBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + for (size_t num_calls = 0; num_calls < 4; ++num_calls) { + std::ostringstream ss; + ss << "Sample rate: " << rate; + ss << ", Num preceeding InsertBlockAndExtractSubFrame calls: " + << num_calls; + + SCOPED_TRACE(ss.str()); + RunWronglyInsertOrderTest(rate, num_calls); + } + } +} + +// Verifiers that the verification for null sub_frame pointer works. +TEST(BlockFramer, NullSubFrameParameter) { + EXPECT_DEATH(BlockFramer(1).InsertBlockAndExtractSubFrame( + std::vector>( + 1, std::vector(kBlockSize, 0.f)), + nullptr), + ""); +} + +#endif + +TEST(BlockFramer, FrameBitexactness) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunFramerTest(rate); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.cc new file mode 100644 index 0000000000..f0b963087c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/block_processor.h" + +#include "api/optional.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block_processor_metrics.h" +#include "modules/audio_processing/aec3/echo_path_variability.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +enum class BlockProcessorApiCall { kCapture, kRender }; + +class BlockProcessorImpl final : public BlockProcessor { + public: + BlockProcessorImpl(int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover); + + ~BlockProcessorImpl() override; + + void ProcessCapture(bool echo_path_gain_change, + bool capture_signal_saturation, + std::vector>* capture_block) override; + + void BufferRender(const std::vector>& block) override; + + void UpdateEchoLeakageStatus(bool leakage_detected) override; + + void GetMetrics(EchoControl::Metrics* metrics) const override; + + private: + static int instance_count_; + bool no_capture_data_received_ = true; + bool no_render_data_received_ = true; + std::unique_ptr data_dumper_; + const size_t sample_rate_hz_; + std::unique_ptr render_buffer_; + std::unique_ptr delay_controller_; + std::unique_ptr echo_remover_; + BlockProcessorMetrics metrics_; + bool render_buffer_overrun_occurred_ = false; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl); +}; + +int BlockProcessorImpl::instance_count_ = 0; + +BlockProcessorImpl::BlockProcessorImpl( + int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover) + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + sample_rate_hz_(sample_rate_hz), + render_buffer_(std::move(render_buffer)), + delay_controller_(std::move(delay_controller)), + echo_remover_(std::move(echo_remover)) { + RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); +} + +BlockProcessorImpl::~BlockProcessorImpl() = default; + +void BlockProcessorImpl::ProcessCapture( + bool echo_path_gain_change, + bool capture_signal_saturation, + std::vector>* capture_block) { + RTC_DCHECK(capture_block); + RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size()); + RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size()); + data_dumper_->DumpRaw("aec3_processblock_call_order", + static_cast(BlockProcessorApiCall::kCapture)); + data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize, + &(*capture_block)[0][0], + LowestBandRate(sample_rate_hz_), 1); + + // Do not start processing until render data has been buffered as that will + // cause the buffers to be wrongly aligned. + no_capture_data_received_ = false; + if (no_render_data_received_) { + return; + } + + data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, + &(*capture_block)[0][0], + LowestBandRate(sample_rate_hz_), 1); + + bool render_buffer_underrun = false; + if (render_buffer_overrun_occurred_) { + // Reset the render buffers and the alignment functionality when there has + // been a render buffer overrun as the buffer alignment may be noncausal. + delay_controller_->Reset(); + render_buffer_->Reset(); + RTC_LOG(LS_WARNING) << "Reset due to detected render buffer overrun."; + } + + // Update the render buffers with new render data, filling the buffers with + // empty blocks when there is no render data available. + render_buffer_underrun = !render_buffer_->UpdateBuffers(); + if (render_buffer_underrun) { + RTC_LOG(LS_WARNING) << "Render API jitter buffer underrun."; + } + + // Compute and and apply the render delay required to achieve proper signal + // alignment. + const size_t old_delay = render_buffer_->Delay(); + const size_t new_delay = delay_controller_->GetDelay( + render_buffer_->GetDownsampledRenderBuffer(), (*capture_block)[0]); + + bool delay_change; + if (new_delay >= kMinEchoPathDelayBlocks) { + render_buffer_->SetDelay(new_delay); + const size_t achieved_delay = render_buffer_->Delay(); + delay_change = old_delay != achieved_delay || old_delay != new_delay || + render_buffer_overrun_occurred_; + + // Inform the delay controller of the actually set delay to allow it to + // properly react to a non-feasible delay. + delay_controller_->SetDelay(achieved_delay); + } else { + delay_controller_->Reset(); + render_buffer_->Reset(); + delay_change = true; + RTC_LOG(LS_WARNING) << "Reset due to noncausal delay."; + } + + // Remove the echo from the capture signal. + echo_remover_->ProcessCapture( + delay_controller_->AlignmentHeadroomSamples(), + EchoPathVariability(echo_path_gain_change, delay_change), + capture_signal_saturation, render_buffer_->GetRenderBuffer(), + capture_block); + + // Update the metrics. + metrics_.UpdateCapture(render_buffer_underrun); + + render_buffer_overrun_occurred_ = false; +} + +void BlockProcessorImpl::BufferRender( + const std::vector>& block) { + RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size()); + RTC_DCHECK_EQ(kBlockSize, block[0].size()); + data_dumper_->DumpRaw("aec3_processblock_call_order", + static_cast(BlockProcessorApiCall::kRender)); + data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize, + &block[0][0], LowestBandRate(sample_rate_hz_), 1); + + no_render_data_received_ = false; + + // Do not start buffer render data until capture data has been received as + // that data may give a false alignment. + if (no_capture_data_received_) { + return; + } + + data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize, + &block[0][0], LowestBandRate(sample_rate_hz_), 1); + + // Buffer the render data. + render_buffer_overrun_occurred_ = !render_buffer_->Insert(block); + + // Update the metrics. + metrics_.UpdateRender(render_buffer_overrun_occurred_); +} + +void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { + echo_remover_->UpdateEchoLeakageStatus(leakage_detected); +} + +void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { + echo_remover_->GetMetrics(metrics); + const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4; + metrics->delay_ms = static_cast(render_buffer_->Delay()) * block_size_ms; +} + +} // namespace + +BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, + int sample_rate_hz) { + std::unique_ptr render_buffer(RenderDelayBuffer::Create( + NumBandsForRate(sample_rate_hz), config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + std::unique_ptr delay_controller( + RenderDelayController::Create(config, sample_rate_hz)); + std::unique_ptr echo_remover( + EchoRemover::Create(config, sample_rate_hz)); + return Create(config, sample_rate_hz, std::move(render_buffer), + std::move(delay_controller), std::move(echo_remover)); +} + +BlockProcessor* BlockProcessor::Create( + const EchoCanceller3Config& config, + int sample_rate_hz, + std::unique_ptr render_buffer) { + std::unique_ptr delay_controller( + RenderDelayController::Create(config, sample_rate_hz)); + std::unique_ptr echo_remover( + EchoRemover::Create(config, sample_rate_hz)); + return Create(config, sample_rate_hz, std::move(render_buffer), + std::move(delay_controller), std::move(echo_remover)); +} + +BlockProcessor* BlockProcessor::Create( + const EchoCanceller3Config& config, + int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover) { + return new BlockProcessorImpl(sample_rate_hz, std::move(render_buffer), + std::move(delay_controller), + std::move(echo_remover)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.h new file mode 100644 index 0000000000..8687bc2282 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_H_ + +#include +#include + +#include "modules/audio_processing/aec3/echo_remover.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/aec3/render_delay_controller.h" + +namespace webrtc { + +// Class for performing echo cancellation on 64 sample blocks of audio data. +class BlockProcessor { + public: + static BlockProcessor* Create(const EchoCanceller3Config& config, + int sample_rate_hz); + // Only used for testing purposes. + static BlockProcessor* Create( + const EchoCanceller3Config& config, + int sample_rate_hz, + std::unique_ptr render_buffer); + static BlockProcessor* Create( + const EchoCanceller3Config& config, + int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover); + + virtual ~BlockProcessor() = default; + + // Get current metrics. + virtual void GetMetrics(EchoControl::Metrics* metrics) const = 0; + + // Processes a block of capture data. + virtual void ProcessCapture( + bool echo_path_gain_change, + bool capture_signal_saturation, + std::vector>* capture_block) = 0; + + // Buffers a block of render data supplied by a FrameBlocker object. + virtual void BufferRender( + const std::vector>& render_block) = 0; + + // Reports whether echo leakage has been detected in the echo canceller + // output. + virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.cc new file mode 100644 index 0000000000..c8bdda7389 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/block_processor_metrics.h" + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +enum class RenderUnderrunCategory { + kNone, + kFew, + kSeveral, + kMany, + kConstant, + kNumCategories +}; + +enum class RenderOverrunCategory { + kNone, + kFew, + kSeveral, + kMany, + kConstant, + kNumCategories +}; + +} // namespace + +void BlockProcessorMetrics::UpdateCapture(bool underrun) { + ++capture_block_counter_; + if (underrun) { + ++render_buffer_underruns_; + } + + if (capture_block_counter_ == kMetricsReportingIntervalBlocks) { + metrics_reported_ = true; + + RenderUnderrunCategory underrun_category; + if (render_buffer_underruns_ == 0) { + underrun_category = RenderUnderrunCategory::kNone; + } else if (render_buffer_underruns_ > (capture_block_counter_ >> 1)) { + underrun_category = RenderUnderrunCategory::kConstant; + } else if (render_buffer_underruns_ > 100) { + underrun_category = RenderUnderrunCategory::kMany; + } else if (render_buffer_underruns_ > 10) { + underrun_category = RenderUnderrunCategory::kSeveral; + } else { + underrun_category = RenderUnderrunCategory::kFew; + } + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.EchoCanceller.RenderUnderruns", + static_cast(underrun_category), + static_cast(RenderUnderrunCategory::kNumCategories)); + + RenderOverrunCategory overrun_category; + if (render_buffer_overruns_ == 0) { + overrun_category = RenderOverrunCategory::kNone; + } else if (render_buffer_overruns_ > (buffer_render_calls_ >> 1)) { + overrun_category = RenderOverrunCategory::kConstant; + } else if (render_buffer_overruns_ > 100) { + overrun_category = RenderOverrunCategory::kMany; + } else if (render_buffer_overruns_ > 10) { + overrun_category = RenderOverrunCategory::kSeveral; + } else { + overrun_category = RenderOverrunCategory::kFew; + } + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.EchoCanceller.RenderOverruns", + static_cast(overrun_category), + static_cast(RenderOverrunCategory::kNumCategories)); + + ResetMetrics(); + capture_block_counter_ = 0; + } else { + metrics_reported_ = false; + } +} + +void BlockProcessorMetrics::UpdateRender(bool overrun) { + ++buffer_render_calls_; + if (overrun) { + ++render_buffer_overruns_; + } +} + +void BlockProcessorMetrics::ResetMetrics() { + render_buffer_underruns_ = 0; + render_buffer_overruns_ = 0; + buffer_render_calls_ = 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.h new file mode 100644 index 0000000000..9b437c0d13 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_METRICS_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_METRICS_H_ + +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Handles the reporting of metrics for the block_processor. +class BlockProcessorMetrics { + public: + BlockProcessorMetrics() = default; + + // Updates the metric with new capture data. + void UpdateCapture(bool underrun); + + // Updates the metric with new render data. + void UpdateRender(bool overrun); + + // Returns true if the metrics have just been reported, otherwise false. + bool MetricsReported() { return metrics_reported_; } + + private: + // Resets the metrics. + void ResetMetrics(); + + int capture_block_counter_ = 0; + bool metrics_reported_ = false; + int render_buffer_underruns_ = 0; + int render_buffer_overruns_ = 0; + int buffer_render_calls_ = 0; + + RTC_DISALLOW_COPY_AND_ASSIGN(BlockProcessorMetrics); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_METRICS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics_unittest.cc new file mode 100644 index 0000000000..7ce8573efa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics_unittest.cc @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block_processor_metrics.h" + +#include "test/gtest.h" + +namespace webrtc { + +// Verify the general functionality of BlockProcessorMetrics. +TEST(BlockProcessorMetrics, NormalUsage) { + BlockProcessorMetrics metrics; + + for (int j = 0; j < 3; ++j) { + for (int k = 0; k < kMetricsReportingIntervalBlocks - 1; ++k) { + metrics.UpdateRender(false); + metrics.UpdateRender(false); + metrics.UpdateCapture(false); + EXPECT_FALSE(metrics.MetricsReported()); + } + metrics.UpdateCapture(false); + EXPECT_TRUE(metrics.MetricsReported()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc new file mode 100644 index 0000000000..18d1f65ce3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/block_processor.h" + +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/mock/mock_echo_remover.h" +#include "modules/audio_processing/aec3/mock/mock_render_delay_buffer.h" +#include "modules/audio_processing/aec3/mock/mock_render_delay_controller.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/checks.h" +#include "rtc_base/random.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using testing::AtLeast; +using testing::Return; +using testing::StrictMock; +using testing::_; + +// Verifies that the basic BlockProcessor functionality works and that the API +// methods are callable. +void RunBasicSetupAndApiCallTest(int sample_rate_hz) { + std::unique_ptr block_processor( + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); + std::vector> block(NumBandsForRate(sample_rate_hz), + std::vector(kBlockSize, 0.f)); + + block_processor->BufferRender(block); + block_processor->ProcessCapture(false, false, &block); + block_processor->UpdateEchoLeakageStatus(false); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +void RunRenderBlockSizeVerificationTest(int sample_rate_hz) { + std::unique_ptr block_processor( + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); + std::vector> block( + NumBandsForRate(sample_rate_hz), std::vector(kBlockSize - 1, 0.f)); + + EXPECT_DEATH(block_processor->BufferRender(block), ""); +} + +void RunCaptureBlockSizeVerificationTest(int sample_rate_hz) { + std::unique_ptr block_processor( + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); + std::vector> block( + NumBandsForRate(sample_rate_hz), std::vector(kBlockSize - 1, 0.f)); + + EXPECT_DEATH(block_processor->ProcessCapture(false, false, &block), ""); +} + +void RunRenderNumBandsVerificationTest(int sample_rate_hz) { + const size_t wrong_num_bands = NumBandsForRate(sample_rate_hz) < 3 + ? NumBandsForRate(sample_rate_hz) + 1 + : 1; + std::unique_ptr block_processor( + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); + std::vector> block(wrong_num_bands, + std::vector(kBlockSize, 0.f)); + + EXPECT_DEATH(block_processor->BufferRender(block), ""); +} + +void RunCaptureNumBandsVerificationTest(int sample_rate_hz) { + const size_t wrong_num_bands = NumBandsForRate(sample_rate_hz) < 3 + ? NumBandsForRate(sample_rate_hz) + 1 + : 1; + std::unique_ptr block_processor( + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); + std::vector> block(wrong_num_bands, + std::vector(kBlockSize, 0.f)); + + EXPECT_DEATH(block_processor->ProcessCapture(false, false, &block), ""); +} +#endif + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +} // namespace + +// Verifies that the delay controller functionality is properly integrated with +// the render delay buffer inside block processor. +// TODO(peah): Activate the unittest once the required code has been landed. +TEST(BlockProcessor, DISABLED_DelayControllerIntegration) { + constexpr size_t kNumBlocks = 310; + constexpr size_t kDelayInSamples = 640; + constexpr size_t kDelayHeadroom = 1; + constexpr size_t kDelayInBlocks = + kDelayInSamples / kBlockSize - kDelayHeadroom; + Random random_generator(42U); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr> + render_delay_buffer_mock( + new StrictMock(rate)); + EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, IsBlockAvailable()) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, SetDelay(kDelayInBlocks)) + .Times(AtLeast(1)); + EXPECT_CALL(*render_delay_buffer_mock, MaxDelay()).WillOnce(Return(30)); + EXPECT_CALL(*render_delay_buffer_mock, Delay()) + .Times(kNumBlocks + 1) + .WillRepeatedly(Return(0)); + std::unique_ptr block_processor(BlockProcessor::Create( + EchoCanceller3Config(), rate, std::move(render_delay_buffer_mock))); + + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::vector> capture_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + DelayBuffer signal_delay_buffer(kDelayInSamples); + for (size_t k = 0; k < kNumBlocks; ++k) { + RandomizeSampleVector(&random_generator, render_block[0]); + signal_delay_buffer.Delay(render_block[0], capture_block[0]); + block_processor->BufferRender(render_block); + block_processor->ProcessCapture(false, false, &capture_block); + } + } +} + +// Verifies that BlockProcessor submodules are called in a proper manner. +TEST(BlockProcessor, DISABLED_SubmoduleIntegration) { + constexpr size_t kNumBlocks = 310; + Random random_generator(42U); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr> + render_delay_buffer_mock( + new StrictMock(rate)); + std::unique_ptr< + testing::StrictMock> + render_delay_controller_mock( + new StrictMock()); + std::unique_ptr> + echo_remover_mock(new StrictMock()); + + EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) + .Times(kNumBlocks - 1) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, IsBlockAvailable()) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, UpdateBuffers()).Times(kNumBlocks); + EXPECT_CALL(*render_delay_buffer_mock, SetDelay(9)).Times(AtLeast(1)); + EXPECT_CALL(*render_delay_buffer_mock, Delay()) + .Times(kNumBlocks) + .WillRepeatedly(Return(0)); + EXPECT_CALL(*render_delay_controller_mock, GetDelay(_, _)) + .Times(kNumBlocks) + .WillRepeatedly(Return(9)); + EXPECT_CALL(*render_delay_controller_mock, AlignmentHeadroomSamples()) + .Times(kNumBlocks); + EXPECT_CALL(*echo_remover_mock, ProcessCapture(_, _, _, _, _)) + .Times(kNumBlocks); + EXPECT_CALL(*echo_remover_mock, UpdateEchoLeakageStatus(_)) + .Times(kNumBlocks); + + std::unique_ptr block_processor(BlockProcessor::Create( + EchoCanceller3Config(), rate, std::move(render_delay_buffer_mock), + std::move(render_delay_controller_mock), std::move(echo_remover_mock))); + + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::vector> capture_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + DelayBuffer signal_delay_buffer(640); + for (size_t k = 0; k < kNumBlocks; ++k) { + RandomizeSampleVector(&random_generator, render_block[0]); + signal_delay_buffer.Delay(render_block[0], capture_block[0]); + block_processor->BufferRender(render_block); + block_processor->ProcessCapture(false, false, &capture_block); + block_processor->UpdateEchoLeakageStatus(false); + } + } +} + +TEST(BlockProcessor, BasicSetupAndApiCalls) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunBasicSetupAndApiCallTest(rate); + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// TODO(gustaf): Re-enable the test once the issue with memory leaks during +// DEATH tests on test bots has been fixed. +TEST(BlockProcessor, DISABLED_VerifyRenderBlockSizeCheck) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunRenderBlockSizeVerificationTest(rate); + } +} + +TEST(BlockProcessor, VerifyCaptureBlockSizeCheck) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunCaptureBlockSizeVerificationTest(rate); + } +} + +TEST(BlockProcessor, VerifyRenderNumBandsCheck) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunRenderNumBandsVerificationTest(rate); + } +} + +// TODO(peah): Verify the check for correct number of bands in the capture +// signal. +TEST(BlockProcessor, VerifyCaptureNumBandsCheck) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunCaptureNumBandsVerificationTest(rate); + } +} + +// Verifiers that the verification for null ProcessCapture input works. +TEST(BlockProcessor, NullProcessCaptureParameter) { + EXPECT_DEATH(std::unique_ptr( + BlockProcessor::Create(EchoCanceller3Config(), 8000)) + ->ProcessCapture(false, false, nullptr), + ""); +} + +// Verifies the check for correct sample rate. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(BlockProcessor, DISABLED_WrongSampleRate) { + EXPECT_DEATH(std::unique_ptr( + BlockProcessor::Create(EchoCanceller3Config(), 8001)), + ""); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.cc new file mode 100644 index 0000000000..9a472f516f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/cascaded_biquad_filter.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +CascadedBiQuadFilter::CascadedBiQuadFilter( + const CascadedBiQuadFilter::BiQuadCoefficients& coefficients, + size_t num_biquads) + : biquad_states_(num_biquads), coefficients_(coefficients) {} + +CascadedBiQuadFilter::~CascadedBiQuadFilter() = default; + +void CascadedBiQuadFilter::Process(rtc::ArrayView x, + rtc::ArrayView y) { + ApplyBiQuad(x, y, &biquad_states_[0]); + for (size_t k = 1; k < biquad_states_.size(); ++k) { + ApplyBiQuad(y, y, &biquad_states_[k]); + } +} + +void CascadedBiQuadFilter::Process(rtc::ArrayView y) { + for (auto& biquad : biquad_states_) { + ApplyBiQuad(y, y, &biquad); + } +} + +void CascadedBiQuadFilter::ApplyBiQuad( + rtc::ArrayView x, + rtc::ArrayView y, + CascadedBiQuadFilter::BiQuadState* biquad_state) { + RTC_DCHECK_EQ(x.size(), y.size()); + RTC_DCHECK(biquad_state); + const auto* c_b = coefficients_.b; + const auto* c_a = coefficients_.a; + auto* m_x = biquad_state->x; + auto* m_y = biquad_state->y; + for (size_t k = 0; k < x.size(); ++k) { + const float tmp = x[k]; + y[k] = c_b[0] * tmp + c_b[1] * m_x[0] + c_b[2] * m_x[1] - c_a[0] * m_y[0] - + c_a[1] * m_y[1]; + m_x[1] = m_x[0]; + m_x[0] = tmp; + m_y[1] = m_y[0]; + m_y[0] = y[k]; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.h new file mode 100644 index 0000000000..aea889ab53 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_CASCADED_BIQUAD_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_CASCADED_BIQUAD_FILTER_H_ + +#include + +#include "api/array_view.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Applies a number of identical biquads in a cascaded manner. The filter +// implementation is direct form 1. +class CascadedBiQuadFilter { + public: + struct BiQuadState { + BiQuadState() : x(), y() {} + float x[2]; + float y[2]; + }; + + struct BiQuadCoefficients { + float b[3]; + float a[2]; + }; + + CascadedBiQuadFilter( + const CascadedBiQuadFilter::BiQuadCoefficients& coefficients, + size_t num_biquads); + ~CascadedBiQuadFilter(); + // Applies the biquads on the values in x in order to form the output in y. + void Process(rtc::ArrayView x, rtc::ArrayView y); + // Applies the biquads on the values in y in an in-place manner. + void Process(rtc::ArrayView y); + + private: + void ApplyBiQuad(rtc::ArrayView x, + rtc::ArrayView y, + CascadedBiQuadFilter::BiQuadState* biquad_state); + + std::vector biquad_states_; + const BiQuadCoefficients coefficients_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CascadedBiQuadFilter); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_CASCADED_BIQUAD_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter_unittest.cc new file mode 100644 index 0000000000..fcb77e1f6e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter_unittest.cc @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/cascaded_biquad_filter.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +// Coefficients for a second order Butterworth high-pass filter with cutoff +// frequency 100 Hz. +const CascadedBiQuadFilter::BiQuadCoefficients kHighPassFilterCoefficients = { + {0.97261f, -1.94523f, 0.97261f}, + {-1.94448f, 0.94598f}}; + +const CascadedBiQuadFilter::BiQuadCoefficients kTransparentCoefficients = { + {1.f, 0.f, 0.f}, + {0.f, 0.f}}; + +const CascadedBiQuadFilter::BiQuadCoefficients kBlockingCoefficients = { + {0.f, 0.f, 0.f}, + {0.f, 0.f}}; + +std::vector CreateInputWithIncreasingValues(size_t vector_length) { + std::vector v(vector_length); + for (size_t k = 0; k < v.size(); ++k) { + v[k] = k; + } + return v; +} + +} // namespace + +// Verifies that the filter applies an effect which removes the input signal. +// The test also verifies that the in-place Process API call works as intended. +TEST(CascadedBiquadFilter, BlockingConfiguration) { + std::vector values = CreateInputWithIncreasingValues(1000); + + CascadedBiQuadFilter filter(kBlockingCoefficients, 1); + filter.Process(values); + + EXPECT_EQ(std::vector(1000, 0.f), values); +} + +// Verifies that the filter is able to form a zero-mean output from a +// non-zeromean input signal when coefficients for a high-pass filter are +// applied. The test also verifies that the filter works with multiple biquads. +TEST(CascadedBiquadFilter, HighPassConfiguration) { + std::vector values(1000); + for (size_t k = 0; k < values.size(); ++k) { + values[k] = 1.f; + } + + CascadedBiQuadFilter filter(kHighPassFilterCoefficients, 2); + filter.Process(values); + + for (size_t k = values.size() / 2; k < values.size(); ++k) { + EXPECT_NEAR(0.f, values[k], 1e-4); + } +} + +// Verifies that the filter is able to produce a transparent effect with no +// impact on the data when the proper coefficients are applied. The test also +// verifies that the non-in-place Process API call works as intended. +TEST(CascadedBiquadFilter, TransparentConfiguration) { + const std::vector input = CreateInputWithIncreasingValues(1000); + std::vector output(input.size()); + + CascadedBiQuadFilter filter(kTransparentCoefficients, 1); + filter.Process(input, output); + + EXPECT_EQ(input, output); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Verifies that the check of the lengths for the input and output works for the +// non-in-place call. +TEST(CascadedBiquadFilter, InputSizeCheckVerification) { + const std::vector input = CreateInputWithIncreasingValues(10); + std::vector output(input.size() - 1); + + CascadedBiQuadFilter filter(kTransparentCoefficients, 1); + EXPECT_DEATH(filter.Process(input, output), ""); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.cc new file mode 100644 index 0000000000..dab40a9d7e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.cc @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/comfort_noise_generator.h" + +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include +#include +#include +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +namespace webrtc { + +namespace { + +// Creates an array of uniformly distributed variables. +void TableRandomValue(int16_t* vector, int16_t vector_length, uint32_t* seed) { + for (int i = 0; i < vector_length; i++) { + seed[0] = (seed[0] * ((int32_t)69069) + 1) & (0x80000000 - 1); + vector[i] = (int16_t)(seed[0] >> 16); + } +} + +} // namespace + +namespace aec3 { + +#if defined(WEBRTC_ARCH_X86_FAMILY) + +void EstimateComfortNoise_SSE2(const std::array& N2, + uint32_t* seed, + FftData* lower_band_noise, + FftData* upper_band_noise) { + FftData* N_low = lower_band_noise; + FftData* N_high = upper_band_noise; + + // Compute square root spectrum. + std::array N; + for (size_t k = 0; k < kFftLengthBy2; k += 4) { + __m128 v = _mm_loadu_ps(&N2[k]); + v = _mm_sqrt_ps(v); + _mm_storeu_ps(&N[k], v); + } + + N[kFftLengthBy2] = sqrtf(N2[kFftLengthBy2]); + + // Compute the noise level for the upper bands. + constexpr float kOneByNumBands = 1.f / (kFftLengthBy2Plus1 / 2 + 1); + constexpr int kFftLengthBy2Plus1By2 = kFftLengthBy2Plus1 / 2; + const float high_band_noise_level = + std::accumulate(N.begin() + kFftLengthBy2Plus1By2, N.end(), 0.f) * + kOneByNumBands; + + // Generate complex noise. + std::array random_values_int; + TableRandomValue(random_values_int.data(), random_values_int.size(), seed); + + std::array sin; + std::array cos; + constexpr float kScale = 6.28318530717959f / 32768.0f; + std::transform(random_values_int.begin(), random_values_int.end(), + sin.begin(), [&](int16_t a) { return -sinf(kScale * a); }); + std::transform(random_values_int.begin(), random_values_int.end(), + cos.begin(), [&](int16_t a) { return cosf(kScale * a); }); + + // Form low-frequency noise via spectral shaping. + N_low->re[0] = N_low->re[kFftLengthBy2] = N_high->re[0] = + N_high->re[kFftLengthBy2] = 0.f; + std::transform(cos.begin(), cos.end(), N.begin() + 1, N_low->re.begin() + 1, + std::multiplies()); + std::transform(sin.begin(), sin.end(), N.begin() + 1, N_low->im.begin() + 1, + std::multiplies()); + + // Form the high-frequency noise via simple levelling. + std::transform(cos.begin(), cos.end(), N_high->re.begin() + 1, + [&](float a) { return high_band_noise_level * a; }); + std::transform(sin.begin(), sin.end(), N_high->im.begin() + 1, + [&](float a) { return high_band_noise_level * a; }); +} + +#endif + +void EstimateComfortNoise(const std::array& N2, + uint32_t* seed, + FftData* lower_band_noise, + FftData* upper_band_noise) { + FftData* N_low = lower_band_noise; + FftData* N_high = upper_band_noise; + + // Compute square root spectrum. + std::array N; + std::transform(N2.begin(), N2.end(), N.begin(), + [](float a) { return sqrtf(a); }); + + // Compute the noise level for the upper bands. + constexpr float kOneByNumBands = 1.f / (kFftLengthBy2Plus1 / 2 + 1); + constexpr int kFftLengthBy2Plus1By2 = kFftLengthBy2Plus1 / 2; + const float high_band_noise_level = + std::accumulate(N.begin() + kFftLengthBy2Plus1By2, N.end(), 0.f) * + kOneByNumBands; + + // Generate complex noise. + std::array random_values_int; + TableRandomValue(random_values_int.data(), random_values_int.size(), seed); + + std::array sin; + std::array cos; + constexpr float kScale = 6.28318530717959f / 32768.0f; + std::transform(random_values_int.begin(), random_values_int.end(), + sin.begin(), [&](int16_t a) { return -sinf(kScale * a); }); + std::transform(random_values_int.begin(), random_values_int.end(), + cos.begin(), [&](int16_t a) { return cosf(kScale * a); }); + + // Form low-frequency noise via spectral shaping. + N_low->re[0] = N_low->re[kFftLengthBy2] = N_high->re[0] = + N_high->re[kFftLengthBy2] = 0.f; + std::transform(cos.begin(), cos.end(), N.begin() + 1, N_low->re.begin() + 1, + std::multiplies()); + std::transform(sin.begin(), sin.end(), N.begin() + 1, N_low->im.begin() + 1, + std::multiplies()); + + // Form the high-frequency noise via simple levelling. + std::transform(cos.begin(), cos.end(), N_high->re.begin() + 1, + [&](float a) { return high_band_noise_level * a; }); + std::transform(sin.begin(), sin.end(), N_high->im.begin() + 1, + [&](float a) { return high_band_noise_level * a; }); +} + +} // namespace aec3 + +ComfortNoiseGenerator::ComfortNoiseGenerator(Aec3Optimization optimization) + : optimization_(optimization), + seed_(42), + N2_initial_(new std::array()) { + N2_initial_->fill(0.f); + Y2_smoothed_.fill(0.f); + N2_.fill(1.0e6f); +} + +ComfortNoiseGenerator::~ComfortNoiseGenerator() = default; + +void ComfortNoiseGenerator::Compute( + const AecState& aec_state, + const std::array& capture_spectrum, + FftData* lower_band_noise, + FftData* upper_band_noise) { + RTC_DCHECK(lower_band_noise); + RTC_DCHECK(upper_band_noise); + const auto& Y2 = capture_spectrum; + + if (!aec_state.SaturatedCapture()) { + // Smooth Y2. + std::transform(Y2_smoothed_.begin(), Y2_smoothed_.end(), Y2.begin(), + Y2_smoothed_.begin(), + [](float a, float b) { return a + 0.1f * (b - a); }); + + if (N2_counter_ > 50) { + // Update N2 from Y2_smoothed. + std::transform(N2_.begin(), N2_.end(), Y2_smoothed_.begin(), N2_.begin(), + [](float a, float b) { + return b < a ? (0.9f * b + 0.1f * a) * 1.0002f + : a * 1.0002f; + }); + } + + if (N2_initial_) { + if (++N2_counter_ == 1000) { + N2_initial_.reset(); + } else { + // Compute the N2_initial from N2. + std::transform( + N2_.begin(), N2_.end(), N2_initial_->begin(), N2_initial_->begin(), + [](float a, float b) { return a > b ? b + 0.001f * (a - b) : a; }); + } + } + } + + // Limit the noise to a floor of -96 dBFS. + constexpr float kNoiseFloor = 440.f; + for (auto& n : N2_) { + n = std::max(n, kNoiseFloor); + } + if (N2_initial_) { + for (auto& n : *N2_initial_) { + n = std::max(n, kNoiseFloor); + } + } + + // Choose N2 estimate to use. + const std::array& N2 = + N2_initial_ ? *N2_initial_ : N2_; + + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: + aec3::EstimateComfortNoise_SSE2(N2, &seed_, lower_band_noise, + upper_band_noise); + break; +#endif + default: + aec3::EstimateComfortNoise(N2, &seed_, lower_band_noise, + upper_band_noise); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h new file mode 100644 index 0000000000..2d998beb94 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_COMFORT_NOISE_GENERATOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_COMFORT_NOISE_GENERATOR_H_ + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace aec3 { +#if defined(WEBRTC_ARCH_X86_FAMILY) + +void EstimateComfortNoise_SSE2(const std::array& N2, + uint32_t* seed, + FftData* lower_band_noise, + FftData* upper_band_noise); +#endif +void EstimateComfortNoise(const std::array& N2, + uint32_t* seed, + FftData* lower_band_noise, + FftData* upper_band_noise); + +} // namespace aec3 + +// Generates the comfort noise. +class ComfortNoiseGenerator { + public: + explicit ComfortNoiseGenerator(Aec3Optimization optimization); + ~ComfortNoiseGenerator(); + + // Computes the comfort noise. + void Compute(const AecState& aec_state, + const std::array& capture_spectrum, + FftData* lower_band_noise, + FftData* upper_band_noise); + + // Returns the estimate of the background noise spectrum. + const std::array& NoiseSpectrum() const { + return N2_; + } + + private: + const Aec3Optimization optimization_; + uint32_t seed_; + std::unique_ptr> N2_initial_; + std::array Y2_smoothed_; + std::array N2_; + int N2_counter_ = 0; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(ComfortNoiseGenerator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_COMFORT_NOISE_GENERATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc new file mode 100644 index 0000000000..46da3eca46 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/comfort_noise_generator.h" + +#include +#include + +#include "rtc_base/random.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace aec3 { +namespace { + +float Power(const FftData& N) { + std::array N2; + N.Spectrum(Aec3Optimization::kNone, &N2); + return std::accumulate(N2.begin(), N2.end(), 0.f) / N2.size(); +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +TEST(ComfortNoiseGenerator, NullLowerBandNoise) { + std::array N2; + FftData noise; + EXPECT_DEATH( + ComfortNoiseGenerator(DetectOptimization()) + .Compute(AecState(EchoCanceller3Config{}), N2, nullptr, &noise), + ""); +} + +TEST(ComfortNoiseGenerator, NullUpperBandNoise) { + std::array N2; + FftData noise; + EXPECT_DEATH( + ComfortNoiseGenerator(DetectOptimization()) + .Compute(AecState(EchoCanceller3Config{}), N2, &noise, nullptr), + ""); +} + +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Verifies that the optimized methods are bitexact to their reference +// counterparts. +TEST(ComfortNoiseGenerator, TestOptimizations) { + if (WebRtc_GetCPUInfo(kSSE2) != 0) { + Random random_generator(42U); + uint32_t seed = 42; + uint32_t seed_SSE2 = 42; + std::array N2; + FftData lower_band_noise; + FftData upper_band_noise; + FftData lower_band_noise_SSE2; + FftData upper_band_noise_SSE2; + for (int k = 0; k < 10; ++k) { + for (size_t j = 0; j < N2.size(); ++j) { + N2[j] = random_generator.Rand() * 1000.f; + } + + EstimateComfortNoise(N2, &seed, &lower_band_noise, &upper_band_noise); + EstimateComfortNoise_SSE2(N2, &seed_SSE2, &lower_band_noise_SSE2, + &upper_band_noise_SSE2); + for (size_t j = 0; j < lower_band_noise.re.size(); ++j) { + EXPECT_NEAR(lower_band_noise.re[j], lower_band_noise_SSE2.re[j], + 0.00001f); + EXPECT_NEAR(upper_band_noise.re[j], upper_band_noise_SSE2.re[j], + 0.00001f); + } + for (size_t j = 1; j < lower_band_noise.re.size() - 1; ++j) { + EXPECT_NEAR(lower_band_noise.im[j], lower_band_noise_SSE2.im[j], + 0.00001f); + EXPECT_NEAR(upper_band_noise.im[j], upper_band_noise_SSE2.im[j], + 0.00001f); + } + } + } +} + +#endif + +TEST(ComfortNoiseGenerator, CorrectLevel) { + ComfortNoiseGenerator cng(DetectOptimization()); + AecState aec_state(EchoCanceller3Config{}); + + std::array N2; + N2.fill(1000.f * 1000.f); + + FftData n_lower; + FftData n_upper; + n_lower.re.fill(0.f); + n_lower.im.fill(0.f); + n_upper.re.fill(0.f); + n_upper.im.fill(0.f); + + // Ensure instantaneous updata to nonzero noise. + cng.Compute(aec_state, N2, &n_lower, &n_upper); + EXPECT_LT(0.f, Power(n_lower)); + EXPECT_LT(0.f, Power(n_upper)); + + for (int k = 0; k < 10000; ++k) { + cng.Compute(aec_state, N2, &n_lower, &n_upper); + } + EXPECT_NEAR(N2[0], Power(n_lower), N2[0] / 10.f); + EXPECT_NEAR(N2[0], Power(n_upper), N2[0] / 10.f); +} + +} // namespace aec3 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.cc new file mode 100644 index 0000000000..135a771a7c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/decimator.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// b, a = signal.butter(2, 3400/8000.0, 'lowpass', analog=False) which are the +// same as b, a = signal.butter(2, 1700/4000.0, 'lowpass', analog=False). +const CascadedBiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients2 = { + {0.22711796f, 0.45423593f, 0.22711796f}, + {-0.27666461f, 0.18513647f}}; +constexpr int kNumFilters2 = 3; + +// b, a = signal.butter(2, 1500/8000.0, 'lowpass', analog=False) which are the +// same as b, a = signal.butter(2, 75/4000.0, 'lowpass', analog=False). +const CascadedBiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients4 = { + {0.0179f, 0.0357f, 0.0179f}, + {-1.5879f, 0.6594f}}; +constexpr int kNumFilters4 = 3; + +// b, a = signal.butter(2, 800/8000.0, 'lowpass', analog=False) which are the +// same as b, a = signal.butter(2, 400/4000.0, 'lowpass', analog=False). +const CascadedBiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients8 = { + {0.02008337f, 0.04016673f, 0.02008337f}, + {-1.56101808f, 0.64135154f}}; +constexpr int kNumFilters8 = 4; + +} // namespace + +Decimator::Decimator(size_t down_sampling_factor) + : down_sampling_factor_(down_sampling_factor), + low_pass_filter_( + down_sampling_factor_ == 4 + ? kLowPassFilterCoefficients4 + : (down_sampling_factor_ == 8 ? kLowPassFilterCoefficients8 + : kLowPassFilterCoefficients2), + down_sampling_factor_ == 4 + ? kNumFilters4 + : (down_sampling_factor_ == 8 ? kNumFilters8 : kNumFilters2)) { + RTC_DCHECK(down_sampling_factor_ == 2 || down_sampling_factor_ == 4 || + down_sampling_factor_ == 8); +} + +void Decimator::Decimate(rtc::ArrayView in, + rtc::ArrayView out) { + RTC_DCHECK_EQ(kBlockSize, in.size()); + RTC_DCHECK_EQ(kBlockSize / down_sampling_factor_, out.size()); + std::array x; + + // Limit the frequency content of the signal to avoid aliasing. + low_pass_filter_.Process(in, x); + + // Downsample the signal. + for (size_t j = 0, k = 0; j < out.size(); ++j, k += down_sampling_factor_) { + RTC_DCHECK_GT(kBlockSize, k); + out[j] = x[k]; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.h new file mode 100644 index 0000000000..7418a26fb0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_DECIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_DECIMATOR_H_ + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/cascaded_biquad_filter.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Provides functionality for decimating a signal. +class Decimator { + public: + explicit Decimator(size_t down_sampling_factor); + + // Downsamples the signal. + void Decimate(rtc::ArrayView in, rtc::ArrayView out); + + private: + const size_t down_sampling_factor_; + CascadedBiQuadFilter low_pass_filter_; + + RTC_DISALLOW_COPY_AND_ASSIGN(Decimator); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_DECIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator_unittest.cc new file mode 100644 index 0000000000..e77a990f9c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator_unittest.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/decimator.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; +constexpr float kPi = 3.141592f; +constexpr size_t kNumStartupBlocks = 50; +constexpr size_t kNumBlocks = 1000; + +void ProduceDecimatedSinusoidalOutputPower(int sample_rate_hz, + size_t down_sampling_factor, + float sinusoidal_frequency_hz, + float* input_power, + float* output_power) { + float input[kBlockSize * kNumBlocks]; + const size_t sub_block_size = kBlockSize / down_sampling_factor; + + // Produce a sinusoid of the specified frequency. + for (size_t k = 0; k < kBlockSize * kNumBlocks; ++k) { + input[k] = + 32767.f * sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz); + } + + Decimator decimator(down_sampling_factor); + std::vector output(sub_block_size * kNumBlocks); + + for (size_t k = 0; k < kNumBlocks; ++k) { + std::vector sub_block(sub_block_size); + + decimator.Decimate( + rtc::ArrayView(&input[k * kBlockSize], kBlockSize), + sub_block); + + std::copy(sub_block.begin(), sub_block.end(), + output.begin() + k * sub_block_size); + } + + ASSERT_GT(kNumBlocks, kNumStartupBlocks); + rtc::ArrayView input_to_evaluate( + &input[kNumStartupBlocks * kBlockSize], + (kNumBlocks - kNumStartupBlocks) * kBlockSize); + rtc::ArrayView output_to_evaluate( + &output[kNumStartupBlocks * sub_block_size], + (kNumBlocks - kNumStartupBlocks) * sub_block_size); + *input_power = + std::inner_product(input_to_evaluate.begin(), input_to_evaluate.end(), + input_to_evaluate.begin(), 0.f) / + input_to_evaluate.size(); + *output_power = + std::inner_product(output_to_evaluate.begin(), output_to_evaluate.end(), + output_to_evaluate.begin(), 0.f) / + output_to_evaluate.size(); +} + +} // namespace + +// Verifies that there is little aliasing from upper frequencies in the +// downsampling. +TEST(Decimator, NoLeakageFromUpperFrequencies) { + float input_power; + float output_power; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto down_sampling_factor : kDownSamplingFactors) { + ProduceDebugText(rate); + ProduceDecimatedSinusoidalOutputPower(rate, down_sampling_factor, + 3.f / 8.f * rate, &input_power, + &output_power); + EXPECT_GT(0.0001f * input_power, output_power); + } + } +} + +// Verifies that the impact of low-frequency content is small during the +// downsampling. +TEST(Decimator, NoImpactOnLowerFrequencies) { + float input_power; + float output_power; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto down_sampling_factor : kDownSamplingFactors) { + ProduceDebugText(rate); + ProduceDecimatedSinusoidalOutputPower(rate, down_sampling_factor, 200.f, + &input_power, &output_power); + EXPECT_LT(0.7f * input_power, output_power); + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Verifies the check for the input size. +TEST(Decimator, WrongInputSize) { + Decimator decimator(4); + std::vector x(std::vector(kBlockSize - 1, 0.f)); + std::array x_downsampled; + EXPECT_DEATH(decimator.Decimate(x, x_downsampled), ""); +} + +// Verifies the check for non-null output parameter. +TEST(Decimator, NullOutput) { + Decimator decimator(4); + std::vector x(std::vector(kBlockSize, 0.f)); + EXPECT_DEATH(decimator.Decimate(x, nullptr), ""); +} + +// Verifies the check for the output size. +TEST(Decimator, WrongOutputSize) { + Decimator decimator(4); + std::vector x(std::vector(kBlockSize, 0.f)); + std::array x_downsampled; + EXPECT_DEATH(decimator.Decimate(x, x_downsampled), ""); +} + +// Verifies the check for the correct downsampling factor. +TEST(Decimator, CorrectDownSamplingFactor) { + EXPECT_DEATH(Decimator(3), ""); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.cc new file mode 100644 index 0000000000..efc733b182 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.cc @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/downsampled_render_buffer.h" + +namespace webrtc { + +DownsampledRenderBuffer::DownsampledRenderBuffer(size_t downsampled_buffer_size) + : buffer(downsampled_buffer_size, 0.f) {} + +DownsampledRenderBuffer::~DownsampledRenderBuffer() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.h new file mode 100644 index 0000000000..531852a0c9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_DOWNSAMPLED_RENDER_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_DOWNSAMPLED_RENDER_BUFFER_H_ + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +// Holds the circular buffer of the downsampled render data. +struct DownsampledRenderBuffer { + explicit DownsampledRenderBuffer(size_t downsampled_buffer_size); + ~DownsampledRenderBuffer(); + std::vector buffer; + int position = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_DOWNSAMPLED_RENDER_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.cc new file mode 100644 index 0000000000..491faa01fb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.cc @@ -0,0 +1,374 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/echo_canceller3.h" + +#include + +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/atomicops.h" + +namespace webrtc { + +namespace { + +enum class EchoCanceller3ApiCall { kCapture, kRender }; + +bool DetectSaturation(rtc::ArrayView y) { + for (auto y_k : y) { + if (y_k >= 32700.0f || y_k <= -32700.0f) { + return true; + } + } + return false; +} + +void FillSubFrameView(AudioBuffer* frame, + size_t sub_frame_index, + std::vector>* sub_frame_view) { + RTC_DCHECK_GE(1, sub_frame_index); + RTC_DCHECK_LE(0, sub_frame_index); + RTC_DCHECK_EQ(frame->num_bands(), sub_frame_view->size()); + for (size_t k = 0; k < sub_frame_view->size(); ++k) { + (*sub_frame_view)[k] = rtc::ArrayView( + &frame->split_bands_f(0)[k][sub_frame_index * kSubFrameLength], + kSubFrameLength); + } +} + +void FillSubFrameView(std::vector>* frame, + size_t sub_frame_index, + std::vector>* sub_frame_view) { + RTC_DCHECK_GE(1, sub_frame_index); + RTC_DCHECK_EQ(frame->size(), sub_frame_view->size()); + for (size_t k = 0; k < frame->size(); ++k) { + (*sub_frame_view)[k] = rtc::ArrayView( + &(*frame)[k][sub_frame_index * kSubFrameLength], kSubFrameLength); + } +} + +void ProcessCaptureFrameContent( + AudioBuffer* capture, + bool level_change, + bool saturated_microphone_signal, + size_t sub_frame_index, + FrameBlocker* capture_blocker, + BlockFramer* output_framer, + BlockProcessor* block_processor, + std::vector>* block, + std::vector>* sub_frame_view) { + FillSubFrameView(capture, sub_frame_index, sub_frame_view); + capture_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); + block_processor->ProcessCapture(level_change, saturated_microphone_signal, + block); + output_framer->InsertBlockAndExtractSubFrame(*block, sub_frame_view); +} + +void ProcessRemainingCaptureFrameContent( + bool level_change, + bool saturated_microphone_signal, + FrameBlocker* capture_blocker, + BlockFramer* output_framer, + BlockProcessor* block_processor, + std::vector>* block) { + if (!capture_blocker->IsBlockAvailable()) { + return; + } + + capture_blocker->ExtractBlock(block); + block_processor->ProcessCapture(level_change, saturated_microphone_signal, + block); + output_framer->InsertBlock(*block); +} + +void BufferRenderFrameContent( + std::vector>* render_frame, + size_t sub_frame_index, + FrameBlocker* render_blocker, + BlockProcessor* block_processor, + std::vector>* block, + std::vector>* sub_frame_view) { + FillSubFrameView(render_frame, sub_frame_index, sub_frame_view); + render_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); + block_processor->BufferRender(*block); +} + +void BufferRemainingRenderFrameContent(FrameBlocker* render_blocker, + BlockProcessor* block_processor, + std::vector>* block) { + if (!render_blocker->IsBlockAvailable()) { + return; + } + render_blocker->ExtractBlock(block); + block_processor->BufferRender(*block); +} + +void CopyBufferIntoFrame(AudioBuffer* buffer, + size_t num_bands, + size_t frame_length, + std::vector>* frame) { + RTC_DCHECK_EQ(num_bands, frame->size()); + RTC_DCHECK_EQ(frame_length, (*frame)[0].size()); + for (size_t k = 0; k < num_bands; ++k) { + rtc::ArrayView buffer_view(&buffer->split_bands_f(0)[k][0], + frame_length); + std::copy(buffer_view.begin(), buffer_view.end(), (*frame)[k].begin()); + } +} + +// [B,A] = butter(2,100/4000,'high') +const CascadedBiQuadFilter::BiQuadCoefficients + kHighPassFilterCoefficients_8kHz = {{0.94598f, -1.89195f, 0.94598f}, + {-1.88903f, 0.89487f}}; +const int kNumberOfHighPassBiQuads_8kHz = 1; + +// [B,A] = butter(2,100/8000,'high') +const CascadedBiQuadFilter::BiQuadCoefficients + kHighPassFilterCoefficients_16kHz = {{0.97261f, -1.94523f, 0.97261f}, + {-1.94448f, 0.94598f}}; +const int kNumberOfHighPassBiQuads_16kHz = 1; + +} // namespace + +class EchoCanceller3::RenderWriter { + public: + RenderWriter(ApmDataDumper* data_dumper, + SwapQueue>, + Aec3RenderQueueItemVerifier>* render_transfer_queue, + std::unique_ptr render_highpass_filter, + int sample_rate_hz, + int frame_length, + int num_bands); + ~RenderWriter(); + void Insert(AudioBuffer* input); + + private: + ApmDataDumper* data_dumper_; + const int sample_rate_hz_; + const size_t frame_length_; + const int num_bands_; + std::unique_ptr render_highpass_filter_; + std::vector> render_queue_input_frame_; + SwapQueue>, Aec3RenderQueueItemVerifier>* + render_transfer_queue_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderWriter); +}; + +EchoCanceller3::RenderWriter::RenderWriter( + ApmDataDumper* data_dumper, + SwapQueue>, Aec3RenderQueueItemVerifier>* + render_transfer_queue, + std::unique_ptr render_highpass_filter, + int sample_rate_hz, + int frame_length, + int num_bands) + : data_dumper_(data_dumper), + sample_rate_hz_(sample_rate_hz), + frame_length_(frame_length), + num_bands_(num_bands), + render_highpass_filter_(std::move(render_highpass_filter)), + render_queue_input_frame_(num_bands_, + std::vector(frame_length_, 0.f)), + render_transfer_queue_(render_transfer_queue) { + RTC_DCHECK(data_dumper); +} + +EchoCanceller3::RenderWriter::~RenderWriter() = default; + +void EchoCanceller3::RenderWriter::Insert(AudioBuffer* input) { + RTC_DCHECK_EQ(1, input->num_channels()); + RTC_DCHECK_EQ(frame_length_, input->num_frames_per_band()); + data_dumper_->DumpWav("aec3_render_input", frame_length_, + &input->split_bands_f(0)[0][0], + LowestBandRate(sample_rate_hz_), 1); + + CopyBufferIntoFrame(input, num_bands_, frame_length_, + &render_queue_input_frame_); + + if (render_highpass_filter_) { + render_highpass_filter_->Process(render_queue_input_frame_[0]); + } + + static_cast(render_transfer_queue_->Insert(&render_queue_input_frame_)); +} + +int EchoCanceller3::instance_count_ = 0; + +EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, + int sample_rate_hz, + bool use_highpass_filter) + : EchoCanceller3(sample_rate_hz, + use_highpass_filter, + std::unique_ptr( + BlockProcessor::Create(config, sample_rate_hz))) {} +EchoCanceller3::EchoCanceller3(int sample_rate_hz, + bool use_highpass_filter, + std::unique_ptr block_processor) + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + sample_rate_hz_(sample_rate_hz), + num_bands_(NumBandsForRate(sample_rate_hz_)), + frame_length_(rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)), + output_framer_(num_bands_), + capture_blocker_(num_bands_), + render_blocker_(num_bands_), + render_transfer_queue_( + kRenderTransferQueueSize, + std::vector>( + num_bands_, + std::vector(frame_length_, 0.f)), + Aec3RenderQueueItemVerifier(num_bands_, frame_length_)), + block_processor_(std::move(block_processor)), + render_queue_output_frame_(num_bands_, + std::vector(frame_length_, 0.f)), + block_(num_bands_, std::vector(kBlockSize, 0.f)), + sub_frame_view_(num_bands_) { + RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); + + std::unique_ptr render_highpass_filter; + if (use_highpass_filter) { + render_highpass_filter.reset(new CascadedBiQuadFilter( + sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz + : kHighPassFilterCoefficients_16kHz, + sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz + : kNumberOfHighPassBiQuads_16kHz)); + capture_highpass_filter_.reset(new CascadedBiQuadFilter( + sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz + : kHighPassFilterCoefficients_16kHz, + sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz + : kNumberOfHighPassBiQuads_16kHz)); + } + + render_writer_.reset( + new RenderWriter(data_dumper_.get(), &render_transfer_queue_, + std::move(render_highpass_filter), sample_rate_hz_, + frame_length_, num_bands_)); + + RTC_DCHECK_EQ(num_bands_, std::max(sample_rate_hz_, 16000) / 16000); + RTC_DCHECK_GE(kMaxNumBands, num_bands_); +} + +EchoCanceller3::~EchoCanceller3() = default; + +void EchoCanceller3::AnalyzeRender(AudioBuffer* render) { + RTC_DCHECK_RUNS_SERIALIZED(&render_race_checker_); + RTC_DCHECK(render); + data_dumper_->DumpRaw("aec3_call_order", + static_cast(EchoCanceller3ApiCall::kRender)); + + return render_writer_->Insert(render); +} + +void EchoCanceller3::AnalyzeCapture(AudioBuffer* capture) { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + RTC_DCHECK(capture); + data_dumper_->DumpWav("aec3_capture_analyze_input", capture->num_frames(), + capture->channels_f()[0], sample_rate_hz_, 1); + + saturated_microphone_signal_ = false; + for (size_t k = 0; k < capture->num_channels(); ++k) { + saturated_microphone_signal_ |= + DetectSaturation(rtc::ArrayView(capture->channels_f()[k], + capture->num_frames())); + if (saturated_microphone_signal_) { + break; + } + } +} + +void EchoCanceller3::ProcessCapture(AudioBuffer* capture, bool level_change) { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + RTC_DCHECK(capture); + RTC_DCHECK_EQ(1u, capture->num_channels()); + RTC_DCHECK_EQ(num_bands_, capture->num_bands()); + RTC_DCHECK_EQ(frame_length_, capture->num_frames_per_band()); + data_dumper_->DumpRaw("aec3_call_order", + static_cast(EchoCanceller3ApiCall::kCapture)); + + rtc::ArrayView capture_lower_band = + rtc::ArrayView(&capture->split_bands_f(0)[0][0], frame_length_); + + data_dumper_->DumpWav("aec3_capture_input", capture_lower_band, + LowestBandRate(sample_rate_hz_), 1); + + EmptyRenderQueue(); + + if (capture_highpass_filter_) { + capture_highpass_filter_->Process(capture_lower_band); + } + + ProcessCaptureFrameContent( + capture, level_change, saturated_microphone_signal_, 0, &capture_blocker_, + &output_framer_, block_processor_.get(), &block_, &sub_frame_view_); + + if (sample_rate_hz_ != 8000) { + ProcessCaptureFrameContent( + capture, level_change, saturated_microphone_signal_, 1, + &capture_blocker_, &output_framer_, block_processor_.get(), &block_, + &sub_frame_view_); + } + + ProcessRemainingCaptureFrameContent( + level_change, saturated_microphone_signal_, &capture_blocker_, + &output_framer_, block_processor_.get(), &block_); + + data_dumper_->DumpWav("aec3_capture_output", frame_length_, + &capture->split_bands_f(0)[0][0], + LowestBandRate(sample_rate_hz_), 1); +} + +EchoControl::Metrics EchoCanceller3::GetMetrics() const { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + Metrics metrics; + block_processor_->GetMetrics(&metrics); + return metrics; +} + +bool EchoCanceller3::Validate(const EchoCanceller3Config& config) { + return true; +} + +void EchoCanceller3::EmptyRenderQueue() { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + bool frame_to_buffer = + render_transfer_queue_.Remove(&render_queue_output_frame_); + while (frame_to_buffer) { + BufferRenderFrameContent(&render_queue_output_frame_, 0, &render_blocker_, + block_processor_.get(), &block_, &sub_frame_view_); + + if (sample_rate_hz_ != 8000) { + BufferRenderFrameContent(&render_queue_output_frame_, 1, &render_blocker_, + block_processor_.get(), &block_, + &sub_frame_view_); + } + + BufferRemainingRenderFrameContent(&render_blocker_, block_processor_.get(), + &block_); + + frame_to_buffer = + render_transfer_queue_.Remove(&render_queue_output_frame_); + } +} + +EchoCanceller3Factory::EchoCanceller3Factory() {} + +EchoCanceller3Factory::EchoCanceller3Factory(const EchoCanceller3Config& config) + : config_(config) { + // Revert to default configuration if needed. + if (!EchoCanceller3::Validate(config_)) { + config_ = EchoCanceller3Config(); + } +} + +std::unique_ptr EchoCanceller3Factory::Create(int sample_rate_hz) { + return std::unique_ptr( + new EchoCanceller3(config_, sample_rate_hz, true)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.h new file mode 100644 index 0000000000..475bacb723 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ + +#include "modules/audio_processing/aec3/block_framer.h" +#include "modules/audio_processing/aec3/block_processor.h" +#include "modules/audio_processing/aec3/cascaded_biquad_filter.h" +#include "modules/audio_processing/aec3/frame_blocker.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/swap_queue.h" + +namespace webrtc { + +// Functor for verifying the invariance of the frames being put into the render +// queue. +class Aec3RenderQueueItemVerifier { + public: + explicit Aec3RenderQueueItemVerifier(size_t num_bands, size_t frame_length) + : num_bands_(num_bands), frame_length_(frame_length) {} + + bool operator()(const std::vector>& v) const { + if (v.size() != num_bands_) { + return false; + } + for (const auto& v_k : v) { + if (v_k.size() != frame_length_) { + return false; + } + } + return true; + } + + private: + const size_t num_bands_; + const size_t frame_length_; +}; + +// Main class for the echo canceller3. +// It does 4 things: +// -Receives 10 ms frames of band-split audio. +// -Optionally applies an anti-hum (high-pass) filter on the +// received signals. +// -Provides the lower level echo canceller functionality with +// blocks of 64 samples of audio data. +// -Partially handles the jitter in the render and capture API +// call sequence. +// +// The class is supposed to be used in a non-concurrent manner apart from the +// AnalyzeRender call which can be called concurrently with the other methods. +class EchoCanceller3 : public EchoControl { + public: + // Normal c-tor to use. + EchoCanceller3(const EchoCanceller3Config& config, + int sample_rate_hz, + bool use_highpass_filter); + // Testing c-tor that is used only for testing purposes. + EchoCanceller3(int sample_rate_hz, + bool use_highpass_filter, + std::unique_ptr block_processor); + ~EchoCanceller3() override; + // Analyzes and stores an internal copy of the split-band domain render + // signal. + void AnalyzeRender(AudioBuffer* farend) override; + // Analyzes the full-band domain capture signal to detect signal saturation. + void AnalyzeCapture(AudioBuffer* capture) override; + // Processes the split-band domain capture signal in order to remove any echo + // present in the signal. + void ProcessCapture(AudioBuffer* capture, bool level_change) override; + // Collect current metrics from the echo canceller. + Metrics GetMetrics() const override; + + // Signals whether an external detector has detected echo leakage from the + // echo canceller. + // Note that in the case echo leakage has been flagged, it should be unflagged + // once it is no longer occurring. + void UpdateEchoLeakageStatus(bool leakage_detected) { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + block_processor_->UpdateEchoLeakageStatus(leakage_detected); + } + + // Validates a config. + static bool Validate(const EchoCanceller3Config& config); + + private: + class RenderWriter; + + // Empties the render SwapQueue. + void EmptyRenderQueue(); + + rtc::RaceChecker capture_race_checker_; + rtc::RaceChecker render_race_checker_; + + // State that is accessed by the AnalyzeRender call. + std::unique_ptr render_writer_ + RTC_GUARDED_BY(render_race_checker_); + + // State that may be accessed by the capture thread. + static int instance_count_; + std::unique_ptr data_dumper_; + const int sample_rate_hz_; + const int num_bands_; + const size_t frame_length_; + BlockFramer output_framer_ RTC_GUARDED_BY(capture_race_checker_); + FrameBlocker capture_blocker_ RTC_GUARDED_BY(capture_race_checker_); + FrameBlocker render_blocker_ RTC_GUARDED_BY(capture_race_checker_); + SwapQueue>, Aec3RenderQueueItemVerifier> + render_transfer_queue_; + std::unique_ptr block_processor_ + RTC_GUARDED_BY(capture_race_checker_); + std::vector> render_queue_output_frame_ + RTC_GUARDED_BY(capture_race_checker_); + std::unique_ptr capture_highpass_filter_ + RTC_GUARDED_BY(capture_race_checker_); + bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) = + false; + std::vector> block_ RTC_GUARDED_BY(capture_race_checker_); + std::vector> sub_frame_view_ + RTC_GUARDED_BY(capture_race_checker_); + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoCanceller3); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc new file mode 100644 index 0000000000..75de48b547 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -0,0 +1,748 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/echo_canceller3.h" + +#include +#include +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block_processor.h" +#include "modules/audio_processing/aec3/frame_blocker.h" +#include "modules/audio_processing/aec3/mock/mock_block_processor.h" +#include "modules/audio_processing/audio_buffer.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using testing::StrictMock; +using testing::_; + +// Populates the frame with linearly increasing sample values for each band, +// with a band-specific offset, in order to allow simple bitexactness +// verification for each band. +void PopulateInputFrame(size_t frame_length, + size_t num_bands, + size_t frame_index, + float* const* frame, + int offset) { + for (size_t k = 0; k < num_bands; ++k) { + for (size_t i = 0; i < frame_length; ++i) { + float value = static_cast(frame_index * frame_length + i) + offset; + frame[k][i] = (value > 0 ? 5000 * k + value : 0); + } + } +} + +// Populates the frame with linearly increasing sample values. +void PopulateInputFrame(size_t frame_length, + size_t frame_index, + float* frame, + int offset) { + for (size_t i = 0; i < frame_length; ++i) { + float value = static_cast(frame_index * frame_length + i) + offset; + frame[i] = std::max(value, 0.f); + } +} + +// Verifies the that samples in the output frame are identical to the samples +// that were produced for the input frame, with an offset in order to compensate +// for buffering delays. +bool VerifyOutputFrameBitexactness(size_t frame_length, + size_t num_bands, + size_t frame_index, + const float* const* frame, + int offset) { + float reference_frame_data[kMaxNumBands][2 * kSubFrameLength]; + float* reference_frame[kMaxNumBands]; + for (size_t k = 0; k < num_bands; ++k) { + reference_frame[k] = &reference_frame_data[k][0]; + } + + PopulateInputFrame(frame_length, num_bands, frame_index, reference_frame, + offset); + for (size_t k = 0; k < num_bands; ++k) { + for (size_t i = 0; i < frame_length; ++i) { + if (reference_frame[k][i] != frame[k][i]) { + return false; + } + } + } + + return true; +} + +// Class for testing that the capture data is properly received by the block +// processor and that the processor data is properly passed to the +// EchoCanceller3 output. +class CaptureTransportVerificationProcessor : public BlockProcessor { + public: + explicit CaptureTransportVerificationProcessor(size_t num_bands) {} + ~CaptureTransportVerificationProcessor() override = default; + + void ProcessCapture(bool level_change, + bool saturated_microphone_signal, + std::vector>* capture_block) override { + } + + void BufferRender(const std::vector>& block) override {} + + void UpdateEchoLeakageStatus(bool leakage_detected) override {} + + void GetMetrics(EchoControl::Metrics* metrics) const override {} + + private: + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CaptureTransportVerificationProcessor); +}; + +// Class for testing that the render data is properly received by the block +// processor. +class RenderTransportVerificationProcessor : public BlockProcessor { + public: + explicit RenderTransportVerificationProcessor(size_t num_bands) {} + ~RenderTransportVerificationProcessor() override = default; + + void ProcessCapture(bool level_change, + bool saturated_microphone_signal, + std::vector>* capture_block) override { + std::vector> render_block = + received_render_blocks_.front(); + received_render_blocks_.pop_front(); + capture_block->swap(render_block); + } + + void BufferRender(const std::vector>& block) override { + received_render_blocks_.push_back(block); + } + + void UpdateEchoLeakageStatus(bool leakage_detected) override {} + + void GetMetrics(EchoControl::Metrics* metrics) const override {} + + private: + std::deque>> received_render_blocks_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderTransportVerificationProcessor); +}; + +class EchoCanceller3Tester { + public: + explicit EchoCanceller3Tester(int sample_rate_hz) + : sample_rate_hz_(sample_rate_hz), + num_bands_(NumBandsForRate(sample_rate_hz_)), + frame_length_(sample_rate_hz_ == 8000 ? 80 : 160), + fullband_frame_length_(rtc::CheckedDivExact(sample_rate_hz_, 100)), + capture_buffer_(fullband_frame_length_, + 1, + fullband_frame_length_, + 1, + fullband_frame_length_), + render_buffer_(fullband_frame_length_, + 1, + fullband_frame_length_, + 1, + fullband_frame_length_) {} + + // Verifies that the capture data is properly received by the block processor + // and that the processor data is properly passed to the EchoCanceller3 + // output. + void RunCaptureTransportVerificationTest() { + EchoCanceller3 aec3( + sample_rate_hz_, false, + std::unique_ptr( + new CaptureTransportVerificationProcessor(num_bands_))); + + for (size_t frame_index = 0; frame_index < kNumFramesToProcess; + ++frame_index) { + aec3.AnalyzeCapture(&capture_buffer_); + OptionalBandSplit(); + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], 0); + PopulateInputFrame(frame_length_, frame_index, + &render_buffer_.channels_f()[0][0], 0); + + aec3.AnalyzeRender(&render_buffer_); + aec3.ProcessCapture(&capture_buffer_, false); + EXPECT_TRUE(VerifyOutputFrameBitexactness( + frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], -64)); + } + } + + // Test method for testing that the render data is properly received by the + // block processor. + void RunRenderTransportVerificationTest() { + EchoCanceller3 aec3( + sample_rate_hz_, false, + std::unique_ptr( + new RenderTransportVerificationProcessor(num_bands_))); + + for (size_t frame_index = 0; frame_index < kNumFramesToProcess; + ++frame_index) { + aec3.AnalyzeCapture(&capture_buffer_); + OptionalBandSplit(); + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], 100); + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &render_buffer_.split_bands_f(0)[0], 0); + + aec3.AnalyzeRender(&render_buffer_); + aec3.ProcessCapture(&capture_buffer_, false); + EXPECT_TRUE(VerifyOutputFrameBitexactness( + frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], -64)); + } + } + + // Verifies that information about echo path changes are properly propagated + // to the block processor. + // The cases tested are: + // -That no set echo path change flags are received when there is no echo path + // change. + // -That set echo path change flags are received and continues to be received + // as long as echo path changes are flagged. + // -That set echo path change flags are no longer received when echo path + // change events stop being flagged. + enum class EchoPathChangeTestVariant { kNone, kOneSticky, kOneNonSticky }; + + void RunEchoPathChangeVerificationTest( + EchoPathChangeTestVariant echo_path_change_test_variant) { + const size_t num_full_blocks_per_frame = + rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100) / kBlockSize; + const size_t expected_num_block_to_process = + (kNumFramesToProcess * + rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)) / + kBlockSize; + std::unique_ptr> + block_processor_mock( + new StrictMock()); + EXPECT_CALL(*block_processor_mock, BufferRender(_)) + .Times(expected_num_block_to_process); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); + + switch (echo_path_change_test_variant) { + case EchoPathChangeTestVariant::kNone: + EXPECT_CALL(*block_processor_mock, ProcessCapture(false, _, _)) + .Times(expected_num_block_to_process); + break; + case EchoPathChangeTestVariant::kOneSticky: + EXPECT_CALL(*block_processor_mock, ProcessCapture(true, _, _)) + .Times(expected_num_block_to_process); + break; + case EchoPathChangeTestVariant::kOneNonSticky: + EXPECT_CALL(*block_processor_mock, ProcessCapture(true, _, _)) + .Times(num_full_blocks_per_frame); + EXPECT_CALL(*block_processor_mock, ProcessCapture(false, _, _)) + .Times(expected_num_block_to_process - num_full_blocks_per_frame); + break; + } + + EchoCanceller3 aec3(sample_rate_hz_, false, + std::move(block_processor_mock)); + + for (size_t frame_index = 0; frame_index < kNumFramesToProcess; + ++frame_index) { + bool echo_path_change = false; + switch (echo_path_change_test_variant) { + case EchoPathChangeTestVariant::kNone: + break; + case EchoPathChangeTestVariant::kOneSticky: + echo_path_change = true; + break; + case EchoPathChangeTestVariant::kOneNonSticky: + if (frame_index == 0) { + echo_path_change = true; + } + break; + } + + aec3.AnalyzeCapture(&capture_buffer_); + OptionalBandSplit(); + + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], 0); + PopulateInputFrame(frame_length_, frame_index, + &render_buffer_.channels_f()[0][0], 0); + + aec3.AnalyzeRender(&render_buffer_); + aec3.ProcessCapture(&capture_buffer_, echo_path_change); + } + } + + // Test for verifying that echo leakage information is being properly passed + // to the processor. + // The cases tested are: + // -That no method calls are received when they should not. + // -That false values are received each time they are flagged. + // -That true values are received each time they are flagged. + // -That a false value is received when flagged after a true value has been + // flagged. + enum class EchoLeakageTestVariant { + kNone, + kFalseSticky, + kTrueSticky, + kTrueNonSticky + }; + + void RunEchoLeakageVerificationTest( + EchoLeakageTestVariant leakage_report_variant) { + const size_t expected_num_block_to_process = + (kNumFramesToProcess * + rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)) / + kBlockSize; + std::unique_ptr> + block_processor_mock( + new StrictMock()); + EXPECT_CALL(*block_processor_mock, BufferRender(_)) + .Times(expected_num_block_to_process); + EXPECT_CALL(*block_processor_mock, ProcessCapture(_, _, _)) + .Times(expected_num_block_to_process); + + switch (leakage_report_variant) { + case EchoLeakageTestVariant::kNone: + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); + break; + case EchoLeakageTestVariant::kFalseSticky: + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(false)) + .Times(1); + break; + case EchoLeakageTestVariant::kTrueSticky: + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(true)) + .Times(1); + break; + case EchoLeakageTestVariant::kTrueNonSticky: { + testing::InSequence s; + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(true)) + .Times(1); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(false)) + .Times(kNumFramesToProcess - 1); + } break; + } + + EchoCanceller3 aec3(sample_rate_hz_, false, + std::move(block_processor_mock)); + + for (size_t frame_index = 0; frame_index < kNumFramesToProcess; + ++frame_index) { + switch (leakage_report_variant) { + case EchoLeakageTestVariant::kNone: + break; + case EchoLeakageTestVariant::kFalseSticky: + if (frame_index == 0) { + aec3.UpdateEchoLeakageStatus(false); + } + break; + case EchoLeakageTestVariant::kTrueSticky: + if (frame_index == 0) { + aec3.UpdateEchoLeakageStatus(true); + } + break; + case EchoLeakageTestVariant::kTrueNonSticky: + if (frame_index == 0) { + aec3.UpdateEchoLeakageStatus(true); + } else { + aec3.UpdateEchoLeakageStatus(false); + } + break; + } + + aec3.AnalyzeCapture(&capture_buffer_); + OptionalBandSplit(); + + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], 0); + PopulateInputFrame(frame_length_, frame_index, + &render_buffer_.channels_f()[0][0], 0); + + aec3.AnalyzeRender(&render_buffer_); + aec3.ProcessCapture(&capture_buffer_, false); + } + } + + // This verifies that saturation information is properly passed to the + // BlockProcessor. + // The cases tested are: + // -That no saturation event is passed to the processor if there is no + // saturation. + // -That one frame with one negative saturated sample value is reported to be + // saturated and that following non-saturated frames are properly reported as + // not being saturated. + // -That one frame with one positive saturated sample value is reported to be + // saturated and that following non-saturated frames are properly reported as + // not being saturated. + enum class SaturationTestVariant { kNone, kOneNegative, kOnePositive }; + + void RunCaptureSaturationVerificationTest( + SaturationTestVariant saturation_variant) { + const size_t num_full_blocks_per_frame = + rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100) / kBlockSize; + const size_t expected_num_block_to_process = + (kNumFramesToProcess * + rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)) / + kBlockSize; + std::unique_ptr> + block_processor_mock( + new StrictMock()); + EXPECT_CALL(*block_processor_mock, BufferRender(_)) + .Times(expected_num_block_to_process); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); + + switch (saturation_variant) { + case SaturationTestVariant::kNone: + EXPECT_CALL(*block_processor_mock, ProcessCapture(_, false, _)) + .Times(expected_num_block_to_process); + break; + case SaturationTestVariant::kOneNegative: { + testing::InSequence s; + EXPECT_CALL(*block_processor_mock, ProcessCapture(_, true, _)) + .Times(num_full_blocks_per_frame); + EXPECT_CALL(*block_processor_mock, ProcessCapture(_, false, _)) + .Times(expected_num_block_to_process - num_full_blocks_per_frame); + } break; + case SaturationTestVariant::kOnePositive: { + testing::InSequence s; + EXPECT_CALL(*block_processor_mock, ProcessCapture(_, true, _)) + .Times(num_full_blocks_per_frame); + EXPECT_CALL(*block_processor_mock, ProcessCapture(_, false, _)) + .Times(expected_num_block_to_process - num_full_blocks_per_frame); + } break; + } + + EchoCanceller3 aec3(sample_rate_hz_, false, + std::move(block_processor_mock)); + for (size_t frame_index = 0; frame_index < kNumFramesToProcess; + ++frame_index) { + for (int k = 0; k < fullband_frame_length_; ++k) { + capture_buffer_.channels_f()[0][k] = 0.f; + } + switch (saturation_variant) { + case SaturationTestVariant::kNone: + break; + case SaturationTestVariant::kOneNegative: + if (frame_index == 0) { + capture_buffer_.channels_f()[0][10] = -32768.f; + } + break; + case SaturationTestVariant::kOnePositive: + if (frame_index == 0) { + capture_buffer_.channels_f()[0][10] = 32767.f; + } + break; + } + + aec3.AnalyzeCapture(&capture_buffer_); + OptionalBandSplit(); + + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], 0); + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &render_buffer_.split_bands_f(0)[0], 0); + + aec3.AnalyzeRender(&render_buffer_); + aec3.ProcessCapture(&capture_buffer_, false); + } + } + + // This test verifies that the swapqueue is able to handle jitter in the + // capture and render API calls. + void RunRenderSwapQueueVerificationTest() { + EchoCanceller3 aec3( + sample_rate_hz_, false, + std::unique_ptr( + new RenderTransportVerificationProcessor(num_bands_))); + + for (size_t frame_index = 0; frame_index < kRenderTransferQueueSize; + ++frame_index) { + if (sample_rate_hz_ > 16000) { + render_buffer_.SplitIntoFrequencyBands(); + } + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &render_buffer_.split_bands_f(0)[0], 0); + + if (sample_rate_hz_ > 16000) { + render_buffer_.SplitIntoFrequencyBands(); + } + + aec3.AnalyzeRender(&render_buffer_); + } + + for (size_t frame_index = 0; frame_index < kRenderTransferQueueSize; + ++frame_index) { + aec3.AnalyzeCapture(&capture_buffer_); + if (sample_rate_hz_ > 16000) { + capture_buffer_.SplitIntoFrequencyBands(); + } + + PopulateInputFrame(frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], 0); + + aec3.ProcessCapture(&capture_buffer_, false); + EXPECT_TRUE(VerifyOutputFrameBitexactness( + frame_length_, num_bands_, frame_index, + &capture_buffer_.split_bands_f(0)[0], -64)); + } + } + + // This test verifies that a buffer overrun in the render swapqueue is + // properly reported. + void RunRenderPipelineSwapQueueOverrunReturnValueTest() { + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, false); + + constexpr size_t kRenderTransferQueueSize = 30; + for (size_t k = 0; k < 2; ++k) { + for (size_t frame_index = 0; frame_index < kRenderTransferQueueSize; + ++frame_index) { + if (sample_rate_hz_ > 16000) { + render_buffer_.SplitIntoFrequencyBands(); + } + PopulateInputFrame(frame_length_, frame_index, + &render_buffer_.channels_f()[0][0], 0); + + if (k == 0) { + aec3.AnalyzeRender(&render_buffer_); + } else { + aec3.AnalyzeRender(&render_buffer_); + } + } + } + } + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + // Verifies the that the check for the number of bands in the AnalyzeRender + // input is correct by adjusting the sample rates of EchoCanceller3 and the + // input AudioBuffer to have a different number of bands. + void RunAnalyzeRenderNumBandsCheckVerification() { + // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a + // way that the number of bands for the rates are different. + const int aec3_sample_rate_hz = sample_rate_hz_ == 48000 ? 32000 : 48000; + EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz, false); + PopulateInputFrame(frame_length_, 0, &render_buffer_.channels_f()[0][0], 0); + + EXPECT_DEATH(aec3.AnalyzeRender(&render_buffer_), ""); + } + + // Verifies the that the check for the number of bands in the ProcessCapture + // input is correct by adjusting the sample rates of EchoCanceller3 and the + // input AudioBuffer to have a different number of bands. + void RunProcessCaptureNumBandsCheckVerification() { + // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a + // way that the number of bands for the rates are different. + const int aec3_sample_rate_hz = sample_rate_hz_ == 48000 ? 32000 : 48000; + EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz, false); + PopulateInputFrame(frame_length_, num_bands_, 0, + &capture_buffer_.split_bands_f(0)[0], 100); + EXPECT_DEATH(aec3.ProcessCapture(&capture_buffer_, false), ""); + } + + // Verifies the that the check for the frame length in the AnalyzeRender input + // is correct by adjusting the sample rates of EchoCanceller3 and the input + // AudioBuffer to have a different frame lengths. + void RunAnalyzeRenderFrameLengthCheckVerification() { + // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a + // way that the band frame lengths are different. + const int aec3_sample_rate_hz = sample_rate_hz_ == 8000 ? 16000 : 8000; + EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz, false); + + OptionalBandSplit(); + PopulateInputFrame(frame_length_, 0, &render_buffer_.channels_f()[0][0], 0); + + EXPECT_DEATH(aec3.AnalyzeRender(&render_buffer_), ""); + } + + // Verifies the that the check for the frame length in the AnalyzeRender input + // is correct by adjusting the sample rates of EchoCanceller3 and the input + // AudioBuffer to have a different frame lengths. + void RunProcessCaptureFrameLengthCheckVerification() { + // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a + // way that the band frame lengths are different. + const int aec3_sample_rate_hz = sample_rate_hz_ == 8000 ? 16000 : 8000; + EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz, false); + + OptionalBandSplit(); + PopulateInputFrame(frame_length_, num_bands_, 0, + &capture_buffer_.split_bands_f(0)[0], 100); + + EXPECT_DEATH(aec3.ProcessCapture(&capture_buffer_, false), ""); + } + +#endif + + private: + void OptionalBandSplit() { + if (sample_rate_hz_ > 16000) { + capture_buffer_.SplitIntoFrequencyBands(); + render_buffer_.SplitIntoFrequencyBands(); + } + } + + static constexpr size_t kNumFramesToProcess = 20; + const int sample_rate_hz_; + const size_t num_bands_; + const size_t frame_length_; + const int fullband_frame_length_; + AudioBuffer capture_buffer_; + AudioBuffer render_buffer_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoCanceller3Tester); +}; + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +std::string ProduceDebugText(int sample_rate_hz, int variant) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz << ", variant: " << variant; + return ss.str(); +} + +} // namespace + +TEST(EchoCanceller3Buffering, CaptureBitexactness) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Tester(rate).RunCaptureTransportVerificationTest(); + } +} + +TEST(EchoCanceller3Buffering, RenderBitexactness) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Tester(rate).RunRenderTransportVerificationTest(); + } +} + +TEST(EchoCanceller3Buffering, RenderSwapQueue) { + for (auto rate : {8000, 16000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Tester(rate).RunRenderSwapQueueVerificationTest(); + } +} + +TEST(EchoCanceller3Buffering, RenderSwapQueueOverrunReturnValue) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Tester(rate) + .RunRenderPipelineSwapQueueOverrunReturnValueTest(); + } +} + +TEST(EchoCanceller3Messaging, CaptureSaturation) { + auto variants = {EchoCanceller3Tester::SaturationTestVariant::kNone, + EchoCanceller3Tester::SaturationTestVariant::kOneNegative, + EchoCanceller3Tester::SaturationTestVariant::kOnePositive}; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto variant : variants) { + SCOPED_TRACE(ProduceDebugText(rate, static_cast(variant))); + EchoCanceller3Tester(rate).RunCaptureSaturationVerificationTest(variant); + } + } +} + +TEST(EchoCanceller3Messaging, EchoPathChange) { + auto variants = { + EchoCanceller3Tester::EchoPathChangeTestVariant::kNone, + EchoCanceller3Tester::EchoPathChangeTestVariant::kOneSticky, + EchoCanceller3Tester::EchoPathChangeTestVariant::kOneNonSticky}; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto variant : variants) { + SCOPED_TRACE(ProduceDebugText(rate, static_cast(variant))); + EchoCanceller3Tester(rate).RunEchoPathChangeVerificationTest(variant); + } + } +} + +TEST(EchoCanceller3Messaging, EchoLeakage) { + auto variants = { + EchoCanceller3Tester::EchoLeakageTestVariant::kNone, + EchoCanceller3Tester::EchoLeakageTestVariant::kFalseSticky, + EchoCanceller3Tester::EchoLeakageTestVariant::kTrueSticky, + EchoCanceller3Tester::EchoLeakageTestVariant::kTrueNonSticky}; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto variant : variants) { + SCOPED_TRACE(ProduceDebugText(rate, static_cast(variant))); + EchoCanceller3Tester(rate).RunEchoLeakageVerificationTest(variant); + } + } +} + +TEST(EchoCanceller3, ConfigValidation) { + EchoCanceller3Config config; + EXPECT_TRUE(EchoCanceller3::Validate(config)); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +TEST(EchoCanceller3InputCheck, WrongCaptureNumBandsCheckVerification) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Tester(rate).RunProcessCaptureNumBandsCheckVerification(); + } +} + +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoCanceller3InputCheck, + DISABLED_WrongRenderFrameLengthCheckVerification) { + for (auto rate : {8000, 16000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Tester(rate).RunAnalyzeRenderFrameLengthCheckVerification(); + } +} + +TEST(EchoCanceller3InputCheck, WrongCaptureFrameLengthCheckVerification) { + for (auto rate : {8000, 16000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Tester(rate).RunProcessCaptureFrameLengthCheckVerification(); + } +} + +// Verifiers that the verification for null input to the render analysis api +// call works. +TEST(EchoCanceller3InputCheck, NullRenderAnalysisParameter) { + EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 8000, false) + .AnalyzeRender(nullptr), + ""); +} + +// Verifiers that the verification for null input to the capture analysis api +// call works. +TEST(EchoCanceller3InputCheck, NullCaptureAnalysisParameter) { + EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 8000, false) + .AnalyzeCapture(nullptr), + ""); +} + +// Verifiers that the verification for null input to the capture processing api +// call works. +TEST(EchoCanceller3InputCheck, NullCaptureProcessingParameter) { + EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 8000, false) + .ProcessCapture(nullptr, false), + ""); +} + +// Verifies the check for correct sample rate. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoCanceller3InputCheck, DISABLED_WrongSampleRate) { + ApmDataDumper data_dumper(0); + EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 8001, false), ""); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc new file mode 100644 index 0000000000..914f2d28eb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/echo_path_delay_estimator.h" + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +EchoPathDelayEstimator::EchoPathDelayEstimator( + ApmDataDumper* data_dumper, + const EchoCanceller3Config& config) + : data_dumper_(data_dumper), + down_sampling_factor_(config.delay.down_sampling_factor), + sub_block_size_(down_sampling_factor_ != 0 + ? kBlockSize / down_sampling_factor_ + : kBlockSize), + capture_decimator_(down_sampling_factor_), + matched_filter_(data_dumper_, + DetectOptimization(), + sub_block_size_, + kMatchedFilterWindowSizeSubBlocks, + config.delay.num_filters, + kMatchedFilterAlignmentShiftSizeSubBlocks, + config.render_levels.poor_excitation_render_limit), + matched_filter_lag_aggregator_(data_dumper_, + matched_filter_.GetMaxFilterLag()) { + RTC_DCHECK(data_dumper); + RTC_DCHECK(down_sampling_factor_ > 0); +} + +EchoPathDelayEstimator::~EchoPathDelayEstimator() = default; + +void EchoPathDelayEstimator::Reset() { + matched_filter_lag_aggregator_.Reset(); + matched_filter_.Reset(); +} + +rtc::Optional EchoPathDelayEstimator::EstimateDelay( + const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture) { + RTC_DCHECK_EQ(kBlockSize, capture.size()); + + std::array downsampled_capture_data; + rtc::ArrayView downsampled_capture(downsampled_capture_data.data(), + sub_block_size_); + data_dumper_->DumpWav("aec3_capture_decimator_input", capture.size(), + capture.data(), 16000, 1); + capture_decimator_.Decimate(capture, downsampled_capture); + data_dumper_->DumpWav("aec3_capture_decimator_output", + downsampled_capture.size(), downsampled_capture.data(), + 16000 / down_sampling_factor_, 1); + matched_filter_.Update(render_buffer, downsampled_capture); + + rtc::Optional aggregated_matched_filter_lag = + matched_filter_lag_aggregator_.Aggregate( + matched_filter_.GetLagEstimates()); + + // TODO(peah): Move this logging outside of this class once EchoCanceller3 + // development is done. + data_dumper_->DumpRaw("aec3_echo_path_delay_estimator_delay", + aggregated_matched_filter_lag + ? static_cast(*aggregated_matched_filter_lag * + down_sampling_factor_) + : -1); + + // Return the detected delay in samples as the aggregated matched filter lag + // compensated by the down sampling factor for the signal being correlated. + return aggregated_matched_filter_lag + ? rtc::Optional(*aggregated_matched_filter_lag * + down_sampling_factor_) + : rtc::nullopt; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h new file mode 100644 index 0000000000..04943ca4ba --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_DELAY_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_DELAY_ESTIMATOR_H_ + +#include + +#include "api/optional.h" +#include "modules/audio_processing/aec3/decimator.h" +#include "modules/audio_processing/aec3/downsampled_render_buffer.h" +#include "modules/audio_processing/aec3/matched_filter.h" +#include "modules/audio_processing/aec3/matched_filter_lag_aggregator.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; + +// Estimates the delay of the echo path. +class EchoPathDelayEstimator { + public: + EchoPathDelayEstimator(ApmDataDumper* data_dumper, + const EchoCanceller3Config& config); + ~EchoPathDelayEstimator(); + + // Resets the estimation. + void Reset(); + + // Produce a delay estimate if such is avaliable. + rtc::Optional EstimateDelay( + const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture); + + // Log delay estimator properties. + void LogDelayEstimationProperties(int sample_rate_hz, size_t shift) const { + matched_filter_.LogFilterProperties(sample_rate_hz, shift, + down_sampling_factor_); + } + + private: + ApmDataDumper* const data_dumper_; + const size_t down_sampling_factor_; + const size_t sub_block_size_; + Decimator capture_decimator_; + MatchedFilter matched_filter_; + MatchedFilterLagAggregator matched_filter_lag_aggregator_; + + RTC_DISALLOW_COPY_AND_ASSIGN(EchoPathDelayEstimator); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_DELAY_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc new file mode 100644 index 0000000000..2dbdb1ccf4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/echo_path_delay_estimator.h" + +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(size_t delay) { + std::ostringstream ss; + ss << "Delay: " << delay; + return ss.str(); +} + +} // namespace + +// Verifies that the basic API calls work. +TEST(EchoPathDelayEstimator, BasicApiCalls) { + ApmDataDumper data_dumper(0); + EchoCanceller3Config config; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + EchoPathDelayEstimator estimator(&data_dumper, config); + std::vector> render(3, std::vector(kBlockSize)); + std::vector capture(kBlockSize); + for (size_t k = 0; k < 100; ++k) { + render_delay_buffer->Insert(render); + estimator.EstimateDelay(render_delay_buffer->GetDownsampledRenderBuffer(), + capture); + } +} + +// Verifies that the delay estimator produces correct delay for artificially +// delayed signals. +TEST(EchoPathDelayEstimator, DelayEstimation) { + Random random_generator(42U); + std::vector> render(3, std::vector(kBlockSize)); + std::vector capture(kBlockSize); + ApmDataDumper data_dumper(0); + constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; + for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = 10; + for (size_t delay_samples : {30, 64, 150, 200, 800, 4000}) { + SCOPED_TRACE(ProduceDebugText(delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + DelayBuffer signal_delay_buffer(delay_samples); + EchoPathDelayEstimator estimator(&data_dumper, config); + + rtc::Optional estimated_delay_samples; + for (size_t k = 0; k < (300 + delay_samples / kBlockSize); ++k) { + RandomizeSampleVector(&random_generator, render[0]); + signal_delay_buffer.Delay(render[0], capture); + render_delay_buffer->Insert(render); + render_delay_buffer->UpdateBuffers(); + estimated_delay_samples = estimator.EstimateDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture); + } + if (estimated_delay_samples) { + // Due to the internal down-sampling done inside the delay estimator + // the estimated delay cannot be expected to be exact to the true delay. + EXPECT_NEAR(delay_samples, *estimated_delay_samples, + config.delay.down_sampling_factor); + } else { + ADD_FAILURE(); + } + } + } +} + +// Verifies that the delay estimator does not produce delay estimates too +// quickly. +TEST(EchoPathDelayEstimator, NoInitialDelayestimates) { + Random random_generator(42U); + EchoCanceller3Config config; + std::vector> render(3, std::vector(kBlockSize)); + std::vector capture(kBlockSize); + ApmDataDumper data_dumper(0); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + + EchoPathDelayEstimator estimator(&data_dumper, config); + for (size_t k = 0; k < 19; ++k) { + RandomizeSampleVector(&random_generator, render[0]); + std::copy(render[0].begin(), render[0].end(), capture.begin()); + render_delay_buffer->Insert(render); + render_delay_buffer->UpdateBuffers(); + EXPECT_FALSE(estimator.EstimateDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture)); + } +} + +// Verifies that the delay estimator does not produce delay estimates for render +// signals of low level. +TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) { + Random random_generator(42U); + EchoCanceller3Config config; + std::vector> render(3, std::vector(kBlockSize)); + std::vector capture(kBlockSize); + ApmDataDumper data_dumper(0); + EchoPathDelayEstimator estimator(&data_dumper, config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + for (size_t k = 0; k < 100; ++k) { + RandomizeSampleVector(&random_generator, render[0]); + for (auto& render_k : render[0]) { + render_k *= 100.f / 32767.f; + } + std::copy(render[0].begin(), render[0].end(), capture.begin()); + render_delay_buffer->Insert(render); + render_delay_buffer->UpdateBuffers(); + EXPECT_FALSE(estimator.EstimateDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture)); + } +} + +// Verifies that the delay estimator does not produce delay estimates for +// uncorrelated signals. +TEST(EchoPathDelayEstimator, NoDelayEstimatesForUncorrelatedSignals) { + Random random_generator(42U); + EchoCanceller3Config config; + std::vector> render(3, std::vector(kBlockSize)); + std::vector capture(kBlockSize); + ApmDataDumper data_dumper(0); + EchoPathDelayEstimator estimator(&data_dumper, config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + for (size_t k = 0; k < 100; ++k) { + RandomizeSampleVector(&random_generator, render[0]); + RandomizeSampleVector(&random_generator, capture); + render_delay_buffer->Insert(render); + render_delay_buffer->UpdateBuffers(); + EXPECT_FALSE(estimator.EstimateDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture)); + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for the render blocksize. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoPathDelayEstimator, DISABLED_WrongRenderBlockSize) { + ApmDataDumper data_dumper(0); + EchoCanceller3Config config; + EchoPathDelayEstimator estimator(&data_dumper, config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + std::vector capture(kBlockSize); + EXPECT_DEATH(estimator.EstimateDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture), + ""); +} + +// Verifies the check for the capture blocksize. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoPathDelayEstimator, WrongCaptureBlockSize) { + ApmDataDumper data_dumper(0); + EchoCanceller3Config config; + EchoPathDelayEstimator estimator(&data_dumper, config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, config.delay.down_sampling_factor, + GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters))); + std::vector capture(std::vector(kBlockSize - 1)); + EXPECT_DEATH(estimator.EstimateDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture), + ""); +} + +// Verifies the check for non-null data dumper. +TEST(EchoPathDelayEstimator, NullDataDumper) { + EXPECT_DEATH(EchoPathDelayEstimator(nullptr, EchoCanceller3Config()), ""); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.cc new file mode 100644 index 0000000000..f63a83006e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.cc @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/echo_path_variability.h" + +namespace webrtc { + +EchoPathVariability::EchoPathVariability(bool gain_change, bool delay_change) + : gain_change(gain_change), delay_change(delay_change) {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.h new file mode 100644 index 0000000000..55915d5b68 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_VARIABILITY_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_VARIABILITY_H_ + +namespace webrtc { + +struct EchoPathVariability { + EchoPathVariability(bool gain_change, bool delay_change); + + bool AudioPathChanged() const { return gain_change || delay_change; } + bool gain_change; + bool delay_change; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_VARIABILITY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability_unittest.cc new file mode 100644 index 0000000000..9a1df78885 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability_unittest.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/echo_path_variability.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(EchoPathVariability, CorrectBehavior) { + // Test correct passing and reporting of the gain change information. + EchoPathVariability v(true, true); + EXPECT_TRUE(v.gain_change); + EXPECT_TRUE(v.delay_change); + EXPECT_TRUE(v.AudioPathChanged()); + + v = EchoPathVariability(true, false); + EXPECT_TRUE(v.gain_change); + EXPECT_FALSE(v.delay_change); + EXPECT_TRUE(v.AudioPathChanged()); + + v = EchoPathVariability(false, true); + EXPECT_FALSE(v.gain_change); + EXPECT_TRUE(v.delay_change); + EXPECT_TRUE(v.AudioPathChanged()); + + v = EchoPathVariability(false, false); + EXPECT_FALSE(v.gain_change); + EXPECT_FALSE(v.delay_change); + EXPECT_FALSE(v.AudioPathChanged()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.cc new file mode 100644 index 0000000000..9adcec51f4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.cc @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/echo_remover.h" + +#include +#include +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/comfort_noise_generator.h" +#include "modules/audio_processing/aec3/echo_path_variability.h" +#include "modules/audio_processing/aec3/echo_remover_metrics.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "modules/audio_processing/aec3/output_selector.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/aec3/residual_echo_estimator.h" +#include "modules/audio_processing/aec3/subtractor.h" +#include "modules/audio_processing/aec3/suppression_filter.h" +#include "modules/audio_processing/aec3/suppression_gain.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +namespace { + +void LinearEchoPower(const FftData& E, + const FftData& Y, + std::array* S2) { + for (size_t k = 0; k < E.re.size(); ++k) { + (*S2)[k] = (Y.re[k] - E.re[k]) * (Y.re[k] - E.re[k]) + + (Y.im[k] - E.im[k]) * (Y.im[k] - E.im[k]); + } +} + +// Class for removing the echo from the capture signal. +class EchoRemoverImpl final : public EchoRemover { + public: + explicit EchoRemoverImpl(const EchoCanceller3Config& config, + int sample_rate_hz); + ~EchoRemoverImpl() override; + + void GetMetrics(EchoControl::Metrics* metrics) const override; + + // Removes the echo from a block of samples from the capture signal. The + // supplied render signal is assumed to be pre-aligned with the capture + // signal. + void ProcessCapture(const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const RenderBuffer& render_buffer, + std::vector>* capture) override; + + // Updates the status on whether echo leakage is detected in the output of the + // echo remover. + void UpdateEchoLeakageStatus(bool leakage_detected) override { + echo_leakage_detected_ = leakage_detected; + } + + private: + static int instance_count_; + const EchoCanceller3Config config_; + const Aec3Fft fft_; + std::unique_ptr data_dumper_; + const Aec3Optimization optimization_; + const int sample_rate_hz_; + Subtractor subtractor_; + SuppressionGain suppression_gain_; + ComfortNoiseGenerator cng_; + SuppressionFilter suppression_filter_; + RenderSignalAnalyzer render_signal_analyzer_; + OutputSelector output_selector_; + ResidualEchoEstimator residual_echo_estimator_; + bool echo_leakage_detected_ = false; + AecState aec_state_; + EchoRemoverMetrics metrics_; + + RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl); +}; + +int EchoRemoverImpl::instance_count_ = 0; + +EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, + int sample_rate_hz) + : config_(config), + fft_(), + data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + optimization_(DetectOptimization()), + sample_rate_hz_(sample_rate_hz), + subtractor_(data_dumper_.get(), optimization_), + suppression_gain_(config_, optimization_), + cng_(optimization_), + suppression_filter_(sample_rate_hz_), + residual_echo_estimator_(config_), + aec_state_(config_) { + RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); +} + +EchoRemoverImpl::~EchoRemoverImpl() = default; + +void EchoRemoverImpl::GetMetrics(EchoControl::Metrics* metrics) const { + // Echo return loss (ERL) is inverted to go from gain to attenuation. + metrics->echo_return_loss = -10.0 * log10(aec_state_.ErlTimeDomain()); + metrics->echo_return_loss_enhancement = + 10.0 * log10(aec_state_.ErleTimeDomain()); +} + +void EchoRemoverImpl::ProcessCapture( + const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const RenderBuffer& render_buffer, + std::vector>* capture) { + const std::vector>& x = render_buffer.MostRecentBlock(); + std::vector>* y = capture; + + RTC_DCHECK(y); + RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_)); + RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_)); + RTC_DCHECK_EQ(x[0].size(), kBlockSize); + RTC_DCHECK_EQ((*y)[0].size(), kBlockSize); + const std::vector& x0 = x[0]; + std::vector& y0 = (*y)[0]; + + data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, &y0[0], + LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, &x0[0], + LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpRaw("aec3_echo_remover_capture_input", y0); + data_dumper_->DumpRaw("aec3_echo_remover_render_input", x0); + + aec_state_.UpdateCaptureSaturation(capture_signal_saturation); + + if (echo_path_variability.AudioPathChanged()) { + subtractor_.HandleEchoPathChange(echo_path_variability); + aec_state_.HandleEchoPathChange(echo_path_variability); + } + + std::array Y2; + std::array R2; + std::array S2_linear; + std::array G; + float high_bands_gain; + FftData Y; + FftData comfort_noise; + FftData high_band_comfort_noise; + SubtractorOutput subtractor_output; + FftData& E_main = subtractor_output.E_main; + auto& E2_main = subtractor_output.E2_main; + auto& E2_shadow = subtractor_output.E2_shadow; + auto& e_main = subtractor_output.e_main; + + // Analyze the render signal. + render_signal_analyzer_.Update(render_buffer, aec_state_.FilterDelay()); + + // Perform linear echo cancellation. + subtractor_.Process(render_buffer, y0, render_signal_analyzer_, aec_state_, + &subtractor_output); + + // Compute spectra. + fft_.ZeroPaddedFft(y0, &Y); + LinearEchoPower(E_main, Y, &S2_linear); + Y.Spectrum(optimization_, &Y2); + + // Update the AEC state information. + aec_state_.Update(subtractor_.FilterFrequencyResponse(), + subtractor_.FilterImpulseResponse(), + subtractor_.ConvergedFilter(), echo_path_delay_samples, + render_buffer, E2_main, Y2, x0, subtractor_output.s_main, + echo_leakage_detected_); + + // Choose the linear output. + output_selector_.FormLinearOutput(!aec_state_.TransparentMode(), e_main, y0); + data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0], + LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpRaw("aec3_output_linear", y0); + const auto& E2 = output_selector_.UseSubtractorOutput() ? E2_main : Y2; + + // Estimate the residual echo power. + residual_echo_estimator_.Estimate(aec_state_, render_buffer, S2_linear, Y2, + &R2); + + // Estimate the comfort noise. + cng_.Compute(aec_state_, Y2, &comfort_noise, &high_band_comfort_noise); + + // A choose and apply echo suppression gain. + suppression_gain_.GetGain(E2, R2, cng_.NoiseSpectrum(), + render_signal_analyzer_, aec_state_, x, + &high_bands_gain, &G); + suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, + high_bands_gain, y); + + // Update the metrics. + metrics_.Update(aec_state_, cng_.NoiseSpectrum(), G); + + // Update the aec state with the aec output characteristics. + aec_state_.UpdateWithOutput(y0); + + // Debug outputs for the purpose of development and analysis. + data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize, + &subtractor_output.s_main[0], + LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpRaw("aec3_output", y0); + data_dumper_->DumpRaw("aec3_narrow_render", + render_signal_analyzer_.NarrowPeakBand() ? 1 : 0); + data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum()); + data_dumper_->DumpRaw("aec3_suppressor_gain", G); + data_dumper_->DumpWav("aec3_output", + rtc::ArrayView(&y0[0], kBlockSize), + LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpRaw("aec3_using_subtractor_output", + output_selector_.UseSubtractorOutput() ? 1 : 0); + data_dumper_->DumpRaw("aec3_E2", E2); + data_dumper_->DumpRaw("aec3_E2_main", E2_main); + data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow); + data_dumper_->DumpRaw("aec3_S2_linear", S2_linear); + data_dumper_->DumpRaw("aec3_Y2", Y2); + data_dumper_->DumpRaw("aec3_X2", render_buffer.Spectrum(0)); + data_dumper_->DumpRaw("aec3_R2", R2); + data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle()); + data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl()); + data_dumper_->DumpRaw("aec3_active_render", aec_state_.ActiveRender()); + data_dumper_->DumpRaw("aec3_usable_linear_estimate", + aec_state_.UsableLinearEstimate()); + data_dumper_->DumpRaw( + "aec3_filter_delay", + aec_state_.FilterDelay() ? *aec_state_.FilterDelay() : -1); + data_dumper_->DumpRaw( + "aec3_external_delay", + aec_state_.ExternalDelay() ? *aec_state_.ExternalDelay() : -1); + data_dumper_->DumpRaw("aec3_capture_saturation", + aec_state_.SaturatedCapture() ? 1 : 0); +} + +} // namespace + +EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config, + int sample_rate_hz) { + return new EchoRemoverImpl(config, sample_rate_hz); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.h new file mode 100644 index 0000000000..7411b5c45c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_ + +#include + +#include "api/optional.h" +#include "modules/audio_processing/aec3/echo_path_variability.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { + +// Class for removing the echo from the capture signal. +class EchoRemover { + public: + static EchoRemover* Create(const EchoCanceller3Config& config, + int sample_rate_hz); + virtual ~EchoRemover() = default; + + // Get current metrics. + virtual void GetMetrics(EchoControl::Metrics* metrics) const = 0; + + // Removes the echo from a block of samples from the capture signal. The + // supplied render signal is assumed to be pre-aligned with the capture + // signal. + virtual void ProcessCapture( + const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const RenderBuffer& render_buffer, + std::vector>* capture) = 0; + + // Updates the status on whether echo leakage is detected in the output of the + // echo remover. + virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc new file mode 100644 index 0000000000..3a71299b1e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/echo_remover_metrics.h" + +#include +#include +#include + +#include "rtc_base/numerics/safe_minmax.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +constexpr float kOneByMetricsCollectionBlocks = 1.f / kMetricsCollectionBlocks; + +} // namespace + +EchoRemoverMetrics::DbMetric::DbMetric() : DbMetric(0.f, 0.f, 0.f) {} +EchoRemoverMetrics::DbMetric::DbMetric(float sum_value, + float floor_value, + float ceil_value) + : sum_value(sum_value), floor_value(floor_value), ceil_value(ceil_value) {} + +void EchoRemoverMetrics::DbMetric::Update(float value) { + sum_value += value; + floor_value = std::min(floor_value, value); + ceil_value = std::max(ceil_value, value); +} + +EchoRemoverMetrics::EchoRemoverMetrics() { + ResetMetrics(); +} + +void EchoRemoverMetrics::ResetMetrics() { + erl_.fill(DbMetric(0.f, 10000.f, 0.000f)); + erle_.fill(DbMetric(0.f, 0.f, 1000.f)); + comfort_noise_.fill(DbMetric(0.f, 100000000.f, 0.f)); + suppressor_gain_.fill(DbMetric(0.f, 1.f, 0.f)); + active_render_count_ = 0; + saturated_capture_ = false; +} + +void EchoRemoverMetrics::Update( + const AecState& aec_state, + const std::array& comfort_noise_spectrum, + const std::array& suppressor_gain) { + metrics_reported_ = false; + if (++block_counter_ <= kMetricsCollectionBlocks) { + aec3::UpdateDbMetric(aec_state.Erl(), &erl_); + aec3::UpdateDbMetric(aec_state.Erle(), &erle_); + aec3::UpdateDbMetric(comfort_noise_spectrum, &comfort_noise_); + aec3::UpdateDbMetric(suppressor_gain, &suppressor_gain_); + active_render_count_ += (aec_state.ActiveRender() ? 1 : 0); + saturated_capture_ = saturated_capture_ || aec_state.SaturatedCapture(); + } else { + // Report the metrics over several frames in order to lower the impact of + // the logarithms involved on the computational complexity. + constexpr int kMetricsCollectionBlocksBy2 = kMetricsCollectionBlocks / 2; + constexpr float kComfortNoiseScaling = 1.f / (kBlockSize * kBlockSize); + switch (block_counter_) { + case kMetricsCollectionBlocks + 1: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErleBand0.Average", + aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, + kOneByMetricsCollectionBlocks, + erle_[0].sum_value), + 0, 19, 20); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErleBand0.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, + erle_[0].ceil_value), + 0, 19, 20); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErleBand0.Min", + aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, + erle_[0].floor_value), + 0, 19, 20); + break; + case kMetricsCollectionBlocks + 2: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErleBand1.Average", + aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, + kOneByMetricsCollectionBlocks, + erle_[1].sum_value), + 0, 19, 20); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErleBand1.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, + erle_[1].ceil_value), + 0, 19, 20); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErleBand1.Min", + aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, + erle_[1].floor_value), + 0, 19, 20); + break; + case kMetricsCollectionBlocks + 3: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErlBand0.Average", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, + kOneByMetricsCollectionBlocks, + erl_[0].sum_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErlBand0.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, + erl_[0].ceil_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErlBand0.Min", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, + erl_[0].floor_value), + 0, 59, 30); + break; + case kMetricsCollectionBlocks + 4: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErlBand1.Average", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, + kOneByMetricsCollectionBlocks, + erl_[1].sum_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErlBand1.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, + erl_[1].ceil_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ErlBand1.Min", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, + erl_[1].floor_value), + 0, 59, 30); + break; + case kMetricsCollectionBlocks + 5: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ComfortNoiseBand0.Average", + aec3::TransformDbMetricForReporting( + true, 0.f, 89.f, -90.3f, + kComfortNoiseScaling * kOneByMetricsCollectionBlocks, + comfort_noise_[0].sum_value), + 0, 89, 45); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ComfortNoiseBand0.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 89.f, -90.3f, + kComfortNoiseScaling, + comfort_noise_[0].ceil_value), + 0, 89, 45); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ComfortNoiseBand0.Min", + aec3::TransformDbMetricForReporting(true, 0.f, 89.f, -90.3f, + kComfortNoiseScaling, + comfort_noise_[0].floor_value), + 0, 89, 45); + break; + case kMetricsCollectionBlocks + 6: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ComfortNoiseBand1.Average", + aec3::TransformDbMetricForReporting( + true, 0.f, 89.f, -90.3f, + kComfortNoiseScaling * kOneByMetricsCollectionBlocks, + comfort_noise_[1].sum_value), + 0, 89, 45); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ComfortNoiseBand1.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 89.f, -90.3f, + kComfortNoiseScaling, + comfort_noise_[1].ceil_value), + 0, 89, 45); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.ComfortNoiseBand1.Min", + aec3::TransformDbMetricForReporting(true, 0.f, 89.f, -90.3f, + kComfortNoiseScaling, + comfort_noise_[1].floor_value), + 0, 89, 45); + break; + case kMetricsCollectionBlocks + 7: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.SuppressorGainBand0.Average", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 0.f, + kOneByMetricsCollectionBlocks, + suppressor_gain_[0].sum_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.SuppressorGainBand0.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 0.f, 1.f, + suppressor_gain_[0].ceil_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.SuppressorGainBand0.Min", + aec3::TransformDbMetricForReporting( + true, 0.f, 59.f, 0.f, 1.f, suppressor_gain_[0].floor_value), + 0, 59, 30); + break; + case kMetricsCollectionBlocks + 8: + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.SuppressorGainBand1.Average", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 0.f, + kOneByMetricsCollectionBlocks, + suppressor_gain_[1].sum_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.SuppressorGainBand1.Max", + aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 0.f, 1.f, + suppressor_gain_[1].ceil_value), + 0, 59, 30); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.SuppressorGainBand1.Min", + aec3::TransformDbMetricForReporting( + true, 0.f, 59.f, 0.f, 1.f, suppressor_gain_[1].floor_value), + 0, 59, 30); + break; + case kMetricsCollectionBlocks + 9: + RTC_HISTOGRAM_BOOLEAN( + "WebRTC.Audio.EchoCanceller.UsableLinearEstimate", + static_cast(aec_state.UsableLinearEstimate() ? 1 : 0)); + RTC_HISTOGRAM_BOOLEAN( + "WebRTC.Audio.EchoCanceller.ActiveRender", + static_cast( + active_render_count_ > kMetricsCollectionBlocksBy2 ? 1 : 0)); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.EchoCanceller.FilterDelay", + aec_state.FilterDelay() ? *aec_state.FilterDelay() + 1 : 0, 0, 30, + 31); + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.EchoCanceller.CaptureSaturation", + static_cast(saturated_capture_ ? 1 : 0)); + metrics_reported_ = true; + RTC_DCHECK_EQ(kMetricsReportingIntervalBlocks, block_counter_); + block_counter_ = 0; + ResetMetrics(); + break; + default: + RTC_NOTREACHED(); + break; + } + } +} + +namespace aec3 { + +void UpdateDbMetric(const std::array& value, + std::array* statistic) { + RTC_DCHECK(statistic); + // Truncation is intended in the band width computation. + constexpr int kNumBands = 2; + constexpr int kBandWidth = 65 / kNumBands; + constexpr float kOneByBandWidth = 1.f / kBandWidth; + RTC_DCHECK_EQ(kNumBands, statistic->size()); + RTC_DCHECK_EQ(65, value.size()); + for (size_t k = 0; k < statistic->size(); ++k) { + float average_band = + std::accumulate(value.begin() + kBandWidth * k, + value.begin() + kBandWidth * (k + 1), 0.f) * + kOneByBandWidth; + (*statistic)[k].Update(average_band); + } +} + +int TransformDbMetricForReporting(bool negate, + float min_value, + float max_value, + float offset, + float scaling, + float value) { + float new_value = 10.f * log10(value * scaling + 1e-10f) + offset; + if (negate) { + new_value = -new_value; + } + return static_cast(rtc::SafeClamp(new_value, min_value, max_value)); +} + +} // namespace aec3 + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h new file mode 100644 index 0000000000..6548ae8aa2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_METRICS_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_METRICS_H_ + +#include "modules/audio_processing/aec3/aec_state.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Handles the reporting of metrics for the echo remover. +class EchoRemoverMetrics { + public: + struct DbMetric { + DbMetric(); + DbMetric(float sum_value, float floor_value, float ceil_value); + void Update(float value); + float sum_value; + float floor_value; + float ceil_value; + }; + + EchoRemoverMetrics(); + + // Updates the metric with new data. + void Update( + const AecState& aec_state, + const std::array& comfort_noise_spectrum, + const std::array& suppressor_gain); + + // Returns true if the metrics have just been reported, otherwise false. + bool MetricsReported() { return metrics_reported_; } + + private: + // Resets the metrics. + void ResetMetrics(); + + int block_counter_ = 0; + std::array erl_; + std::array erle_; + std::array comfort_noise_; + std::array suppressor_gain_; + int active_render_count_ = 0; + bool saturated_capture_ = false; + bool metrics_reported_ = false; + + RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverMetrics); +}; + +namespace aec3 { + +// Updates a banded metric of type DbMetric with the values in the supplied +// array. +void UpdateDbMetric(const std::array& value, + std::array* statistic); + +// Transforms a DbMetric from the linear domain into the logarithmic domain. +int TransformDbMetricForReporting(bool negate, + float min_value, + float max_value, + float offset, + float scaling, + float value); + +} // namespace aec3 + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_METRICS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc new file mode 100644 index 0000000000..b8b20d7722 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/echo_remover_metrics.h" + +#include + +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "test/gtest.h" + +namespace webrtc { + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for non-null input. +TEST(UpdateDbMetric, NullValue) { + std::array value; + value.fill(0.f); + EXPECT_DEATH(aec3::UpdateDbMetric(value, nullptr), ""); +} + +#endif + +// Verifies the updating functionality of UpdateDbMetric. +TEST(UpdateDbMetric, Updating) { + std::array value; + std::array statistic; + statistic.fill(EchoRemoverMetrics::DbMetric(0.f, 100.f, -100.f)); + constexpr float kValue0 = 10.f; + constexpr float kValue1 = 20.f; + std::fill(value.begin(), value.begin() + 32, kValue0); + std::fill(value.begin() + 32, value.begin() + 64, kValue1); + + aec3::UpdateDbMetric(value, &statistic); + EXPECT_FLOAT_EQ(kValue0, statistic[0].sum_value); + EXPECT_FLOAT_EQ(kValue0, statistic[0].ceil_value); + EXPECT_FLOAT_EQ(kValue0, statistic[0].floor_value); + EXPECT_FLOAT_EQ(kValue1, statistic[1].sum_value); + EXPECT_FLOAT_EQ(kValue1, statistic[1].ceil_value); + EXPECT_FLOAT_EQ(kValue1, statistic[1].floor_value); + + aec3::UpdateDbMetric(value, &statistic); + EXPECT_FLOAT_EQ(2.f * kValue0, statistic[0].sum_value); + EXPECT_FLOAT_EQ(kValue0, statistic[0].ceil_value); + EXPECT_FLOAT_EQ(kValue0, statistic[0].floor_value); + EXPECT_FLOAT_EQ(2.f * kValue1, statistic[1].sum_value); + EXPECT_FLOAT_EQ(kValue1, statistic[1].ceil_value); + EXPECT_FLOAT_EQ(kValue1, statistic[1].floor_value); +} + +// Verifies that the TransformDbMetricForReporting method produces the desired +// output for values for dBFS. +TEST(TransformDbMetricForReporting, DbFsScaling) { + std::array x; + FftData X; + std::array X2; + Aec3Fft fft; + x.fill(1000.f); + fft.ZeroPaddedFft(x, &X); + X.Spectrum(Aec3Optimization::kNone, &X2); + + float offset = -10.f * log10(32768.f * 32768.f); + EXPECT_NEAR(offset, -90.3f, 0.1f); + EXPECT_EQ( + static_cast(30.3f), + aec3::TransformDbMetricForReporting( + true, 0.f, 90.f, offset, 1.f / (kBlockSize * kBlockSize), X2[0])); +} + +// Verifies that the TransformDbMetricForReporting method is able to properly +// limit the output. +TEST(TransformDbMetricForReporting, Limits) { + EXPECT_EQ( + 0, + aec3::TransformDbMetricForReporting(false, 0.f, 10.f, 0.f, 1.f, 0.001f)); + EXPECT_EQ( + 10, + aec3::TransformDbMetricForReporting(false, 0.f, 10.f, 0.f, 1.f, 100.f)); +} + +// Verifies that the TransformDbMetricForReporting method is able to properly +// negate output. +TEST(TransformDbMetricForReporting, Negate) { + EXPECT_EQ( + 10, + aec3::TransformDbMetricForReporting(true, -20.f, 20.f, 0.f, 1.f, 0.1f)); + EXPECT_EQ( + -10, + aec3::TransformDbMetricForReporting(true, -20.f, 20.f, 0.f, 1.f, 10.f)); +} + +// Verify the Update functionality of DbMetric. +TEST(DbMetric, Update) { + EchoRemoverMetrics::DbMetric metric(0.f, 20.f, -20.f); + constexpr int kNumValues = 100; + constexpr float kValue = 10.f; + for (int k = 0; k < kNumValues; ++k) { + metric.Update(kValue); + } + EXPECT_FLOAT_EQ(kValue * kNumValues, metric.sum_value); + EXPECT_FLOAT_EQ(kValue, metric.ceil_value); + EXPECT_FLOAT_EQ(kValue, metric.floor_value); +} + +// Verify the constructor functionality of DbMetric. +TEST(DbMetric, Constructor) { + EchoRemoverMetrics::DbMetric metric; + EXPECT_FLOAT_EQ(0.f, metric.sum_value); + EXPECT_FLOAT_EQ(0.f, metric.ceil_value); + EXPECT_FLOAT_EQ(0.f, metric.floor_value); + + metric = EchoRemoverMetrics::DbMetric(1.f, 2.f, 3.f); + EXPECT_FLOAT_EQ(1.f, metric.sum_value); + EXPECT_FLOAT_EQ(2.f, metric.floor_value); + EXPECT_FLOAT_EQ(3.f, metric.ceil_value); +} + +// Verify the general functionality of EchoRemoverMetrics. +TEST(EchoRemoverMetrics, NormalUsage) { + EchoRemoverMetrics metrics; + AecState aec_state(EchoCanceller3Config{}); + std::array comfort_noise_spectrum; + std::array suppressor_gain; + comfort_noise_spectrum.fill(10.f); + suppressor_gain.fill(1.f); + for (int j = 0; j < 3; ++j) { + for (int k = 0; k < kMetricsReportingIntervalBlocks - 1; ++k) { + metrics.Update(aec_state, comfort_noise_spectrum, suppressor_gain); + EXPECT_FALSE(metrics.MetricsReported()); + } + metrics.Update(aec_state, comfort_noise_spectrum, suppressor_gain); + EXPECT_TRUE(metrics.MetricsReported()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc new file mode 100644 index 0000000000..24b50e8c95 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/echo_remover.h" + +#include +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +std::string ProduceDebugText(int sample_rate_hz, int delay) { + std::ostringstream ss(ProduceDebugText(sample_rate_hz)); + ss << ", Delay: " << delay; + return ss.str(); +} + +constexpr size_t kDownSamplingFactor = 4; +constexpr size_t kNumMatchedFilters = 4; + +} // namespace + +// Verifies the basic API call sequence +TEST(EchoRemover, BasicApiCalls) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr remover( + EchoRemover::Create(EchoCanceller3Config(), rate)); + std::unique_ptr render_buffer(RenderDelayBuffer::Create( + NumBandsForRate(rate), kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + + std::vector> render(NumBandsForRate(rate), + std::vector(kBlockSize, 0.f)); + std::vector> capture( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (size_t k = 0; k < 100; ++k) { + EchoPathVariability echo_path_variability(k % 3 == 0 ? true : false, + k % 5 == 0 ? true : false); + rtc::Optional echo_path_delay_samples = + (k % 6 == 0 ? rtc::Optional(k * 10) + : rtc::nullopt); + render_buffer->Insert(render); + render_buffer->UpdateBuffers(); + remover->ProcessCapture(echo_path_delay_samples, echo_path_variability, + k % 2 == 0 ? true : false, + render_buffer->GetRenderBuffer(), &capture); + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for the samplerate. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoRemover, DISABLED_WrongSampleRate) { + EXPECT_DEATH(std::unique_ptr( + EchoRemover::Create(EchoCanceller3Config(), 8001)), + ""); +} + +// Verifies the check for the capture block size. +TEST(EchoRemover, WrongCaptureBlockSize) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr remover( + EchoRemover::Create(EchoCanceller3Config(), rate)); + std::unique_ptr render_buffer(RenderDelayBuffer::Create( + NumBandsForRate(rate), kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::vector> capture( + NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH(remover->ProcessCapture( + echo_path_delay_samples, echo_path_variability, false, + render_buffer->GetRenderBuffer(), &capture), + ""); + } +} + +// Verifies the check for the number of capture bands. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed.c +TEST(EchoRemover, DISABLED_WrongCaptureNumBands) { + for (auto rate : {16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr remover( + EchoRemover::Create(EchoCanceller3Config(), rate)); + std::unique_ptr render_buffer(RenderDelayBuffer::Create( + NumBandsForRate(rate), kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::vector> capture( + NumBandsForRate(rate == 48000 ? 16000 : rate + 16000), + std::vector(kBlockSize, 0.f)); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH(remover->ProcessCapture( + echo_path_delay_samples, echo_path_variability, false, + render_buffer->GetRenderBuffer(), &capture), + ""); + } +} + +// Verifies the check for non-null capture block. +TEST(EchoRemover, NullCapture) { + std::unique_ptr remover( + EchoRemover::Create(EchoCanceller3Config(), 8000)); + std::unique_ptr render_buffer(RenderDelayBuffer::Create( + 3, kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH( + remover->ProcessCapture(echo_path_delay_samples, echo_path_variability, + false, render_buffer->GetRenderBuffer(), nullptr), + ""); +} + +#endif + +// Performs a sanity check that the echo_remover is able to properly +// remove echoes. +TEST(EchoRemover, BasicEchoRemoval) { + constexpr int kNumBlocksToProcess = 500; + Random random_generator(42U); + for (auto rate : {8000, 16000, 32000, 48000}) { + std::vector> x(NumBandsForRate(rate), + std::vector(kBlockSize, 0.f)); + std::vector> y(NumBandsForRate(rate), + std::vector(kBlockSize, 0.f)); + EchoPathVariability echo_path_variability(false, false); + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + std::unique_ptr remover( + EchoRemover::Create(EchoCanceller3Config(), rate)); + std::unique_ptr render_buffer( + RenderDelayBuffer::Create( + NumBandsForRate(rate), kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, + kNumMatchedFilters))); + std::vector>> delay_buffers(x.size()); + for (size_t j = 0; j < x.size(); ++j) { + delay_buffers[j].reset(new DelayBuffer(delay_samples)); + } + + float input_energy = 0.f; + float output_energy = 0.f; + for (int k = 0; k < kNumBlocksToProcess; ++k) { + const bool silence = k < 100 || (k % 100 >= 10); + + for (size_t j = 0; j < x.size(); ++j) { + if (silence) { + std::fill(x[j].begin(), x[j].end(), 0.f); + } else { + RandomizeSampleVector(&random_generator, x[j]); + } + delay_buffers[j]->Delay(x[j], y[j]); + } + + if (k > kNumBlocksToProcess / 2) { + for (size_t j = 0; j < x.size(); ++j) { + input_energy = std::inner_product(y[j].begin(), y[j].end(), + y[j].begin(), input_energy); + } + } + + render_buffer->Insert(x); + render_buffer->UpdateBuffers(); + + remover->ProcessCapture(delay_samples, echo_path_variability, false, + render_buffer->GetRenderBuffer(), &y); + + if (k > kNumBlocksToProcess / 2) { + for (size_t j = 0; j < x.size(); ++j) { + output_energy = std::inner_product(y[j].begin(), y[j].end(), + y[j].begin(), output_energy); + } + } + } + EXPECT_GT(input_energy, 10.f * output_energy); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.cc new file mode 100644 index 0000000000..3f12ba41a1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.cc @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/erl_estimator.h" + +#include +#include + +namespace webrtc { + +namespace { + +constexpr float kMinErl = 0.01f; +constexpr float kMaxErl = 1000.f; + +} // namespace + +ErlEstimator::ErlEstimator() { + erl_.fill(kMaxErl); + hold_counters_.fill(0); + erl_time_domain_ = kMaxErl; + hold_counter_time_domain_ = 0; +} + +ErlEstimator::~ErlEstimator() = default; + +void ErlEstimator::Update( + const std::array& render_spectrum, + const std::array& capture_spectrum) { + const auto& X2 = render_spectrum; + const auto& Y2 = capture_spectrum; + + // Corresponds to WGN of power -46 dBFS. + constexpr float kX2Min = 44015068.0f; + + // Update the estimates in a maximum statistics manner. + for (size_t k = 1; k < kFftLengthBy2; ++k) { + if (X2[k] > kX2Min) { + const float new_erl = Y2[k] / X2[k]; + if (new_erl < erl_[k]) { + hold_counters_[k - 1] = 1000; + erl_[k] += 0.1f * (new_erl - erl_[k]); + erl_[k] = std::max(erl_[k], kMinErl); + } + } + } + + std::for_each(hold_counters_.begin(), hold_counters_.end(), + [](int& a) { --a; }); + std::transform(hold_counters_.begin(), hold_counters_.end(), erl_.begin() + 1, + erl_.begin() + 1, [](int a, float b) { + return a > 0 ? b : std::min(kMaxErl, 2.f * b); + }); + + erl_[0] = erl_[1]; + erl_[kFftLengthBy2] = erl_[kFftLengthBy2 - 1]; + + // Compute ERL over all frequency bins. + const float X2_sum = std::accumulate(X2.begin(), X2.end(), 0.0f); + + if (X2_sum > kX2Min * X2.size()) { + const float Y2_sum = std::accumulate(Y2.begin(), Y2.end(), 0.0f); + const float new_erl = Y2_sum / X2_sum; + if (new_erl < erl_time_domain_) { + hold_counter_time_domain_ = 1000; + erl_time_domain_ += 0.1f * (new_erl - erl_time_domain_); + erl_time_domain_ = std::max(erl_time_domain_, kMinErl); + } + } + + --hold_counter_time_domain_; + erl_time_domain_ = (hold_counter_time_domain_ > 0) + ? erl_time_domain_ + : std::min(kMaxErl, 2.f * erl_time_domain_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.h new file mode 100644 index 0000000000..24b3f4b104 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ERL_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ERL_ESTIMATOR_H_ + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Estimates the echo return loss based on the signal spectra. +class ErlEstimator { + public: + ErlEstimator(); + ~ErlEstimator(); + + // Updates the ERL estimate. + void Update(const std::array& render_spectrum, + const std::array& capture_spectrum); + + // Returns the most recent ERL estimate. + const std::array& Erl() const { return erl_; } + float ErlTimeDomain() const { return erl_time_domain_; } + + private: + std::array erl_; + std::array hold_counters_; + float erl_time_domain_; + int hold_counter_time_domain_; + + RTC_DISALLOW_COPY_AND_ASSIGN(ErlEstimator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ERL_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator_unittest.cc new file mode 100644 index 0000000000..a406581001 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator_unittest.cc @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/erl_estimator.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +void VerifyErl(const std::array& erl, + float erl_time_domain, + float reference) { + std::for_each(erl.begin(), erl.end(), + [reference](float a) { EXPECT_NEAR(reference, a, 0.001); }); + EXPECT_NEAR(reference, erl_time_domain, 0.001); +} + +} // namespace + +// Verifies that the correct ERL estimates are achieved. +TEST(ErlEstimator, Estimates) { + std::array X2; + std::array Y2; + + ErlEstimator estimator; + + // Verifies that the ERL estimate is properly reduced to lower values. + X2.fill(500 * 1000.f * 1000.f); + Y2.fill(10 * X2[0]); + for (size_t k = 0; k < 200; ++k) { + estimator.Update(X2, Y2); + } + VerifyErl(estimator.Erl(), estimator.ErlTimeDomain(), 10.f); + + // Verifies that the ERL is not immediately increased when the ERL in the data + // increases. + Y2.fill(10000 * X2[0]); + for (size_t k = 0; k < 998; ++k) { + estimator.Update(X2, Y2); + } + VerifyErl(estimator.Erl(), estimator.ErlTimeDomain(), 10.f); + + // Verifies that the rate of increase is 3 dB. + estimator.Update(X2, Y2); + VerifyErl(estimator.Erl(), estimator.ErlTimeDomain(), 20.f); + + // Verifies that the maximum ERL is achieved when there are no low RLE + // estimates. + for (size_t k = 0; k < 1000; ++k) { + estimator.Update(X2, Y2); + } + VerifyErl(estimator.Erl(), estimator.ErlTimeDomain(), 1000.f); + + // Verifies that the ERL estimate is is not updated for low-level signals + X2.fill(1000.f * 1000.f); + Y2.fill(10 * X2[0]); + for (size_t k = 0; k < 200; ++k) { + estimator.Update(X2, Y2); + } + VerifyErl(estimator.Erl(), estimator.ErlTimeDomain(), 1000.f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.cc new file mode 100644 index 0000000000..385e6dd7e6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/erle_estimator.h" + +#include +#include + +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +ErleEstimator::ErleEstimator(float min_erle, + float max_erle_lf, + float max_erle_hf) + : min_erle_(min_erle), + max_erle_lf_(max_erle_lf), + max_erle_hf_(max_erle_hf) { + erle_.fill(min_erle_); + hold_counters_.fill(0); + erle_time_domain_ = min_erle_; + hold_counter_time_domain_ = 0; +} + +ErleEstimator::~ErleEstimator() = default; + +void ErleEstimator::Update( + const std::array& render_spectrum, + const std::array& capture_spectrum, + const std::array& subtractor_spectrum) { + const auto& X2 = render_spectrum; + const auto& Y2 = capture_spectrum; + const auto& E2 = subtractor_spectrum; + + // Corresponds of WGN of power -46 dBFS. + constexpr float kX2Min = 44015068.0f; + + // Update the estimates in a clamped minimum statistics manner. + auto erle_update = [&](size_t start, size_t stop, float max_erle) { + for (size_t k = start; k < stop; ++k) { + if (X2[k] > kX2Min && E2[k] > 0.f) { + const float new_erle = Y2[k] / E2[k]; + if (new_erle > erle_[k]) { + hold_counters_[k - 1] = 100; + erle_[k] += 0.1f * (new_erle - erle_[k]); + erle_[k] = rtc::SafeClamp(erle_[k], min_erle_, max_erle); + } + } + } + }; + erle_update(1, kFftLengthBy2 / 2, max_erle_lf_); + erle_update(kFftLengthBy2 / 2, kFftLengthBy2, max_erle_hf_); + + std::for_each(hold_counters_.begin(), hold_counters_.end(), + [](int& a) { --a; }); + std::transform(hold_counters_.begin(), hold_counters_.end(), + erle_.begin() + 1, erle_.begin() + 1, [&](int a, float b) { + return a > 0 ? b : std::max(min_erle_, 0.97f * b); + }); + + erle_[0] = erle_[1]; + erle_[kFftLengthBy2] = erle_[kFftLengthBy2 - 1]; + + // Compute ERLE over all frequency bins. + const float X2_sum = std::accumulate(X2.begin(), X2.end(), 0.0f); + const float E2_sum = std::accumulate(E2.begin(), E2.end(), 0.0f); + if (X2_sum > kX2Min * X2.size() && E2_sum > 0.f) { + const float Y2_sum = std::accumulate(Y2.begin(), Y2.end(), 0.0f); + const float new_erle = Y2_sum / E2_sum; + if (new_erle > erle_time_domain_) { + hold_counter_time_domain_ = 100; + erle_time_domain_ += 0.1f * (new_erle - erle_time_domain_); + erle_time_domain_ = + rtc::SafeClamp(erle_time_domain_, min_erle_, max_erle_lf_); + } + } + --hold_counter_time_domain_; + erle_time_domain_ = (hold_counter_time_domain_ > 0) + ? erle_time_domain_ + : std::max(min_erle_, 0.97f * erle_time_domain_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.h new file mode 100644 index 0000000000..d88b11bbb8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_ERLE_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_ERLE_ESTIMATOR_H_ + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Estimates the echo return loss enhancement based on the signal spectra. +class ErleEstimator { + public: + ErleEstimator(float min_erle, float max_erle_lf, float max_erle_hf); + ~ErleEstimator(); + + // Updates the ERLE estimate. + void Update(const std::array& render_spectrum, + const std::array& capture_spectrum, + const std::array& subtractor_spectrum); + + // Returns the most recent ERLE estimate. + const std::array& Erle() const { return erle_; } + float ErleTimeDomain() const { return erle_time_domain_; } + + private: + std::array erle_; + std::array hold_counters_; + float erle_time_domain_; + int hold_counter_time_domain_; + const float min_erle_; + const float max_erle_lf_; + const float max_erle_hf_; + + RTC_DISALLOW_COPY_AND_ASSIGN(ErleEstimator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_ERLE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator_unittest.cc new file mode 100644 index 0000000000..f3dd7d9bbb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator_unittest.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/erle_estimator.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kLowFrequencyLimit = kFftLengthBy2 / 2; + +void VerifyErle(const std::array& erle, + float erle_time_domain, + float reference_lf, + float reference_hf) { + std::for_each( + erle.begin(), erle.begin() + kLowFrequencyLimit, + [reference_lf](float a) { EXPECT_NEAR(reference_lf, a, 0.001); }); + std::for_each( + erle.begin() + kLowFrequencyLimit, erle.end(), + [reference_hf](float a) { EXPECT_NEAR(reference_hf, a, 0.001); }); + EXPECT_NEAR(reference_lf, erle_time_domain, 0.001); +} + +} // namespace + +// Verifies that the correct ERLE estimates are achieved. +TEST(ErleEstimator, Estimates) { + std::array X2; + std::array E2; + std::array Y2; + + ErleEstimator estimator(1.f, 8.f, 1.5f); + + // Verifies that the ERLE estimate is properley increased to higher values. + X2.fill(500 * 1000.f * 1000.f); + E2.fill(1000.f * 1000.f); + Y2.fill(10 * E2[0]); + for (size_t k = 0; k < 200; ++k) { + estimator.Update(X2, Y2, E2); + } + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f); + + // Verifies that the ERLE is not immediately decreased when the ERLE in the + // data decreases. + Y2.fill(0.1f * E2[0]); + for (size_t k = 0; k < 98; ++k) { + estimator.Update(X2, Y2, E2); + } + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f); + + // Verifies that the minimum ERLE is eventually achieved. + for (size_t k = 0; k < 1000; ++k) { + estimator.Update(X2, Y2, E2); + } + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f); + + // Verifies that the ERLE estimate is is not updated for low-level render + // signals. + X2.fill(1000.f * 1000.f); + Y2.fill(10 * E2[0]); + for (size_t k = 0; k < 200; ++k) { + estimator.Update(X2, Y2, E2); + } + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data.h new file mode 100644 index 0000000000..a5c51bf342 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_FFT_DATA_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_FFT_DATA_H_ + +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +// Struct that holds imaginary data produced from 128 point real-valued FFTs. +struct FftData { + // Copies the data in src. + void Assign(const FftData& src) { + std::copy(src.re.begin(), src.re.end(), re.begin()); + std::copy(src.im.begin(), src.im.end(), im.begin()); + im[0] = im[kFftLengthBy2] = 0; + } + + // Clears all the imaginary. + void Clear() { + re.fill(0.f); + im.fill(0.f); + } + + // Computes the power spectrum of the data. + void Spectrum(Aec3Optimization optimization, + std::array* power_spectrum) const { + RTC_DCHECK(power_spectrum); + switch (optimization) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: { + constexpr int kNumFourBinBands = kFftLengthBy2 / 4; + constexpr int kLimit = kNumFourBinBands * 4; + for (size_t k = 0; k < kLimit; k += 4) { + const __m128 r = _mm_loadu_ps(&re[k]); + const __m128 i = _mm_loadu_ps(&im[k]); + const __m128 ii = _mm_mul_ps(i, i); + const __m128 rr = _mm_mul_ps(r, r); + const __m128 rrii = _mm_add_ps(rr, ii); + _mm_storeu_ps(&(*power_spectrum)[k], rrii); + } + (*power_spectrum)[kFftLengthBy2] = + re[kFftLengthBy2] * re[kFftLengthBy2] + + im[kFftLengthBy2] * im[kFftLengthBy2]; + } break; +#endif + default: + std::transform(re.begin(), re.end(), im.begin(), + power_spectrum->begin(), + [](float a, float b) { return a * a + b * b; }); + } + } + + // Copy the data from an interleaved array. + void CopyFromPackedArray(const std::array& v) { + re[0] = v[0]; + re[kFftLengthBy2] = v[1]; + im[0] = im[kFftLengthBy2] = 0; + for (size_t k = 1, j = 2; k < kFftLengthBy2; ++k) { + re[k] = v[j++]; + im[k] = v[j++]; + } + } + + // Copies the data into an interleaved array. + void CopyToPackedArray(std::array* v) const { + RTC_DCHECK(v); + (*v)[0] = re[0]; + (*v)[1] = re[kFftLengthBy2]; + for (size_t k = 1, j = 2; k < kFftLengthBy2; ++k) { + (*v)[j++] = re[k]; + (*v)[j++] = im[k]; + } + } + + std::array re; + std::array im; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_FFT_DATA_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc new file mode 100644 index 0000000000..d969744c04 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/fft_data.h" + +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Verifies that the optimized methods are bitexact to their reference +// counterparts. +TEST(FftData, TestOptimizations) { + if (WebRtc_GetCPUInfo(kSSE2) != 0) { + FftData x; + + for (size_t k = 0; k < x.re.size(); ++k) { + x.re[k] = k + 1; + } + + x.im[0] = x.im[x.im.size() - 1] = 0.f; + for (size_t k = 1; k < x.im.size() - 1; ++k) { + x.im[k] = 2.f * (k + 1); + } + + std::array spectrum; + std::array spectrum_sse2; + x.Spectrum(Aec3Optimization::kNone, &spectrum); + x.Spectrum(Aec3Optimization::kSse2, &spectrum_sse2); + EXPECT_EQ(spectrum, spectrum_sse2); + } +} +#endif + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for null output in CopyToPackedArray. +TEST(FftData, NonNullCopyToPackedArrayOutput) { + EXPECT_DEATH(FftData().CopyToPackedArray(nullptr), ""); +} + +// Verifies the check for null output in Spectrum. +TEST(FftData, NonNullSpectrumOutput) { + EXPECT_DEATH(FftData().Spectrum(Aec3Optimization::kNone, nullptr), ""); +} + +#endif + +// Verifies that the Assign method properly copies the data from the source and +// ensures that the imaginary components for the DC and Nyquist bins are 0. +TEST(FftData, Assign) { + FftData x; + FftData y; + + x.re.fill(1.f); + x.im.fill(2.f); + y.Assign(x); + EXPECT_EQ(x.re, y.re); + EXPECT_EQ(0.f, y.im[0]); + EXPECT_EQ(0.f, y.im[x.im.size() - 1]); + for (size_t k = 1; k < x.im.size() - 1; ++k) { + EXPECT_EQ(x.im[k], y.im[k]); + } +} + +// Verifies that the Clear method properly clears all the data. +TEST(FftData, Clear) { + FftData x_ref; + FftData x; + + x_ref.re.fill(0.f); + x_ref.im.fill(0.f); + + x.re.fill(1.f); + x.im.fill(2.f); + x.Clear(); + + EXPECT_EQ(x_ref.re, x.re); + EXPECT_EQ(x_ref.im, x.im); +} + +// Verifies that the spectrum is correctly computed. +TEST(FftData, Spectrum) { + FftData x; + + for (size_t k = 0; k < x.re.size(); ++k) { + x.re[k] = k + 1; + } + + x.im[0] = x.im[x.im.size() - 1] = 0.f; + for (size_t k = 1; k < x.im.size() - 1; ++k) { + x.im[k] = 2.f * (k + 1); + } + + std::array spectrum; + x.Spectrum(Aec3Optimization::kNone, &spectrum); + + EXPECT_EQ(x.re[0] * x.re[0], spectrum[0]); + EXPECT_EQ(x.re[spectrum.size() - 1] * x.re[spectrum.size() - 1], + spectrum[spectrum.size() - 1]); + for (size_t k = 1; k < spectrum.size() - 1; ++k) { + EXPECT_EQ(x.re[k] * x.re[k] + x.im[k] * x.im[k], spectrum[k]); + } +} + +// Verifies that the functionality in CopyToPackedArray works as intended. +TEST(FftData, CopyToPackedArray) { + FftData x; + std::array x_packed; + + for (size_t k = 0; k < x.re.size(); ++k) { + x.re[k] = k + 1; + } + + x.im[0] = x.im[x.im.size() - 1] = 0.f; + for (size_t k = 1; k < x.im.size() - 1; ++k) { + x.im[k] = 2.f * (k + 1); + } + + x.CopyToPackedArray(&x_packed); + + EXPECT_EQ(x.re[0], x_packed[0]); + EXPECT_EQ(x.re[x.re.size() - 1], x_packed[1]); + for (size_t k = 1; k < x_packed.size() / 2; ++k) { + EXPECT_EQ(x.re[k], x_packed[2 * k]); + EXPECT_EQ(x.im[k], x_packed[2 * k + 1]); + } +} + +// Verifies that the functionality in CopyFromPackedArray works as intended +// (relies on that the functionality in CopyToPackedArray has been verified in +// the test above). +TEST(FftData, CopyFromPackedArray) { + FftData x_ref; + FftData x; + std::array x_packed; + + for (size_t k = 0; k < x_ref.re.size(); ++k) { + x_ref.re[k] = k + 1; + } + + x_ref.im[0] = x_ref.im[x_ref.im.size() - 1] = 0.f; + for (size_t k = 1; k < x_ref.im.size() - 1; ++k) { + x_ref.im[k] = 2.f * (k + 1); + } + + x_ref.CopyToPackedArray(&x_packed); + x.CopyFromPackedArray(x_packed); + + EXPECT_EQ(x_ref.re, x.re); + EXPECT_EQ(x_ref.im, x.im); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.cc new file mode 100644 index 0000000000..0a0c0e2fae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/frame_blocker.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +FrameBlocker::FrameBlocker(size_t num_bands) + : num_bands_(num_bands), buffer_(num_bands_) { + for (auto& b : buffer_) { + b.reserve(kBlockSize); + RTC_DCHECK(b.empty()); + } +} + +FrameBlocker::~FrameBlocker() = default; + +void FrameBlocker::InsertSubFrameAndExtractBlock( + const std::vector>& sub_frame, + std::vector>* block) { + RTC_DCHECK(block); + RTC_DCHECK_EQ(num_bands_, block->size()); + RTC_DCHECK_EQ(num_bands_, sub_frame.size()); + for (size_t i = 0; i < num_bands_; ++i) { + RTC_DCHECK_GE(kBlockSize - 16, buffer_[i].size()); + RTC_DCHECK_EQ(kBlockSize, (*block)[i].size()); + RTC_DCHECK_EQ(kSubFrameLength, sub_frame[i].size()); + const int samples_to_block = kBlockSize - buffer_[i].size(); + (*block)[i].clear(); + (*block)[i].insert((*block)[i].begin(), buffer_[i].begin(), + buffer_[i].end()); + (*block)[i].insert((*block)[i].begin() + buffer_[i].size(), + sub_frame[i].begin(), + sub_frame[i].begin() + samples_to_block); + buffer_[i].clear(); + buffer_[i].insert(buffer_[i].begin(), + sub_frame[i].begin() + samples_to_block, + sub_frame[i].end()); + } +} + +bool FrameBlocker::IsBlockAvailable() const { + return kBlockSize == buffer_[0].size(); +} + +void FrameBlocker::ExtractBlock(std::vector>* block) { + RTC_DCHECK(block); + RTC_DCHECK_EQ(num_bands_, block->size()); + RTC_DCHECK(IsBlockAvailable()); + for (size_t i = 0; i < num_bands_; ++i) { + RTC_DCHECK_EQ(kBlockSize, buffer_[i].size()); + RTC_DCHECK_EQ(kBlockSize, (*block)[i].size()); + (*block)[i].clear(); + (*block)[i].insert((*block)[i].begin(), buffer_[i].begin(), + buffer_[i].end()); + buffer_[i].clear(); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.h new file mode 100644 index 0000000000..08e1e1dd23 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_FRAME_BLOCKER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_FRAME_BLOCKER_H_ + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Class for producing 64 sample multiband blocks from frames consisting of 1 or +// 2 subframes of 80 samples. +class FrameBlocker { + public: + explicit FrameBlocker(size_t num_bands); + ~FrameBlocker(); + // Inserts one 80 sample multiband subframe from the multiband frame and + // extracts one 64 sample multiband block. + void InsertSubFrameAndExtractBlock( + const std::vector>& sub_frame, + std::vector>* block); + // Reports whether a multiband block of 64 samples is available for + // extraction. + bool IsBlockAvailable() const; + // Extracts a multiband block of 64 samples. + void ExtractBlock(std::vector>* block); + + private: + const size_t num_bands_; + std::vector> buffer_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBlocker); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_FRAME_BLOCKER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc new file mode 100644 index 0000000000..6e73d4bc8f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc @@ -0,0 +1,341 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/frame_blocker.h" + +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block_framer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +float ComputeSampleValue(size_t chunk_counter, + size_t chunk_size, + size_t band, + size_t sample_index, + int offset) { + float value = + static_cast(chunk_counter * chunk_size + sample_index) + offset; + return value > 0 ? 5000 * band + value : 0; +} + +void FillSubFrame(size_t sub_frame_counter, + int offset, + std::vector>* sub_frame) { + for (size_t k = 0; k < sub_frame->size(); ++k) { + for (size_t i = 0; i < (*sub_frame)[0].size(); ++i) { + (*sub_frame)[k][i] = + ComputeSampleValue(sub_frame_counter, kSubFrameLength, k, i, offset); + } + } +} + +void FillSubFrameView(size_t sub_frame_counter, + int offset, + std::vector>* sub_frame, + std::vector>* sub_frame_view) { + FillSubFrame(sub_frame_counter, offset, sub_frame); + for (size_t k = 0; k < sub_frame_view->size(); ++k) { + (*sub_frame_view)[k] = + rtc::ArrayView(&(*sub_frame)[k][0], (*sub_frame)[k].size()); + } +} + +bool VerifySubFrame(size_t sub_frame_counter, + int offset, + const std::vector>& sub_frame_view) { + std::vector> reference_sub_frame( + sub_frame_view.size(), std::vector(sub_frame_view[0].size(), 0.f)); + FillSubFrame(sub_frame_counter, offset, &reference_sub_frame); + for (size_t k = 0; k < sub_frame_view.size(); ++k) { + for (size_t i = 0; i < sub_frame_view[k].size(); ++i) { + if (reference_sub_frame[k][i] != sub_frame_view[k][i]) { + return false; + } + } + } + return true; +} + +bool VerifyBlock(size_t block_counter, + int offset, + const std::vector>& block) { + for (size_t k = 0; k < block.size(); ++k) { + for (size_t i = 0; i < block[k].size(); ++i) { + const float reference_value = + ComputeSampleValue(block_counter, kBlockSize, k, i, offset); + if (reference_value != block[k][i]) { + return false; + } + } + } + return true; +} + +// Verifies that the FrameBlocker properly forms blocks out of the frames. +void RunBlockerTest(int sample_rate_hz) { + constexpr size_t kNumSubFramesToProcess = 20; + const size_t num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> block(num_bands, + std::vector(kBlockSize, 0.f)); + std::vector> input_sub_frame( + num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> input_sub_frame_view(num_bands); + FrameBlocker blocker(num_bands); + + size_t block_counter = 0; + for (size_t sub_frame_index = 0; sub_frame_index < kNumSubFramesToProcess; + ++sub_frame_index) { + FillSubFrameView(sub_frame_index, 0, &input_sub_frame, + &input_sub_frame_view); + + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &block); + VerifyBlock(block_counter++, 0, block); + + if ((sub_frame_index + 1) % 4 == 0) { + EXPECT_TRUE(blocker.IsBlockAvailable()); + } else { + EXPECT_FALSE(blocker.IsBlockAvailable()); + } + if (blocker.IsBlockAvailable()) { + blocker.ExtractBlock(&block); + VerifyBlock(block_counter++, 0, block); + } + } +} + +// Verifies that the FrameBlocker and BlockFramer work well together and produce +// the expected output. +void RunBlockerAndFramerTest(int sample_rate_hz) { + const size_t kNumSubFramesToProcess = 20; + const size_t num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> block(num_bands, + std::vector(kBlockSize, 0.f)); + std::vector> input_sub_frame( + num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> output_sub_frame( + num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> output_sub_frame_view(num_bands); + std::vector> input_sub_frame_view(num_bands); + FrameBlocker blocker(num_bands); + BlockFramer framer(num_bands); + + for (size_t sub_frame_index = 0; sub_frame_index < kNumSubFramesToProcess; + ++sub_frame_index) { + FillSubFrameView(sub_frame_index, 0, &input_sub_frame, + &input_sub_frame_view); + FillSubFrameView(sub_frame_index, 0, &output_sub_frame, + &output_sub_frame_view); + + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &block); + framer.InsertBlockAndExtractSubFrame(block, &output_sub_frame_view); + + if ((sub_frame_index + 1) % 4 == 0) { + EXPECT_TRUE(blocker.IsBlockAvailable()); + } else { + EXPECT_FALSE(blocker.IsBlockAvailable()); + } + if (blocker.IsBlockAvailable()) { + blocker.ExtractBlock(&block); + framer.InsertBlock(block); + } + EXPECT_TRUE(VerifySubFrame(sub_frame_index, -64, output_sub_frame_view)); + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Verifies that the FrameBlocker crashes if the InsertSubFrameAndExtractBlock +// method is called for inputs with the wrong number of bands or band lengths. +void RunWronglySizedInsertAndExtractParametersTest(int sample_rate_hz, + size_t num_block_bands, + size_t block_length, + size_t num_sub_frame_bands, + size_t sub_frame_length) { + const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> block(num_block_bands, + std::vector(block_length, 0.f)); + std::vector> input_sub_frame( + num_sub_frame_bands, std::vector(sub_frame_length, 0.f)); + std::vector> input_sub_frame_view( + input_sub_frame.size()); + FillSubFrameView(0, 0, &input_sub_frame, &input_sub_frame_view); + FrameBlocker blocker(correct_num_bands); + EXPECT_DEATH( + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &block), ""); +} + +// Verifies that the FrameBlocker crashes if the ExtractBlock method is called +// for inputs with the wrong number of bands or band lengths. +void RunWronglySizedExtractParameterTest(int sample_rate_hz, + size_t num_block_bands, + size_t block_length) { + const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> correct_block( + correct_num_bands, std::vector(kBlockSize, 0.f)); + std::vector> wrong_block( + num_block_bands, std::vector(block_length, 0.f)); + std::vector> input_sub_frame( + correct_num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> input_sub_frame_view( + input_sub_frame.size()); + FillSubFrameView(0, 0, &input_sub_frame, &input_sub_frame_view); + FrameBlocker blocker(correct_num_bands); + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &correct_block); + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &correct_block); + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &correct_block); + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &correct_block); + + EXPECT_DEATH(blocker.ExtractBlock(&wrong_block), ""); +} + +// Verifies that the FrameBlocker crashes if the ExtractBlock method is called +// after a wrong number of previous InsertSubFrameAndExtractBlock method calls +// have been made. +void RunWrongExtractOrderTest(int sample_rate_hz, + size_t num_preceeding_api_calls) { + const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); + + std::vector> block(correct_num_bands, + std::vector(kBlockSize, 0.f)); + std::vector> input_sub_frame( + correct_num_bands, std::vector(kSubFrameLength, 0.f)); + std::vector> input_sub_frame_view( + input_sub_frame.size()); + FillSubFrameView(0, 0, &input_sub_frame, &input_sub_frame_view); + FrameBlocker blocker(correct_num_bands); + for (size_t k = 0; k < num_preceeding_api_calls; ++k) { + blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &block); + } + + EXPECT_DEATH(blocker.ExtractBlock(&block), ""); +} +#endif + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(FrameBlocker, WrongNumberOfBandsInBlockForInsertSubFrameAndExtractBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, wrong_num_bands, kBlockSize, correct_num_bands, kSubFrameLength); + } +} + +TEST(FrameBlocker, + WrongNumberOfBandsInSubFrameForInsertSubFrameAndExtractBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_bands, kBlockSize, wrong_num_bands, kSubFrameLength); + } +} + +TEST(FrameBlocker, + WrongNumberOfSamplesInBlockForInsertSubFrameAndExtractBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_bands, kBlockSize - 1, correct_num_bands, + kSubFrameLength); + } +} + +TEST(FrameBlocker, + WrongNumberOfSamplesInSubFrameForInsertSubFrameAndExtractBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertAndExtractParametersTest(rate, correct_num_bands, + kBlockSize, correct_num_bands, + kSubFrameLength - 1); + } +} + +TEST(FrameBlocker, WrongNumberOfBandsInBlockForExtractBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedExtractParameterTest(rate, wrong_num_bands, kBlockSize); + } +} + +TEST(FrameBlocker, WrongNumberOfSamplesInBlockForExtractBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedExtractParameterTest(rate, correct_num_bands, + kBlockSize - 1); + } +} + +TEST(FrameBlocker, WrongNumberOfPreceedingApiCallsForExtractBlock) { + for (auto rate : {8000, 16000, 32000, 48000}) { + for (size_t num_calls = 0; num_calls < 4; ++num_calls) { + std::ostringstream ss; + ss << "Sample rate: " << rate; + ss << ", Num preceeding InsertSubFrameAndExtractBlock calls: " + << num_calls; + + SCOPED_TRACE(ss.str()); + RunWrongExtractOrderTest(rate, num_calls); + } + } +} + +// Verifiers that the verification for null sub_frame pointer works. +TEST(FrameBlocker, NullBlockParameter) { + std::vector> sub_frame( + 1, std::vector(kSubFrameLength, 0.f)); + std::vector> sub_frame_view(sub_frame.size()); + FillSubFrameView(0, 0, &sub_frame, &sub_frame_view); + EXPECT_DEATH( + FrameBlocker(1).InsertSubFrameAndExtractBlock(sub_frame_view, nullptr), + ""); +} + +#endif + +TEST(FrameBlocker, BlockBitexactness) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunBlockerTest(rate); + } +} + +TEST(FrameBlocker, BlockerAndFramer) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + RunBlockerAndFramerTest(rate); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.cc new file mode 100644 index 0000000000..1dd2a20b27 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/main_filter_update_gain.h" + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +constexpr float kHErrorInitial = 10000.f; +constexpr int kPoorExcitationCounterInitial = 1000; + +} // namespace + +int MainFilterUpdateGain::instance_count_ = 0; + +MainFilterUpdateGain::MainFilterUpdateGain() + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + poor_excitation_counter_(kPoorExcitationCounterInitial) { + H_error_.fill(kHErrorInitial); +} + +MainFilterUpdateGain::~MainFilterUpdateGain() {} + +void MainFilterUpdateGain::HandleEchoPathChange() { + H_error_.fill(kHErrorInitial); + poor_excitation_counter_ = kPoorExcitationCounterInitial; + call_counter_ = 0; +} + +void MainFilterUpdateGain::Compute( + const RenderBuffer& render_buffer, + const RenderSignalAnalyzer& render_signal_analyzer, + const SubtractorOutput& subtractor_output, + const AdaptiveFirFilter& filter, + bool saturated_capture_signal, + FftData* gain_fft) { + RTC_DCHECK(gain_fft); + // Introducing shorter notation to improve readability. + const FftData& E_main = subtractor_output.E_main; + const auto& E2_main = subtractor_output.E2_main; + const auto& E2_shadow = subtractor_output.E2_shadow; + FftData* G = gain_fft; + const size_t size_partitions = filter.SizePartitions(); + const auto& X2 = render_buffer.SpectralSum(size_partitions); + const auto& erl = filter.Erl(); + + ++call_counter_; + + if (render_signal_analyzer.PoorSignalExcitation()) { + poor_excitation_counter_ = 0; + } + + // Do not update the filter if the render is not sufficiently excited. + if (++poor_excitation_counter_ < size_partitions || + saturated_capture_signal || call_counter_ <= size_partitions) { + G->re.fill(0.f); + G->im.fill(0.f); + } else { + // Corresponds to WGN of power -39 dBFS. + constexpr float kNoiseGatePower = 220075344.f; + std::array mu; + // mu = H_error / (0.5* H_error* X2 + n * E2). + for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { + mu[k] = X2[k] > kNoiseGatePower + ? H_error_[k] / (0.5f * H_error_[k] * X2[k] + + size_partitions * E2_main[k]) + : 0.f; + } + + // Avoid updating the filter close to narrow bands in the render signals. + render_signal_analyzer.MaskRegionsAroundNarrowBands(&mu); + + // H_error = H_error - 0.5 * mu * X2 * H_error. + for (size_t k = 0; k < H_error_.size(); ++k) { + H_error_[k] -= 0.5f * mu[k] * X2[k] * H_error_[k]; + } + + // G = mu * E. + std::transform(mu.begin(), mu.end(), E_main.re.begin(), G->re.begin(), + std::multiplies()); + std::transform(mu.begin(), mu.end(), E_main.im.begin(), G->im.begin(), + std::multiplies()); + } + + // H_error = H_error + factor * erl. + std::array H_error_increase; + constexpr float kErlScaleAccurate = 1.f / 100.0f; + constexpr float kErlScaleInaccurate = 1.f / 60.0f; + std::transform(E2_shadow.begin(), E2_shadow.end(), E2_main.begin(), + H_error_increase.begin(), [&](float a, float b) { + return a >= b ? kErlScaleAccurate : kErlScaleInaccurate; + }); + std::transform(erl.begin(), erl.end(), H_error_increase.begin(), + H_error_increase.begin(), std::multiplies()); + std::transform(H_error_.begin(), H_error_.end(), H_error_increase.begin(), + H_error_.begin(), + [&](float a, float b) { return std::max(a + b, 0.1f); }); + + data_dumper_->DumpRaw("aec3_main_gain_H_error", H_error_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.h new file mode 100644 index 0000000000..756a5d0e5d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MAIN_FILTER_UPDATE_GAIN_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MAIN_FILTER_UPDATE_GAIN_H_ + +#include +#include + +#include "modules/audio_processing/aec3/adaptive_fir_filter.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_signal_analyzer.h" +#include "modules/audio_processing/aec3/subtractor_output.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; + +// Provides functionality for computing the adaptive gain for the main filter. +class MainFilterUpdateGain { + public: + MainFilterUpdateGain(); + ~MainFilterUpdateGain(); + + // Takes action in the case of a known echo path change. + void HandleEchoPathChange(); + + // Computes the gain. + void Compute(const RenderBuffer& render_buffer, + const RenderSignalAnalyzer& render_signal_analyzer, + const SubtractorOutput& subtractor_output, + const AdaptiveFirFilter& filter, + bool saturated_capture_signal, + FftData* gain_fft); + + private: + static int instance_count_; + std::unique_ptr data_dumper_; + std::array H_error_; + size_t poor_excitation_counter_; + size_t call_counter_ = 0; + RTC_DISALLOW_COPY_AND_ASSIGN(MainFilterUpdateGain); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MAIN_FILTER_UPDATE_GAIN_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc new file mode 100644 index 0000000000..203731a929 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/main_filter_update_gain.h" + +#include +#include +#include + +#include "modules/audio_processing/aec3/adaptive_fir_filter.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_signal_analyzer.h" +#include "modules/audio_processing/aec3/shadow_filter_update_gain.h" +#include "modules/audio_processing/aec3/subtractor_output.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +// Method for performing the simulations needed to test the main filter update +// gain functionality. +void RunFilterUpdateTest(int num_blocks_to_process, + size_t delay_samples, + const std::vector& blocks_with_echo_path_changes, + const std::vector& blocks_with_saturation, + bool use_silent_render_in_second_half, + std::array* e_last_block, + std::array* y_last_block, + FftData* G_last_block) { + ApmDataDumper data_dumper(42); + AdaptiveFirFilter main_filter(9, DetectOptimization(), &data_dumper); + AdaptiveFirFilter shadow_filter(9, DetectOptimization(), &data_dumper); + Aec3Fft fft; + RenderBuffer render_buffer( + Aec3Optimization::kNone, 3, main_filter.SizePartitions(), + std::vector(1, main_filter.SizePartitions())); + std::array x_old; + x_old.fill(0.f); + ShadowFilterUpdateGain shadow_gain; + MainFilterUpdateGain main_gain; + Random random_generator(42U); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector y(kBlockSize, 0.f); + AecState aec_state(EchoCanceller3Config{}); + RenderSignalAnalyzer render_signal_analyzer; + std::array s_scratch; + std::array s; + FftData S; + FftData G; + SubtractorOutput output; + output.Reset(); + FftData& E_main = output.E_main; + FftData E_shadow; + std::array Y2; + std::array& E2_main = output.E2_main; + std::array& e_main = output.e_main; + std::array& e_shadow = output.e_shadow; + Y2.fill(0.f); + + constexpr float kScale = 1.0f / kFftLengthBy2; + + DelayBuffer delay_buffer(delay_samples); + for (int k = 0; k < num_blocks_to_process; ++k) { + // Handle echo path changes. + if (std::find(blocks_with_echo_path_changes.begin(), + blocks_with_echo_path_changes.end(), + k) != blocks_with_echo_path_changes.end()) { + main_filter.HandleEchoPathChange(); + } + + // Handle saturation. + const bool saturation = + std::find(blocks_with_saturation.begin(), blocks_with_saturation.end(), + k) != blocks_with_saturation.end(); + + // Create the render signal. + if (use_silent_render_in_second_half && k > num_blocks_to_process / 2) { + std::fill(x[0].begin(), x[0].end(), 0.f); + } else { + RandomizeSampleVector(&random_generator, x[0]); + } + delay_buffer.Delay(x[0], y); + render_buffer.Insert(x); + render_signal_analyzer.Update(render_buffer, aec_state.FilterDelay()); + + // Apply the main filter. + main_filter.Filter(render_buffer, &S); + fft.Ifft(S, &s_scratch); + std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, + e_main.begin(), + [&](float a, float b) { return a - b * kScale; }); + std::for_each(e_main.begin(), e_main.end(), + [](float& a) { a = rtc::SafeClamp(a, -32768.f, 32767.f); }); + fft.ZeroPaddedFft(e_main, &E_main); + for (size_t k = 0; k < kBlockSize; ++k) { + s[k] = kScale * s_scratch[k + kFftLengthBy2]; + } + + // Apply the shadow filter. + shadow_filter.Filter(render_buffer, &S); + fft.Ifft(S, &s_scratch); + std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, + e_shadow.begin(), + [&](float a, float b) { return a - b * kScale; }); + std::for_each(e_shadow.begin(), e_shadow.end(), + [](float& a) { a = rtc::SafeClamp(a, -32768.f, 32767.f); }); + fft.ZeroPaddedFft(e_shadow, &E_shadow); + + // Compute spectra for future use. + E_main.Spectrum(Aec3Optimization::kNone, &output.E2_main); + E_shadow.Spectrum(Aec3Optimization::kNone, &output.E2_shadow); + + // Adapt the shadow filter. + shadow_gain.Compute(render_buffer, render_signal_analyzer, E_shadow, + shadow_filter.SizePartitions(), saturation, &G); + shadow_filter.Adapt(render_buffer, G); + + // Adapt the main filter + main_gain.Compute(render_buffer, render_signal_analyzer, output, + main_filter, saturation, &G); + main_filter.Adapt(render_buffer, G); + + // Update the delay. + aec_state.HandleEchoPathChange(EchoPathVariability(false, false)); + aec_state.Update(main_filter.FilterFrequencyResponse(), + main_filter.FilterImpulseResponse(), true, rtc::nullopt, + render_buffer, E2_main, Y2, x[0], s, false); + } + + std::copy(e_main.begin(), e_main.end(), e_last_block->begin()); + std::copy(y.begin(), y.end(), y_last_block->begin()); + std::copy(G.re.begin(), G.re.end(), G_last_block->re.begin()); + std::copy(G.im.begin(), G.im.end(), G_last_block->im.begin()); +} + +std::string ProduceDebugText(size_t delay) { + std::ostringstream ss; + ss << "Delay: " << delay; + return ss.str(); +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies that the check for non-null output gain parameter works. +TEST(MainFilterUpdateGain, NullDataOutputGain) { + ApmDataDumper data_dumper(42); + AdaptiveFirFilter filter(9, DetectOptimization(), &data_dumper); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, + filter.SizePartitions(), + std::vector(1, filter.SizePartitions())); + RenderSignalAnalyzer analyzer; + SubtractorOutput output; + MainFilterUpdateGain gain; + EXPECT_DEATH( + gain.Compute(render_buffer, analyzer, output, filter, false, nullptr), + ""); +} + +#endif + +// Verifies that the gain formed causes the filter using it to converge. +TEST(MainFilterUpdateGain, GainCausesFilterToConverge) { + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(delay_samples)); + + std::array e; + std::array y; + FftData G; + + RunFilterUpdateTest(500, delay_samples, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G); + + // Verify that the main filter is able to perform well. + EXPECT_LT(1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f), + std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); + } +} + +// Verifies that the magnitude of the gain on average decreases for a +// persistently exciting signal. +TEST(MainFilterUpdateGain, DecreasingGain) { + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; + + std::array e; + std::array y; + FftData G_a; + FftData G_b; + FftData G_c; + std::array G_a_power; + std::array G_b_power; + std::array G_c_power; + + RunFilterUpdateTest(100, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_a); + RunFilterUpdateTest(200, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_b); + RunFilterUpdateTest(300, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_c); + + G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); + G_c.Spectrum(Aec3Optimization::kNone, &G_c_power); + + EXPECT_GT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), + std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); + + EXPECT_GT(std::accumulate(G_b_power.begin(), G_b_power.end(), 0.), + std::accumulate(G_c_power.begin(), G_c_power.end(), 0.)); +} + +// Verifies that the gain is zero when there is saturation and that the internal +// error estimates cause the gain to increase after a period of saturation. +TEST(MainFilterUpdateGain, SaturationBehavior) { + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; + for (int k = 99; k < 200; ++k) { + blocks_with_saturation.push_back(k); + } + + std::array e; + std::array y; + FftData G_a; + FftData G_b; + FftData G_a_ref; + G_a_ref.re.fill(0.f); + G_a_ref.im.fill(0.f); + + std::array G_a_power; + std::array G_b_power; + + RunFilterUpdateTest(100, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_a); + + EXPECT_EQ(G_a_ref.re, G_a.re); + EXPECT_EQ(G_a_ref.im, G_a.im); + + RunFilterUpdateTest(99, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_a); + RunFilterUpdateTest(201, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_b); + + G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); + + EXPECT_LT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), + std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); +} + +// Verifies that the gain increases after an echo path change. +TEST(MainFilterUpdateGain, EchoPathChangeBehavior) { + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; + blocks_with_echo_path_changes.push_back(99); + + std::array e; + std::array y; + FftData G_a; + FftData G_b; + std::array G_a_power; + std::array G_b_power; + + RunFilterUpdateTest(99, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_a); + RunFilterUpdateTest(100, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_b); + + G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); + + EXPECT_LT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), + std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.cc new file mode 100644 index 0000000000..4906bf9db7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.cc @@ -0,0 +1,443 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/matched_filter.h" + +#if defined(WEBRTC_HAS_NEON) +#include +#endif +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace aec3 { + +#if defined(WEBRTC_HAS_NEON) + +void MatchedFilterCore_NEON(size_t x_start_index, + float x2_sum_threshold, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum) { + const int h_size = static_cast(h.size()); + const int x_size = static_cast(x.size()); + RTC_DCHECK_EQ(0, h_size % 4); + + // Process for all samples in the sub-block. + for (size_t i = 0; i < y.size(); ++i) { + // Apply the matched filter as filter * x, and compute x * x. + + RTC_DCHECK_GT(x_size, x_start_index); + const float* x_p = &x[x_start_index]; + const float* h_p = &h[0]; + + // Initialize values for the accumulation. + float32x4_t s_128 = vdupq_n_f32(0); + float32x4_t x2_sum_128 = vdupq_n_f32(0); + float x2_sum = 0.f; + float s = 0; + + // Compute loop chunk sizes until, and after, the wraparound of the circular + // buffer for x. + const int chunk1 = + std::min(h_size, static_cast(x_size - x_start_index)); + + // Perform the loop in two chunks. + const int chunk2 = h_size - chunk1; + for (int limit : {chunk1, chunk2}) { + // Perform 128 bit vector operations. + const int limit_by_4 = limit >> 2; + for (int k = limit_by_4; k > 0; --k, h_p += 4, x_p += 4) { + // Load the data into 128 bit vectors. + const float32x4_t x_k = vld1q_f32(x_p); + const float32x4_t h_k = vld1q_f32(h_p); + // Compute and accumulate x * x and h * x. + x2_sum_128 = vmlaq_f32(x2_sum_128, x_k, x_k); + s_128 = vmlaq_f32(s_128, h_k, x_k); + } + + // Perform non-vector operations for any remaining items. + for (int k = limit - limit_by_4 * 4; k > 0; --k, ++h_p, ++x_p) { + const float x_k = *x_p; + x2_sum += x_k * x_k; + s += *h_p * x_k; + } + + x_p = &x[0]; + } + + // Combine the accumulated vector and scalar values. + float* v = reinterpret_cast(&x2_sum_128); + x2_sum += v[0] + v[1] + v[2] + v[3]; + v = reinterpret_cast(&s_128); + s += v[0] + v[1] + v[2] + v[3]; + + // Compute the matched filter error. + const float e = std::min(32767.f, std::max(-32768.f, y[i] - s)); + *error_sum += e * e; + + // Update the matched filter estimate in an NLMS manner. + if (x2_sum > x2_sum_threshold) { + RTC_DCHECK_LT(0.f, x2_sum); + const float alpha = 0.7f * e / x2_sum; + const float32x4_t alpha_128 = vmovq_n_f32(alpha); + + // filter = filter + 0.7 * (y - filter * x) / x * x. + float* h_p = &h[0]; + x_p = &x[x_start_index]; + + // Perform the loop in two chunks. + for (int limit : {chunk1, chunk2}) { + // Perform 128 bit vector operations. + const int limit_by_4 = limit >> 2; + for (int k = limit_by_4; k > 0; --k, h_p += 4, x_p += 4) { + // Load the data into 128 bit vectors. + float32x4_t h_k = vld1q_f32(h_p); + const float32x4_t x_k = vld1q_f32(x_p); + // Compute h = h + alpha * x. + h_k = vmlaq_f32(h_k, alpha_128, x_k); + + // Store the result. + vst1q_f32(h_p, h_k); + } + + // Perform non-vector operations for any remaining items. + for (int k = limit - limit_by_4 * 4; k > 0; --k, ++h_p, ++x_p) { + *h_p += alpha * *x_p; + } + + x_p = &x[0]; + } + + *filters_updated = true; + } + + x_start_index = x_start_index > 0 ? x_start_index - 1 : x_size - 1; + } +} + +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) + +void MatchedFilterCore_SSE2(size_t x_start_index, + float x2_sum_threshold, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum) { + const int h_size = static_cast(h.size()); + const int x_size = static_cast(x.size()); + RTC_DCHECK_EQ(0, h_size % 4); + + // Process for all samples in the sub-block. + for (size_t i = 0; i < y.size(); ++i) { + // Apply the matched filter as filter * x, and compute x * x. + + RTC_DCHECK_GT(x_size, x_start_index); + const float* x_p = &x[x_start_index]; + const float* h_p = &h[0]; + + // Initialize values for the accumulation. + __m128 s_128 = _mm_set1_ps(0); + __m128 x2_sum_128 = _mm_set1_ps(0); + float x2_sum = 0.f; + float s = 0; + + // Compute loop chunk sizes until, and after, the wraparound of the circular + // buffer for x. + const int chunk1 = + std::min(h_size, static_cast(x_size - x_start_index)); + + // Perform the loop in two chunks. + const int chunk2 = h_size - chunk1; + for (int limit : {chunk1, chunk2}) { + // Perform 128 bit vector operations. + const int limit_by_4 = limit >> 2; + for (int k = limit_by_4; k > 0; --k, h_p += 4, x_p += 4) { + // Load the data into 128 bit vectors. + const __m128 x_k = _mm_loadu_ps(x_p); + const __m128 h_k = _mm_loadu_ps(h_p); + const __m128 xx = _mm_mul_ps(x_k, x_k); + // Compute and accumulate x * x and h * x. + x2_sum_128 = _mm_add_ps(x2_sum_128, xx); + const __m128 hx = _mm_mul_ps(h_k, x_k); + s_128 = _mm_add_ps(s_128, hx); + } + + // Perform non-vector operations for any remaining items. + for (int k = limit - limit_by_4 * 4; k > 0; --k, ++h_p, ++x_p) { + const float x_k = *x_p; + x2_sum += x_k * x_k; + s += *h_p * x_k; + } + + x_p = &x[0]; + } + + // Combine the accumulated vector and scalar values. + float* v = reinterpret_cast(&x2_sum_128); + x2_sum += v[0] + v[1] + v[2] + v[3]; + v = reinterpret_cast(&s_128); + s += v[0] + v[1] + v[2] + v[3]; + + // Compute the matched filter error. + const float e = std::min(32767.f, std::max(-32768.f, y[i] - s)); + *error_sum += e * e; + + // Update the matched filter estimate in an NLMS manner. + if (x2_sum > x2_sum_threshold) { + RTC_DCHECK_LT(0.f, x2_sum); + const float alpha = 0.7f * e / x2_sum; + const __m128 alpha_128 = _mm_set1_ps(alpha); + + // filter = filter + 0.7 * (y - filter * x) / x * x. + float* h_p = &h[0]; + x_p = &x[x_start_index]; + + // Perform the loop in two chunks. + for (int limit : {chunk1, chunk2}) { + // Perform 128 bit vector operations. + const int limit_by_4 = limit >> 2; + for (int k = limit_by_4; k > 0; --k, h_p += 4, x_p += 4) { + // Load the data into 128 bit vectors. + __m128 h_k = _mm_loadu_ps(h_p); + const __m128 x_k = _mm_loadu_ps(x_p); + + // Compute h = h + alpha * x. + const __m128 alpha_x = _mm_mul_ps(alpha_128, x_k); + h_k = _mm_add_ps(h_k, alpha_x); + + // Store the result. + _mm_storeu_ps(h_p, h_k); + } + + // Perform non-vector operations for any remaining items. + for (int k = limit - limit_by_4 * 4; k > 0; --k, ++h_p, ++x_p) { + *h_p += alpha * *x_p; + } + + x_p = &x[0]; + } + + *filters_updated = true; + } + + x_start_index = x_start_index > 0 ? x_start_index - 1 : x_size - 1; + } +} +#endif + +void MatchedFilterCore(size_t x_start_index, + float x2_sum_threshold, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum) { + // Process for all samples in the sub-block. + for (size_t i = 0; i < y.size(); ++i) { + // Apply the matched filter as filter * x, and compute x * x. + float x2_sum = 0.f; + float s = 0; + size_t x_index = x_start_index; + for (size_t k = 0; k < h.size(); ++k) { + x2_sum += x[x_index] * x[x_index]; + s += h[k] * x[x_index]; + x_index = x_index < (x.size() - 1) ? x_index + 1 : 0; + } + + // Compute the matched filter error. + const float e = std::min(32767.f, std::max(-32768.f, y[i] - s)); + (*error_sum) += e * e; + + // Update the matched filter estimate in an NLMS manner. + if (x2_sum > x2_sum_threshold) { + RTC_DCHECK_LT(0.f, x2_sum); + const float alpha = 0.7f * e / x2_sum; + + // filter = filter + 0.7 * (y - filter * x) / x * x. + size_t x_index = x_start_index; + for (size_t k = 0; k < h.size(); ++k) { + h[k] += alpha * x[x_index]; + x_index = x_index < (x.size() - 1) ? x_index + 1 : 0; + } + *filters_updated = true; + } + + x_start_index = x_start_index > 0 ? x_start_index - 1 : x.size() - 1; + } +} + +} // namespace aec3 + +MatchedFilter::MatchedFilter(ApmDataDumper* data_dumper, + Aec3Optimization optimization, + size_t sub_block_size, + size_t window_size_sub_blocks, + int num_matched_filters, + size_t alignment_shift_sub_blocks, + float excitation_limit) + : data_dumper_(data_dumper), + optimization_(optimization), + sub_block_size_(sub_block_size), + filter_intra_lag_shift_(alignment_shift_sub_blocks * sub_block_size_), + filters_( + num_matched_filters, + std::vector(window_size_sub_blocks * sub_block_size_, 0.f)), + lag_estimates_(num_matched_filters), + filters_offsets_(num_matched_filters, 0), + excitation_limit_(excitation_limit) { + RTC_DCHECK(data_dumper); + RTC_DCHECK_LT(0, window_size_sub_blocks); + RTC_DCHECK((kBlockSize % sub_block_size) == 0); + RTC_DCHECK((sub_block_size % 4) == 0); +} + +MatchedFilter::~MatchedFilter() = default; + +void MatchedFilter::Reset() { + for (auto& f : filters_) { + std::fill(f.begin(), f.end(), 0.f); + } + + for (auto& l : lag_estimates_) { + l = MatchedFilter::LagEstimate(); + } +} + +void MatchedFilter::Update(const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture) { + RTC_DCHECK_EQ(sub_block_size_, capture.size()); + auto& y = capture; + + const float x2_sum_threshold = + filters_[0].size() * excitation_limit_ * excitation_limit_; + + // Apply all matched filters. + size_t alignment_shift = 0; + for (size_t n = 0; n < filters_.size(); ++n) { + float error_sum = 0.f; + bool filters_updated = false; + + size_t x_start_index = + (render_buffer.position + alignment_shift + sub_block_size_ - 1) % + render_buffer.buffer.size(); + + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: + aec3::MatchedFilterCore_SSE2(x_start_index, x2_sum_threshold, + render_buffer.buffer, y, filters_[n], + &filters_updated, &error_sum); + break; +#endif +#if defined(WEBRTC_HAS_NEON) + case Aec3Optimization::kNeon: + aec3::MatchedFilterCore_NEON(x_start_index, x2_sum_threshold, + render_buffer.buffer, y, filters_[n], + &filters_updated, &error_sum); + break; +#endif + default: + aec3::MatchedFilterCore(x_start_index, x2_sum_threshold, + render_buffer.buffer, y, filters_[n], + &filters_updated, &error_sum); + } + + // Compute anchor for the matched filter error. + const float error_sum_anchor = + std::inner_product(y.begin(), y.end(), y.begin(), 0.f); + + // Estimate the lag in the matched filter as the distance to the portion in + // the filter that contributes the most to the matched filter output. This + // is detected as the peak of the matched filter. + const size_t lag_estimate = std::distance( + filters_[n].begin(), + std::max_element( + filters_[n].begin(), filters_[n].end(), + [](float a, float b) -> bool { return a * a < b * b; })); + + // Update the lag estimates for the matched filter. + const float kMatchingFilterThreshold = 0.2f; + lag_estimates_[n] = LagEstimate( + error_sum_anchor - error_sum, + (lag_estimate > 2 && lag_estimate < (filters_[n].size() - 10) && + error_sum < kMatchingFilterThreshold * error_sum_anchor), + lag_estimate + alignment_shift, filters_updated); + + RTC_DCHECK_GE(10, filters_.size()); + switch (n) { + case 0: + data_dumper_->DumpRaw("aec3_correlator_0_h", filters_[0]); + break; + case 1: + data_dumper_->DumpRaw("aec3_correlator_1_h", filters_[1]); + break; + case 2: + data_dumper_->DumpRaw("aec3_correlator_2_h", filters_[2]); + break; + case 3: + data_dumper_->DumpRaw("aec3_correlator_3_h", filters_[3]); + break; + case 4: + data_dumper_->DumpRaw("aec3_correlator_4_h", filters_[4]); + break; + case 5: + data_dumper_->DumpRaw("aec3_correlator_5_h", filters_[5]); + break; + case 6: + data_dumper_->DumpRaw("aec3_correlator_6_h", filters_[6]); + break; + case 7: + data_dumper_->DumpRaw("aec3_correlator_7_h", filters_[7]); + break; + case 8: + data_dumper_->DumpRaw("aec3_correlator_8_h", filters_[8]); + break; + case 9: + data_dumper_->DumpRaw("aec3_correlator_9_h", filters_[9]); + break; + default: + RTC_NOTREACHED(); + } + + alignment_shift += filter_intra_lag_shift_; + } +} + +void MatchedFilter::LogFilterProperties(int sample_rate_hz, + size_t shift, + size_t downsampling_factor) const { + size_t alignment_shift = 0; + const int fs_by_1000 = LowestBandRate(sample_rate_hz) / 1000; + for (size_t k = 0; k < filters_.size(); ++k) { + int start = static_cast(alignment_shift * downsampling_factor); + int end = static_cast((alignment_shift + filters_[k].size()) * + downsampling_factor); + RTC_LOG(LS_INFO) << "Filter " << k << ": start: " + << (start - static_cast(shift)) / fs_by_1000 + << " ms, end: " + << (end - static_cast(shift)) / fs_by_1000 << " ms."; + alignment_shift += filter_intra_lag_shift_; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.h new file mode 100644 index 0000000000..c9bdc462e8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_H_ + +#include +#include +#include + +#include "api/optional.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/downsampled_render_buffer.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace aec3 { + +#if defined(WEBRTC_HAS_NEON) + +// Filter core for the matched filter that is optimized for NEON. +void MatchedFilterCore_NEON(size_t x_start_index, + float x2_sum_threshold, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum); + +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) + +// Filter core for the matched filter that is optimized for SSE2. +void MatchedFilterCore_SSE2(size_t x_start_index, + float x2_sum_threshold, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum); + +#endif + +// Filter core for the matched filter. +void MatchedFilterCore(size_t x_start_index, + float x2_sum_threshold, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum); + +} // namespace aec3 + +class ApmDataDumper; + +// Produces recursively updated cross-correlation estimates for several signal +// shifts where the intra-shift spacing is uniform. +class MatchedFilter { + public: + // Stores properties for the lag estimate corresponding to a particular signal + // shift. + struct LagEstimate { + LagEstimate() = default; + LagEstimate(float accuracy, bool reliable, size_t lag, bool updated) + : accuracy(accuracy), reliable(reliable), lag(lag), updated(updated) {} + + float accuracy = 0.f; + bool reliable = false; + size_t lag = 0; + bool updated = false; + }; + + MatchedFilter(ApmDataDumper* data_dumper, + Aec3Optimization optimization, + size_t sub_block_size, + size_t window_size_sub_blocks, + int num_matched_filters, + size_t alignment_shift_sub_blocks, + float excitation_limit); + + ~MatchedFilter(); + + // Updates the correlation with the values in the capture buffer. + void Update(const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture); + + // Resets the matched filter. + void Reset(); + + // Returns the current lag estimates. + rtc::ArrayView GetLagEstimates() const { + return lag_estimates_; + } + + // Returns the maximum filter lag. + size_t GetMaxFilterLag() const { + return filters_.size() * filter_intra_lag_shift_ + filters_[0].size(); + } + + // Log matched filter properties. + void LogFilterProperties(int sample_rate_hz, + size_t shift, + size_t downsampling_factor) const; + + private: + ApmDataDumper* const data_dumper_; + const Aec3Optimization optimization_; + const size_t sub_block_size_; + const size_t filter_intra_lag_shift_; + std::vector> filters_; + std::vector lag_estimates_; + std::vector filters_offsets_; + const float excitation_limit_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(MatchedFilter); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc new file mode 100644 index 0000000000..92cb4f7736 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/matched_filter_lag_aggregator.h" + +#include "modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { + +MatchedFilterLagAggregator::MatchedFilterLagAggregator( + ApmDataDumper* data_dumper, + size_t max_filter_lag) + : data_dumper_(data_dumper), histogram_(max_filter_lag + 1, 0) { + RTC_DCHECK(data_dumper); + histogram_data_.fill(0); +} + +MatchedFilterLagAggregator::~MatchedFilterLagAggregator() = default; + +void MatchedFilterLagAggregator::Reset() { + std::fill(histogram_.begin(), histogram_.end(), 0); + histogram_data_.fill(0); + histogram_data_index_ = 0; +} + +rtc::Optional MatchedFilterLagAggregator::Aggregate( + rtc::ArrayView lag_estimates) { + // Choose the strongest lag estimate as the best one. + float best_accuracy = 0.f; + int best_lag_estimate_index = -1; + for (size_t k = 0; k < lag_estimates.size(); ++k) { + if (lag_estimates[k].updated && lag_estimates[k].reliable) { + if (lag_estimates[k].accuracy > best_accuracy) { + best_accuracy = lag_estimates[k].accuracy; + best_lag_estimate_index = static_cast(k); + } + } + } + + // TODO(peah): Remove this logging once all development is done. + data_dumper_->DumpRaw("aec3_echo_path_delay_estimator_best_index", + best_lag_estimate_index); + + if (best_lag_estimate_index != -1) { + RTC_DCHECK_GT(histogram_.size(), histogram_data_[histogram_data_index_]); + RTC_DCHECK_LE(0, histogram_data_[histogram_data_index_]); + --histogram_[histogram_data_[histogram_data_index_]]; + + histogram_data_[histogram_data_index_] = + lag_estimates[best_lag_estimate_index].lag; + + RTC_DCHECK_GT(histogram_.size(), histogram_data_[histogram_data_index_]); + RTC_DCHECK_LE(0, histogram_data_[histogram_data_index_]); + ++histogram_[histogram_data_[histogram_data_index_]]; + + histogram_data_index_ = + (histogram_data_index_ + 1) % histogram_data_.size(); + + const int candidate = + std::distance(histogram_.begin(), + std::max_element(histogram_.begin(), histogram_.end())); + + if (histogram_[candidate] > 25) { + return candidate; + } + } + return rtc::nullopt; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h new file mode 100644 index 0000000000..c5dd24700e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_LAG_AGGREGATOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_LAG_AGGREGATOR_H_ + +#include + +#include "api/optional.h" +#include "modules/audio_processing/aec3/matched_filter.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; + +// Aggregates lag estimates produced by the MatchedFilter class into a single +// reliable combined lag estimate. +class MatchedFilterLagAggregator { + public: + MatchedFilterLagAggregator(ApmDataDumper* data_dumper, size_t max_filter_lag); + ~MatchedFilterLagAggregator(); + + // Resets the aggregator. + void Reset(); + + // Aggregates the provided lag estimates. + rtc::Optional Aggregate( + rtc::ArrayView lag_estimates); + + private: + ApmDataDumper* const data_dumper_; + std::vector histogram_; + std::array histogram_data_; + int histogram_data_index_ = 0; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(MatchedFilterLagAggregator); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_LAG_AGGREGATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc new file mode 100644 index 0000000000..985ed43427 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/matched_filter_lag_aggregator.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr size_t kNumLagsBeforeDetection = 25; + +} // namespace + +// Verifies that the most accurate lag estimate is chosen. +TEST(MatchedFilterLagAggregator, MostAccurateLagChosen) { + constexpr size_t kLag1 = 5; + constexpr size_t kLag2 = 10; + ApmDataDumper data_dumper(0); + std::vector lag_estimates(2); + MatchedFilterLagAggregator aggregator(&data_dumper, std::max(kLag1, kLag2)); + lag_estimates[0] = MatchedFilter::LagEstimate(1.f, true, kLag1, true); + lag_estimates[1] = MatchedFilter::LagEstimate(0.5f, true, kLag2, true); + + for (size_t k = 0; k < kNumLagsBeforeDetection; ++k) { + EXPECT_FALSE(aggregator.Aggregate(lag_estimates)); + } + + rtc::Optional aggregated_lag = aggregator.Aggregate(lag_estimates); + EXPECT_TRUE(aggregated_lag); + EXPECT_EQ(kLag1, *aggregated_lag); + + lag_estimates[0] = MatchedFilter::LagEstimate(0.5f, true, kLag1, true); + lag_estimates[1] = MatchedFilter::LagEstimate(1.f, true, kLag2, true); + + for (size_t k = 0; k < kNumLagsBeforeDetection; ++k) { + aggregated_lag = aggregator.Aggregate(lag_estimates); + EXPECT_TRUE(aggregated_lag); + EXPECT_EQ(kLag1, *aggregated_lag); + } + + aggregated_lag = aggregator.Aggregate(lag_estimates); + aggregated_lag = aggregator.Aggregate(lag_estimates); + EXPECT_TRUE(aggregated_lag); + EXPECT_EQ(kLag2, *aggregated_lag); +} + +// Verifies that varying lag estimates causes lag estimates to not be deemed +// reliable. +TEST(MatchedFilterLagAggregator, + LagEstimateInvarianceRequiredForAggregatedLag) { + ApmDataDumper data_dumper(0); + std::vector lag_estimates(1); + MatchedFilterLagAggregator aggregator(&data_dumper, 100); + for (size_t k = 0; k < kNumLagsBeforeDetection * 100; ++k) { + lag_estimates[0] = MatchedFilter::LagEstimate(1.f, true, k % 100, true); + rtc::Optional aggregated_lag = aggregator.Aggregate(lag_estimates); + EXPECT_FALSE(aggregated_lag); + } +} + +// Verifies that lag estimate updates are required to produce an updated lag +// aggregate. +TEST(MatchedFilterLagAggregator, + DISABLED_LagEstimateUpdatesRequiredForAggregatedLag) { + constexpr size_t kLag = 5; + ApmDataDumper data_dumper(0); + std::vector lag_estimates(1); + MatchedFilterLagAggregator aggregator(&data_dumper, kLag); + for (size_t k = 0; k < kNumLagsBeforeDetection * 10; ++k) { + lag_estimates[0] = MatchedFilter::LagEstimate(1.f, true, kLag, false); + rtc::Optional aggregated_lag = aggregator.Aggregate(lag_estimates); + EXPECT_FALSE(aggregated_lag); + EXPECT_EQ(kLag, *aggregated_lag); + } +} + +// Verifies that an aggregated lag is persistent if the lag estimates do not +// change and that an aggregated lag is not produced without gaining lag +// estimate confidence. +TEST(MatchedFilterLagAggregator, DISABLED_PersistentAggregatedLag) { + constexpr size_t kLag1 = 5; + constexpr size_t kLag2 = 10; + ApmDataDumper data_dumper(0); + std::vector lag_estimates(1); + MatchedFilterLagAggregator aggregator(&data_dumper, std::max(kLag1, kLag2)); + rtc::Optional aggregated_lag; + for (size_t k = 0; k < kNumLagsBeforeDetection; ++k) { + lag_estimates[0] = MatchedFilter::LagEstimate(1.f, true, kLag1, true); + aggregated_lag = aggregator.Aggregate(lag_estimates); + } + EXPECT_TRUE(aggregated_lag); + EXPECT_EQ(kLag1, *aggregated_lag); + + for (size_t k = 0; k < kNumLagsBeforeDetection * 40; ++k) { + lag_estimates[0] = MatchedFilter::LagEstimate(1.f, false, kLag2, true); + aggregated_lag = aggregator.Aggregate(lag_estimates); + EXPECT_TRUE(aggregated_lag); + EXPECT_EQ(kLag1, *aggregated_lag); + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for non-null data dumper. +TEST(MatchedFilterLagAggregator, NullDataDumper) { + EXPECT_DEATH(MatchedFilterLagAggregator(nullptr, 10), ""); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc new file mode 100644 index 0000000000..06004190ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/matched_filter.h" + +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/decimator.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/random.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" + +namespace webrtc { +namespace aec3 { +namespace { + +std::string ProduceDebugText(size_t delay, size_t down_sampling_factor) { + std::ostringstream ss; + ss << "Delay: " << delay; + ss << ", Down sampling factor: " << down_sampling_factor; + return ss.str(); +} + +constexpr size_t kNumMatchedFilters = 10; +constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; +constexpr size_t kWindowSizeSubBlocks = 32; +constexpr size_t kAlignmentShiftSubBlocks = kWindowSizeSubBlocks * 3 / 4; + +} // namespace + +#if defined(WEBRTC_HAS_NEON) +// Verifies that the optimized methods for NEON are similar to their reference +// counterparts. +TEST(MatchedFilter, TestNeonOptimizations) { + Random random_generator(42U); + for (auto down_sampling_factor : kDownSamplingFactors) { + const size_t sub_block_size = kBlockSize / down_sampling_factor; + + std::vector x(2000); + RandomizeSampleVector(&random_generator, x); + std::vector y(sub_block_size); + std::vector h_NEON(512); + std::vector h(512); + int x_index = 0; + for (int k = 0; k < 1000; ++k) { + RandomizeSampleVector(&random_generator, y); + + bool filters_updated = false; + float error_sum = 0.f; + bool filters_updated_NEON = false; + float error_sum_NEON = 0.f; + + MatchedFilterCore_NEON(x_index, h.size() * 150.f * 150.f, x, y, h_NEON, + &filters_updated_NEON, &error_sum_NEON); + + MatchedFilterCore(x_index, h.size() * 150.f * 150.f, x, y, h, + &filters_updated, &error_sum); + + EXPECT_EQ(filters_updated, filters_updated_NEON); + EXPECT_NEAR(error_sum, error_sum_NEON, error_sum / 100000.f); + + for (size_t j = 0; j < h.size(); ++j) { + EXPECT_NEAR(h[j], h_NEON[j], 0.00001f); + } + + x_index = (x_index + sub_block_size) % x.size(); + } + } +} +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) +// Verifies that the optimized methods for SSE2 are bitexact to their reference +// counterparts. +TEST(MatchedFilter, TestSse2Optimizations) { + bool use_sse2 = (WebRtc_GetCPUInfo(kSSE2) != 0); + if (use_sse2) { + Random random_generator(42U); + for (auto down_sampling_factor : kDownSamplingFactors) { + const size_t sub_block_size = kBlockSize / down_sampling_factor; + std::vector x(2000); + RandomizeSampleVector(&random_generator, x); + std::vector y(sub_block_size); + std::vector h_SSE2(512); + std::vector h(512); + int x_index = 0; + for (int k = 0; k < 1000; ++k) { + RandomizeSampleVector(&random_generator, y); + + bool filters_updated = false; + float error_sum = 0.f; + bool filters_updated_SSE2 = false; + float error_sum_SSE2 = 0.f; + + MatchedFilterCore_SSE2(x_index, h.size() * 150.f * 150.f, x, y, h_SSE2, + &filters_updated_SSE2, &error_sum_SSE2); + + MatchedFilterCore(x_index, h.size() * 150.f * 150.f, x, y, h, + &filters_updated, &error_sum); + + EXPECT_EQ(filters_updated, filters_updated_SSE2); + EXPECT_NEAR(error_sum, error_sum_SSE2, error_sum / 100000.f); + + for (size_t j = 0; j < h.size(); ++j) { + EXPECT_NEAR(h[j], h_SSE2[j], 0.00001f); + } + + x_index = (x_index + sub_block_size) % x.size(); + } + } + } +} + +#endif + +// Verifies that the matched filter produces proper lag estimates for +// artificially +// delayed signals. +TEST(MatchedFilter, LagEstimation) { + Random random_generator(42U); + for (auto down_sampling_factor : kDownSamplingFactors) { + const size_t sub_block_size = kBlockSize / down_sampling_factor; + + std::vector> render(3, + std::vector(kBlockSize, 0.f)); + std::array capture; + capture.fill(0.f); + ApmDataDumper data_dumper(0); + for (size_t delay_samples : {5, 64, 150, 200, 800, 1000}) { + SCOPED_TRACE(ProduceDebugText(delay_samples, down_sampling_factor)); + Decimator capture_decimator(down_sampling_factor); + DelayBuffer signal_delay_buffer(down_sampling_factor * + delay_samples); + MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size, + kWindowSizeSubBlocks, kNumMatchedFilters, + kAlignmentShiftSubBlocks, 150); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, + kNumMatchedFilters), + GetRenderDelayBufferSize(down_sampling_factor, + kNumMatchedFilters))); + + // Analyze the correlation between render and capture. + for (size_t k = 0; k < (300 + delay_samples / sub_block_size); ++k) { + RandomizeSampleVector(&random_generator, render[0]); + signal_delay_buffer.Delay(render[0], capture); + render_delay_buffer->Insert(render); + render_delay_buffer->UpdateBuffers(); + std::array downsampled_capture_data; + rtc::ArrayView downsampled_capture( + downsampled_capture_data.data(), sub_block_size); + capture_decimator.Decimate(capture, downsampled_capture); + filter.Update(render_delay_buffer->GetDownsampledRenderBuffer(), + downsampled_capture); + } + + // Obtain the lag estimates. + auto lag_estimates = filter.GetLagEstimates(); + + // Find which lag estimate should be the most accurate. + rtc::Optional expected_most_accurate_lag_estimate; + size_t alignment_shift_sub_blocks = 0; + for (size_t k = 0; k < kNumMatchedFilters; ++k) { + if ((alignment_shift_sub_blocks + 3 * kWindowSizeSubBlocks / 4) * + sub_block_size > + delay_samples) { + expected_most_accurate_lag_estimate = k > 0 ? k - 1 : 0; + break; + } + alignment_shift_sub_blocks += kAlignmentShiftSubBlocks; + } + ASSERT_TRUE(expected_most_accurate_lag_estimate); + + // Verify that the expected most accurate lag estimate is the most + // accurate estimate. + for (size_t k = 0; k < kNumMatchedFilters; ++k) { + if (k != *expected_most_accurate_lag_estimate && + k != (*expected_most_accurate_lag_estimate + 1)) { + EXPECT_TRUE( + lag_estimates[*expected_most_accurate_lag_estimate].accuracy > + lag_estimates[k].accuracy || + !lag_estimates[k].reliable || + !lag_estimates[*expected_most_accurate_lag_estimate].reliable); + } + } + + // Verify that all lag estimates are updated as expected for signals + // containing strong noise. + for (auto& le : lag_estimates) { + EXPECT_TRUE(le.updated); + } + + // Verify that the expected most accurate lag estimate is reliable. + EXPECT_TRUE( + lag_estimates[*expected_most_accurate_lag_estimate].reliable || + lag_estimates[std::min(*expected_most_accurate_lag_estimate + 1, + lag_estimates.size() - 1)] + .reliable); + + // Verify that the expected most accurate lag estimate is correct. + if (lag_estimates[*expected_most_accurate_lag_estimate].reliable) { + EXPECT_TRUE(delay_samples == + lag_estimates[*expected_most_accurate_lag_estimate].lag); + } else { + EXPECT_TRUE( + delay_samples == + lag_estimates[std::min(*expected_most_accurate_lag_estimate + 1, + lag_estimates.size() - 1)] + .lag); + } + } + } +} + +// Verifies that the matched filter does not produce reliable and accurate +// estimates for uncorrelated render and capture signals. +TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) { + Random random_generator(42U); + for (auto down_sampling_factor : kDownSamplingFactors) { + const size_t sub_block_size = kBlockSize / down_sampling_factor; + + std::vector> render(3, + std::vector(kBlockSize, 0.f)); + std::array capture_data; + rtc::ArrayView capture(capture_data.data(), sub_block_size); + std::fill(capture.begin(), capture.end(), 0.f); + ApmDataDumper data_dumper(0); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, kNumMatchedFilters), + GetRenderDelayBufferSize(down_sampling_factor, + kNumMatchedFilters))); + MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size, + kWindowSizeSubBlocks, kNumMatchedFilters, + kAlignmentShiftSubBlocks, 150); + + // Analyze the correlation between render and capture. + for (size_t k = 0; k < 100; ++k) { + RandomizeSampleVector(&random_generator, render[0]); + RandomizeSampleVector(&random_generator, capture); + render_delay_buffer->Insert(render); + filter.Update(render_delay_buffer->GetDownsampledRenderBuffer(), capture); + } + + // Obtain the lag estimates. + auto lag_estimates = filter.GetLagEstimates(); + EXPECT_EQ(kNumMatchedFilters, lag_estimates.size()); + + // Verify that no lag estimates are reliable. + for (auto& le : lag_estimates) { + EXPECT_FALSE(le.reliable); + } + } +} + +// Verifies that the matched filter does not produce updated lag estimates for +// render signals of low level. +TEST(MatchedFilter, LagNotUpdatedForLowLevelRender) { + Random random_generator(42U); + for (auto down_sampling_factor : kDownSamplingFactors) { + const size_t sub_block_size = kBlockSize / down_sampling_factor; + + std::vector> render(3, + std::vector(kBlockSize, 0.f)); + std::array capture; + capture.fill(0.f); + ApmDataDumper data_dumper(0); + MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size, + kWindowSizeSubBlocks, kNumMatchedFilters, + kAlignmentShiftSubBlocks, 150); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + 3, down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, kNumMatchedFilters), + GetRenderDelayBufferSize(down_sampling_factor, + kNumMatchedFilters))); + Decimator capture_decimator(down_sampling_factor); + + // Analyze the correlation between render and capture. + for (size_t k = 0; k < 100; ++k) { + RandomizeSampleVector(&random_generator, render[0]); + for (auto& render_k : render[0]) { + render_k *= 149.f / 32767.f; + } + std::copy(render[0].begin(), render[0].end(), capture.begin()); + std::array downsampled_capture_data; + rtc::ArrayView downsampled_capture(downsampled_capture_data.data(), + sub_block_size); + capture_decimator.Decimate(capture, downsampled_capture); + filter.Update(render_delay_buffer->GetDownsampledRenderBuffer(), + downsampled_capture); + } + + // Obtain the lag estimates. + auto lag_estimates = filter.GetLagEstimates(); + EXPECT_EQ(kNumMatchedFilters, lag_estimates.size()); + + // Verify that no lag estimates are updated and that no lag estimates are + // reliable. + for (auto& le : lag_estimates) { + EXPECT_FALSE(le.updated); + EXPECT_FALSE(le.reliable); + } + } +} + +// Verifies that the correct number of lag estimates are produced for a certain +// number of alignment shifts. +TEST(MatchedFilter, NumberOfLagEstimates) { + ApmDataDumper data_dumper(0); + for (auto down_sampling_factor : kDownSamplingFactors) { + const size_t sub_block_size = kBlockSize / down_sampling_factor; + for (size_t num_matched_filters = 0; num_matched_filters < 10; + ++num_matched_filters) { + MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size, + 32, num_matched_filters, 1, 150); + EXPECT_EQ(num_matched_filters, filter.GetLagEstimates().size()); + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for non-zero windows size. +TEST(MatchedFilter, ZeroWindowSize) { + ApmDataDumper data_dumper(0); + EXPECT_DEATH( + MatchedFilter(&data_dumper, DetectOptimization(), 16, 0, 1, 1, 150), ""); +} + +// Verifies the check for non-null data dumper. +TEST(MatchedFilter, NullDataDumper) { + EXPECT_DEATH(MatchedFilter(nullptr, DetectOptimization(), 16, 1, 1, 1, 150), + ""); +} + +// Verifies the check for that the sub block size is a multiple of 4. +// TODO(peah): Activate the unittest once the required code has been landed. +TEST(MatchedFilter, DISABLED_BlockSizeMultipleOf4) { + ApmDataDumper data_dumper(0); + EXPECT_DEATH( + MatchedFilter(&data_dumper, DetectOptimization(), 15, 1, 1, 1, 150), ""); +} + +// Verifies the check for that there is an integer number of sub blocks that add +// up to a block size. +// TODO(peah): Activate the unittest once the required code has been landed. +TEST(MatchedFilter, DISABLED_SubBlockSizeAddsUpToBlockSize) { + ApmDataDumper data_dumper(0); + EXPECT_DEATH( + MatchedFilter(&data_dumper, DetectOptimization(), 12, 1, 1, 1, 150), ""); +} + +#endif + +} // namespace aec3 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h new file mode 100644 index 0000000000..5fff456185 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_BLOCK_PROCESSOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_BLOCK_PROCESSOR_H_ + +#include + +#include "modules/audio_processing/aec3/block_processor.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockBlockProcessor : public BlockProcessor { + public: + virtual ~MockBlockProcessor() {} + + MOCK_METHOD3(ProcessCapture, + void(bool level_change, + bool saturated_microphone_signal, + std::vector>* capture_block)); + MOCK_METHOD1(BufferRender, + void(const std::vector>& block)); + MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); + MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_BLOCK_PROCESSOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h new file mode 100644 index 0000000000..44d3778e6d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_ECHO_REMOVER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_ECHO_REMOVER_H_ + +#include + +#include "api/optional.h" +#include "modules/audio_processing/aec3/echo_path_variability.h" +#include "modules/audio_processing/aec3/echo_remover.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockEchoRemover : public EchoRemover { + public: + virtual ~MockEchoRemover() = default; + + MOCK_METHOD5(ProcessCapture, + void(const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const RenderBuffer& render_buffer, + std::vector>* capture)); + + MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); + MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_ECHO_REMOVER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h new file mode 100644 index 0000000000..6b5870901d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_BUFFER_H_ + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/downsampled_render_buffer.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockRenderDelayBuffer : public RenderDelayBuffer { + public: + explicit MockRenderDelayBuffer(int sample_rate_hz) + : render_buffer_(Aec3Optimization::kNone, + NumBandsForRate(sample_rate_hz), + GetRenderDelayBufferSize(4, 4), + std::vector(1, kAdaptiveFilterLength)), + downsampled_render_buffer_(GetDownSampledBufferSize(4, 4)) { + ON_CALL(*this, GetRenderBuffer()) + .WillByDefault( + testing::Invoke(this, &MockRenderDelayBuffer::FakeGetRenderBuffer)); + ON_CALL(*this, GetDownsampledRenderBuffer()) + .WillByDefault(testing::Invoke( + this, &MockRenderDelayBuffer::FakeGetDownsampledRenderBuffer)); + } + virtual ~MockRenderDelayBuffer() = default; + + MOCK_METHOD0(Reset, void()); + MOCK_METHOD1(Insert, bool(const std::vector>& block)); + MOCK_METHOD0(UpdateBuffers, bool()); + MOCK_METHOD1(SetDelay, void(size_t delay)); + MOCK_CONST_METHOD0(Delay, size_t()); + MOCK_CONST_METHOD0(MaxDelay, size_t()); + MOCK_CONST_METHOD0(IsBlockAvailable, bool()); + MOCK_CONST_METHOD0(GetRenderBuffer, const RenderBuffer&()); + MOCK_CONST_METHOD0(GetDownsampledRenderBuffer, + const DownsampledRenderBuffer&()); + + private: + const RenderBuffer& FakeGetRenderBuffer() const { return render_buffer_; } + const DownsampledRenderBuffer& FakeGetDownsampledRenderBuffer() const { + return downsampled_render_buffer_; + } + RenderBuffer render_buffer_; + DownsampledRenderBuffer downsampled_render_buffer_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h new file mode 100644 index 0000000000..b1f1cbe6cf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_CONTROLLER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_CONTROLLER_H_ + +#include "api/array_view.h" +#include "api/optional.h" +#include "modules/audio_processing/aec3/downsampled_render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_controller.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockRenderDelayController : public RenderDelayController { + public: + virtual ~MockRenderDelayController() = default; + + MOCK_METHOD0(Reset, void()); + MOCK_METHOD1(SetDelay, void(size_t render_delay)); + MOCK_METHOD2(GetDelay, + size_t(const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture)); + MOCK_CONST_METHOD0(AlignmentHeadroomSamples, rtc::Optional()); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.cc new file mode 100644 index 0000000000..4f547d98d9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/output_selector.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Performs the transition between the signals in a smooth manner. +void SmoothFrameTransition(bool from_y_to_e, + rtc::ArrayView e, + rtc::ArrayView y) { + RTC_DCHECK_LT(0u, e.size()); + RTC_DCHECK_EQ(y.size(), e.size()); + + const float change_factor = (from_y_to_e ? 1.f : -1.f) / e.size(); + float averaging = from_y_to_e ? 0.f : 1.f; + for (size_t k = 0; k < e.size(); ++k) { + y[k] += averaging * (e[k] - y[k]); + averaging += change_factor; + } + RTC_DCHECK_EQ(from_y_to_e ? 1.f : 0.f, averaging); +} + +} // namespace + +OutputSelector::OutputSelector() = default; + +OutputSelector::~OutputSelector() = default; + +void OutputSelector::FormLinearOutput( + bool use_subtractor_output, + rtc::ArrayView subtractor_output, + rtc::ArrayView capture) { + RTC_DCHECK_EQ(subtractor_output.size(), capture.size()); + rtc::ArrayView& e_main = subtractor_output; + rtc::ArrayView y = capture; + + if (use_subtractor_output != use_subtractor_output_) { + use_subtractor_output_ = use_subtractor_output; + SmoothFrameTransition(use_subtractor_output_, e_main, y); + } else if (use_subtractor_output_) { + std::copy(e_main.begin(), e_main.end(), y.begin()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.h new file mode 100644 index 0000000000..a406c61745 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_OUTPUT_SELECTOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_OUTPUT_SELECTOR_H_ + +#include "api/array_view.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Performs the selection between which of the linear aec output and the +// microphone signal should be used as the echo suppressor output. +class OutputSelector { + public: + OutputSelector(); + ~OutputSelector(); + + // Forms the most appropriate output signal. + void FormLinearOutput(bool use_subtractor_output, + rtc::ArrayView subtractor_output, + rtc::ArrayView capture); + + // Returns true if the linear aec output is the one used. + bool UseSubtractorOutput() const { return use_subtractor_output_; } + + private: + bool use_subtractor_output_ = false; + RTC_DISALLOW_COPY_AND_ASSIGN(OutputSelector); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_OUTPUT_SELECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector_unittest.cc new file mode 100644 index 0000000000..c7add1c838 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector_unittest.cc @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/output_selector.h" + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "test/gtest.h" + +namespace webrtc { + +// Verifies that the switching between the signals in the output works as +// intended. +TEST(OutputSelector, ProperSwitching) { + OutputSelector selector; + + std::array y; + std::array e; + std::array e_ref; + std::array y_ref; + auto init_blocks = [](std::array* e, + std::array* y) { + e->fill(10.f); + y->fill(20.f); + }; + + init_blocks(&e_ref, &y_ref); + + init_blocks(&e, &y); + selector.FormLinearOutput(false, e, y); + EXPECT_EQ(y_ref, y); + + init_blocks(&e, &y); + selector.FormLinearOutput(true, e, y); + EXPECT_NE(e_ref, y); + EXPECT_NE(y_ref, y); + + init_blocks(&e, &y); + selector.FormLinearOutput(true, e, y); + EXPECT_EQ(e_ref, y); + + init_blocks(&e, &y); + selector.FormLinearOutput(true, e, y); + EXPECT_EQ(e_ref, y); + + init_blocks(&e, &y); + selector.FormLinearOutput(false, e, y); + EXPECT_NE(e_ref, y); + EXPECT_NE(y_ref, y); + + init_blocks(&e, &y); + selector.FormLinearOutput(false, e, y); + EXPECT_EQ(y_ref, y); + + init_blocks(&e, &y); + selector.FormLinearOutput(false, e, y); + EXPECT_EQ(y_ref, y); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.cc new file mode 100644 index 0000000000..fa86ea6b36 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.cc @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_buffer.h" + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +RenderBuffer::RenderBuffer(Aec3Optimization optimization, + size_t num_bands, + size_t num_partitions, + const std::vector num_ffts_for_spectral_sums) + : optimization_(optimization), + fft_buffer_(num_partitions), + spectrum_buffer_(num_partitions, std::array()), + spectral_sums_(num_ffts_for_spectral_sums.size(), + std::array()), + last_block_(num_bands, std::vector(kBlockSize, 0.f)), + fft_() { + // Current implementation only allows a maximum of one spectral sum lengths. + RTC_DCHECK_EQ(1, num_ffts_for_spectral_sums.size()); + spectral_sums_length_ = num_ffts_for_spectral_sums[0]; + RTC_DCHECK_GE(fft_buffer_.size(), spectral_sums_length_); + + Clear(); +} + +RenderBuffer::~RenderBuffer() = default; + +void RenderBuffer::Clear() { + position_ = 0; + for (auto& sum : spectral_sums_) { + sum.fill(0.f); + } + + for (auto& spectrum : spectrum_buffer_) { + spectrum.fill(0.f); + } + + for (auto& fft : fft_buffer_) { + fft.Clear(); + } + + for (auto& b : last_block_) { + std::fill(b.begin(), b.end(), 0.f); + } +} + +void RenderBuffer::Insert(const std::vector>& block) { + // Compute the FFT of the data in the lowest band. + FftData X; + fft_.PaddedFft(block[0], last_block_[0], &X); + + // Copy the last render frame. + RTC_DCHECK_EQ(last_block_.size(), block.size()); + for (size_t k = 0; k < block.size(); ++k) { + RTC_DCHECK_EQ(last_block_[k].size(), block[k].size()); + std::copy(block[k].begin(), block[k].end(), last_block_[k].begin()); + } + + // Insert X into the buffer. + position_ = position_ > 0 ? position_ - 1 : fft_buffer_.size() - 1; + fft_buffer_[position_].Assign(X); + + // Compute and insert the spectrum for the FFT into the spectrum buffer. + X.Spectrum(optimization_, &spectrum_buffer_[position_]); + + // Pre-compute and cache the spectral sums. + std::copy(spectrum_buffer_[position_].begin(), + spectrum_buffer_[position_].end(), spectral_sums_[0].begin()); + size_t position = (position_ + 1) % fft_buffer_.size(); + for (size_t j = 1; j < spectral_sums_length_; ++j) { + const std::array& spectrum = + spectrum_buffer_[position]; + + for (size_t k = 0; k < spectral_sums_[0].size(); ++k) { + spectral_sums_[0][k] += spectrum[k]; + } + + position = position < (fft_buffer_.size() - 1) ? position + 1 : 0; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.h new file mode 100644 index 0000000000..3288ff36ad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_RENDER_BUFFER_H_ + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Provides a buffer of the render data for the echo remover. +class RenderBuffer { + public: + // The constructor takes, besides from the other parameters, a vector + // containing the number of FFTs that will be included in the spectral sums in + // the call to SpectralSum. + RenderBuffer(Aec3Optimization optimization, + size_t num_bands, + size_t size, + const std::vector num_ffts_for_spectral_sums); + ~RenderBuffer(); + + // Clears the buffer. + void Clear(); + + // Insert a block into the buffer. + void Insert(const std::vector>& block); + + // Gets the last inserted block. + const std::vector>& MostRecentBlock() const { + return last_block_; + } + + // Get the spectrum from one of the FFTs in the buffer + const std::array& Spectrum( + size_t buffer_offset_ffts) const { + return spectrum_buffer_[(position_ + buffer_offset_ffts) % + fft_buffer_.size()]; + } + + // Returns the sum of the spectrums for a certain number of FFTs. + const std::array& SpectralSum( + size_t num_ffts) const { + RTC_DCHECK_EQ(spectral_sums_length_, num_ffts); + return spectral_sums_[0]; + } + + // Returns the circular buffer. + rtc::ArrayView Buffer() const { return fft_buffer_; } + + // Returns the current position in the circular buffer + size_t Position() const { return position_; } + + private: + const Aec3Optimization optimization_; + std::vector fft_buffer_; + std::vector> spectrum_buffer_; + size_t spectral_sums_length_; + std::vector> spectral_sums_; + size_t position_ = 0; + std::vector> last_block_; + const Aec3Fft fft_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderBuffer); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_RENDER_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc new file mode 100644 index 0000000000..1498f4ea25 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_buffer.h" + +#include +#include +#include + +#include "test/gtest.h" + +namespace webrtc { + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for the provided numbers of Ffts to include in the +// spectral sum. +TEST(RenderBuffer, TooLargeNumberOfSpectralSums) { + EXPECT_DEATH( + RenderBuffer(Aec3Optimization::kNone, 3, 1, std::vector(2, 1)), + ""); +} + +TEST(RenderBuffer, TooSmallNumberOfSpectralSums) { + EXPECT_DEATH( + RenderBuffer(Aec3Optimization::kNone, 3, 1, std::vector()), ""); +} + +// Verifies the feasibility check for the provided number of Ffts to include in +// the spectral. +TEST(RenderBuffer, FeasibleNumberOfFftsInSum) { + EXPECT_DEATH( + RenderBuffer(Aec3Optimization::kNone, 3, 1, std::vector(1, 2)), + ""); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc new file mode 100644 index 0000000000..d2ead63b02 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_delay_buffer.h" + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block_processor.h" +#include "modules/audio_processing/aec3/decimator.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +class ApiCallJitterBuffer { + public: + explicit ApiCallJitterBuffer(size_t num_bands) { + buffer_.fill(std::vector>( + num_bands, std::vector(kBlockSize, 0.f))); + } + + ~ApiCallJitterBuffer() = default; + + void Reset() { + size_ = 0; + last_insert_index_ = 0; + } + + void Insert(const std::vector>& block) { + RTC_DCHECK_LT(size_, buffer_.size()); + last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); + RTC_DCHECK_EQ(buffer_[last_insert_index_].size(), block.size()); + RTC_DCHECK_EQ(buffer_[last_insert_index_][0].size(), block[0].size()); + for (size_t k = 0; k < block.size(); ++k) { + std::copy(block[k].begin(), block[k].end(), + buffer_[last_insert_index_][k].begin()); + } + ++size_; + } + + void Remove(std::vector>* block) { + RTC_DCHECK_LT(0, size_); + --size_; + const size_t extract_index = + (last_insert_index_ - size_ + buffer_.size()) % buffer_.size(); + for (size_t k = 0; k < block->size(); ++k) { + std::copy(buffer_[extract_index][k].begin(), + buffer_[extract_index][k].end(), (*block)[k].begin()); + } + } + + size_t Size() const { return size_; } + bool Full() const { return size_ >= (buffer_.size()); } + bool Empty() const { return size_ == 0; } + + private: + std::array>, kMaxApiCallsJitterBlocks> buffer_; + size_t size_ = 0; + int last_insert_index_ = 0; +}; + +class RenderDelayBufferImpl final : public RenderDelayBuffer { + public: + RenderDelayBufferImpl(size_t num_bands, + size_t down_sampling_factor, + size_t downsampled_render_buffer_size, + size_t render_delay_buffer_size); + ~RenderDelayBufferImpl() override; + + void Reset() override; + bool Insert(const std::vector>& block) override; + bool UpdateBuffers() override; + void SetDelay(size_t delay) override; + size_t Delay() const override { return delay_; } + + const RenderBuffer& GetRenderBuffer() const override { return fft_buffer_; } + + const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override { + return downsampled_render_buffer_; + } + + private: + static int instance_count_; + std::unique_ptr data_dumper_; + const Aec3Optimization optimization_; + const size_t down_sampling_factor_; + const size_t sub_block_size_; + std::vector>> buffer_; + size_t delay_ = 0; + size_t last_insert_index_ = 0; + RenderBuffer fft_buffer_; + DownsampledRenderBuffer downsampled_render_buffer_; + Decimator render_decimator_; + ApiCallJitterBuffer api_call_jitter_buffer_; + const std::vector> zero_block_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayBufferImpl); +}; + +int RenderDelayBufferImpl::instance_count_ = 0; + +RenderDelayBufferImpl::RenderDelayBufferImpl( + size_t num_bands, + size_t down_sampling_factor, + size_t downsampled_render_buffer_size, + size_t render_delay_buffer_size) + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + optimization_(DetectOptimization()), + down_sampling_factor_(down_sampling_factor), + sub_block_size_(down_sampling_factor_ > 0 + ? kBlockSize / down_sampling_factor + : kBlockSize), + buffer_( + render_delay_buffer_size, + std::vector>(num_bands, + std::vector(kBlockSize, 0.f))), + fft_buffer_( + optimization_, + num_bands, + std::max(kUnknownDelayRenderWindowSize, kAdaptiveFilterLength), + std::vector(1, kAdaptiveFilterLength)), + downsampled_render_buffer_(downsampled_render_buffer_size), + render_decimator_(down_sampling_factor_), + api_call_jitter_buffer_(num_bands), + zero_block_(num_bands, std::vector(kBlockSize, 0.f)) { + RTC_DCHECK_LT(buffer_.size(), downsampled_render_buffer_.buffer.size()); +} + +RenderDelayBufferImpl::~RenderDelayBufferImpl() = default; + +void RenderDelayBufferImpl::Reset() { + // Empty all data in the buffers. + delay_ = 0; + last_insert_index_ = 0; + downsampled_render_buffer_.position = 0; + std::fill(downsampled_render_buffer_.buffer.begin(), + downsampled_render_buffer_.buffer.end(), 0.f); + fft_buffer_.Clear(); + api_call_jitter_buffer_.Reset(); + for (auto& c : buffer_) { + for (auto& b : c) { + std::fill(b.begin(), b.end(), 0.f); + } + } +} + +bool RenderDelayBufferImpl::Insert( + const std::vector>& block) { + RTC_DCHECK_EQ(block.size(), buffer_[0].size()); + RTC_DCHECK_EQ(block[0].size(), buffer_[0][0].size()); + + if (api_call_jitter_buffer_.Full()) { + // Report buffer overrun and let the caller handle the overrun. + return false; + } + api_call_jitter_buffer_.Insert(block); + + return true; +} + +bool RenderDelayBufferImpl::UpdateBuffers() { + bool underrun = true; + // Update the buffers with a new block if such is available, otherwise insert + // a block of silence. + if (api_call_jitter_buffer_.Size() > 0) { + last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); + api_call_jitter_buffer_.Remove(&buffer_[last_insert_index_]); + underrun = false; + } + + downsampled_render_buffer_.position = + (downsampled_render_buffer_.position - sub_block_size_ + + downsampled_render_buffer_.buffer.size()) % + downsampled_render_buffer_.buffer.size(); + + rtc::ArrayView input( + underrun ? zero_block_[0].data() : buffer_[last_insert_index_][0].data(), + kBlockSize); + rtc::ArrayView output(downsampled_render_buffer_.buffer.data() + + downsampled_render_buffer_.position, + sub_block_size_); + data_dumper_->DumpWav("aec3_render_decimator_input", input.size(), + input.data(), 16000, 1); + render_decimator_.Decimate(input, output); + data_dumper_->DumpWav("aec3_render_decimator_output", output.size(), + output.data(), 16000 / down_sampling_factor_, 1); + for (size_t k = 0; k < output.size() / 2; ++k) { + float tmp = output[k]; + output[k] = output[output.size() - 1 - k]; + output[output.size() - 1 - k] = tmp; + } + + if (underrun) { + fft_buffer_.Insert(zero_block_); + } else { + fft_buffer_.Insert(buffer_[(last_insert_index_ - delay_ + buffer_.size()) % + buffer_.size()]); + } + return !underrun; +} + +void RenderDelayBufferImpl::SetDelay(size_t delay) { + if (delay_ == delay) { + return; + } + + // If there is a new delay set, clear the fft buffer. + fft_buffer_.Clear(); + + if ((buffer_.size() - 1) < delay) { + // If the desired delay is larger than the delay buffer, shorten the delay + // buffer size to achieve the desired alignment with the available buffer + // size. + downsampled_render_buffer_.position = + (downsampled_render_buffer_.position + + sub_block_size_ * (delay - (buffer_.size() - 1))) % + downsampled_render_buffer_.buffer.size(); + + last_insert_index_ = + (last_insert_index_ - (delay - (buffer_.size() - 1)) + buffer_.size()) % + buffer_.size(); + delay_ = buffer_.size() - 1; + } else { + delay_ = delay; + } +} + +} // namespace + +RenderDelayBuffer* RenderDelayBuffer::Create( + size_t num_bands, + size_t down_sampling_factor, + size_t downsampled_render_buffer_size, + size_t render_delay_buffer_size) { + return new RenderDelayBufferImpl(num_bands, down_sampling_factor, + downsampled_render_buffer_size, + render_delay_buffer_size); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.h new file mode 100644 index 0000000000..8f5de40752 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_BUFFER_H_ + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/downsampled_render_buffer.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "modules/audio_processing/aec3/render_buffer.h" + +namespace webrtc { + +// Class for buffering the incoming render blocks such that these may be +// extracted with a specified delay. +class RenderDelayBuffer { + public: + static RenderDelayBuffer* Create(size_t num_bands, + size_t down_sampling_factor, + size_t downsampled_render_buffer_size, + size_t render_delay_buffer_size); + virtual ~RenderDelayBuffer() = default; + + // Resets the buffer data. + virtual void Reset() = 0; + + // Inserts a block into the buffer and returns true if the insert is + // successful. + virtual bool Insert(const std::vector>& block) = 0; + + // Updates the buffers one step based on the specified buffer delay. Returns + // true if there was no overrun, otherwise returns false. + virtual bool UpdateBuffers() = 0; + + // Sets the buffer delay. + virtual void SetDelay(size_t delay) = 0; + + // Gets the buffer delay. + virtual size_t Delay() const = 0; + + // Returns the render buffer for the echo remover. + virtual const RenderBuffer& GetRenderBuffer() const = 0; + + // Returns the downsampled render buffer. + virtual const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc new file mode 100644 index 0000000000..3e0abea753 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_delay_buffer.h" + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +constexpr size_t kDownSamplingFactor = 4; +constexpr size_t kNumMatchedFilters = 4; + +} // namespace + +// Verifies that the buffer overflow is correctly reported. +TEST(RenderDelayBuffer, BufferOverflow) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + NumBandsForRate(rate), kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (size_t k = 0; k < kMaxApiCallsJitterBlocks; ++k) { + EXPECT_TRUE(delay_buffer->Insert(block_to_insert)); + } + EXPECT_FALSE(delay_buffer->Insert(block_to_insert)); + } +} + +// Verifies that the check for available block works. +TEST(RenderDelayBuffer, AvailableBlock) { + constexpr size_t kNumBands = 1; + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + kNumBands, kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::vector> input_block( + kNumBands, std::vector(kBlockSize, 1.f)); + EXPECT_TRUE(delay_buffer->Insert(input_block)); + delay_buffer->UpdateBuffers(); +} + +// Verifies the SetDelay method. +TEST(RenderDelayBuffer, SetDelay) { + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 1, kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + EXPECT_EQ(0u, delay_buffer->Delay()); + for (size_t delay = 0; delay < 20; ++delay) { + delay_buffer->SetDelay(delay); + EXPECT_EQ(delay, delay_buffer->Delay()); + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for feasible delay. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(RenderDelayBuffer, DISABLED_WrongDelay) { + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 3, kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + EXPECT_DEATH(delay_buffer->SetDelay(21), ""); +} + +// Verifies the check for the number of bands in the inserted blocks. +TEST(RenderDelayBuffer, WrongNumberOfBands) { + for (auto rate : {16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + NumBandsForRate(rate), kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::vector> block_to_insert( + NumBandsForRate(rate < 48000 ? rate + 16000 : 16000), + std::vector(kBlockSize, 0.f)); + EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); + } +} + +// Verifies the check of the length of the inserted blocks. +TEST(RenderDelayBuffer, WrongBlockLength) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 3, kDownSamplingFactor, + GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), + GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); + EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); + } +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.cc new file mode 100644 index 0000000000..2c1f263ee7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.cc @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/render_delay_controller.h" + +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/echo_path_delay_estimator.h" +#include "modules/audio_processing/aec3/render_delay_controller_metrics.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +namespace { + +class RenderDelayControllerImpl final : public RenderDelayController { + public: + RenderDelayControllerImpl(const EchoCanceller3Config& config, + int sample_rate_hz); + ~RenderDelayControllerImpl() override; + void Reset() override; + void SetDelay(size_t render_delay) override; + size_t GetDelay(const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture) override; + rtc::Optional AlignmentHeadroomSamples() const override { + return headroom_samples_; + } + + private: + static int instance_count_; + std::unique_ptr data_dumper_; + const size_t default_delay_; + size_t delay_; + size_t blocks_since_last_delay_estimate_ = 300000; + int echo_path_delay_samples_; + size_t align_call_counter_ = 0; + rtc::Optional headroom_samples_; + std::vector capture_delay_buffer_; + int capture_delay_buffer_index_ = 0; + RenderDelayControllerMetrics metrics_; + EchoPathDelayEstimator delay_estimator_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl); +}; + +size_t ComputeNewBufferDelay(size_t current_delay, + size_t echo_path_delay_samples) { + // The below division is not exact and the truncation is intended. + const int echo_path_delay_blocks = echo_path_delay_samples / kBlockSize; + constexpr int kDelayHeadroomBlocks = 1; + + // Compute the buffer delay increase required to achieve the desired latency. + size_t new_delay = std::max(echo_path_delay_blocks - kDelayHeadroomBlocks, 0); + + // Add hysteresis. + if (new_delay == current_delay + 1) { + new_delay = current_delay; + } + + return new_delay; +} + +int RenderDelayControllerImpl::instance_count_ = 0; + +RenderDelayControllerImpl::RenderDelayControllerImpl( + const EchoCanceller3Config& config, + int sample_rate_hz) + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + default_delay_( + std::max(config.delay.default_delay, kMinEchoPathDelayBlocks)), + delay_(default_delay_), + echo_path_delay_samples_(default_delay_ * kBlockSize), + capture_delay_buffer_(kBlockSize * (kMaxApiCallsJitterBlocks + 2), 0.f), + delay_estimator_(data_dumper_.get(), config) { + RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); + delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, + capture_delay_buffer_.size()); +} + +RenderDelayControllerImpl::~RenderDelayControllerImpl() = default; + +void RenderDelayControllerImpl::Reset() { + delay_ = default_delay_; + blocks_since_last_delay_estimate_ = 300000; + echo_path_delay_samples_ = delay_ * kBlockSize; + align_call_counter_ = 0; + headroom_samples_ = rtc::nullopt; + std::fill(capture_delay_buffer_.begin(), capture_delay_buffer_.end(), 0.f); + delay_estimator_.Reset(); +} + +void RenderDelayControllerImpl::SetDelay(size_t render_delay) { + if (delay_ != render_delay) { + // If a the delay set does not match the actual delay, reset the delay + // controller. + Reset(); + delay_ = render_delay; + } +} + +size_t RenderDelayControllerImpl::GetDelay( + const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture) { + RTC_DCHECK_EQ(kBlockSize, capture.size()); + + ++align_call_counter_; + + // Estimate the delay with a delayed capture signal in order to catch + // noncausal delays. + RTC_DCHECK_LT(capture_delay_buffer_index_ + kBlockSize - 1, + capture_delay_buffer_.size()); + const rtc::Optional echo_path_delay_samples_shifted = + delay_estimator_.EstimateDelay( + render_buffer, + rtc::ArrayView( + &capture_delay_buffer_[capture_delay_buffer_index_], kBlockSize)); + std::copy(capture.begin(), capture.end(), + capture_delay_buffer_.begin() + capture_delay_buffer_index_); + capture_delay_buffer_index_ = + (capture_delay_buffer_index_ + kBlockSize) % capture_delay_buffer_.size(); + + if (echo_path_delay_samples_shifted) { + blocks_since_last_delay_estimate_ = 0; + + // Correct for the capture signal delay. + const int echo_path_delay_samples_corrected = + static_cast(*echo_path_delay_samples_shifted) - + static_cast(capture_delay_buffer_.size()); + echo_path_delay_samples_ = std::max(0, echo_path_delay_samples_corrected); + + // Compute and set new render delay buffer delay. + const size_t new_delay = + ComputeNewBufferDelay(delay_, echo_path_delay_samples_); + if (align_call_counter_ > kNumBlocksPerSecond) { + delay_ = new_delay; + + // Update render delay buffer headroom. + if (echo_path_delay_samples_corrected >= 0) { + const int headroom = echo_path_delay_samples_ - delay_ * kBlockSize; + RTC_DCHECK_LE(0, headroom); + headroom_samples_ = headroom; + } else { + headroom_samples_ = rtc::nullopt; + } + } + + metrics_.Update(echo_path_delay_samples_, delay_); + } else { + metrics_.Update(rtc::nullopt, delay_); + } + + data_dumper_->DumpRaw("aec3_render_delay_controller_delay", 1, + &echo_path_delay_samples_); + data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay", delay_); + + return delay_; +} + +} // namespace + +RenderDelayController* RenderDelayController::Create( + const EchoCanceller3Config& config, + int sample_rate_hz) { + return new RenderDelayControllerImpl(config, sample_rate_hz); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.h new file mode 100644 index 0000000000..e971b5656a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_ + +#include "api/array_view.h" +#include "api/optional.h" +#include "modules/audio_processing/aec3/downsampled_render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { + +// Class for aligning the render and capture signal using a RenderDelayBuffer. +class RenderDelayController { + public: + static RenderDelayController* Create(const EchoCanceller3Config& config, + int sample_rate_hz); + virtual ~RenderDelayController() = default; + + // Resets the delay controller. + virtual void Reset() = 0; + + // Receives the externally used delay. + virtual void SetDelay(size_t render_delay) = 0; + + // Aligns the render buffer content with the capture signal. + virtual size_t GetDelay(const DownsampledRenderBuffer& render_buffer, + rtc::ArrayView capture) = 0; + + // Returns an approximate value for the headroom in the buffer alignment. + virtual rtc::Optional AlignmentHeadroomSamples() const = 0; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc new file mode 100644 index 0000000000..696ac29a73 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_delay_controller_metrics.h" + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +enum class DelayReliabilityCategory { + kNone, + kPoor, + kMedium, + kGood, + kExcellent, + kNumCategories +}; +enum class DelayChangesCategory { + kNone, + kFew, + kSeveral, + kMany, + kConstant, + kNumCategories +}; + +} // namespace + +void RenderDelayControllerMetrics::Update(rtc::Optional delay_samples, + size_t buffer_delay_blocks) { + ++call_counter_; + + if (!initial_update) { + if (delay_samples) { + ++reliable_delay_estimate_counter_; + size_t delay_blocks = (*delay_samples) / kBlockSize; + + if (delay_blocks != delay_blocks_) { + ++delay_change_counter_; + delay_blocks_ = delay_blocks; + } + } + } else if (++initial_call_counter_ == 5 * kNumBlocksPerSecond) { + initial_update = false; + } + + if (call_counter_ == kMetricsReportingIntervalBlocks) { + int value_to_report = static_cast(delay_blocks_); + value_to_report = std::min(124, value_to_report); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.EchoPathDelay", + value_to_report, 0, 124, 125); + + value_to_report = static_cast(buffer_delay_blocks); + value_to_report = std::min(124, value_to_report); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.BufferDelay", + value_to_report, 0, 124, 125); + + DelayReliabilityCategory delay_reliability; + if (reliable_delay_estimate_counter_ == 0) { + delay_reliability = DelayReliabilityCategory::kNone; + } else if (reliable_delay_estimate_counter_ > (call_counter_ >> 1)) { + delay_reliability = DelayReliabilityCategory::kExcellent; + } else if (reliable_delay_estimate_counter_ > 100) { + delay_reliability = DelayReliabilityCategory::kGood; + } else if (reliable_delay_estimate_counter_ > 10) { + delay_reliability = DelayReliabilityCategory::kMedium; + } else { + delay_reliability = DelayReliabilityCategory::kPoor; + } + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.EchoCanceller.ReliableDelayEstimates", + static_cast(delay_reliability), + static_cast(DelayReliabilityCategory::kNumCategories)); + + DelayChangesCategory delay_changes; + if (delay_change_counter_ == 0) { + delay_changes = DelayChangesCategory::kNone; + } else if (delay_change_counter_ > 10) { + delay_changes = DelayChangesCategory::kConstant; + } else if (delay_change_counter_ > 5) { + delay_changes = DelayChangesCategory::kMany; + } else if (delay_change_counter_ > 2) { + delay_changes = DelayChangesCategory::kSeveral; + } else { + delay_changes = DelayChangesCategory::kFew; + } + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.EchoCanceller.DelayChanges", + static_cast(delay_changes), + static_cast(DelayChangesCategory::kNumCategories)); + + metrics_reported_ = true; + call_counter_ = 0; + ResetMetrics(); + } else { + metrics_reported_ = false; + } +} + +void RenderDelayControllerMetrics::ResetMetrics() { + delay_change_counter_ = 0; + reliable_delay_estimate_counter_ = 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h new file mode 100644 index 0000000000..a93f0764ed --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_METRICS_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_METRICS_H_ + +#include "api/optional.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Handles the reporting of metrics for the render delay controller. +class RenderDelayControllerMetrics { + public: + RenderDelayControllerMetrics() = default; + + // Updates the metric with new data. + void Update(rtc::Optional delay_samples, size_t buffer_delay_blocks); + + // Returns true if the metrics have just been reported, otherwise false. + bool MetricsReported() { return metrics_reported_; } + + private: + // Resets the metrics. + void ResetMetrics(); + + size_t delay_blocks_ = 0; + int reliable_delay_estimate_counter_ = 0; + int delay_change_counter_ = 0; + int call_counter_ = 0; + int initial_call_counter_ = 0; + bool metrics_reported_ = false; + bool initial_update = true; + + RTC_DISALLOW_COPY_AND_ASSIGN(RenderDelayControllerMetrics); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_METRICS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc new file mode 100644 index 0000000000..433c249554 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_delay_controller_metrics.h" +#include "api/optional.h" +#include "modules/audio_processing/aec3/aec3_common.h" + +#include "test/gtest.h" + +namespace webrtc { + +// Verify the general functionality of RenderDelayControllerMetrics. +TEST(RenderDelayControllerMetrics, NormalUsage) { + RenderDelayControllerMetrics metrics; + + for (int j = 0; j < 3; ++j) { + for (int k = 0; k < kMetricsReportingIntervalBlocks - 1; ++k) { + metrics.Update(rtc::nullopt, 0); + EXPECT_FALSE(metrics.MetricsReported()); + } + metrics.Update(rtc::nullopt, 0); + EXPECT_TRUE(metrics.MetricsReported()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc new file mode 100644 index 0000000000..2e36d22484 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_delay_controller.h" + +#include +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block_processor.h" +#include "modules/audio_processing/aec3/decimator.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +std::string ProduceDebugText(int sample_rate_hz, size_t delay) { + std::ostringstream ss; + ss << ProduceDebugText(sample_rate_hz) << ", Delay: " << delay; + return ss.str(); +} + +constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; + +} // namespace + +// Verifies the output of GetDelay when there are no AnalyzeRender calls. +TEST(RenderDelayController, NoRenderSignal) { + std::vector block(kBlockSize, 0.f); + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create( + NumBandsForRate(rate), down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, + num_matched_filters), + GetRenderDelayBufferSize(down_sampling_factor, + num_matched_filters))); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + for (size_t k = 0; k < 100; ++k) { + EXPECT_EQ(kMinEchoPathDelayBlocks, + delay_controller->GetDelay( + delay_buffer->GetDownsampledRenderBuffer(), block)); + } + } + } + } +} + +// Verifies the basic API call sequence. +TEST(RenderDelayController, BasicApiCalls) { + std::vector capture_block(kBlockSize, 0.f); + size_t delay_blocks = 0; + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + for (auto rate : {8000, 16000, 32000, 48000}) { + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + NumBandsForRate(rate), down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, + num_matched_filters), + GetRenderDelayBufferSize(down_sampling_factor, + num_matched_filters))); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + for (size_t k = 0; k < 10; ++k) { + render_delay_buffer->Insert(render_block); + render_delay_buffer->UpdateBuffers(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture_block); + } + EXPECT_FALSE(delay_controller->AlignmentHeadroomSamples()); + EXPECT_EQ(kMinEchoPathDelayBlocks, delay_blocks); + } + } + } +} + +// Verifies that the RenderDelayController is able to align the signals for +// simple timeshifts between the signals. +TEST(RenderDelayController, Alignment) { + Random random_generator(42U); + std::vector capture_block(kBlockSize, 0.f); + size_t delay_blocks = 0; + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + for (auto rate : {8000, 16000, 32000, 48000}) { + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + + for (size_t delay_samples : {15, 50, 150, 200, 800, 4000}) { + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + NumBandsForRate(rate), down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, + num_matched_filters), + GetRenderDelayBufferSize(down_sampling_factor, + num_matched_filters))); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + DelayBuffer signal_delay_buffer(delay_samples); + for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) { + RandomizeSampleVector(&random_generator, render_block[0]); + signal_delay_buffer.Delay(render_block[0], capture_block); + render_delay_buffer->Insert(render_block); + render_delay_buffer->UpdateBuffers(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + capture_block); + } + + constexpr int kDelayHeadroomBlocks = 1; + size_t expected_delay_blocks = + std::max(0, static_cast(delay_samples / kBlockSize) - + kDelayHeadroomBlocks); + + EXPECT_EQ(expected_delay_blocks, delay_blocks); + + const rtc::Optional headroom_samples = + delay_controller->AlignmentHeadroomSamples(); + ASSERT_TRUE(headroom_samples); + EXPECT_NEAR(delay_samples - delay_blocks * kBlockSize, + *headroom_samples, 4); + } + } + } + } +} + +// Verifies that the RenderDelayController is able to properly handle noncausal +// delays. +TEST(RenderDelayController, NonCausalAlignment) { + Random random_generator(42U); + size_t delay_blocks = 0; + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + for (auto rate : {8000, 16000, 32000, 48000}) { + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::vector> capture_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + + for (int delay_samples : {-15, -50, -150, -200}) { + SCOPED_TRACE(ProduceDebugText(rate, -delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + NumBandsForRate(rate), down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, + num_matched_filters), + GetRenderDelayBufferSize(down_sampling_factor, + num_matched_filters))); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + DelayBuffer signal_delay_buffer(-delay_samples); + for (int k = 0; + k < (400 - delay_samples / static_cast(kBlockSize)); ++k) { + RandomizeSampleVector(&random_generator, capture_block[0]); + signal_delay_buffer.Delay(capture_block[0], render_block[0]); + render_delay_buffer->Insert(render_block); + render_delay_buffer->UpdateBuffers(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + capture_block[0]); + } + + EXPECT_EQ(0u, delay_blocks); + + const rtc::Optional headroom_samples = + delay_controller->AlignmentHeadroomSamples(); + ASSERT_FALSE(headroom_samples); + } + } + } + } +} + +// Verifies that the RenderDelayController is able to align the signals for +// simple timeshifts between the signals when there is jitter in the API calls. +TEST(RenderDelayController, AlignmentWithJitter) { + Random random_generator(42U); + std::vector capture_block(kBlockSize, 0.f); + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + for (auto rate : {8000, 16000, 32000, 48000}) { + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (size_t delay_samples : {15, 50, 300, 800}) { + size_t delay_blocks = 0; + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + NumBandsForRate(rate), down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, + num_matched_filters), + GetRenderDelayBufferSize(down_sampling_factor, + num_matched_filters))); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + DelayBuffer signal_delay_buffer(delay_samples); + for (size_t j = 0; j < (1000 + delay_samples / kBlockSize) / + kMaxApiCallsJitterBlocks + + 1; + ++j) { + std::vector> capture_block_buffer; + for (size_t k = 0; k < (kMaxApiCallsJitterBlocks - 1); ++k) { + RandomizeSampleVector(&random_generator, render_block[0]); + signal_delay_buffer.Delay(render_block[0], capture_block); + capture_block_buffer.push_back(capture_block); + render_delay_buffer->Insert(render_block); + } + for (size_t k = 0; k < (kMaxApiCallsJitterBlocks - 1); ++k) { + render_delay_buffer->UpdateBuffers(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + capture_block_buffer[k]); + } + } + + constexpr int kDelayHeadroomBlocks = 1; + size_t expected_delay_blocks = + std::max(0, static_cast(delay_samples / kBlockSize) - + kDelayHeadroomBlocks); + if (expected_delay_blocks < 2) { + expected_delay_blocks = 0; + } + + EXPECT_EQ(expected_delay_blocks, delay_blocks); + + const rtc::Optional headroom_samples = + delay_controller->AlignmentHeadroomSamples(); + ASSERT_TRUE(headroom_samples); + EXPECT_NEAR(delay_samples - delay_blocks * kBlockSize, + *headroom_samples, 4); + } + } + } + } +} + +// Verifies the initial value for the AlignmentHeadroomSamples. +TEST(RenderDelayController, InitialHeadroom) { + std::vector render_block(kBlockSize, 0.f); + std::vector capture_block(kBlockSize, 0.f); + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create( + NumBandsForRate(rate), down_sampling_factor, + GetDownSampledBufferSize(down_sampling_factor, + num_matched_filters), + GetRenderDelayBufferSize(down_sampling_factor, + num_matched_filters))); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + EXPECT_FALSE(delay_controller->AlignmentHeadroomSamples()); + } + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for the capture signal block size. +TEST(RenderDelayController, WrongCaptureSize) { + std::vector block(kBlockSize - 1, 0.f); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(NumBandsForRate(rate), 4, + GetDownSampledBufferSize(4, 4), + GetRenderDelayBufferSize(4, 4))); + EXPECT_DEATH( + std::unique_ptr( + RenderDelayController::Create(EchoCanceller3Config(), rate)) + ->GetDelay(render_delay_buffer->GetDownsampledRenderBuffer(), + block), + ""); + } +} + +// Verifies the check for correct sample rate. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(RenderDelayController, DISABLED_WrongSampleRate) { + for (auto rate : {-1, 0, 8001, 16001}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(NumBandsForRate(rate), 4, + GetDownSampledBufferSize(4, 4), + GetRenderDelayBufferSize(4, 4))); + EXPECT_DEATH( + std::unique_ptr( + RenderDelayController::Create(EchoCanceller3Config(), rate)), + ""); + } +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc new file mode 100644 index 0000000000..22aa352320 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_signal_analyzer.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { +constexpr size_t kCounterThreshold = 5; + +// Identifies local bands with narrow characteristics. +void IdentifySmallNarrowBandRegions( + const RenderBuffer& render_buffer, + const rtc::Optional& delay_partitions, + std::array* narrow_band_counters) { + if (!delay_partitions) { + narrow_band_counters->fill(0); + return; + } + + const std::array& X2 = + render_buffer.Spectrum(*delay_partitions); + + for (size_t k = 1; k < (X2.size() - 1); ++k) { + (*narrow_band_counters)[k - 1] = X2[k] > 3 * std::max(X2[k - 1], X2[k + 1]) + ? (*narrow_band_counters)[k - 1] + 1 + : 0; + } +} + +// Identifies whether the signal has a single strong narrow-band component. +void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer, + rtc::Optional* narrow_peak_band, + size_t* narrow_peak_counter) { + const auto X2_latest = render_buffer.Spectrum(0); + + // Identify the spectral peak. + const int peak_bin = static_cast( + std::max_element(X2_latest.begin(), X2_latest.end()) - X2_latest.begin()); + + // Compute the level around the peak. + float non_peak_power = 0.f; + for (int k = std::max(5, peak_bin - 14); k < peak_bin - 4; ++k) { + non_peak_power = std::max(X2_latest[k], non_peak_power); + } + for (int k = peak_bin + 5; + k < std::min(peak_bin + 15, static_cast(kFftLengthBy2Plus1)); ++k) { + non_peak_power = std::max(X2_latest[k], non_peak_power); + } + + // Assess the render signal strength + const std::vector>& x_latest = + render_buffer.MostRecentBlock(); + auto result0 = std::minmax_element(x_latest[0].begin(), x_latest[0].end()); + float max_abs = std::max(fabs(*result0.first), fabs(*result0.second)); + + if (x_latest.size() > 1) { + const auto result1 = + std::minmax_element(x_latest[1].begin(), x_latest[1].end()); + max_abs = + std::max(max_abs, static_cast(std::max(fabs(*result1.first), + fabs(*result1.second)))); + } + + // Detect whether the spectal peak has as strong narrowband nature. + if (peak_bin > 6 && max_abs > 100 && + X2_latest[peak_bin] > 100 * non_peak_power) { + *narrow_peak_band = peak_bin; + *narrow_peak_counter = 0; + } else { + if (*narrow_peak_band && ++(*narrow_peak_counter) > 7) { + *narrow_peak_band = rtc::nullopt; + } + } +} + +} // namespace + +RenderSignalAnalyzer::RenderSignalAnalyzer() { + narrow_band_counters_.fill(0); +} +RenderSignalAnalyzer::~RenderSignalAnalyzer() = default; + +void RenderSignalAnalyzer::Update( + const RenderBuffer& render_buffer, + const rtc::Optional& delay_partitions) { + // Identify bands of narrow nature. + IdentifySmallNarrowBandRegions(render_buffer, delay_partitions, + &narrow_band_counters_); + + // Identify the presence of a strong narrow band. + IdentifyStrongNarrowBandComponent(render_buffer, &narrow_peak_band_, + &narrow_peak_counter_); +} + +void RenderSignalAnalyzer::MaskRegionsAroundNarrowBands( + std::array* v) const { + RTC_DCHECK(v); + + // Set v to zero around narrow band signal regions. + if (narrow_band_counters_[0] > kCounterThreshold) { + (*v)[1] = (*v)[0] = 0.f; + } + for (size_t k = 2; k < kFftLengthBy2 - 1; ++k) { + if (narrow_band_counters_[k - 1] > kCounterThreshold) { + (*v)[k - 2] = (*v)[k - 1] = (*v)[k] = (*v)[k + 1] = (*v)[k + 2] = 0.f; + } + } + if (narrow_band_counters_[kFftLengthBy2 - 2] > kCounterThreshold) { + (*v)[kFftLengthBy2] = (*v)[kFftLengthBy2 - 1] = 0.f; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h new file mode 100644 index 0000000000..64d74f4e3f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_SIGNAL_ANALYZER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_RENDER_SIGNAL_ANALYZER_H_ + +#include +#include + +#include "api/optional.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Provides functionality for analyzing the properties of the render signal. +class RenderSignalAnalyzer { + public: + RenderSignalAnalyzer(); + ~RenderSignalAnalyzer(); + + // Updates the render signal analysis with the most recent render signal. + void Update(const RenderBuffer& render_buffer, + const rtc::Optional& delay_partitions); + + // Returns true if the render signal is poorly exciting. + bool PoorSignalExcitation() const { + RTC_DCHECK_LT(2, narrow_band_counters_.size()); + return std::any_of(narrow_band_counters_.begin(), + narrow_band_counters_.end(), + [](size_t a) { return a > 10; }); + } + + // Zeros the array around regions with narrow bands signal characteristics. + void MaskRegionsAroundNarrowBands( + std::array* v) const; + + rtc::Optional NarrowPeakBand() const { return narrow_peak_band_; } + + private: + std::array narrow_band_counters_; + rtc::Optional narrow_peak_band_; + size_t narrow_peak_counter_; + + RTC_DISALLOW_COPY_AND_ASSIGN(RenderSignalAnalyzer); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_RENDER_SIGNAL_ANALYZER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc new file mode 100644 index 0000000000..7e01f3fc44 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/render_signal_analyzer.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/fft_data.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr float kPi = 3.141592f; + +void ProduceSinusoid(int sample_rate_hz, + float sinusoidal_frequency_hz, + size_t* sample_counter, + rtc::ArrayView x) { + // Produce a sinusoid of the specified frequency. + for (size_t k = *sample_counter, j = 0; k < (*sample_counter + kBlockSize); + ++k, ++j) { + x[j] = + 32767.f * sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz); + } + *sample_counter = *sample_counter + kBlockSize; +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Verifies that the check for non-null output parameter works. +TEST(RenderSignalAnalyzer, NullMaskOutput) { + RenderSignalAnalyzer analyzer; + EXPECT_DEATH(analyzer.MaskRegionsAroundNarrowBands(nullptr), ""); +} + +#endif + +// Verify that no narrow bands are detected in a Gaussian noise signal. +TEST(RenderSignalAnalyzer, NoFalseDetectionOfNarrowBands) { + RenderSignalAnalyzer analyzer; + Random random_generator(42U); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::array x_old; + FftData X; + Aec3Fft fft; + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 1, + std::vector(1, 1)); + std::array mask; + x_old.fill(0.f); + + for (size_t k = 0; k < 100; ++k) { + RandomizeSampleVector(&random_generator, x[0]); + fft.PaddedFft(x[0], x_old, &X); + render_buffer.Insert(x); + analyzer.Update(render_buffer, 0); + } + + mask.fill(1.f); + analyzer.MaskRegionsAroundNarrowBands(&mask); + EXPECT_TRUE( + std::all_of(mask.begin(), mask.end(), [](float a) { return a == 1.f; })); + EXPECT_FALSE(analyzer.PoorSignalExcitation()); +} + +// Verify that a sinusiod signal is detected as narrow bands. +TEST(RenderSignalAnalyzer, NarrowBandDetection) { + RenderSignalAnalyzer analyzer; + Random random_generator(42U); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::array x_old; + Aec3Fft fft; + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 1, + std::vector(1, 1)); + std::array mask; + x_old.fill(0.f); + constexpr int kSinusFrequencyBin = 32; + + auto generate_sinusoid_test = [&](bool known_delay) { + size_t sample_counter = 0; + for (size_t k = 0; k < 100; ++k) { + ProduceSinusoid(16000, 16000 / 2 * kSinusFrequencyBin / kFftLengthBy2, + &sample_counter, x[0]); + render_buffer.Insert(x); + analyzer.Update(render_buffer, known_delay ? rtc::Optional(0) + : rtc::nullopt); + } + }; + + generate_sinusoid_test(true); + mask.fill(1.f); + analyzer.MaskRegionsAroundNarrowBands(&mask); + for (int k = 0; k < static_cast(mask.size()); ++k) { + EXPECT_EQ(abs(k - kSinusFrequencyBin) <= 2 ? 0.f : 1.f, mask[k]); + } + EXPECT_TRUE(analyzer.PoorSignalExcitation()); + + // Verify that no bands are detected as narrow when the delay is unknown. + generate_sinusoid_test(false); + mask.fill(1.f); + analyzer.MaskRegionsAroundNarrowBands(&mask); + std::for_each(mask.begin(), mask.end(), [](float a) { EXPECT_EQ(1.f, a); }); + EXPECT_FALSE(analyzer.PoorSignalExcitation()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc new file mode 100644 index 0000000000..ba65684fad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/residual_echo_estimator.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Estimates the echo generating signal power as gated maximal power over a time +// window. +void EchoGeneratingPower(const RenderBuffer& render_buffer, + size_t min_delay, + size_t max_delay, + std::array* X2) { + X2->fill(0.f); + for (size_t k = min_delay; k <= max_delay; ++k) { + std::transform(X2->begin(), X2->end(), render_buffer.Spectrum(k).begin(), + X2->begin(), + [](float a, float b) { return std::max(a, b); }); + } + + // Apply soft noise gate of -78 dBFS. + static constexpr float kNoiseGatePower = 27509.42f; + std::for_each(X2->begin(), X2->end(), [](float& a) { + if (kNoiseGatePower > a) { + a = std::max(0.f, a - 0.3f * (kNoiseGatePower - a)); + } + }); +} + +constexpr int kNoiseFloorCounterMax = 50; +constexpr float kNoiseFloorMin = 10.f * 10.f * 128.f * 128.f; + +// Updates estimate for the power of the stationary noise component in the +// render signal. +void RenderNoisePower( + const RenderBuffer& render_buffer, + std::array* X2_noise_floor, + std::array* X2_noise_floor_counter) { + RTC_DCHECK(X2_noise_floor); + RTC_DCHECK(X2_noise_floor_counter); + + const auto render_power = render_buffer.Spectrum(0); + RTC_DCHECK_EQ(X2_noise_floor->size(), render_power.size()); + RTC_DCHECK_EQ(X2_noise_floor_counter->size(), render_power.size()); + + // Estimate the stationary noise power in a minimum statistics manner. + for (size_t k = 0; k < render_power.size(); ++k) { + // Decrease rapidly. + if (render_power[k] < (*X2_noise_floor)[k]) { + (*X2_noise_floor)[k] = render_power[k]; + (*X2_noise_floor_counter)[k] = 0; + } else { + // Increase in a delayed, leaky manner. + if ((*X2_noise_floor_counter)[k] >= kNoiseFloorCounterMax) { + (*X2_noise_floor)[k] = + std::max((*X2_noise_floor)[k] * 1.1f, kNoiseFloorMin); + } else { + ++(*X2_noise_floor_counter)[k]; + } + } + } +} + +} // namespace + +ResidualEchoEstimator::ResidualEchoEstimator(const EchoCanceller3Config& config) + : config_(config) { + Reset(); +} + +ResidualEchoEstimator::~ResidualEchoEstimator() = default; + +void ResidualEchoEstimator::Estimate( + const AecState& aec_state, + const RenderBuffer& render_buffer, + const std::array& S2_linear, + const std::array& Y2, + std::array* R2) { + RTC_DCHECK(R2); + + // Estimate the power of the stationary noise in the render signal. + RenderNoisePower(render_buffer, &X2_noise_floor_, &X2_noise_floor_counter_); + + // Estimate the residual echo power. + if (aec_state.LinearEchoEstimate()) { + RTC_DCHECK(aec_state.FilterDelay()); + const int filter_delay = *aec_state.FilterDelay(); + LinearEstimate(S2_linear, aec_state.Erle(), filter_delay, R2); + AddEchoReverb(S2_linear, aec_state.SaturatedEcho(), filter_delay, + aec_state.ReverbDecay(), R2); + + // If the echo is saturated, estimate the echo power as the maximum echo + // power with a leakage factor. + if (aec_state.SaturatedEcho()) { + R2->fill((*std::max_element(R2->begin(), R2->end())) * 100.f); + } + } else { + const rtc::Optional delay = + aec_state.ExternalDelay() + ? (aec_state.FilterDelay() ? aec_state.FilterDelay() + : aec_state.ExternalDelay()) + : rtc::Optional(); + + // Estimate the echo generating signal power. + std::array X2; + if (aec_state.ExternalDelay() && aec_state.FilterDelay()) { + RTC_DCHECK(delay); + const int delay_use = static_cast(*delay); + + // Computes the spectral power over the blocks surrounding the delay. + constexpr int kKnownDelayRenderWindowSize = 5; + // TODO(peah): Add lookahead since that was what was there initially. + static_assert( + kUnknownDelayRenderWindowSize >= kKnownDelayRenderWindowSize, + "Requirement to ensure that the render buffer is overrun"); + EchoGeneratingPower( + render_buffer, std::max(0, delay_use - 1), + std::min(kKnownDelayRenderWindowSize - 1, delay_use + 1), &X2); + } else { + // Computes the spectral power over the latest blocks. + // TODO(peah): Add lookahead since that was what was there initially. + EchoGeneratingPower(render_buffer, 0, kUnknownDelayRenderWindowSize - 1, + &X2); + } + + // Subtract the stationary noise power to avoid stationary noise causing + // excessive echo suppression. + std::transform( + X2.begin(), X2.end(), X2_noise_floor_.begin(), X2.begin(), + [](float a, float b) { return std::max(0.f, a - 10.f * b); }); + + NonLinearEstimate( + aec_state.SufficientFilterUpdates(), aec_state.SaturatedEcho(), + config_.ep_strength.bounded_erl, aec_state.TransparentMode(), + aec_state.InitialState(), X2, Y2, R2); + + if (aec_state.ExternalDelay() && aec_state.FilterDelay() && + aec_state.SaturatedEcho()) { + AddEchoReverb(*R2, aec_state.SaturatedEcho(), + std::min(static_cast(kAdaptiveFilterLength), + delay.value_or(kAdaptiveFilterLength)), + aec_state.ReverbDecay(), R2); + } + } + + // If the echo is deemed inaudible, set the residual echo to zero. + if (aec_state.InaudibleEcho()) { + R2->fill(0.f); + R2_old_.fill(0.f); + R2_hold_counter_.fill(0.f); + } + + std::copy(R2->begin(), R2->end(), R2_old_.begin()); +} + +void ResidualEchoEstimator::Reset() { + X2_noise_floor_counter_.fill(kNoiseFloorCounterMax); + X2_noise_floor_.fill(kNoiseFloorMin); + R2_reverb_.fill(0.f); + R2_old_.fill(0.f); + R2_hold_counter_.fill(0.f); + for (auto& S2_k : S2_old_) { + S2_k.fill(0.f); + } +} + +void ResidualEchoEstimator::LinearEstimate( + const std::array& S2_linear, + const std::array& erle, + size_t delay, + std::array* R2) { + std::fill(R2_hold_counter_.begin(), R2_hold_counter_.end(), 10.f); + std::transform(erle.begin(), erle.end(), S2_linear.begin(), R2->begin(), + [](float a, float b) { + RTC_DCHECK_LT(0.f, a); + return b / a; + }); +} + +void ResidualEchoEstimator::NonLinearEstimate( + bool sufficient_filter_updates, + bool saturated_echo, + bool bounded_erl, + bool transparent_mode, + bool initial_state, + const std::array& X2, + const std::array& Y2, + std::array* R2) { + float echo_path_gain_lf; + float echo_path_gain_mf; + float echo_path_gain_hf; + + // Set echo path gains. + if (saturated_echo) { + // If the echo could be saturated, use a very conservative gain. + echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 10000.f; + } else if (sufficient_filter_updates && !bounded_erl) { + // If the filter should have been able to converge, and no assumption is + // possible on the ERL, use a low gain. + echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.01f; + } else if ((sufficient_filter_updates && bounded_erl) || transparent_mode) { + // If the filter should have been able to converge, and and it is known that + // the ERL is bounded, use a very low gain. + echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.001f; + } else if (!initial_state) { + // If the AEC is no longer in an initial state, assume a weak echo path. + echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.01f; + } else { + // In the initial state, use conservative gains. + echo_path_gain_lf = config_.ep_strength.lf; + echo_path_gain_mf = config_.ep_strength.mf; + echo_path_gain_hf = config_.ep_strength.hf; + } + + // Compute preliminary residual echo. + std::transform( + X2.begin(), X2.begin() + 12, R2->begin(), + [echo_path_gain_lf](float a) { return a * echo_path_gain_lf; }); + std::transform( + X2.begin() + 12, X2.begin() + 25, R2->begin() + 12, + [echo_path_gain_mf](float a) { return a * echo_path_gain_mf; }); + std::transform( + X2.begin() + 25, X2.end(), R2->begin() + 25, + [echo_path_gain_hf](float a) { return a * echo_path_gain_hf; }); + + for (size_t k = 0; k < R2->size(); ++k) { + // Update hold counter. + R2_hold_counter_[k] = R2_old_[k] < (*R2)[k] ? 0 : R2_hold_counter_[k] + 1; + + // Compute the residual echo by holding a maximum echo powers and an echo + // fading corresponding to a room with an RT60 value of about 50 ms. + (*R2)[k] = R2_hold_counter_[k] < 2 + ? std::max((*R2)[k], R2_old_[k]) + : std::min((*R2)[k] + R2_old_[k] * 0.1f, Y2[k]); + } +} + +void ResidualEchoEstimator::AddEchoReverb( + const std::array& S2, + bool saturated_echo, + size_t delay, + float reverb_decay_factor, + std::array* R2) { + // Compute the decay factor for how much the echo has decayed before leaving + // the region covered by the linear model. + auto integer_power = [](float base, int exp) { + float result = 1.f; + for (int k = 0; k < exp; ++k) { + result *= base; + } + return result; + }; + RTC_DCHECK_LE(delay, S2_old_.size()); + const float reverb_decay_for_delay = + integer_power(reverb_decay_factor, S2_old_.size() - delay); + + // Update the estimate of the reverberant residual echo power. + S2_old_index_ = S2_old_index_ > 0 ? S2_old_index_ - 1 : S2_old_.size() - 1; + const auto& S2_end = S2_old_[S2_old_index_]; + std::transform( + S2_end.begin(), S2_end.end(), R2_reverb_.begin(), R2_reverb_.begin(), + [reverb_decay_for_delay, reverb_decay_factor](float a, float b) { + return (b + a * reverb_decay_for_delay) * reverb_decay_factor; + }); + + // Update the buffer of old echo powers. + if (saturated_echo) { + S2_old_[S2_old_index_].fill((*std::max_element(S2.begin(), S2.end())) * + 100.f); + } else { + std::copy(S2.begin(), S2.end(), S2_old_[S2_old_index_].begin()); + } + + // Add the power of the echo reverb to the residual echo power. + std::transform(R2->begin(), R2->end(), R2_reverb_.begin(), R2->begin(), + std::plus()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h new file mode 100644 index 0000000000..d4c4307eb5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_RESIDUAL_ECHO_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_RESIDUAL_ECHO_ESTIMATOR_H_ + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ResidualEchoEstimator { + public: + explicit ResidualEchoEstimator(const EchoCanceller3Config& config); + ~ResidualEchoEstimator(); + + void Estimate(const AecState& aec_state, + const RenderBuffer& render_buffer, + const std::array& S2_linear, + const std::array& Y2, + std::array* R2); + + private: + // Resets the state. + void Reset(); + + // Estimates the residual echo power based on the echo return loss enhancement + // (ERLE) and the linear power estimate. + void LinearEstimate(const std::array& S2_linear, + const std::array& erle, + size_t delay, + std::array* R2); + + // Estimates the residual echo power based on the estimate of the echo path + // gain. + void NonLinearEstimate(bool sufficient_filter_updates, + bool saturated_echo, + bool bounded_erl, + bool transparent_mode, + bool initial_state, + const std::array& X2, + const std::array& Y2, + std::array* R2); + + // Adds the estimated unmodelled echo power to the residual echo power + // estimate. + void AddEchoReverb(const std::array& S2, + bool saturated_echo, + size_t delay, + float reverb_decay_factor, + std::array* R2); + + std::array R2_old_; + std::array R2_hold_counter_; + std::array R2_reverb_; + int S2_old_index_ = 0; + std::array, kAdaptiveFilterLength> + S2_old_; + std::array X2_noise_floor_; + std::array X2_noise_floor_counter_; + const EchoCanceller3Config config_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(ResidualEchoEstimator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_RESIDUAL_ECHO_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc new file mode 100644 index 0000000000..b85bc1d936 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/residual_echo_estimator.h" + +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies that the check for non-null output residual echo power works. +TEST(ResidualEchoEstimator, NullResidualEchoPowerOutput) { + AecState aec_state(EchoCanceller3Config{}); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 10, + std::vector(1, 10)); + std::vector> H2; + std::array S2_linear; + std::array Y2; + EXPECT_DEATH(ResidualEchoEstimator(EchoCanceller3Config{}) + .Estimate(aec_state, render_buffer, S2_linear, Y2, nullptr), + ""); +} + +#endif + +TEST(ResidualEchoEstimator, BasicTest) { + ResidualEchoEstimator estimator(EchoCanceller3Config{}); + EchoCanceller3Config config; + config.ep_strength.default_len = 0.f; + AecState aec_state(config); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 10, + std::vector(1, 10)); + std::array E2_main; + std::array E2_shadow; + std::array S2_linear; + std::array S2_fallback; + std::array Y2; + std::array R2; + EchoPathVariability echo_path_variability(false, false); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector> H2(10); + Random random_generator(42U); + FftData X; + std::array x_old; + std::array s; + Aec3Fft fft; + + for (auto& H2_k : H2) { + H2_k.fill(0.01f); + } + H2[2].fill(10.f); + H2[2][0] = 0.1f; + + std::array h; + h.fill(0.f); + + s.fill(100.f); + + constexpr float kLevel = 10.f; + E2_shadow.fill(kLevel); + E2_main.fill(kLevel); + S2_linear.fill(kLevel); + S2_fallback.fill(kLevel); + Y2.fill(kLevel); + + for (int k = 0; k < 2000; ++k) { + RandomizeSampleVector(&random_generator, x[0]); + std::for_each(x[0].begin(), x[0].end(), [](float& a) { a /= 30.f; }); + fft.PaddedFft(x[0], x_old, &X); + render_buffer.Insert(x); + + aec_state.HandleEchoPathChange(echo_path_variability); + aec_state.Update(H2, h, true, 2, render_buffer, E2_main, Y2, x[0], s, + false); + + estimator.Estimate(aec_state, render_buffer, S2_linear, Y2, &R2); + } + std::for_each(R2.begin(), R2.end(), + [&](float a) { EXPECT_NEAR(kLevel, a, 0.1f); }); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.cc new file mode 100644 index 0000000000..db393a78b2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.cc @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/shadow_filter_update_gain.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +void ShadowFilterUpdateGain::HandleEchoPathChange() { + // TODO(peah): Check whether this counter should instead be initialized to a + // large value. + poor_signal_excitation_counter_ = 0; + call_counter_ = 0; +} + +void ShadowFilterUpdateGain::Compute( + const RenderBuffer& render_buffer, + const RenderSignalAnalyzer& render_signal_analyzer, + const FftData& E_shadow, + size_t size_partitions, + bool saturated_capture_signal, + FftData* G) { + RTC_DCHECK(G); + ++call_counter_; + + if (render_signal_analyzer.PoorSignalExcitation()) { + poor_signal_excitation_counter_ = 0; + } + + // Do not update the filter if the render is not sufficiently excited. + if (++poor_signal_excitation_counter_ < size_partitions || + saturated_capture_signal || call_counter_ <= size_partitions) { + G->re.fill(0.f); + G->im.fill(0.f); + return; + } + + // Compute mu. + // Corresponds to WGN of power -39 dBFS. + constexpr float kNoiseGatePower = 220075344.f; + constexpr float kMuFixed = .5f; + std::array mu; + const auto& X2 = render_buffer.SpectralSum(size_partitions); + std::transform(X2.begin(), X2.end(), mu.begin(), [&](float a) { + return a > kNoiseGatePower ? kMuFixed / a : 0.f; + }); + + // Avoid updating the filter close to narrow bands in the render signals. + render_signal_analyzer.MaskRegionsAroundNarrowBands(&mu); + + // G = mu * E * X2. + std::transform(mu.begin(), mu.end(), E_shadow.re.begin(), G->re.begin(), + std::multiplies()); + std::transform(mu.begin(), mu.end(), E_shadow.im.begin(), G->im.begin(), + std::multiplies()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.h new file mode 100644 index 0000000000..8c72cf487b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_SHADOW_FILTER_UPDATE_GAIN_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_SHADOW_FILTER_UPDATE_GAIN_H_ + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_signal_analyzer.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Provides functionality for computing the fixed gain for the shadow filter. +class ShadowFilterUpdateGain { + public: + // Takes action in the case of a known echo path change. + void HandleEchoPathChange(); + + // Computes the gain. + void Compute(const RenderBuffer& render_buffer, + const RenderSignalAnalyzer& render_signal_analyzer, + const FftData& E_shadow, + size_t size_partitions, + bool saturated_capture_signal, + FftData* G); + + private: + // TODO(peah): Check whether this counter should instead be initialized to a + // large value. + size_t poor_signal_excitation_counter_ = 0; + size_t call_counter_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_SHADOW_FILTER_UPDATE_GAIN_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc new file mode 100644 index 0000000000..b89fc718ac --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/shadow_filter_update_gain.h" + +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/adaptive_fir_filter.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +// Method for performing the simulations needed to test the main filter update +// gain functionality. +void RunFilterUpdateTest(int num_blocks_to_process, + size_t delay_samples, + const std::vector& blocks_with_saturation, + std::array* e_last_block, + std::array* y_last_block, + FftData* G_last_block) { + ApmDataDumper data_dumper(42); + AdaptiveFirFilter main_filter(9, DetectOptimization(), &data_dumper); + AdaptiveFirFilter shadow_filter(9, DetectOptimization(), &data_dumper); + Aec3Fft fft; + RenderBuffer render_buffer( + Aec3Optimization::kNone, 3, main_filter.SizePartitions(), + std::vector(1, main_filter.SizePartitions())); + std::array x_old; + x_old.fill(0.f); + ShadowFilterUpdateGain shadow_gain; + Random random_generator(42U); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector y(kBlockSize, 0.f); + AecState aec_state(EchoCanceller3Config{}); + RenderSignalAnalyzer render_signal_analyzer; + std::array s; + FftData S; + FftData G; + FftData E_shadow; + std::array e_shadow; + + constexpr float kScale = 1.0f / kFftLengthBy2; + + DelayBuffer delay_buffer(delay_samples); + for (int k = 0; k < num_blocks_to_process; ++k) { + // Handle saturation. + bool saturation = + std::find(blocks_with_saturation.begin(), blocks_with_saturation.end(), + k) != blocks_with_saturation.end(); + + // Create the render signal. + RandomizeSampleVector(&random_generator, x[0]); + delay_buffer.Delay(x[0], y); + render_buffer.Insert(x); + render_signal_analyzer.Update(render_buffer, delay_samples / kBlockSize); + + shadow_filter.Filter(render_buffer, &S); + fft.Ifft(S, &s); + std::transform(y.begin(), y.end(), s.begin() + kFftLengthBy2, + e_shadow.begin(), + [&](float a, float b) { return a - b * kScale; }); + std::for_each(e_shadow.begin(), e_shadow.end(), + [](float& a) { a = rtc::SafeClamp(a, -32768.f, 32767.f); }); + fft.ZeroPaddedFft(e_shadow, &E_shadow); + + shadow_gain.Compute(render_buffer, render_signal_analyzer, E_shadow, + shadow_filter.SizePartitions(), saturation, &G); + shadow_filter.Adapt(render_buffer, G); + } + + std::copy(e_shadow.begin(), e_shadow.end(), e_last_block->begin()); + std::copy(y.begin(), y.end(), y_last_block->begin()); + std::copy(G.re.begin(), G.re.end(), G_last_block->re.begin()); + std::copy(G.im.begin(), G.im.end(), G_last_block->im.begin()); +} + +std::string ProduceDebugText(size_t delay) { + std::ostringstream ss; + ss << ", Delay: " << delay; + return ss.str(); +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies that the check for non-null output gain parameter works. +TEST(ShadowFilterUpdateGain, NullDataOutputGain) { + ApmDataDumper data_dumper(42); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 1, + std::vector(1, 1)); + RenderSignalAnalyzer analyzer; + FftData E; + ShadowFilterUpdateGain gain; + EXPECT_DEATH(gain.Compute(render_buffer, analyzer, E, 1, false, nullptr), ""); +} + +#endif + +// Verifies that the gain formed causes the filter using it to converge. +TEST(ShadowFilterUpdateGain, GainCausesFilterToConverge) { + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(delay_samples)); + + std::array e; + std::array y; + FftData G; + + RunFilterUpdateTest(500, delay_samples, blocks_with_saturation, &e, &y, &G); + + // Verify that the main filter is able to perform well. + EXPECT_LT(1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f), + std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); + } +} + +// Verifies that the magnitude of the gain on average decreases for a +// persistently exciting signal. +TEST(ShadowFilterUpdateGain, DecreasingGain) { + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; + + std::array e; + std::array y; + FftData G_a; + FftData G_b; + FftData G_c; + std::array G_a_power; + std::array G_b_power; + std::array G_c_power; + + RunFilterUpdateTest(100, 65, blocks_with_saturation, &e, &y, &G_a); + RunFilterUpdateTest(200, 65, blocks_with_saturation, &e, &y, &G_b); + RunFilterUpdateTest(300, 65, blocks_with_saturation, &e, &y, &G_c); + + G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); + G_c.Spectrum(Aec3Optimization::kNone, &G_c_power); + + EXPECT_GT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), + std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); + + EXPECT_GT(std::accumulate(G_b_power.begin(), G_b_power.end(), 0.), + std::accumulate(G_c_power.begin(), G_c_power.end(), 0.)); +} + +// Verifies that the gain is zero when there is saturation. +TEST(ShadowFilterUpdateGain, SaturationBehavior) { + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; + for (int k = 99; k < 200; ++k) { + blocks_with_saturation.push_back(k); + } + + std::array e; + std::array y; + FftData G_a; + FftData G_a_ref; + G_a_ref.re.fill(0.f); + G_a_ref.im.fill(0.f); + + RunFilterUpdateTest(100, 65, blocks_with_saturation, &e, &y, &G_a); + + EXPECT_EQ(G_a_ref.re, G_a.re); + EXPECT_EQ(G_a_ref.im, G_a.im); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.cc new file mode 100644 index 0000000000..90e14f4f27 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.cc @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/subtractor.h" + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +namespace { + +void PredictionError(const Aec3Fft& fft, + const FftData& S, + rtc::ArrayView y, + std::array* e, + FftData* E, + std::array* s) { + std::array s_scratch; + fft.Ifft(S, &s_scratch); + constexpr float kScale = 1.0f / kFftLengthBy2; + std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, + e->begin(), [&](float a, float b) { return a - b * kScale; }); + std::for_each(e->begin(), e->end(), + [](float& a) { a = rtc::SafeClamp(a, -32768.f, 32767.f); }); + fft.ZeroPaddedFft(*e, E); + + if (s) { + for (size_t k = 0; k < s->size(); ++k) { + (*s)[k] = kScale * s_scratch[k + kFftLengthBy2]; + } + } +} +} // namespace + +Subtractor::Subtractor(ApmDataDumper* data_dumper, + Aec3Optimization optimization) + : fft_(), + data_dumper_(data_dumper), + optimization_(optimization), + main_filter_(kAdaptiveFilterLength, optimization, data_dumper_), + shadow_filter_(kAdaptiveFilterLength, optimization, data_dumper_) { + RTC_DCHECK(data_dumper_); +} + +Subtractor::~Subtractor() = default; + +void Subtractor::HandleEchoPathChange( + const EchoPathVariability& echo_path_variability) { + if (echo_path_variability.delay_change) { + main_filter_.HandleEchoPathChange(); + shadow_filter_.HandleEchoPathChange(); + G_main_.HandleEchoPathChange(); + G_shadow_.HandleEchoPathChange(); + converged_filter_ = false; + } +} + +void Subtractor::Process(const RenderBuffer& render_buffer, + const rtc::ArrayView capture, + const RenderSignalAnalyzer& render_signal_analyzer, + const AecState& aec_state, + SubtractorOutput* output) { + RTC_DCHECK_EQ(kBlockSize, capture.size()); + rtc::ArrayView y = capture; + FftData& E_main = output->E_main; + FftData E_shadow; + std::array& e_main = output->e_main; + std::array& e_shadow = output->e_shadow; + + FftData S; + FftData& G = S; + + // Form the output of the main filter. + main_filter_.Filter(render_buffer, &S); + PredictionError(fft_, S, y, &e_main, &E_main, &output->s_main); + + // Form the output of the shadow filter. + shadow_filter_.Filter(render_buffer, &S); + PredictionError(fft_, S, y, &e_shadow, &E_shadow, nullptr); + + + if (!converged_filter_) { + const auto sum_of_squares = [](float a, float b) { return a + b * b; }; + const float e2_main = + std::accumulate(e_main.begin(), e_main.end(), 0.f, sum_of_squares); + const float e2_shadow = + std::accumulate(e_shadow.begin(), e_shadow.end(), 0.f, sum_of_squares); + const float y2 = std::accumulate(y.begin(), y.end(), 0.f, sum_of_squares); + + if (y2 > kBlockSize * 50.f * 50.f) { + converged_filter_ = (e2_main > 0.3 * y2 || e2_shadow > 0.1 * y2); + } + } + + // Compute spectra for future use. + E_main.Spectrum(optimization_, &output->E2_main); + E_shadow.Spectrum(optimization_, &output->E2_shadow); + + // Update the main filter. + G_main_.Compute(render_buffer, render_signal_analyzer, *output, main_filter_, + aec_state.SaturatedCapture(), &G); + main_filter_.Adapt(render_buffer, G); + data_dumper_->DumpRaw("aec3_subtractor_G_main", G.re); + data_dumper_->DumpRaw("aec3_subtractor_G_main", G.im); + + // Update the shadow filter. + G_shadow_.Compute(render_buffer, render_signal_analyzer, E_shadow, + shadow_filter_.SizePartitions(), + aec_state.SaturatedCapture(), &G); + shadow_filter_.Adapt(render_buffer, G); + + data_dumper_->DumpRaw("aec3_subtractor_G_shadow", G.re); + data_dumper_->DumpRaw("aec3_subtractor_G_shadow", G.im); + + main_filter_.DumpFilter("aec3_subtractor_H_main"); + shadow_filter_.DumpFilter("aec3_subtractor_H_shadow"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.h new file mode 100644 index 0000000000..fe7928ea8a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_H_ + +#include +#include +#include + +#include "modules/audio_processing/aec3/adaptive_fir_filter.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/echo_path_variability.h" +#include "modules/audio_processing/aec3/main_filter_update_gain.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/shadow_filter_update_gain.h" +#include "modules/audio_processing/aec3/subtractor_output.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/utility/ooura_fft.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Proves linear echo cancellation functionality +class Subtractor { + public: + Subtractor(ApmDataDumper* data_dumper, Aec3Optimization optimization); + ~Subtractor(); + + // Performs the echo subtraction. + void Process(const RenderBuffer& render_buffer, + const rtc::ArrayView capture, + const RenderSignalAnalyzer& render_signal_analyzer, + const AecState& aec_state, + SubtractorOutput* output); + + void HandleEchoPathChange(const EchoPathVariability& echo_path_variability); + + // Returns the block-wise frequency response for the main adaptive filter. + const std::vector>& + FilterFrequencyResponse() const { + return main_filter_.FilterFrequencyResponse(); + } + + // Returns the estimate of the impulse response for the main adaptive filter. + const std::array& + FilterImpulseResponse() const { + return main_filter_.FilterImpulseResponse(); + } + + bool ConvergedFilter() const { return converged_filter_; } + + private: + const Aec3Fft fft_; + ApmDataDumper* data_dumper_; + const Aec3Optimization optimization_; + AdaptiveFirFilter main_filter_; + AdaptiveFirFilter shadow_filter_; + MainFilterUpdateGain G_main_; + ShadowFilterUpdateGain G_shadow_; + bool converged_filter_ = false; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(Subtractor); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_output.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_output.h new file mode 100644 index 0000000000..8655665b35 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_output.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_OUTPUT_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_OUTPUT_H_ + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/fft_data.h" + +namespace webrtc { + +// Stores the values being returned from the echo subtractor. +struct SubtractorOutput { + std::array s_main; + std::array e_main; + std::array e_shadow; + FftData E_main; + std::array E2_main; + std::array E2_shadow; + + void Reset() { + s_main.fill(0.f); + e_main.fill(0.f); + e_shadow.fill(0.f); + E_main.re.fill(0.f); + E_main.im.fill(0.f); + E2_main.fill(0.f); + E2_shadow.fill(0.f); + } +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_OUTPUT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc new file mode 100644 index 0000000000..b10421b428 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/subtractor.h" + +#include +#include +#include + +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/test/echo_canceller_test_tools.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +float RunSubtractorTest(int num_blocks_to_process, + int delay_samples, + bool uncorrelated_inputs, + const std::vector& blocks_with_echo_path_changes) { + ApmDataDumper data_dumper(42); + Subtractor subtractor(&data_dumper, DetectOptimization()); + std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector y(kBlockSize, 0.f); + std::array x_old; + SubtractorOutput output; + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, kAdaptiveFilterLength, + std::vector(1, kAdaptiveFilterLength)); + RenderSignalAnalyzer render_signal_analyzer; + Random random_generator(42U); + Aec3Fft fft; + std::array Y2; + std::array E2_main; + std::array E2_shadow; + AecState aec_state(EchoCanceller3Config{}); + x_old.fill(0.f); + Y2.fill(0.f); + E2_main.fill(0.f); + E2_shadow.fill(0.f); + + DelayBuffer delay_buffer(delay_samples); + for (int k = 0; k < num_blocks_to_process; ++k) { + RandomizeSampleVector(&random_generator, x[0]); + if (uncorrelated_inputs) { + RandomizeSampleVector(&random_generator, y); + } else { + delay_buffer.Delay(x[0], y); + } + render_buffer.Insert(x); + render_signal_analyzer.Update(render_buffer, aec_state.FilterDelay()); + + // Handle echo path changes. + if (std::find(blocks_with_echo_path_changes.begin(), + blocks_with_echo_path_changes.end(), + k) != blocks_with_echo_path_changes.end()) { + subtractor.HandleEchoPathChange(EchoPathVariability(true, true)); + } + subtractor.Process(render_buffer, y, render_signal_analyzer, aec_state, + &output); + + aec_state.HandleEchoPathChange(EchoPathVariability(false, false)); + aec_state.Update(subtractor.FilterFrequencyResponse(), + subtractor.FilterImpulseResponse(), + subtractor.ConvergedFilter(), delay_samples / kBlockSize, + render_buffer, E2_main, Y2, x[0], output.s_main, false); + } + + const float output_power = std::inner_product( + output.e_main.begin(), output.e_main.end(), output.e_main.begin(), 0.f); + const float y_power = std::inner_product(y.begin(), y.end(), y.begin(), 0.f); + if (y_power == 0.f) { + ADD_FAILURE(); + return -1.0; + } + return output_power / y_power; +} + +std::string ProduceDebugText(size_t delay) { + std::ostringstream ss; + ss << "Delay: " << delay; + return ss.str(); +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies that the check for non data dumper works. +TEST(Subtractor, NullDataDumper) { + EXPECT_DEATH(Subtractor(nullptr, DetectOptimization()), ""); +} + +// Verifies the check for null subtractor output. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(Subtractor, DISABLED_NullOutput) { + ApmDataDumper data_dumper(42); + Subtractor subtractor(&data_dumper, DetectOptimization()); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, kAdaptiveFilterLength, + std::vector(1, kAdaptiveFilterLength)); + RenderSignalAnalyzer render_signal_analyzer; + std::vector y(kBlockSize, 0.f); + + EXPECT_DEATH(subtractor.Process(render_buffer, y, render_signal_analyzer, + AecState(EchoCanceller3Config{}), nullptr), + ""); +} + +// Verifies the check for the capture signal size. +TEST(Subtractor, WrongCaptureSize) { + ApmDataDumper data_dumper(42); + Subtractor subtractor(&data_dumper, DetectOptimization()); + RenderBuffer render_buffer(Aec3Optimization::kNone, 3, kAdaptiveFilterLength, + std::vector(1, kAdaptiveFilterLength)); + RenderSignalAnalyzer render_signal_analyzer; + std::vector y(kBlockSize - 1, 0.f); + SubtractorOutput output; + + EXPECT_DEATH(subtractor.Process(render_buffer, y, render_signal_analyzer, + AecState(EchoCanceller3Config{}), &output), + ""); +} + +#endif + +// Verifies that the subtractor is able to converge on correlated data. +TEST(Subtractor, Convergence) { + std::vector blocks_with_echo_path_changes; + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(delay_samples)); + + float echo_to_nearend_power = RunSubtractorTest( + 100, delay_samples, false, blocks_with_echo_path_changes); + EXPECT_GT(0.1f, echo_to_nearend_power); + } +} + +// Verifies that the subtractor does not converge on uncorrelated signals. +TEST(Subtractor, NonConvergenceOnUncorrelatedSignals) { + std::vector blocks_with_echo_path_changes; + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(delay_samples)); + + float echo_to_nearend_power = RunSubtractorTest( + 100, delay_samples, true, blocks_with_echo_path_changes); + EXPECT_NEAR(1.f, echo_to_nearend_power, 0.05); + } +} + +// Verifies that the subtractor is properly reset when there is an echo path +// change. +TEST(Subtractor, EchoPathChangeReset) { + std::vector blocks_with_echo_path_changes; + blocks_with_echo_path_changes.push_back(99); + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(delay_samples)); + + float echo_to_nearend_power = RunSubtractorTest( + 100, delay_samples, false, blocks_with_echo_path_changes); + EXPECT_NEAR(1.f, echo_to_nearend_power, 0.0000001f); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.cc new file mode 100644 index 0000000000..8c92bf5762 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.cc @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/suppression_filter.h" + +#include +#include +#include +#include +#include + +#include "modules/audio_processing/utility/ooura_fft.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { +namespace { + +// Hanning window from Matlab command win = sqrt(hanning(128)). +const float kSqrtHanning[kFftLength] = { + 0.00000000000000f, 0.02454122852291f, 0.04906767432742f, 0.07356456359967f, + 0.09801714032956f, 0.12241067519922f, 0.14673047445536f, 0.17096188876030f, + 0.19509032201613f, 0.21910124015687f, 0.24298017990326f, 0.26671275747490f, + 0.29028467725446f, 0.31368174039889f, 0.33688985339222f, 0.35989503653499f, + 0.38268343236509f, 0.40524131400499f, 0.42755509343028f, 0.44961132965461f, + 0.47139673682600f, 0.49289819222978f, 0.51410274419322f, 0.53499761988710f, + 0.55557023301960f, 0.57580819141785f, 0.59569930449243f, 0.61523159058063f, + 0.63439328416365f, 0.65317284295378f, 0.67155895484702f, 0.68954054473707f, + 0.70710678118655f, 0.72424708295147f, 0.74095112535496f, 0.75720884650648f, + 0.77301045336274f, 0.78834642762661f, 0.80320753148064f, 0.81758481315158f, + 0.83146961230255f, 0.84485356524971f, 0.85772861000027f, 0.87008699110871f, + 0.88192126434835f, 0.89322430119552f, 0.90398929312344f, 0.91420975570353f, + 0.92387953251129f, 0.93299279883474f, 0.94154406518302f, 0.94952818059304f, + 0.95694033573221f, 0.96377606579544f, 0.97003125319454f, 0.97570213003853f, + 0.98078528040323f, 0.98527764238894f, 0.98917650996478f, 0.99247953459871f, + 0.99518472667220f, 0.99729045667869f, 0.99879545620517f, 0.99969881869620f, + 1.00000000000000f, 0.99969881869620f, 0.99879545620517f, 0.99729045667869f, + 0.99518472667220f, 0.99247953459871f, 0.98917650996478f, 0.98527764238894f, + 0.98078528040323f, 0.97570213003853f, 0.97003125319454f, 0.96377606579544f, + 0.95694033573221f, 0.94952818059304f, 0.94154406518302f, 0.93299279883474f, + 0.92387953251129f, 0.91420975570353f, 0.90398929312344f, 0.89322430119552f, + 0.88192126434835f, 0.87008699110871f, 0.85772861000027f, 0.84485356524971f, + 0.83146961230255f, 0.81758481315158f, 0.80320753148064f, 0.78834642762661f, + 0.77301045336274f, 0.75720884650648f, 0.74095112535496f, 0.72424708295147f, + 0.70710678118655f, 0.68954054473707f, 0.67155895484702f, 0.65317284295378f, + 0.63439328416365f, 0.61523159058063f, 0.59569930449243f, 0.57580819141785f, + 0.55557023301960f, 0.53499761988710f, 0.51410274419322f, 0.49289819222978f, + 0.47139673682600f, 0.44961132965461f, 0.42755509343028f, 0.40524131400499f, + 0.38268343236509f, 0.35989503653499f, 0.33688985339222f, 0.31368174039889f, + 0.29028467725446f, 0.26671275747490f, 0.24298017990326f, 0.21910124015687f, + 0.19509032201613f, 0.17096188876030f, 0.14673047445536f, 0.12241067519922f, + 0.09801714032956f, 0.07356456359967f, 0.04906767432742f, 0.02454122852291f}; + +} // namespace + +SuppressionFilter::SuppressionFilter(int sample_rate_hz) + : sample_rate_hz_(sample_rate_hz), + fft_(), + e_output_old_(NumBandsForRate(sample_rate_hz_)) { + RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); + e_input_old_.fill(0.f); + std::for_each(e_output_old_.begin(), e_output_old_.end(), + [](std::array& a) { a.fill(0.f); }); +} + +SuppressionFilter::~SuppressionFilter() = default; + +void SuppressionFilter::ApplyGain( + const FftData& comfort_noise, + const FftData& comfort_noise_high_band, + const std::array& suppression_gain, + float high_bands_gain, + std::vector>* e) { + RTC_DCHECK(e); + RTC_DCHECK_EQ(e->size(), NumBandsForRate(sample_rate_hz_)); + FftData E; + std::array e_extended; + constexpr float kIfftNormalization = 2.f / kFftLength; + + // Analysis filterbank. + std::transform(e_input_old_.begin(), e_input_old_.end(), + std::begin(kSqrtHanning), e_extended.begin(), + std::multiplies()); + std::transform((*e)[0].begin(), (*e)[0].end(), + std::begin(kSqrtHanning) + kFftLengthBy2, + e_extended.begin() + kFftLengthBy2, std::multiplies()); + std::copy((*e)[0].begin(), (*e)[0].end(), e_input_old_.begin()); + fft_.Fft(&e_extended, &E); + + // Apply gain. + std::transform(suppression_gain.begin(), suppression_gain.end(), E.re.begin(), + E.re.begin(), std::multiplies()); + std::transform(suppression_gain.begin(), suppression_gain.end(), E.im.begin(), + E.im.begin(), std::multiplies()); + + // Compute and add the comfort noise. + std::array scaled_comfort_noise; + std::transform(suppression_gain.begin(), suppression_gain.end(), + comfort_noise.re.begin(), scaled_comfort_noise.begin(), + [](float a, float b) { return std::max(1.f - a, 0.f) * b; }); + std::transform(scaled_comfort_noise.begin(), scaled_comfort_noise.end(), + E.re.begin(), E.re.begin(), std::plus()); + std::transform(suppression_gain.begin(), suppression_gain.end(), + comfort_noise.im.begin(), scaled_comfort_noise.begin(), + [](float a, float b) { return std::max(1.f - a, 0.f) * b; }); + std::transform(scaled_comfort_noise.begin(), scaled_comfort_noise.end(), + E.im.begin(), E.im.begin(), std::plus()); + + // Synthesis filterbank. + fft_.Ifft(E, &e_extended); + std::transform(e_output_old_[0].begin(), e_output_old_[0].end(), + std::begin(kSqrtHanning) + kFftLengthBy2, (*e)[0].begin(), + [&](float a, float b) { return kIfftNormalization * a * b; }); + std::transform(e_extended.begin(), e_extended.begin() + kFftLengthBy2, + std::begin(kSqrtHanning), e_extended.begin(), + [&](float a, float b) { return kIfftNormalization * a * b; }); + std::transform((*e)[0].begin(), (*e)[0].end(), e_extended.begin(), + (*e)[0].begin(), std::plus()); + std::for_each((*e)[0].begin(), (*e)[0].end(), [](float& x_k) { + x_k = rtc::SafeClamp(x_k, -32768.f, 32767.f); + }); + std::copy(e_extended.begin() + kFftLengthBy2, e_extended.begin() + kFftLength, + std::begin(e_output_old_[0])); + + if (e->size() > 1) { + // Form time-domain high-band noise. + std::array time_domain_high_band_noise; + std::transform(comfort_noise_high_band.re.begin(), + comfort_noise_high_band.re.end(), E.re.begin(), + [&](float a) { return kIfftNormalization * a; }); + std::transform(comfort_noise_high_band.im.begin(), + comfort_noise_high_band.im.end(), E.im.begin(), + [&](float a) { return kIfftNormalization * a; }); + fft_.Ifft(E, &time_domain_high_band_noise); + + // Scale and apply the noise to the signals. + const float high_bands_noise_scaling = + 0.4f * std::max(1.f - high_bands_gain, 0.f); + + std::transform( + (*e)[1].begin(), (*e)[1].end(), time_domain_high_band_noise.begin(), + (*e)[1].begin(), [&](float a, float b) { + return std::max( + std::min(b * high_bands_noise_scaling + high_bands_gain * a, + 32767.0f), + -32768.0f); + }); + + if (e->size() > 2) { + RTC_DCHECK_EQ(3, e->size()); + std::for_each((*e)[2].begin(), (*e)[2].end(), [&](float& a) { + a = rtc::SafeClamp(a * high_bands_gain, -32768.f, 32767.f); + }); + } + + std::array tmp; + for (size_t k = 1; k < e->size(); ++k) { + std::copy((*e)[k].begin(), (*e)[k].end(), tmp.begin()); + std::copy(e_output_old_[k].begin(), e_output_old_[k].end(), + (*e)[k].begin()); + std::copy(tmp.begin(), tmp.end(), e_output_old_[k].begin()); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.h new file mode 100644 index 0000000000..5f91dea28f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_SUPPRESSION_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_SUPPRESSION_FILTER_H_ + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class SuppressionFilter { + public: + explicit SuppressionFilter(int sample_rate_hz); + ~SuppressionFilter(); + void ApplyGain(const FftData& comfort_noise, + const FftData& comfort_noise_high_bands, + const std::array& suppression_gain, + float high_bands_gain, + std::vector>* e); + + private: + const int sample_rate_hz_; + const OouraFft ooura_fft_; + const Aec3Fft fft_; + std::array e_input_old_; + std::vector> e_output_old_; + RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionFilter); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_SUPPRESSION_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc new file mode 100644 index 0000000000..51b3f91f2a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/suppression_filter.h" + +#include +#include +#include + +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr float kPi = 3.141592f; + +void ProduceSinusoid(int sample_rate_hz, + float sinusoidal_frequency_hz, + size_t* sample_counter, + rtc::ArrayView x) { + // Produce a sinusoid of the specified frequency. + for (size_t k = *sample_counter, j = 0; k < (*sample_counter + kBlockSize); + ++k, ++j) { + x[j] = + 32767.f * sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz); + } + *sample_counter = *sample_counter + kBlockSize; +} + +} // namespace + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for null suppressor output. +TEST(SuppressionFilter, NullOutput) { + FftData cn; + FftData cn_high_bands; + std::array gain; + + EXPECT_DEATH(SuppressionFilter(16000).ApplyGain(cn, cn_high_bands, gain, 1.0f, + nullptr), + ""); +} + +// Verifies the check for allowed sample rate. +TEST(SuppressionFilter, ProperSampleRate) { + EXPECT_DEATH(SuppressionFilter(16001), ""); +} + +#endif + +// Verifies that no comfort noise is added when the gain is 1. +TEST(SuppressionFilter, ComfortNoiseInUnityGain) { + SuppressionFilter filter(48000); + FftData cn; + FftData cn_high_bands; + std::array gain; + + gain.fill(1.f); + cn.re.fill(1.f); + cn.im.fill(1.f); + cn_high_bands.re.fill(1.f); + cn_high_bands.im.fill(1.f); + + std::vector> e(3, std::vector(kBlockSize, 0.f)); + std::vector> e_ref = e; + filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e); + + for (size_t k = 0; k < e.size(); ++k) { + EXPECT_EQ(e_ref[k], e[k]); + } +} + +// Verifies that the suppressor is able to suppress a signal. +TEST(SuppressionFilter, SignalSuppression) { + SuppressionFilter filter(48000); + FftData cn; + FftData cn_high_bands; + std::array gain; + std::vector> e(3, std::vector(kBlockSize, 0.f)); + + gain.fill(1.f); + std::for_each(gain.begin() + 10, gain.end(), [](float& a) { a = 0.f; }); + + cn.re.fill(0.f); + cn.im.fill(0.f); + cn_high_bands.re.fill(0.f); + cn_high_bands.im.fill(0.f); + + size_t sample_counter = 0; + + float e0_input = 0.f; + float e0_output = 0.f; + for (size_t k = 0; k < 100; ++k) { + ProduceSinusoid(16000, 16000 * 40 / kFftLengthBy2 / 2, &sample_counter, + e[0]); + e0_input = + std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_input); + filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e); + e0_output = + std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_output); + } + + EXPECT_LT(e0_output, e0_input / 1000.f); +} + +// Verifies that the suppressor is able to pass through a desired signal while +// applying suppressing for some frequencies. +TEST(SuppressionFilter, SignalTransparency) { + SuppressionFilter filter(48000); + FftData cn; + FftData cn_high_bands; + std::array gain; + std::vector> e(3, std::vector(kBlockSize, 0.f)); + + gain.fill(1.f); + std::for_each(gain.begin() + 30, gain.end(), [](float& a) { a = 0.f; }); + + cn.re.fill(0.f); + cn.im.fill(0.f); + cn_high_bands.re.fill(0.f); + cn_high_bands.im.fill(0.f); + + size_t sample_counter = 0; + + float e0_input = 0.f; + float e0_output = 0.f; + for (size_t k = 0; k < 100; ++k) { + ProduceSinusoid(16000, 16000 * 10 / kFftLengthBy2 / 2, &sample_counter, + e[0]); + e0_input = + std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_input); + filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e); + e0_output = + std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_output); + } + + EXPECT_LT(0.9f * e0_input, e0_output); +} + +// Verifies that the suppressor delay. +TEST(SuppressionFilter, Delay) { + SuppressionFilter filter(48000); + FftData cn; + FftData cn_high_bands; + std::array gain; + std::vector> e(3, std::vector(kBlockSize, 0.f)); + + gain.fill(1.f); + + cn.re.fill(0.f); + cn.im.fill(0.f); + cn_high_bands.re.fill(0.f); + cn_high_bands.im.fill(0.f); + + for (size_t k = 0; k < 100; ++k) { + for (size_t j = 0; j < 3; ++j) { + for (size_t i = 0; i < kBlockSize; ++i) { + e[j][i] = k * kBlockSize + i; + } + } + + filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e); + if (k > 2) { + for (size_t j = 0; j < 2; ++j) { + for (size_t i = 0; i < kBlockSize; ++i) { + EXPECT_NEAR(k * kBlockSize + i - kBlockSize, e[j][i], 0.01); + } + } + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.cc new file mode 100644 index 0000000000..ae4d4ace23 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.cc @@ -0,0 +1,426 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/suppression_gain.h" + +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include +#include +#include +#include + +#include "modules/audio_processing/aec3/vector_math.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Reduce gain to avoid narrow band echo leakage. +void NarrowBandAttenuation(int narrow_bin, + std::array* gain) { + const int upper_bin = + std::min(narrow_bin + 6, static_cast(kFftLengthBy2Plus1 - 1)); + for (int k = std::max(0, narrow_bin - 6); k <= upper_bin; ++k) { + (*gain)[k] = std::min((*gain)[k], 0.001f); + } +} + +// Adjust the gains according to the presence of known external filters. +void AdjustForExternalFilters(std::array* gain) { + // Limit the low frequency gains to avoid the impact of the high-pass filter + // on the lower-frequency gain influencing the overall achieved gain. + (*gain)[0] = (*gain)[1] = std::min((*gain)[1], (*gain)[2]); + + // Limit the high frequency gains to avoid the impact of the anti-aliasing + // filter on the upper-frequency gains influencing the overall achieved + // gain. TODO(peah): Update this when new anti-aliasing filters are + // implemented. + constexpr size_t kAntiAliasingImpactLimit = (64 * 2000) / 8000; + const float min_upper_gain = (*gain)[kAntiAliasingImpactLimit]; + std::for_each( + gain->begin() + kAntiAliasingImpactLimit, gain->end() - 1, + [min_upper_gain](float& a) { a = std::min(a, min_upper_gain); }); + (*gain)[kFftLengthBy2] = (*gain)[kFftLengthBy2Minus1]; +} + +// Computes the gain to apply for the bands beyond the first band. +float UpperBandsGain( + const rtc::Optional& narrow_peak_band, + bool saturated_echo, + const std::vector>& render, + const std::array& low_band_gain) { + RTC_DCHECK_LT(0, render.size()); + if (render.size() == 1) { + return 1.f; + } + + if (narrow_peak_band && + (*narrow_peak_band > static_cast(kFftLengthBy2Plus1 - 10))) { + return 0.001f; + } + + constexpr size_t kLowBandGainLimit = kFftLengthBy2 / 2; + const float gain_below_8_khz = *std::min_element( + low_band_gain.begin() + kLowBandGainLimit, low_band_gain.end()); + + // Always attenuate the upper bands when there is saturated echo. + if (saturated_echo) { + return std::min(0.001f, gain_below_8_khz); + } + + // Compute the upper and lower band energies. + const auto sum_of_squares = [](float a, float b) { return a + b * b; }; + const float low_band_energy = + std::accumulate(render[0].begin(), render[0].end(), 0.f, sum_of_squares); + float high_band_energy = 0.f; + for (size_t k = 1; k < render.size(); ++k) { + const float energy = std::accumulate(render[k].begin(), render[k].end(), + 0.f, sum_of_squares); + high_band_energy = std::max(high_band_energy, energy); + } + + // If there is more power in the lower frequencies than the upper frequencies, + // or if the power in upper frequencies is low, do not bound the gain in the + // upper bands. + float anti_howling_gain; + constexpr float kThreshold = kBlockSize * 10.f * 10.f / 4.f; + if (high_band_energy < std::max(low_band_energy, kThreshold)) { + anti_howling_gain = 1.f; + } else { + // In all other cases, bound the gain for upper frequencies. + RTC_DCHECK_LE(low_band_energy, high_band_energy); + RTC_DCHECK_NE(0.f, high_band_energy); + anti_howling_gain = 0.01f * sqrtf(low_band_energy / high_band_energy); + } + + // Choose the gain as the minimum of the lower and upper gains. + return std::min(gain_below_8_khz, anti_howling_gain); +} + +// Limits the gain increase. +void UpdateMaxGainIncrease( + const EchoCanceller3Config& config, + size_t no_saturation_counter, + bool low_noise_render, + bool linear_echo_estimate, + const std::array& last_echo, + const std::array& echo, + const std::array& last_gain, + const std::array& new_gain, + std::array* gain_increase) { + float max_increasing; + float max_decreasing; + float rate_increasing; + float rate_decreasing; + float min_increasing; + float min_decreasing; + + auto& param = config.gain_updates; + if (linear_echo_estimate) { + max_increasing = param.nonlinear.max_inc; + max_decreasing = param.nonlinear.max_dec; + rate_increasing = param.nonlinear.rate_inc; + rate_decreasing = param.nonlinear.rate_dec; + min_increasing = param.nonlinear.min_inc; + min_decreasing = param.nonlinear.min_dec; + } else if (low_noise_render) { + max_increasing = param.low_noise.max_inc; + max_decreasing = param.low_noise.max_dec; + rate_increasing = param.low_noise.rate_inc; + rate_decreasing = param.low_noise.rate_dec; + min_increasing = param.low_noise.min_inc; + min_decreasing = param.low_noise.min_dec; + } else if (no_saturation_counter > 10) { + max_increasing = param.normal.max_inc; + max_decreasing = param.normal.max_dec; + rate_increasing = param.normal.rate_inc; + rate_decreasing = param.normal.rate_dec; + min_increasing = param.normal.min_inc; + min_decreasing = param.normal.min_dec; + } else { + max_increasing = param.saturation.max_inc; + max_decreasing = param.saturation.max_dec; + rate_increasing = param.saturation.rate_inc; + rate_decreasing = param.saturation.rate_dec; + min_increasing = param.saturation.min_inc; + min_decreasing = param.saturation.min_dec; + } + + for (size_t k = 0; k < new_gain.size(); ++k) { + if (echo[k] > last_echo[k]) { + (*gain_increase)[k] = + new_gain[k] > last_gain[k] + ? std::min(max_increasing, (*gain_increase)[k] * rate_increasing) + : min_increasing; + } else { + (*gain_increase)[k] = + new_gain[k] > last_gain[k] + ? std::min(max_decreasing, (*gain_increase)[k] * rate_decreasing) + : min_decreasing; + } + } +} + +// Computes the gain to reduce the echo to a non audible level. +void GainToNoAudibleEcho( + const EchoCanceller3Config& config, + bool low_noise_render, + bool saturated_echo, + bool saturating_echo_path, + bool linear_echo_estimate, + const std::array& nearend, + const std::array& echo, + const std::array& masker, + const std::array& min_gain, + const std::array& max_gain, + const std::array& one_by_echo, + std::array* gain) { + float nearend_masking_margin = 0.f; + if (linear_echo_estimate) { + nearend_masking_margin = + low_noise_render + ? config.gain_mask.m9 + : (saturated_echo ? config.gain_mask.m2 : config.gain_mask.m3); + } else { + nearend_masking_margin = config.gain_mask.m7; + } + + RTC_DCHECK_LE(0.f, nearend_masking_margin); + RTC_DCHECK_GT(1.f, nearend_masking_margin); + const float one_by_one_minus_nearend_masking_margin = + 1.f / (1.0f - nearend_masking_margin); + + const float masker_margin = + linear_echo_estimate ? config.gain_mask.m1 : config.gain_mask.m8; + + for (size_t k = 0; k < gain->size(); ++k) { + const float unity_gain_masker = std::max(nearend[k], masker[k]); + RTC_DCHECK_LE(0.f, nearend_masking_margin * unity_gain_masker); + if (echo[k] <= nearend_masking_margin * unity_gain_masker || + unity_gain_masker <= 0.f) { + (*gain)[k] = 1.f; + } else { + RTC_DCHECK_LT(0.f, unity_gain_masker); + (*gain)[k] = std::max(0.f, (1.f - 5.f * echo[k] / unity_gain_masker) * + one_by_one_minus_nearend_masking_margin); + (*gain)[k] = + std::max(masker_margin * masker[k] * one_by_echo[k], (*gain)[k]); + } + + (*gain)[k] = std::min(std::max((*gain)[k], min_gain[k]), max_gain[k]); + } +} + +// TODO(peah): Make adaptive to take the actual filter error into account. +constexpr size_t kUpperAccurateBandPlus1 = 29; + +// Computes the signal output power that masks the echo signal. +void MaskingPower(const EchoCanceller3Config& config, + const std::array& nearend, + const std::array& comfort_noise, + const std::array& last_masker, + const std::array& gain, + std::array* masker) { + std::array side_band_masker; + float max_nearend_after_gain = 0.f; + for (size_t k = 0; k < gain.size(); ++k) { + const float nearend_after_gain = nearend[k] * gain[k]; + max_nearend_after_gain = + std::max(max_nearend_after_gain, nearend_after_gain); + side_band_masker[k] = nearend_after_gain + comfort_noise[k]; + (*masker)[k] = comfort_noise[k] + config.gain_mask.m4 * last_masker[k]; + } + + // Apply masking only between lower frequency bands. + RTC_DCHECK_LT(kUpperAccurateBandPlus1, gain.size()); + for (size_t k = 1; k < kUpperAccurateBandPlus1; ++k) { + (*masker)[k] += config.gain_mask.m5 * + (side_band_masker[k - 1] + side_band_masker[k + 1]); + } + + // Add full-band masking as a minimum value for the masker. + const float min_masker = max_nearend_after_gain * config.gain_mask.m6; + std::for_each(masker->begin(), masker->end(), + [min_masker](float& a) { a = std::max(a, min_masker); }); +} + +// Limits the gain in the frequencies for which the adaptive filter has not +// converged. Currently, these frequencies are not hardcoded to the frequencies +// which are typically not excited by speech. +// TODO(peah): Make adaptive to take the actual filter error into account. +void AdjustNonConvergedFrequencies( + std::array* gain) { + constexpr float oneByBandsInSum = + 1 / static_cast(kUpperAccurateBandPlus1 - 20); + const float hf_gain_bound = + std::accumulate(gain->begin() + 20, + gain->begin() + kUpperAccurateBandPlus1, 0.f) * + oneByBandsInSum; + + std::for_each(gain->begin() + kUpperAccurateBandPlus1, gain->end(), + [hf_gain_bound](float& a) { a = std::min(a, hf_gain_bound); }); +} + +} // namespace + +// TODO(peah): Add further optimizations, in particular for the divisions. +void SuppressionGain::LowerBandGain( + bool low_noise_render, + const rtc::Optional& narrow_peak_band, + bool saturated_echo, + bool saturating_echo_path, + bool linear_echo_estimate, + const std::array& nearend, + const std::array& echo, + const std::array& comfort_noise, + std::array* gain) { + // Count the number of blocks since saturation. + no_saturation_counter_ = saturated_echo ? 0 : no_saturation_counter_ + 1; + + // Precompute 1/echo (note that when the echo is zero, the precomputed value + // is never used). + std::array one_by_echo; + std::transform(echo.begin(), echo.end(), one_by_echo.begin(), + [](float a) { return a > 0.f ? 1.f / a : 1.f; }); + + // Compute the minimum gain as the attenuating gain to put the signal just + // above the zero sample values. + std::array min_gain; + const float min_echo_power = + low_noise_render ? config_.echo_audibility.low_render_limit + : config_.echo_audibility.normal_render_limit; + if (no_saturation_counter_ > 10) { + for (size_t k = 0; k < nearend.size(); ++k) { + const float denom = std::min(nearend[k], echo[k]); + min_gain[k] = denom > 0.f ? min_echo_power / denom : 1.f; + min_gain[k] = std::min(min_gain[k], 1.f); + } + } else { + min_gain.fill(0.f); + } + + // Compute the maximum gain by limiting the gain increase from the previous + // gain. + std::array max_gain; + for (size_t k = 0; k < gain->size(); ++k) { + max_gain[k] = std::min(std::max(last_gain_[k] * gain_increase_[k], + config_.gain_updates.floor_first_increase), + 1.f); + } + + // Iteratively compute the gain required to attenuate the echo to a non + // noticeable level. + gain->fill(0.f); + for (int k = 0; k < 2; ++k) { + std::array masker; + MaskingPower(config_, nearend, comfort_noise, last_masker_, *gain, &masker); + GainToNoAudibleEcho(config_, low_noise_render, saturated_echo, + saturating_echo_path, linear_echo_estimate, nearend, + echo, masker, min_gain, max_gain, one_by_echo, gain); + AdjustForExternalFilters(gain); + if (narrow_peak_band) { + NarrowBandAttenuation(*narrow_peak_band, gain); + } + } + + // Adjust the gain for frequencies which have not yet converged. + AdjustNonConvergedFrequencies(gain); + + // Update the allowed maximum gain increase. + UpdateMaxGainIncrease(config_, no_saturation_counter_, low_noise_render, + linear_echo_estimate, last_echo_, echo, last_gain_, + *gain, &gain_increase_); + + // Adjust gain dynamics. + const float gain_bound = + std::max(0.001f, *std::min_element(gain->begin(), gain->end()) * 10000.f); + std::for_each(gain->begin(), gain->end(), + [gain_bound](float& a) { a = std::min(a, gain_bound); }); + + // Store data required for the gain computation of the next block. + std::copy(echo.begin(), echo.end(), last_echo_.begin()); + std::copy(gain->begin(), gain->end(), last_gain_.begin()); + MaskingPower(config_, nearend, comfort_noise, last_masker_, *gain, + &last_masker_); + aec3::VectorMath(optimization_).Sqrt(*gain); +} + +SuppressionGain::SuppressionGain(const EchoCanceller3Config& config, + Aec3Optimization optimization) + : optimization_(optimization), config_(config) { + last_gain_.fill(1.f); + last_masker_.fill(0.f); + gain_increase_.fill(1.f); + last_echo_.fill(0.f); +} + +void SuppressionGain::GetGain( + const std::array& nearend, + const std::array& echo, + const std::array& comfort_noise, + const RenderSignalAnalyzer& render_signal_analyzer, + const AecState& aec_state, + const std::vector>& render, + float* high_bands_gain, + std::array* low_band_gain) { + RTC_DCHECK(high_bands_gain); + RTC_DCHECK(low_band_gain); + + const bool saturated_echo = aec_state.SaturatedEcho(); + const bool saturating_echo_path = aec_state.SaturatingEchoPath(); + const bool force_zero_gain = aec_state.ForcedZeroGain(); + const bool linear_echo_estimate = aec_state.LinearEchoEstimate(); + + if (force_zero_gain) { + last_gain_.fill(0.f); + std::copy(comfort_noise.begin(), comfort_noise.end(), last_masker_.begin()); + low_band_gain->fill(0.f); + gain_increase_.fill(1.f); + *high_bands_gain = 0.f; + return; + } + + bool low_noise_render = low_render_detector_.Detect(render); + + // Compute gain for the lower band. + const rtc::Optional narrow_peak_band = + render_signal_analyzer.NarrowPeakBand(); + LowerBandGain(low_noise_render, narrow_peak_band, saturated_echo, + saturating_echo_path, linear_echo_estimate, nearend, echo, + comfort_noise, low_band_gain); + + // Compute the gain for the upper bands. + *high_bands_gain = + UpperBandsGain(narrow_peak_band, saturated_echo, render, *low_band_gain); +} + +// Detects when the render signal can be considered to have low power and +// consist of stationary noise. +bool SuppressionGain::LowNoiseRenderDetector::Detect( + const std::vector>& render) { + float x2_sum = 0.f; + float x2_max = 0.f; + for (auto x_k : render[0]) { + const float x2 = x_k * x_k; + x2_sum += x2; + x2_max = std::max(x2_max, x2); + } + + constexpr float kThreshold = 50.f * 50.f * 64.f; + const bool low_noise_render = + average_power_ < kThreshold && x2_max < 3 * average_power_; + average_power_ = average_power_ * 0.9f + x2_sum * 0.1f; + return low_noise_render; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.h new file mode 100644 index 0000000000..6f21f71961 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_SUPPRESSION_GAIN_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_SUPPRESSION_GAIN_H_ + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/render_signal_analyzer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class SuppressionGain { + public: + SuppressionGain(const EchoCanceller3Config& config, + Aec3Optimization optimization); + void GetGain(const std::array& nearend, + const std::array& echo, + const std::array& comfort_noise, + const RenderSignalAnalyzer& render_signal_analyzer, + const AecState& aec_state, + const std::vector>& render, + float* high_bands_gain, + std::array* low_band_gain); + + private: + void LowerBandGain(bool stationary_with_low_power, + const rtc::Optional& narrow_peak_band, + bool saturated_echo, + bool saturating_echo_path, + bool linear_echo_estimate, + const std::array& nearend, + const std::array& echo, + const std::array& comfort_noise, + std::array* gain); + + class LowNoiseRenderDetector { + public: + bool Detect(const std::vector>& render); + + private: + float average_power_ = 32768.f * 32768.f; + }; + + const Aec3Optimization optimization_; + std::array last_gain_; + std::array last_masker_; + std::array gain_increase_; + std::array last_echo_; + + LowNoiseRenderDetector low_render_detector_; + size_t no_saturation_counter_ = 0; + const EchoCanceller3Config config_; + RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionGain); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_SUPPRESSION_GAIN_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc new file mode 100644 index 0000000000..9fee6a24a9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/suppression_gain.h" + +#include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/subtractor.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace aec3 { + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies that the check for non-null output gains works. +TEST(SuppressionGain, NullOutputGains) { + std::array E2; + std::array R2; + std::array N2; + E2.fill(0.f); + R2.fill(0.f); + N2.fill(0.f); + float high_bands_gain; + AecState aec_state(EchoCanceller3Config{}); + EXPECT_DEATH(SuppressionGain(EchoCanceller3Config{}, DetectOptimization()) + .GetGain(E2, R2, N2, RenderSignalAnalyzer(), aec_state, + std::vector>( + 3, std::vector(kBlockSize, 0.f)), + &high_bands_gain, nullptr), + ""); +} + +#endif + +// Does a sanity check that the gains are correctly computed. +TEST(SuppressionGain, BasicGainComputation) { + SuppressionGain suppression_gain(EchoCanceller3Config(), + DetectOptimization()); + RenderSignalAnalyzer analyzer; + float high_bands_gain; + std::array E2; + std::array Y2; + std::array R2; + std::array N2; + std::array g; + std::array s; + std::vector> x(1, std::vector(kBlockSize, 0.f)); + AecState aec_state(EchoCanceller3Config{}); + ApmDataDumper data_dumper(42); + Subtractor subtractor(&data_dumper, DetectOptimization()); + RenderBuffer render_buffer( + DetectOptimization(), 1, + std::max(kUnknownDelayRenderWindowSize, kAdaptiveFilterLength), + std::vector(1, kAdaptiveFilterLength)); + + // Verify the functionality for forcing a zero gain. + E2.fill(1000000000.f); + R2.fill(10000000000000.f); + N2.fill(0.f); + s.fill(10.f); + aec_state.Update( + subtractor.FilterFrequencyResponse(), subtractor.FilterImpulseResponse(), + subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, x[0], s, false); + suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x, &high_bands_gain, + &g); + std::for_each(g.begin(), g.end(), [](float a) { EXPECT_FLOAT_EQ(0.f, a); }); + EXPECT_FLOAT_EQ(0.f, high_bands_gain); + + // Ensure that a strong noise is detected to mask any echoes. + E2.fill(10.f); + Y2.fill(10.f); + R2.fill(0.1f); + N2.fill(100.f); + // Ensure that the gain is no longer forced to zero. + for (int k = 0; k <= kNumBlocksPerSecond / 5 + 1; ++k) { + aec_state.Update(subtractor.FilterFrequencyResponse(), + subtractor.FilterImpulseResponse(), + subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, + x[0], s, false); + } + + for (int k = 0; k < 100; ++k) { + aec_state.Update(subtractor.FilterFrequencyResponse(), + subtractor.FilterImpulseResponse(), + subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, + x[0], s, false); + suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x, + &high_bands_gain, &g); + } + std::for_each(g.begin(), g.end(), + [](float a) { EXPECT_NEAR(1.f, a, 0.001); }); + + // Ensure that a strong nearend is detected to mask any echoes. + E2.fill(100.f); + Y2.fill(100.f); + R2.fill(0.1f); + N2.fill(0.f); + for (int k = 0; k < 100; ++k) { + aec_state.Update(subtractor.FilterFrequencyResponse(), + subtractor.FilterImpulseResponse(), + subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, + x[0], s, false); + suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x, + &high_bands_gain, &g); + } + std::for_each(g.begin(), g.end(), + [](float a) { EXPECT_NEAR(1.f, a, 0.001); }); + + // Ensure that a strong echo is suppressed. + E2.fill(1000000000.f); + R2.fill(10000000000000.f); + N2.fill(0.f); + for (int k = 0; k < 10; ++k) { + suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x, + &high_bands_gain, &g); + } + std::for_each(g.begin(), g.end(), + [](float a) { EXPECT_NEAR(0.f, a, 0.001); }); + +} + +} // namespace aec3 +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math.h new file mode 100644 index 0000000000..0672b513f3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math.h @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_VECTOR_MATH_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_VECTOR_MATH_H_ + +#include "typedefs.h" // NOLINT(build/include) +#if defined(WEBRTC_HAS_NEON) +#include +#endif +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif +#include +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace aec3 { + +// Provides optimizations for mathematical operations based on vectors. +class VectorMath { + public: + explicit VectorMath(Aec3Optimization optimization) + : optimization_(optimization) {} + + // Elementwise square root. + void Sqrt(rtc::ArrayView x) { + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: { + const int x_size = static_cast(x.size()); + const int vector_limit = x_size >> 2; + + int j = 0; + for (; j < vector_limit * 4; j += 4) { + __m128 g = _mm_loadu_ps(&x[j]); + g = _mm_sqrt_ps(g); + _mm_storeu_ps(&x[j], g); + } + + for (; j < x_size; ++j) { + x[j] = sqrtf(x[j]); + } + } break; +#endif +#if defined(WEBRTC_HAS_NEON) + case Aec3Optimization::kNeon: { + const int x_size = static_cast(x.size()); + const int vector_limit = x_size >> 2; + + int j = 0; + for (; j < vector_limit * 4; j += 4) { + float32x4_t g = vld1q_f32(&x[j]); +#if !defined(WEBRTC_ARCH_ARM64) + float32x4_t y = vrsqrteq_f32(g); + + // Code to handle sqrt(0). + // If the input to sqrtf() is zero, a zero will be returned. + // If the input to vrsqrteq_f32() is zero, positive infinity is + // returned. + const uint32x4_t vec_p_inf = vdupq_n_u32(0x7F800000); + // check for divide by zero + const uint32x4_t div_by_zero = + vceqq_u32(vec_p_inf, vreinterpretq_u32_f32(y)); + // zero out the positive infinity results + y = vreinterpretq_f32_u32( + vandq_u32(vmvnq_u32(div_by_zero), vreinterpretq_u32_f32(y))); + // from arm documentation + // The Newton-Raphson iteration: + // y[n+1] = y[n] * (3 - d * (y[n] * y[n])) / 2) + // converges to (1/√d) if y0 is the result of VRSQRTE applied to d. + // + // Note: The precision did not improve after 2 iterations. + for (int i = 0; i < 2; i++) { + y = vmulq_f32(vrsqrtsq_f32(vmulq_f32(y, y), g), y); + } + // sqrt(g) = g * 1/sqrt(g) + g = vmulq_f32(g, y); +#else + g = vsqrtq_f32(g); +#endif + vst1q_f32(&x[j], g); + } + + for (; j < x_size; ++j) { + x[j] = sqrtf(x[j]); + } + } +#endif + break; + default: + std::for_each(x.begin(), x.end(), [](float& a) { a = sqrtf(a); }); + } + } + + // Elementwise vector multiplication z = x * y. + void Multiply(rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView z) { + RTC_DCHECK_EQ(z.size(), x.size()); + RTC_DCHECK_EQ(z.size(), y.size()); + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: { + const int x_size = static_cast(x.size()); + const int vector_limit = x_size >> 2; + + int j = 0; + for (; j < vector_limit * 4; j += 4) { + const __m128 x_j = _mm_loadu_ps(&x[j]); + const __m128 y_j = _mm_loadu_ps(&y[j]); + const __m128 z_j = _mm_mul_ps(x_j, y_j); + _mm_storeu_ps(&z[j], z_j); + } + + for (; j < x_size; ++j) { + z[j] = x[j] * y[j]; + } + } break; +#endif +#if defined(WEBRTC_HAS_NEON) + case Aec3Optimization::kNeon: { + const int x_size = static_cast(x.size()); + const int vector_limit = x_size >> 2; + + int j = 0; + for (; j < vector_limit * 4; j += 4) { + const float32x4_t x_j = vld1q_f32(&x[j]); + const float32x4_t y_j = vld1q_f32(&y[j]); + const float32x4_t z_j = vmulq_f32(x_j, y_j); + vst1q_f32(&z[j], z_j); + } + + for (; j < x_size; ++j) { + z[j] = x[j] * y[j]; + } + } break; +#endif + default: + std::transform(x.begin(), x.end(), y.begin(), z.begin(), + std::multiplies()); + } + } + + // Elementwise vector accumulation z += x. + void Accumulate(rtc::ArrayView x, rtc::ArrayView z) { + RTC_DCHECK_EQ(z.size(), x.size()); + switch (optimization_) { +#if defined(WEBRTC_ARCH_X86_FAMILY) + case Aec3Optimization::kSse2: { + const int x_size = static_cast(x.size()); + const int vector_limit = x_size >> 2; + + int j = 0; + for (; j < vector_limit * 4; j += 4) { + const __m128 x_j = _mm_loadu_ps(&x[j]); + __m128 z_j = _mm_loadu_ps(&z[j]); + z_j = _mm_add_ps(x_j, z_j); + _mm_storeu_ps(&z[j], z_j); + } + + for (; j < x_size; ++j) { + z[j] += x[j]; + } + } break; +#endif +#if defined(WEBRTC_HAS_NEON) + case Aec3Optimization::kNeon: { + const int x_size = static_cast(x.size()); + const int vector_limit = x_size >> 2; + + int j = 0; + for (; j < vector_limit * 4; j += 4) { + const float32x4_t x_j = vld1q_f32(&x[j]); + float32x4_t z_j = vld1q_f32(&z[j]); + z_j = vaddq_f32(z_j, x_j); + vst1q_f32(&z[j], z_j); + } + + for (; j < x_size; ++j) { + z[j] += x[j]; + } + } break; +#endif + default: + std::transform(x.begin(), x.end(), z.begin(), z.begin(), + std::plus()); + } + } + + private: + Aec3Optimization optimization_; +}; + +} // namespace aec3 + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_VECTOR_MATH_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math_unittest.cc new file mode 100644 index 0000000000..6bf60ecb64 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/vector_math_unittest.cc @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/vector_math.h" + +#include + +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#if defined(WEBRTC_HAS_NEON) + +TEST(VectorMath, Sqrt) { + std::array x; + std::array z; + std::array z_neon; + + for (size_t k = 0; k < x.size(); ++k) { + x[k] = (2.f / 3.f) * k; + } + + std::copy(x.begin(), x.end(), z.begin()); + aec3::VectorMath(Aec3Optimization::kNone).Sqrt(z); + std::copy(x.begin(), x.end(), z_neon.begin()); + aec3::VectorMath(Aec3Optimization::kNeon).Sqrt(z_neon); + for (size_t k = 0; k < z.size(); ++k) { + EXPECT_NEAR(z[k], z_neon[k], 0.0001f); + EXPECT_NEAR(sqrtf(x[k]), z_neon[k], 0.0001f); + } +} + +TEST(VectorMath, Multiply) { + std::array x; + std::array y; + std::array z; + std::array z_neon; + + for (size_t k = 0; k < x.size(); ++k) { + x[k] = k; + y[k] = (2.f / 3.f) * k; + } + + aec3::VectorMath(Aec3Optimization::kNone).Multiply(x, y, z); + aec3::VectorMath(Aec3Optimization::kNeon).Multiply(x, y, z_neon); + for (size_t k = 0; k < z.size(); ++k) { + EXPECT_FLOAT_EQ(z[k], z_neon[k]); + EXPECT_FLOAT_EQ(x[k] * y[k], z_neon[k]); + } +} + +TEST(VectorMath, Accumulate) { + std::array x; + std::array z; + std::array z_neon; + + for (size_t k = 0; k < x.size(); ++k) { + x[k] = k; + z[k] = z_neon[k] = 2.f * k; + } + + aec3::VectorMath(Aec3Optimization::kNone).Accumulate(x, z); + aec3::VectorMath(Aec3Optimization::kNeon).Accumulate(x, z_neon); + for (size_t k = 0; k < z.size(); ++k) { + EXPECT_FLOAT_EQ(z[k], z_neon[k]); + EXPECT_FLOAT_EQ(x[k] + 2.f * x[k], z_neon[k]); + } +} +#endif + +#if defined(WEBRTC_ARCH_X86_FAMILY) + +TEST(VectorMath, Sqrt) { + if (WebRtc_GetCPUInfo(kSSE2) != 0) { + std::array x; + std::array z; + std::array z_sse2; + + for (size_t k = 0; k < x.size(); ++k) { + x[k] = (2.f / 3.f) * k; + } + + std::copy(x.begin(), x.end(), z.begin()); + aec3::VectorMath(Aec3Optimization::kNone).Sqrt(z); + std::copy(x.begin(), x.end(), z_sse2.begin()); + aec3::VectorMath(Aec3Optimization::kSse2).Sqrt(z_sse2); + EXPECT_EQ(z, z_sse2); + for (size_t k = 0; k < z.size(); ++k) { + EXPECT_FLOAT_EQ(z[k], z_sse2[k]); + EXPECT_FLOAT_EQ(sqrtf(x[k]), z_sse2[k]); + } + } +} + +TEST(VectorMath, Multiply) { + if (WebRtc_GetCPUInfo(kSSE2) != 0) { + std::array x; + std::array y; + std::array z; + std::array z_sse2; + + for (size_t k = 0; k < x.size(); ++k) { + x[k] = k; + y[k] = (2.f / 3.f) * k; + } + + aec3::VectorMath(Aec3Optimization::kNone).Multiply(x, y, z); + aec3::VectorMath(Aec3Optimization::kSse2).Multiply(x, y, z_sse2); + for (size_t k = 0; k < z.size(); ++k) { + EXPECT_FLOAT_EQ(z[k], z_sse2[k]); + EXPECT_FLOAT_EQ(x[k] * y[k], z_sse2[k]); + } + } +} + +TEST(VectorMath, Accumulate) { + if (WebRtc_GetCPUInfo(kSSE2) != 0) { + std::array x; + std::array z; + std::array z_sse2; + + for (size_t k = 0; k < x.size(); ++k) { + x[k] = k; + z[k] = z_sse2[k] = 2.f * k; + } + + aec3::VectorMath(Aec3Optimization::kNone).Accumulate(x, z); + aec3::VectorMath(Aec3Optimization::kSse2).Accumulate(x, z_sse2); + for (size_t k = 0; k < z.size(); ++k) { + EXPECT_FLOAT_EQ(z[k], z_sse2[k]); + EXPECT_FLOAT_EQ(x[k] + 2.f * x[k], z_sse2[k]); + } + } +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/BUILD.gn b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/BUILD.gn new file mode 100644 index 0000000000..818a9bf27d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/BUILD.gn @@ -0,0 +1,114 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") # This contains def of 'rtc_enable_protobuf' + +rtc_source_set("aec_dump") { + sources = [ + "aec_dump_factory.h", + ] + + public_deps = [ + "..:aec_dump_interface", + ] + + deps = [ + "../../../rtc_base:rtc_base_approved", + ] +} + +rtc_source_set("mock_aec_dump") { + testonly = true + sources = [ + "mock_aec_dump.cc", + "mock_aec_dump.h", + ] + + deps = [ + "..:aec_dump_interface", + ] + public_deps = [ + "../..:module_api", + "../../../test:test_support", + "//testing/gmock", + ] +} + +rtc_source_set("mock_aec_dump_unittests") { + testonly = true + + sources = [ + "aec_dump_integration_test.cc", + ] + + deps = [ + ":mock_aec_dump", + "..:audio_processing", + "../../../rtc_base:rtc_base_approved", + "//testing/gtest", + ] +} + +if (rtc_enable_protobuf) { + rtc_source_set("aec_dump_impl") { + sources = [ + "aec_dump_impl.cc", + "aec_dump_impl.h", + "capture_stream_info.cc", + "capture_stream_info.h", + "write_to_file_task.cc", + "write_to_file_task.h", + ] + + public = [] + + public_deps = [ + ":aec_dump", + "..:aec_dump_interface", + ] + + deps = [ + "../../../modules:module_api", + "../../../rtc_base:protobuf_utils", + "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:rtc_task_queue", + "../../../system_wrappers", + ] + + deps += [ "../:audioproc_debug_proto" ] + } + + rtc_source_set("aec_dump_unittests") { + testonly = true + defines = [] + deps = [ + ":aec_dump_impl", + "..:aec_dump_interface", + "..:audioproc_debug_proto", + "../../../modules:module_api", + "../../../rtc_base:rtc_task_queue", + "../../../test:test_support", + "//testing/gtest", + ] + sources = [ + "aec_dump_unittest.cc", + ] + } +} + +rtc_source_set("null_aec_dump_factory") { + assert_no_deps = [ ":aec_dump_impl" ] + sources = [ + "null_aec_dump_factory.cc", + ] + + public_deps = [ + ":aec_dump", + "..:aec_dump_interface", + ] +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h new file mode 100644 index 0000000000..e3f00f67a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_FACTORY_H_ +#define MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_FACTORY_H_ + +#include +#include + +#include "modules/audio_processing/include/aec_dump.h" +#include "rtc_base/platform_file.h" + +namespace rtc { +class TaskQueue; +} // namespace rtc + +namespace webrtc { + +class AecDumpFactory { + public: + // The |worker_queue| may not be null and must outlive the created + // AecDump instance. |max_log_size_bytes == -1| means the log size + // will be unlimited. |handle| may not be null. The AecDump takes + // responsibility for |handle| and closes it in the destructor. A + // non-null return value indicates that the file has been + // sucessfully opened. + static std::unique_ptr Create(rtc::PlatformFile file, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue); + static std::unique_ptr Create(std::string file_name, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue); + static std::unique_ptr Create(FILE* handle, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_FACTORY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_gn/moz.build new file mode 100644 index 0000000000..a708b3b78f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_gn/moz.build @@ -0,0 +1,179 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +Library("aec_dump_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc new file mode 100644 index 0000000000..594bf85cd9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/aec_dump/aec_dump_impl.h" + +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { + +namespace { +void CopyFromConfigToEvent(const webrtc::InternalAPMConfig& config, + webrtc::audioproc::Config* pb_cfg) { + pb_cfg->set_aec_enabled(config.aec_enabled); + pb_cfg->set_aec_delay_agnostic_enabled(config.aec_delay_agnostic_enabled); + pb_cfg->set_aec_drift_compensation_enabled( + config.aec_drift_compensation_enabled); + pb_cfg->set_aec_extended_filter_enabled(config.aec_extended_filter_enabled); + pb_cfg->set_aec_suppression_level(config.aec_suppression_level); + + pb_cfg->set_aecm_enabled(config.aecm_enabled); + pb_cfg->set_aecm_comfort_noise_enabled(config.aecm_comfort_noise_enabled); + pb_cfg->set_aecm_routing_mode(config.aecm_routing_mode); + + pb_cfg->set_agc_enabled(config.agc_enabled); + pb_cfg->set_agc_mode(config.agc_mode); + pb_cfg->set_agc_limiter_enabled(config.agc_limiter_enabled); + pb_cfg->set_noise_robust_agc_enabled(config.noise_robust_agc_enabled); + + pb_cfg->set_hpf_enabled(config.hpf_enabled); + + pb_cfg->set_ns_enabled(config.ns_enabled); + pb_cfg->set_ns_level(config.ns_level); + + pb_cfg->set_transient_suppression_enabled( + config.transient_suppression_enabled); + pb_cfg->set_intelligibility_enhancer_enabled( + config.intelligibility_enhancer_enabled); + + pb_cfg->set_experiments_description(config.experiments_description); +} + +} // namespace + +AecDumpImpl::AecDumpImpl(std::unique_ptr debug_file, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue) + : debug_file_(std::move(debug_file)), + num_bytes_left_for_log_(max_log_size_bytes), + worker_queue_(worker_queue), + capture_stream_info_(CreateWriteToFileTask()) {} + +AecDumpImpl::~AecDumpImpl() { + // Block until all tasks have finished running. + rtc::Event thread_sync_event(false /* manual_reset */, false); + worker_queue_->PostTask([&thread_sync_event] { thread_sync_event.Set(); }); + // Wait until the event has been signaled with .Set(). By then all + // pending tasks will have finished. + thread_sync_event.Wait(rtc::Event::kForever); +} + +void AecDumpImpl::WriteInitMessage( + const InternalAPMStreamsConfig& streams_config) { + auto task = CreateWriteToFileTask(); + auto* event = task->GetEvent(); + event->set_type(audioproc::Event::INIT); + audioproc::Init* msg = event->mutable_init(); + + msg->set_sample_rate(streams_config.input_sample_rate); + msg->set_output_sample_rate(streams_config.output_sample_rate); + msg->set_reverse_sample_rate(streams_config.render_input_sample_rate); + msg->set_reverse_output_sample_rate(streams_config.render_output_sample_rate); + + msg->set_num_input_channels( + static_cast(streams_config.input_num_channels)); + msg->set_num_output_channels( + static_cast(streams_config.output_num_channels)); + msg->set_num_reverse_channels( + static_cast(streams_config.render_input_num_channels)); + msg->set_num_reverse_output_channels( + streams_config.render_output_num_channels); + + worker_queue_->PostTask(std::unique_ptr(std::move(task))); +} + +void AecDumpImpl::AddCaptureStreamInput(const FloatAudioFrame& src) { + capture_stream_info_.AddInput(src); +} + +void AecDumpImpl::AddCaptureStreamOutput(const FloatAudioFrame& src) { + capture_stream_info_.AddOutput(src); +} + +void AecDumpImpl::AddCaptureStreamInput(const AudioFrame& frame) { + capture_stream_info_.AddInput(frame); +} + +void AecDumpImpl::AddCaptureStreamOutput(const AudioFrame& frame) { + capture_stream_info_.AddOutput(frame); +} + +void AecDumpImpl::AddAudioProcessingState(const AudioProcessingState& state) { + capture_stream_info_.AddAudioProcessingState(state); +} + +void AecDumpImpl::WriteCaptureStreamMessage() { + auto task = capture_stream_info_.GetTask(); + RTC_DCHECK(task); + worker_queue_->PostTask(std::unique_ptr(std::move(task))); + capture_stream_info_.SetTask(CreateWriteToFileTask()); +} + +void AecDumpImpl::WriteRenderStreamMessage(const AudioFrame& frame) { + auto task = CreateWriteToFileTask(); + auto* event = task->GetEvent(); + + event->set_type(audioproc::Event::REVERSE_STREAM); + audioproc::ReverseStream* msg = event->mutable_reverse_stream(); + const size_t data_size = + sizeof(int16_t) * frame.samples_per_channel_ * frame.num_channels_; + msg->set_data(frame.data(), data_size); + + worker_queue_->PostTask(std::unique_ptr(std::move(task))); +} + +void AecDumpImpl::WriteRenderStreamMessage(const FloatAudioFrame& src) { + auto task = CreateWriteToFileTask(); + auto* event = task->GetEvent(); + + event->set_type(audioproc::Event::REVERSE_STREAM); + + audioproc::ReverseStream* msg = event->mutable_reverse_stream(); + + for (size_t i = 0; i < src.num_channels(); ++i) { + const auto& channel_view = src.channel(i); + msg->add_channel(channel_view.begin(), sizeof(float) * channel_view.size()); + } + + worker_queue_->PostTask(std::unique_ptr(std::move(task))); +} + +void AecDumpImpl::WriteConfig(const InternalAPMConfig& config) { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + auto task = CreateWriteToFileTask(); + auto* event = task->GetEvent(); + event->set_type(audioproc::Event::CONFIG); + CopyFromConfigToEvent(config, event->mutable_config()); + worker_queue_->PostTask(std::unique_ptr(std::move(task))); +} + +std::unique_ptr AecDumpImpl::CreateWriteToFileTask() { + return rtc::MakeUnique(debug_file_.get(), + &num_bytes_left_for_log_); +} + +std::unique_ptr AecDumpFactory::Create(rtc::PlatformFile file, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue) { + RTC_DCHECK(worker_queue); + std::unique_ptr debug_file(FileWrapper::Create()); + FILE* handle = rtc::FdopenPlatformFileForWriting(file); + if (!handle) { + return nullptr; + } + if (!debug_file->OpenFromFileHandle(handle)) { + return nullptr; + } + return rtc::MakeUnique(std::move(debug_file), max_log_size_bytes, + worker_queue); +} + +std::unique_ptr AecDumpFactory::Create(std::string file_name, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue) { + RTC_DCHECK(worker_queue); + std::unique_ptr debug_file(FileWrapper::Create()); + if (!debug_file->OpenFile(file_name.c_str(), false)) { + return nullptr; + } + return rtc::MakeUnique(std::move(debug_file), max_log_size_bytes, + worker_queue); +} + +std::unique_ptr AecDumpFactory::Create(FILE* handle, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue) { + RTC_DCHECK(worker_queue); + RTC_DCHECK(handle); + std::unique_ptr debug_file(FileWrapper::Create()); + if (!debug_file->OpenFromFileHandle(handle)) { + return nullptr; + } + return rtc::MakeUnique(std::move(debug_file), max_log_size_bytes, + worker_queue); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.h new file mode 100644 index 0000000000..5be876b6b7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_impl.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_IMPL_H_ + +#include +#include +#include + +#include "modules/audio_processing/aec_dump/capture_stream_info.h" +#include "modules/audio_processing/aec_dump/write_to_file_task.h" +#include "modules/audio_processing/include/aec_dump.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/platform_file.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/file_wrapper.h" + +// Files generated at build-time by the protobuf compiler. +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" +#else +#include "modules/audio_processing/debug.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() + +namespace rtc { +class TaskQueue; +} // namespace rtc + +namespace webrtc { + +// Task-queue based implementation of AecDump. It is thread safe by +// relying on locks in TaskQueue. +class AecDumpImpl : public AecDump { + public: + // Does member variables initialization shared across all c-tors. + AecDumpImpl(std::unique_ptr debug_file, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue); + + ~AecDumpImpl() override; + + void WriteInitMessage(const InternalAPMStreamsConfig& api_format) override; + + void AddCaptureStreamInput(const FloatAudioFrame& src) override; + void AddCaptureStreamOutput(const FloatAudioFrame& src) override; + void AddCaptureStreamInput(const AudioFrame& frame) override; + void AddCaptureStreamOutput(const AudioFrame& frame) override; + void AddAudioProcessingState(const AudioProcessingState& state) override; + void WriteCaptureStreamMessage() override; + + void WriteRenderStreamMessage(const AudioFrame& frame) override; + void WriteRenderStreamMessage(const FloatAudioFrame& src) override; + + void WriteConfig(const InternalAPMConfig& config) override; + + private: + std::unique_ptr CreateWriteToFileTask(); + + std::unique_ptr debug_file_; + int64_t num_bytes_left_for_log_ = 0; + rtc::RaceChecker race_checker_; + rtc::TaskQueue* worker_queue_; + CaptureStreamInfo capture_stream_info_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_integration_test.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_integration_test.cc new file mode 100644 index 0000000000..720e4f5c34 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_integration_test.cc @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/aec_dump/mock_aec_dump.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/ptr_util.h" + +using testing::_; +using testing::AtLeast; +using testing::Exactly; +using testing::Matcher; +using testing::StrictMock; + +namespace { +std::unique_ptr CreateAudioProcessing() { + webrtc::Config config; + std::unique_ptr apm( + webrtc::AudioProcessing::Create(config)); + RTC_DCHECK(apm); + return apm; +} + +std::unique_ptr CreateMockAecDump() { + auto mock_aec_dump = + rtc::MakeUnique>(); + EXPECT_CALL(*mock_aec_dump.get(), WriteConfig(_)).Times(AtLeast(1)); + EXPECT_CALL(*mock_aec_dump.get(), WriteInitMessage(_)).Times(AtLeast(1)); + return std::unique_ptr(std::move(mock_aec_dump)); +} + +std::unique_ptr CreateFakeFrame() { + auto fake_frame = rtc::MakeUnique(); + fake_frame->num_channels_ = 1; + fake_frame->sample_rate_hz_ = 48000; + fake_frame->samples_per_channel_ = 480; + return fake_frame; +} + +} // namespace + +TEST(AecDumpIntegration, ConfigurationAndInitShouldBeLogged) { + auto apm = CreateAudioProcessing(); + + apm->AttachAecDump(CreateMockAecDump()); +} + +TEST(AecDumpIntegration, + RenderStreamShouldBeLoggedOnceEveryProcessReverseStream) { + auto apm = CreateAudioProcessing(); + auto mock_aec_dump = CreateMockAecDump(); + auto fake_frame = CreateFakeFrame(); + + EXPECT_CALL(*mock_aec_dump.get(), + WriteRenderStreamMessage(Matcher(_))) + .Times(Exactly(1)); + + apm->AttachAecDump(std::move(mock_aec_dump)); + apm->ProcessReverseStream(fake_frame.get()); +} + +TEST(AecDumpIntegration, CaptureStreamShouldBeLoggedOnceEveryProcessStream) { + auto apm = CreateAudioProcessing(); + auto mock_aec_dump = CreateMockAecDump(); + auto fake_frame = CreateFakeFrame(); + + EXPECT_CALL(*mock_aec_dump.get(), + AddCaptureStreamInput(Matcher(_))) + .Times(AtLeast(1)); + + EXPECT_CALL(*mock_aec_dump.get(), + AddCaptureStreamOutput(Matcher(_))) + .Times(Exactly(1)); + + EXPECT_CALL(*mock_aec_dump.get(), AddAudioProcessingState(_)) + .Times(Exactly(1)); + + EXPECT_CALL(*mock_aec_dump.get(), WriteCaptureStreamMessage()) + .Times(Exactly(1)); + + apm->AttachAecDump(std::move(mock_aec_dump)); + apm->ProcessStream(fake_frame.get()); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc new file mode 100644 index 0000000000..965ac03bd7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" + +#include "modules/include/module_common_types.h" +#include "rtc_base/task_queue.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +TEST(AecDumper, APICallsDoNotCrash) { + // Note order of initialization: Task queue has to be initialized + // before AecDump. + rtc::TaskQueue file_writer_queue("file_writer_queue"); + + const std::string filename = + webrtc::test::TempFilename(webrtc::test::OutputPath(), "aec_dump"); + + { + std::unique_ptr aec_dump = + webrtc::AecDumpFactory::Create(filename, -1, &file_writer_queue); + + const webrtc::AudioFrame frame; + aec_dump->WriteRenderStreamMessage(frame); + + aec_dump->AddCaptureStreamInput(frame); + aec_dump->AddCaptureStreamOutput(frame); + + aec_dump->WriteCaptureStreamMessage(); + + webrtc::InternalAPMConfig apm_config; + aec_dump->WriteConfig(apm_config); + + webrtc::InternalAPMStreamsConfig streams_config; + aec_dump->WriteInitMessage(streams_config); + } + // Remove file after the AecDump d-tor has finished. + ASSERT_EQ(0, remove(filename.c_str())); +} + +TEST(AecDumper, WriteToFile) { + rtc::TaskQueue file_writer_queue("file_writer_queue"); + + const std::string filename = + webrtc::test::TempFilename(webrtc::test::OutputPath(), "aec_dump"); + + { + std::unique_ptr aec_dump = + webrtc::AecDumpFactory::Create(filename, -1, &file_writer_queue); + const webrtc::AudioFrame frame; + aec_dump->WriteRenderStreamMessage(frame); + } + + // Verify the file has been written after the AecDump d-tor has + // finished. + FILE* fid = fopen(filename.c_str(), "r"); + ASSERT_TRUE(fid != NULL); + + // Clean it up. + ASSERT_EQ(0, fclose(fid)); + ASSERT_EQ(0, remove(filename.c_str())); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.cc new file mode 100644 index 0000000000..e3284d8822 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.cc @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec_dump/capture_stream_info.h" + +namespace webrtc { +CaptureStreamInfo::CaptureStreamInfo(std::unique_ptr task) + : task_(std::move(task)) { + RTC_DCHECK(task_); + task_->GetEvent()->set_type(audioproc::Event::STREAM); +} + +CaptureStreamInfo::~CaptureStreamInfo() = default; + +void CaptureStreamInfo::AddInput(const FloatAudioFrame& src) { + RTC_DCHECK(task_); + auto* stream = task_->GetEvent()->mutable_stream(); + + for (size_t i = 0; i < src.num_channels(); ++i) { + const auto& channel_view = src.channel(i); + stream->add_input_channel(channel_view.begin(), + sizeof(float) * channel_view.size()); + } +} + +void CaptureStreamInfo::AddOutput(const FloatAudioFrame& src) { + RTC_DCHECK(task_); + auto* stream = task_->GetEvent()->mutable_stream(); + + for (size_t i = 0; i < src.num_channels(); ++i) { + const auto& channel_view = src.channel(i); + stream->add_output_channel(channel_view.begin(), + sizeof(float) * channel_view.size()); + } +} + +void CaptureStreamInfo::AddInput(const AudioFrame& frame) { + RTC_DCHECK(task_); + auto* stream = task_->GetEvent()->mutable_stream(); + const size_t data_size = + sizeof(int16_t) * frame.samples_per_channel_ * frame.num_channels_; + stream->set_input_data(frame.data(), data_size); +} + +void CaptureStreamInfo::AddOutput(const AudioFrame& frame) { + RTC_DCHECK(task_); + auto* stream = task_->GetEvent()->mutable_stream(); + const size_t data_size = + sizeof(int16_t) * frame.samples_per_channel_ * frame.num_channels_; + stream->set_output_data(frame.data(), data_size); +} + +void CaptureStreamInfo::AddAudioProcessingState( + const AecDump::AudioProcessingState& state) { + RTC_DCHECK(task_); + auto* stream = task_->GetEvent()->mutable_stream(); + stream->set_delay(state.delay); + stream->set_drift(state.drift); + stream->set_level(state.level); + stream->set_keypress(state.keypress); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.h new file mode 100644 index 0000000000..9999c3fbd0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/capture_stream_info.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_CAPTURE_STREAM_INFO_H_ +#define MODULES_AUDIO_PROCESSING_AEC_DUMP_CAPTURE_STREAM_INFO_H_ + +#include +#include +#include + +#include "modules/audio_processing/aec_dump/write_to_file_task.h" +#include "modules/audio_processing/include/aec_dump.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/logging.h" + +// Files generated at build-time by the protobuf compiler. +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" +#else +#include "modules/audio_processing/debug.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() + +namespace webrtc { + +class CaptureStreamInfo { + public: + explicit CaptureStreamInfo(std::unique_ptr task); + ~CaptureStreamInfo(); + void AddInput(const FloatAudioFrame& src); + void AddOutput(const FloatAudioFrame& src); + + void AddInput(const AudioFrame& frame); + void AddOutput(const AudioFrame& frame); + + void AddAudioProcessingState(const AecDump::AudioProcessingState& state); + + std::unique_ptr GetTask() { + RTC_DCHECK(task_); + return std::move(task_); + } + + void SetTask(std::unique_ptr task) { + RTC_DCHECK(!task_); + RTC_DCHECK(task); + task_ = std::move(task); + task_->GetEvent()->set_type(audioproc::Event::STREAM); + } + + private: + std::unique_ptr task_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_DUMP_CAPTURE_STREAM_INFO_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.cc new file mode 100644 index 0000000000..aa89e45fe8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.cc @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec_dump/mock_aec_dump.h" + +namespace webrtc { + +namespace test { + +MockAecDump::MockAecDump() = default; +MockAecDump::~MockAecDump() = default; +} +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h new file mode 100644 index 0000000000..6df6f2849c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_MOCK_AEC_DUMP_H_ +#define MODULES_AUDIO_PROCESSING_AEC_DUMP_MOCK_AEC_DUMP_H_ + +#include + +#include "modules/audio_processing/include/aec_dump.h" +#include "modules/include/module_common_types.h" +#include "test/gmock.h" + +namespace webrtc { + +namespace test { + +class MockAecDump : public AecDump { + public: + MockAecDump(); + virtual ~MockAecDump(); + + MOCK_METHOD1(WriteInitMessage, + void(const InternalAPMStreamsConfig& streams_config)); + + MOCK_METHOD1(AddCaptureStreamInput, void(const FloatAudioFrame& src)); + MOCK_METHOD1(AddCaptureStreamOutput, void(const FloatAudioFrame& src)); + MOCK_METHOD1(AddCaptureStreamInput, void(const AudioFrame& frame)); + MOCK_METHOD1(AddCaptureStreamOutput, void(const AudioFrame& frame)); + MOCK_METHOD1(AddAudioProcessingState, + void(const AudioProcessingState& state)); + MOCK_METHOD0(WriteCaptureStreamMessage, void()); + + MOCK_METHOD1(WriteRenderStreamMessage, void(const AudioFrame& frame)); + MOCK_METHOD1(WriteRenderStreamMessage, void(const FloatAudioFrame& src)); + + MOCK_METHOD1(WriteConfig, void(const InternalAPMConfig& config)); +}; + +} // namespace test + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_DUMP_MOCK_AEC_DUMP_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc new file mode 100644 index 0000000000..5623e24aa5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "modules/audio_processing/include/aec_dump.h" + +namespace webrtc { + +std::unique_ptr AecDumpFactory::Create(rtc::PlatformFile file, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue) { + return nullptr; +} + +std::unique_ptr AecDumpFactory::Create(std::string file_name, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue) { + return nullptr; +} + +std::unique_ptr AecDumpFactory::Create(FILE* handle, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue) { + return nullptr; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory_gn/moz.build new file mode 100644 index 0000000000..610edcdef5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("null_aec_dump_factory_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.cc new file mode 100644 index 0000000000..5fdfd4ec82 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.cc @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec_dump/write_to_file_task.h" + +#include "rtc_base/protobuf_utils.h" + +namespace webrtc { + +WriteToFileTask::WriteToFileTask(webrtc::FileWrapper* debug_file, + int64_t* num_bytes_left_for_log) + : debug_file_(debug_file), + num_bytes_left_for_log_(num_bytes_left_for_log) {} + +WriteToFileTask::~WriteToFileTask() = default; + +audioproc::Event* WriteToFileTask::GetEvent() { + return &event_; +} + +bool WriteToFileTask::IsRoomForNextEvent(size_t event_byte_size) const { + int64_t next_message_size = event_byte_size + sizeof(int32_t); + return (*num_bytes_left_for_log_ < 0) || + (*num_bytes_left_for_log_ >= next_message_size); +} + +void WriteToFileTask::UpdateBytesLeft(size_t event_byte_size) { + RTC_DCHECK(IsRoomForNextEvent(event_byte_size)); + if (*num_bytes_left_for_log_ >= 0) { + *num_bytes_left_for_log_ -= (sizeof(int32_t) + event_byte_size); + } +} + +bool WriteToFileTask::Run() { + if (!debug_file_->is_open()) { + return true; + } + + ProtoString event_string; + event_.SerializeToString(&event_string); + + const size_t event_byte_size = event_.ByteSize(); + + if (!IsRoomForNextEvent(event_byte_size)) { + debug_file_->CloseFile(); + return true; + } + + UpdateBytesLeft(event_byte_size); + + // Write message preceded by its size. + if (!debug_file_->Write(&event_byte_size, sizeof(int32_t))) { + RTC_NOTREACHED(); + } + if (!debug_file_->Write(event_string.data(), event_string.length())) { + RTC_NOTREACHED(); + } + return true; // Delete task from queue at once. +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.h new file mode 100644 index 0000000000..7301473247 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump/write_to_file_task.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_WRITE_TO_FILE_TASK_H_ +#define MODULES_AUDIO_PROCESSING_AEC_DUMP_WRITE_TO_FILE_TASK_H_ + +#include +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/platform_file.h" +#include "rtc_base/task_queue.h" +#include "system_wrappers/include/file_wrapper.h" + +// Files generated at build-time by the protobuf compiler. +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" +#else +#include "modules/audio_processing/debug.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() + +namespace webrtc { + +class WriteToFileTask : public rtc::QueuedTask { + public: + WriteToFileTask(webrtc::FileWrapper* debug_file, + int64_t* num_bytes_left_for_log); + ~WriteToFileTask() override; + + audioproc::Event* GetEvent(); + + private: + bool IsRoomForNextEvent(size_t event_byte_size) const; + + void UpdateBytesLeft(size_t event_byte_size); + + bool Run() override; + + webrtc::FileWrapper* debug_file_; + audioproc::Event event_; + int64_t* num_bytes_left_for_log_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_DUMP_WRITE_TO_FILE_TASK_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump_interface_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump_interface_gn/moz.build new file mode 100644 index 0000000000..0f2de68370 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aec_dump_interface_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("aec_dump_interface_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.cc new file mode 100644 index 0000000000..d69dc1ce76 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.cc @@ -0,0 +1,1232 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aecm/aecm_core.h" + +#include +#include + +extern "C" { +#include "common_audio/ring_buffer.h" +#include "common_audio/signal_processing/include/real_fft.h" +} +#include "modules/audio_processing/aecm/echo_control_mobile.h" +#include "modules/audio_processing/utility/delay_estimator_wrapper.h" +extern "C" { +#include "system_wrappers/include/cpu_features_wrapper.h" +} + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "typedefs.h" // NOLINT(build/include) + +#ifdef AEC_DEBUG +FILE *dfile; +FILE *testfile; +#endif + +const int16_t WebRtcAecm_kCosTable[] = { + 8192, 8190, 8187, 8180, 8172, 8160, 8147, 8130, 8112, + 8091, 8067, 8041, 8012, 7982, 7948, 7912, 7874, 7834, + 7791, 7745, 7697, 7647, 7595, 7540, 7483, 7424, 7362, + 7299, 7233, 7164, 7094, 7021, 6947, 6870, 6791, 6710, + 6627, 6542, 6455, 6366, 6275, 6182, 6087, 5991, 5892, + 5792, 5690, 5586, 5481, 5374, 5265, 5155, 5043, 4930, + 4815, 4698, 4580, 4461, 4341, 4219, 4096, 3971, 3845, + 3719, 3591, 3462, 3331, 3200, 3068, 2935, 2801, 2667, + 2531, 2395, 2258, 2120, 1981, 1842, 1703, 1563, 1422, + 1281, 1140, 998, 856, 713, 571, 428, 285, 142, + 0, -142, -285, -428, -571, -713, -856, -998, -1140, + -1281, -1422, -1563, -1703, -1842, -1981, -2120, -2258, -2395, + -2531, -2667, -2801, -2935, -3068, -3200, -3331, -3462, -3591, + -3719, -3845, -3971, -4095, -4219, -4341, -4461, -4580, -4698, + -4815, -4930, -5043, -5155, -5265, -5374, -5481, -5586, -5690, + -5792, -5892, -5991, -6087, -6182, -6275, -6366, -6455, -6542, + -6627, -6710, -6791, -6870, -6947, -7021, -7094, -7164, -7233, + -7299, -7362, -7424, -7483, -7540, -7595, -7647, -7697, -7745, + -7791, -7834, -7874, -7912, -7948, -7982, -8012, -8041, -8067, + -8091, -8112, -8130, -8147, -8160, -8172, -8180, -8187, -8190, + -8191, -8190, -8187, -8180, -8172, -8160, -8147, -8130, -8112, + -8091, -8067, -8041, -8012, -7982, -7948, -7912, -7874, -7834, + -7791, -7745, -7697, -7647, -7595, -7540, -7483, -7424, -7362, + -7299, -7233, -7164, -7094, -7021, -6947, -6870, -6791, -6710, + -6627, -6542, -6455, -6366, -6275, -6182, -6087, -5991, -5892, + -5792, -5690, -5586, -5481, -5374, -5265, -5155, -5043, -4930, + -4815, -4698, -4580, -4461, -4341, -4219, -4096, -3971, -3845, + -3719, -3591, -3462, -3331, -3200, -3068, -2935, -2801, -2667, + -2531, -2395, -2258, -2120, -1981, -1842, -1703, -1563, -1422, + -1281, -1140, -998, -856, -713, -571, -428, -285, -142, + 0, 142, 285, 428, 571, 713, 856, 998, 1140, + 1281, 1422, 1563, 1703, 1842, 1981, 2120, 2258, 2395, + 2531, 2667, 2801, 2935, 3068, 3200, 3331, 3462, 3591, + 3719, 3845, 3971, 4095, 4219, 4341, 4461, 4580, 4698, + 4815, 4930, 5043, 5155, 5265, 5374, 5481, 5586, 5690, + 5792, 5892, 5991, 6087, 6182, 6275, 6366, 6455, 6542, + 6627, 6710, 6791, 6870, 6947, 7021, 7094, 7164, 7233, + 7299, 7362, 7424, 7483, 7540, 7595, 7647, 7697, 7745, + 7791, 7834, 7874, 7912, 7948, 7982, 8012, 8041, 8067, + 8091, 8112, 8130, 8147, 8160, 8172, 8180, 8187, 8190 +}; + +const int16_t WebRtcAecm_kSinTable[] = { + 0, 142, 285, 428, 571, 713, 856, 998, + 1140, 1281, 1422, 1563, 1703, 1842, 1981, 2120, + 2258, 2395, 2531, 2667, 2801, 2935, 3068, 3200, + 3331, 3462, 3591, 3719, 3845, 3971, 4095, 4219, + 4341, 4461, 4580, 4698, 4815, 4930, 5043, 5155, + 5265, 5374, 5481, 5586, 5690, 5792, 5892, 5991, + 6087, 6182, 6275, 6366, 6455, 6542, 6627, 6710, + 6791, 6870, 6947, 7021, 7094, 7164, 7233, 7299, + 7362, 7424, 7483, 7540, 7595, 7647, 7697, 7745, + 7791, 7834, 7874, 7912, 7948, 7982, 8012, 8041, + 8067, 8091, 8112, 8130, 8147, 8160, 8172, 8180, + 8187, 8190, 8191, 8190, 8187, 8180, 8172, 8160, + 8147, 8130, 8112, 8091, 8067, 8041, 8012, 7982, + 7948, 7912, 7874, 7834, 7791, 7745, 7697, 7647, + 7595, 7540, 7483, 7424, 7362, 7299, 7233, 7164, + 7094, 7021, 6947, 6870, 6791, 6710, 6627, 6542, + 6455, 6366, 6275, 6182, 6087, 5991, 5892, 5792, + 5690, 5586, 5481, 5374, 5265, 5155, 5043, 4930, + 4815, 4698, 4580, 4461, 4341, 4219, 4096, 3971, + 3845, 3719, 3591, 3462, 3331, 3200, 3068, 2935, + 2801, 2667, 2531, 2395, 2258, 2120, 1981, 1842, + 1703, 1563, 1422, 1281, 1140, 998, 856, 713, + 571, 428, 285, 142, 0, -142, -285, -428, + -571, -713, -856, -998, -1140, -1281, -1422, -1563, + -1703, -1842, -1981, -2120, -2258, -2395, -2531, -2667, + -2801, -2935, -3068, -3200, -3331, -3462, -3591, -3719, + -3845, -3971, -4095, -4219, -4341, -4461, -4580, -4698, + -4815, -4930, -5043, -5155, -5265, -5374, -5481, -5586, + -5690, -5792, -5892, -5991, -6087, -6182, -6275, -6366, + -6455, -6542, -6627, -6710, -6791, -6870, -6947, -7021, + -7094, -7164, -7233, -7299, -7362, -7424, -7483, -7540, + -7595, -7647, -7697, -7745, -7791, -7834, -7874, -7912, + -7948, -7982, -8012, -8041, -8067, -8091, -8112, -8130, + -8147, -8160, -8172, -8180, -8187, -8190, -8191, -8190, + -8187, -8180, -8172, -8160, -8147, -8130, -8112, -8091, + -8067, -8041, -8012, -7982, -7948, -7912, -7874, -7834, + -7791, -7745, -7697, -7647, -7595, -7540, -7483, -7424, + -7362, -7299, -7233, -7164, -7094, -7021, -6947, -6870, + -6791, -6710, -6627, -6542, -6455, -6366, -6275, -6182, + -6087, -5991, -5892, -5792, -5690, -5586, -5481, -5374, + -5265, -5155, -5043, -4930, -4815, -4698, -4580, -4461, + -4341, -4219, -4096, -3971, -3845, -3719, -3591, -3462, + -3331, -3200, -3068, -2935, -2801, -2667, -2531, -2395, + -2258, -2120, -1981, -1842, -1703, -1563, -1422, -1281, + -1140, -998, -856, -713, -571, -428, -285, -142 +}; + +// Initialization table for echo channel in 8 kHz +static const int16_t kChannelStored8kHz[PART_LEN1] = { + 2040, 1815, 1590, 1498, 1405, 1395, 1385, 1418, + 1451, 1506, 1562, 1644, 1726, 1804, 1882, 1918, + 1953, 1982, 2010, 2025, 2040, 2034, 2027, 2021, + 2014, 1997, 1980, 1925, 1869, 1800, 1732, 1683, + 1635, 1604, 1572, 1545, 1517, 1481, 1444, 1405, + 1367, 1331, 1294, 1270, 1245, 1239, 1233, 1247, + 1260, 1282, 1303, 1338, 1373, 1407, 1441, 1470, + 1499, 1524, 1549, 1565, 1582, 1601, 1621, 1649, + 1676 +}; + +// Initialization table for echo channel in 16 kHz +static const int16_t kChannelStored16kHz[PART_LEN1] = { + 2040, 1590, 1405, 1385, 1451, 1562, 1726, 1882, + 1953, 2010, 2040, 2027, 2014, 1980, 1869, 1732, + 1635, 1572, 1517, 1444, 1367, 1294, 1245, 1233, + 1260, 1303, 1373, 1441, 1499, 1549, 1582, 1621, + 1676, 1741, 1802, 1861, 1921, 1983, 2040, 2102, + 2170, 2265, 2375, 2515, 2651, 2781, 2922, 3075, + 3253, 3471, 3738, 3976, 4151, 4258, 4308, 4288, + 4270, 4253, 4237, 4179, 4086, 3947, 3757, 3484, + 3153 +}; + +// Moves the pointer to the next entry and inserts |far_spectrum| and +// corresponding Q-domain in its buffer. +// +// Inputs: +// - self : Pointer to the delay estimation instance +// - far_spectrum : Pointer to the far end spectrum +// - far_q : Q-domain of far end spectrum +// +void WebRtcAecm_UpdateFarHistory(AecmCore* self, + uint16_t* far_spectrum, + int far_q) { + // Get new buffer position + self->far_history_pos++; + if (self->far_history_pos >= MAX_DELAY) { + self->far_history_pos = 0; + } + // Update Q-domain buffer + self->far_q_domains[self->far_history_pos] = far_q; + // Update far end spectrum buffer + memcpy(&(self->far_history[self->far_history_pos * PART_LEN1]), + far_spectrum, + sizeof(uint16_t) * PART_LEN1); +} + +// Returns a pointer to the far end spectrum aligned to current near end +// spectrum. The function WebRtc_DelayEstimatorProcessFix(...) should have been +// called before AlignedFarend(...). Otherwise, you get the pointer to the +// previous frame. The memory is only valid until the next call of +// WebRtc_DelayEstimatorProcessFix(...). +// +// Inputs: +// - self : Pointer to the AECM instance. +// - delay : Current delay estimate. +// +// Output: +// - far_q : The Q-domain of the aligned far end spectrum +// +// Return value: +// - far_spectrum : Pointer to the aligned far end spectrum +// NULL - Error +// +const uint16_t* WebRtcAecm_AlignedFarend(AecmCore* self, + int* far_q, + int delay) { + int buffer_position = 0; + RTC_DCHECK(self); + buffer_position = self->far_history_pos - delay; + + // Check buffer position + if (buffer_position < 0) { + buffer_position += MAX_DELAY; + } + // Get Q-domain + *far_q = self->far_q_domains[buffer_position]; + // Return far end spectrum + return &(self->far_history[buffer_position * PART_LEN1]); +} + +// Declare function pointers. +CalcLinearEnergies WebRtcAecm_CalcLinearEnergies; +StoreAdaptiveChannel WebRtcAecm_StoreAdaptiveChannel; +ResetAdaptiveChannel WebRtcAecm_ResetAdaptiveChannel; + +AecmCore* WebRtcAecm_CreateCore() { + AecmCore* aecm = static_cast(malloc(sizeof(AecmCore))); + + aecm->farFrameBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, + sizeof(int16_t)); + if (!aecm->farFrameBuf) + { + WebRtcAecm_FreeCore(aecm); + return NULL; + } + + aecm->nearNoisyFrameBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, + sizeof(int16_t)); + if (!aecm->nearNoisyFrameBuf) + { + WebRtcAecm_FreeCore(aecm); + return NULL; + } + + aecm->nearCleanFrameBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, + sizeof(int16_t)); + if (!aecm->nearCleanFrameBuf) + { + WebRtcAecm_FreeCore(aecm); + return NULL; + } + + aecm->outFrameBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, + sizeof(int16_t)); + if (!aecm->outFrameBuf) + { + WebRtcAecm_FreeCore(aecm); + return NULL; + } + + aecm->delay_estimator_farend = WebRtc_CreateDelayEstimatorFarend(PART_LEN1, + MAX_DELAY); + if (aecm->delay_estimator_farend == NULL) { + WebRtcAecm_FreeCore(aecm); + return NULL; + } + aecm->delay_estimator = + WebRtc_CreateDelayEstimator(aecm->delay_estimator_farend, 0); + if (aecm->delay_estimator == NULL) { + WebRtcAecm_FreeCore(aecm); + return NULL; + } + // TODO(bjornv): Explicitly disable robust delay validation until no + // performance regression has been established. Then remove the line. + WebRtc_enable_robust_validation(aecm->delay_estimator, 0); + + aecm->real_fft = WebRtcSpl_CreateRealFFT(PART_LEN_SHIFT); + if (aecm->real_fft == NULL) { + WebRtcAecm_FreeCore(aecm); + return NULL; + } + + // Init some aecm pointers. 16 and 32 byte alignment is only necessary + // for Neon code currently. + aecm->xBuf = (int16_t*) (((uintptr_t)aecm->xBuf_buf + 31) & ~ 31); + aecm->dBufClean = (int16_t*) (((uintptr_t)aecm->dBufClean_buf + 31) & ~ 31); + aecm->dBufNoisy = (int16_t*) (((uintptr_t)aecm->dBufNoisy_buf + 31) & ~ 31); + aecm->outBuf = (int16_t*) (((uintptr_t)aecm->outBuf_buf + 15) & ~ 15); + aecm->channelStored = (int16_t*) (((uintptr_t) + aecm->channelStored_buf + 15) & ~ 15); + aecm->channelAdapt16 = (int16_t*) (((uintptr_t) + aecm->channelAdapt16_buf + 15) & ~ 15); + aecm->channelAdapt32 = (int32_t*) (((uintptr_t) + aecm->channelAdapt32_buf + 31) & ~ 31); + + return aecm; +} + +void WebRtcAecm_InitEchoPathCore(AecmCore* aecm, const int16_t* echo_path) { + int i = 0; + + // Reset the stored channel + memcpy(aecm->channelStored, echo_path, sizeof(int16_t) * PART_LEN1); + // Reset the adapted channels + memcpy(aecm->channelAdapt16, echo_path, sizeof(int16_t) * PART_LEN1); + for (i = 0; i < PART_LEN1; i++) + { + aecm->channelAdapt32[i] = (int32_t)aecm->channelAdapt16[i] << 16; + } + + // Reset channel storing variables + aecm->mseAdaptOld = 1000; + aecm->mseStoredOld = 1000; + aecm->mseThreshold = WEBRTC_SPL_WORD32_MAX; + aecm->mseChannelCount = 0; +} + +static void CalcLinearEnergiesC(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est, + uint32_t* far_energy, + uint32_t* echo_energy_adapt, + uint32_t* echo_energy_stored) { + int i; + + // Get energy for the delayed far end signal and estimated + // echo using both stored and adapted channels. + for (i = 0; i < PART_LEN1; i++) + { + echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + far_spectrum[i]); + (*far_energy) += (uint32_t)(far_spectrum[i]); + *echo_energy_adapt += aecm->channelAdapt16[i] * far_spectrum[i]; + (*echo_energy_stored) += (uint32_t)echo_est[i]; + } +} + +static void StoreAdaptiveChannelC(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est) { + int i; + + // During startup we store the channel every block. + memcpy(aecm->channelStored, aecm->channelAdapt16, sizeof(int16_t) * PART_LEN1); + // Recalculate echo estimate + for (i = 0; i < PART_LEN; i += 4) + { + echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + far_spectrum[i]); + echo_est[i + 1] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 1], + far_spectrum[i + 1]); + echo_est[i + 2] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 2], + far_spectrum[i + 2]); + echo_est[i + 3] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 3], + far_spectrum[i + 3]); + } + echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + far_spectrum[i]); +} + +static void ResetAdaptiveChannelC(AecmCore* aecm) { + int i; + + // The stored channel has a significantly lower MSE than the adaptive one for + // two consecutive calculations. Reset the adaptive channel. + memcpy(aecm->channelAdapt16, aecm->channelStored, + sizeof(int16_t) * PART_LEN1); + // Restore the W32 channel + for (i = 0; i < PART_LEN; i += 4) + { + aecm->channelAdapt32[i] = (int32_t)aecm->channelStored[i] << 16; + aecm->channelAdapt32[i + 1] = (int32_t)aecm->channelStored[i + 1] << 16; + aecm->channelAdapt32[i + 2] = (int32_t)aecm->channelStored[i + 2] << 16; + aecm->channelAdapt32[i + 3] = (int32_t)aecm->channelStored[i + 3] << 16; + } + aecm->channelAdapt32[i] = (int32_t)aecm->channelStored[i] << 16; +} + +// Initialize function pointers for ARM Neon platform. +#if defined(WEBRTC_HAS_NEON) +static void WebRtcAecm_InitNeon(void) +{ + WebRtcAecm_StoreAdaptiveChannel = WebRtcAecm_StoreAdaptiveChannelNeon; + WebRtcAecm_ResetAdaptiveChannel = WebRtcAecm_ResetAdaptiveChannelNeon; + WebRtcAecm_CalcLinearEnergies = WebRtcAecm_CalcLinearEnergiesNeon; +} +#endif + +// Initialize function pointers for MIPS platform. +#if defined(MIPS32_LE) +static void WebRtcAecm_InitMips(void) +{ +#if defined(MIPS_DSP_R1_LE) + WebRtcAecm_StoreAdaptiveChannel = WebRtcAecm_StoreAdaptiveChannel_mips; + WebRtcAecm_ResetAdaptiveChannel = WebRtcAecm_ResetAdaptiveChannel_mips; +#endif + WebRtcAecm_CalcLinearEnergies = WebRtcAecm_CalcLinearEnergies_mips; +} +#endif + +// WebRtcAecm_InitCore(...) +// +// This function initializes the AECM instant created with WebRtcAecm_CreateCore(...) +// Input: +// - aecm : Pointer to the Echo Suppression instance +// - samplingFreq : Sampling Frequency +// +// Output: +// - aecm : Initialized instance +// +// Return value : 0 - Ok +// -1 - Error +// +int WebRtcAecm_InitCore(AecmCore* const aecm, int samplingFreq) { + int i = 0; + int32_t tmp32 = PART_LEN1 * PART_LEN1; + int16_t tmp16 = PART_LEN1; + + if (samplingFreq != 8000 && samplingFreq != 16000) + { + samplingFreq = 8000; + return -1; + } + // sanity check of sampling frequency + aecm->mult = (int16_t)samplingFreq / 8000; + + aecm->farBufWritePos = 0; + aecm->farBufReadPos = 0; + aecm->knownDelay = 0; + aecm->lastKnownDelay = 0; + + WebRtc_InitBuffer(aecm->farFrameBuf); + WebRtc_InitBuffer(aecm->nearNoisyFrameBuf); + WebRtc_InitBuffer(aecm->nearCleanFrameBuf); + WebRtc_InitBuffer(aecm->outFrameBuf); + + memset(aecm->xBuf_buf, 0, sizeof(aecm->xBuf_buf)); + memset(aecm->dBufClean_buf, 0, sizeof(aecm->dBufClean_buf)); + memset(aecm->dBufNoisy_buf, 0, sizeof(aecm->dBufNoisy_buf)); + memset(aecm->outBuf_buf, 0, sizeof(aecm->outBuf_buf)); + + aecm->seed = 666; + aecm->totCount = 0; + + if (WebRtc_InitDelayEstimatorFarend(aecm->delay_estimator_farend) != 0) { + return -1; + } + if (WebRtc_InitDelayEstimator(aecm->delay_estimator) != 0) { + return -1; + } + // Set far end histories to zero + memset(aecm->far_history, 0, sizeof(uint16_t) * PART_LEN1 * MAX_DELAY); + memset(aecm->far_q_domains, 0, sizeof(int) * MAX_DELAY); + aecm->far_history_pos = MAX_DELAY; + + aecm->nlpFlag = 1; + aecm->fixedDelay = -1; + + aecm->dfaCleanQDomain = 0; + aecm->dfaCleanQDomainOld = 0; + aecm->dfaNoisyQDomain = 0; + aecm->dfaNoisyQDomainOld = 0; + + memset(aecm->nearLogEnergy, 0, sizeof(aecm->nearLogEnergy)); + aecm->farLogEnergy = 0; + memset(aecm->echoAdaptLogEnergy, 0, sizeof(aecm->echoAdaptLogEnergy)); + memset(aecm->echoStoredLogEnergy, 0, sizeof(aecm->echoStoredLogEnergy)); + + // Initialize the echo channels with a stored shape. + if (samplingFreq == 8000) + { + WebRtcAecm_InitEchoPathCore(aecm, kChannelStored8kHz); + } + else + { + WebRtcAecm_InitEchoPathCore(aecm, kChannelStored16kHz); + } + + memset(aecm->echoFilt, 0, sizeof(aecm->echoFilt)); + memset(aecm->nearFilt, 0, sizeof(aecm->nearFilt)); + aecm->noiseEstCtr = 0; + + aecm->cngMode = AecmTrue; + + memset(aecm->noiseEstTooLowCtr, 0, sizeof(aecm->noiseEstTooLowCtr)); + memset(aecm->noiseEstTooHighCtr, 0, sizeof(aecm->noiseEstTooHighCtr)); + // Shape the initial noise level to an approximate pink noise. + for (i = 0; i < (PART_LEN1 >> 1) - 1; i++) + { + aecm->noiseEst[i] = (tmp32 << 8); + tmp16--; + tmp32 -= (int32_t)((tmp16 << 1) + 1); + } + for (; i < PART_LEN1; i++) + { + aecm->noiseEst[i] = (tmp32 << 8); + } + + aecm->farEnergyMin = WEBRTC_SPL_WORD16_MAX; + aecm->farEnergyMax = WEBRTC_SPL_WORD16_MIN; + aecm->farEnergyMaxMin = 0; + aecm->farEnergyVAD = FAR_ENERGY_MIN; // This prevents false speech detection at the + // beginning. + aecm->farEnergyMSE = 0; + aecm->currentVADValue = 0; + aecm->vadUpdateCount = 0; + aecm->firstVAD = 1; + + aecm->startupState = 0; + aecm->supGain = SUPGAIN_DEFAULT; + aecm->supGainOld = SUPGAIN_DEFAULT; + + aecm->supGainErrParamA = SUPGAIN_ERROR_PARAM_A; + aecm->supGainErrParamD = SUPGAIN_ERROR_PARAM_D; + aecm->supGainErrParamDiffAB = SUPGAIN_ERROR_PARAM_A - SUPGAIN_ERROR_PARAM_B; + aecm->supGainErrParamDiffBD = SUPGAIN_ERROR_PARAM_B - SUPGAIN_ERROR_PARAM_D; + + // Assert a preprocessor definition at compile-time. It's an assumption + // used in assembly code, so check the assembly files before any change. + static_assert(PART_LEN % 16 == 0, "PART_LEN is not a multiple of 16"); + + // Initialize function pointers. + WebRtcAecm_CalcLinearEnergies = CalcLinearEnergiesC; + WebRtcAecm_StoreAdaptiveChannel = StoreAdaptiveChannelC; + WebRtcAecm_ResetAdaptiveChannel = ResetAdaptiveChannelC; + +#if defined(WEBRTC_HAS_NEON) + WebRtcAecm_InitNeon(); +#endif + +#if defined(MIPS32_LE) + WebRtcAecm_InitMips(); +#endif + return 0; +} + +// TODO(bjornv): This function is currently not used. Add support for these +// parameters from a higher level +int WebRtcAecm_Control(AecmCore* aecm, int delay, int nlpFlag) { + aecm->nlpFlag = nlpFlag; + aecm->fixedDelay = delay; + + return 0; +} + +void WebRtcAecm_FreeCore(AecmCore* aecm) { + if (aecm == NULL) { + return; + } + + WebRtc_FreeBuffer(aecm->farFrameBuf); + WebRtc_FreeBuffer(aecm->nearNoisyFrameBuf); + WebRtc_FreeBuffer(aecm->nearCleanFrameBuf); + WebRtc_FreeBuffer(aecm->outFrameBuf); + + WebRtc_FreeDelayEstimator(aecm->delay_estimator); + WebRtc_FreeDelayEstimatorFarend(aecm->delay_estimator_farend); + WebRtcSpl_FreeRealFFT(aecm->real_fft); + + free(aecm); +} + +int WebRtcAecm_ProcessFrame(AecmCore* aecm, + const int16_t* farend, + const int16_t* nearendNoisy, + const int16_t* nearendClean, + int16_t* out) { + int16_t outBlock_buf[PART_LEN + 8]; // Align buffer to 8-byte boundary. + int16_t* outBlock = (int16_t*) (((uintptr_t) outBlock_buf + 15) & ~ 15); + + int16_t farFrame[FRAME_LEN]; + const int16_t* out_ptr = NULL; + int size = 0; + + // Buffer the current frame. + // Fetch an older one corresponding to the delay. + WebRtcAecm_BufferFarFrame(aecm, farend, FRAME_LEN); + WebRtcAecm_FetchFarFrame(aecm, farFrame, FRAME_LEN, aecm->knownDelay); + + // Buffer the synchronized far and near frames, + // to pass the smaller blocks individually. + WebRtc_WriteBuffer(aecm->farFrameBuf, farFrame, FRAME_LEN); + WebRtc_WriteBuffer(aecm->nearNoisyFrameBuf, nearendNoisy, FRAME_LEN); + if (nearendClean != NULL) + { + WebRtc_WriteBuffer(aecm->nearCleanFrameBuf, nearendClean, FRAME_LEN); + } + + // Process as many blocks as possible. + while (WebRtc_available_read(aecm->farFrameBuf) >= PART_LEN) + { + int16_t far_block[PART_LEN]; + const int16_t* far_block_ptr = NULL; + int16_t near_noisy_block[PART_LEN]; + const int16_t* near_noisy_block_ptr = NULL; + + WebRtc_ReadBuffer(aecm->farFrameBuf, (void**) &far_block_ptr, far_block, + PART_LEN); + WebRtc_ReadBuffer(aecm->nearNoisyFrameBuf, + (void**) &near_noisy_block_ptr, + near_noisy_block, + PART_LEN); + if (nearendClean != NULL) + { + int16_t near_clean_block[PART_LEN]; + const int16_t* near_clean_block_ptr = NULL; + + WebRtc_ReadBuffer(aecm->nearCleanFrameBuf, + (void**) &near_clean_block_ptr, + near_clean_block, + PART_LEN); + if (WebRtcAecm_ProcessBlock(aecm, + far_block_ptr, + near_noisy_block_ptr, + near_clean_block_ptr, + outBlock) == -1) + { + return -1; + } + } else + { + if (WebRtcAecm_ProcessBlock(aecm, + far_block_ptr, + near_noisy_block_ptr, + NULL, + outBlock) == -1) + { + return -1; + } + } + + WebRtc_WriteBuffer(aecm->outFrameBuf, outBlock, PART_LEN); + } + + // Stuff the out buffer if we have less than a frame to output. + // This should only happen for the first frame. + size = (int) WebRtc_available_read(aecm->outFrameBuf); + if (size < FRAME_LEN) + { + WebRtc_MoveReadPtr(aecm->outFrameBuf, size - FRAME_LEN); + } + + // Obtain an output frame. + WebRtc_ReadBuffer(aecm->outFrameBuf, (void**) &out_ptr, out, FRAME_LEN); + if (out_ptr != out) { + // ReadBuffer() hasn't copied to |out| in this case. + memcpy(out, out_ptr, FRAME_LEN * sizeof(int16_t)); + } + + return 0; +} + +// WebRtcAecm_AsymFilt(...) +// +// Performs asymmetric filtering. +// +// Inputs: +// - filtOld : Previous filtered value. +// - inVal : New input value. +// - stepSizePos : Step size when we have a positive contribution. +// - stepSizeNeg : Step size when we have a negative contribution. +// +// Output: +// +// Return: - Filtered value. +// +int16_t WebRtcAecm_AsymFilt(const int16_t filtOld, const int16_t inVal, + const int16_t stepSizePos, + const int16_t stepSizeNeg) +{ + int16_t retVal; + + if ((filtOld == WEBRTC_SPL_WORD16_MAX) | (filtOld == WEBRTC_SPL_WORD16_MIN)) + { + return inVal; + } + retVal = filtOld; + if (filtOld > inVal) + { + retVal -= (filtOld - inVal) >> stepSizeNeg; + } else + { + retVal += (inVal - filtOld) >> stepSizePos; + } + + return retVal; +} + +// ExtractFractionPart(a, zeros) +// +// returns the fraction part of |a|, with |zeros| number of leading zeros, as an +// int16_t scaled to Q8. There is no sanity check of |a| in the sense that the +// number of zeros match. +static int16_t ExtractFractionPart(uint32_t a, int zeros) { + return (int16_t)(((a << zeros) & 0x7FFFFFFF) >> 23); +} + +// Calculates and returns the log of |energy| in Q8. The input |energy| is +// supposed to be in Q(|q_domain|). +static int16_t LogOfEnergyInQ8(uint32_t energy, int q_domain) { + static const int16_t kLogLowValue = PART_LEN_SHIFT << 7; + int16_t log_energy_q8 = kLogLowValue; + if (energy > 0) { + int zeros = WebRtcSpl_NormU32(energy); + int16_t frac = ExtractFractionPart(energy, zeros); + // log2 of |energy| in Q8. + log_energy_q8 += ((31 - zeros) << 8) + frac - (q_domain << 8); + } + return log_energy_q8; +} + +// WebRtcAecm_CalcEnergies(...) +// +// This function calculates the log of energies for nearend, farend and estimated +// echoes. There is also an update of energy decision levels, i.e. internal VAD. +// +// +// @param aecm [i/o] Handle of the AECM instance. +// @param far_spectrum [in] Pointer to farend spectrum. +// @param far_q [in] Q-domain of farend spectrum. +// @param nearEner [in] Near end energy for current block in +// Q(aecm->dfaQDomain). +// @param echoEst [out] Estimated echo in Q(xfa_q+RESOLUTION_CHANNEL16). +// +void WebRtcAecm_CalcEnergies(AecmCore* aecm, + const uint16_t* far_spectrum, + const int16_t far_q, + const uint32_t nearEner, + int32_t* echoEst) { + // Local variables + uint32_t tmpAdapt = 0; + uint32_t tmpStored = 0; + uint32_t tmpFar = 0; + + int i; + + int16_t tmp16; + int16_t increase_max_shifts = 4; + int16_t decrease_max_shifts = 11; + int16_t increase_min_shifts = 11; + int16_t decrease_min_shifts = 3; + + // Get log of near end energy and store in buffer + + // Shift buffer + memmove(aecm->nearLogEnergy + 1, aecm->nearLogEnergy, + sizeof(int16_t) * (MAX_BUF_LEN - 1)); + + // Logarithm of integrated magnitude spectrum (nearEner) + aecm->nearLogEnergy[0] = LogOfEnergyInQ8(nearEner, aecm->dfaNoisyQDomain); + + WebRtcAecm_CalcLinearEnergies(aecm, far_spectrum, echoEst, &tmpFar, &tmpAdapt, &tmpStored); + + // Shift buffers + memmove(aecm->echoAdaptLogEnergy + 1, aecm->echoAdaptLogEnergy, + sizeof(int16_t) * (MAX_BUF_LEN - 1)); + memmove(aecm->echoStoredLogEnergy + 1, aecm->echoStoredLogEnergy, + sizeof(int16_t) * (MAX_BUF_LEN - 1)); + + // Logarithm of delayed far end energy + aecm->farLogEnergy = LogOfEnergyInQ8(tmpFar, far_q); + + // Logarithm of estimated echo energy through adapted channel + aecm->echoAdaptLogEnergy[0] = LogOfEnergyInQ8(tmpAdapt, + RESOLUTION_CHANNEL16 + far_q); + + // Logarithm of estimated echo energy through stored channel + aecm->echoStoredLogEnergy[0] = + LogOfEnergyInQ8(tmpStored, RESOLUTION_CHANNEL16 + far_q); + + // Update farend energy levels (min, max, vad, mse) + if (aecm->farLogEnergy > FAR_ENERGY_MIN) + { + if (aecm->startupState == 0) + { + increase_max_shifts = 2; + decrease_min_shifts = 2; + increase_min_shifts = 8; + } + + aecm->farEnergyMin = WebRtcAecm_AsymFilt(aecm->farEnergyMin, aecm->farLogEnergy, + increase_min_shifts, decrease_min_shifts); + aecm->farEnergyMax = WebRtcAecm_AsymFilt(aecm->farEnergyMax, aecm->farLogEnergy, + increase_max_shifts, decrease_max_shifts); + aecm->farEnergyMaxMin = (aecm->farEnergyMax - aecm->farEnergyMin); + + // Dynamic VAD region size + tmp16 = 2560 - aecm->farEnergyMin; + if (tmp16 > 0) + { + tmp16 = (int16_t)((tmp16 * FAR_ENERGY_VAD_REGION) >> 9); + } else + { + tmp16 = 0; + } + tmp16 += FAR_ENERGY_VAD_REGION; + + if ((aecm->startupState == 0) | (aecm->vadUpdateCount > 1024)) + { + // In startup phase or VAD update halted + aecm->farEnergyVAD = aecm->farEnergyMin + tmp16; + } else + { + if (aecm->farEnergyVAD > aecm->farLogEnergy) + { + aecm->farEnergyVAD += + (aecm->farLogEnergy + tmp16 - aecm->farEnergyVAD) >> 6; + aecm->vadUpdateCount = 0; + } else + { + aecm->vadUpdateCount++; + } + } + // Put MSE threshold higher than VAD + aecm->farEnergyMSE = aecm->farEnergyVAD + (1 << 8); + } + + // Update VAD variables + if (aecm->farLogEnergy > aecm->farEnergyVAD) + { + if ((aecm->startupState == 0) | (aecm->farEnergyMaxMin > FAR_ENERGY_DIFF)) + { + // We are in startup or have significant dynamics in input speech level + aecm->currentVADValue = 1; + } + } else + { + aecm->currentVADValue = 0; + } + if ((aecm->currentVADValue) && (aecm->firstVAD)) + { + aecm->firstVAD = 0; + if (aecm->echoAdaptLogEnergy[0] > aecm->nearLogEnergy[0]) + { + // The estimated echo has higher energy than the near end signal. + // This means that the initialization was too aggressive. Scale + // down by a factor 8 + for (i = 0; i < PART_LEN1; i++) + { + aecm->channelAdapt16[i] >>= 3; + } + // Compensate the adapted echo energy level accordingly. + aecm->echoAdaptLogEnergy[0] -= (3 << 8); + aecm->firstVAD = 1; + } + } +} + +// WebRtcAecm_CalcStepSize(...) +// +// This function calculates the step size used in channel estimation +// +// +// @param aecm [in] Handle of the AECM instance. +// @param mu [out] (Return value) Stepsize in log2(), i.e. number of shifts. +// +// +int16_t WebRtcAecm_CalcStepSize(AecmCore* const aecm) { + int32_t tmp32; + int16_t tmp16; + int16_t mu = MU_MAX; + + // Here we calculate the step size mu used in the + // following NLMS based Channel estimation algorithm + if (!aecm->currentVADValue) + { + // Far end energy level too low, no channel update + mu = 0; + } else if (aecm->startupState > 0) + { + if (aecm->farEnergyMin >= aecm->farEnergyMax) + { + mu = MU_MIN; + } else + { + tmp16 = (aecm->farLogEnergy - aecm->farEnergyMin); + tmp32 = tmp16 * MU_DIFF; + tmp32 = WebRtcSpl_DivW32W16(tmp32, aecm->farEnergyMaxMin); + mu = MU_MIN - 1 - (int16_t)(tmp32); + // The -1 is an alternative to rounding. This way we get a larger + // stepsize, so we in some sense compensate for truncation in NLMS + } + if (mu < MU_MAX) + { + mu = MU_MAX; // Equivalent with maximum step size of 2^-MU_MAX + } + } + + return mu; +} + +// WebRtcAecm_UpdateChannel(...) +// +// This function performs channel estimation. NLMS and decision on channel storage. +// +// +// @param aecm [i/o] Handle of the AECM instance. +// @param far_spectrum [in] Absolute value of the farend signal in Q(far_q) +// @param far_q [in] Q-domain of the farend signal +// @param dfa [in] Absolute value of the nearend signal (Q[aecm->dfaQDomain]) +// @param mu [in] NLMS step size. +// @param echoEst [i/o] Estimated echo in Q(far_q+RESOLUTION_CHANNEL16). +// +void WebRtcAecm_UpdateChannel(AecmCore* aecm, + const uint16_t* far_spectrum, + const int16_t far_q, + const uint16_t* const dfa, + const int16_t mu, + int32_t* echoEst) { + uint32_t tmpU32no1, tmpU32no2; + int32_t tmp32no1, tmp32no2; + int32_t mseStored; + int32_t mseAdapt; + + int i; + + int16_t zerosFar, zerosNum, zerosCh, zerosDfa; + int16_t shiftChFar, shiftNum, shift2ResChan; + int16_t tmp16no1; + int16_t xfaQ, dfaQ; + + // This is the channel estimation algorithm. It is base on NLMS but has a variable step + // length, which was calculated above. + if (mu) + { + for (i = 0; i < PART_LEN1; i++) + { + // Determine norm of channel and farend to make sure we don't get overflow in + // multiplication + zerosCh = WebRtcSpl_NormU32(aecm->channelAdapt32[i]); + zerosFar = WebRtcSpl_NormU32((uint32_t)far_spectrum[i]); + if (zerosCh + zerosFar > 31) + { + // Multiplication is safe + tmpU32no1 = WEBRTC_SPL_UMUL_32_16(aecm->channelAdapt32[i], + far_spectrum[i]); + shiftChFar = 0; + } else + { + // We need to shift down before multiplication + shiftChFar = 32 - zerosCh - zerosFar; + tmpU32no1 = rtc::dchecked_cast( + aecm->channelAdapt32[i] >> shiftChFar) * far_spectrum[i]; + } + // Determine Q-domain of numerator + zerosNum = WebRtcSpl_NormU32(tmpU32no1); + if (dfa[i]) + { + zerosDfa = WebRtcSpl_NormU32((uint32_t)dfa[i]); + } else + { + zerosDfa = 32; + } + tmp16no1 = zerosDfa - 2 + aecm->dfaNoisyQDomain - + RESOLUTION_CHANNEL32 - far_q + shiftChFar; + if (zerosNum > tmp16no1 + 1) + { + xfaQ = tmp16no1; + dfaQ = zerosDfa - 2; + } else + { + xfaQ = zerosNum - 2; + dfaQ = RESOLUTION_CHANNEL32 + far_q - aecm->dfaNoisyQDomain - + shiftChFar + xfaQ; + } + // Add in the same Q-domain + tmpU32no1 = WEBRTC_SPL_SHIFT_W32(tmpU32no1, xfaQ); + tmpU32no2 = WEBRTC_SPL_SHIFT_W32((uint32_t)dfa[i], dfaQ); + tmp32no1 = (int32_t)tmpU32no2 - (int32_t)tmpU32no1; + zerosNum = WebRtcSpl_NormW32(tmp32no1); + if ((tmp32no1) && (far_spectrum[i] > (CHANNEL_VAD << far_q))) + { + // + // Update is needed + // + // This is what we would like to compute + // + // tmp32no1 = dfa[i] - (aecm->channelAdapt[i] * far_spectrum[i]) + // tmp32norm = (i + 1) + // aecm->channelAdapt[i] += (2^mu) * tmp32no1 + // / (tmp32norm * far_spectrum[i]) + // + + // Make sure we don't get overflow in multiplication. + if (zerosNum + zerosFar > 31) + { + if (tmp32no1 > 0) + { + tmp32no2 = (int32_t)WEBRTC_SPL_UMUL_32_16(tmp32no1, + far_spectrum[i]); + } else + { + tmp32no2 = -(int32_t)WEBRTC_SPL_UMUL_32_16(-tmp32no1, + far_spectrum[i]); + } + shiftNum = 0; + } else + { + shiftNum = 32 - (zerosNum + zerosFar); + if (tmp32no1 > 0) + { + tmp32no2 = (tmp32no1 >> shiftNum) * far_spectrum[i]; + } else + { + tmp32no2 = -((-tmp32no1 >> shiftNum) * far_spectrum[i]); + } + } + // Normalize with respect to frequency bin + tmp32no2 = WebRtcSpl_DivW32W16(tmp32no2, i + 1); + // Make sure we are in the right Q-domain + shift2ResChan = shiftNum + shiftChFar - xfaQ - mu - ((30 - zerosFar) << 1); + if (WebRtcSpl_NormW32(tmp32no2) < shift2ResChan) + { + tmp32no2 = WEBRTC_SPL_WORD32_MAX; + } else + { + tmp32no2 = WEBRTC_SPL_SHIFT_W32(tmp32no2, shift2ResChan); + } + aecm->channelAdapt32[i] = + WebRtcSpl_AddSatW32(aecm->channelAdapt32[i], tmp32no2); + if (aecm->channelAdapt32[i] < 0) + { + // We can never have negative channel gain + aecm->channelAdapt32[i] = 0; + } + aecm->channelAdapt16[i] = + (int16_t)(aecm->channelAdapt32[i] >> 16); + } + } + } + // END: Adaptive channel update + + // Determine if we should store or restore the channel + if ((aecm->startupState == 0) & (aecm->currentVADValue)) + { + // During startup we store the channel every block, + // and we recalculate echo estimate + WebRtcAecm_StoreAdaptiveChannel(aecm, far_spectrum, echoEst); + } else + { + if (aecm->farLogEnergy < aecm->farEnergyMSE) + { + aecm->mseChannelCount = 0; + } else + { + aecm->mseChannelCount++; + } + // Enough data for validation. Store channel if we can. + if (aecm->mseChannelCount >= (MIN_MSE_COUNT + 10)) + { + // We have enough data. + // Calculate MSE of "Adapt" and "Stored" versions. + // It is actually not MSE, but average absolute error. + mseStored = 0; + mseAdapt = 0; + for (i = 0; i < MIN_MSE_COUNT; i++) + { + tmp32no1 = ((int32_t)aecm->echoStoredLogEnergy[i] + - (int32_t)aecm->nearLogEnergy[i]); + tmp32no2 = WEBRTC_SPL_ABS_W32(tmp32no1); + mseStored += tmp32no2; + + tmp32no1 = ((int32_t)aecm->echoAdaptLogEnergy[i] + - (int32_t)aecm->nearLogEnergy[i]); + tmp32no2 = WEBRTC_SPL_ABS_W32(tmp32no1); + mseAdapt += tmp32no2; + } + if (((mseStored << MSE_RESOLUTION) < (MIN_MSE_DIFF * mseAdapt)) + & ((aecm->mseStoredOld << MSE_RESOLUTION) < (MIN_MSE_DIFF + * aecm->mseAdaptOld))) + { + // The stored channel has a significantly lower MSE than the adaptive one for + // two consecutive calculations. Reset the adaptive channel. + WebRtcAecm_ResetAdaptiveChannel(aecm); + } else if (((MIN_MSE_DIFF * mseStored) > (mseAdapt << MSE_RESOLUTION)) & (mseAdapt + < aecm->mseThreshold) & (aecm->mseAdaptOld < aecm->mseThreshold)) + { + // The adaptive channel has a significantly lower MSE than the stored one. + // The MSE for the adaptive channel has also been low for two consecutive + // calculations. Store the adaptive channel. + WebRtcAecm_StoreAdaptiveChannel(aecm, far_spectrum, echoEst); + + // Update threshold + if (aecm->mseThreshold == WEBRTC_SPL_WORD32_MAX) + { + aecm->mseThreshold = (mseAdapt + aecm->mseAdaptOld); + } else + { + int scaled_threshold = aecm->mseThreshold * 5 / 8; + aecm->mseThreshold += + ((mseAdapt - scaled_threshold) * 205) >> 8; + } + + } + + // Reset counter + aecm->mseChannelCount = 0; + + // Store the MSE values. + aecm->mseStoredOld = mseStored; + aecm->mseAdaptOld = mseAdapt; + } + } + // END: Determine if we should store or reset channel estimate. +} + +// CalcSuppressionGain(...) +// +// This function calculates the suppression gain that is used in the Wiener filter. +// +// +// @param aecm [i/n] Handle of the AECM instance. +// @param supGain [out] (Return value) Suppression gain with which to scale the noise +// level (Q14). +// +// +int16_t WebRtcAecm_CalcSuppressionGain(AecmCore* const aecm) { + int32_t tmp32no1; + + int16_t supGain = SUPGAIN_DEFAULT; + int16_t tmp16no1; + int16_t dE = 0; + + // Determine suppression gain used in the Wiener filter. The gain is based on a mix of far + // end energy and echo estimation error. + // Adjust for the far end signal level. A low signal level indicates no far end signal, + // hence we set the suppression gain to 0 + if (!aecm->currentVADValue) + { + supGain = 0; + } else + { + // Adjust for possible double talk. If we have large variations in estimation error we + // likely have double talk (or poor channel). + tmp16no1 = (aecm->nearLogEnergy[0] - aecm->echoStoredLogEnergy[0] - ENERGY_DEV_OFFSET); + dE = WEBRTC_SPL_ABS_W16(tmp16no1); + + if (dE < ENERGY_DEV_TOL) + { + // Likely no double talk. The better estimation, the more we can suppress signal. + // Update counters + if (dE < SUPGAIN_EPC_DT) + { + tmp32no1 = aecm->supGainErrParamDiffAB * dE; + tmp32no1 += (SUPGAIN_EPC_DT >> 1); + tmp16no1 = (int16_t)WebRtcSpl_DivW32W16(tmp32no1, SUPGAIN_EPC_DT); + supGain = aecm->supGainErrParamA - tmp16no1; + } else + { + tmp32no1 = aecm->supGainErrParamDiffBD * (ENERGY_DEV_TOL - dE); + tmp32no1 += ((ENERGY_DEV_TOL - SUPGAIN_EPC_DT) >> 1); + tmp16no1 = (int16_t)WebRtcSpl_DivW32W16(tmp32no1, (ENERGY_DEV_TOL + - SUPGAIN_EPC_DT)); + supGain = aecm->supGainErrParamD + tmp16no1; + } + } else + { + // Likely in double talk. Use default value + supGain = aecm->supGainErrParamD; + } + } + + if (supGain > aecm->supGainOld) + { + tmp16no1 = supGain; + } else + { + tmp16no1 = aecm->supGainOld; + } + aecm->supGainOld = supGain; + if (tmp16no1 < aecm->supGain) + { + aecm->supGain += (int16_t)((tmp16no1 - aecm->supGain) >> 4); + } else + { + aecm->supGain += (int16_t)((tmp16no1 - aecm->supGain) >> 4); + } + + // END: Update suppression gain + + return aecm->supGain; +} + +void WebRtcAecm_BufferFarFrame(AecmCore* const aecm, + const int16_t* const farend, + const int farLen) { + int writeLen = farLen, writePos = 0; + + // Check if the write position must be wrapped + while (aecm->farBufWritePos + writeLen > FAR_BUF_LEN) + { + // Write to remaining buffer space before wrapping + writeLen = FAR_BUF_LEN - aecm->farBufWritePos; + memcpy(aecm->farBuf + aecm->farBufWritePos, farend + writePos, + sizeof(int16_t) * writeLen); + aecm->farBufWritePos = 0; + writePos = writeLen; + writeLen = farLen - writeLen; + } + + memcpy(aecm->farBuf + aecm->farBufWritePos, farend + writePos, + sizeof(int16_t) * writeLen); + aecm->farBufWritePos += writeLen; +} + +void WebRtcAecm_FetchFarFrame(AecmCore* const aecm, + int16_t* const farend, + const int farLen, + const int knownDelay) { + int readLen = farLen; + int readPos = 0; + int delayChange = knownDelay - aecm->lastKnownDelay; + + aecm->farBufReadPos -= delayChange; + + // Check if delay forces a read position wrap + while (aecm->farBufReadPos < 0) + { + aecm->farBufReadPos += FAR_BUF_LEN; + } + while (aecm->farBufReadPos > FAR_BUF_LEN - 1) + { + aecm->farBufReadPos -= FAR_BUF_LEN; + } + + aecm->lastKnownDelay = knownDelay; + + // Check if read position must be wrapped + while (aecm->farBufReadPos + readLen > FAR_BUF_LEN) + { + + // Read from remaining buffer space before wrapping + readLen = FAR_BUF_LEN - aecm->farBufReadPos; + memcpy(farend + readPos, aecm->farBuf + aecm->farBufReadPos, + sizeof(int16_t) * readLen); + aecm->farBufReadPos = 0; + readPos = readLen; + readLen = farLen - readLen; + } + memcpy(farend + readPos, aecm->farBuf + aecm->farBufReadPos, + sizeof(int16_t) * readLen); + aecm->farBufReadPos += readLen; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.h new file mode 100644 index 0000000000..feb997e9fa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.h @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Performs echo control (suppression) with fft routines in fixed-point. + +#ifndef MODULES_AUDIO_PROCESSING_AECM_AECM_CORE_H_ +#define MODULES_AUDIO_PROCESSING_AECM_AECM_CORE_H_ + +extern "C" { +#include "common_audio/ring_buffer.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +} +#include "modules/audio_processing/aecm/aecm_defines.h" +#include "typedefs.h" // NOLINT(build/include) + +#ifdef _MSC_VER // visual c++ +#define ALIGN8_BEG __declspec(align(8)) +#define ALIGN8_END +#else // gcc or icc +#define ALIGN8_BEG +#define ALIGN8_END __attribute__((aligned(8))) +#endif + +typedef struct { + int16_t real; + int16_t imag; +} ComplexInt16; + +typedef struct { + int farBufWritePos; + int farBufReadPos; + int knownDelay; + int lastKnownDelay; + int firstVAD; // Parameter to control poorly initialized channels + + RingBuffer* farFrameBuf; + RingBuffer* nearNoisyFrameBuf; + RingBuffer* nearCleanFrameBuf; + RingBuffer* outFrameBuf; + + int16_t farBuf[FAR_BUF_LEN]; + + int16_t mult; + uint32_t seed; + + // Delay estimation variables + void* delay_estimator_farend; + void* delay_estimator; + uint16_t currentDelay; + // Far end history variables + // TODO(bjornv): Replace |far_history| with ring_buffer. + uint16_t far_history[PART_LEN1 * MAX_DELAY]; + int far_history_pos; + int far_q_domains[MAX_DELAY]; + + int16_t nlpFlag; + int16_t fixedDelay; + + uint32_t totCount; + + int16_t dfaCleanQDomain; + int16_t dfaCleanQDomainOld; + int16_t dfaNoisyQDomain; + int16_t dfaNoisyQDomainOld; + + int16_t nearLogEnergy[MAX_BUF_LEN]; + int16_t farLogEnergy; + int16_t echoAdaptLogEnergy[MAX_BUF_LEN]; + int16_t echoStoredLogEnergy[MAX_BUF_LEN]; + + // The extra 16 or 32 bytes in the following buffers are for alignment based + // Neon code. + // It's designed this way since the current GCC compiler can't align a + // buffer in 16 or 32 byte boundaries properly. + int16_t channelStored_buf[PART_LEN1 + 8]; + int16_t channelAdapt16_buf[PART_LEN1 + 8]; + int32_t channelAdapt32_buf[PART_LEN1 + 8]; + int16_t xBuf_buf[PART_LEN2 + 16]; // farend + int16_t dBufClean_buf[PART_LEN2 + 16]; // nearend + int16_t dBufNoisy_buf[PART_LEN2 + 16]; // nearend + int16_t outBuf_buf[PART_LEN + 8]; + + // Pointers to the above buffers + int16_t *channelStored; + int16_t *channelAdapt16; + int32_t *channelAdapt32; + int16_t *xBuf; + int16_t *dBufClean; + int16_t *dBufNoisy; + int16_t *outBuf; + + int32_t echoFilt[PART_LEN1]; + int16_t nearFilt[PART_LEN1]; + int32_t noiseEst[PART_LEN1]; + int noiseEstTooLowCtr[PART_LEN1]; + int noiseEstTooHighCtr[PART_LEN1]; + int16_t noiseEstCtr; + int16_t cngMode; + + int32_t mseAdaptOld; + int32_t mseStoredOld; + int32_t mseThreshold; + + int16_t farEnergyMin; + int16_t farEnergyMax; + int16_t farEnergyMaxMin; + int16_t farEnergyVAD; + int16_t farEnergyMSE; + int currentVADValue; + int16_t vadUpdateCount; + + int16_t startupState; + int16_t mseChannelCount; + int16_t supGain; + int16_t supGainOld; + + int16_t supGainErrParamA; + int16_t supGainErrParamD; + int16_t supGainErrParamDiffAB; + int16_t supGainErrParamDiffBD; + + struct RealFFT* real_fft; + +#ifdef AEC_DEBUG + FILE *farFile; + FILE *nearFile; + FILE *outFile; +#endif +} AecmCore; + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_CreateCore() +// +// Allocates the memory needed by the AECM. The memory needs to be +// initialized separately using the WebRtcAecm_InitCore() function. +// Returns a pointer to the instance and a nullptr at failure. +AecmCore* WebRtcAecm_CreateCore(); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_InitCore(...) +// +// This function initializes the AECM instant created with +// WebRtcAecm_CreateCore() +// Input: +// - aecm : Pointer to the AECM instance +// - samplingFreq : Sampling Frequency +// +// Output: +// - aecm : Initialized instance +// +// Return value : 0 - Ok +// -1 - Error +// +int WebRtcAecm_InitCore(AecmCore* const aecm, int samplingFreq); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_FreeCore(...) +// +// This function releases the memory allocated by WebRtcAecm_CreateCore() +// Input: +// - aecm : Pointer to the AECM instance +// +void WebRtcAecm_FreeCore(AecmCore* aecm); + +int WebRtcAecm_Control(AecmCore* aecm, int delay, int nlpFlag); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_InitEchoPathCore(...) +// +// This function resets the echo channel adaptation with the specified channel. +// Input: +// - aecm : Pointer to the AECM instance +// - echo_path : Pointer to the data that should initialize the echo +// path +// +// Output: +// - aecm : Initialized instance +// +void WebRtcAecm_InitEchoPathCore(AecmCore* aecm, const int16_t* echo_path); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_ProcessFrame(...) +// +// This function processes frames and sends blocks to +// WebRtcAecm_ProcessBlock(...) +// +// Inputs: +// - aecm : Pointer to the AECM instance +// - farend : In buffer containing one frame of echo signal +// - nearendNoisy : In buffer containing one frame of nearend+echo signal +// without NS +// - nearendClean : In buffer containing one frame of nearend+echo signal +// with NS +// +// Output: +// - out : Out buffer, one frame of nearend signal : +// +// +int WebRtcAecm_ProcessFrame(AecmCore* aecm, + const int16_t* farend, + const int16_t* nearendNoisy, + const int16_t* nearendClean, + int16_t* out); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_ProcessBlock(...) +// +// This function is called for every block within one frame +// This function is called by WebRtcAecm_ProcessFrame(...) +// +// Inputs: +// - aecm : Pointer to the AECM instance +// - farend : In buffer containing one block of echo signal +// - nearendNoisy : In buffer containing one frame of nearend+echo signal +// without NS +// - nearendClean : In buffer containing one frame of nearend+echo signal +// with NS +// +// Output: +// - out : Out buffer, one block of nearend signal : +// +// +int WebRtcAecm_ProcessBlock(AecmCore* aecm, + const int16_t* farend, + const int16_t* nearendNoisy, + const int16_t* noisyClean, + int16_t* out); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_BufferFarFrame() +// +// Inserts a frame of data into farend buffer. +// +// Inputs: +// - aecm : Pointer to the AECM instance +// - farend : In buffer containing one frame of farend signal +// - farLen : Length of frame +// +void WebRtcAecm_BufferFarFrame(AecmCore* const aecm, + const int16_t* const farend, + const int farLen); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_FetchFarFrame() +// +// Read the farend buffer to account for known delay +// +// Inputs: +// - aecm : Pointer to the AECM instance +// - farend : In buffer containing one frame of farend signal +// - farLen : Length of frame +// - knownDelay : known delay +// +void WebRtcAecm_FetchFarFrame(AecmCore* const aecm, + int16_t* const farend, + const int farLen, + const int knownDelay); + +// All the functions below are intended to be private + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_UpdateFarHistory() +// +// Moves the pointer to the next entry and inserts |far_spectrum| and +// corresponding Q-domain in its buffer. +// +// Inputs: +// - self : Pointer to the delay estimation instance +// - far_spectrum : Pointer to the far end spectrum +// - far_q : Q-domain of far end spectrum +// +void WebRtcAecm_UpdateFarHistory(AecmCore* self, + uint16_t* far_spectrum, + int far_q); + +//////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_AlignedFarend() +// +// Returns a pointer to the far end spectrum aligned to current near end +// spectrum. The function WebRtc_DelayEstimatorProcessFix(...) should have been +// called before AlignedFarend(...). Otherwise, you get the pointer to the +// previous frame. The memory is only valid until the next call of +// WebRtc_DelayEstimatorProcessFix(...). +// +// Inputs: +// - self : Pointer to the AECM instance. +// - delay : Current delay estimate. +// +// Output: +// - far_q : The Q-domain of the aligned far end spectrum +// +// Return value: +// - far_spectrum : Pointer to the aligned far end spectrum +// NULL - Error +// +const uint16_t* WebRtcAecm_AlignedFarend(AecmCore* self, int* far_q, int delay); + +/////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_CalcSuppressionGain() +// +// This function calculates the suppression gain that is used in the +// Wiener filter. +// +// Inputs: +// - aecm : Pointer to the AECM instance. +// +// Return value: +// - supGain : Suppression gain with which to scale the noise +// level (Q14). +// +int16_t WebRtcAecm_CalcSuppressionGain(AecmCore* const aecm); + +/////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_CalcEnergies() +// +// This function calculates the log of energies for nearend, farend and +// estimated echoes. There is also an update of energy decision levels, +// i.e. internal VAD. +// +// Inputs: +// - aecm : Pointer to the AECM instance. +// - far_spectrum : Pointer to farend spectrum. +// - far_q : Q-domain of farend spectrum. +// - nearEner : Near end energy for current block in +// Q(aecm->dfaQDomain). +// +// Output: +// - echoEst : Estimated echo in Q(xfa_q+RESOLUTION_CHANNEL16). +// +void WebRtcAecm_CalcEnergies(AecmCore* aecm, + const uint16_t* far_spectrum, + const int16_t far_q, + const uint32_t nearEner, + int32_t* echoEst); + +/////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_CalcStepSize() +// +// This function calculates the step size used in channel estimation +// +// Inputs: +// - aecm : Pointer to the AECM instance. +// +// Return value: +// - mu : Stepsize in log2(), i.e. number of shifts. +// +int16_t WebRtcAecm_CalcStepSize(AecmCore* const aecm); + +/////////////////////////////////////////////////////////////////////////////// +// WebRtcAecm_UpdateChannel(...) +// +// This function performs channel estimation. +// NLMS and decision on channel storage. +// +// Inputs: +// - aecm : Pointer to the AECM instance. +// - far_spectrum : Absolute value of the farend signal in Q(far_q) +// - far_q : Q-domain of the farend signal +// - dfa : Absolute value of the nearend signal +// (Q[aecm->dfaQDomain]) +// - mu : NLMS step size. +// Input/Output: +// - echoEst : Estimated echo in Q(far_q+RESOLUTION_CHANNEL16). +// +void WebRtcAecm_UpdateChannel(AecmCore* aecm, + const uint16_t* far_spectrum, + const int16_t far_q, + const uint16_t* const dfa, + const int16_t mu, + int32_t* echoEst); + +extern const int16_t WebRtcAecm_kCosTable[]; +extern const int16_t WebRtcAecm_kSinTable[]; + +/////////////////////////////////////////////////////////////////////////////// +// Some function pointers, for internal functions shared by ARM NEON and +// generic C code. +// +typedef void (*CalcLinearEnergies)(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echoEst, + uint32_t* far_energy, + uint32_t* echo_energy_adapt, + uint32_t* echo_energy_stored); +extern CalcLinearEnergies WebRtcAecm_CalcLinearEnergies; + +typedef void (*StoreAdaptiveChannel)(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est); +extern StoreAdaptiveChannel WebRtcAecm_StoreAdaptiveChannel; + +typedef void (*ResetAdaptiveChannel)(AecmCore* aecm); +extern ResetAdaptiveChannel WebRtcAecm_ResetAdaptiveChannel; + +// For the above function pointers, functions for generic platforms are declared +// and defined as static in file aecm_core.c, while those for ARM Neon platforms +// are declared below and defined in file aecm_core_neon.c. +#if defined(WEBRTC_HAS_NEON) +void WebRtcAecm_CalcLinearEnergiesNeon(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est, + uint32_t* far_energy, + uint32_t* echo_energy_adapt, + uint32_t* echo_energy_stored); + +void WebRtcAecm_StoreAdaptiveChannelNeon(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est); + +void WebRtcAecm_ResetAdaptiveChannelNeon(AecmCore* aecm); +#endif + +#if defined(MIPS32_LE) +void WebRtcAecm_CalcLinearEnergies_mips(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est, + uint32_t* far_energy, + uint32_t* echo_energy_adapt, + uint32_t* echo_energy_stored); +#if defined(MIPS_DSP_R1_LE) +void WebRtcAecm_StoreAdaptiveChannel_mips(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est); + +void WebRtcAecm_ResetAdaptiveChannel_mips(AecmCore* aecm); +#endif +#endif + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_c.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_c.cc new file mode 100644 index 0000000000..7fd6a499ad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_c.cc @@ -0,0 +1,773 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aecm/aecm_core.h" + +#include +#include + +extern "C" { +#include "common_audio/ring_buffer.h" +#include "common_audio/signal_processing/include/real_fft.h" +} +#include "modules/audio_processing/aecm/echo_control_mobile.h" +#include "modules/audio_processing/utility/delay_estimator_wrapper.h" +extern "C" { +#include "system_wrappers/include/cpu_features_wrapper.h" +} + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/sanitizer.h" +#include "typedefs.h" // NOLINT(build/include) + +// Square root of Hanning window in Q14. +static const ALIGN8_BEG int16_t WebRtcAecm_kSqrtHanning[] ALIGN8_END = { + 0, 399, 798, 1196, 1594, 1990, 2386, 2780, 3172, + 3562, 3951, 4337, 4720, 5101, 5478, 5853, 6224, + 6591, 6954, 7313, 7668, 8019, 8364, 8705, 9040, + 9370, 9695, 10013, 10326, 10633, 10933, 11227, 11514, + 11795, 12068, 12335, 12594, 12845, 13089, 13325, 13553, + 13773, 13985, 14189, 14384, 14571, 14749, 14918, 15079, + 15231, 15373, 15506, 15631, 15746, 15851, 15947, 16034, + 16111, 16179, 16237, 16286, 16325, 16354, 16373, 16384 +}; + +#ifdef AECM_WITH_ABS_APPROX +//Q15 alpha = 0.99439986968132 const Factor for magnitude approximation +static const uint16_t kAlpha1 = 32584; +//Q15 beta = 0.12967166976970 const Factor for magnitude approximation +static const uint16_t kBeta1 = 4249; +//Q15 alpha = 0.94234827210087 const Factor for magnitude approximation +static const uint16_t kAlpha2 = 30879; +//Q15 beta = 0.33787806009150 const Factor for magnitude approximation +static const uint16_t kBeta2 = 11072; +//Q15 alpha = 0.82247698684306 const Factor for magnitude approximation +static const uint16_t kAlpha3 = 26951; +//Q15 beta = 0.57762063060713 const Factor for magnitude approximation +static const uint16_t kBeta3 = 18927; +#endif + +static const int16_t kNoiseEstQDomain = 15; +static const int16_t kNoiseEstIncCount = 5; + +static void ComfortNoise(AecmCore* aecm, + const uint16_t* dfa, + ComplexInt16* out, + const int16_t* lambda); + +static void WindowAndFFT(AecmCore* aecm, + int16_t* fft, + const int16_t* time_signal, + ComplexInt16* freq_signal, + int time_signal_scaling) { + int i = 0; + + // FFT of signal + for (i = 0; i < PART_LEN; i++) { + // Window time domain signal and insert into real part of + // transformation array |fft| + int16_t scaled_time_signal = time_signal[i] << time_signal_scaling; + fft[i] = (int16_t)((scaled_time_signal * WebRtcAecm_kSqrtHanning[i]) >> 14); + scaled_time_signal = time_signal[i + PART_LEN] << time_signal_scaling; + fft[PART_LEN + i] = (int16_t)(( + scaled_time_signal * WebRtcAecm_kSqrtHanning[PART_LEN - i]) >> 14); + } + + // Do forward FFT, then take only the first PART_LEN complex samples, + // and change signs of the imaginary parts. + WebRtcSpl_RealForwardFFT(aecm->real_fft, fft, (int16_t*)freq_signal); + for (i = 0; i < PART_LEN; i++) { + freq_signal[i].imag = -freq_signal[i].imag; + } +} + +static void InverseFFTAndWindow(AecmCore* aecm, + int16_t* fft, + ComplexInt16* efw, + int16_t* output, + const int16_t* nearendClean) { + int i, j, outCFFT; + int32_t tmp32no1; + // Reuse |efw| for the inverse FFT output after transferring + // the contents to |fft|. + int16_t* ifft_out = (int16_t*)efw; + + // Synthesis + for (i = 1, j = 2; i < PART_LEN; i += 1, j += 2) { + fft[j] = efw[i].real; + fft[j + 1] = -efw[i].imag; + } + fft[0] = efw[0].real; + fft[1] = -efw[0].imag; + + fft[PART_LEN2] = efw[PART_LEN].real; + fft[PART_LEN2 + 1] = -efw[PART_LEN].imag; + + // Inverse FFT. Keep outCFFT to scale the samples in the next block. + outCFFT = WebRtcSpl_RealInverseFFT(aecm->real_fft, fft, ifft_out); + for (i = 0; i < PART_LEN; i++) { + ifft_out[i] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + ifft_out[i], WebRtcAecm_kSqrtHanning[i], 14); + tmp32no1 = WEBRTC_SPL_SHIFT_W32((int32_t)ifft_out[i], + outCFFT - aecm->dfaCleanQDomain); + output[i] = (int16_t)WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, + tmp32no1 + aecm->outBuf[i], + WEBRTC_SPL_WORD16_MIN); + + tmp32no1 = (ifft_out[PART_LEN + i] * + WebRtcAecm_kSqrtHanning[PART_LEN - i]) >> 14; + tmp32no1 = WEBRTC_SPL_SHIFT_W32(tmp32no1, + outCFFT - aecm->dfaCleanQDomain); + aecm->outBuf[i] = (int16_t)WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, + tmp32no1, + WEBRTC_SPL_WORD16_MIN); + } + + // Copy the current block to the old position + // (aecm->outBuf is shifted elsewhere) + memcpy(aecm->xBuf, aecm->xBuf + PART_LEN, sizeof(int16_t) * PART_LEN); + memcpy(aecm->dBufNoisy, + aecm->dBufNoisy + PART_LEN, + sizeof(int16_t) * PART_LEN); + if (nearendClean != NULL) + { + memcpy(aecm->dBufClean, + aecm->dBufClean + PART_LEN, + sizeof(int16_t) * PART_LEN); + } +} + +// Transforms a time domain signal into the frequency domain, outputting the +// complex valued signal, absolute value and sum of absolute values. +// +// time_signal [in] Pointer to time domain signal +// freq_signal_real [out] Pointer to real part of frequency domain array +// freq_signal_imag [out] Pointer to imaginary part of frequency domain +// array +// freq_signal_abs [out] Pointer to absolute value of frequency domain +// array +// freq_signal_sum_abs [out] Pointer to the sum of all absolute values in +// the frequency domain array +// return value The Q-domain of current frequency values +// +static int TimeToFrequencyDomain(AecmCore* aecm, + const int16_t* time_signal, + ComplexInt16* freq_signal, + uint16_t* freq_signal_abs, + uint32_t* freq_signal_sum_abs) { + int i = 0; + int time_signal_scaling = 0; + + int32_t tmp32no1 = 0; + int32_t tmp32no2 = 0; + + // In fft_buf, +16 for 32-byte alignment. + int16_t fft_buf[PART_LEN4 + 16]; + int16_t *fft = (int16_t *) (((uintptr_t) fft_buf + 31) & ~31); + + int16_t tmp16no1; +#ifndef WEBRTC_ARCH_ARM_V7 + int16_t tmp16no2; +#endif +#ifdef AECM_WITH_ABS_APPROX + int16_t max_value = 0; + int16_t min_value = 0; + uint16_t alpha = 0; + uint16_t beta = 0; +#endif + +#ifdef AECM_DYNAMIC_Q + tmp16no1 = WebRtcSpl_MaxAbsValueW16(time_signal, PART_LEN2); + time_signal_scaling = WebRtcSpl_NormW16(tmp16no1); +#endif + + WindowAndFFT(aecm, fft, time_signal, freq_signal, time_signal_scaling); + + // Extract imaginary and real part, calculate the magnitude for + // all frequency bins + freq_signal[0].imag = 0; + freq_signal[PART_LEN].imag = 0; + freq_signal_abs[0] = (uint16_t)WEBRTC_SPL_ABS_W16(freq_signal[0].real); + freq_signal_abs[PART_LEN] = (uint16_t)WEBRTC_SPL_ABS_W16( + freq_signal[PART_LEN].real); + (*freq_signal_sum_abs) = (uint32_t)(freq_signal_abs[0]) + + (uint32_t)(freq_signal_abs[PART_LEN]); + + for (i = 1; i < PART_LEN; i++) + { + if (freq_signal[i].real == 0) + { + freq_signal_abs[i] = (uint16_t)WEBRTC_SPL_ABS_W16(freq_signal[i].imag); + } + else if (freq_signal[i].imag == 0) + { + freq_signal_abs[i] = (uint16_t)WEBRTC_SPL_ABS_W16(freq_signal[i].real); + } + else + { + // Approximation for magnitude of complex fft output + // magn = sqrt(real^2 + imag^2) + // magn ~= alpha * max(|imag|,|real|) + beta * min(|imag|,|real|) + // + // The parameters alpha and beta are stored in Q15 + +#ifdef AECM_WITH_ABS_APPROX + tmp16no1 = WEBRTC_SPL_ABS_W16(freq_signal[i].real); + tmp16no2 = WEBRTC_SPL_ABS_W16(freq_signal[i].imag); + + if(tmp16no1 > tmp16no2) + { + max_value = tmp16no1; + min_value = tmp16no2; + } else + { + max_value = tmp16no2; + min_value = tmp16no1; + } + + // Magnitude in Q(-6) + if ((max_value >> 2) > min_value) + { + alpha = kAlpha1; + beta = kBeta1; + } else if ((max_value >> 1) > min_value) + { + alpha = kAlpha2; + beta = kBeta2; + } else + { + alpha = kAlpha3; + beta = kBeta3; + } + tmp16no1 = (int16_t)((max_value * alpha) >> 15); + tmp16no2 = (int16_t)((min_value * beta) >> 15); + freq_signal_abs[i] = (uint16_t)tmp16no1 + (uint16_t)tmp16no2; +#else +#ifdef WEBRTC_ARCH_ARM_V7 + __asm __volatile( + "smulbb %[tmp32no1], %[real], %[real]\n\t" + "smlabb %[tmp32no2], %[imag], %[imag], %[tmp32no1]\n\t" + :[tmp32no1]"+&r"(tmp32no1), + [tmp32no2]"=r"(tmp32no2) + :[real]"r"(freq_signal[i].real), + [imag]"r"(freq_signal[i].imag) + ); +#else + tmp16no1 = WEBRTC_SPL_ABS_W16(freq_signal[i].real); + tmp16no2 = WEBRTC_SPL_ABS_W16(freq_signal[i].imag); + tmp32no1 = tmp16no1 * tmp16no1; + tmp32no2 = tmp16no2 * tmp16no2; + tmp32no2 = WebRtcSpl_AddSatW32(tmp32no1, tmp32no2); +#endif // WEBRTC_ARCH_ARM_V7 + tmp32no1 = WebRtcSpl_SqrtFloor(tmp32no2); + + freq_signal_abs[i] = (uint16_t)tmp32no1; +#endif // AECM_WITH_ABS_APPROX + } + (*freq_signal_sum_abs) += (uint32_t)freq_signal_abs[i]; + } + + return time_signal_scaling; +} + +int RTC_NO_SANITIZE("signed-integer-overflow") // bugs.webrtc.org/8200 +WebRtcAecm_ProcessBlock(AecmCore* aecm, + const int16_t* farend, + const int16_t* nearendNoisy, + const int16_t* nearendClean, + int16_t* output) { + int i; + + uint32_t xfaSum; + uint32_t dfaNoisySum; + uint32_t dfaCleanSum; + uint32_t echoEst32Gained; + uint32_t tmpU32; + + int32_t tmp32no1; + + uint16_t xfa[PART_LEN1]; + uint16_t dfaNoisy[PART_LEN1]; + uint16_t dfaClean[PART_LEN1]; + uint16_t* ptrDfaClean = dfaClean; + const uint16_t* far_spectrum_ptr = NULL; + + // 32 byte aligned buffers (with +8 or +16). + // TODO(kma): define fft with ComplexInt16. + int16_t fft_buf[PART_LEN4 + 2 + 16]; // +2 to make a loop safe. + int32_t echoEst32_buf[PART_LEN1 + 8]; + int32_t dfw_buf[PART_LEN2 + 8]; + int32_t efw_buf[PART_LEN2 + 8]; + + int16_t* fft = (int16_t*) (((uintptr_t) fft_buf + 31) & ~ 31); + int32_t* echoEst32 = (int32_t*) (((uintptr_t) echoEst32_buf + 31) & ~ 31); + ComplexInt16* dfw = (ComplexInt16*)(((uintptr_t)dfw_buf + 31) & ~31); + ComplexInt16* efw = (ComplexInt16*)(((uintptr_t)efw_buf + 31) & ~31); + + int16_t hnl[PART_LEN1]; + int16_t numPosCoef = 0; + int16_t nlpGain = ONE_Q14; + int delay; + int16_t tmp16no1; + int16_t tmp16no2; + int16_t mu; + int16_t supGain; + int16_t zeros32, zeros16; + int16_t zerosDBufNoisy, zerosDBufClean, zerosXBuf; + int far_q; + int16_t resolutionDiff, qDomainDiff, dfa_clean_q_domain_diff; + + const int kMinPrefBand = 4; + const int kMaxPrefBand = 24; + int32_t avgHnl32 = 0; + + // Determine startup state. There are three states: + // (0) the first CONV_LEN blocks + // (1) another CONV_LEN blocks + // (2) the rest + + if (aecm->startupState < 2) + { + aecm->startupState = (aecm->totCount >= CONV_LEN) + + (aecm->totCount >= CONV_LEN2); + } + // END: Determine startup state + + // Buffer near and far end signals + memcpy(aecm->xBuf + PART_LEN, farend, sizeof(int16_t) * PART_LEN); + memcpy(aecm->dBufNoisy + PART_LEN, nearendNoisy, sizeof(int16_t) * PART_LEN); + if (nearendClean != NULL) + { + memcpy(aecm->dBufClean + PART_LEN, + nearendClean, + sizeof(int16_t) * PART_LEN); + } + + // Transform far end signal from time domain to frequency domain. + far_q = TimeToFrequencyDomain(aecm, + aecm->xBuf, + dfw, + xfa, + &xfaSum); + + // Transform noisy near end signal from time domain to frequency domain. + zerosDBufNoisy = TimeToFrequencyDomain(aecm, + aecm->dBufNoisy, + dfw, + dfaNoisy, + &dfaNoisySum); + aecm->dfaNoisyQDomainOld = aecm->dfaNoisyQDomain; + aecm->dfaNoisyQDomain = (int16_t)zerosDBufNoisy; + + + if (nearendClean == NULL) + { + ptrDfaClean = dfaNoisy; + aecm->dfaCleanQDomainOld = aecm->dfaNoisyQDomainOld; + aecm->dfaCleanQDomain = aecm->dfaNoisyQDomain; + dfaCleanSum = dfaNoisySum; + } else + { + // Transform clean near end signal from time domain to frequency domain. + zerosDBufClean = TimeToFrequencyDomain(aecm, + aecm->dBufClean, + dfw, + dfaClean, + &dfaCleanSum); + aecm->dfaCleanQDomainOld = aecm->dfaCleanQDomain; + aecm->dfaCleanQDomain = (int16_t)zerosDBufClean; + } + + // Get the delay + // Save far-end history and estimate delay + WebRtcAecm_UpdateFarHistory(aecm, xfa, far_q); + if (WebRtc_AddFarSpectrumFix(aecm->delay_estimator_farend, + xfa, + PART_LEN1, + far_q) == -1) { + return -1; + } + delay = WebRtc_DelayEstimatorProcessFix(aecm->delay_estimator, + dfaNoisy, + PART_LEN1, + zerosDBufNoisy); + if (delay == -1) + { + return -1; + } + else if (delay == -2) + { + // If the delay is unknown, we assume zero. + // NOTE: this will have to be adjusted if we ever add lookahead. + delay = 0; + } + + if (aecm->fixedDelay >= 0) + { + // Use fixed delay + delay = aecm->fixedDelay; + } + + // Get aligned far end spectrum + far_spectrum_ptr = WebRtcAecm_AlignedFarend(aecm, &far_q, delay); + zerosXBuf = (int16_t) far_q; + if (far_spectrum_ptr == NULL) + { + return -1; + } + + // Calculate log(energy) and update energy threshold levels + WebRtcAecm_CalcEnergies(aecm, + far_spectrum_ptr, + zerosXBuf, + dfaNoisySum, + echoEst32); + + // Calculate stepsize + mu = WebRtcAecm_CalcStepSize(aecm); + + // Update counters + aecm->totCount++; + + // This is the channel estimation algorithm. + // It is base on NLMS but has a variable step length, + // which was calculated above. + WebRtcAecm_UpdateChannel(aecm, + far_spectrum_ptr, + zerosXBuf, + dfaNoisy, + mu, + echoEst32); + supGain = WebRtcAecm_CalcSuppressionGain(aecm); + + + // Calculate Wiener filter hnl[] + for (i = 0; i < PART_LEN1; i++) + { + // Far end signal through channel estimate in Q8 + // How much can we shift right to preserve resolution + tmp32no1 = echoEst32[i] - aecm->echoFilt[i]; + aecm->echoFilt[i] += + rtc::dchecked_cast((int64_t{tmp32no1} * 50) >> 8); + + zeros32 = WebRtcSpl_NormW32(aecm->echoFilt[i]) + 1; + zeros16 = WebRtcSpl_NormW16(supGain) + 1; + if (zeros32 + zeros16 > 16) + { + // Multiplication is safe + // Result in + // Q(RESOLUTION_CHANNEL+RESOLUTION_SUPGAIN+ + // aecm->xfaQDomainBuf[diff]) + echoEst32Gained = WEBRTC_SPL_UMUL_32_16((uint32_t)aecm->echoFilt[i], + (uint16_t)supGain); + resolutionDiff = 14 - RESOLUTION_CHANNEL16 - RESOLUTION_SUPGAIN; + resolutionDiff += (aecm->dfaCleanQDomain - zerosXBuf); + } else + { + tmp16no1 = 17 - zeros32 - zeros16; + resolutionDiff = 14 + tmp16no1 - RESOLUTION_CHANNEL16 - + RESOLUTION_SUPGAIN; + resolutionDiff += (aecm->dfaCleanQDomain - zerosXBuf); + if (zeros32 > tmp16no1) + { + echoEst32Gained = WEBRTC_SPL_UMUL_32_16((uint32_t)aecm->echoFilt[i], + supGain >> tmp16no1); + } else + { + // Result in Q-(RESOLUTION_CHANNEL+RESOLUTION_SUPGAIN-16) + echoEst32Gained = (aecm->echoFilt[i] >> tmp16no1) * supGain; + } + } + + zeros16 = WebRtcSpl_NormW16(aecm->nearFilt[i]); + RTC_DCHECK_GE(zeros16, 0); // |zeros16| is a norm, hence non-negative. + dfa_clean_q_domain_diff = aecm->dfaCleanQDomain - aecm->dfaCleanQDomainOld; + if (zeros16 < dfa_clean_q_domain_diff && aecm->nearFilt[i]) { + tmp16no1 = aecm->nearFilt[i] << zeros16; + qDomainDiff = zeros16 - dfa_clean_q_domain_diff; + tmp16no2 = ptrDfaClean[i] >> -qDomainDiff; + } else { + tmp16no1 = dfa_clean_q_domain_diff < 0 + ? aecm->nearFilt[i] >> -dfa_clean_q_domain_diff + : aecm->nearFilt[i] << dfa_clean_q_domain_diff; + qDomainDiff = 0; + tmp16no2 = ptrDfaClean[i]; + } + tmp32no1 = (int32_t)(tmp16no2 - tmp16no1); + tmp16no2 = (int16_t)(tmp32no1 >> 4); + tmp16no2 += tmp16no1; + zeros16 = WebRtcSpl_NormW16(tmp16no2); + if ((tmp16no2) & (-qDomainDiff > zeros16)) { + aecm->nearFilt[i] = WEBRTC_SPL_WORD16_MAX; + } else { + aecm->nearFilt[i] = qDomainDiff < 0 ? tmp16no2 << -qDomainDiff + : tmp16no2 >> qDomainDiff; + } + + // Wiener filter coefficients, resulting hnl in Q14 + if (echoEst32Gained == 0) + { + hnl[i] = ONE_Q14; + } else if (aecm->nearFilt[i] == 0) + { + hnl[i] = 0; + } else + { + // Multiply the suppression gain + // Rounding + echoEst32Gained += (uint32_t)(aecm->nearFilt[i] >> 1); + tmpU32 = WebRtcSpl_DivU32U16(echoEst32Gained, + (uint16_t)aecm->nearFilt[i]); + + // Current resolution is + // Q-(RESOLUTION_CHANNEL+RESOLUTION_SUPGAIN- max(0,17-zeros16- zeros32)) + // Make sure we are in Q14 + tmp32no1 = (int32_t)WEBRTC_SPL_SHIFT_W32(tmpU32, resolutionDiff); + if (tmp32no1 > ONE_Q14) + { + hnl[i] = 0; + } else if (tmp32no1 < 0) + { + hnl[i] = ONE_Q14; + } else + { + // 1-echoEst/dfa + hnl[i] = ONE_Q14 - (int16_t)tmp32no1; + if (hnl[i] < 0) + { + hnl[i] = 0; + } + } + } + if (hnl[i]) + { + numPosCoef++; + } + } + // Only in wideband. Prevent the gain in upper band from being larger than + // in lower band. + if (aecm->mult == 2) + { + // TODO(bjornv): Investigate if the scaling of hnl[i] below can cause + // speech distortion in double-talk. + for (i = 0; i < PART_LEN1; i++) + { + hnl[i] = (int16_t)((hnl[i] * hnl[i]) >> 14); + } + + for (i = kMinPrefBand; i <= kMaxPrefBand; i++) + { + avgHnl32 += (int32_t)hnl[i]; + } + RTC_DCHECK_GT(kMaxPrefBand - kMinPrefBand + 1, 0); + avgHnl32 /= (kMaxPrefBand - kMinPrefBand + 1); + + for (i = kMaxPrefBand; i < PART_LEN1; i++) + { + if (hnl[i] > (int16_t)avgHnl32) + { + hnl[i] = (int16_t)avgHnl32; + } + } + } + + // Calculate NLP gain, result is in Q14 + if (aecm->nlpFlag) + { + for (i = 0; i < PART_LEN1; i++) + { + // Truncate values close to zero and one. + if (hnl[i] > NLP_COMP_HIGH) + { + hnl[i] = ONE_Q14; + } else if (hnl[i] < NLP_COMP_LOW) + { + hnl[i] = 0; + } + + // Remove outliers + if (numPosCoef < 3) + { + nlpGain = 0; + } else + { + nlpGain = ONE_Q14; + } + + // NLP + if ((hnl[i] == ONE_Q14) && (nlpGain == ONE_Q14)) + { + hnl[i] = ONE_Q14; + } else + { + hnl[i] = (int16_t)((hnl[i] * nlpGain) >> 14); + } + + // multiply with Wiener coefficients + efw[i].real = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, + hnl[i], 14)); + efw[i].imag = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, + hnl[i], 14)); + } + } + else + { + // multiply with Wiener coefficients + for (i = 0; i < PART_LEN1; i++) + { + efw[i].real = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, + hnl[i], 14)); + efw[i].imag = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, + hnl[i], 14)); + } + } + + if (aecm->cngMode == AecmTrue) + { + ComfortNoise(aecm, ptrDfaClean, efw, hnl); + } + + InverseFFTAndWindow(aecm, fft, efw, output, nearendClean); + + return 0; +} + +static void ComfortNoise(AecmCore* aecm, + const uint16_t* dfa, + ComplexInt16* out, + const int16_t* lambda) { + int16_t i; + int16_t tmp16; + int32_t tmp32; + + int16_t randW16[PART_LEN]; + int16_t uReal[PART_LEN1]; + int16_t uImag[PART_LEN1]; + int32_t outLShift32; + int16_t noiseRShift16[PART_LEN1]; + + int16_t shiftFromNearToNoise = kNoiseEstQDomain - aecm->dfaCleanQDomain; + int16_t minTrackShift; + + RTC_DCHECK_GE(shiftFromNearToNoise, 0); + RTC_DCHECK_LT(shiftFromNearToNoise, 16); + + if (aecm->noiseEstCtr < 100) + { + // Track the minimum more quickly initially. + aecm->noiseEstCtr++; + minTrackShift = 6; + } else + { + minTrackShift = 9; + } + + // Estimate noise power. + for (i = 0; i < PART_LEN1; i++) + { + // Shift to the noise domain. + tmp32 = (int32_t)dfa[i]; + outLShift32 = tmp32 << shiftFromNearToNoise; + + if (outLShift32 < aecm->noiseEst[i]) + { + // Reset "too low" counter + aecm->noiseEstTooLowCtr[i] = 0; + // Track the minimum. + if (aecm->noiseEst[i] < (1 << minTrackShift)) + { + // For small values, decrease noiseEst[i] every + // |kNoiseEstIncCount| block. The regular approach below can not + // go further down due to truncation. + aecm->noiseEstTooHighCtr[i]++; + if (aecm->noiseEstTooHighCtr[i] >= kNoiseEstIncCount) + { + aecm->noiseEst[i]--; + aecm->noiseEstTooHighCtr[i] = 0; // Reset the counter + } + } + else + { + aecm->noiseEst[i] -= ((aecm->noiseEst[i] - outLShift32) + >> minTrackShift); + } + } else + { + // Reset "too high" counter + aecm->noiseEstTooHighCtr[i] = 0; + // Ramp slowly upwards until we hit the minimum again. + if ((aecm->noiseEst[i] >> 19) > 0) + { + // Avoid overflow. + // Multiplication with 2049 will cause wrap around. Scale + // down first and then multiply + aecm->noiseEst[i] >>= 11; + aecm->noiseEst[i] *= 2049; + } + else if ((aecm->noiseEst[i] >> 11) > 0) + { + // Large enough for relative increase + aecm->noiseEst[i] *= 2049; + aecm->noiseEst[i] >>= 11; + } + else + { + // Make incremental increases based on size every + // |kNoiseEstIncCount| block + aecm->noiseEstTooLowCtr[i]++; + if (aecm->noiseEstTooLowCtr[i] >= kNoiseEstIncCount) + { + aecm->noiseEst[i] += (aecm->noiseEst[i] >> 9) + 1; + aecm->noiseEstTooLowCtr[i] = 0; // Reset counter + } + } + } + } + + for (i = 0; i < PART_LEN1; i++) + { + tmp32 = aecm->noiseEst[i] >> shiftFromNearToNoise; + if (tmp32 > 32767) + { + tmp32 = 32767; + aecm->noiseEst[i] = tmp32 << shiftFromNearToNoise; + } + noiseRShift16[i] = (int16_t)tmp32; + + tmp16 = ONE_Q14 - lambda[i]; + noiseRShift16[i] = (int16_t)((tmp16 * noiseRShift16[i]) >> 14); + } + + // Generate a uniform random array on [0 2^15-1]. + WebRtcSpl_RandUArray(randW16, PART_LEN, &aecm->seed); + + // Generate noise according to estimated energy. + uReal[0] = 0; // Reject LF noise. + uImag[0] = 0; + for (i = 1; i < PART_LEN1; i++) + { + // Get a random index for the cos and sin tables over [0 359]. + tmp16 = (int16_t)((359 * randW16[i - 1]) >> 15); + + // Tables are in Q13. + uReal[i] = (int16_t)((noiseRShift16[i] * WebRtcAecm_kCosTable[tmp16]) >> + 13); + uImag[i] = (int16_t)((-noiseRShift16[i] * WebRtcAecm_kSinTable[tmp16]) >> + 13); + } + uImag[PART_LEN] = 0; + + for (i = 0; i < PART_LEN1; i++) + { + out[i].real = WebRtcSpl_AddSatW16(out[i].real, uReal[i]); + out[i].imag = WebRtcSpl_AddSatW16(out[i].imag, uImag[i]); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc new file mode 100644 index 0000000000..58e5ec5e35 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc @@ -0,0 +1,1566 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aecm/aecm_core.h" + +#include "modules/audio_processing/aecm/echo_control_mobile.h" +#include "modules/audio_processing/utility/delay_estimator_wrapper.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +static const ALIGN8_BEG int16_t WebRtcAecm_kSqrtHanning[] ALIGN8_END = { + 0, 399, 798, 1196, 1594, 1990, 2386, 2780, 3172, + 3562, 3951, 4337, 4720, 5101, 5478, 5853, 6224, + 6591, 6954, 7313, 7668, 8019, 8364, 8705, 9040, + 9370, 9695, 10013, 10326, 10633, 10933, 11227, 11514, + 11795, 12068, 12335, 12594, 12845, 13089, 13325, 13553, + 13773, 13985, 14189, 14384, 14571, 14749, 14918, 15079, + 15231, 15373, 15506, 15631, 15746, 15851, 15947, 16034, + 16111, 16179, 16237, 16286, 16325, 16354, 16373, 16384 +}; + +static const int16_t kNoiseEstQDomain = 15; +static const int16_t kNoiseEstIncCount = 5; + +static int16_t coefTable[] = { + 0, 4, 256, 260, 128, 132, 384, 388, + 64, 68, 320, 324, 192, 196, 448, 452, + 32, 36, 288, 292, 160, 164, 416, 420, + 96, 100, 352, 356, 224, 228, 480, 484, + 16, 20, 272, 276, 144, 148, 400, 404, + 80, 84, 336, 340, 208, 212, 464, 468, + 48, 52, 304, 308, 176, 180, 432, 436, + 112, 116, 368, 372, 240, 244, 496, 500, + 8, 12, 264, 268, 136, 140, 392, 396, + 72, 76, 328, 332, 200, 204, 456, 460, + 40, 44, 296, 300, 168, 172, 424, 428, + 104, 108, 360, 364, 232, 236, 488, 492, + 24, 28, 280, 284, 152, 156, 408, 412, + 88, 92, 344, 348, 216, 220, 472, 476, + 56, 60, 312, 316, 184, 188, 440, 444, + 120, 124, 376, 380, 248, 252, 504, 508 +}; + +static int16_t coefTable_ifft[] = { + 0, 512, 256, 508, 128, 252, 384, 380, + 64, 124, 320, 444, 192, 188, 448, 316, + 32, 60, 288, 476, 160, 220, 416, 348, + 96, 92, 352, 412, 224, 156, 480, 284, + 16, 28, 272, 492, 144, 236, 400, 364, + 80, 108, 336, 428, 208, 172, 464, 300, + 48, 44, 304, 460, 176, 204, 432, 332, + 112, 76, 368, 396, 240, 140, 496, 268, + 8, 12, 264, 500, 136, 244, 392, 372, + 72, 116, 328, 436, 200, 180, 456, 308, + 40, 52, 296, 468, 168, 212, 424, 340, + 104, 84, 360, 404, 232, 148, 488, 276, + 24, 20, 280, 484, 152, 228, 408, 356, + 88, 100, 344, 420, 216, 164, 472, 292, + 56, 36, 312, 452, 184, 196, 440, 324, + 120, 68, 376, 388, 248, 132, 504, 260 +}; + +static void ComfortNoise(AecmCore* aecm, + const uint16_t* dfa, + ComplexInt16* out, + const int16_t* lambda); + +static void WindowAndFFT(AecmCore* aecm, + int16_t* fft, + const int16_t* time_signal, + ComplexInt16* freq_signal, + int time_signal_scaling) { + int i, j; + int32_t tmp1, tmp2, tmp3, tmp4; + int16_t* pfrfi; + ComplexInt16* pfreq_signal; + int16_t f_coef, s_coef; + int32_t load_ptr, store_ptr1, store_ptr2, shift, shift1; + int32_t hann, hann1, coefs; + + memset(fft, 0, sizeof(int16_t) * PART_LEN4); + + // FFT of signal + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[shift], %[time_signal_scaling], -14 \n\t" + "addiu %[i], $zero, 64 \n\t" + "addiu %[load_ptr], %[time_signal], 0 \n\t" + "addiu %[hann], %[hanning], 0 \n\t" + "addiu %[hann1], %[hanning], 128 \n\t" + "addiu %[coefs], %[coefTable], 0 \n\t" + "bltz %[shift], 2f \n\t" + " negu %[shift1], %[shift] \n\t" + "1: \n\t" + "lh %[tmp1], 0(%[load_ptr]) \n\t" + "lh %[tmp2], 0(%[hann]) \n\t" + "lh %[tmp3], 128(%[load_ptr]) \n\t" + "lh %[tmp4], 0(%[hann1]) \n\t" + "addiu %[i], %[i], -1 \n\t" + "mul %[tmp1], %[tmp1], %[tmp2] \n\t" + "mul %[tmp3], %[tmp3], %[tmp4] \n\t" + "lh %[f_coef], 0(%[coefs]) \n\t" + "lh %[s_coef], 2(%[coefs]) \n\t" + "addiu %[load_ptr], %[load_ptr], 2 \n\t" + "addiu %[hann], %[hann], 2 \n\t" + "addiu %[hann1], %[hann1], -2 \n\t" + "addu %[store_ptr1], %[fft], %[f_coef] \n\t" + "addu %[store_ptr2], %[fft], %[s_coef] \n\t" + "sllv %[tmp1], %[tmp1], %[shift] \n\t" + "sllv %[tmp3], %[tmp3], %[shift] \n\t" + "sh %[tmp1], 0(%[store_ptr1]) \n\t" + "sh %[tmp3], 0(%[store_ptr2]) \n\t" + "bgtz %[i], 1b \n\t" + " addiu %[coefs], %[coefs], 4 \n\t" + "b 3f \n\t" + " nop \n\t" + "2: \n\t" + "lh %[tmp1], 0(%[load_ptr]) \n\t" + "lh %[tmp2], 0(%[hann]) \n\t" + "lh %[tmp3], 128(%[load_ptr]) \n\t" + "lh %[tmp4], 0(%[hann1]) \n\t" + "addiu %[i], %[i], -1 \n\t" + "mul %[tmp1], %[tmp1], %[tmp2] \n\t" + "mul %[tmp3], %[tmp3], %[tmp4] \n\t" + "lh %[f_coef], 0(%[coefs]) \n\t" + "lh %[s_coef], 2(%[coefs]) \n\t" + "addiu %[load_ptr], %[load_ptr], 2 \n\t" + "addiu %[hann], %[hann], 2 \n\t" + "addiu %[hann1], %[hann1], -2 \n\t" + "addu %[store_ptr1], %[fft], %[f_coef] \n\t" + "addu %[store_ptr2], %[fft], %[s_coef] \n\t" + "srav %[tmp1], %[tmp1], %[shift1] \n\t" + "srav %[tmp3], %[tmp3], %[shift1] \n\t" + "sh %[tmp1], 0(%[store_ptr1]) \n\t" + "sh %[tmp3], 0(%[store_ptr2]) \n\t" + "bgtz %[i], 2b \n\t" + " addiu %[coefs], %[coefs], 4 \n\t" + "3: \n\t" + ".set pop \n\t" + : [load_ptr] "=&r" (load_ptr), [shift] "=&r" (shift), [hann] "=&r" (hann), + [hann1] "=&r" (hann1), [shift1] "=&r" (shift1), [coefs] "=&r" (coefs), + [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [tmp3] "=&r" (tmp3), + [tmp4] "=&r" (tmp4), [i] "=&r" (i), [f_coef] "=&r" (f_coef), + [s_coef] "=&r" (s_coef), [store_ptr1] "=&r" (store_ptr1), + [store_ptr2] "=&r" (store_ptr2) + : [time_signal] "r" (time_signal), [coefTable] "r" (coefTable), + [time_signal_scaling] "r" (time_signal_scaling), + [hanning] "r" (WebRtcAecm_kSqrtHanning), [fft] "r" (fft) + : "memory", "hi", "lo" + ); + + WebRtcSpl_ComplexFFT(fft, PART_LEN_SHIFT, 1); + pfrfi = fft; + pfreq_signal = freq_signal; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[j], $zero, 128 \n\t" + "1: \n\t" + "lh %[tmp1], 0(%[pfrfi]) \n\t" + "lh %[tmp2], 2(%[pfrfi]) \n\t" + "lh %[tmp3], 4(%[pfrfi]) \n\t" + "lh %[tmp4], 6(%[pfrfi]) \n\t" + "subu %[tmp2], $zero, %[tmp2] \n\t" + "sh %[tmp1], 0(%[pfreq_signal]) \n\t" + "sh %[tmp2], 2(%[pfreq_signal]) \n\t" + "subu %[tmp4], $zero, %[tmp4] \n\t" + "sh %[tmp3], 4(%[pfreq_signal]) \n\t" + "sh %[tmp4], 6(%[pfreq_signal]) \n\t" + "lh %[tmp1], 8(%[pfrfi]) \n\t" + "lh %[tmp2], 10(%[pfrfi]) \n\t" + "lh %[tmp3], 12(%[pfrfi]) \n\t" + "lh %[tmp4], 14(%[pfrfi]) \n\t" + "addiu %[j], %[j], -8 \n\t" + "subu %[tmp2], $zero, %[tmp2] \n\t" + "sh %[tmp1], 8(%[pfreq_signal]) \n\t" + "sh %[tmp2], 10(%[pfreq_signal]) \n\t" + "subu %[tmp4], $zero, %[tmp4] \n\t" + "sh %[tmp3], 12(%[pfreq_signal]) \n\t" + "sh %[tmp4], 14(%[pfreq_signal]) \n\t" + "addiu %[pfreq_signal], %[pfreq_signal], 16 \n\t" + "bgtz %[j], 1b \n\t" + " addiu %[pfrfi], %[pfrfi], 16 \n\t" + ".set pop \n\t" + : [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [tmp3] "=&r" (tmp3), + [j] "=&r" (j), [pfrfi] "+r" (pfrfi), [pfreq_signal] "+r" (pfreq_signal), + [tmp4] "=&r" (tmp4) + : + : "memory" + ); +} + +static void InverseFFTAndWindow(AecmCore* aecm, + int16_t* fft, + ComplexInt16* efw, + int16_t* output, + const int16_t* nearendClean) { + int i, outCFFT; + int32_t tmp1, tmp2, tmp3, tmp4, tmp_re, tmp_im; + int16_t* pcoefTable_ifft = coefTable_ifft; + int16_t* pfft = fft; + int16_t* ppfft = fft; + ComplexInt16* pefw = efw; + int32_t out_aecm; + int16_t* paecm_buf = aecm->outBuf; + const int16_t* p_kSqrtHanning = WebRtcAecm_kSqrtHanning; + const int16_t* pp_kSqrtHanning = &WebRtcAecm_kSqrtHanning[PART_LEN]; + int16_t* output1 = output; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[i], $zero, 64 \n\t" + "1: \n\t" + "lh %[tmp1], 0(%[pcoefTable_ifft]) \n\t" + "lh %[tmp2], 2(%[pcoefTable_ifft]) \n\t" + "lh %[tmp_re], 0(%[pefw]) \n\t" + "lh %[tmp_im], 2(%[pefw]) \n\t" + "addu %[pfft], %[fft], %[tmp2] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "addu %[pfft], %[fft], %[tmp1] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "subu %[tmp_im], $zero, %[tmp_im] \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "lh %[tmp1], 4(%[pcoefTable_ifft]) \n\t" + "lh %[tmp2], 6(%[pcoefTable_ifft]) \n\t" + "lh %[tmp_re], 4(%[pefw]) \n\t" + "lh %[tmp_im], 6(%[pefw]) \n\t" + "addu %[pfft], %[fft], %[tmp2] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "addu %[pfft], %[fft], %[tmp1] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "subu %[tmp_im], $zero, %[tmp_im] \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "lh %[tmp1], 8(%[pcoefTable_ifft]) \n\t" + "lh %[tmp2], 10(%[pcoefTable_ifft]) \n\t" + "lh %[tmp_re], 8(%[pefw]) \n\t" + "lh %[tmp_im], 10(%[pefw]) \n\t" + "addu %[pfft], %[fft], %[tmp2] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "addu %[pfft], %[fft], %[tmp1] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "subu %[tmp_im], $zero, %[tmp_im] \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "lh %[tmp1], 12(%[pcoefTable_ifft]) \n\t" + "lh %[tmp2], 14(%[pcoefTable_ifft]) \n\t" + "lh %[tmp_re], 12(%[pefw]) \n\t" + "lh %[tmp_im], 14(%[pefw]) \n\t" + "addu %[pfft], %[fft], %[tmp2] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "addu %[pfft], %[fft], %[tmp1] \n\t" + "sh %[tmp_re], 0(%[pfft]) \n\t" + "subu %[tmp_im], $zero, %[tmp_im] \n\t" + "sh %[tmp_im], 2(%[pfft]) \n\t" + "addiu %[pcoefTable_ifft], %[pcoefTable_ifft], 16 \n\t" + "addiu %[i], %[i], -4 \n\t" + "bgtz %[i], 1b \n\t" + " addiu %[pefw], %[pefw], 16 \n\t" + ".set pop \n\t" + : [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [pfft] "+r" (pfft), + [i] "=&r" (i), [tmp_re] "=&r" (tmp_re), [tmp_im] "=&r" (tmp_im), + [pefw] "+r" (pefw), [pcoefTable_ifft] "+r" (pcoefTable_ifft), + [fft] "+r" (fft) + : + : "memory" + ); + + fft[2] = efw[PART_LEN].real; + fft[3] = -efw[PART_LEN].imag; + + outCFFT = WebRtcSpl_ComplexIFFT(fft, PART_LEN_SHIFT, 1); + pfft = fft; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[i], $zero, 128 \n\t" + "1: \n\t" + "lh %[tmp1], 0(%[ppfft]) \n\t" + "lh %[tmp2], 4(%[ppfft]) \n\t" + "lh %[tmp3], 8(%[ppfft]) \n\t" + "lh %[tmp4], 12(%[ppfft]) \n\t" + "addiu %[i], %[i], -4 \n\t" + "sh %[tmp1], 0(%[pfft]) \n\t" + "sh %[tmp2], 2(%[pfft]) \n\t" + "sh %[tmp3], 4(%[pfft]) \n\t" + "sh %[tmp4], 6(%[pfft]) \n\t" + "addiu %[ppfft], %[ppfft], 16 \n\t" + "bgtz %[i], 1b \n\t" + " addiu %[pfft], %[pfft], 8 \n\t" + ".set pop \n\t" + : [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [pfft] "+r" (pfft), + [i] "=&r" (i), [tmp3] "=&r" (tmp3), [tmp4] "=&r" (tmp4), + [ppfft] "+r" (ppfft) + : + : "memory" + ); + + pfft = fft; + out_aecm = (int32_t)(outCFFT - aecm->dfaCleanQDomain); + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[i], $zero, 64 \n\t" + "11: \n\t" + "lh %[tmp1], 0(%[pfft]) \n\t" + "lh %[tmp2], 0(%[p_kSqrtHanning]) \n\t" + "addiu %[i], %[i], -2 \n\t" + "mul %[tmp1], %[tmp1], %[tmp2] \n\t" + "lh %[tmp3], 2(%[pfft]) \n\t" + "lh %[tmp4], 2(%[p_kSqrtHanning]) \n\t" + "mul %[tmp3], %[tmp3], %[tmp4] \n\t" + "addiu %[tmp1], %[tmp1], 8192 \n\t" + "sra %[tmp1], %[tmp1], 14 \n\t" + "addiu %[tmp3], %[tmp3], 8192 \n\t" + "sra %[tmp3], %[tmp3], 14 \n\t" + "bgez %[out_aecm], 1f \n\t" + " negu %[tmp2], %[out_aecm] \n\t" + "srav %[tmp1], %[tmp1], %[tmp2] \n\t" + "b 2f \n\t" + " srav %[tmp3], %[tmp3], %[tmp2] \n\t" + "1: \n\t" + "sllv %[tmp1], %[tmp1], %[out_aecm] \n\t" + "sllv %[tmp3], %[tmp3], %[out_aecm] \n\t" + "2: \n\t" + "lh %[tmp4], 0(%[paecm_buf]) \n\t" + "lh %[tmp2], 2(%[paecm_buf]) \n\t" + "addu %[tmp3], %[tmp3], %[tmp2] \n\t" + "addu %[tmp1], %[tmp1], %[tmp4] \n\t" +#if defined(MIPS_DSP_R1_LE) + "shll_s.w %[tmp1], %[tmp1], 16 \n\t" + "sra %[tmp1], %[tmp1], 16 \n\t" + "shll_s.w %[tmp3], %[tmp3], 16 \n\t" + "sra %[tmp3], %[tmp3], 16 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "sra %[tmp4], %[tmp1], 31 \n\t" + "sra %[tmp2], %[tmp1], 15 \n\t" + "beq %[tmp4], %[tmp2], 3f \n\t" + " ori %[tmp2], $zero, 0x7fff \n\t" + "xor %[tmp1], %[tmp2], %[tmp4] \n\t" + "3: \n\t" + "sra %[tmp2], %[tmp3], 31 \n\t" + "sra %[tmp4], %[tmp3], 15 \n\t" + "beq %[tmp2], %[tmp4], 4f \n\t" + " ori %[tmp4], $zero, 0x7fff \n\t" + "xor %[tmp3], %[tmp4], %[tmp2] \n\t" + "4: \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sh %[tmp1], 0(%[pfft]) \n\t" + "sh %[tmp1], 0(%[output1]) \n\t" + "sh %[tmp3], 2(%[pfft]) \n\t" + "sh %[tmp3], 2(%[output1]) \n\t" + "lh %[tmp1], 128(%[pfft]) \n\t" + "lh %[tmp2], 0(%[pp_kSqrtHanning]) \n\t" + "mul %[tmp1], %[tmp1], %[tmp2] \n\t" + "lh %[tmp3], 130(%[pfft]) \n\t" + "lh %[tmp4], -2(%[pp_kSqrtHanning]) \n\t" + "mul %[tmp3], %[tmp3], %[tmp4] \n\t" + "sra %[tmp1], %[tmp1], 14 \n\t" + "sra %[tmp3], %[tmp3], 14 \n\t" + "bgez %[out_aecm], 5f \n\t" + " negu %[tmp2], %[out_aecm] \n\t" + "srav %[tmp3], %[tmp3], %[tmp2] \n\t" + "b 6f \n\t" + " srav %[tmp1], %[tmp1], %[tmp2] \n\t" + "5: \n\t" + "sllv %[tmp1], %[tmp1], %[out_aecm] \n\t" + "sllv %[tmp3], %[tmp3], %[out_aecm] \n\t" + "6: \n\t" +#if defined(MIPS_DSP_R1_LE) + "shll_s.w %[tmp1], %[tmp1], 16 \n\t" + "sra %[tmp1], %[tmp1], 16 \n\t" + "shll_s.w %[tmp3], %[tmp3], 16 \n\t" + "sra %[tmp3], %[tmp3], 16 \n\t" +#else // #if defined(MIPS_DSP_R1_LE) + "sra %[tmp4], %[tmp1], 31 \n\t" + "sra %[tmp2], %[tmp1], 15 \n\t" + "beq %[tmp4], %[tmp2], 7f \n\t" + " ori %[tmp2], $zero, 0x7fff \n\t" + "xor %[tmp1], %[tmp2], %[tmp4] \n\t" + "7: \n\t" + "sra %[tmp2], %[tmp3], 31 \n\t" + "sra %[tmp4], %[tmp3], 15 \n\t" + "beq %[tmp2], %[tmp4], 8f \n\t" + " ori %[tmp4], $zero, 0x7fff \n\t" + "xor %[tmp3], %[tmp4], %[tmp2] \n\t" + "8: \n\t" +#endif // #if defined(MIPS_DSP_R1_LE) + "sh %[tmp1], 0(%[paecm_buf]) \n\t" + "sh %[tmp3], 2(%[paecm_buf]) \n\t" + "addiu %[output1], %[output1], 4 \n\t" + "addiu %[paecm_buf], %[paecm_buf], 4 \n\t" + "addiu %[pfft], %[pfft], 4 \n\t" + "addiu %[p_kSqrtHanning], %[p_kSqrtHanning], 4 \n\t" + "bgtz %[i], 11b \n\t" + " addiu %[pp_kSqrtHanning], %[pp_kSqrtHanning], -4 \n\t" + ".set pop \n\t" + : [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [pfft] "+r" (pfft), + [output1] "+r" (output1), [tmp3] "=&r" (tmp3), [tmp4] "=&r" (tmp4), + [paecm_buf] "+r" (paecm_buf), [i] "=&r" (i), + [pp_kSqrtHanning] "+r" (pp_kSqrtHanning), + [p_kSqrtHanning] "+r" (p_kSqrtHanning) + : [out_aecm] "r" (out_aecm), + [WebRtcAecm_kSqrtHanning] "r" (WebRtcAecm_kSqrtHanning) + : "hi", "lo","memory" + ); + + // Copy the current block to the old position + // (aecm->outBuf is shifted elsewhere) + memcpy(aecm->xBuf, aecm->xBuf + PART_LEN, sizeof(int16_t) * PART_LEN); + memcpy(aecm->dBufNoisy, + aecm->dBufNoisy + PART_LEN, + sizeof(int16_t) * PART_LEN); + if (nearendClean != NULL) { + memcpy(aecm->dBufClean, + aecm->dBufClean + PART_LEN, + sizeof(int16_t) * PART_LEN); + } +} + +void WebRtcAecm_CalcLinearEnergies_mips(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est, + uint32_t* far_energy, + uint32_t* echo_energy_adapt, + uint32_t* echo_energy_stored) { + int i; + uint32_t par1 = (*far_energy); + uint32_t par2 = (*echo_energy_adapt); + uint32_t par3 = (*echo_energy_stored); + int16_t* ch_stored_p = &(aecm->channelStored[0]); + int16_t* ch_adapt_p = &(aecm->channelAdapt16[0]); + uint16_t* spectrum_p = (uint16_t*)(&(far_spectrum[0])); + int32_t* echo_p = &(echo_est[0]); + int32_t temp0, stored0, echo0, adept0, spectrum0; + int32_t stored1, adept1, spectrum1, echo1, temp1; + + // Get energy for the delayed far end signal and estimated + // echo using both stored and adapted channels. + for (i = 0; i < PART_LEN; i+= 4) { + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lh %[stored0], 0(%[ch_stored_p]) \n\t" + "lhu %[adept0], 0(%[ch_adapt_p]) \n\t" + "lhu %[spectrum0], 0(%[spectrum_p]) \n\t" + "lh %[stored1], 2(%[ch_stored_p]) \n\t" + "lhu %[adept1], 2(%[ch_adapt_p]) \n\t" + "lhu %[spectrum1], 2(%[spectrum_p]) \n\t" + "mul %[echo0], %[stored0], %[spectrum0] \n\t" + "mul %[temp0], %[adept0], %[spectrum0] \n\t" + "mul %[echo1], %[stored1], %[spectrum1] \n\t" + "mul %[temp1], %[adept1], %[spectrum1] \n\t" + "addu %[par1], %[par1], %[spectrum0] \n\t" + "addu %[par1], %[par1], %[spectrum1] \n\t" + "addiu %[echo_p], %[echo_p], 16 \n\t" + "addu %[par3], %[par3], %[echo0] \n\t" + "addu %[par2], %[par2], %[temp0] \n\t" + "addu %[par3], %[par3], %[echo1] \n\t" + "addu %[par2], %[par2], %[temp1] \n\t" + "usw %[echo0], -16(%[echo_p]) \n\t" + "usw %[echo1], -12(%[echo_p]) \n\t" + "lh %[stored0], 4(%[ch_stored_p]) \n\t" + "lhu %[adept0], 4(%[ch_adapt_p]) \n\t" + "lhu %[spectrum0], 4(%[spectrum_p]) \n\t" + "lh %[stored1], 6(%[ch_stored_p]) \n\t" + "lhu %[adept1], 6(%[ch_adapt_p]) \n\t" + "lhu %[spectrum1], 6(%[spectrum_p]) \n\t" + "mul %[echo0], %[stored0], %[spectrum0] \n\t" + "mul %[temp0], %[adept0], %[spectrum0] \n\t" + "mul %[echo1], %[stored1], %[spectrum1] \n\t" + "mul %[temp1], %[adept1], %[spectrum1] \n\t" + "addu %[par1], %[par1], %[spectrum0] \n\t" + "addu %[par1], %[par1], %[spectrum1] \n\t" + "addiu %[ch_stored_p], %[ch_stored_p], 8 \n\t" + "addiu %[ch_adapt_p], %[ch_adapt_p], 8 \n\t" + "addiu %[spectrum_p], %[spectrum_p], 8 \n\t" + "addu %[par3], %[par3], %[echo0] \n\t" + "addu %[par2], %[par2], %[temp0] \n\t" + "addu %[par3], %[par3], %[echo1] \n\t" + "addu %[par2], %[par2], %[temp1] \n\t" + "usw %[echo0], -8(%[echo_p]) \n\t" + "usw %[echo1], -4(%[echo_p]) \n\t" + ".set pop \n\t" + : [temp0] "=&r" (temp0), [stored0] "=&r" (stored0), + [adept0] "=&r" (adept0), [spectrum0] "=&r" (spectrum0), + [echo0] "=&r" (echo0), [echo_p] "+r" (echo_p), [par3] "+r" (par3), + [par1] "+r" (par1), [par2] "+r" (par2), [stored1] "=&r" (stored1), + [adept1] "=&r" (adept1), [echo1] "=&r" (echo1), + [spectrum1] "=&r" (spectrum1), [temp1] "=&r" (temp1), + [ch_stored_p] "+r" (ch_stored_p), [ch_adapt_p] "+r" (ch_adapt_p), + [spectrum_p] "+r" (spectrum_p) + : + : "hi", "lo", "memory" + ); + } + + echo_est[PART_LEN] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[PART_LEN], + far_spectrum[PART_LEN]); + par1 += (uint32_t)(far_spectrum[PART_LEN]); + par2 += aecm->channelAdapt16[PART_LEN] * far_spectrum[PART_LEN]; + par3 += (uint32_t)echo_est[PART_LEN]; + + (*far_energy) = par1; + (*echo_energy_adapt) = par2; + (*echo_energy_stored) = par3; +} + +#if defined(MIPS_DSP_R1_LE) +void WebRtcAecm_StoreAdaptiveChannel_mips(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est) { + int i; + int16_t* temp1; + uint16_t* temp8; + int32_t temp0, temp2, temp3, temp4, temp5, temp6; + int32_t* temp7 = &(echo_est[0]); + temp1 = &(aecm->channelStored[0]); + temp8 = (uint16_t*)(&far_spectrum[0]); + + // During startup we store the channel every block. + memcpy(aecm->channelStored, aecm->channelAdapt16, + sizeof(int16_t) * PART_LEN1); + // Recalculate echo estimate + for (i = 0; i < PART_LEN; i += 4) { + __asm __volatile ( + "ulw %[temp0], 0(%[temp8]) \n\t" + "ulw %[temp2], 0(%[temp1]) \n\t" + "ulw %[temp4], 4(%[temp8]) \n\t" + "ulw %[temp5], 4(%[temp1]) \n\t" + "muleq_s.w.phl %[temp3], %[temp2], %[temp0] \n\t" + "muleq_s.w.phr %[temp0], %[temp2], %[temp0] \n\t" + "muleq_s.w.phl %[temp6], %[temp5], %[temp4] \n\t" + "muleq_s.w.phr %[temp4], %[temp5], %[temp4] \n\t" + "addiu %[temp7], %[temp7], 16 \n\t" + "addiu %[temp1], %[temp1], 8 \n\t" + "addiu %[temp8], %[temp8], 8 \n\t" + "sra %[temp3], %[temp3], 1 \n\t" + "sra %[temp0], %[temp0], 1 \n\t" + "sra %[temp6], %[temp6], 1 \n\t" + "sra %[temp4], %[temp4], 1 \n\t" + "usw %[temp3], -12(%[temp7]) \n\t" + "usw %[temp0], -16(%[temp7]) \n\t" + "usw %[temp6], -4(%[temp7]) \n\t" + "usw %[temp4], -8(%[temp7]) \n\t" + : [temp0] "=&r" (temp0), [temp2] "=&r" (temp2), [temp3] "=&r" (temp3), + [temp4] "=&r" (temp4), [temp5] "=&r" (temp5), [temp6] "=&r" (temp6), + [temp1] "+r" (temp1), [temp8] "+r" (temp8), [temp7] "+r" (temp7) + : + : "hi", "lo", "memory" + ); + } + echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + far_spectrum[i]); +} + +void WebRtcAecm_ResetAdaptiveChannel_mips(AecmCore* aecm) { + int i; + int32_t* temp3; + int16_t* temp0; + int32_t temp1, temp2, temp4, temp5; + + temp0 = &(aecm->channelStored[0]); + temp3 = &(aecm->channelAdapt32[0]); + + // The stored channel has a significantly lower MSE than the adaptive one for + // two consecutive calculations. Reset the adaptive channel. + memcpy(aecm->channelAdapt16, + aecm->channelStored, + sizeof(int16_t) * PART_LEN1); + + // Restore the W32 channel + for (i = 0; i < PART_LEN; i += 4) { + __asm __volatile ( + "ulw %[temp1], 0(%[temp0]) \n\t" + "ulw %[temp4], 4(%[temp0]) \n\t" + "preceq.w.phl %[temp2], %[temp1] \n\t" + "preceq.w.phr %[temp1], %[temp1] \n\t" + "preceq.w.phl %[temp5], %[temp4] \n\t" + "preceq.w.phr %[temp4], %[temp4] \n\t" + "addiu %[temp0], %[temp0], 8 \n\t" + "usw %[temp2], 4(%[temp3]) \n\t" + "usw %[temp1], 0(%[temp3]) \n\t" + "usw %[temp5], 12(%[temp3]) \n\t" + "usw %[temp4], 8(%[temp3]) \n\t" + "addiu %[temp3], %[temp3], 16 \n\t" + : [temp1] "=&r" (temp1), [temp2] "=&r" (temp2), + [temp4] "=&r" (temp4), [temp5] "=&r" (temp5), + [temp3] "+r" (temp3), [temp0] "+r" (temp0) + : + : "memory" + ); + } + + aecm->channelAdapt32[i] = (int32_t)aecm->channelStored[i] << 16; +} +#endif // #if defined(MIPS_DSP_R1_LE) + +// Transforms a time domain signal into the frequency domain, outputting the +// complex valued signal, absolute value and sum of absolute values. +// +// time_signal [in] Pointer to time domain signal +// freq_signal_real [out] Pointer to real part of frequency domain array +// freq_signal_imag [out] Pointer to imaginary part of frequency domain +// array +// freq_signal_abs [out] Pointer to absolute value of frequency domain +// array +// freq_signal_sum_abs [out] Pointer to the sum of all absolute values in +// the frequency domain array +// return value The Q-domain of current frequency values +// +static int TimeToFrequencyDomain(AecmCore* aecm, + const int16_t* time_signal, + ComplexInt16* freq_signal, + uint16_t* freq_signal_abs, + uint32_t* freq_signal_sum_abs) { + int i = 0; + int time_signal_scaling = 0; + + // In fft_buf, +16 for 32-byte alignment. + int16_t fft_buf[PART_LEN4 + 16]; + int16_t *fft = (int16_t *) (((uintptr_t) fft_buf + 31) & ~31); + + int16_t tmp16no1; +#if !defined(MIPS_DSP_R2_LE) + int32_t tmp32no1; + int32_t tmp32no2; + int16_t tmp16no2; +#else + int32_t tmp32no10, tmp32no11, tmp32no12, tmp32no13; + int32_t tmp32no20, tmp32no21, tmp32no22, tmp32no23; + int16_t* freqp; + uint16_t* freqabsp; + uint32_t freqt0, freqt1, freqt2, freqt3; + uint32_t freqs; +#endif + +#ifdef AECM_DYNAMIC_Q + tmp16no1 = WebRtcSpl_MaxAbsValueW16(time_signal, PART_LEN2); + time_signal_scaling = WebRtcSpl_NormW16(tmp16no1); +#endif + + WindowAndFFT(aecm, fft, time_signal, freq_signal, time_signal_scaling); + + // Extract imaginary and real part, + // calculate the magnitude for all frequency bins + freq_signal[0].imag = 0; + freq_signal[PART_LEN].imag = 0; + freq_signal[PART_LEN].real = fft[PART_LEN2]; + freq_signal_abs[0] = (uint16_t)WEBRTC_SPL_ABS_W16(freq_signal[0].real); + freq_signal_abs[PART_LEN] = (uint16_t)WEBRTC_SPL_ABS_W16( + freq_signal[PART_LEN].real); + (*freq_signal_sum_abs) = (uint32_t)(freq_signal_abs[0]) + + (uint32_t)(freq_signal_abs[PART_LEN]); + +#if !defined(MIPS_DSP_R2_LE) + for (i = 1; i < PART_LEN; i++) { + if (freq_signal[i].real == 0) + { + freq_signal_abs[i] = (uint16_t)WEBRTC_SPL_ABS_W16( + freq_signal[i].imag); + } + else if (freq_signal[i].imag == 0) + { + freq_signal_abs[i] = (uint16_t)WEBRTC_SPL_ABS_W16( + freq_signal[i].real); + } + else + { + // Approximation for magnitude of complex fft output + // magn = sqrt(real^2 + imag^2) + // magn ~= alpha * max(|imag|,|real|) + beta * min(|imag|,|real|) + // + // The parameters alpha and beta are stored in Q15 + tmp16no1 = WEBRTC_SPL_ABS_W16(freq_signal[i].real); + tmp16no2 = WEBRTC_SPL_ABS_W16(freq_signal[i].imag); + tmp32no1 = tmp16no1 * tmp16no1; + tmp32no2 = tmp16no2 * tmp16no2; + tmp32no2 = WebRtcSpl_AddSatW32(tmp32no1, tmp32no2); + tmp32no1 = WebRtcSpl_SqrtFloor(tmp32no2); + + freq_signal_abs[i] = (uint16_t)tmp32no1; + } + (*freq_signal_sum_abs) += (uint32_t)freq_signal_abs[i]; + } +#else // #if !defined(MIPS_DSP_R2_LE) + freqs = (uint32_t)(freq_signal_abs[0]) + + (uint32_t)(freq_signal_abs[PART_LEN]); + freqp = &(freq_signal[1].real); + + __asm __volatile ( + "lw %[freqt0], 0(%[freqp]) \n\t" + "lw %[freqt1], 4(%[freqp]) \n\t" + "lw %[freqt2], 8(%[freqp]) \n\t" + "mult $ac0, $zero, $zero \n\t" + "mult $ac1, $zero, $zero \n\t" + "mult $ac2, $zero, $zero \n\t" + "dpaq_s.w.ph $ac0, %[freqt0], %[freqt0] \n\t" + "dpaq_s.w.ph $ac1, %[freqt1], %[freqt1] \n\t" + "dpaq_s.w.ph $ac2, %[freqt2], %[freqt2] \n\t" + "addiu %[freqp], %[freqp], 12 \n\t" + "extr.w %[tmp32no20], $ac0, 1 \n\t" + "extr.w %[tmp32no21], $ac1, 1 \n\t" + "extr.w %[tmp32no22], $ac2, 1 \n\t" + : [freqt0] "=&r" (freqt0), [freqt1] "=&r" (freqt1), + [freqt2] "=&r" (freqt2), [freqp] "+r" (freqp), + [tmp32no20] "=r" (tmp32no20), [tmp32no21] "=r" (tmp32no21), + [tmp32no22] "=r" (tmp32no22) + : + : "memory", "hi", "lo", "$ac1hi", "$ac1lo", "$ac2hi", "$ac2lo" + ); + + tmp32no10 = WebRtcSpl_SqrtFloor(tmp32no20); + tmp32no11 = WebRtcSpl_SqrtFloor(tmp32no21); + tmp32no12 = WebRtcSpl_SqrtFloor(tmp32no22); + freq_signal_abs[1] = (uint16_t)tmp32no10; + freq_signal_abs[2] = (uint16_t)tmp32no11; + freq_signal_abs[3] = (uint16_t)tmp32no12; + freqs += (uint32_t)tmp32no10; + freqs += (uint32_t)tmp32no11; + freqs += (uint32_t)tmp32no12; + freqabsp = &(freq_signal_abs[4]); + for (i = 4; i < PART_LEN; i+=4) + { + __asm __volatile ( + "ulw %[freqt0], 0(%[freqp]) \n\t" + "ulw %[freqt1], 4(%[freqp]) \n\t" + "ulw %[freqt2], 8(%[freqp]) \n\t" + "ulw %[freqt3], 12(%[freqp]) \n\t" + "mult $ac0, $zero, $zero \n\t" + "mult $ac1, $zero, $zero \n\t" + "mult $ac2, $zero, $zero \n\t" + "mult $ac3, $zero, $zero \n\t" + "dpaq_s.w.ph $ac0, %[freqt0], %[freqt0] \n\t" + "dpaq_s.w.ph $ac1, %[freqt1], %[freqt1] \n\t" + "dpaq_s.w.ph $ac2, %[freqt2], %[freqt2] \n\t" + "dpaq_s.w.ph $ac3, %[freqt3], %[freqt3] \n\t" + "addiu %[freqp], %[freqp], 16 \n\t" + "addiu %[freqabsp], %[freqabsp], 8 \n\t" + "extr.w %[tmp32no20], $ac0, 1 \n\t" + "extr.w %[tmp32no21], $ac1, 1 \n\t" + "extr.w %[tmp32no22], $ac2, 1 \n\t" + "extr.w %[tmp32no23], $ac3, 1 \n\t" + : [freqt0] "=&r" (freqt0), [freqt1] "=&r" (freqt1), + [freqt2] "=&r" (freqt2), [freqt3] "=&r" (freqt3), + [tmp32no20] "=r" (tmp32no20), [tmp32no21] "=r" (tmp32no21), + [tmp32no22] "=r" (tmp32no22), [tmp32no23] "=r" (tmp32no23), + [freqabsp] "+r" (freqabsp), [freqp] "+r" (freqp) + : + : "memory", "hi", "lo", "$ac1hi", "$ac1lo", + "$ac2hi", "$ac2lo", "$ac3hi", "$ac3lo" + ); + + tmp32no10 = WebRtcSpl_SqrtFloor(tmp32no20); + tmp32no11 = WebRtcSpl_SqrtFloor(tmp32no21); + tmp32no12 = WebRtcSpl_SqrtFloor(tmp32no22); + tmp32no13 = WebRtcSpl_SqrtFloor(tmp32no23); + + __asm __volatile ( + "sh %[tmp32no10], -8(%[freqabsp]) \n\t" + "sh %[tmp32no11], -6(%[freqabsp]) \n\t" + "sh %[tmp32no12], -4(%[freqabsp]) \n\t" + "sh %[tmp32no13], -2(%[freqabsp]) \n\t" + "addu %[freqs], %[freqs], %[tmp32no10] \n\t" + "addu %[freqs], %[freqs], %[tmp32no11] \n\t" + "addu %[freqs], %[freqs], %[tmp32no12] \n\t" + "addu %[freqs], %[freqs], %[tmp32no13] \n\t" + : [freqs] "+r" (freqs) + : [tmp32no10] "r" (tmp32no10), [tmp32no11] "r" (tmp32no11), + [tmp32no12] "r" (tmp32no12), [tmp32no13] "r" (tmp32no13), + [freqabsp] "r" (freqabsp) + : "memory" + ); + } + + (*freq_signal_sum_abs) = freqs; +#endif + + return time_signal_scaling; +} + +int WebRtcAecm_ProcessBlock(AecmCore* aecm, + const int16_t* farend, + const int16_t* nearendNoisy, + const int16_t* nearendClean, + int16_t* output) { + int i; + uint32_t xfaSum; + uint32_t dfaNoisySum; + uint32_t dfaCleanSum; + uint32_t echoEst32Gained; + uint32_t tmpU32; + int32_t tmp32no1; + + uint16_t xfa[PART_LEN1]; + uint16_t dfaNoisy[PART_LEN1]; + uint16_t dfaClean[PART_LEN1]; + uint16_t* ptrDfaClean = dfaClean; + const uint16_t* far_spectrum_ptr = NULL; + + // 32 byte aligned buffers (with +8 or +16). + int16_t fft_buf[PART_LEN4 + 2 + 16]; // +2 to make a loop safe. + int32_t echoEst32_buf[PART_LEN1 + 8]; + int32_t dfw_buf[PART_LEN2 + 8]; + int32_t efw_buf[PART_LEN2 + 8]; + + int16_t* fft = (int16_t*)(((uint32_t)fft_buf + 31) & ~ 31); + int32_t* echoEst32 = (int32_t*)(((uint32_t)echoEst32_buf + 31) & ~ 31); + ComplexInt16* dfw = (ComplexInt16*)(((uint32_t)dfw_buf + 31) & ~31); + ComplexInt16* efw = (ComplexInt16*)(((uint32_t)efw_buf + 31) & ~31); + + int16_t hnl[PART_LEN1]; + int16_t numPosCoef = 0; + int delay; + int16_t tmp16no1; + int16_t tmp16no2; + int16_t mu; + int16_t supGain; + int16_t zeros32, zeros16; + int16_t zerosDBufNoisy, zerosDBufClean, zerosXBuf; + int far_q; + int16_t resolutionDiff, qDomainDiff, dfa_clean_q_domain_diff; + + const int kMinPrefBand = 4; + const int kMaxPrefBand = 24; + int32_t avgHnl32 = 0; + + int32_t temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8; + int16_t* ptr; + int16_t* ptr1; + int16_t* er_ptr; + int16_t* dr_ptr; + + ptr = &hnl[0]; + ptr1 = &hnl[0]; + er_ptr = &efw[0].real; + dr_ptr = &dfw[0].real; + + // Determine startup state. There are three states: + // (0) the first CONV_LEN blocks + // (1) another CONV_LEN blocks + // (2) the rest + + if (aecm->startupState < 2) { + aecm->startupState = (aecm->totCount >= CONV_LEN) + + (aecm->totCount >= CONV_LEN2); + } + // END: Determine startup state + + // Buffer near and far end signals + memcpy(aecm->xBuf + PART_LEN, farend, sizeof(int16_t) * PART_LEN); + memcpy(aecm->dBufNoisy + PART_LEN, + nearendNoisy, + sizeof(int16_t) * PART_LEN); + if (nearendClean != NULL) { + memcpy(aecm->dBufClean + PART_LEN, + nearendClean, + sizeof(int16_t) * PART_LEN); + } + + // Transform far end signal from time domain to frequency domain. + far_q = TimeToFrequencyDomain(aecm, + aecm->xBuf, + dfw, + xfa, + &xfaSum); + + // Transform noisy near end signal from time domain to frequency domain. + zerosDBufNoisy = TimeToFrequencyDomain(aecm, + aecm->dBufNoisy, + dfw, + dfaNoisy, + &dfaNoisySum); + aecm->dfaNoisyQDomainOld = aecm->dfaNoisyQDomain; + aecm->dfaNoisyQDomain = (int16_t)zerosDBufNoisy; + + if (nearendClean == NULL) { + ptrDfaClean = dfaNoisy; + aecm->dfaCleanQDomainOld = aecm->dfaNoisyQDomainOld; + aecm->dfaCleanQDomain = aecm->dfaNoisyQDomain; + dfaCleanSum = dfaNoisySum; + } else { + // Transform clean near end signal from time domain to frequency domain. + zerosDBufClean = TimeToFrequencyDomain(aecm, + aecm->dBufClean, + dfw, + dfaClean, + &dfaCleanSum); + aecm->dfaCleanQDomainOld = aecm->dfaCleanQDomain; + aecm->dfaCleanQDomain = (int16_t)zerosDBufClean; + } + + // Get the delay + // Save far-end history and estimate delay + WebRtcAecm_UpdateFarHistory(aecm, xfa, far_q); + + if (WebRtc_AddFarSpectrumFix(aecm->delay_estimator_farend, xfa, PART_LEN1, + far_q) == -1) { + return -1; + } + delay = WebRtc_DelayEstimatorProcessFix(aecm->delay_estimator, + dfaNoisy, + PART_LEN1, + zerosDBufNoisy); + if (delay == -1) { + return -1; + } + else if (delay == -2) { + // If the delay is unknown, we assume zero. + // NOTE: this will have to be adjusted if we ever add lookahead. + delay = 0; + } + + if (aecm->fixedDelay >= 0) { + // Use fixed delay + delay = aecm->fixedDelay; + } + + // Get aligned far end spectrum + far_spectrum_ptr = WebRtcAecm_AlignedFarend(aecm, &far_q, delay); + zerosXBuf = (int16_t) far_q; + + if (far_spectrum_ptr == NULL) { + return -1; + } + + // Calculate log(energy) and update energy threshold levels + WebRtcAecm_CalcEnergies(aecm, + far_spectrum_ptr, + zerosXBuf, + dfaNoisySum, + echoEst32); + // Calculate stepsize + mu = WebRtcAecm_CalcStepSize(aecm); + + // Update counters + aecm->totCount++; + + // This is the channel estimation algorithm. + // It is base on NLMS but has a variable step length, + // which was calculated above. + WebRtcAecm_UpdateChannel(aecm, + far_spectrum_ptr, + zerosXBuf, + dfaNoisy, + mu, + echoEst32); + + supGain = WebRtcAecm_CalcSuppressionGain(aecm); + + // Calculate Wiener filter hnl[] + for (i = 0; i < PART_LEN1; i++) { + // Far end signal through channel estimate in Q8 + // How much can we shift right to preserve resolution + tmp32no1 = echoEst32[i] - aecm->echoFilt[i]; + aecm->echoFilt[i] += + rtc::dchecked_cast((int64_t{tmp32no1} * 50) >> 8); + + zeros32 = WebRtcSpl_NormW32(aecm->echoFilt[i]) + 1; + zeros16 = WebRtcSpl_NormW16(supGain) + 1; + if (zeros32 + zeros16 > 16) { + // Multiplication is safe + // Result in + // Q(RESOLUTION_CHANNEL+RESOLUTION_SUPGAIN+aecm->xfaQDomainBuf[diff]) + echoEst32Gained = WEBRTC_SPL_UMUL_32_16((uint32_t)aecm->echoFilt[i], + (uint16_t)supGain); + resolutionDiff = 14 - RESOLUTION_CHANNEL16 - RESOLUTION_SUPGAIN; + resolutionDiff += (aecm->dfaCleanQDomain - zerosXBuf); + } else { + tmp16no1 = 17 - zeros32 - zeros16; + resolutionDiff = 14 + tmp16no1 - RESOLUTION_CHANNEL16 - + RESOLUTION_SUPGAIN; + resolutionDiff += (aecm->dfaCleanQDomain - zerosXBuf); + if (zeros32 > tmp16no1) { + echoEst32Gained = WEBRTC_SPL_UMUL_32_16( + (uint32_t)aecm->echoFilt[i], + supGain >> tmp16no1); + } else { + // Result in Q-(RESOLUTION_CHANNEL+RESOLUTION_SUPGAIN-16) + echoEst32Gained = (aecm->echoFilt[i] >> tmp16no1) * supGain; + } + } + + zeros16 = WebRtcSpl_NormW16(aecm->nearFilt[i]); + RTC_DCHECK_GE(zeros16, 0); // |zeros16| is a norm, hence non-negative. + dfa_clean_q_domain_diff = aecm->dfaCleanQDomain - aecm->dfaCleanQDomainOld; + if (zeros16 < dfa_clean_q_domain_diff && aecm->nearFilt[i]) { + tmp16no1 = aecm->nearFilt[i] << zeros16; + qDomainDiff = zeros16 - dfa_clean_q_domain_diff; + tmp16no2 = ptrDfaClean[i] >> -qDomainDiff; + } else { + tmp16no1 = dfa_clean_q_domain_diff < 0 + ? aecm->nearFilt[i] >> -dfa_clean_q_domain_diff + : aecm->nearFilt[i] << dfa_clean_q_domain_diff; + qDomainDiff = 0; + tmp16no2 = ptrDfaClean[i]; + } + + tmp32no1 = (int32_t)(tmp16no2 - tmp16no1); + tmp16no2 = (int16_t)(tmp32no1 >> 4); + tmp16no2 += tmp16no1; + zeros16 = WebRtcSpl_NormW16(tmp16no2); + if ((tmp16no2) & (-qDomainDiff > zeros16)) { + aecm->nearFilt[i] = WEBRTC_SPL_WORD16_MAX; + } else { + aecm->nearFilt[i] = qDomainDiff < 0 ? tmp16no2 << -qDomainDiff + : tmp16no2 >> qDomainDiff; + } + + // Wiener filter coefficients, resulting hnl in Q14 + if (echoEst32Gained == 0) { + hnl[i] = ONE_Q14; + numPosCoef++; + } else if (aecm->nearFilt[i] == 0) { + hnl[i] = 0; + } else { + // Multiply the suppression gain + // Rounding + echoEst32Gained += (uint32_t)(aecm->nearFilt[i] >> 1); + tmpU32 = WebRtcSpl_DivU32U16(echoEst32Gained, + (uint16_t)aecm->nearFilt[i]); + + // Current resolution is + // Q-(RESOLUTION_CHANNEL + RESOLUTION_SUPGAIN + // - max(0, 17 - zeros16 - zeros32)) + // Make sure we are in Q14 + tmp32no1 = (int32_t)WEBRTC_SPL_SHIFT_W32(tmpU32, resolutionDiff); + if (tmp32no1 > ONE_Q14) { + hnl[i] = 0; + } else if (tmp32no1 < 0) { + hnl[i] = ONE_Q14; + numPosCoef++; + } else { + // 1-echoEst/dfa + hnl[i] = ONE_Q14 - (int16_t)tmp32no1; + if (hnl[i] <= 0) { + hnl[i] = 0; + } else { + numPosCoef++; + } + } + } + } + + // Only in wideband. Prevent the gain in upper band from being larger than + // in lower band. + if (aecm->mult == 2) { + // TODO(bjornv): Investigate if the scaling of hnl[i] below can cause + // speech distortion in double-talk. + for (i = 0; i < (PART_LEN1 >> 3); i++) { + __asm __volatile ( + "lh %[temp1], 0(%[ptr1]) \n\t" + "lh %[temp2], 2(%[ptr1]) \n\t" + "lh %[temp3], 4(%[ptr1]) \n\t" + "lh %[temp4], 6(%[ptr1]) \n\t" + "lh %[temp5], 8(%[ptr1]) \n\t" + "lh %[temp6], 10(%[ptr1]) \n\t" + "lh %[temp7], 12(%[ptr1]) \n\t" + "lh %[temp8], 14(%[ptr1]) \n\t" + "mul %[temp1], %[temp1], %[temp1] \n\t" + "mul %[temp2], %[temp2], %[temp2] \n\t" + "mul %[temp3], %[temp3], %[temp3] \n\t" + "mul %[temp4], %[temp4], %[temp4] \n\t" + "mul %[temp5], %[temp5], %[temp5] \n\t" + "mul %[temp6], %[temp6], %[temp6] \n\t" + "mul %[temp7], %[temp7], %[temp7] \n\t" + "mul %[temp8], %[temp8], %[temp8] \n\t" + "sra %[temp1], %[temp1], 14 \n\t" + "sra %[temp2], %[temp2], 14 \n\t" + "sra %[temp3], %[temp3], 14 \n\t" + "sra %[temp4], %[temp4], 14 \n\t" + "sra %[temp5], %[temp5], 14 \n\t" + "sra %[temp6], %[temp6], 14 \n\t" + "sra %[temp7], %[temp7], 14 \n\t" + "sra %[temp8], %[temp8], 14 \n\t" + "sh %[temp1], 0(%[ptr1]) \n\t" + "sh %[temp2], 2(%[ptr1]) \n\t" + "sh %[temp3], 4(%[ptr1]) \n\t" + "sh %[temp4], 6(%[ptr1]) \n\t" + "sh %[temp5], 8(%[ptr1]) \n\t" + "sh %[temp6], 10(%[ptr1]) \n\t" + "sh %[temp7], 12(%[ptr1]) \n\t" + "sh %[temp8], 14(%[ptr1]) \n\t" + "addiu %[ptr1], %[ptr1], 16 \n\t" + : [temp1] "=&r" (temp1), [temp2] "=&r" (temp2), [temp3] "=&r" (temp3), + [temp4] "=&r" (temp4), [temp5] "=&r" (temp5), [temp6] "=&r" (temp6), + [temp7] "=&r" (temp7), [temp8] "=&r" (temp8), [ptr1] "+r" (ptr1) + : + : "memory", "hi", "lo" + ); + } + for(i = 0; i < (PART_LEN1 & 7); i++) { + __asm __volatile ( + "lh %[temp1], 0(%[ptr1]) \n\t" + "mul %[temp1], %[temp1], %[temp1] \n\t" + "sra %[temp1], %[temp1], 14 \n\t" + "sh %[temp1], 0(%[ptr1]) \n\t" + "addiu %[ptr1], %[ptr1], 2 \n\t" + : [temp1] "=&r" (temp1), [ptr1] "+r" (ptr1) + : + : "memory", "hi", "lo" + ); + } + + for (i = kMinPrefBand; i <= kMaxPrefBand; i++) { + avgHnl32 += (int32_t)hnl[i]; + } + + RTC_DCHECK_GT(kMaxPrefBand - kMinPrefBand + 1, 0); + avgHnl32 /= (kMaxPrefBand - kMinPrefBand + 1); + + for (i = kMaxPrefBand; i < PART_LEN1; i++) { + if (hnl[i] > (int16_t)avgHnl32) { + hnl[i] = (int16_t)avgHnl32; + } + } + } + + // Calculate NLP gain, result is in Q14 + if (aecm->nlpFlag) { + if (numPosCoef < 3) { + for (i = 0; i < PART_LEN1; i++) { + efw[i].real = 0; + efw[i].imag = 0; + hnl[i] = 0; + } + } else { + for (i = 0; i < PART_LEN1; i++) { +#if defined(MIPS_DSP_R1_LE) + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lh %[temp1], 0(%[ptr]) \n\t" + "lh %[temp2], 0(%[dr_ptr]) \n\t" + "slti %[temp4], %[temp1], 0x4001 \n\t" + "beqz %[temp4], 3f \n\t" + " lh %[temp3], 2(%[dr_ptr]) \n\t" + "slti %[temp5], %[temp1], 3277 \n\t" + "bnez %[temp5], 2f \n\t" + " addiu %[dr_ptr], %[dr_ptr], 4 \n\t" + "mul %[temp2], %[temp2], %[temp1] \n\t" + "mul %[temp3], %[temp3], %[temp1] \n\t" + "shra_r.w %[temp2], %[temp2], 14 \n\t" + "shra_r.w %[temp3], %[temp3], 14 \n\t" + "b 4f \n\t" + " nop \n\t" + "2: \n\t" + "addu %[temp1], $zero, $zero \n\t" + "addu %[temp2], $zero, $zero \n\t" + "addu %[temp3], $zero, $zero \n\t" + "b 1f \n\t" + " nop \n\t" + "3: \n\t" + "addiu %[temp1], $0, 0x4000 \n\t" + "1: \n\t" + "sh %[temp1], 0(%[ptr]) \n\t" + "4: \n\t" + "sh %[temp2], 0(%[er_ptr]) \n\t" + "sh %[temp3], 2(%[er_ptr]) \n\t" + "addiu %[ptr], %[ptr], 2 \n\t" + "addiu %[er_ptr], %[er_ptr], 4 \n\t" + ".set pop \n\t" + : [temp1] "=&r" (temp1), [temp2] "=&r" (temp2), [temp3] "=&r" (temp3), + [temp4] "=&r" (temp4), [temp5] "=&r" (temp5), [ptr] "+r" (ptr), + [er_ptr] "+r" (er_ptr), [dr_ptr] "+r" (dr_ptr) + : + : "memory", "hi", "lo" + ); +#else + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lh %[temp1], 0(%[ptr]) \n\t" + "lh %[temp2], 0(%[dr_ptr]) \n\t" + "slti %[temp4], %[temp1], 0x4001 \n\t" + "beqz %[temp4], 3f \n\t" + " lh %[temp3], 2(%[dr_ptr]) \n\t" + "slti %[temp5], %[temp1], 3277 \n\t" + "bnez %[temp5], 2f \n\t" + " addiu %[dr_ptr], %[dr_ptr], 4 \n\t" + "mul %[temp2], %[temp2], %[temp1] \n\t" + "mul %[temp3], %[temp3], %[temp1] \n\t" + "addiu %[temp2], %[temp2], 0x2000 \n\t" + "addiu %[temp3], %[temp3], 0x2000 \n\t" + "sra %[temp2], %[temp2], 14 \n\t" + "sra %[temp3], %[temp3], 14 \n\t" + "b 4f \n\t" + " nop \n\t" + "2: \n\t" + "addu %[temp1], $zero, $zero \n\t" + "addu %[temp2], $zero, $zero \n\t" + "addu %[temp3], $zero, $zero \n\t" + "b 1f \n\t" + " nop \n\t" + "3: \n\t" + "addiu %[temp1], $0, 0x4000 \n\t" + "1: \n\t" + "sh %[temp1], 0(%[ptr]) \n\t" + "4: \n\t" + "sh %[temp2], 0(%[er_ptr]) \n\t" + "sh %[temp3], 2(%[er_ptr]) \n\t" + "addiu %[ptr], %[ptr], 2 \n\t" + "addiu %[er_ptr], %[er_ptr], 4 \n\t" + ".set pop \n\t" + : [temp1] "=&r" (temp1), [temp2] "=&r" (temp2), [temp3] "=&r" (temp3), + [temp4] "=&r" (temp4), [temp5] "=&r" (temp5), [ptr] "+r" (ptr), + [er_ptr] "+r" (er_ptr), [dr_ptr] "+r" (dr_ptr) + : + : "memory", "hi", "lo" + ); +#endif + } + } + } + else { + // multiply with Wiener coefficients + for (i = 0; i < PART_LEN1; i++) { + efw[i].real = (int16_t) + (WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, + hnl[i], + 14)); + efw[i].imag = (int16_t) + (WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, + hnl[i], + 14)); + } + } + + if (aecm->cngMode == AecmTrue) { + ComfortNoise(aecm, ptrDfaClean, efw, hnl); + } + + InverseFFTAndWindow(aecm, fft, efw, output, nearendClean); + + return 0; +} + +// Generate comfort noise and add to output signal. +static void ComfortNoise(AecmCore* aecm, + const uint16_t* dfa, + ComplexInt16* out, + const int16_t* lambda) { + int16_t i; + int16_t tmp16, tmp161, tmp162, tmp163, nrsh1, nrsh2; + int32_t tmp32, tmp321, tnoise, tnoise1; + int32_t tmp322, tmp323, *tmp1; + int16_t* dfap; + int16_t* lambdap; + const int32_t c2049 = 2049; + const int32_t c359 = 359; + const int32_t c114 = ONE_Q14; + + int16_t randW16[PART_LEN]; + int16_t uReal[PART_LEN1]; + int16_t uImag[PART_LEN1]; + int32_t outLShift32; + + int16_t shiftFromNearToNoise = kNoiseEstQDomain - aecm->dfaCleanQDomain; + int16_t minTrackShift = 9; + + RTC_DCHECK_GE(shiftFromNearToNoise, 0); + RTC_DCHECK_LT(shiftFromNearToNoise, 16); + + if (aecm->noiseEstCtr < 100) { + // Track the minimum more quickly initially. + aecm->noiseEstCtr++; + minTrackShift = 6; + } + + // Generate a uniform random array on [0 2^15-1]. + WebRtcSpl_RandUArray(randW16, PART_LEN, &aecm->seed); + int16_t* randW16p = (int16_t*)randW16; +#if defined (MIPS_DSP_R1_LE) + int16_t* kCosTablep = (int16_t*)WebRtcAecm_kCosTable; + int16_t* kSinTablep = (int16_t*)WebRtcAecm_kSinTable; +#endif // #if defined(MIPS_DSP_R1_LE) + tmp1 = (int32_t*)aecm->noiseEst + 1; + dfap = (int16_t*)dfa + 1; + lambdap = (int16_t*)lambda + 1; + // Estimate noise power. + for (i = 1; i < PART_LEN1; i+=2) { + // Shift to the noise domain. + __asm __volatile ( + "lh %[tmp32], 0(%[dfap]) \n\t" + "lw %[tnoise], 0(%[tmp1]) \n\t" + "sllv %[outLShift32], %[tmp32], %[shiftFromNearToNoise] \n\t" + : [tmp32] "=&r" (tmp32), [outLShift32] "=r" (outLShift32), + [tnoise] "=&r" (tnoise) + : [tmp1] "r" (tmp1), [dfap] "r" (dfap), + [shiftFromNearToNoise] "r" (shiftFromNearToNoise) + : "memory" + ); + + if (outLShift32 < tnoise) { + // Reset "too low" counter + aecm->noiseEstTooLowCtr[i] = 0; + // Track the minimum. + if (tnoise < (1 << minTrackShift)) { + // For small values, decrease noiseEst[i] every + // |kNoiseEstIncCount| block. The regular approach below can not + // go further down due to truncation. + aecm->noiseEstTooHighCtr[i]++; + if (aecm->noiseEstTooHighCtr[i] >= kNoiseEstIncCount) { + tnoise--; + aecm->noiseEstTooHighCtr[i] = 0; // Reset the counter + } + } else { + __asm __volatile ( + "subu %[tmp32], %[tnoise], %[outLShift32] \n\t" + "srav %[tmp32], %[tmp32], %[minTrackShift] \n\t" + "subu %[tnoise], %[tnoise], %[tmp32] \n\t" + : [tmp32] "=&r" (tmp32), [tnoise] "+r" (tnoise) + : [outLShift32] "r" (outLShift32), [minTrackShift] "r" (minTrackShift) + ); + } + } else { + // Reset "too high" counter + aecm->noiseEstTooHighCtr[i] = 0; + // Ramp slowly upwards until we hit the minimum again. + if ((tnoise >> 19) <= 0) { + if ((tnoise >> 11) > 0) { + // Large enough for relative increase + __asm __volatile ( + "mul %[tnoise], %[tnoise], %[c2049] \n\t" + "sra %[tnoise], %[tnoise], 11 \n\t" + : [tnoise] "+r" (tnoise) + : [c2049] "r" (c2049) + : "hi", "lo" + ); + } else { + // Make incremental increases based on size every + // |kNoiseEstIncCount| block + aecm->noiseEstTooLowCtr[i]++; + if (aecm->noiseEstTooLowCtr[i] >= kNoiseEstIncCount) { + __asm __volatile ( + "sra %[tmp32], %[tnoise], 9 \n\t" + "addi %[tnoise], %[tnoise], 1 \n\t" + "addu %[tnoise], %[tnoise], %[tmp32] \n\t" + : [tnoise] "+r" (tnoise), [tmp32] "=&r" (tmp32) + : + ); + aecm->noiseEstTooLowCtr[i] = 0; // Reset counter + } + } + } else { + // Avoid overflow. + // Multiplication with 2049 will cause wrap around. Scale + // down first and then multiply + __asm __volatile ( + "sra %[tnoise], %[tnoise], 11 \n\t" + "mul %[tnoise], %[tnoise], %[c2049] \n\t" + : [tnoise] "+r" (tnoise) + : [c2049] "r" (c2049) + : "hi", "lo" + ); + } + } + + // Shift to the noise domain. + __asm __volatile ( + "lh %[tmp32], 2(%[dfap]) \n\t" + "lw %[tnoise1], 4(%[tmp1]) \n\t" + "addiu %[dfap], %[dfap], 4 \n\t" + "sllv %[outLShift32], %[tmp32], %[shiftFromNearToNoise] \n\t" + : [tmp32] "=&r" (tmp32), [dfap] "+r" (dfap), + [outLShift32] "=r" (outLShift32), [tnoise1] "=&r" (tnoise1) + : [tmp1] "r" (tmp1), [shiftFromNearToNoise] "r" (shiftFromNearToNoise) + : "memory" + ); + + if (outLShift32 < tnoise1) { + // Reset "too low" counter + aecm->noiseEstTooLowCtr[i + 1] = 0; + // Track the minimum. + if (tnoise1 < (1 << minTrackShift)) { + // For small values, decrease noiseEst[i] every + // |kNoiseEstIncCount| block. The regular approach below can not + // go further down due to truncation. + aecm->noiseEstTooHighCtr[i + 1]++; + if (aecm->noiseEstTooHighCtr[i + 1] >= kNoiseEstIncCount) { + tnoise1--; + aecm->noiseEstTooHighCtr[i + 1] = 0; // Reset the counter + } + } else { + __asm __volatile ( + "subu %[tmp32], %[tnoise1], %[outLShift32] \n\t" + "srav %[tmp32], %[tmp32], %[minTrackShift] \n\t" + "subu %[tnoise1], %[tnoise1], %[tmp32] \n\t" + : [tmp32] "=&r" (tmp32), [tnoise1] "+r" (tnoise1) + : [outLShift32] "r" (outLShift32), [minTrackShift] "r" (minTrackShift) + ); + } + } else { + // Reset "too high" counter + aecm->noiseEstTooHighCtr[i + 1] = 0; + // Ramp slowly upwards until we hit the minimum again. + if ((tnoise1 >> 19) <= 0) { + if ((tnoise1 >> 11) > 0) { + // Large enough for relative increase + __asm __volatile ( + "mul %[tnoise1], %[tnoise1], %[c2049] \n\t" + "sra %[tnoise1], %[tnoise1], 11 \n\t" + : [tnoise1] "+r" (tnoise1) + : [c2049] "r" (c2049) + : "hi", "lo" + ); + } else { + // Make incremental increases based on size every + // |kNoiseEstIncCount| block + aecm->noiseEstTooLowCtr[i + 1]++; + if (aecm->noiseEstTooLowCtr[i + 1] >= kNoiseEstIncCount) { + __asm __volatile ( + "sra %[tmp32], %[tnoise1], 9 \n\t" + "addi %[tnoise1], %[tnoise1], 1 \n\t" + "addu %[tnoise1], %[tnoise1], %[tmp32] \n\t" + : [tnoise1] "+r" (tnoise1), [tmp32] "=&r" (tmp32) + : + ); + aecm->noiseEstTooLowCtr[i + 1] = 0; // Reset counter + } + } + } else { + // Avoid overflow. + // Multiplication with 2049 will cause wrap around. Scale + // down first and then multiply + __asm __volatile ( + "sra %[tnoise1], %[tnoise1], 11 \n\t" + "mul %[tnoise1], %[tnoise1], %[c2049] \n\t" + : [tnoise1] "+r" (tnoise1) + : [c2049] "r" (c2049) + : "hi", "lo" + ); + } + } + + __asm __volatile ( + "lh %[tmp16], 0(%[lambdap]) \n\t" + "lh %[tmp161], 2(%[lambdap]) \n\t" + "sw %[tnoise], 0(%[tmp1]) \n\t" + "sw %[tnoise1], 4(%[tmp1]) \n\t" + "subu %[tmp16], %[c114], %[tmp16] \n\t" + "subu %[tmp161], %[c114], %[tmp161] \n\t" + "srav %[tmp32], %[tnoise], %[shiftFromNearToNoise] \n\t" + "srav %[tmp321], %[tnoise1], %[shiftFromNearToNoise] \n\t" + "addiu %[lambdap], %[lambdap], 4 \n\t" + "addiu %[tmp1], %[tmp1], 8 \n\t" + : [tmp16] "=&r" (tmp16), [tmp161] "=&r" (tmp161), [tmp1] "+r" (tmp1), + [tmp32] "=&r" (tmp32), [tmp321] "=&r" (tmp321), [lambdap] "+r" (lambdap) + : [tnoise] "r" (tnoise), [tnoise1] "r" (tnoise1), [c114] "r" (c114), + [shiftFromNearToNoise] "r" (shiftFromNearToNoise) + : "memory" + ); + + if (tmp32 > 32767) { + tmp32 = 32767; + aecm->noiseEst[i] = tmp32 << shiftFromNearToNoise; + } + if (tmp321 > 32767) { + tmp321 = 32767; + aecm->noiseEst[i+1] = tmp321 << shiftFromNearToNoise; + } + + __asm __volatile ( + "mul %[tmp32], %[tmp32], %[tmp16] \n\t" + "mul %[tmp321], %[tmp321], %[tmp161] \n\t" + "sra %[nrsh1], %[tmp32], 14 \n\t" + "sra %[nrsh2], %[tmp321], 14 \n\t" + : [nrsh1] "=&r" (nrsh1), [nrsh2] "=r" (nrsh2) + : [tmp16] "r" (tmp16), [tmp161] "r" (tmp161), [tmp32] "r" (tmp32), + [tmp321] "r" (tmp321) + : "memory", "hi", "lo" + ); + + __asm __volatile ( + "lh %[tmp32], 0(%[randW16p]) \n\t" + "lh %[tmp321], 2(%[randW16p]) \n\t" + "addiu %[randW16p], %[randW16p], 4 \n\t" + "mul %[tmp32], %[tmp32], %[c359] \n\t" + "mul %[tmp321], %[tmp321], %[c359] \n\t" + "sra %[tmp16], %[tmp32], 15 \n\t" + "sra %[tmp161], %[tmp321], 15 \n\t" + : [randW16p] "+r" (randW16p), [tmp32] "=&r" (tmp32), + [tmp16] "=r" (tmp16), [tmp161] "=r" (tmp161), [tmp321] "=&r" (tmp321) + : [c359] "r" (c359) + : "memory", "hi", "lo" + ); + +#if !defined(MIPS_DSP_R1_LE) + tmp32 = WebRtcAecm_kCosTable[tmp16]; + tmp321 = WebRtcAecm_kSinTable[tmp16]; + tmp322 = WebRtcAecm_kCosTable[tmp161]; + tmp323 = WebRtcAecm_kSinTable[tmp161]; +#else + __asm __volatile ( + "sll %[tmp16], %[tmp16], 1 \n\t" + "sll %[tmp161], %[tmp161], 1 \n\t" + "lhx %[tmp32], %[tmp16](%[kCosTablep]) \n\t" + "lhx %[tmp321], %[tmp16](%[kSinTablep]) \n\t" + "lhx %[tmp322], %[tmp161](%[kCosTablep]) \n\t" + "lhx %[tmp323], %[tmp161](%[kSinTablep]) \n\t" + : [tmp32] "=&r" (tmp32), [tmp321] "=&r" (tmp321), + [tmp322] "=&r" (tmp322), [tmp323] "=&r" (tmp323) + : [kCosTablep] "r" (kCosTablep), [tmp16] "r" (tmp16), + [tmp161] "r" (tmp161), [kSinTablep] "r" (kSinTablep) + : "memory" + ); +#endif + __asm __volatile ( + "mul %[tmp32], %[tmp32], %[nrsh1] \n\t" + "negu %[tmp162], %[nrsh1] \n\t" + "mul %[tmp322], %[tmp322], %[nrsh2] \n\t" + "negu %[tmp163], %[nrsh2] \n\t" + "sra %[tmp32], %[tmp32], 13 \n\t" + "mul %[tmp321], %[tmp321], %[tmp162] \n\t" + "sra %[tmp322], %[tmp322], 13 \n\t" + "mul %[tmp323], %[tmp323], %[tmp163] \n\t" + "sra %[tmp321], %[tmp321], 13 \n\t" + "sra %[tmp323], %[tmp323], 13 \n\t" + : [tmp32] "+r" (tmp32), [tmp321] "+r" (tmp321), [tmp162] "=&r" (tmp162), + [tmp322] "+r" (tmp322), [tmp323] "+r" (tmp323), [tmp163] "=&r" (tmp163) + : [nrsh1] "r" (nrsh1), [nrsh2] "r" (nrsh2) + : "hi", "lo" + ); + // Tables are in Q13. + uReal[i] = (int16_t)tmp32; + uImag[i] = (int16_t)tmp321; + uReal[i + 1] = (int16_t)tmp322; + uImag[i + 1] = (int16_t)tmp323; + } + + int32_t tt, sgn; + tt = out[0].real; + sgn = ((int)tt) >> 31; + out[0].real = sgn == (int16_t)(tt >> 15) ? (int16_t)tt : (16384 ^ sgn); + tt = out[0].imag; + sgn = ((int)tt) >> 31; + out[0].imag = sgn == (int16_t)(tt >> 15) ? (int16_t)tt : (16384 ^ sgn); + for (i = 1; i < PART_LEN; i++) { + tt = out[i].real + uReal[i]; + sgn = ((int)tt) >> 31; + out[i].real = sgn == (int16_t)(tt >> 15) ? (int16_t)tt : (16384 ^ sgn); + tt = out[i].imag + uImag[i]; + sgn = ((int)tt) >> 31; + out[i].imag = sgn == (int16_t)(tt >> 15) ? (int16_t)tt : (16384 ^ sgn); + } + tt = out[PART_LEN].real + uReal[PART_LEN]; + sgn = ((int)tt) >> 31; + out[PART_LEN].real = sgn == (int16_t)(tt >> 15) ? (int16_t)tt : (16384 ^ sgn); + tt = out[PART_LEN].imag; + sgn = ((int)tt) >> 31; + out[PART_LEN].imag = sgn == (int16_t)(tt >> 15) ? (int16_t)tt : (16384 ^ sgn); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_neon.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_neon.cc new file mode 100644 index 0000000000..ca7211f738 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_neon.cc @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aecm/aecm_core.h" + +#include + +#include "common_audio/signal_processing/include/real_fft.h" +#include "rtc_base/checks.h" + +// TODO(kma): Re-write the corresponding assembly file, the offset +// generating script and makefile, to replace these C functions. + +static inline void AddLanes(uint32_t* ptr, uint32x4_t v) { +#if defined(WEBRTC_ARCH_ARM64) + *(ptr) = vaddvq_u32(v); +#else + uint32x2_t tmp_v; + tmp_v = vadd_u32(vget_low_u32(v), vget_high_u32(v)); + tmp_v = vpadd_u32(tmp_v, tmp_v); + *(ptr) = vget_lane_u32(tmp_v, 0); +#endif +} + +void WebRtcAecm_CalcLinearEnergiesNeon(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est, + uint32_t* far_energy, + uint32_t* echo_energy_adapt, + uint32_t* echo_energy_stored) { + int16_t* start_stored_p = aecm->channelStored; + int16_t* start_adapt_p = aecm->channelAdapt16; + int32_t* echo_est_p = echo_est; + const int16_t* end_stored_p = aecm->channelStored + PART_LEN; + const uint16_t* far_spectrum_p = far_spectrum; + int16x8_t store_v, adapt_v; + uint16x8_t spectrum_v; + uint32x4_t echo_est_v_low, echo_est_v_high; + uint32x4_t far_energy_v, echo_stored_v, echo_adapt_v; + + far_energy_v = vdupq_n_u32(0); + echo_adapt_v = vdupq_n_u32(0); + echo_stored_v = vdupq_n_u32(0); + + // Get energy for the delayed far end signal and estimated + // echo using both stored and adapted channels. + // The C code: + // for (i = 0; i < PART_LEN1; i++) { + // echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + // far_spectrum[i]); + // (*far_energy) += (uint32_t)(far_spectrum[i]); + // *echo_energy_adapt += aecm->channelAdapt16[i] * far_spectrum[i]; + // (*echo_energy_stored) += (uint32_t)echo_est[i]; + // } + while (start_stored_p < end_stored_p) { + spectrum_v = vld1q_u16(far_spectrum_p); + adapt_v = vld1q_s16(start_adapt_p); + store_v = vld1q_s16(start_stored_p); + + far_energy_v = vaddw_u16(far_energy_v, vget_low_u16(spectrum_v)); + far_energy_v = vaddw_u16(far_energy_v, vget_high_u16(spectrum_v)); + + echo_est_v_low = vmull_u16(vreinterpret_u16_s16(vget_low_s16(store_v)), + vget_low_u16(spectrum_v)); + echo_est_v_high = vmull_u16(vreinterpret_u16_s16(vget_high_s16(store_v)), + vget_high_u16(spectrum_v)); + vst1q_s32(echo_est_p, vreinterpretq_s32_u32(echo_est_v_low)); + vst1q_s32(echo_est_p + 4, vreinterpretq_s32_u32(echo_est_v_high)); + + echo_stored_v = vaddq_u32(echo_est_v_low, echo_stored_v); + echo_stored_v = vaddq_u32(echo_est_v_high, echo_stored_v); + + echo_adapt_v = vmlal_u16(echo_adapt_v, + vreinterpret_u16_s16(vget_low_s16(adapt_v)), + vget_low_u16(spectrum_v)); + echo_adapt_v = vmlal_u16(echo_adapt_v, + vreinterpret_u16_s16(vget_high_s16(adapt_v)), + vget_high_u16(spectrum_v)); + + start_stored_p += 8; + start_adapt_p += 8; + far_spectrum_p += 8; + echo_est_p += 8; + } + + AddLanes(far_energy, far_energy_v); + AddLanes(echo_energy_stored, echo_stored_v); + AddLanes(echo_energy_adapt, echo_adapt_v); + + echo_est[PART_LEN] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[PART_LEN], + far_spectrum[PART_LEN]); + *echo_energy_stored += (uint32_t)echo_est[PART_LEN]; + *far_energy += (uint32_t)far_spectrum[PART_LEN]; + *echo_energy_adapt += aecm->channelAdapt16[PART_LEN] * far_spectrum[PART_LEN]; +} + +void WebRtcAecm_StoreAdaptiveChannelNeon(AecmCore* aecm, + const uint16_t* far_spectrum, + int32_t* echo_est) { + RTC_DCHECK_EQ(0, (uintptr_t)echo_est % 32); + RTC_DCHECK_EQ(0, (uintptr_t)aecm->channelStored % 16); + RTC_DCHECK_EQ(0, (uintptr_t)aecm->channelAdapt16 % 16); + + // This is C code of following optimized code. + // During startup we store the channel every block. + // memcpy(aecm->channelStored, + // aecm->channelAdapt16, + // sizeof(int16_t) * PART_LEN1); + // Recalculate echo estimate + // for (i = 0; i < PART_LEN; i += 4) { + // echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + // far_spectrum[i]); + // echo_est[i + 1] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 1], + // far_spectrum[i + 1]); + // echo_est[i + 2] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 2], + // far_spectrum[i + 2]); + // echo_est[i + 3] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i + 3], + // far_spectrum[i + 3]); + // } + // echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + // far_spectrum[i]); + const uint16_t* far_spectrum_p = far_spectrum; + int16_t* start_adapt_p = aecm->channelAdapt16; + int16_t* start_stored_p = aecm->channelStored; + const int16_t* end_stored_p = aecm->channelStored + PART_LEN; + int32_t* echo_est_p = echo_est; + + uint16x8_t far_spectrum_v; + int16x8_t adapt_v; + uint32x4_t echo_est_v_low, echo_est_v_high; + + while (start_stored_p < end_stored_p) { + far_spectrum_v = vld1q_u16(far_spectrum_p); + adapt_v = vld1q_s16(start_adapt_p); + + vst1q_s16(start_stored_p, adapt_v); + + echo_est_v_low = vmull_u16(vget_low_u16(far_spectrum_v), + vget_low_u16(vreinterpretq_u16_s16(adapt_v))); + echo_est_v_high = vmull_u16(vget_high_u16(far_spectrum_v), + vget_high_u16(vreinterpretq_u16_s16(adapt_v))); + + vst1q_s32(echo_est_p, vreinterpretq_s32_u32(echo_est_v_low)); + vst1q_s32(echo_est_p + 4, vreinterpretq_s32_u32(echo_est_v_high)); + + far_spectrum_p += 8; + start_adapt_p += 8; + start_stored_p += 8; + echo_est_p += 8; + } + aecm->channelStored[PART_LEN] = aecm->channelAdapt16[PART_LEN]; + echo_est[PART_LEN] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[PART_LEN], + far_spectrum[PART_LEN]); +} + +void WebRtcAecm_ResetAdaptiveChannelNeon(AecmCore* aecm) { + RTC_DCHECK_EQ(0, (uintptr_t)aecm->channelStored % 16); + RTC_DCHECK_EQ(0, (uintptr_t)aecm->channelAdapt16 % 16); + RTC_DCHECK_EQ(0, (uintptr_t)aecm->channelAdapt32 % 32); + + // The C code of following optimized code. + // for (i = 0; i < PART_LEN1; i++) { + // aecm->channelAdapt16[i] = aecm->channelStored[i]; + // aecm->channelAdapt32[i] = WEBRTC_SPL_LSHIFT_W32( + // (int32_t)aecm->channelStored[i], 16); + // } + + int16_t* start_stored_p = aecm->channelStored; + int16_t* start_adapt16_p = aecm->channelAdapt16; + int32_t* start_adapt32_p = aecm->channelAdapt32; + const int16_t* end_stored_p = start_stored_p + PART_LEN; + + int16x8_t stored_v; + int32x4_t adapt32_v_low, adapt32_v_high; + + while (start_stored_p < end_stored_p) { + stored_v = vld1q_s16(start_stored_p); + vst1q_s16(start_adapt16_p, stored_v); + + adapt32_v_low = vshll_n_s16(vget_low_s16(stored_v), 16); + adapt32_v_high = vshll_n_s16(vget_high_s16(stored_v), 16); + + vst1q_s32(start_adapt32_p, adapt32_v_low); + vst1q_s32(start_adapt32_p + 4, adapt32_v_high); + + start_stored_p += 8; + start_adapt16_p += 8; + start_adapt32_p += 8; + } + aecm->channelAdapt16[PART_LEN] = aecm->channelStored[PART_LEN]; + aecm->channelAdapt32[PART_LEN] = (int32_t)aecm->channelStored[PART_LEN] << 16; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_defines.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_defines.h new file mode 100644 index 0000000000..ae2d2bc03a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_defines.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AECM_AECM_DEFINES_H_ +#define MODULES_AUDIO_PROCESSING_AECM_AECM_DEFINES_H_ + +#define AECM_DYNAMIC_Q /* Turn on/off dynamic Q-domain. */ + +/* Algorithm parameters */ +#define FRAME_LEN 80 /* Total frame length, 10 ms. */ + +#define PART_LEN 64 /* Length of partition. */ +#define PART_LEN_SHIFT 7 /* Length of (PART_LEN * 2) in base 2. */ + +#define PART_LEN1 (PART_LEN + 1) /* Unique fft coefficients. */ +#define PART_LEN2 (PART_LEN << 1) /* Length of partition * 2. */ +#define PART_LEN4 (PART_LEN << 2) /* Length of partition * 4. */ +#define FAR_BUF_LEN PART_LEN4 /* Length of buffers. */ +#define MAX_DELAY 100 + +/* Counter parameters */ +#define CONV_LEN 512 /* Convergence length used at startup. */ +#define CONV_LEN2 (CONV_LEN << 1) /* Used at startup. */ + +/* Energy parameters */ +#define MAX_BUF_LEN 64 /* History length of energy signals. */ +#define FAR_ENERGY_MIN 1025 /* Lowest Far energy level: At least 2 */ + /* in energy. */ +#define FAR_ENERGY_DIFF 929 /* Allowed difference between max */ + /* and min. */ +#define ENERGY_DEV_OFFSET 0 /* The energy error offset in Q8. */ +#define ENERGY_DEV_TOL 400 /* The energy estimation tolerance (Q8). */ +#define FAR_ENERGY_VAD_REGION 230 /* Far VAD tolerance region. */ + +/* Stepsize parameters */ +#define MU_MIN 10 /* Min stepsize 2^-MU_MIN (far end energy */ + /* dependent). */ +#define MU_MAX 1 /* Max stepsize 2^-MU_MAX (far end energy */ + /* dependent). */ +#define MU_DIFF 9 /* MU_MIN - MU_MAX */ + +/* Channel parameters */ +#define MIN_MSE_COUNT 20 /* Min number of consecutive blocks with enough */ + /* far end energy to compare channel estimates. */ +#define MIN_MSE_DIFF 29 /* The ratio between adapted and stored channel to */ + /* accept a new storage (0.8 in Q-MSE_RESOLUTION). */ +#define MSE_RESOLUTION 5 /* MSE parameter resolution. */ +#define RESOLUTION_CHANNEL16 12 /* W16 Channel in Q-RESOLUTION_CHANNEL16. */ +#define RESOLUTION_CHANNEL32 28 /* W32 Channel in Q-RESOLUTION_CHANNEL. */ +#define CHANNEL_VAD 16 /* Minimum energy in frequency band */ + /* to update channel. */ + +/* Suppression gain parameters: SUPGAIN parameters in Q-(RESOLUTION_SUPGAIN). */ +#define RESOLUTION_SUPGAIN 8 /* Channel in Q-(RESOLUTION_SUPGAIN). */ +#define SUPGAIN_DEFAULT (1 << RESOLUTION_SUPGAIN) /* Default. */ +#define SUPGAIN_ERROR_PARAM_A 3072 /* Estimation error parameter */ + /* (Maximum gain) (8 in Q8). */ +#define SUPGAIN_ERROR_PARAM_B 1536 /* Estimation error parameter */ + /* (Gain before going down). */ +#define SUPGAIN_ERROR_PARAM_D SUPGAIN_DEFAULT /* Estimation error parameter */ + /* (Should be the same as Default) (1 in Q8). */ +#define SUPGAIN_EPC_DT 200 /* SUPGAIN_ERROR_PARAM_C * ENERGY_DEV_TOL */ + +/* Defines for "check delay estimation" */ +#define CORR_WIDTH 31 /* Number of samples to correlate over. */ +#define CORR_MAX 16 /* Maximum correlation offset. */ +#define CORR_MAX_BUF 63 +#define CORR_DEV 4 +#define CORR_MAX_LEVEL 20 +#define CORR_MAX_LOW 4 +#define CORR_BUF_LEN (CORR_MAX << 1) + 1 +/* Note that CORR_WIDTH + 2*CORR_MAX <= MAX_BUF_LEN. */ + +#define ONE_Q14 (1 << 14) + +/* NLP defines */ +#define NLP_COMP_LOW 3277 /* 0.2 in Q14 */ +#define NLP_COMP_HIGH ONE_Q14 /* 1 in Q14 */ + +#endif diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.cc new file mode 100644 index 0000000000..880f0aa65a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.cc @@ -0,0 +1,648 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aecm/echo_control_mobile.h" + +#ifdef AEC_DEBUG +#include +#endif +#include + +extern "C" { +#include "common_audio/ring_buffer.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +} +#include "modules/audio_processing/aecm/aecm_core.h" + +#define BUF_SIZE_FRAMES 50 // buffer size (frames) +// Maximum length of resampled signal. Must be an integer multiple of frames +// (ceil(1/(1 + MIN_SKEW)*2) + 1)*FRAME_LEN +// The factor of 2 handles wb, and the + 1 is as a safety margin +#define MAX_RESAMP_LEN (5 * FRAME_LEN) + +static const size_t kBufSizeSamp = BUF_SIZE_FRAMES * FRAME_LEN; // buffer size (samples) +static const int kSampMsNb = 8; // samples per ms in nb +// Target suppression levels for nlp modes +// log{0.001, 0.00001, 0.00000001} +static const int kInitCheck = 42; + +typedef struct +{ + int sampFreq; + int scSampFreq; + short bufSizeStart; + int knownDelay; + + // Stores the last frame added to the farend buffer + short farendOld[2][FRAME_LEN]; + short initFlag; // indicates if AEC has been initialized + + // Variables used for averaging far end buffer size + short counter; + short sum; + short firstVal; + short checkBufSizeCtr; + + // Variables used for delay shifts + short msInSndCardBuf; + short filtDelay; + int timeForDelayChange; + int ECstartup; + int checkBuffSize; + int delayChange; + short lastDelayDiff; + + int16_t echoMode; + +#ifdef AEC_DEBUG + FILE *bufFile; + FILE *delayFile; + FILE *preCompFile; + FILE *postCompFile; +#endif // AEC_DEBUG + // Structures + RingBuffer *farendBuf; + + AecmCore* aecmCore; +} AecMobile; + +// Estimates delay to set the position of the farend buffer read pointer +// (controlled by knownDelay) +static int WebRtcAecm_EstBufDelay(AecMobile* aecm, short msInSndCardBuf); + +// Stuffs the farend buffer if the estimated delay is too large +static int WebRtcAecm_DelayComp(AecMobile* aecm); + +void* WebRtcAecm_Create() { + AecMobile* aecm = static_cast(malloc(sizeof(AecMobile))); + + WebRtcSpl_Init(); + + aecm->aecmCore = WebRtcAecm_CreateCore(); + if (!aecm->aecmCore) { + WebRtcAecm_Free(aecm); + return NULL; + } + + aecm->farendBuf = WebRtc_CreateBuffer(kBufSizeSamp, + sizeof(int16_t)); + if (!aecm->farendBuf) + { + WebRtcAecm_Free(aecm); + return NULL; + } + + aecm->initFlag = 0; + +#ifdef AEC_DEBUG + aecm->aecmCore->farFile = fopen("aecFar.pcm","wb"); + aecm->aecmCore->nearFile = fopen("aecNear.pcm","wb"); + aecm->aecmCore->outFile = fopen("aecOut.pcm","wb"); + //aecm->aecmCore->outLpFile = fopen("aecOutLp.pcm","wb"); + + aecm->bufFile = fopen("aecBuf.dat", "wb"); + aecm->delayFile = fopen("aecDelay.dat", "wb"); + aecm->preCompFile = fopen("preComp.pcm", "wb"); + aecm->postCompFile = fopen("postComp.pcm", "wb"); +#endif // AEC_DEBUG + return aecm; +} + +void WebRtcAecm_Free(void* aecmInst) { + AecMobile* aecm = static_cast(aecmInst); + + if (aecm == NULL) { + return; + } + +#ifdef AEC_DEBUG + fclose(aecm->aecmCore->farFile); + fclose(aecm->aecmCore->nearFile); + fclose(aecm->aecmCore->outFile); + //fclose(aecm->aecmCore->outLpFile); + + fclose(aecm->bufFile); + fclose(aecm->delayFile); + fclose(aecm->preCompFile); + fclose(aecm->postCompFile); +#endif // AEC_DEBUG + WebRtcAecm_FreeCore(aecm->aecmCore); + WebRtc_FreeBuffer(aecm->farendBuf); + free(aecm); +} + +int32_t WebRtcAecm_Init(void *aecmInst, int32_t sampFreq) +{ + AecMobile* aecm = static_cast(aecmInst); + AecmConfig aecConfig; + + if (aecm == NULL) + { + return -1; + } + + if (sampFreq != 8000 && sampFreq != 16000) + { + return AECM_BAD_PARAMETER_ERROR; + } + aecm->sampFreq = sampFreq; + + // Initialize AECM core + if (WebRtcAecm_InitCore(aecm->aecmCore, aecm->sampFreq) == -1) + { + return AECM_UNSPECIFIED_ERROR; + } + + // Initialize farend buffer + WebRtc_InitBuffer(aecm->farendBuf); + + aecm->initFlag = kInitCheck; // indicates that initialization has been done + + aecm->delayChange = 1; + + aecm->sum = 0; + aecm->counter = 0; + aecm->checkBuffSize = 1; + aecm->firstVal = 0; + + aecm->ECstartup = 1; + aecm->bufSizeStart = 0; + aecm->checkBufSizeCtr = 0; + aecm->filtDelay = 0; + aecm->timeForDelayChange = 0; + aecm->knownDelay = 0; + aecm->lastDelayDiff = 0; + + memset(&aecm->farendOld[0][0], 0, 160); + + // Default settings. + aecConfig.cngMode = AecmTrue; + aecConfig.echoMode = 3; + + if (WebRtcAecm_set_config(aecm, aecConfig) == -1) + { + return AECM_UNSPECIFIED_ERROR; + } + + return 0; +} + +// Returns any error that is caused when buffering the +// farend signal. +int32_t WebRtcAecm_GetBufferFarendError(void *aecmInst, const int16_t *farend, + size_t nrOfSamples) { + AecMobile* aecm = static_cast(aecmInst); + + if (aecm == NULL) + return -1; + + if (farend == NULL) + return AECM_NULL_POINTER_ERROR; + + if (aecm->initFlag != kInitCheck) + return AECM_UNINITIALIZED_ERROR; + + if (nrOfSamples != 80 && nrOfSamples != 160) + return AECM_BAD_PARAMETER_ERROR; + + return 0; +} + + +int32_t WebRtcAecm_BufferFarend(void *aecmInst, const int16_t *farend, + size_t nrOfSamples) { + AecMobile* aecm = static_cast(aecmInst); + + const int32_t err = + WebRtcAecm_GetBufferFarendError(aecmInst, farend, nrOfSamples); + + if (err != 0) + return err; + + // TODO(unknown): Is this really a good idea? + if (!aecm->ECstartup) + { + WebRtcAecm_DelayComp(aecm); + } + + WebRtc_WriteBuffer(aecm->farendBuf, farend, nrOfSamples); + + return 0; +} + +int32_t WebRtcAecm_Process(void *aecmInst, const int16_t *nearendNoisy, + const int16_t *nearendClean, int16_t *out, + size_t nrOfSamples, int16_t msInSndCardBuf) +{ + AecMobile* aecm = static_cast(aecmInst); + int32_t retVal = 0; + size_t i; + short nmbrOfFilledBuffers; + size_t nBlocks10ms; + size_t nFrames; +#ifdef AEC_DEBUG + short msInAECBuf; +#endif + + if (aecm == NULL) + { + return -1; + } + + if (nearendNoisy == NULL) + { + return AECM_NULL_POINTER_ERROR; + } + + if (out == NULL) + { + return AECM_NULL_POINTER_ERROR; + } + + if (aecm->initFlag != kInitCheck) + { + return AECM_UNINITIALIZED_ERROR; + } + + if (nrOfSamples != 80 && nrOfSamples != 160) + { + return AECM_BAD_PARAMETER_ERROR; + } + + if (msInSndCardBuf < 0) + { + msInSndCardBuf = 0; + retVal = AECM_BAD_PARAMETER_WARNING; + } else if (msInSndCardBuf > 500) + { + msInSndCardBuf = 500; + retVal = AECM_BAD_PARAMETER_WARNING; + } + msInSndCardBuf += 10; + aecm->msInSndCardBuf = msInSndCardBuf; + + nFrames = nrOfSamples / FRAME_LEN; + nBlocks10ms = nFrames / aecm->aecmCore->mult; + + if (aecm->ECstartup) + { + if (nearendClean == NULL) + { + if (out != nearendNoisy) + { + memcpy(out, nearendNoisy, sizeof(short) * nrOfSamples); + } + } else if (out != nearendClean) + { + memcpy(out, nearendClean, sizeof(short) * nrOfSamples); + } + + nmbrOfFilledBuffers = + (short) WebRtc_available_read(aecm->farendBuf) / FRAME_LEN; + // The AECM is in the start up mode + // AECM is disabled until the soundcard buffer and farend buffers are OK + + // Mechanism to ensure that the soundcard buffer is reasonably stable. + if (aecm->checkBuffSize) + { + aecm->checkBufSizeCtr++; + // Before we fill up the far end buffer we require the amount of data on the + // sound card to be stable (+/-8 ms) compared to the first value. This + // comparison is made during the following 4 consecutive frames. If it seems + // to be stable then we start to fill up the far end buffer. + + if (aecm->counter == 0) + { + aecm->firstVal = aecm->msInSndCardBuf; + aecm->sum = 0; + } + + if (abs(aecm->firstVal - aecm->msInSndCardBuf) + < WEBRTC_SPL_MAX(0.2 * aecm->msInSndCardBuf, kSampMsNb)) + { + aecm->sum += aecm->msInSndCardBuf; + aecm->counter++; + } else + { + aecm->counter = 0; + } + + if (aecm->counter * nBlocks10ms >= 6) + { + // The farend buffer size is determined in blocks of 80 samples + // Use 75% of the average value of the soundcard buffer + aecm->bufSizeStart + = WEBRTC_SPL_MIN((3 * aecm->sum + * aecm->aecmCore->mult) / (aecm->counter * 40), BUF_SIZE_FRAMES); + // buffersize has now been determined + aecm->checkBuffSize = 0; + } + + if (aecm->checkBufSizeCtr * nBlocks10ms > 50) + { + // for really bad sound cards, don't disable echocanceller for more than 0.5 sec + aecm->bufSizeStart = WEBRTC_SPL_MIN((3 * aecm->msInSndCardBuf + * aecm->aecmCore->mult) / 40, BUF_SIZE_FRAMES); + aecm->checkBuffSize = 0; + } + } + + // if checkBuffSize changed in the if-statement above + if (!aecm->checkBuffSize) + { + // soundcard buffer is now reasonably stable + // When the far end buffer is filled with approximately the same amount of + // data as the amount on the sound card we end the start up phase and start + // to cancel echoes. + + if (nmbrOfFilledBuffers == aecm->bufSizeStart) + { + aecm->ECstartup = 0; // Enable the AECM + } else if (nmbrOfFilledBuffers > aecm->bufSizeStart) + { + WebRtc_MoveReadPtr(aecm->farendBuf, + (int) WebRtc_available_read(aecm->farendBuf) + - (int) aecm->bufSizeStart * FRAME_LEN); + aecm->ECstartup = 0; + } + } + + } else + { + // AECM is enabled + + // Note only 1 block supported for nb and 2 blocks for wb + for (i = 0; i < nFrames; i++) + { + int16_t farend[FRAME_LEN]; + const int16_t* farend_ptr = NULL; + + nmbrOfFilledBuffers = + (short) WebRtc_available_read(aecm->farendBuf) / FRAME_LEN; + + // Check that there is data in the far end buffer + if (nmbrOfFilledBuffers > 0) + { + // Get the next 80 samples from the farend buffer + WebRtc_ReadBuffer(aecm->farendBuf, (void**) &farend_ptr, farend, + FRAME_LEN); + + // Always store the last frame for use when we run out of data + memcpy(&(aecm->farendOld[i][0]), farend_ptr, + FRAME_LEN * sizeof(short)); + } else + { + // We have no data so we use the last played frame + memcpy(farend, &(aecm->farendOld[i][0]), FRAME_LEN * sizeof(short)); + farend_ptr = farend; + } + + // Call buffer delay estimator when all data is extracted, + // i,e. i = 0 for NB and i = 1 for WB + if ((i == 0 && aecm->sampFreq == 8000) || (i == 1 && aecm->sampFreq == 16000)) + { + WebRtcAecm_EstBufDelay(aecm, aecm->msInSndCardBuf); + } + + // Call the AECM + /*WebRtcAecm_ProcessFrame(aecm->aecmCore, farend, &nearend[FRAME_LEN * i], + &out[FRAME_LEN * i], aecm->knownDelay);*/ + if (WebRtcAecm_ProcessFrame(aecm->aecmCore, + farend_ptr, + &nearendNoisy[FRAME_LEN * i], + (nearendClean + ? &nearendClean[FRAME_LEN * i] + : NULL), + &out[FRAME_LEN * i]) == -1) + return -1; + } + } + +#ifdef AEC_DEBUG + msInAECBuf = (short) WebRtc_available_read(aecm->farendBuf) / + (kSampMsNb * aecm->aecmCore->mult); + fwrite(&msInAECBuf, 2, 1, aecm->bufFile); + fwrite(&(aecm->knownDelay), sizeof(aecm->knownDelay), 1, aecm->delayFile); +#endif + + return retVal; +} + +int32_t WebRtcAecm_set_config(void *aecmInst, AecmConfig config) +{ + AecMobile* aecm = static_cast(aecmInst); + + if (aecm == NULL) + { + return -1; + } + + if (aecm->initFlag != kInitCheck) + { + return AECM_UNINITIALIZED_ERROR; + } + + if (config.cngMode != AecmFalse && config.cngMode != AecmTrue) + { + return AECM_BAD_PARAMETER_ERROR; + } + aecm->aecmCore->cngMode = config.cngMode; + + if (config.echoMode < 0 || config.echoMode > 4) + { + return AECM_BAD_PARAMETER_ERROR; + } + aecm->echoMode = config.echoMode; + + if (aecm->echoMode == 0) + { + aecm->aecmCore->supGain = SUPGAIN_DEFAULT >> 3; + aecm->aecmCore->supGainOld = SUPGAIN_DEFAULT >> 3; + aecm->aecmCore->supGainErrParamA = SUPGAIN_ERROR_PARAM_A >> 3; + aecm->aecmCore->supGainErrParamD = SUPGAIN_ERROR_PARAM_D >> 3; + aecm->aecmCore->supGainErrParamDiffAB = (SUPGAIN_ERROR_PARAM_A >> 3) + - (SUPGAIN_ERROR_PARAM_B >> 3); + aecm->aecmCore->supGainErrParamDiffBD = (SUPGAIN_ERROR_PARAM_B >> 3) + - (SUPGAIN_ERROR_PARAM_D >> 3); + } else if (aecm->echoMode == 1) + { + aecm->aecmCore->supGain = SUPGAIN_DEFAULT >> 2; + aecm->aecmCore->supGainOld = SUPGAIN_DEFAULT >> 2; + aecm->aecmCore->supGainErrParamA = SUPGAIN_ERROR_PARAM_A >> 2; + aecm->aecmCore->supGainErrParamD = SUPGAIN_ERROR_PARAM_D >> 2; + aecm->aecmCore->supGainErrParamDiffAB = (SUPGAIN_ERROR_PARAM_A >> 2) + - (SUPGAIN_ERROR_PARAM_B >> 2); + aecm->aecmCore->supGainErrParamDiffBD = (SUPGAIN_ERROR_PARAM_B >> 2) + - (SUPGAIN_ERROR_PARAM_D >> 2); + } else if (aecm->echoMode == 2) + { + aecm->aecmCore->supGain = SUPGAIN_DEFAULT >> 1; + aecm->aecmCore->supGainOld = SUPGAIN_DEFAULT >> 1; + aecm->aecmCore->supGainErrParamA = SUPGAIN_ERROR_PARAM_A >> 1; + aecm->aecmCore->supGainErrParamD = SUPGAIN_ERROR_PARAM_D >> 1; + aecm->aecmCore->supGainErrParamDiffAB = (SUPGAIN_ERROR_PARAM_A >> 1) + - (SUPGAIN_ERROR_PARAM_B >> 1); + aecm->aecmCore->supGainErrParamDiffBD = (SUPGAIN_ERROR_PARAM_B >> 1) + - (SUPGAIN_ERROR_PARAM_D >> 1); + } else if (aecm->echoMode == 3) + { + aecm->aecmCore->supGain = SUPGAIN_DEFAULT; + aecm->aecmCore->supGainOld = SUPGAIN_DEFAULT; + aecm->aecmCore->supGainErrParamA = SUPGAIN_ERROR_PARAM_A; + aecm->aecmCore->supGainErrParamD = SUPGAIN_ERROR_PARAM_D; + aecm->aecmCore->supGainErrParamDiffAB = SUPGAIN_ERROR_PARAM_A - SUPGAIN_ERROR_PARAM_B; + aecm->aecmCore->supGainErrParamDiffBD = SUPGAIN_ERROR_PARAM_B - SUPGAIN_ERROR_PARAM_D; + } else if (aecm->echoMode == 4) + { + aecm->aecmCore->supGain = SUPGAIN_DEFAULT << 1; + aecm->aecmCore->supGainOld = SUPGAIN_DEFAULT << 1; + aecm->aecmCore->supGainErrParamA = SUPGAIN_ERROR_PARAM_A << 1; + aecm->aecmCore->supGainErrParamD = SUPGAIN_ERROR_PARAM_D << 1; + aecm->aecmCore->supGainErrParamDiffAB = (SUPGAIN_ERROR_PARAM_A << 1) + - (SUPGAIN_ERROR_PARAM_B << 1); + aecm->aecmCore->supGainErrParamDiffBD = (SUPGAIN_ERROR_PARAM_B << 1) + - (SUPGAIN_ERROR_PARAM_D << 1); + } + + return 0; +} + +int32_t WebRtcAecm_InitEchoPath(void* aecmInst, + const void* echo_path, + size_t size_bytes) +{ + AecMobile* aecm = static_cast(aecmInst); + const int16_t* echo_path_ptr = static_cast(echo_path); + + if (aecmInst == NULL) { + return -1; + } + if (echo_path == NULL) { + return AECM_NULL_POINTER_ERROR; + } + if (size_bytes != WebRtcAecm_echo_path_size_bytes()) + { + // Input channel size does not match the size of AECM + return AECM_BAD_PARAMETER_ERROR; + } + if (aecm->initFlag != kInitCheck) + { + return AECM_UNINITIALIZED_ERROR; + } + + WebRtcAecm_InitEchoPathCore(aecm->aecmCore, echo_path_ptr); + + return 0; +} + +int32_t WebRtcAecm_GetEchoPath(void* aecmInst, + void* echo_path, + size_t size_bytes) +{ + AecMobile* aecm = static_cast(aecmInst); + int16_t* echo_path_ptr = static_cast(echo_path); + + if (aecmInst == NULL) { + return -1; + } + if (echo_path == NULL) { + return AECM_NULL_POINTER_ERROR; + } + if (size_bytes != WebRtcAecm_echo_path_size_bytes()) + { + // Input channel size does not match the size of AECM + return AECM_BAD_PARAMETER_ERROR; + } + if (aecm->initFlag != kInitCheck) + { + return AECM_UNINITIALIZED_ERROR; + } + + memcpy(echo_path_ptr, aecm->aecmCore->channelStored, size_bytes); + return 0; +} + +size_t WebRtcAecm_echo_path_size_bytes() +{ + return (PART_LEN1 * sizeof(int16_t)); +} + + +static int WebRtcAecm_EstBufDelay(AecMobile* aecm, short msInSndCardBuf) { + short delayNew, nSampSndCard; + short nSampFar = (short) WebRtc_available_read(aecm->farendBuf); + short diff; + + nSampSndCard = msInSndCardBuf * kSampMsNb * aecm->aecmCore->mult; + + delayNew = nSampSndCard - nSampFar; + + if (delayNew < FRAME_LEN) + { + WebRtc_MoveReadPtr(aecm->farendBuf, FRAME_LEN); + delayNew += FRAME_LEN; + } + + aecm->filtDelay = WEBRTC_SPL_MAX(0, (8 * aecm->filtDelay + 2 * delayNew) / 10); + + diff = aecm->filtDelay - aecm->knownDelay; + if (diff > 224) + { + if (aecm->lastDelayDiff < 96) + { + aecm->timeForDelayChange = 0; + } else + { + aecm->timeForDelayChange++; + } + } else if (diff < 96 && aecm->knownDelay > 0) + { + if (aecm->lastDelayDiff > 224) + { + aecm->timeForDelayChange = 0; + } else + { + aecm->timeForDelayChange++; + } + } else + { + aecm->timeForDelayChange = 0; + } + aecm->lastDelayDiff = diff; + + if (aecm->timeForDelayChange > 25) + { + aecm->knownDelay = WEBRTC_SPL_MAX((int)aecm->filtDelay - 160, 0); + } + return 0; +} + +static int WebRtcAecm_DelayComp(AecMobile* aecm) { + int nSampFar = (int) WebRtc_available_read(aecm->farendBuf); + int nSampSndCard, delayNew, nSampAdd; + const int maxStuffSamp = 10 * FRAME_LEN; + + nSampSndCard = aecm->msInSndCardBuf * kSampMsNb * aecm->aecmCore->mult; + delayNew = nSampSndCard - nSampFar; + + if (delayNew > FAR_BUF_LEN - FRAME_LEN * aecm->aecmCore->mult) + { + // The difference of the buffer sizes is larger than the maximum + // allowed known delay. Compensate by stuffing the buffer. + nSampAdd = (int)(WEBRTC_SPL_MAX(((nSampSndCard >> 1) - nSampFar), + FRAME_LEN)); + nSampAdd = WEBRTC_SPL_MIN(nSampAdd, maxStuffSamp); + + WebRtc_MoveReadPtr(aecm->farendBuf, -nSampAdd); + aecm->delayChange = 1; // the delay needs to be updated + } + + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.h b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.h new file mode 100644 index 0000000000..e0091c3e57 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.h @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AECM_ECHO_CONTROL_MOBILE_H_ +#define MODULES_AUDIO_PROCESSING_AECM_ECHO_CONTROL_MOBILE_H_ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +enum { + AecmFalse = 0, + AecmTrue +}; + +// Errors +#define AECM_UNSPECIFIED_ERROR 12000 +#define AECM_UNSUPPORTED_FUNCTION_ERROR 12001 +#define AECM_UNINITIALIZED_ERROR 12002 +#define AECM_NULL_POINTER_ERROR 12003 +#define AECM_BAD_PARAMETER_ERROR 12004 + +// Warnings +#define AECM_BAD_PARAMETER_WARNING 12100 + +typedef struct { + int16_t cngMode; // AECM_FALSE, AECM_TRUE (default) + int16_t echoMode; // 0, 1, 2, 3 (default), 4 +} AecmConfig; + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Allocates the memory needed by the AECM. The memory needs to be + * initialized separately using the WebRtcAecm_Init() function. + * Returns a pointer to the instance and a nullptr at failure. + */ +void* WebRtcAecm_Create(); + +/* + * This function releases the memory allocated by WebRtcAecm_Create() + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + */ +void WebRtcAecm_Free(void* aecmInst); + +/* + * Initializes an AECM instance. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + * int32_t sampFreq Sampling frequency of data + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 1200-12004,12100: error/warning + */ +int32_t WebRtcAecm_Init(void* aecmInst, int32_t sampFreq); + +/* + * Inserts an 80 or 160 sample block of data into the farend buffer. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + * int16_t* farend In buffer containing one frame of + * farend signal + * int16_t nrOfSamples Number of samples in farend buffer + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 1200-12004,12100: error/warning + */ +int32_t WebRtcAecm_BufferFarend(void* aecmInst, + const int16_t* farend, + size_t nrOfSamples); + +/* + * Reports any errors that would arise when buffering a farend buffer. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + * int16_t* farend In buffer containing one frame of + * farend signal + * int16_t nrOfSamples Number of samples in farend buffer + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 1200-12004,12100: error/warning + */ +int32_t WebRtcAecm_GetBufferFarendError(void* aecmInst, + const int16_t* farend, + size_t nrOfSamples); + +/* + * Runs the AECM on an 80 or 160 sample blocks of data. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + * int16_t* nearendNoisy In buffer containing one frame of + * reference nearend+echo signal. If + * noise reduction is active, provide + * the noisy signal here. + * int16_t* nearendClean In buffer containing one frame of + * nearend+echo signal. If noise + * reduction is active, provide the + * clean signal here. Otherwise pass a + * NULL pointer. + * int16_t nrOfSamples Number of samples in nearend buffer + * int16_t msInSndCardBuf Delay estimate for sound card and + * system buffers + * + * Outputs Description + * ------------------------------------------------------------------- + * int16_t* out Out buffer, one frame of processed nearend + * int32_t return 0: OK + * 1200-12004,12100: error/warning + */ +int32_t WebRtcAecm_Process(void* aecmInst, + const int16_t* nearendNoisy, + const int16_t* nearendClean, + int16_t* out, + size_t nrOfSamples, + int16_t msInSndCardBuf); + +/* + * This function enables the user to set certain parameters on-the-fly + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + * AecmConfig config Config instance that contains all + * properties to be set + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 1200-12004,12100: error/warning + */ +int32_t WebRtcAecm_set_config(void* aecmInst, AecmConfig config); + +/* + * This function enables the user to set the echo path on-the-fly. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + * void* echo_path Pointer to the echo path to be set + * size_t size_bytes Size in bytes of the echo path + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 1200-12004,12100: error/warning + */ +int32_t WebRtcAecm_InitEchoPath(void* aecmInst, + const void* echo_path, + size_t size_bytes); + +/* + * This function enables the user to get the currently used echo path + * on-the-fly + * + * Inputs Description + * ------------------------------------------------------------------- + * void* aecmInst Pointer to the AECM instance + * void* echo_path Pointer to echo path + * size_t size_bytes Size in bytes of the echo path + * + * Outputs Description + * ------------------------------------------------------------------- + * int32_t return 0: OK + * 1200-12004,12100: error/warning + */ +int32_t WebRtcAecm_GetEchoPath(void* aecmInst, + void* echo_path, + size_t size_bytes); + +/* + * This function enables the user to get the echo path size in bytes + * + * Outputs Description + * ------------------------------------------------------------------- + * size_t return Size in bytes + */ +size_t WebRtcAecm_echo_path_size_bytes(); + + +#ifdef __cplusplus +} +#endif +#endif // MODULES_AUDIO_PROCESSING_AECM_ECHO_CONTROL_MOBILE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.cc new file mode 100644 index 0000000000..0c6478e803 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc/agc.h" + +#include +#include + +#include +#include + +#include "modules/audio_processing/agc/loudness_histogram.h" +#include "modules/audio_processing/agc/utility.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +const int kDefaultLevelDbfs = -18; +const int kNumAnalysisFrames = 100; +const double kActivityThreshold = 0.3; + +} // namespace + +Agc::Agc() + : target_level_loudness_(Dbfs2Loudness(kDefaultLevelDbfs)), + target_level_dbfs_(kDefaultLevelDbfs), + histogram_(LoudnessHistogram::Create(kNumAnalysisFrames)), + inactive_histogram_(LoudnessHistogram::Create()) {} + +Agc::~Agc() {} + +float Agc::AnalyzePreproc(const int16_t* audio, size_t length) { + RTC_DCHECK_GT(length, 0); + size_t num_clipped = 0; + for (size_t i = 0; i < length; ++i) { + if (audio[i] == 32767 || audio[i] == -32768) + ++num_clipped; + } + return 1.0f * num_clipped / length; +} + +int Agc::Process(const int16_t* audio, size_t length, int sample_rate_hz) { + vad_.ProcessChunk(audio, length, sample_rate_hz); + const std::vector& rms = vad_.chunkwise_rms(); + const std::vector& probabilities = + vad_.chunkwise_voice_probabilities(); + RTC_DCHECK_EQ(rms.size(), probabilities.size()); + for (size_t i = 0; i < rms.size(); ++i) { + histogram_->Update(rms[i], probabilities[i]); + } + return 0; +} + +bool Agc::GetRmsErrorDb(int* error) { + if (!error) { + RTC_NOTREACHED(); + return false; + } + + if (histogram_->num_updates() < kNumAnalysisFrames) { + // We haven't yet received enough frames. + return false; + } + + if (histogram_->AudioContent() < kNumAnalysisFrames * kActivityThreshold) { + // We are likely in an inactive segment. + return false; + } + + double loudness = Linear2Loudness(histogram_->CurrentRms()); + *error = std::floor(Loudness2Db(target_level_loudness_ - loudness) + 0.5); + histogram_->Reset(); + return true; +} + +void Agc::Reset() { + histogram_->Reset(); +} + +int Agc::set_target_level_dbfs(int level) { + // TODO(turajs): just some arbitrary sanity check. We can come up with better + // limits. The upper limit should be chosen such that the risk of clipping is + // low. The lower limit should not result in a too quiet signal. + if (level >= 0 || level <= -100) + return -1; + target_level_dbfs_ = level; + target_level_loudness_ = Dbfs2Loudness(level); + return 0; +} + +int Agc::target_level_dbfs() const { + return target_level_dbfs_; +} + +float Agc::voice_probability() const { + return vad_.last_voice_probability(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.h new file mode 100644 index 0000000000..3a8d5c8122 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_AGC_H_ +#define MODULES_AUDIO_PROCESSING_AGC_AGC_H_ + +#include + +#include "modules/audio_processing/vad/voice_activity_detector.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioFrame; +class LoudnessHistogram; + +class Agc { + public: + Agc(); + virtual ~Agc(); + + // Returns the proportion of samples in the buffer which are at full-scale + // (and presumably clipped). + virtual float AnalyzePreproc(const int16_t* audio, size_t length); + // |audio| must be mono; in a multi-channel stream, provide the first (usually + // left) channel. + virtual int Process(const int16_t* audio, size_t length, int sample_rate_hz); + + // Retrieves the difference between the target RMS level and the current + // signal RMS level in dB. Returns true if an update is available and false + // otherwise, in which case |error| should be ignored and no action taken. + virtual bool GetRmsErrorDb(int* error); + virtual void Reset(); + + virtual int set_target_level_dbfs(int level); + virtual int target_level_dbfs() const; + virtual float voice_probability() const; + + private: + double target_level_loudness_; + int target_level_dbfs_; + std::unique_ptr histogram_; + std::unique_ptr inactive_histogram_; + VoiceActivityDetector vad_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC_AGC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.cc new file mode 100644 index 0000000000..706d4ab037 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.cc @@ -0,0 +1,461 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc/agc_manager_direct.h" + +#include + +#ifdef WEBRTC_AGC_DEBUG_DUMP +#include +#endif + +#include "modules/audio_processing/agc/gain_map_internal.h" +#include "modules/audio_processing/gain_control_impl.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +// Amount the microphone level is lowered with every clipping event. +const int kClippedLevelStep = 15; +// Proportion of clipped samples required to declare a clipping event. +const float kClippedRatioThreshold = 0.1f; +// Time in frames to wait after a clipping event before checking again. +const int kClippedWaitFrames = 300; + +// Amount of error we tolerate in the microphone level (presumably due to OS +// quantization) before we assume the user has manually adjusted the microphone. +const int kLevelQuantizationSlack = 25; + +const int kDefaultCompressionGain = 7; +const int kMaxCompressionGain = 12; +const int kMinCompressionGain = 2; +// Controls the rate of compression changes towards the target. +const float kCompressionGainStep = 0.05f; + +const int kMaxMicLevel = 255; +static_assert(kGainMapSize > kMaxMicLevel, "gain map too small"); +const int kMinMicLevel = 12; + +// Prevent very large microphone level changes. +const int kMaxResidualGainChange = 15; + +// Maximum additional gain allowed to compensate for microphone level +// restrictions from clipping events. +const int kSurplusCompressionGain = 6; + +int ClampLevel(int mic_level) { + return rtc::SafeClamp(mic_level, kMinMicLevel, kMaxMicLevel); +} + +int LevelFromGainError(int gain_error, int level) { + RTC_DCHECK_GE(level, 0); + RTC_DCHECK_LE(level, kMaxMicLevel); + if (gain_error == 0) { + return level; + } + // TODO(ajm): Could be made more efficient with a binary search. + int new_level = level; + if (gain_error > 0) { + while (kGainMap[new_level] - kGainMap[level] < gain_error && + new_level < kMaxMicLevel) { + ++new_level; + } + } else { + while (kGainMap[new_level] - kGainMap[level] > gain_error && + new_level > kMinMicLevel) { + --new_level; + } + } + return new_level; +} + +} // namespace + +// Facility for dumping debug audio files. All methods are no-ops in the +// default case where WEBRTC_AGC_DEBUG_DUMP is undefined. +class DebugFile { +#ifdef WEBRTC_AGC_DEBUG_DUMP + public: + explicit DebugFile(const char* filename) + : file_(fopen(filename, "wb")) { + RTC_DCHECK(file_); + } + ~DebugFile() { + fclose(file_); + } + void Write(const int16_t* data, size_t length_samples) { + fwrite(data, 1, length_samples * sizeof(int16_t), file_); + } + private: + FILE* file_; +#else + public: + explicit DebugFile(const char* filename) { + } + ~DebugFile() { + } + void Write(const int16_t* data, size_t length_samples) { + } +#endif // WEBRTC_AGC_DEBUG_DUMP +}; + +AgcManagerDirect::AgcManagerDirect(GainControl* gctrl, + VolumeCallbacks* volume_callbacks, + int startup_min_level, + int clipped_level_min) + : agc_(new Agc()), + gctrl_(gctrl), + volume_callbacks_(volume_callbacks), + frames_since_clipped_(kClippedWaitFrames), + level_(0), + max_level_(kMaxMicLevel), + max_compression_gain_(kMaxCompressionGain), + target_compression_(kDefaultCompressionGain), + compression_(target_compression_), + compression_accumulator_(compression_), + capture_muted_(false), + check_volume_on_next_process_(true), // Check at startup. + startup_(true), + startup_min_level_(ClampLevel(startup_min_level)), + clipped_level_min_(clipped_level_min), + file_preproc_(new DebugFile("agc_preproc.pcm")), + file_postproc_(new DebugFile("agc_postproc.pcm")) {} + +AgcManagerDirect::AgcManagerDirect(Agc* agc, + GainControl* gctrl, + VolumeCallbacks* volume_callbacks, + int startup_min_level, + int clipped_level_min) + : agc_(agc), + gctrl_(gctrl), + volume_callbacks_(volume_callbacks), + frames_since_clipped_(kClippedWaitFrames), + level_(0), + max_level_(kMaxMicLevel), + max_compression_gain_(kMaxCompressionGain), + target_compression_(kDefaultCompressionGain), + compression_(target_compression_), + compression_accumulator_(compression_), + capture_muted_(false), + check_volume_on_next_process_(true), // Check at startup. + startup_(true), + startup_min_level_(ClampLevel(startup_min_level)), + clipped_level_min_(clipped_level_min), + file_preproc_(new DebugFile("agc_preproc.pcm")), + file_postproc_(new DebugFile("agc_postproc.pcm")) {} + +AgcManagerDirect::~AgcManagerDirect() {} + +int AgcManagerDirect::Initialize() { + max_level_ = kMaxMicLevel; + max_compression_gain_ = kMaxCompressionGain; + target_compression_ = kDefaultCompressionGain; + compression_ = target_compression_; + compression_accumulator_ = compression_; + capture_muted_ = false; + check_volume_on_next_process_ = true; + // TODO(bjornv): Investigate if we need to reset |startup_| as well. For + // example, what happens when we change devices. + + if (gctrl_->set_mode(GainControl::kFixedDigital) != 0) { + RTC_LOG(LS_ERROR) << "set_mode(GainControl::kFixedDigital) failed."; + return -1; + } + if (gctrl_->set_target_level_dbfs(2) != 0) { + RTC_LOG(LS_ERROR) << "set_target_level_dbfs(2) failed."; + return -1; + } + if (gctrl_->set_compression_gain_db(kDefaultCompressionGain) != 0) { + RTC_LOG(LS_ERROR) + << "set_compression_gain_db(kDefaultCompressionGain) failed."; + return -1; + } + if (gctrl_->enable_limiter(true) != 0) { + RTC_LOG(LS_ERROR) << "enable_limiter(true) failed."; + return -1; + } + return 0; +} + +void AgcManagerDirect::AnalyzePreProcess(int16_t* audio, + int num_channels, + size_t samples_per_channel) { + size_t length = num_channels * samples_per_channel; + if (capture_muted_) { + return; + } + + file_preproc_->Write(audio, length); + + if (frames_since_clipped_ < kClippedWaitFrames) { + ++frames_since_clipped_; + return; + } + + // Check for clipped samples, as the AGC has difficulty detecting pitch + // under clipping distortion. We do this in the preprocessing phase in order + // to catch clipped echo as well. + // + // If we find a sufficiently clipped frame, drop the current microphone level + // and enforce a new maximum level, dropped the same amount from the current + // maximum. This harsh treatment is an effort to avoid repeated clipped echo + // events. As compensation for this restriction, the maximum compression + // gain is increased, through SetMaxLevel(). + float clipped_ratio = agc_->AnalyzePreproc(audio, length); + if (clipped_ratio > kClippedRatioThreshold) { + RTC_LOG(LS_INFO) << "[agc] Clipping detected. clipped_ratio=" + << clipped_ratio; + // Always decrease the maximum level, even if the current level is below + // threshold. + SetMaxLevel(std::max(clipped_level_min_, max_level_ - kClippedLevelStep)); + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.AgcClippingAdjustmentAllowed", + level_ - kClippedLevelStep >= clipped_level_min_); + if (level_ > clipped_level_min_) { + // Don't try to adjust the level if we're already below the limit. As + // a consequence, if the user has brought the level above the limit, we + // will still not react until the postproc updates the level. + SetLevel(std::max(clipped_level_min_, level_ - kClippedLevelStep)); + // Reset the AGC since the level has changed. + agc_->Reset(); + } + frames_since_clipped_ = 0; + } +} + +void AgcManagerDirect::Process(const int16_t* audio, + size_t length, + int sample_rate_hz) { + if (capture_muted_) { + return; + } + + if (check_volume_on_next_process_) { + check_volume_on_next_process_ = false; + // We have to wait until the first process call to check the volume, + // because Chromium doesn't guarantee it to be valid any earlier. + CheckVolumeAndReset(); + } + + if (agc_->Process(audio, length, sample_rate_hz) != 0) { + RTC_LOG(LS_ERROR) << "Agc::Process failed"; + RTC_NOTREACHED(); + } + + UpdateGain(); + UpdateCompressor(); + + file_postproc_->Write(audio, length); +} + +void AgcManagerDirect::SetLevel(int new_level) { + int voe_level = volume_callbacks_->GetMicVolume(); + if (voe_level < 0) { + return; + } + if (voe_level == 0) { + RTC_LOG(LS_INFO) + << "[agc] VolumeCallbacks returned level=0, taking no action."; + return; + } + if (voe_level > kMaxMicLevel) { + RTC_LOG(LS_ERROR) << "VolumeCallbacks returned an invalid level=" + << voe_level; + return; + } + + if (voe_level > level_ + kLevelQuantizationSlack || + voe_level < level_ - kLevelQuantizationSlack) { + RTC_LOG(LS_INFO) << "[agc] Mic volume was manually adjusted. Updating " + << "stored level from " << level_ << " to " << voe_level; + level_ = voe_level; + // Always allow the user to increase the volume. + if (level_ > max_level_) { + SetMaxLevel(level_); + } + // Take no action in this case, since we can't be sure when the volume + // was manually adjusted. The compressor will still provide some of the + // desired gain change. + agc_->Reset(); + return; + } + + new_level = std::min(new_level, max_level_); + if (new_level == level_) { + return; + } + + volume_callbacks_->SetMicVolume(new_level); + RTC_LOG(LS_INFO) << "[agc] voe_level=" << voe_level << ", " + << "level_=" << level_ << ", " + << "new_level=" << new_level; + level_ = new_level; +} + +void AgcManagerDirect::SetMaxLevel(int level) { + RTC_DCHECK_GE(level, clipped_level_min_); + max_level_ = level; + // Scale the |kSurplusCompressionGain| linearly across the restricted + // level range. + max_compression_gain_ = + kMaxCompressionGain + std::floor((1.f * kMaxMicLevel - max_level_) / + (kMaxMicLevel - clipped_level_min_) * + kSurplusCompressionGain + + 0.5f); + RTC_LOG(LS_INFO) << "[agc] max_level_=" << max_level_ + << ", max_compression_gain_=" << max_compression_gain_; +} + +void AgcManagerDirect::SetCaptureMuted(bool muted) { + if (capture_muted_ == muted) { + return; + } + capture_muted_ = muted; + + if (!muted) { + // When we unmute, we should reset things to be safe. + check_volume_on_next_process_ = true; + } +} + +float AgcManagerDirect::voice_probability() { + return agc_->voice_probability(); +} + +int AgcManagerDirect::CheckVolumeAndReset() { + int level = volume_callbacks_->GetMicVolume(); + if (level < 0) { + return -1; + } + // Reasons for taking action at startup: + // 1) A person starting a call is expected to be heard. + // 2) Independent of interpretation of |level| == 0 we should raise it so the + // AGC can do its job properly. + if (level == 0 && !startup_) { + RTC_LOG(LS_INFO) + << "[agc] VolumeCallbacks returned level=0, taking no action."; + return 0; + } + if (level > kMaxMicLevel) { + RTC_LOG(LS_ERROR) << "VolumeCallbacks returned an invalid level=" << level; + return -1; + } + RTC_LOG(LS_INFO) << "[agc] Initial GetMicVolume()=" << level; + + int minLevel = startup_ ? startup_min_level_ : kMinMicLevel; + if (level < minLevel) { + level = minLevel; + RTC_LOG(LS_INFO) << "[agc] Initial volume too low, raising to " << level; + volume_callbacks_->SetMicVolume(level); + } + agc_->Reset(); + level_ = level; + startup_ = false; + return 0; +} + +// Requests the RMS error from AGC and distributes the required gain change +// between the digital compression stage and volume slider. We use the +// compressor first, providing a slack region around the current slider +// position to reduce movement. +// +// If the slider needs to be moved, we check first if the user has adjusted +// it, in which case we take no action and cache the updated level. +void AgcManagerDirect::UpdateGain() { + int rms_error = 0; + if (!agc_->GetRmsErrorDb(&rms_error)) { + // No error update ready. + return; + } + // The compressor will always add at least kMinCompressionGain. In effect, + // this adjusts our target gain upward by the same amount and rms_error + // needs to reflect that. + rms_error += kMinCompressionGain; + + // Handle as much error as possible with the compressor first. + int raw_compression = + rtc::SafeClamp(rms_error, kMinCompressionGain, max_compression_gain_); + + // Deemphasize the compression gain error. Move halfway between the current + // target and the newly received target. This serves to soften perceptible + // intra-talkspurt adjustments, at the cost of some adaptation speed. + if ((raw_compression == max_compression_gain_ && + target_compression_ == max_compression_gain_ - 1) || + (raw_compression == kMinCompressionGain && + target_compression_ == kMinCompressionGain + 1)) { + // Special case to allow the target to reach the endpoints of the + // compression range. The deemphasis would otherwise halt it at 1 dB shy. + target_compression_ = raw_compression; + } else { + target_compression_ = (raw_compression - target_compression_) / 2 + + target_compression_; + } + + // Residual error will be handled by adjusting the volume slider. Use the + // raw rather than deemphasized compression here as we would otherwise + // shrink the amount of slack the compressor provides. + const int residual_gain = + rtc::SafeClamp(rms_error - raw_compression, -kMaxResidualGainChange, + kMaxResidualGainChange); + RTC_LOG(LS_INFO) << "[agc] rms_error=" << rms_error << ", " + << "target_compression=" << target_compression_ << ", " + << "residual_gain=" << residual_gain; + if (residual_gain == 0) + return; + + int old_level = level_; + SetLevel(LevelFromGainError(residual_gain, level_)); + if (old_level != level_) { + // level_ was updated by SetLevel; log the new value. + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.AgcSetLevel", level_, 1, + kMaxMicLevel, 50); + } +} + +void AgcManagerDirect::UpdateCompressor() { + if (compression_ == target_compression_) { + return; + } + + // Adapt the compression gain slowly towards the target, in order to avoid + // highly perceptible changes. + if (target_compression_ > compression_) { + compression_accumulator_ += kCompressionGainStep; + } else { + compression_accumulator_ -= kCompressionGainStep; + } + + // The compressor accepts integer gains in dB. Adjust the gain when + // we've come within half a stepsize of the nearest integer. (We don't + // check for equality due to potential floating point imprecision). + int new_compression = compression_; + int nearest_neighbor = std::floor(compression_accumulator_ + 0.5); + if (std::fabs(compression_accumulator_ - nearest_neighbor) < + kCompressionGainStep / 2) { + new_compression = nearest_neighbor; + } + + // Set the new compression gain. + if (new_compression != compression_) { + compression_ = new_compression; + compression_accumulator_ = new_compression; + if (gctrl_->set_compression_gain_db(compression_) != 0) { + RTC_LOG(LS_ERROR) << "set_compression_gain_db(" << compression_ + << ") failed."; + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.h new file mode 100644 index 0000000000..03d2607f85 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_AGC_MANAGER_DIRECT_H_ +#define MODULES_AUDIO_PROCESSING_AGC_AGC_MANAGER_DIRECT_H_ + +#include + +#include "modules/audio_processing/agc/agc.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class AudioFrame; +class DebugFile; +class GainControl; + +// Callbacks that need to be injected into AgcManagerDirect to read and control +// the volume values. This is done to remove the VoiceEngine dependency in +// AgcManagerDirect. +// TODO(aluebs): Remove VolumeCallbacks. +class VolumeCallbacks { + public: + virtual ~VolumeCallbacks() {} + virtual void SetMicVolume(int volume) = 0; + virtual int GetMicVolume() = 0; +}; + +// Direct interface to use AGC to set volume and compression values. +// AudioProcessing uses this interface directly to integrate the callback-less +// AGC. +// +// This class is not thread-safe. +class AgcManagerDirect final { + public: + // AgcManagerDirect will configure GainControl internally. The user is + // responsible for processing the audio using it after the call to Process. + // The operating range of startup_min_level is [12, 255] and any input value + // outside that range will be clamped. + AgcManagerDirect(GainControl* gctrl, + VolumeCallbacks* volume_callbacks, + int startup_min_level, + int clipped_level_min); + // Dependency injection for testing. Don't delete |agc| as the memory is owned + // by the manager. + AgcManagerDirect(Agc* agc, + GainControl* gctrl, + VolumeCallbacks* volume_callbacks, + int startup_min_level, + int clipped_level_min); + ~AgcManagerDirect(); + + int Initialize(); + void AnalyzePreProcess(int16_t* audio, + int num_channels, + size_t samples_per_channel); + void Process(const int16_t* audio, size_t length, int sample_rate_hz); + + // Call when the capture stream has been muted/unmuted. This causes the + // manager to disregard all incoming audio; chances are good it's background + // noise to which we'd like to avoid adapting. + void SetCaptureMuted(bool muted); + bool capture_muted() { return capture_muted_; } + + float voice_probability(); + + private: + // Sets a new microphone level, after first checking that it hasn't been + // updated by the user, in which case no action is taken. + void SetLevel(int new_level); + + // Set the maximum level the AGC is allowed to apply. Also updates the + // maximum compression gain to compensate. The level must be at least + // |kClippedLevelMin|. + void SetMaxLevel(int level); + + int CheckVolumeAndReset(); + void UpdateGain(); + void UpdateCompressor(); + + std::unique_ptr agc_; + GainControl* gctrl_; + VolumeCallbacks* volume_callbacks_; + + int frames_since_clipped_; + int level_; + int max_level_; + int max_compression_gain_; + int target_compression_; + int compression_; + float compression_accumulator_; + bool capture_muted_; + bool check_volume_on_next_process_; + bool startup_; + int startup_min_level_; + const int clipped_level_min_; + + std::unique_ptr file_preproc_; + std::unique_ptr file_postproc_; + + RTC_DISALLOW_COPY_AND_ASSIGN(AgcManagerDirect); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC_AGC_MANAGER_DIRECT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc new file mode 100644 index 0000000000..6a8af6aa71 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc @@ -0,0 +1,682 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc/agc_manager_direct.h" + +#include "common_types.h" // NOLINT(build/include) +#include "modules/audio_processing/agc/mock_agc.h" +#include "modules/audio_processing/include/mock_audio_processing.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::DoAll; +using ::testing::Return; +using ::testing::SetArgPointee; + +namespace webrtc { +namespace { + +const int kSampleRateHz = 32000; +const int kNumChannels = 1; +const int kSamplesPerChannel = kSampleRateHz / 100; +const int kInitialVolume = 128; +constexpr int kClippedMin = 165; // Arbitrary, but different from the default. +const float kAboveClippedThreshold = 0.2f; + +class TestVolumeCallbacks : public VolumeCallbacks { + public: + TestVolumeCallbacks() : volume_(0) {} + void SetMicVolume(int volume) override { volume_ = volume; } + int GetMicVolume() override { return volume_; } + + private: + int volume_; +}; + +} // namespace + +class AgcManagerDirectTest : public ::testing::Test { + protected: + AgcManagerDirectTest() + : agc_(new MockAgc), + manager_(agc_, &gctrl_, &volume_, kInitialVolume, kClippedMin) { + ExpectInitialize(); + manager_.Initialize(); + } + + void FirstProcess() { + EXPECT_CALL(*agc_, Reset()); + EXPECT_CALL(*agc_, GetRmsErrorDb(_)).WillOnce(Return(false)); + CallProcess(1); + } + + void SetVolumeAndProcess(int volume) { + volume_.SetMicVolume(volume); + FirstProcess(); + } + + void ExpectCheckVolumeAndReset(int volume) { + volume_.SetMicVolume(volume); + EXPECT_CALL(*agc_, Reset()); + } + + void ExpectInitialize() { + EXPECT_CALL(gctrl_, set_mode(GainControl::kFixedDigital)); + EXPECT_CALL(gctrl_, set_target_level_dbfs(2)); + EXPECT_CALL(gctrl_, set_compression_gain_db(7)); + EXPECT_CALL(gctrl_, enable_limiter(true)); + } + + void CallProcess(int num_calls) { + for (int i = 0; i < num_calls; ++i) { + EXPECT_CALL(*agc_, Process(_, _, _)).WillOnce(Return(0)); + manager_.Process(nullptr, kSamplesPerChannel, kSampleRateHz); + } + } + + void CallPreProc(int num_calls) { + for (int i = 0; i < num_calls; ++i) { + manager_.AnalyzePreProcess(nullptr, kNumChannels, kSamplesPerChannel); + } + } + + MockAgc* agc_; + test::MockGainControl gctrl_; + TestVolumeCallbacks volume_; + AgcManagerDirect manager_; +}; + +TEST_F(AgcManagerDirectTest, StartupMinVolumeConfigurationIsRespected) { + FirstProcess(); + EXPECT_EQ(kInitialVolume, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, MicVolumeResponseToRmsError) { + FirstProcess(); + + // Compressor default; no residual error. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(5), Return(true))); + CallProcess(1); + + // Inside the compressor's window; no change of volume. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(10), Return(true))); + CallProcess(1); + + // Above the compressor's window; volume should be increased. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))); + CallProcess(1); + EXPECT_EQ(130, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(20), Return(true))); + CallProcess(1); + EXPECT_EQ(168, volume_.GetMicVolume()); + + // Inside the compressor's window; no change of volume. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(5), Return(true))); + CallProcess(1); + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(0), Return(true))); + CallProcess(1); + + // Below the compressor's window; volume should be decreased. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-1), Return(true))); + CallProcess(1); + EXPECT_EQ(167, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-1), Return(true))); + CallProcess(1); + EXPECT_EQ(163, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-9), Return(true))); + CallProcess(1); + EXPECT_EQ(129, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, MicVolumeIsLimited) { + FirstProcess(); + + // Maximum upwards change is limited. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(30), Return(true))); + CallProcess(1); + EXPECT_EQ(183, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(30), Return(true))); + CallProcess(1); + EXPECT_EQ(243, volume_.GetMicVolume()); + + // Won't go higher than the maximum. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(30), Return(true))); + CallProcess(1); + EXPECT_EQ(255, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-1), Return(true))); + CallProcess(1); + EXPECT_EQ(254, volume_.GetMicVolume()); + + // Maximum downwards change is limited. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-40), Return(true))); + CallProcess(1); + EXPECT_EQ(194, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-40), Return(true))); + CallProcess(1); + EXPECT_EQ(137, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-40), Return(true))); + CallProcess(1); + EXPECT_EQ(88, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-40), Return(true))); + CallProcess(1); + EXPECT_EQ(54, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-40), Return(true))); + CallProcess(1); + EXPECT_EQ(33, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-40), Return(true))); + CallProcess(1); + EXPECT_EQ(18, volume_.GetMicVolume()); + + // Won't go lower than the minimum. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-40), Return(true))); + CallProcess(1); + EXPECT_EQ(12, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, CompressorStepsTowardsTarget) { + FirstProcess(); + + // Compressor default; no call to set_compression_gain_db. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(5), Return(true))) + .WillRepeatedly(Return(false)); + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(20); + + // Moves slowly upwards. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(9), Return(true))) + .WillRepeatedly(Return(false)); + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(8)).WillOnce(Return(0)); + CallProcess(1); + + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(9)).WillOnce(Return(0)); + CallProcess(1); + + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(20); + + // Moves slowly downward, then reverses before reaching the original target. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(5), Return(true))) + .WillRepeatedly(Return(false)); + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(8)).WillOnce(Return(0)); + CallProcess(1); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(9), Return(true))) + .WillRepeatedly(Return(false)); + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(9)).WillOnce(Return(0)); + CallProcess(1); + + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(20); +} + +TEST_F(AgcManagerDirectTest, CompressorErrorIsDeemphasized) { + FirstProcess(); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(10), Return(true))) + .WillRepeatedly(Return(false)); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(8)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(9)).WillOnce(Return(0)); + CallProcess(1); + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(20); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(0), Return(true))) + .WillRepeatedly(Return(false)); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(8)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(7)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(6)).WillOnce(Return(0)); + CallProcess(1); + EXPECT_CALL(gctrl_, set_compression_gain_db(_)).Times(0); + CallProcess(20); +} + +TEST_F(AgcManagerDirectTest, CompressorReachesMaximum) { + FirstProcess(); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(10), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(10), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(10), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(10), Return(true))) + .WillRepeatedly(Return(false)); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(8)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(9)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(10)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(11)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(12)).WillOnce(Return(0)); + CallProcess(1); +} + +TEST_F(AgcManagerDirectTest, CompressorReachesMinimum) { + FirstProcess(); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(0), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(0), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(0), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(0), Return(true))) + .WillRepeatedly(Return(false)); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(6)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(5)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(4)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(3)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(2)).WillOnce(Return(0)); + CallProcess(1); +} + +TEST_F(AgcManagerDirectTest, NoActionWhileMuted) { + manager_.SetCaptureMuted(true); + manager_.Process(nullptr, kSamplesPerChannel, kSampleRateHz); +} + +TEST_F(AgcManagerDirectTest, UnmutingChecksVolumeWithoutRaising) { + FirstProcess(); + + manager_.SetCaptureMuted(true); + manager_.SetCaptureMuted(false); + ExpectCheckVolumeAndReset(127); + // SetMicVolume should not be called. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)).WillOnce(Return(false)); + CallProcess(1); + EXPECT_EQ(127, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, UnmutingRaisesTooLowVolume) { + FirstProcess(); + + manager_.SetCaptureMuted(true); + manager_.SetCaptureMuted(false); + ExpectCheckVolumeAndReset(11); + EXPECT_CALL(*agc_, GetRmsErrorDb(_)).WillOnce(Return(false)); + CallProcess(1); + EXPECT_EQ(12, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, ManualLevelChangeResultsInNoSetMicCall) { + FirstProcess(); + + // Change outside of compressor's range, which would normally trigger a call + // to SetMicVolume. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))); + // GetMicVolume returns a value outside of the quantization slack, indicating + // a manual volume change. + volume_.SetMicVolume(154); + // SetMicVolume should not be called. + EXPECT_CALL(*agc_, Reset()).Times(1); + CallProcess(1); + EXPECT_EQ(154, volume_.GetMicVolume()); + + // Do the same thing, except downwards now. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-1), Return(true))); + volume_.SetMicVolume(100); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallProcess(1); + EXPECT_EQ(100, volume_.GetMicVolume()); + + // And finally verify the AGC continues working without a manual change. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-1), Return(true))); + CallProcess(1); + EXPECT_EQ(99, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, RecoveryAfterManualLevelChangeFromMax) { + FirstProcess(); + + // Force the mic up to max volume. Takes a few steps due to the residual + // gain limitation. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(30), Return(true))); + CallProcess(1); + EXPECT_EQ(183, volume_.GetMicVolume()); + CallProcess(1); + EXPECT_EQ(243, volume_.GetMicVolume()); + CallProcess(1); + EXPECT_EQ(255, volume_.GetMicVolume()); + + // Manual change does not result in SetMicVolume call. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-1), Return(true))); + volume_.SetMicVolume(50); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallProcess(1); + EXPECT_EQ(50, volume_.GetMicVolume()); + + // Continues working as usual afterwards. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(20), Return(true))); + CallProcess(1); + EXPECT_EQ(69, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, RecoveryAfterManualLevelChangeBelowMin) { + FirstProcess(); + + // Manual change below min. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-1), Return(true))); + // Don't set to zero, which will cause AGC to take no action. + volume_.SetMicVolume(1); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallProcess(1); + EXPECT_EQ(1, volume_.GetMicVolume()); + + // Continues working as usual afterwards. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))); + CallProcess(1); + EXPECT_EQ(2, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(30), Return(true))); + CallProcess(1); + EXPECT_EQ(11, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(20), Return(true))); + CallProcess(1); + EXPECT_EQ(18, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, NoClippingHasNoImpact) { + FirstProcess(); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)).WillRepeatedly(Return(0)); + CallPreProc(100); + EXPECT_EQ(128, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, ClippingUnderThresholdHasNoImpact) { + FirstProcess(); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)).WillOnce(Return(0.099)); + CallPreProc(1); + EXPECT_EQ(128, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, ClippingLowersVolume) { + SetVolumeAndProcess(255); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)).WillOnce(Return(0.101)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(240, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, WaitingPeriodBetweenClippingChecks) { + SetVolumeAndProcess(255); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(240, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillRepeatedly(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(0); + CallPreProc(300); + EXPECT_EQ(240, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(225, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, ClippingLoweringIsLimited) { + SetVolumeAndProcess(180); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(kClippedMin, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillRepeatedly(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(0); + CallPreProc(1000); + EXPECT_EQ(kClippedMin, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, ClippingMaxIsRespectedWhenEqualToLevel) { + SetVolumeAndProcess(255); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(240, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(30), Return(true))); + CallProcess(10); + EXPECT_EQ(240, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, ClippingMaxIsRespectedWhenHigherThanLevel) { + SetVolumeAndProcess(200); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(185, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(40), Return(true))); + CallProcess(1); + EXPECT_EQ(240, volume_.GetMicVolume()); + CallProcess(10); + EXPECT_EQ(240, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, MaxCompressionIsIncreasedAfterClipping) { + SetVolumeAndProcess(210); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(195, volume_.GetMicVolume()); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(11), Return(true))) + .WillRepeatedly(Return(false)); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(8)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(9)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(10)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(11)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(12)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(13)).WillOnce(Return(0)); + CallProcess(1); + + // Continue clipping until we hit the maximum surplus compression. + CallPreProc(300); + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(180, volume_.GetMicVolume()); + + CallPreProc(300); + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(kClippedMin, volume_.GetMicVolume()); + + // Current level is now at the minimum, but the maximum allowed level still + // has more to decrease. + CallPreProc(300); + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + CallPreProc(1); + + CallPreProc(300); + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + CallPreProc(1); + + CallPreProc(300); + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + CallPreProc(1); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(16), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(16), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(16), Return(true))) + .WillOnce(DoAll(SetArgPointee<0>(16), Return(true))) + .WillRepeatedly(Return(false)); + CallProcess(19); + EXPECT_CALL(gctrl_, set_compression_gain_db(14)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(15)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(16)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(17)).WillOnce(Return(0)); + CallProcess(20); + EXPECT_CALL(gctrl_, set_compression_gain_db(18)).WillOnce(Return(0)); + CallProcess(1); +} + +TEST_F(AgcManagerDirectTest, UserCanRaiseVolumeAfterClipping) { + SetVolumeAndProcess(225); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallPreProc(1); + EXPECT_EQ(210, volume_.GetMicVolume()); + + // High enough error to trigger a volume check. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(14), Return(true))); + // User changed the volume. + volume_.SetMicVolume(250); + EXPECT_CALL(*agc_, Reset()).Times(1); + CallProcess(1); + EXPECT_EQ(250, volume_.GetMicVolume()); + + // Move down... + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(-10), Return(true))); + CallProcess(1); + EXPECT_EQ(210, volume_.GetMicVolume()); + // And back up to the new max established by the user. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(40), Return(true))); + CallProcess(1); + EXPECT_EQ(250, volume_.GetMicVolume()); + // Will not move above new maximum. + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillOnce(DoAll(SetArgPointee<0>(30), Return(true))); + CallProcess(1); + EXPECT_EQ(250, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, ClippingDoesNotPullLowVolumeBackUp) { + SetVolumeAndProcess(80); + + EXPECT_CALL(*agc_, AnalyzePreproc(_, _)) + .WillOnce(Return(kAboveClippedThreshold)); + EXPECT_CALL(*agc_, Reset()).Times(0); + int initial_volume = volume_.GetMicVolume(); + CallPreProc(1); + EXPECT_EQ(initial_volume, volume_.GetMicVolume()); +} + +TEST_F(AgcManagerDirectTest, TakesNoActionOnZeroMicVolume) { + FirstProcess(); + + EXPECT_CALL(*agc_, GetRmsErrorDb(_)) + .WillRepeatedly(DoAll(SetArgPointee<0>(30), Return(true))); + volume_.SetMicVolume(0); + CallProcess(10); + EXPECT_EQ(0, volume_.GetMicVolume()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/gain_map_internal.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/gain_map_internal.h new file mode 100644 index 0000000000..f09c748b59 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/gain_map_internal.h @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_GAIN_MAP_INTERNAL_H_ +#define MODULES_AUDIO_PROCESSING_AGC_GAIN_MAP_INTERNAL_H_ + +static const int kGainMapSize = 256; +// Uses parameters: si = 2, sf = 0.25, D = 8/256 +static const int kGainMap[kGainMapSize] = { + -56, + -54, + -52, + -50, + -48, + -47, + -45, + -43, + -42, + -40, + -38, + -37, + -35, + -34, + -33, + -31, + -30, + -29, + -27, + -26, + -25, + -24, + -23, + -22, + -20, + -19, + -18, + -17, + -16, + -15, + -14, + -14, + -13, + -12, + -11, + -10, + -9, + -8, + -8, + -7, + -6, + -5, + -5, + -4, + -3, + -2, + -2, + -1, + 0, + 0, + 1, + 1, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 6, + 7, + 7, + 8, + 8, + 9, + 9, + 10, + 10, + 11, + 11, + 12, + 12, + 13, + 13, + 13, + 14, + 14, + 15, + 15, + 15, + 16, + 16, + 17, + 17, + 17, + 18, + 18, + 18, + 19, + 19, + 19, + 20, + 20, + 21, + 21, + 21, + 22, + 22, + 22, + 23, + 23, + 23, + 24, + 24, + 24, + 24, + 25, + 25, + 25, + 26, + 26, + 26, + 27, + 27, + 27, + 28, + 28, + 28, + 28, + 29, + 29, + 29, + 30, + 30, + 30, + 30, + 31, + 31, + 31, + 32, + 32, + 32, + 32, + 33, + 33, + 33, + 33, + 34, + 34, + 34, + 35, + 35, + 35, + 35, + 36, + 36, + 36, + 36, + 37, + 37, + 37, + 38, + 38, + 38, + 38, + 39, + 39, + 39, + 39, + 40, + 40, + 40, + 40, + 41, + 41, + 41, + 41, + 42, + 42, + 42, + 42, + 43, + 43, + 43, + 44, + 44, + 44, + 44, + 45, + 45, + 45, + 45, + 46, + 46, + 46, + 46, + 47, + 47, + 47, + 47, + 48, + 48, + 48, + 48, + 49, + 49, + 49, + 49, + 50, + 50, + 50, + 50, + 51, + 51, + 51, + 51, + 52, + 52, + 52, + 52, + 53, + 53, + 53, + 53, + 54, + 54, + 54, + 54, + 55, + 55, + 55, + 55, + 56, + 56, + 56, + 56, + 57, + 57, + 57, + 57, + 58, + 58, + 58, + 58, + 59, + 59, + 59, + 59, + 60, + 60, + 60, + 60, + 61, + 61, + 61, + 61, + 62, + 62, + 62, + 62, + 63, + 63, + 63, + 63, + 64 +}; + +#endif // MODULES_AUDIO_PROCESSING_AGC_GAIN_MAP_INTERNAL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.c b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.c new file mode 100644 index 0000000000..662e88b6e7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.c @@ -0,0 +1,1390 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* analog_agc.c + * + * Using a feedback system, determines an appropriate analog volume level + * given an input signal and current volume level. Targets a conservative + * signal level and is intended for use with a digital AGC to apply + * additional gain. + * + */ + +#include "modules/audio_processing/agc/legacy/analog_agc.h" + +#include +#ifdef WEBRTC_AGC_DEBUG_DUMP +#include +#endif + +#include "rtc_base/checks.h" + +/* The slope of in Q13*/ +static const int16_t kSlope1[8] = {21793, 12517, 7189, 4129, + 2372, 1362, 472, 78}; + +/* The offset in Q14 */ +static const int16_t kOffset1[8] = {25395, 23911, 22206, 20737, + 19612, 18805, 17951, 17367}; + +/* The slope of in Q13*/ +static const int16_t kSlope2[8] = {2063, 1731, 1452, 1218, 1021, 857, 597, 337}; + +/* The offset in Q14 */ +static const int16_t kOffset2[8] = {18432, 18379, 18290, 18177, + 18052, 17920, 17670, 17286}; + +static const int16_t kMuteGuardTimeMs = 8000; +static const int16_t kInitCheck = 42; +static const size_t kNumSubframes = 10; + +/* Default settings if config is not used */ +#define AGC_DEFAULT_TARGET_LEVEL 3 +#define AGC_DEFAULT_COMP_GAIN 9 +/* This is the target level for the analog part in ENV scale. To convert to RMS + * scale you + * have to add OFFSET_ENV_TO_RMS. + */ +#define ANALOG_TARGET_LEVEL 11 +#define ANALOG_TARGET_LEVEL_2 5 // ANALOG_TARGET_LEVEL / 2 +/* Offset between RMS scale (analog part) and ENV scale (digital part). This + * value actually + * varies with the FIXED_ANALOG_TARGET_LEVEL, hence we should in the future + * replace it with + * a table. + */ +#define OFFSET_ENV_TO_RMS 9 +/* The reference input level at which the digital part gives an output of + * targetLevelDbfs + * (desired level) if we have no compression gain. This level should be set high + * enough not + * to compress the peaks due to the dynamics. + */ +#define DIGITAL_REF_AT_0_COMP_GAIN 4 +/* Speed of reference level decrease. + */ +#define DIFF_REF_TO_ANALOG 5 + +#ifdef MIC_LEVEL_FEEDBACK +#define NUM_BLOCKS_IN_SAT_BEFORE_CHANGE_TARGET 7 +#endif +/* Size of analog gain table */ +#define GAIN_TBL_LEN 32 +/* Matlab code: + * fprintf(1, '\t%i, %i, %i, %i,\n', round(10.^(linspace(0,10,32)/20) * 2^12)); + */ +/* Q12 */ +static const uint16_t kGainTableAnalog[GAIN_TBL_LEN] = { + 4096, 4251, 4412, 4579, 4752, 4932, 5118, 5312, 5513, 5722, 5938, + 6163, 6396, 6638, 6889, 7150, 7420, 7701, 7992, 8295, 8609, 8934, + 9273, 9623, 9987, 10365, 10758, 11165, 11587, 12025, 12480, 12953}; + +/* Gain/Suppression tables for virtual Mic (in Q10) */ +static const uint16_t kGainTableVirtualMic[128] = { + 1052, 1081, 1110, 1141, 1172, 1204, 1237, 1271, 1305, 1341, 1378, + 1416, 1454, 1494, 1535, 1577, 1620, 1664, 1710, 1757, 1805, 1854, + 1905, 1957, 2010, 2065, 2122, 2180, 2239, 2301, 2364, 2428, 2495, + 2563, 2633, 2705, 2779, 2855, 2933, 3013, 3096, 3180, 3267, 3357, + 3449, 3543, 3640, 3739, 3842, 3947, 4055, 4166, 4280, 4397, 4517, + 4640, 4767, 4898, 5032, 5169, 5311, 5456, 5605, 5758, 5916, 6078, + 6244, 6415, 6590, 6770, 6956, 7146, 7341, 7542, 7748, 7960, 8178, + 8402, 8631, 8867, 9110, 9359, 9615, 9878, 10148, 10426, 10711, 11004, + 11305, 11614, 11932, 12258, 12593, 12938, 13292, 13655, 14029, 14412, 14807, + 15212, 15628, 16055, 16494, 16945, 17409, 17885, 18374, 18877, 19393, 19923, + 20468, 21028, 21603, 22194, 22801, 23425, 24065, 24724, 25400, 26095, 26808, + 27541, 28295, 29069, 29864, 30681, 31520, 32382}; +static const uint16_t kSuppressionTableVirtualMic[128] = { + 1024, 1006, 988, 970, 952, 935, 918, 902, 886, 870, 854, 839, 824, 809, 794, + 780, 766, 752, 739, 726, 713, 700, 687, 675, 663, 651, 639, 628, 616, 605, + 594, 584, 573, 563, 553, 543, 533, 524, 514, 505, 496, 487, 478, 470, 461, + 453, 445, 437, 429, 421, 414, 406, 399, 392, 385, 378, 371, 364, 358, 351, + 345, 339, 333, 327, 321, 315, 309, 304, 298, 293, 288, 283, 278, 273, 268, + 263, 258, 254, 249, 244, 240, 236, 232, 227, 223, 219, 215, 211, 208, 204, + 200, 197, 193, 190, 186, 183, 180, 176, 173, 170, 167, 164, 161, 158, 155, + 153, 150, 147, 145, 142, 139, 137, 134, 132, 130, 127, 125, 123, 121, 118, + 116, 114, 112, 110, 108, 106, 104, 102}; + +/* Table for target energy levels. Values in Q(-7) + * Matlab code + * targetLevelTable = fprintf('%d,\t%d,\t%d,\t%d,\n', + * round((32767*10.^(-(0:63)'/20)).^2*16/2^7) */ + +static const int32_t kTargetLevelTable[64] = { + 134209536, 106606424, 84680493, 67264106, 53429779, 42440782, 33711911, + 26778323, 21270778, 16895980, 13420954, 10660642, 8468049, 6726411, + 5342978, 4244078, 3371191, 2677832, 2127078, 1689598, 1342095, + 1066064, 846805, 672641, 534298, 424408, 337119, 267783, + 212708, 168960, 134210, 106606, 84680, 67264, 53430, + 42441, 33712, 26778, 21271, 16896, 13421, 10661, + 8468, 6726, 5343, 4244, 3371, 2678, 2127, + 1690, 1342, 1066, 847, 673, 534, 424, + 337, 268, 213, 169, 134, 107, 85, + 67}; + +int WebRtcAgc_AddMic(void* state, + int16_t* const* in_mic, + size_t num_bands, + size_t samples) { + int32_t nrg, max_nrg, sample, tmp32; + int32_t* ptr; + uint16_t targetGainIdx, gain; + size_t i; + int16_t n, L, tmp16, tmp_speech[16]; + LegacyAgc* stt; + stt = (LegacyAgc*)state; + + if (stt->fs == 8000) { + L = 8; + if (samples != 80) { + return -1; + } + } else { + L = 16; + if (samples != 160) { + return -1; + } + } + + /* apply slowly varying digital gain */ + if (stt->micVol > stt->maxAnalog) { + /* |maxLevel| is strictly >= |micVol|, so this condition should be + * satisfied here, ensuring there is no divide-by-zero. */ + RTC_DCHECK_GT(stt->maxLevel, stt->maxAnalog); + + /* Q1 */ + tmp16 = (int16_t)(stt->micVol - stt->maxAnalog); + tmp32 = (GAIN_TBL_LEN - 1) * tmp16; + tmp16 = (int16_t)(stt->maxLevel - stt->maxAnalog); + targetGainIdx = tmp32 / tmp16; + RTC_DCHECK_LT(targetGainIdx, GAIN_TBL_LEN); + + /* Increment through the table towards the target gain. + * If micVol drops below maxAnalog, we allow the gain + * to be dropped immediately. */ + if (stt->gainTableIdx < targetGainIdx) { + stt->gainTableIdx++; + } else if (stt->gainTableIdx > targetGainIdx) { + stt->gainTableIdx--; + } + + /* Q12 */ + gain = kGainTableAnalog[stt->gainTableIdx]; + + for (i = 0; i < samples; i++) { + size_t j; + for (j = 0; j < num_bands; ++j) { + sample = (in_mic[j][i] * gain) >> 12; + if (sample > 32767) { + in_mic[j][i] = 32767; + } else if (sample < -32768) { + in_mic[j][i] = -32768; + } else { + in_mic[j][i] = (int16_t)sample; + } + } + } + } else { + stt->gainTableIdx = 0; + } + + /* compute envelope */ + if (stt->inQueue > 0) { + ptr = stt->env[1]; + } else { + ptr = stt->env[0]; + } + + for (i = 0; i < kNumSubframes; i++) { + /* iterate over samples */ + max_nrg = 0; + for (n = 0; n < L; n++) { + nrg = in_mic[0][i * L + n] * in_mic[0][i * L + n]; + if (nrg > max_nrg) { + max_nrg = nrg; + } + } + ptr[i] = max_nrg; + } + + /* compute energy */ + if (stt->inQueue > 0) { + ptr = stt->Rxx16w32_array[1]; + } else { + ptr = stt->Rxx16w32_array[0]; + } + + for (i = 0; i < kNumSubframes / 2; i++) { + if (stt->fs == 16000) { + WebRtcSpl_DownsampleBy2(&in_mic[0][i * 32], 32, tmp_speech, + stt->filterState); + } else { + memcpy(tmp_speech, &in_mic[0][i * 16], 16 * sizeof(short)); + } + /* Compute energy in blocks of 16 samples */ + ptr[i] = WebRtcSpl_DotProductWithScale(tmp_speech, tmp_speech, 16, 4); + } + + /* update queue information */ + if (stt->inQueue == 0) { + stt->inQueue = 1; + } else { + stt->inQueue = 2; + } + + /* call VAD (use low band only) */ + WebRtcAgc_ProcessVad(&stt->vadMic, in_mic[0], samples); + + return 0; +} + +int WebRtcAgc_AddFarend(void* state, const int16_t* in_far, size_t samples) { + LegacyAgc* stt = (LegacyAgc*)state; + + int err = WebRtcAgc_GetAddFarendError(state, samples); + + if (err != 0) + return err; + + return WebRtcAgc_AddFarendToDigital(&stt->digitalAgc, in_far, samples); +} + +int WebRtcAgc_GetAddFarendError(void* state, size_t samples) { + LegacyAgc* stt; + stt = (LegacyAgc*)state; + + if (stt == NULL) + return -1; + + if (stt->fs == 8000) { + if (samples != 80) + return -1; + } else if (stt->fs == 16000 || stt->fs == 32000 || stt->fs == 48000) { + if (samples != 160) + return -1; + } else { + return -1; + } + + return 0; +} + +int WebRtcAgc_VirtualMic(void* agcInst, + int16_t* const* in_near, + size_t num_bands, + size_t samples, + int32_t micLevelIn, + int32_t* micLevelOut) { + int32_t tmpFlt, micLevelTmp, gainIdx; + uint16_t gain; + size_t ii, j; + LegacyAgc* stt; + + uint32_t nrg; + size_t sampleCntr; + uint32_t frameNrg = 0; + uint32_t frameNrgLimit = 5500; + int16_t numZeroCrossing = 0; + const int16_t kZeroCrossingLowLim = 15; + const int16_t kZeroCrossingHighLim = 20; + + stt = (LegacyAgc*)agcInst; + + /* + * Before applying gain decide if this is a low-level signal. + * The idea is that digital AGC will not adapt to low-level + * signals. + */ + if (stt->fs != 8000) { + frameNrgLimit = frameNrgLimit << 1; + } + + frameNrg = (uint32_t)(in_near[0][0] * in_near[0][0]); + for (sampleCntr = 1; sampleCntr < samples; sampleCntr++) { + // increment frame energy if it is less than the limit + // the correct value of the energy is not important + if (frameNrg < frameNrgLimit) { + nrg = (uint32_t)(in_near[0][sampleCntr] * in_near[0][sampleCntr]); + frameNrg += nrg; + } + + // Count the zero crossings + numZeroCrossing += + ((in_near[0][sampleCntr] ^ in_near[0][sampleCntr - 1]) < 0); + } + + if ((frameNrg < 500) || (numZeroCrossing <= 5)) { + stt->lowLevelSignal = 1; + } else if (numZeroCrossing <= kZeroCrossingLowLim) { + stt->lowLevelSignal = 0; + } else if (frameNrg <= frameNrgLimit) { + stt->lowLevelSignal = 1; + } else if (numZeroCrossing >= kZeroCrossingHighLim) { + stt->lowLevelSignal = 1; + } else { + stt->lowLevelSignal = 0; + } + + micLevelTmp = micLevelIn << stt->scale; + /* Set desired level */ + gainIdx = stt->micVol; + if (stt->micVol > stt->maxAnalog) { + gainIdx = stt->maxAnalog; + } + if (micLevelTmp != stt->micRef) { + /* Something has happened with the physical level, restart. */ + stt->micRef = micLevelTmp; + stt->micVol = 127; + *micLevelOut = 127; + stt->micGainIdx = 127; + gainIdx = 127; + } + /* Pre-process the signal to emulate the microphone level. */ + /* Take one step at a time in the gain table. */ + if (gainIdx > 127) { + gain = kGainTableVirtualMic[gainIdx - 128]; + } else { + gain = kSuppressionTableVirtualMic[127 - gainIdx]; + } + for (ii = 0; ii < samples; ii++) { + tmpFlt = (in_near[0][ii] * gain) >> 10; + if (tmpFlt > 32767) { + tmpFlt = 32767; + gainIdx--; + if (gainIdx >= 127) { + gain = kGainTableVirtualMic[gainIdx - 127]; + } else { + gain = kSuppressionTableVirtualMic[127 - gainIdx]; + } + } + if (tmpFlt < -32768) { + tmpFlt = -32768; + gainIdx--; + if (gainIdx >= 127) { + gain = kGainTableVirtualMic[gainIdx - 127]; + } else { + gain = kSuppressionTableVirtualMic[127 - gainIdx]; + } + } + in_near[0][ii] = (int16_t)tmpFlt; + for (j = 1; j < num_bands; ++j) { + tmpFlt = (in_near[j][ii] * gain) >> 10; + if (tmpFlt > 32767) { + tmpFlt = 32767; + } + if (tmpFlt < -32768) { + tmpFlt = -32768; + } + in_near[j][ii] = (int16_t)tmpFlt; + } + } + /* Set the level we (finally) used */ + stt->micGainIdx = gainIdx; + // *micLevelOut = stt->micGainIdx; + *micLevelOut = stt->micGainIdx >> stt->scale; + /* Add to Mic as if it was the output from a true microphone */ + if (WebRtcAgc_AddMic(agcInst, in_near, num_bands, samples) != 0) { + return -1; + } + return 0; +} + +void WebRtcAgc_UpdateAgcThresholds(LegacyAgc* stt) { + int16_t tmp16; +#ifdef MIC_LEVEL_FEEDBACK + int zeros; + + if (stt->micLvlSat) { + /* Lower the analog target level since we have reached its maximum */ + zeros = WebRtcSpl_NormW32(stt->Rxx160_LPw32); + stt->targetIdxOffset = (3 * zeros - stt->targetIdx - 2) / 4; + } +#endif + + /* Set analog target level in envelope dBOv scale */ + tmp16 = (DIFF_REF_TO_ANALOG * stt->compressionGaindB) + ANALOG_TARGET_LEVEL_2; + tmp16 = WebRtcSpl_DivW32W16ResW16((int32_t)tmp16, ANALOG_TARGET_LEVEL); + stt->analogTarget = DIGITAL_REF_AT_0_COMP_GAIN + tmp16; + if (stt->analogTarget < DIGITAL_REF_AT_0_COMP_GAIN) { + stt->analogTarget = DIGITAL_REF_AT_0_COMP_GAIN; + } + if (stt->agcMode == kAgcModeFixedDigital) { + /* Adjust for different parameter interpretation in FixedDigital mode */ + stt->analogTarget = stt->compressionGaindB; + } +#ifdef MIC_LEVEL_FEEDBACK + stt->analogTarget += stt->targetIdxOffset; +#endif + /* Since the offset between RMS and ENV is not constant, we should make this + * into a + * table, but for now, we'll stick with a constant, tuned for the chosen + * analog + * target level. + */ + stt->targetIdx = ANALOG_TARGET_LEVEL + OFFSET_ENV_TO_RMS; +#ifdef MIC_LEVEL_FEEDBACK + stt->targetIdx += stt->targetIdxOffset; +#endif + /* Analog adaptation limits */ + /* analogTargetLevel = round((32767*10^(-targetIdx/20))^2*16/2^7) */ + stt->analogTargetLevel = + RXX_BUFFER_LEN * kTargetLevelTable[stt->targetIdx]; /* ex. -20 dBov */ + stt->startUpperLimit = + RXX_BUFFER_LEN * kTargetLevelTable[stt->targetIdx - 1]; /* -19 dBov */ + stt->startLowerLimit = + RXX_BUFFER_LEN * kTargetLevelTable[stt->targetIdx + 1]; /* -21 dBov */ + stt->upperPrimaryLimit = + RXX_BUFFER_LEN * kTargetLevelTable[stt->targetIdx - 2]; /* -18 dBov */ + stt->lowerPrimaryLimit = + RXX_BUFFER_LEN * kTargetLevelTable[stt->targetIdx + 2]; /* -22 dBov */ + stt->upperSecondaryLimit = + RXX_BUFFER_LEN * kTargetLevelTable[stt->targetIdx - 5]; /* -15 dBov */ + stt->lowerSecondaryLimit = + RXX_BUFFER_LEN * kTargetLevelTable[stt->targetIdx + 5]; /* -25 dBov */ + stt->upperLimit = stt->startUpperLimit; + stt->lowerLimit = stt->startLowerLimit; +} + +void WebRtcAgc_SaturationCtrl(LegacyAgc* stt, + uint8_t* saturated, + int32_t* env) { + int16_t i, tmpW16; + + /* Check if the signal is saturated */ + for (i = 0; i < 10; i++) { + tmpW16 = (int16_t)(env[i] >> 20); + if (tmpW16 > 875) { + stt->envSum += tmpW16; + } + } + + if (stt->envSum > 25000) { + *saturated = 1; + stt->envSum = 0; + } + + /* stt->envSum *= 0.99; */ + stt->envSum = (int16_t)((stt->envSum * 32440) >> 15); +} + +void WebRtcAgc_ZeroCtrl(LegacyAgc* stt, int32_t* inMicLevel, int32_t* env) { + int16_t i; + int64_t tmp = 0; + int32_t midVal; + + /* Is the input signal zero? */ + for (i = 0; i < 10; i++) { + tmp += env[i]; + } + + /* Each block is allowed to have a few non-zero + * samples. + */ + if (tmp < 500) { + stt->msZero += 10; + } else { + stt->msZero = 0; + } + + if (stt->muteGuardMs > 0) { + stt->muteGuardMs -= 10; + } + + if (stt->msZero > 500) { + stt->msZero = 0; + + /* Increase microphone level only if it's less than 50% */ + midVal = (stt->maxAnalog + stt->minLevel + 1) / 2; + if (*inMicLevel < midVal) { + /* *inMicLevel *= 1.1; */ + *inMicLevel = (1126 * *inMicLevel) >> 10; + /* Reduces risk of a muted mic repeatedly triggering excessive levels due + * to zero signal detection. */ + *inMicLevel = WEBRTC_SPL_MIN(*inMicLevel, stt->zeroCtrlMax); + stt->micVol = *inMicLevel; + } + +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, + "\t\tAGC->zeroCntrl, frame %d: 500 ms under threshold," + " micVol: %d\n", + stt->fcount, stt->micVol); +#endif + + stt->activeSpeech = 0; + stt->Rxx16_LPw32Max = 0; + + /* The AGC has a tendency (due to problems with the VAD parameters), to + * vastly increase the volume after a muting event. This timer prevents + * upwards adaptation for a short period. */ + stt->muteGuardMs = kMuteGuardTimeMs; + } +} + +void WebRtcAgc_SpeakerInactiveCtrl(LegacyAgc* stt) { + /* Check if the near end speaker is inactive. + * If that is the case the VAD threshold is + * increased since the VAD speech model gets + * more sensitive to any sound after a long + * silence. + */ + + int32_t tmp32; + int16_t vadThresh; + + if (stt->vadMic.stdLongTerm < 2500) { + stt->vadThreshold = 1500; + } else { + vadThresh = kNormalVadThreshold; + if (stt->vadMic.stdLongTerm < 4500) { + /* Scale between min and max threshold */ + vadThresh += (4500 - stt->vadMic.stdLongTerm) / 2; + } + + /* stt->vadThreshold = (31 * stt->vadThreshold + vadThresh) / 32; */ + tmp32 = vadThresh + 31 * stt->vadThreshold; + stt->vadThreshold = (int16_t)(tmp32 >> 5); + } +} + +void WebRtcAgc_ExpCurve(int16_t volume, int16_t* index) { + // volume in Q14 + // index in [0-7] + /* 8 different curves */ + if (volume > 5243) { + if (volume > 7864) { + if (volume > 12124) { + *index = 7; + } else { + *index = 6; + } + } else { + if (volume > 6554) { + *index = 5; + } else { + *index = 4; + } + } + } else { + if (volume > 2621) { + if (volume > 3932) { + *index = 3; + } else { + *index = 2; + } + } else { + if (volume > 1311) { + *index = 1; + } else { + *index = 0; + } + } + } +} + +int32_t WebRtcAgc_ProcessAnalog(void* state, + int32_t inMicLevel, + int32_t* outMicLevel, + int16_t vadLogRatio, + int16_t echo, + uint8_t* saturationWarning) { + uint32_t tmpU32; + int32_t Rxx16w32, tmp32; + int32_t inMicLevelTmp, lastMicVol; + int16_t i; + uint8_t saturated = 0; + LegacyAgc* stt; + + stt = (LegacyAgc*)state; + inMicLevelTmp = inMicLevel << stt->scale; + + if (inMicLevelTmp > stt->maxAnalog) { +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "\tAGC->ProcessAnalog, frame %d: micLvl > maxAnalog\n", + stt->fcount); +#endif + return -1; + } else if (inMicLevelTmp < stt->minLevel) { +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "\tAGC->ProcessAnalog, frame %d: micLvl < minLevel\n", + stt->fcount); +#endif + return -1; + } + + if (stt->firstCall == 0) { + int32_t tmpVol; + stt->firstCall = 1; + tmp32 = ((stt->maxLevel - stt->minLevel) * 51) >> 9; + tmpVol = (stt->minLevel + tmp32); + + /* If the mic level is very low at start, increase it! */ + if ((inMicLevelTmp < tmpVol) && (stt->agcMode == kAgcModeAdaptiveAnalog)) { + inMicLevelTmp = tmpVol; + } + stt->micVol = inMicLevelTmp; + } + + /* Set the mic level to the previous output value if there is digital input + * gain */ + if ((inMicLevelTmp == stt->maxAnalog) && (stt->micVol > stt->maxAnalog)) { + inMicLevelTmp = stt->micVol; + } + + /* If the mic level was manually changed to a very low value raise it! */ + if ((inMicLevelTmp != stt->micVol) && (inMicLevelTmp < stt->minOutput)) { + tmp32 = ((stt->maxLevel - stt->minLevel) * 51) >> 9; + inMicLevelTmp = (stt->minLevel + tmp32); + stt->micVol = inMicLevelTmp; +#ifdef MIC_LEVEL_FEEDBACK +// stt->numBlocksMicLvlSat = 0; +#endif +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, + "\tAGC->ProcessAnalog, frame %d: micLvl < minLevel by manual" + " decrease, raise vol\n", + stt->fcount); +#endif + } + + if (inMicLevelTmp != stt->micVol) { + if (inMicLevel == stt->lastInMicLevel) { + // We requested a volume adjustment, but it didn't occur. This is + // probably due to a coarse quantization of the volume slider. + // Restore the requested value to prevent getting stuck. + inMicLevelTmp = stt->micVol; + } else { + // As long as the value changed, update to match. + stt->micVol = inMicLevelTmp; + } + } + + if (inMicLevelTmp > stt->maxLevel) { + // Always allow the user to raise the volume above the maxLevel. + stt->maxLevel = inMicLevelTmp; + } + + // Store last value here, after we've taken care of manual updates etc. + stt->lastInMicLevel = inMicLevel; + lastMicVol = stt->micVol; + + /* Checks if the signal is saturated. Also a check if individual samples + * are larger than 12000 is done. If they are the counter for increasing + * the volume level is set to -100ms + */ + WebRtcAgc_SaturationCtrl(stt, &saturated, stt->env[0]); + + /* The AGC is always allowed to lower the level if the signal is saturated */ + if (saturated == 1) { + /* Lower the recording level + * Rxx160_LP is adjusted down because it is so slow it could + * cause the AGC to make wrong decisions. */ + /* stt->Rxx160_LPw32 *= 0.875; */ + stt->Rxx160_LPw32 = (stt->Rxx160_LPw32 / 8) * 7; + + stt->zeroCtrlMax = stt->micVol; + + /* stt->micVol *= 0.903; */ + tmp32 = inMicLevelTmp - stt->minLevel; + tmpU32 = WEBRTC_SPL_UMUL(29591, (uint32_t)(tmp32)); + stt->micVol = (tmpU32 >> 15) + stt->minLevel; + if (stt->micVol > lastMicVol - 2) { + stt->micVol = lastMicVol - 2; + } + inMicLevelTmp = stt->micVol; + +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, + "\tAGC->ProcessAnalog, frame %d: saturated, micVol = %d\n", + stt->fcount, stt->micVol); +#endif + + if (stt->micVol < stt->minOutput) { + *saturationWarning = 1; + } + + /* Reset counter for decrease of volume level to avoid + * decreasing too much. The saturation control can still + * lower the level if needed. */ + stt->msTooHigh = -100; + + /* Enable the control mechanism to ensure that our measure, + * Rxx160_LP, is in the correct range. This must be done since + * the measure is very slow. */ + stt->activeSpeech = 0; + stt->Rxx16_LPw32Max = 0; + + /* Reset to initial values */ + stt->msecSpeechInnerChange = kMsecSpeechInner; + stt->msecSpeechOuterChange = kMsecSpeechOuter; + stt->changeToSlowMode = 0; + + stt->muteGuardMs = 0; + + stt->upperLimit = stt->startUpperLimit; + stt->lowerLimit = stt->startLowerLimit; +#ifdef MIC_LEVEL_FEEDBACK +// stt->numBlocksMicLvlSat = 0; +#endif + } + + /* Check if the input speech is zero. If so the mic volume + * is increased. On some computers the input is zero up as high + * level as 17% */ + WebRtcAgc_ZeroCtrl(stt, &inMicLevelTmp, stt->env[0]); + + /* Check if the near end speaker is inactive. + * If that is the case the VAD threshold is + * increased since the VAD speech model gets + * more sensitive to any sound after a long + * silence. + */ + WebRtcAgc_SpeakerInactiveCtrl(stt); + + for (i = 0; i < 5; i++) { + /* Computed on blocks of 16 samples */ + + Rxx16w32 = stt->Rxx16w32_array[0][i]; + + /* Rxx160w32 in Q(-7) */ + tmp32 = (Rxx16w32 - stt->Rxx16_vectorw32[stt->Rxx16pos]) >> 3; + stt->Rxx160w32 = stt->Rxx160w32 + tmp32; + stt->Rxx16_vectorw32[stt->Rxx16pos] = Rxx16w32; + + /* Circular buffer */ + stt->Rxx16pos++; + if (stt->Rxx16pos == RXX_BUFFER_LEN) { + stt->Rxx16pos = 0; + } + + /* Rxx16_LPw32 in Q(-4) */ + tmp32 = (Rxx16w32 - stt->Rxx16_LPw32) >> kAlphaShortTerm; + stt->Rxx16_LPw32 = (stt->Rxx16_LPw32) + tmp32; + + if (vadLogRatio > stt->vadThreshold) { + /* Speech detected! */ + + /* Check if Rxx160_LP is in the correct range. If + * it is too high/low then we set it to the maximum of + * Rxx16_LPw32 during the first 200ms of speech. + */ + if (stt->activeSpeech < 250) { + stt->activeSpeech += 2; + + if (stt->Rxx16_LPw32 > stt->Rxx16_LPw32Max) { + stt->Rxx16_LPw32Max = stt->Rxx16_LPw32; + } + } else if (stt->activeSpeech == 250) { + stt->activeSpeech += 2; + tmp32 = stt->Rxx16_LPw32Max >> 3; + stt->Rxx160_LPw32 = tmp32 * RXX_BUFFER_LEN; + } + + tmp32 = (stt->Rxx160w32 - stt->Rxx160_LPw32) >> kAlphaLongTerm; + stt->Rxx160_LPw32 = stt->Rxx160_LPw32 + tmp32; + + if (stt->Rxx160_LPw32 > stt->upperSecondaryLimit) { + stt->msTooHigh += 2; + stt->msTooLow = 0; + stt->changeToSlowMode = 0; + + if (stt->msTooHigh > stt->msecSpeechOuterChange) { + stt->msTooHigh = 0; + + /* Lower the recording level */ + /* Multiply by 0.828125 which corresponds to decreasing ~0.8dB */ + tmp32 = stt->Rxx160_LPw32 >> 6; + stt->Rxx160_LPw32 = tmp32 * 53; + + /* Reduce the max gain to avoid excessive oscillation + * (but never drop below the maximum analog level). + */ + stt->maxLevel = (15 * stt->maxLevel + stt->micVol) / 16; + stt->maxLevel = WEBRTC_SPL_MAX(stt->maxLevel, stt->maxAnalog); + + stt->zeroCtrlMax = stt->micVol; + + /* 0.95 in Q15 */ + tmp32 = inMicLevelTmp - stt->minLevel; + tmpU32 = WEBRTC_SPL_UMUL(31130, (uint32_t)(tmp32)); + stt->micVol = (tmpU32 >> 15) + stt->minLevel; + if (stt->micVol > lastMicVol - 1) { + stt->micVol = lastMicVol - 1; + } + inMicLevelTmp = stt->micVol; + + /* Enable the control mechanism to ensure that our measure, + * Rxx160_LP, is in the correct range. + */ + stt->activeSpeech = 0; + stt->Rxx16_LPw32Max = 0; +#ifdef MIC_LEVEL_FEEDBACK +// stt->numBlocksMicLvlSat = 0; +#endif +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, + "\tAGC->ProcessAnalog, frame %d: measure >" + " 2ndUpperLim, micVol = %d, maxLevel = %d\n", + stt->fcount, stt->micVol, stt->maxLevel); +#endif + } + } else if (stt->Rxx160_LPw32 > stt->upperLimit) { + stt->msTooHigh += 2; + stt->msTooLow = 0; + stt->changeToSlowMode = 0; + + if (stt->msTooHigh > stt->msecSpeechInnerChange) { + /* Lower the recording level */ + stt->msTooHigh = 0; + /* Multiply by 0.828125 which corresponds to decreasing ~0.8dB */ + stt->Rxx160_LPw32 = (stt->Rxx160_LPw32 / 64) * 53; + + /* Reduce the max gain to avoid excessive oscillation + * (but never drop below the maximum analog level). + */ + stt->maxLevel = (15 * stt->maxLevel + stt->micVol) / 16; + stt->maxLevel = WEBRTC_SPL_MAX(stt->maxLevel, stt->maxAnalog); + + stt->zeroCtrlMax = stt->micVol; + + /* 0.965 in Q15 */ + tmp32 = inMicLevelTmp - stt->minLevel; + tmpU32 = + WEBRTC_SPL_UMUL(31621, (uint32_t)(inMicLevelTmp - stt->minLevel)); + stt->micVol = (tmpU32 >> 15) + stt->minLevel; + if (stt->micVol > lastMicVol - 1) { + stt->micVol = lastMicVol - 1; + } + inMicLevelTmp = stt->micVol; + +#ifdef MIC_LEVEL_FEEDBACK +// stt->numBlocksMicLvlSat = 0; +#endif +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, + "\tAGC->ProcessAnalog, frame %d: measure >" + " UpperLim, micVol = %d, maxLevel = %d\n", + stt->fcount, stt->micVol, stt->maxLevel); +#endif + } + } else if (stt->Rxx160_LPw32 < stt->lowerSecondaryLimit) { + stt->msTooHigh = 0; + stt->changeToSlowMode = 0; + stt->msTooLow += 2; + + if (stt->msTooLow > stt->msecSpeechOuterChange) { + /* Raise the recording level */ + int16_t index, weightFIX; + int16_t volNormFIX = 16384; // =1 in Q14. + + stt->msTooLow = 0; + + /* Normalize the volume level */ + tmp32 = (inMicLevelTmp - stt->minLevel) << 14; + if (stt->maxInit != stt->minLevel) { + volNormFIX = tmp32 / (stt->maxInit - stt->minLevel); + } + + /* Find correct curve */ + WebRtcAgc_ExpCurve(volNormFIX, &index); + + /* Compute weighting factor for the volume increase, 32^(-2*X)/2+1.05 + */ + weightFIX = + kOffset1[index] - (int16_t)((kSlope1[index] * volNormFIX) >> 13); + + /* stt->Rxx160_LPw32 *= 1.047 [~0.2 dB]; */ + stt->Rxx160_LPw32 = (stt->Rxx160_LPw32 / 64) * 67; + + tmp32 = inMicLevelTmp - stt->minLevel; + tmpU32 = + ((uint32_t)weightFIX * (uint32_t)(inMicLevelTmp - stt->minLevel)); + stt->micVol = (tmpU32 >> 14) + stt->minLevel; + if (stt->micVol < lastMicVol + 2) { + stt->micVol = lastMicVol + 2; + } + + inMicLevelTmp = stt->micVol; + +#ifdef MIC_LEVEL_FEEDBACK + /* Count ms in level saturation */ + // if (stt->micVol > stt->maxAnalog) { + if (stt->micVol > 150) { + /* mic level is saturated */ + stt->numBlocksMicLvlSat++; + fprintf(stderr, "Sat mic Level: %d\n", stt->numBlocksMicLvlSat); + } +#endif +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, + "\tAGC->ProcessAnalog, frame %d: measure <" + " 2ndLowerLim, micVol = %d\n", + stt->fcount, stt->micVol); +#endif + } + } else if (stt->Rxx160_LPw32 < stt->lowerLimit) { + stt->msTooHigh = 0; + stt->changeToSlowMode = 0; + stt->msTooLow += 2; + + if (stt->msTooLow > stt->msecSpeechInnerChange) { + /* Raise the recording level */ + int16_t index, weightFIX; + int16_t volNormFIX = 16384; // =1 in Q14. + + stt->msTooLow = 0; + + /* Normalize the volume level */ + tmp32 = (inMicLevelTmp - stt->minLevel) << 14; + if (stt->maxInit != stt->minLevel) { + volNormFIX = tmp32 / (stt->maxInit - stt->minLevel); + } + + /* Find correct curve */ + WebRtcAgc_ExpCurve(volNormFIX, &index); + + /* Compute weighting factor for the volume increase, (3.^(-2.*X))/8+1 + */ + weightFIX = + kOffset2[index] - (int16_t)((kSlope2[index] * volNormFIX) >> 13); + + /* stt->Rxx160_LPw32 *= 1.047 [~0.2 dB]; */ + stt->Rxx160_LPw32 = (stt->Rxx160_LPw32 / 64) * 67; + + tmp32 = inMicLevelTmp - stt->minLevel; + tmpU32 = + ((uint32_t)weightFIX * (uint32_t)(inMicLevelTmp - stt->minLevel)); + stt->micVol = (tmpU32 >> 14) + stt->minLevel; + if (stt->micVol < lastMicVol + 1) { + stt->micVol = lastMicVol + 1; + } + + inMicLevelTmp = stt->micVol; + +#ifdef MIC_LEVEL_FEEDBACK + /* Count ms in level saturation */ + // if (stt->micVol > stt->maxAnalog) { + if (stt->micVol > 150) { + /* mic level is saturated */ + stt->numBlocksMicLvlSat++; + fprintf(stderr, "Sat mic Level: %d\n", stt->numBlocksMicLvlSat); + } +#endif +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, + "\tAGC->ProcessAnalog, frame %d: measure < LowerLim, micVol " + "= %d\n", + stt->fcount, stt->micVol); +#endif + } + } else { + /* The signal is inside the desired range which is: + * lowerLimit < Rxx160_LP/640 < upperLimit + */ + if (stt->changeToSlowMode > 4000) { + stt->msecSpeechInnerChange = 1000; + stt->msecSpeechOuterChange = 500; + stt->upperLimit = stt->upperPrimaryLimit; + stt->lowerLimit = stt->lowerPrimaryLimit; + } else { + stt->changeToSlowMode += 2; // in milliseconds + } + stt->msTooLow = 0; + stt->msTooHigh = 0; + + stt->micVol = inMicLevelTmp; + } +#ifdef MIC_LEVEL_FEEDBACK + if (stt->numBlocksMicLvlSat > NUM_BLOCKS_IN_SAT_BEFORE_CHANGE_TARGET) { + stt->micLvlSat = 1; + fprintf(stderr, "target before = %d (%d)\n", stt->analogTargetLevel, + stt->targetIdx); + WebRtcAgc_UpdateAgcThresholds(stt); + WebRtcAgc_CalculateGainTable( + &(stt->digitalAgc.gainTable[0]), stt->compressionGaindB, + stt->targetLevelDbfs, stt->limiterEnable, stt->analogTarget); + stt->numBlocksMicLvlSat = 0; + stt->micLvlSat = 0; + fprintf(stderr, "target offset = %d\n", stt->targetIdxOffset); + fprintf(stderr, "target after = %d (%d)\n", stt->analogTargetLevel, + stt->targetIdx); + } +#endif + } + } + + /* Ensure gain is not increased in presence of echo or after a mute event + * (but allow the zeroCtrl() increase on the frame of a mute detection). + */ + if (echo == 1 || + (stt->muteGuardMs > 0 && stt->muteGuardMs < kMuteGuardTimeMs)) { + if (stt->micVol > lastMicVol) { + stt->micVol = lastMicVol; + } + } + + /* limit the gain */ + if (stt->micVol > stt->maxLevel) { + stt->micVol = stt->maxLevel; + } else if (stt->micVol < stt->minOutput) { + stt->micVol = stt->minOutput; + } + + *outMicLevel = WEBRTC_SPL_MIN(stt->micVol, stt->maxAnalog) >> stt->scale; + + return 0; +} + +int WebRtcAgc_Process(void* agcInst, + const int16_t* const* in_near, + size_t num_bands, + size_t samples, + int16_t* const* out, + int32_t inMicLevel, + int32_t* outMicLevel, + int16_t echo, + uint8_t* saturationWarning) { + LegacyAgc* stt; + + stt = (LegacyAgc*)agcInst; + + // + if (stt == NULL) { + return -1; + } + // + + if (stt->fs == 8000) { + if (samples != 80) { + return -1; + } + } else if (stt->fs == 16000 || stt->fs == 32000 || stt->fs == 48000) { + if (samples != 160) { + return -1; + } + } else { + return -1; + } + + *saturationWarning = 0; + // TODO(minyue): PUT IN RANGE CHECKING FOR INPUT LEVELS + *outMicLevel = inMicLevel; + +#ifdef WEBRTC_AGC_DEBUG_DUMP + stt->fcount++; +#endif + + if (WebRtcAgc_ProcessDigital(&stt->digitalAgc, in_near, num_bands, out, + stt->fs, stt->lowLevelSignal) == -1) { +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "AGC->Process, frame %d: Error from DigAGC\n\n", + stt->fcount); +#endif + return -1; + } + if (stt->agcMode < kAgcModeFixedDigital && + (stt->lowLevelSignal == 0 || stt->agcMode != kAgcModeAdaptiveDigital)) { + if (WebRtcAgc_ProcessAnalog(agcInst, inMicLevel, outMicLevel, + stt->vadMic.logRatio, echo, + saturationWarning) == -1) { + return -1; + } + } +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->agcLog, "%5d\t%d\t%d\t%d\t%d\n", stt->fcount, inMicLevel, + *outMicLevel, stt->maxLevel, stt->micVol); +#endif + + /* update queue */ + if (stt->inQueue > 1) { + memcpy(stt->env[0], stt->env[1], 10 * sizeof(int32_t)); + memcpy(stt->Rxx16w32_array[0], stt->Rxx16w32_array[1], 5 * sizeof(int32_t)); + } + + if (stt->inQueue > 0) { + stt->inQueue--; + } + + return 0; +} + +int WebRtcAgc_set_config(void* agcInst, WebRtcAgcConfig agcConfig) { + LegacyAgc* stt; + stt = (LegacyAgc*)agcInst; + + if (stt == NULL) { + return -1; + } + + if (stt->initFlag != kInitCheck) { + stt->lastError = AGC_UNINITIALIZED_ERROR; + return -1; + } + + if (agcConfig.limiterEnable != kAgcFalse && + agcConfig.limiterEnable != kAgcTrue) { + stt->lastError = AGC_BAD_PARAMETER_ERROR; + return -1; + } + stt->limiterEnable = agcConfig.limiterEnable; + stt->compressionGaindB = agcConfig.compressionGaindB; + if ((agcConfig.targetLevelDbfs < 0) || (agcConfig.targetLevelDbfs > 31)) { + stt->lastError = AGC_BAD_PARAMETER_ERROR; + return -1; + } + stt->targetLevelDbfs = agcConfig.targetLevelDbfs; + + if (stt->agcMode == kAgcModeFixedDigital) { + /* Adjust for different parameter interpretation in FixedDigital mode */ + stt->compressionGaindB += agcConfig.targetLevelDbfs; + } + + /* Update threshold levels for analog adaptation */ + WebRtcAgc_UpdateAgcThresholds(stt); + + /* Recalculate gain table */ + if (WebRtcAgc_CalculateGainTable( + &(stt->digitalAgc.gainTable[0]), stt->compressionGaindB, + stt->targetLevelDbfs, stt->limiterEnable, stt->analogTarget) == -1) { +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "AGC->set_config, frame %d: Error from calcGainTable\n\n", + stt->fcount); +#endif + return -1; + } + /* Store the config in a WebRtcAgcConfig */ + stt->usedConfig.compressionGaindB = agcConfig.compressionGaindB; + stt->usedConfig.limiterEnable = agcConfig.limiterEnable; + stt->usedConfig.targetLevelDbfs = agcConfig.targetLevelDbfs; + + return 0; +} + +int WebRtcAgc_get_config(void* agcInst, WebRtcAgcConfig* config) { + LegacyAgc* stt; + stt = (LegacyAgc*)agcInst; + + if (stt == NULL) { + return -1; + } + + if (config == NULL) { + stt->lastError = AGC_NULL_POINTER_ERROR; + return -1; + } + + if (stt->initFlag != kInitCheck) { + stt->lastError = AGC_UNINITIALIZED_ERROR; + return -1; + } + + config->limiterEnable = stt->usedConfig.limiterEnable; + config->targetLevelDbfs = stt->usedConfig.targetLevelDbfs; + config->compressionGaindB = stt->usedConfig.compressionGaindB; + + return 0; +} + +void* WebRtcAgc_Create() { + LegacyAgc* stt = malloc(sizeof(LegacyAgc)); + +#ifdef WEBRTC_AGC_DEBUG_DUMP + stt->fpt = fopen("./agc_test_log.txt", "wt"); + stt->agcLog = fopen("./agc_debug_log.txt", "wt"); + stt->digitalAgc.logFile = fopen("./agc_log.txt", "wt"); +#endif + + stt->initFlag = 0; + stt->lastError = 0; + + return stt; +} + +void WebRtcAgc_Free(void* state) { + LegacyAgc* stt; + + stt = (LegacyAgc*)state; +#ifdef WEBRTC_AGC_DEBUG_DUMP + fclose(stt->fpt); + fclose(stt->agcLog); + fclose(stt->digitalAgc.logFile); +#endif + free(stt); +} + +/* minLevel - Minimum volume level + * maxLevel - Maximum volume level + */ +int WebRtcAgc_Init(void* agcInst, + int32_t minLevel, + int32_t maxLevel, + int16_t agcMode, + uint32_t fs) { + int32_t max_add, tmp32; + int16_t i; + int tmpNorm; + LegacyAgc* stt; + + /* typecast state pointer */ + stt = (LegacyAgc*)agcInst; + + if (WebRtcAgc_InitDigital(&stt->digitalAgc, agcMode) != 0) { + stt->lastError = AGC_UNINITIALIZED_ERROR; + return -1; + } + + /* Analog AGC variables */ + stt->envSum = 0; + +/* mode = 0 - Only saturation protection + * 1 - Analog Automatic Gain Control [-targetLevelDbfs (default -3 + * dBOv)] + * 2 - Digital Automatic Gain Control [-targetLevelDbfs (default -3 + * dBOv)] + * 3 - Fixed Digital Gain [compressionGaindB (default 8 dB)] + */ +#ifdef WEBRTC_AGC_DEBUG_DUMP + stt->fcount = 0; + fprintf(stt->fpt, "AGC->Init\n"); +#endif + if (agcMode < kAgcModeUnchanged || agcMode > kAgcModeFixedDigital) { +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "AGC->Init: error, incorrect mode\n\n"); +#endif + return -1; + } + stt->agcMode = agcMode; + stt->fs = fs; + + /* initialize input VAD */ + WebRtcAgc_InitVad(&stt->vadMic); + + /* If the volume range is smaller than 0-256 then + * the levels are shifted up to Q8-domain */ + tmpNorm = WebRtcSpl_NormU32((uint32_t)maxLevel); + stt->scale = tmpNorm - 23; + if (stt->scale < 0) { + stt->scale = 0; + } + // TODO(bjornv): Investigate if we really need to scale up a small range now + // when we have + // a guard against zero-increments. For now, we do not support scale up (scale + // = 0). + stt->scale = 0; + maxLevel <<= stt->scale; + minLevel <<= stt->scale; + + /* Make minLevel and maxLevel static in AdaptiveDigital */ + if (stt->agcMode == kAgcModeAdaptiveDigital) { + minLevel = 0; + maxLevel = 255; + stt->scale = 0; + } + /* The maximum supplemental volume range is based on a vague idea + * of how much lower the gain will be than the real analog gain. */ + max_add = (maxLevel - minLevel) / 4; + + /* Minimum/maximum volume level that can be set */ + stt->minLevel = minLevel; + stt->maxAnalog = maxLevel; + stt->maxLevel = maxLevel + max_add; + stt->maxInit = stt->maxLevel; + + stt->zeroCtrlMax = stt->maxAnalog; + stt->lastInMicLevel = 0; + + /* Initialize micVol parameter */ + stt->micVol = stt->maxAnalog; + if (stt->agcMode == kAgcModeAdaptiveDigital) { + stt->micVol = 127; /* Mid-point of mic level */ + } + stt->micRef = stt->micVol; + stt->micGainIdx = 127; +#ifdef MIC_LEVEL_FEEDBACK + stt->numBlocksMicLvlSat = 0; + stt->micLvlSat = 0; +#endif +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "AGC->Init: minLevel = %d, maxAnalog = %d, maxLevel = %d\n", + stt->minLevel, stt->maxAnalog, stt->maxLevel); +#endif + + /* Minimum output volume is 4% higher than the available lowest volume level + */ + tmp32 = ((stt->maxLevel - stt->minLevel) * 10) >> 8; + stt->minOutput = (stt->minLevel + tmp32); + + stt->msTooLow = 0; + stt->msTooHigh = 0; + stt->changeToSlowMode = 0; + stt->firstCall = 0; + stt->msZero = 0; + stt->muteGuardMs = 0; + stt->gainTableIdx = 0; + + stt->msecSpeechInnerChange = kMsecSpeechInner; + stt->msecSpeechOuterChange = kMsecSpeechOuter; + + stt->activeSpeech = 0; + stt->Rxx16_LPw32Max = 0; + + stt->vadThreshold = kNormalVadThreshold; + stt->inActive = 0; + + for (i = 0; i < RXX_BUFFER_LEN; i++) { + stt->Rxx16_vectorw32[i] = (int32_t)1000; /* -54dBm0 */ + } + stt->Rxx160w32 = + 125 * RXX_BUFFER_LEN; /* (stt->Rxx16_vectorw32[0]>>3) = 125 */ + + stt->Rxx16pos = 0; + stt->Rxx16_LPw32 = (int32_t)16284; /* Q(-4) */ + + for (i = 0; i < 5; i++) { + stt->Rxx16w32_array[0][i] = 0; + } + for (i = 0; i < 10; i++) { + stt->env[0][i] = 0; + stt->env[1][i] = 0; + } + stt->inQueue = 0; + +#ifdef MIC_LEVEL_FEEDBACK + stt->targetIdxOffset = 0; +#endif + + WebRtcSpl_MemSetW32(stt->filterState, 0, 8); + + stt->initFlag = kInitCheck; + // Default config settings. + stt->defaultConfig.limiterEnable = kAgcTrue; + stt->defaultConfig.targetLevelDbfs = AGC_DEFAULT_TARGET_LEVEL; + stt->defaultConfig.compressionGaindB = AGC_DEFAULT_COMP_GAIN; + + if (WebRtcAgc_set_config(stt, stt->defaultConfig) == -1) { + stt->lastError = AGC_UNSPECIFIED_ERROR; + return -1; + } + stt->Rxx160_LPw32 = stt->analogTargetLevel; // Initialize rms value + + stt->lowLevelSignal = 0; + + /* Only positive values are allowed that are not too large */ + if ((minLevel >= maxLevel) || (maxLevel & 0xFC000000)) { +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "minLevel, maxLevel value(s) are invalid\n\n"); +#endif + return -1; + } else { +#ifdef WEBRTC_AGC_DEBUG_DUMP + fprintf(stt->fpt, "\n"); +#endif + return 0; + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.h new file mode 100644 index 0000000000..1fed3779ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_LEGACY_ANALOG_AGC_H_ +#define MODULES_AUDIO_PROCESSING_AGC_LEGACY_ANALOG_AGC_H_ + +//#define MIC_LEVEL_FEEDBACK +#ifdef WEBRTC_AGC_DEBUG_DUMP +#include +#endif + +#include "modules/audio_processing/agc/legacy/digital_agc.h" +#include "modules/audio_processing/agc/legacy/gain_control.h" +#include "typedefs.h" // NOLINT(build/include) + +/* Analog Automatic Gain Control variables: + * Constant declarations (inner limits inside which no changes are done) + * In the beginning the range is narrower to widen as soon as the measure + * 'Rxx160_LP' is inside it. Currently the starting limits are -22.2+/-1dBm0 + * and the final limits -22.2+/-2.5dBm0. These levels makes the speech signal + * go towards -25.4dBm0 (-31.4dBov). Tuned with wbfile-31.4dBov.pcm + * The limits are created by running the AGC with a file having the desired + * signal level and thereafter plotting Rxx160_LP in the dBm0-domain defined + * by out=10*log10(in/260537279.7); Set the target level to the average level + * of our measure Rxx160_LP. Remember that the levels are in blocks of 16 in + * Q(-7). (Example matlab code: round(db2pow(-21.2)*16/2^7) ) + */ +#define RXX_BUFFER_LEN 10 + +static const int16_t kMsecSpeechInner = 520; +static const int16_t kMsecSpeechOuter = 340; + +static const int16_t kNormalVadThreshold = 400; + +static const int16_t kAlphaShortTerm = 6; // 1 >> 6 = 0.0156 +static const int16_t kAlphaLongTerm = 10; // 1 >> 10 = 0.000977 + +typedef struct { + // Configurable parameters/variables + uint32_t fs; // Sampling frequency + int16_t compressionGaindB; // Fixed gain level in dB + int16_t targetLevelDbfs; // Target level in -dBfs of envelope (default -3) + int16_t agcMode; // Hard coded mode (adaptAna/adaptDig/fixedDig) + uint8_t limiterEnable; // Enabling limiter (on/off (default off)) + WebRtcAgcConfig defaultConfig; + WebRtcAgcConfig usedConfig; + + // General variables + int16_t initFlag; + int16_t lastError; + + // Target level parameters + // Based on the above: analogTargetLevel = round((32767*10^(-22/20))^2*16/2^7) + int32_t analogTargetLevel; // = RXX_BUFFER_LEN * 846805; -22 dBfs + int32_t startUpperLimit; // = RXX_BUFFER_LEN * 1066064; -21 dBfs + int32_t startLowerLimit; // = RXX_BUFFER_LEN * 672641; -23 dBfs + int32_t upperPrimaryLimit; // = RXX_BUFFER_LEN * 1342095; -20 dBfs + int32_t lowerPrimaryLimit; // = RXX_BUFFER_LEN * 534298; -24 dBfs + int32_t upperSecondaryLimit; // = RXX_BUFFER_LEN * 2677832; -17 dBfs + int32_t lowerSecondaryLimit; // = RXX_BUFFER_LEN * 267783; -27 dBfs + uint16_t targetIdx; // Table index for corresponding target level +#ifdef MIC_LEVEL_FEEDBACK + uint16_t targetIdxOffset; // Table index offset for level compensation +#endif + int16_t analogTarget; // Digital reference level in ENV scale + + // Analog AGC specific variables + int32_t filterState[8]; // For downsampling wb to nb + int32_t upperLimit; // Upper limit for mic energy + int32_t lowerLimit; // Lower limit for mic energy + int32_t Rxx160w32; // Average energy for one frame + int32_t Rxx16_LPw32; // Low pass filtered subframe energies + int32_t Rxx160_LPw32; // Low pass filtered frame energies + int32_t Rxx16_LPw32Max; // Keeps track of largest energy subframe + int32_t Rxx16_vectorw32[RXX_BUFFER_LEN]; // Array with subframe energies + int32_t Rxx16w32_array[2][5]; // Energy values of microphone signal + int32_t env[2][10]; // Envelope values of subframes + + int16_t Rxx16pos; // Current position in the Rxx16_vectorw32 + int16_t envSum; // Filtered scaled envelope in subframes + int16_t vadThreshold; // Threshold for VAD decision + int16_t inActive; // Inactive time in milliseconds + int16_t msTooLow; // Milliseconds of speech at a too low level + int16_t msTooHigh; // Milliseconds of speech at a too high level + int16_t changeToSlowMode; // Change to slow mode after some time at target + int16_t firstCall; // First call to the process-function + int16_t msZero; // Milliseconds of zero input + int16_t msecSpeechOuterChange; // Min ms of speech between volume changes + int16_t msecSpeechInnerChange; // Min ms of speech between volume changes + int16_t activeSpeech; // Milliseconds of active speech + int16_t muteGuardMs; // Counter to prevent mute action + int16_t inQueue; // 10 ms batch indicator + + // Microphone level variables + int32_t micRef; // Remember ref. mic level for virtual mic + uint16_t gainTableIdx; // Current position in virtual gain table + int32_t micGainIdx; // Gain index of mic level to increase slowly + int32_t micVol; // Remember volume between frames + int32_t maxLevel; // Max possible vol level, incl dig gain + int32_t maxAnalog; // Maximum possible analog volume level + int32_t maxInit; // Initial value of "max" + int32_t minLevel; // Minimum possible volume level + int32_t minOutput; // Minimum output volume level + int32_t zeroCtrlMax; // Remember max gain => don't amp low input + int32_t lastInMicLevel; + + int16_t scale; // Scale factor for internal volume levels +#ifdef MIC_LEVEL_FEEDBACK + int16_t numBlocksMicLvlSat; + uint8_t micLvlSat; +#endif + // Structs for VAD and digital_agc + AgcVad vadMic; + DigitalAgc digitalAgc; + +#ifdef WEBRTC_AGC_DEBUG_DUMP + FILE* fpt; + FILE* agcLog; + int32_t fcount; +#endif + + int16_t lowLevelSignal; +} LegacyAgc; + +#endif // MODULES_AUDIO_PROCESSING_AGC_LEGACY_ANALOG_AGC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.c b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.c new file mode 100644 index 0000000000..3269a17ce4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.c @@ -0,0 +1,703 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* digital_agc.c + * + */ + +#include "modules/audio_processing/agc/legacy/digital_agc.h" + +#include +#ifdef WEBRTC_AGC_DEBUG_DUMP +#include +#endif + +#include "rtc_base/checks.h" +#include "modules/audio_processing/agc/legacy/gain_control.h" + +// To generate the gaintable, copy&paste the following lines to a Matlab window: +// MaxGain = 6; MinGain = 0; CompRatio = 3; Knee = 1; +// zeros = 0:31; lvl = 2.^(1-zeros); +// A = -10*log10(lvl) * (CompRatio - 1) / CompRatio; +// B = MaxGain - MinGain; +// gains = round(2^16*10.^(0.05 * (MinGain + B * ( +// log(exp(-Knee*A)+exp(-Knee*B)) - log(1+exp(-Knee*B)) ) / +// log(1/(1+exp(Knee*B)))))); +// fprintf(1, '\t%i, %i, %i, %i,\n', gains); +// % Matlab code for plotting the gain and input/output level characteristic +// (copy/paste the following 3 lines): +// in = 10*log10(lvl); out = 20*log10(gains/65536); +// subplot(121); plot(in, out); axis([-30, 0, -5, 20]); grid on; xlabel('Input +// (dB)'); ylabel('Gain (dB)'); +// subplot(122); plot(in, in+out); axis([-30, 0, -30, 5]); grid on; +// xlabel('Input (dB)'); ylabel('Output (dB)'); +// zoom on; + +// Generator table for y=log2(1+e^x) in Q8. +enum { kGenFuncTableSize = 128 }; +static const uint16_t kGenFuncTable[kGenFuncTableSize] = { + 256, 485, 786, 1126, 1484, 1849, 2217, 2586, 2955, 3324, 3693, + 4063, 4432, 4801, 5171, 5540, 5909, 6279, 6648, 7017, 7387, 7756, + 8125, 8495, 8864, 9233, 9603, 9972, 10341, 10711, 11080, 11449, 11819, + 12188, 12557, 12927, 13296, 13665, 14035, 14404, 14773, 15143, 15512, 15881, + 16251, 16620, 16989, 17359, 17728, 18097, 18466, 18836, 19205, 19574, 19944, + 20313, 20682, 21052, 21421, 21790, 22160, 22529, 22898, 23268, 23637, 24006, + 24376, 24745, 25114, 25484, 25853, 26222, 26592, 26961, 27330, 27700, 28069, + 28438, 28808, 29177, 29546, 29916, 30285, 30654, 31024, 31393, 31762, 32132, + 32501, 32870, 33240, 33609, 33978, 34348, 34717, 35086, 35456, 35825, 36194, + 36564, 36933, 37302, 37672, 38041, 38410, 38780, 39149, 39518, 39888, 40257, + 40626, 40996, 41365, 41734, 42104, 42473, 42842, 43212, 43581, 43950, 44320, + 44689, 45058, 45428, 45797, 46166, 46536, 46905}; + +static const int16_t kAvgDecayTime = 250; // frames; < 3000 + +int32_t WebRtcAgc_CalculateGainTable(int32_t* gainTable, // Q16 + int16_t digCompGaindB, // Q0 + int16_t targetLevelDbfs, // Q0 + uint8_t limiterEnable, + int16_t analogTarget) // Q0 +{ + // This function generates the compressor gain table used in the fixed digital + // part. + uint32_t tmpU32no1, tmpU32no2, absInLevel, logApprox; + int32_t inLevel, limiterLvl; + int32_t tmp32, tmp32no1, tmp32no2, numFIX, den, y32; + const uint16_t kLog10 = 54426; // log2(10) in Q14 + const uint16_t kLog10_2 = 49321; // 10*log10(2) in Q14 + const uint16_t kLogE_1 = 23637; // log2(e) in Q14 + uint16_t constMaxGain; + uint16_t tmpU16, intPart, fracPart; + const int16_t kCompRatio = 3; + const int16_t kSoftLimiterLeft = 1; + int16_t limiterOffset = 0; // Limiter offset + int16_t limiterIdx, limiterLvlX; + int16_t constLinApprox, zeroGainLvl, maxGain, diffGain; + int16_t i, tmp16, tmp16no1; + int zeros, zerosScale; + + // Constants + // kLogE_1 = 23637; // log2(e) in Q14 + // kLog10 = 54426; // log2(10) in Q14 + // kLog10_2 = 49321; // 10*log10(2) in Q14 + + // Calculate maximum digital gain and zero gain level + tmp32no1 = (digCompGaindB - analogTarget) * (kCompRatio - 1); + tmp16no1 = analogTarget - targetLevelDbfs; + tmp16no1 += + WebRtcSpl_DivW32W16ResW16(tmp32no1 + (kCompRatio >> 1), kCompRatio); + maxGain = WEBRTC_SPL_MAX(tmp16no1, (analogTarget - targetLevelDbfs)); + tmp32no1 = maxGain * kCompRatio; + zeroGainLvl = digCompGaindB; + zeroGainLvl -= WebRtcSpl_DivW32W16ResW16(tmp32no1 + ((kCompRatio - 1) >> 1), + kCompRatio - 1); + if ((digCompGaindB <= analogTarget) && (limiterEnable)) { + zeroGainLvl += (analogTarget - digCompGaindB + kSoftLimiterLeft); + limiterOffset = 0; + } + + // Calculate the difference between maximum gain and gain at 0dB0v: + // diffGain = maxGain + (compRatio-1)*zeroGainLvl/compRatio + // = (compRatio-1)*digCompGaindB/compRatio + tmp32no1 = digCompGaindB * (kCompRatio - 1); + diffGain = + WebRtcSpl_DivW32W16ResW16(tmp32no1 + (kCompRatio >> 1), kCompRatio); + if (diffGain < 0 || diffGain >= kGenFuncTableSize) { + RTC_DCHECK(0); + return -1; + } + + // Calculate the limiter level and index: + // limiterLvlX = analogTarget - limiterOffset + // limiterLvl = targetLevelDbfs + limiterOffset/compRatio + limiterLvlX = analogTarget - limiterOffset; + limiterIdx = 2 + WebRtcSpl_DivW32W16ResW16((int32_t)limiterLvlX * (1 << 13), + kLog10_2 / 2); + tmp16no1 = + WebRtcSpl_DivW32W16ResW16(limiterOffset + (kCompRatio >> 1), kCompRatio); + limiterLvl = targetLevelDbfs + tmp16no1; + + // Calculate (through table lookup): + // constMaxGain = log2(1+2^(log2(e)*diffGain)); (in Q8) + constMaxGain = kGenFuncTable[diffGain]; // in Q8 + + // Calculate a parameter used to approximate the fractional part of 2^x with a + // piecewise linear function in Q14: + // constLinApprox = round(3/2*(4*(3-2*sqrt(2))/(log(2)^2)-0.5)*2^14); + constLinApprox = 22817; // in Q14 + + // Calculate a denominator used in the exponential part to convert from dB to + // linear scale: + // den = 20*constMaxGain (in Q8) + den = WEBRTC_SPL_MUL_16_U16(20, constMaxGain); // in Q8 + + for (i = 0; i < 32; i++) { + // Calculate scaled input level (compressor): + // inLevel = + // fix((-constLog10_2*(compRatio-1)*(1-i)+fix(compRatio/2))/compRatio) + tmp16 = (int16_t)((kCompRatio - 1) * (i - 1)); // Q0 + tmp32 = WEBRTC_SPL_MUL_16_U16(tmp16, kLog10_2) + 1; // Q14 + inLevel = WebRtcSpl_DivW32W16(tmp32, kCompRatio); // Q14 + + // Calculate diffGain-inLevel, to map using the genFuncTable + inLevel = (int32_t)diffGain * (1 << 14) - inLevel; // Q14 + + // Make calculations on abs(inLevel) and compensate for the sign afterwards. + absInLevel = (uint32_t)WEBRTC_SPL_ABS_W32(inLevel); // Q14 + + // LUT with interpolation + intPart = (uint16_t)(absInLevel >> 14); + fracPart = + (uint16_t)(absInLevel & 0x00003FFF); // extract the fractional part + tmpU16 = kGenFuncTable[intPart + 1] - kGenFuncTable[intPart]; // Q8 + tmpU32no1 = tmpU16 * fracPart; // Q22 + tmpU32no1 += (uint32_t)kGenFuncTable[intPart] << 14; // Q22 + logApprox = tmpU32no1 >> 8; // Q14 + // Compensate for negative exponent using the relation: + // log2(1 + 2^-x) = log2(1 + 2^x) - x + if (inLevel < 0) { + zeros = WebRtcSpl_NormU32(absInLevel); + zerosScale = 0; + if (zeros < 15) { + // Not enough space for multiplication + tmpU32no2 = absInLevel >> (15 - zeros); // Q(zeros-1) + tmpU32no2 = WEBRTC_SPL_UMUL_32_16(tmpU32no2, kLogE_1); // Q(zeros+13) + if (zeros < 9) { + zerosScale = 9 - zeros; + tmpU32no1 >>= zerosScale; // Q(zeros+13) + } else { + tmpU32no2 >>= zeros - 9; // Q22 + } + } else { + tmpU32no2 = WEBRTC_SPL_UMUL_32_16(absInLevel, kLogE_1); // Q28 + tmpU32no2 >>= 6; // Q22 + } + logApprox = 0; + if (tmpU32no2 < tmpU32no1) { + logApprox = (tmpU32no1 - tmpU32no2) >> (8 - zerosScale); // Q14 + } + } + numFIX = (maxGain * constMaxGain) * (1 << 6); // Q14 + numFIX -= (int32_t)logApprox * diffGain; // Q14 + + // Calculate ratio + // Shift |numFIX| as much as possible. + // Ensure we avoid wrap-around in |den| as well. + if (numFIX > (den >> 8) || -numFIX > (den >> 8)) // |den| is Q8. + { + zeros = WebRtcSpl_NormW32(numFIX); + } else { + zeros = WebRtcSpl_NormW32(den) + 8; + } + numFIX *= 1 << zeros; // Q(14+zeros) + + // Shift den so we end up in Qy1 + tmp32no1 = WEBRTC_SPL_SHIFT_W32(den, zeros - 9); // Q(zeros - 1) + y32 = numFIX / tmp32no1; // in Q15 + // This is to do rounding in Q14. + y32 = y32 >= 0 ? (y32 + 1) >> 1 : -((-y32 + 1) >> 1); + + if (limiterEnable && (i < limiterIdx)) { + tmp32 = WEBRTC_SPL_MUL_16_U16(i - 1, kLog10_2); // Q14 + tmp32 -= limiterLvl * (1 << 14); // Q14 + y32 = WebRtcSpl_DivW32W16(tmp32 + 10, 20); + } + if (y32 > 39000) { + tmp32 = (y32 >> 1) * kLog10 + 4096; // in Q27 + tmp32 >>= 13; // In Q14. + } else { + tmp32 = y32 * kLog10 + 8192; // in Q28 + tmp32 >>= 14; // In Q14. + } + tmp32 += 16 << 14; // in Q14 (Make sure final output is in Q16) + + // Calculate power + if (tmp32 > 0) { + intPart = (int16_t)(tmp32 >> 14); + fracPart = (uint16_t)(tmp32 & 0x00003FFF); // in Q14 + if ((fracPart >> 13) != 0) { + tmp16 = (2 << 14) - constLinApprox; + tmp32no2 = (1 << 14) - fracPart; + tmp32no2 *= tmp16; + tmp32no2 >>= 13; + tmp32no2 = (1 << 14) - tmp32no2; + } else { + tmp16 = constLinApprox - (1 << 14); + tmp32no2 = (fracPart * tmp16) >> 13; + } + fracPart = (uint16_t)tmp32no2; + gainTable[i] = + (1 << intPart) + WEBRTC_SPL_SHIFT_W32(fracPart, intPart - 14); + } else { + gainTable[i] = 0; + } + } + + return 0; +} + +int32_t WebRtcAgc_InitDigital(DigitalAgc* stt, int16_t agcMode) { + if (agcMode == kAgcModeFixedDigital) { + // start at minimum to find correct gain faster + stt->capacitorSlow = 0; + } else { + // start out with 0 dB gain + stt->capacitorSlow = 134217728; // (int32_t)(0.125f * 32768.0f * 32768.0f); + } + stt->capacitorFast = 0; + stt->gain = 65536; + stt->gatePrevious = 0; + stt->agcMode = agcMode; +#ifdef WEBRTC_AGC_DEBUG_DUMP + stt->frameCounter = 0; +#endif + + // initialize VADs + WebRtcAgc_InitVad(&stt->vadNearend); + WebRtcAgc_InitVad(&stt->vadFarend); + + return 0; +} + +int32_t WebRtcAgc_AddFarendToDigital(DigitalAgc* stt, + const int16_t* in_far, + size_t nrSamples) { + RTC_DCHECK(stt); + // VAD for far end + WebRtcAgc_ProcessVad(&stt->vadFarend, in_far, nrSamples); + + return 0; +} + +int32_t WebRtcAgc_ProcessDigital(DigitalAgc* stt, + const int16_t* const* in_near, + size_t num_bands, + int16_t* const* out, + uint32_t FS, + int16_t lowlevelSignal) { + // array for gains (one value per ms, incl start & end) + int32_t gains[11]; + + int32_t out_tmp, tmp32; + int32_t env[10]; + int32_t max_nrg; + int32_t cur_level; + int32_t gain32, delta; + int16_t logratio; + int16_t lower_thr, upper_thr; + int16_t zeros = 0, zeros_fast, frac = 0; + int16_t decay; + int16_t gate, gain_adj; + int16_t k; + size_t n, i, L; + int16_t L2; // samples/subframe + + // determine number of samples per ms + if (FS == 8000) { + L = 8; + L2 = 3; + } else if (FS == 16000 || FS == 32000 || FS == 48000) { + L = 16; + L2 = 4; + } else { + return -1; + } + + for (i = 0; i < num_bands; ++i) { + if (in_near[i] != out[i]) { + // Only needed if they don't already point to the same place. + memcpy(out[i], in_near[i], 10 * L * sizeof(in_near[i][0])); + } + } + // VAD for near end + logratio = WebRtcAgc_ProcessVad(&stt->vadNearend, out[0], L * 10); + + // Account for far end VAD + if (stt->vadFarend.counter > 10) { + tmp32 = 3 * logratio; + logratio = (int16_t)((tmp32 - stt->vadFarend.logRatio) >> 2); + } + + // Determine decay factor depending on VAD + // upper_thr = 1.0f; + // lower_thr = 0.25f; + upper_thr = 1024; // Q10 + lower_thr = 0; // Q10 + if (logratio > upper_thr) { + // decay = -2^17 / DecayTime; -> -65 + decay = -65; + } else if (logratio < lower_thr) { + decay = 0; + } else { + // decay = (int16_t)(((lower_thr - logratio) + // * (2^27/(DecayTime*(upper_thr-lower_thr)))) >> 10); + // SUBSTITUTED: 2^27/(DecayTime*(upper_thr-lower_thr)) -> 65 + tmp32 = (lower_thr - logratio) * 65; + decay = (int16_t)(tmp32 >> 10); + } + + // adjust decay factor for long silence (detected as low standard deviation) + // This is only done in the adaptive modes + if (stt->agcMode != kAgcModeFixedDigital) { + if (stt->vadNearend.stdLongTerm < 4000) { + decay = 0; + } else if (stt->vadNearend.stdLongTerm < 8096) { + // decay = (int16_t)(((stt->vadNearend.stdLongTerm - 4000) * decay) >> + // 12); + tmp32 = (stt->vadNearend.stdLongTerm - 4000) * decay; + decay = (int16_t)(tmp32 >> 12); + } + + if (lowlevelSignal != 0) { + decay = 0; + } + } +#ifdef WEBRTC_AGC_DEBUG_DUMP + stt->frameCounter++; + fprintf(stt->logFile, "%5.2f\t%d\t%d\t%d\t", (float)(stt->frameCounter) / 100, + logratio, decay, stt->vadNearend.stdLongTerm); +#endif + // Find max amplitude per sub frame + // iterate over sub frames + for (k = 0; k < 10; k++) { + // iterate over samples + max_nrg = 0; + for (n = 0; n < L; n++) { + int32_t nrg = out[0][k * L + n] * out[0][k * L + n]; + if (nrg > max_nrg) { + max_nrg = nrg; + } + } + env[k] = max_nrg; + } + + // Calculate gain per sub frame + gains[0] = stt->gain; + for (k = 0; k < 10; k++) { + // Fast envelope follower + // decay time = -131000 / -1000 = 131 (ms) + stt->capacitorFast = + AGC_SCALEDIFF32(-1000, stt->capacitorFast, stt->capacitorFast); + if (env[k] > stt->capacitorFast) { + stt->capacitorFast = env[k]; + } + // Slow envelope follower + if (env[k] > stt->capacitorSlow) { + // increase capacitorSlow + stt->capacitorSlow = AGC_SCALEDIFF32(500, (env[k] - stt->capacitorSlow), + stt->capacitorSlow); + } else { + // decrease capacitorSlow + stt->capacitorSlow = + AGC_SCALEDIFF32(decay, stt->capacitorSlow, stt->capacitorSlow); + } + + // use maximum of both capacitors as current level + if (stt->capacitorFast > stt->capacitorSlow) { + cur_level = stt->capacitorFast; + } else { + cur_level = stt->capacitorSlow; + } + // Translate signal level into gain, using a piecewise linear approximation + // find number of leading zeros + zeros = WebRtcSpl_NormU32((uint32_t)cur_level); + if (cur_level == 0) { + zeros = 31; + } + tmp32 = ((uint32_t)cur_level << zeros) & 0x7FFFFFFF; + frac = (int16_t)(tmp32 >> 19); // Q12. + tmp32 = (stt->gainTable[zeros - 1] - stt->gainTable[zeros]) * frac; + gains[k + 1] = stt->gainTable[zeros] + (tmp32 >> 12); +#ifdef WEBRTC_AGC_DEBUG_DUMP + if (k == 0) { + fprintf(stt->logFile, "%d\t%d\t%d\t%d\t%d\n", env[0], cur_level, + stt->capacitorFast, stt->capacitorSlow, zeros); + } +#endif + } + + // Gate processing (lower gain during absence of speech) + zeros = (zeros << 9) - (frac >> 3); + // find number of leading zeros + zeros_fast = WebRtcSpl_NormU32((uint32_t)stt->capacitorFast); + if (stt->capacitorFast == 0) { + zeros_fast = 31; + } + tmp32 = ((uint32_t)stt->capacitorFast << zeros_fast) & 0x7FFFFFFF; + zeros_fast <<= 9; + zeros_fast -= (int16_t)(tmp32 >> 22); + + gate = 1000 + zeros_fast - zeros - stt->vadNearend.stdShortTerm; + + if (gate < 0) { + stt->gatePrevious = 0; + } else { + tmp32 = stt->gatePrevious * 7; + gate = (int16_t)((gate + tmp32) >> 3); + stt->gatePrevious = gate; + } + // gate < 0 -> no gate + // gate > 2500 -> max gate + if (gate > 0) { + if (gate < 2500) { + gain_adj = (2500 - gate) >> 5; + } else { + gain_adj = 0; + } + for (k = 0; k < 10; k++) { + if ((gains[k + 1] - stt->gainTable[0]) > 8388608) { + // To prevent wraparound + tmp32 = (gains[k + 1] - stt->gainTable[0]) >> 8; + tmp32 *= 178 + gain_adj; + } else { + tmp32 = (gains[k + 1] - stt->gainTable[0]) * (178 + gain_adj); + tmp32 >>= 8; + } + gains[k + 1] = stt->gainTable[0] + tmp32; + } + } + + // Limit gain to avoid overload distortion + for (k = 0; k < 10; k++) { + // To prevent wrap around + zeros = 10; + if (gains[k + 1] > 47453132) { + zeros = 16 - WebRtcSpl_NormW32(gains[k + 1]); + } + gain32 = (gains[k + 1] >> zeros) + 1; + gain32 *= gain32; + // check for overflow + while (AGC_MUL32((env[k] >> 12) + 1, gain32) > + WEBRTC_SPL_SHIFT_W32((int32_t)32767, 2 * (1 - zeros + 10))) { + // multiply by 253/256 ==> -0.1 dB + if (gains[k + 1] > 8388607) { + // Prevent wrap around + gains[k + 1] = (gains[k + 1] / 256) * 253; + } else { + gains[k + 1] = (gains[k + 1] * 253) / 256; + } + gain32 = (gains[k + 1] >> zeros) + 1; + gain32 *= gain32; + } + } + // gain reductions should be done 1 ms earlier than gain increases + for (k = 1; k < 10; k++) { + if (gains[k] > gains[k + 1]) { + gains[k] = gains[k + 1]; + } + } + // save start gain for next frame + stt->gain = gains[10]; + + // Apply gain + // handle first sub frame separately + delta = (gains[1] - gains[0]) * (1 << (4 - L2)); + gain32 = gains[0] * (1 << 4); + // iterate over samples + for (n = 0; n < L; n++) { + for (i = 0; i < num_bands; ++i) { + tmp32 = out[i][n] * ((gain32 + 127) >> 7); + out_tmp = tmp32 >> 16; + if (out_tmp > 4095) { + out[i][n] = (int16_t)32767; + } else if (out_tmp < -4096) { + out[i][n] = (int16_t)-32768; + } else { + tmp32 = out[i][n] * (gain32 >> 4); + out[i][n] = (int16_t)(tmp32 >> 16); + } + } + // + + gain32 += delta; + } + // iterate over subframes + for (k = 1; k < 10; k++) { + delta = (gains[k + 1] - gains[k]) * (1 << (4 - L2)); + gain32 = gains[k] * (1 << 4); + // iterate over samples + for (n = 0; n < L; n++) { + for (i = 0; i < num_bands; ++i) { + int64_t tmp64 = ((int64_t)(out[i][k * L + n])) * (gain32 >> 4); + tmp64 = tmp64 >> 16; + if (tmp64 > 32767) { + out[i][k * L + n] = 32767; + } + else if (tmp64 < -32768) { + out[i][k * L + n] = -32768; + } + else { + out[i][k * L + n] = (int16_t)(tmp64); + } + } + gain32 += delta; + } + } + + return 0; +} + +void WebRtcAgc_InitVad(AgcVad* state) { + int16_t k; + + state->HPstate = 0; // state of high pass filter + state->logRatio = 0; // log( P(active) / P(inactive) ) + // average input level (Q10) + state->meanLongTerm = 15 << 10; + + // variance of input level (Q8) + state->varianceLongTerm = 500 << 8; + + state->stdLongTerm = 0; // standard deviation of input level in dB + // short-term average input level (Q10) + state->meanShortTerm = 15 << 10; + + // short-term variance of input level (Q8) + state->varianceShortTerm = 500 << 8; + + state->stdShortTerm = + 0; // short-term standard deviation of input level in dB + state->counter = 3; // counts updates + for (k = 0; k < 8; k++) { + // downsampling filter + state->downState[k] = 0; + } +} + +int16_t WebRtcAgc_ProcessVad(AgcVad* state, // (i) VAD state + const int16_t* in, // (i) Speech signal + size_t nrSamples) // (i) number of samples +{ + uint32_t nrg; + int32_t out, tmp32, tmp32b; + uint16_t tmpU16; + int16_t k, subfr, tmp16; + int16_t buf1[8]; + int16_t buf2[4]; + int16_t HPstate; + int16_t zeros, dB; + + // process in 10 sub frames of 1 ms (to save on memory) + nrg = 0; + HPstate = state->HPstate; + for (subfr = 0; subfr < 10; subfr++) { + // downsample to 4 kHz + if (nrSamples == 160) { + for (k = 0; k < 8; k++) { + tmp32 = (int32_t)in[2 * k] + (int32_t)in[2 * k + 1]; + tmp32 >>= 1; + buf1[k] = (int16_t)tmp32; + } + in += 16; + + WebRtcSpl_DownsampleBy2(buf1, 8, buf2, state->downState); + } else { + WebRtcSpl_DownsampleBy2(in, 8, buf2, state->downState); + in += 8; + } + + // high pass filter and compute energy + for (k = 0; k < 4; k++) { + out = buf2[k] + HPstate; + tmp32 = 600 * out; + HPstate = (int16_t)((tmp32 >> 10) - buf2[k]); + + // Add 'out * out / 2**6' to 'nrg' in a non-overflowing + // way. Guaranteed to work as long as 'out * out / 2**6' fits in + // an int32_t. + nrg += out * (out / (1 << 6)); + nrg += out * (out % (1 << 6)) / (1 << 6); + } + } + state->HPstate = HPstate; + + // find number of leading zeros + if (!(0xFFFF0000 & nrg)) { + zeros = 16; + } else { + zeros = 0; + } + if (!(0xFF000000 & (nrg << zeros))) { + zeros += 8; + } + if (!(0xF0000000 & (nrg << zeros))) { + zeros += 4; + } + if (!(0xC0000000 & (nrg << zeros))) { + zeros += 2; + } + if (!(0x80000000 & (nrg << zeros))) { + zeros += 1; + } + + // energy level (range {-32..30}) (Q10) + dB = (15 - zeros) * (1 << 11); + + // Update statistics + + if (state->counter < kAvgDecayTime) { + // decay time = AvgDecTime * 10 ms + state->counter++; + } + + // update short-term estimate of mean energy level (Q10) + tmp32 = state->meanShortTerm * 15 + dB; + state->meanShortTerm = (int16_t)(tmp32 >> 4); + + // update short-term estimate of variance in energy level (Q8) + tmp32 = (dB * dB) >> 12; + tmp32 += state->varianceShortTerm * 15; + state->varianceShortTerm = tmp32 / 16; + + // update short-term estimate of standard deviation in energy level (Q10) + tmp32 = state->meanShortTerm * state->meanShortTerm; + tmp32 = (state->varianceShortTerm << 12) - tmp32; + state->stdShortTerm = (int16_t)WebRtcSpl_Sqrt(tmp32); + + // update long-term estimate of mean energy level (Q10) + tmp32 = state->meanLongTerm * state->counter + dB; + state->meanLongTerm = + WebRtcSpl_DivW32W16ResW16(tmp32, WebRtcSpl_AddSatW16(state->counter, 1)); + + // update long-term estimate of variance in energy level (Q8) + tmp32 = (dB * dB) >> 12; + tmp32 += state->varianceLongTerm * state->counter; + state->varianceLongTerm = + WebRtcSpl_DivW32W16(tmp32, WebRtcSpl_AddSatW16(state->counter, 1)); + + // update long-term estimate of standard deviation in energy level (Q10) + tmp32 = state->meanLongTerm * state->meanLongTerm; + tmp32 = (state->varianceLongTerm << 12) - tmp32; + state->stdLongTerm = (int16_t)WebRtcSpl_Sqrt(tmp32); + + // update voice activity measure (Q10) + tmp16 = 3 << 12; + // TODO(bjornv): (dB - state->meanLongTerm) can overflow, e.g., in + // ApmTest.Process unit test. Previously the macro WEBRTC_SPL_MUL_16_16() + // was used, which did an intermediate cast to (int16_t), hence losing + // significant bits. This cause logRatio to max out positive, rather than + // negative. This is a bug, but has very little significance. + tmp32 = tmp16 * (int16_t)(dB - state->meanLongTerm); + tmp32 = WebRtcSpl_DivW32W16(tmp32, state->stdLongTerm); + tmpU16 = (13 << 12); + tmp32b = WEBRTC_SPL_MUL_16_U16(state->logRatio, tmpU16); + tmp32 += tmp32b >> 10; + + state->logRatio = (int16_t)(tmp32 >> 6); + + // limit + if (state->logRatio > 2048) { + state->logRatio = 2048; + } + if (state->logRatio < -2048) { + state->logRatio = -2048; + } + + return state->logRatio; // Q10 +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.h new file mode 100644 index 0000000000..af6cf48837 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_LEGACY_DIGITAL_AGC_H_ +#define MODULES_AUDIO_PROCESSING_AGC_LEGACY_DIGITAL_AGC_H_ + +#ifdef WEBRTC_AGC_DEBUG_DUMP +#include +#endif +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "typedefs.h" // NOLINT(build/include) + +// the 32 most significant bits of A(19) * B(26) >> 13 +#define AGC_MUL32(A, B) (((B) >> 13) * (A) + (((0x00001FFF & (B)) * (A)) >> 13)) +// C + the 32 most significant bits of A * B +#define AGC_SCALEDIFF32(A, B, C) \ + ((C) + ((B) >> 16) * (A) + (((0x0000FFFF & (B)) * (A)) >> 16)) + +typedef struct { + int32_t downState[8]; + int16_t HPstate; + int16_t counter; + int16_t logRatio; // log( P(active) / P(inactive) ) (Q10) + int16_t meanLongTerm; // Q10 + int32_t varianceLongTerm; // Q8 + int16_t stdLongTerm; // Q10 + int16_t meanShortTerm; // Q10 + int32_t varianceShortTerm; // Q8 + int16_t stdShortTerm; // Q10 +} AgcVad; // total = 54 bytes + +typedef struct { + int32_t capacitorSlow; + int32_t capacitorFast; + int32_t gain; + int32_t gainTable[32]; + int16_t gatePrevious; + int16_t agcMode; + AgcVad vadNearend; + AgcVad vadFarend; +#ifdef WEBRTC_AGC_DEBUG_DUMP + FILE* logFile; + int frameCounter; +#endif +} DigitalAgc; + +int32_t WebRtcAgc_InitDigital(DigitalAgc* digitalAgcInst, int16_t agcMode); + +int32_t WebRtcAgc_ProcessDigital(DigitalAgc* digitalAgcInst, + const int16_t* const* inNear, + size_t num_bands, + int16_t* const* out, + uint32_t FS, + int16_t lowLevelSignal); + +int32_t WebRtcAgc_AddFarendToDigital(DigitalAgc* digitalAgcInst, + const int16_t* inFar, + size_t nrSamples); + +void WebRtcAgc_InitVad(AgcVad* vadInst); + +int16_t WebRtcAgc_ProcessVad(AgcVad* vadInst, // (i) VAD state + const int16_t* in, // (i) Speech signal + size_t nrSamples); // (i) number of samples + +int32_t WebRtcAgc_CalculateGainTable(int32_t* gainTable, // Q16 + int16_t compressionGaindB, // Q0 (in dB) + int16_t targetLevelDbfs, // Q0 (in dB) + uint8_t limiterEnable, + int16_t analogTarget); + +#endif // MODULES_AUDIO_PROCESSING_AGC_LEGACY_DIGITAL_AGC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/gain_control.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/gain_control.h new file mode 100644 index 0000000000..0f121b143a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/gain_control.h @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_LEGACY_GAIN_CONTROL_H_ +#define MODULES_AUDIO_PROCESSING_AGC_LEGACY_GAIN_CONTROL_H_ + +#include "typedefs.h" // NOLINT(build/include) + +// Errors +#define AGC_UNSPECIFIED_ERROR 18000 +#define AGC_UNSUPPORTED_FUNCTION_ERROR 18001 +#define AGC_UNINITIALIZED_ERROR 18002 +#define AGC_NULL_POINTER_ERROR 18003 +#define AGC_BAD_PARAMETER_ERROR 18004 + +// Warnings +#define AGC_BAD_PARAMETER_WARNING 18050 + +enum { + kAgcModeUnchanged, + kAgcModeAdaptiveAnalog, + kAgcModeAdaptiveDigital, + kAgcModeFixedDigital +}; + +enum { kAgcFalse = 0, kAgcTrue }; + +typedef struct { + int16_t targetLevelDbfs; // default 3 (-3 dBOv) + int16_t compressionGaindB; // default 9 dB + uint8_t limiterEnable; // default kAgcTrue (on) +} WebRtcAgcConfig; + +#if defined(__cplusplus) +extern "C" { +#endif + +/* + * This function analyses the number of samples passed to + * farend and produces any error code that could arise. + * + * Input: + * - agcInst : AGC instance. + * - samples : Number of samples in input vector. + * + * Return value: + * : 0 - Normal operation. + * : -1 - Error. + */ +int WebRtcAgc_GetAddFarendError(void* state, size_t samples); + +/* + * This function processes a 10 ms frame of far-end speech to determine + * if there is active speech. The length of the input speech vector must be + * given in samples (80 when FS=8000, and 160 when FS=16000, FS=32000 or + * FS=48000). + * + * Input: + * - agcInst : AGC instance. + * - inFar : Far-end input speech vector + * - samples : Number of samples in input vector + * + * Return value: + * : 0 - Normal operation. + * : -1 - Error + */ +int WebRtcAgc_AddFarend(void* agcInst, const int16_t* inFar, size_t samples); + +/* + * This function processes a 10 ms frame of microphone speech to determine + * if there is active speech. The length of the input speech vector must be + * given in samples (80 when FS=8000, and 160 when FS=16000, FS=32000 or + * FS=48000). For very low input levels, the input signal is increased in level + * by multiplying and overwriting the samples in inMic[]. + * + * This function should be called before any further processing of the + * near-end microphone signal. + * + * Input: + * - agcInst : AGC instance. + * - inMic : Microphone input speech vector for each band + * - num_bands : Number of bands in input vector + * - samples : Number of samples in input vector + * + * Return value: + * : 0 - Normal operation. + * : -1 - Error + */ +int WebRtcAgc_AddMic(void* agcInst, + int16_t* const* inMic, + size_t num_bands, + size_t samples); + +/* + * This function replaces the analog microphone with a virtual one. + * It is a digital gain applied to the input signal and is used in the + * agcAdaptiveDigital mode where no microphone level is adjustable. The length + * of the input speech vector must be given in samples (80 when FS=8000, and 160 + * when FS=16000, FS=32000 or FS=48000). + * + * Input: + * - agcInst : AGC instance. + * - inMic : Microphone input speech vector for each band + * - num_bands : Number of bands in input vector + * - samples : Number of samples in input vector + * - micLevelIn : Input level of microphone (static) + * + * Output: + * - inMic : Microphone output after processing (L band) + * - inMic_H : Microphone output after processing (H band) + * - micLevelOut : Adjusted microphone level after processing + * + * Return value: + * : 0 - Normal operation. + * : -1 - Error + */ +int WebRtcAgc_VirtualMic(void* agcInst, + int16_t* const* inMic, + size_t num_bands, + size_t samples, + int32_t micLevelIn, + int32_t* micLevelOut); + +/* + * This function processes a 10 ms frame and adjusts (normalizes) the gain both + * analog and digitally. The gain adjustments are done only during active + * periods of speech. The length of the speech vectors must be given in samples + * (80 when FS=8000, and 160 when FS=16000, FS=32000 or FS=48000). The echo + * parameter can be used to ensure the AGC will not adjust upward in the + * presence of echo. + * + * This function should be called after processing the near-end microphone + * signal, in any case after any echo cancellation. + * + * Input: + * - agcInst : AGC instance + * - inNear : Near-end input speech vector for each band + * - num_bands : Number of bands in input/output vector + * - samples : Number of samples in input/output vector + * - inMicLevel : Current microphone volume level + * - echo : Set to 0 if the signal passed to add_mic is + * almost certainly free of echo; otherwise set + * to 1. If you have no information regarding echo + * set to 0. + * + * Output: + * - outMicLevel : Adjusted microphone volume level + * - out : Gain-adjusted near-end speech vector + * : May be the same vector as the input. + * - saturationWarning : A returned value of 1 indicates a saturation event + * has occurred and the volume cannot be further + * reduced. Otherwise will be set to 0. + * + * Return value: + * : 0 - Normal operation. + * : -1 - Error + */ +int WebRtcAgc_Process(void* agcInst, + const int16_t* const* inNear, + size_t num_bands, + size_t samples, + int16_t* const* out, + int32_t inMicLevel, + int32_t* outMicLevel, + int16_t echo, + uint8_t* saturationWarning); + +/* + * This function sets the config parameters (targetLevelDbfs, + * compressionGaindB and limiterEnable). + * + * Input: + * - agcInst : AGC instance + * - config : config struct + * + * Output: + * + * Return value: + * : 0 - Normal operation. + * : -1 - Error + */ +int WebRtcAgc_set_config(void* agcInst, WebRtcAgcConfig config); + +/* + * This function returns the config parameters (targetLevelDbfs, + * compressionGaindB and limiterEnable). + * + * Input: + * - agcInst : AGC instance + * + * Output: + * - config : config struct + * + * Return value: + * : 0 - Normal operation. + * : -1 - Error + */ +int WebRtcAgc_get_config(void* agcInst, WebRtcAgcConfig* config); + +/* + * This function creates and returns an AGC instance, which will contain the + * state information for one (duplex) channel. + */ +void* WebRtcAgc_Create(); + +/* + * This function frees the AGC instance created at the beginning. + * + * Input: + * - agcInst : AGC instance. + */ +void WebRtcAgc_Free(void* agcInst); + +/* + * This function initializes an AGC instance. + * + * Input: + * - agcInst : AGC instance. + * - minLevel : Minimum possible mic level + * - maxLevel : Maximum possible mic level + * - agcMode : 0 - Unchanged + * : 1 - Adaptive Analog Automatic Gain Control -3dBOv + * : 2 - Adaptive Digital Automatic Gain Control -3dBOv + * : 3 - Fixed Digital Gain 0dB + * - fs : Sampling frequency + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcAgc_Init(void* agcInst, + int32_t minLevel, + int32_t maxLevel, + int16_t agcMode, + uint32_t fs); + +#if defined(__cplusplus) +} +#endif + +#endif // MODULES_AUDIO_PROCESSING_AGC_LEGACY_GAIN_CONTROL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.cc new file mode 100644 index 0000000000..63d5f7cad0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.cc @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc/loudness_histogram.h" + +#include +#include + +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +static const double kHistBinCenters[] = { + 7.59621091765857e-02, 9.02036021061016e-02, 1.07115112009343e-01, + 1.27197217770508e-01, 1.51044347572047e-01, 1.79362373905283e-01, + 2.12989507320644e-01, 2.52921107370304e-01, 3.00339145144454e-01, + 3.56647189489147e-01, 4.23511952494003e-01, 5.02912623991786e-01, + 5.97199455365749e-01, 7.09163326739184e-01, 8.42118356728544e-01, + 1.00000000000000e+00, 1.18748153630660e+00, 1.41011239906908e+00, + 1.67448243801153e+00, 1.98841697800836e+00, 2.36120844786349e+00, + 2.80389143520905e+00, 3.32956930911896e+00, 3.95380207843188e+00, + 4.69506696634852e+00, 5.57530533426190e+00, 6.62057214370769e+00, + 7.86180718043869e+00, 9.33575086877358e+00, 1.10860317842269e+01, + 1.31644580546776e+01, 1.56325508754123e+01, 1.85633655299256e+01, + 2.20436538184971e+01, 2.61764319021997e+01, 3.10840295702492e+01, + 3.69117111886792e+01, 4.38319755100383e+01, 5.20496616180135e+01, + 6.18080121423973e+01, 7.33958732149108e+01, 8.71562442838066e+01, + 1.03496430860848e+02, 1.22900100720889e+02, 1.45941600416277e+02, + 1.73302955873365e+02, 2.05794060286978e+02, 2.44376646872353e+02, + 2.90192756065437e+02, 3.44598539797631e+02, 4.09204403447902e+02, + 4.85922673669740e+02, 5.77024203055553e+02, 6.85205587130498e+02, + 8.13668983291589e+02, 9.66216894324125e+02, 1.14736472207740e+03, + 1.36247442287647e+03, 1.61791322085579e+03, 1.92124207711260e+03, + 2.28143949334655e+03, 2.70916727454970e+03, 3.21708611729384e+03, + 3.82023036499473e+03, 4.53645302286906e+03, 5.38695420497926e+03, + 6.39690865534207e+03, 7.59621091765857e+03, 9.02036021061016e+03, + 1.07115112009343e+04, 1.27197217770508e+04, 1.51044347572047e+04, + 1.79362373905283e+04, 2.12989507320644e+04, 2.52921107370304e+04, + 3.00339145144454e+04, 3.56647189489147e+04}; + +static const double kProbQDomain = 1024.0; +// Loudness of -15 dB (smallest expected loudness) in log domain, +// loudness_db = 13.5 * log10(rms); +static const double kLogDomainMinBinCenter = -2.57752062648587; +// Loudness step of 1 dB in log domain +static const double kLogDomainStepSizeInverse = 5.81954605750359; + +static const int kTransientWidthThreshold = 7; +static const double kLowProbabilityThreshold = 0.2; + +static const int kLowProbThresholdQ10 = + static_cast(kLowProbabilityThreshold * kProbQDomain); + +LoudnessHistogram::LoudnessHistogram() + : num_updates_(0), + audio_content_q10_(0), + bin_count_q10_(), + activity_probability_(), + hist_bin_index_(), + buffer_index_(0), + buffer_is_full_(false), + len_circular_buffer_(0), + len_high_activity_(0) { + static_assert( + kHistSize == sizeof(kHistBinCenters) / sizeof(kHistBinCenters[0]), + "histogram bin centers incorrect size"); +} + +LoudnessHistogram::LoudnessHistogram(int window_size) + : num_updates_(0), + audio_content_q10_(0), + bin_count_q10_(), + activity_probability_(new int[window_size]), + hist_bin_index_(new int[window_size]), + buffer_index_(0), + buffer_is_full_(false), + len_circular_buffer_(window_size), + len_high_activity_(0) {} + +LoudnessHistogram::~LoudnessHistogram() {} + +void LoudnessHistogram::Update(double rms, double activity_probaility) { + // If circular histogram is activated then remove the oldest entry. + if (len_circular_buffer_ > 0) + RemoveOldestEntryAndUpdate(); + + // Find the corresponding bin. + int hist_index = GetBinIndex(rms); + // To Q10 domain. + int prob_q10 = + static_cast(floor(activity_probaility * kProbQDomain)); + InsertNewestEntryAndUpdate(prob_q10, hist_index); +} + +// Doing nothing if buffer is not full, yet. +void LoudnessHistogram::RemoveOldestEntryAndUpdate() { + RTC_DCHECK_GT(len_circular_buffer_, 0); + // Do nothing if circular buffer is not full. + if (!buffer_is_full_) + return; + + int oldest_prob = activity_probability_[buffer_index_]; + int oldest_hist_index = hist_bin_index_[buffer_index_]; + UpdateHist(-oldest_prob, oldest_hist_index); +} + +void LoudnessHistogram::RemoveTransient() { + // Don't expect to be here if high-activity region is longer than + // |kTransientWidthThreshold| or there has not been any transient. + RTC_DCHECK_LE(len_high_activity_, kTransientWidthThreshold); + int index = + (buffer_index_ > 0) ? (buffer_index_ - 1) : len_circular_buffer_ - 1; + while (len_high_activity_ > 0) { + UpdateHist(-activity_probability_[index], hist_bin_index_[index]); + activity_probability_[index] = 0; + index = (index > 0) ? (index - 1) : (len_circular_buffer_ - 1); + len_high_activity_--; + } +} + +void LoudnessHistogram::InsertNewestEntryAndUpdate(int activity_prob_q10, + int hist_index) { + // Update the circular buffer if it is enabled. + if (len_circular_buffer_ > 0) { + // Removing transient. + if (activity_prob_q10 <= kLowProbThresholdQ10) { + // Lower than threshold probability, set it to zero. + activity_prob_q10 = 0; + // Check if this has been a transient. + if (len_high_activity_ <= kTransientWidthThreshold) + RemoveTransient(); // Remove this transient. + len_high_activity_ = 0; + } else if (len_high_activity_ <= kTransientWidthThreshold) { + len_high_activity_++; + } + // Updating the circular buffer. + activity_probability_[buffer_index_] = activity_prob_q10; + hist_bin_index_[buffer_index_] = hist_index; + // Increment the buffer index and check for wrap-around. + buffer_index_++; + if (buffer_index_ >= len_circular_buffer_) { + buffer_index_ = 0; + buffer_is_full_ = true; + } + } + + num_updates_++; + if (num_updates_ < 0) + num_updates_--; + + UpdateHist(activity_prob_q10, hist_index); +} + +void LoudnessHistogram::UpdateHist(int activity_prob_q10, int hist_index) { + bin_count_q10_[hist_index] += activity_prob_q10; + audio_content_q10_ += activity_prob_q10; +} + +double LoudnessHistogram::AudioContent() const { + return audio_content_q10_ / kProbQDomain; +} + +LoudnessHistogram* LoudnessHistogram::Create() { + return new LoudnessHistogram; +} + +LoudnessHistogram* LoudnessHistogram::Create(int window_size) { + if (window_size < 0) + return NULL; + return new LoudnessHistogram(window_size); +} + +void LoudnessHistogram::Reset() { + // Reset the histogram, audio-content and number of updates. + memset(bin_count_q10_, 0, sizeof(bin_count_q10_)); + audio_content_q10_ = 0; + num_updates_ = 0; + // Empty the circular buffer. + buffer_index_ = 0; + buffer_is_full_ = false; + len_high_activity_ = 0; +} + +int LoudnessHistogram::GetBinIndex(double rms) { + // First exclude overload cases. + if (rms <= kHistBinCenters[0]) { + return 0; + } else if (rms >= kHistBinCenters[kHistSize - 1]) { + return kHistSize - 1; + } else { + // The quantizer is uniform in log domain. Alternatively we could do binary + // search in linear domain. + double rms_log = log(rms); + + int index = static_cast( + floor((rms_log - kLogDomainMinBinCenter) * kLogDomainStepSizeInverse)); + // The final decision is in linear domain. + double b = 0.5 * (kHistBinCenters[index] + kHistBinCenters[index + 1]); + if (rms > b) { + return index + 1; + } + return index; + } +} + +double LoudnessHistogram::CurrentRms() const { + double p; + double mean_val = 0; + if (audio_content_q10_ > 0) { + double p_total_inverse = 1. / static_cast(audio_content_q10_); + for (int n = 0; n < kHistSize; n++) { + p = static_cast(bin_count_q10_[n]) * p_total_inverse; + mean_val += p * kHistBinCenters[n]; + } + } else { + mean_val = kHistBinCenters[0]; + } + return mean_val; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.h new file mode 100644 index 0000000000..ab45276f06 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_LOUDNESS_HISTOGRAM_H_ +#define MODULES_AUDIO_PROCESSING_AGC_LOUDNESS_HISTOGRAM_H_ + +#include + +#include + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// This class implements the histogram of loudness with circular buffers so that +// the histogram tracks the last T seconds of the loudness. +class LoudnessHistogram { + public: + // Create a non-sliding LoudnessHistogram. + static LoudnessHistogram* Create(); + + // Create a sliding LoudnessHistogram, i.e. the histogram represents the last + // |window_size| samples. + static LoudnessHistogram* Create(int window_size); + ~LoudnessHistogram(); + + // Insert RMS and the corresponding activity probability. + void Update(double rms, double activity_probability); + + // Reset the histogram, forget the past. + void Reset(); + + // Current loudness, which is actually the mean of histogram in loudness + // domain. + double CurrentRms() const; + + // Sum of the histogram content. + double AudioContent() const; + + // Number of times the histogram has been updated. + int num_updates() const { return num_updates_; } + + private: + LoudnessHistogram(); + explicit LoudnessHistogram(int window); + + // Find the histogram bin associated with the given |rms|. + int GetBinIndex(double rms); + + void RemoveOldestEntryAndUpdate(); + void InsertNewestEntryAndUpdate(int activity_prob_q10, int hist_index); + void UpdateHist(int activity_prob_q10, int hist_index); + void RemoveTransient(); + + // Number of histogram bins. + static const int kHistSize = 77; + + // Number of times the histogram is updated + int num_updates_; + // Audio content, this should be equal to the sum of the components of + // |bin_count_q10_|. + int64_t audio_content_q10_; + + // LoudnessHistogram of input RMS in Q10 with |kHistSize_| bins. In each + // 'Update(),' we increment the associated histogram-bin with the given + // probability. The increment is implemented in Q10 to avoid rounding errors. + int64_t bin_count_q10_[kHistSize]; + + // Circular buffer for probabilities + std::unique_ptr activity_probability_; + // Circular buffer for histogram-indices of probabilities. + std::unique_ptr hist_bin_index_; + // Current index of circular buffer, where the newest data will be written to, + // therefore, pointing to the oldest data if buffer is full. + int buffer_index_; + // Indicating if buffer is full and we had a wrap around. + int buffer_is_full_; + // Size of circular buffer. + int len_circular_buffer_; + int len_high_activity_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC_LOUDNESS_HISTOGRAM_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram_unittest.cc new file mode 100644 index 0000000000..8c617107ff --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram_unittest.cc @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Use CreateHistUnittestFile.m to generate the input file. + +#include "modules/audio_processing/agc/loudness_histogram.h" + +#include +#include +#include +#include + +#include "modules/audio_processing/agc/utility.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +struct InputOutput { + double rms; + double activity_probability; + double audio_content; + double loudness; +}; + +const double kRelativeErrTol = 1e-10; + +class LoudnessHistogramTest : public ::testing::Test { + protected: + void RunTest(bool enable_circular_buff, const char* filename); + + private: + void TestClean(); + std::unique_ptr hist_; +}; + +void LoudnessHistogramTest::TestClean() { + EXPECT_EQ(hist_->CurrentRms(), 7.59621091765857e-02); + EXPECT_EQ(hist_->AudioContent(), 0); + EXPECT_EQ(hist_->num_updates(), 0); +} + +void LoudnessHistogramTest::RunTest(bool enable_circular_buff, + const char* filename) { + FILE* in_file = fopen(filename, "rb"); + ASSERT_TRUE(in_file != NULL); + if (enable_circular_buff) { + int buffer_size; + EXPECT_EQ(fread(&buffer_size, sizeof(buffer_size), 1, in_file), 1u); + hist_.reset(LoudnessHistogram::Create(buffer_size)); + } else { + hist_.reset(LoudnessHistogram::Create()); + } + TestClean(); + + InputOutput io; + int num_updates = 0; + int num_reset = 0; + while (fread(&io, sizeof(InputOutput), 1, in_file) == 1) { + if (io.rms < 0) { + // We have to reset. + hist_->Reset(); + TestClean(); + num_updates = 0; + num_reset++; + // Read the next chunk of input. + if (fread(&io, sizeof(InputOutput), 1, in_file) != 1) + break; + } + hist_->Update(io.rms, io.activity_probability); + num_updates++; + EXPECT_EQ(hist_->num_updates(), num_updates); + double audio_content = hist_->AudioContent(); + + double abs_err = + std::min(audio_content, io.audio_content) * kRelativeErrTol; + + ASSERT_NEAR(audio_content, io.audio_content, abs_err); + double current_loudness = Linear2Loudness(hist_->CurrentRms()); + abs_err = + std::min(fabs(current_loudness), fabs(io.loudness)) * kRelativeErrTol; + ASSERT_NEAR(current_loudness, io.loudness, abs_err); + } + fclose(in_file); +} + +TEST_F(LoudnessHistogramTest, ActiveCircularBuffer) { + RunTest(true, test::ResourcePath( + "audio_processing/agc/agc_with_circular_buffer", "dat") + .c_str()); +} + +TEST_F(LoudnessHistogramTest, InactiveCircularBuffer) { + RunTest(false, test::ResourcePath( + "audio_processing/agc/agc_no_circular_buffer", "dat") + .c_str()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/mock_agc.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/mock_agc.h new file mode 100644 index 0000000000..36901af50c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/mock_agc.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_MOCK_AGC_H_ +#define MODULES_AUDIO_PROCESSING_AGC_MOCK_AGC_H_ + +#include "modules/audio_processing/agc/agc.h" + +#include "modules/include/module_common_types.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockAgc : public Agc { + public: + MOCK_METHOD2(AnalyzePreproc, float(const int16_t* audio, size_t length)); + MOCK_METHOD3(Process, int(const int16_t* audio, size_t length, + int sample_rate_hz)); + MOCK_METHOD1(GetRmsErrorDb, bool(int* error)); + MOCK_METHOD0(Reset, void()); + MOCK_METHOD1(set_target_level_dbfs, int(int level)); + MOCK_CONST_METHOD0(target_level_dbfs, int()); + MOCK_METHOD1(EnableStandaloneVad, void(bool enable)); + MOCK_CONST_METHOD0(standalone_vad_enabled, bool()); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC_MOCK_AGC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.cc new file mode 100644 index 0000000000..554a8377aa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.cc @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc/utility.h" + +#include + +static const double kLog10 = 2.30258509299; +static const double kLinear2DbScale = 20.0 / kLog10; +static const double kLinear2LoudnessScale = 13.4 / kLog10; + +double Loudness2Db(double loudness) { + return loudness * kLinear2DbScale / kLinear2LoudnessScale; +} + +double Linear2Loudness(double rms) { + if (rms == 0) + return -15; + return kLinear2LoudnessScale * log(rms); +} + +double Db2Loudness(double db) { + return db * kLinear2LoudnessScale / kLinear2DbScale; +} + +double Dbfs2Loudness(double dbfs) { + return Db2Loudness(90 + dbfs); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.h new file mode 100644 index 0000000000..8ba87b6d55 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_UTILITY_H_ +#define MODULES_AUDIO_PROCESSING_AGC_UTILITY_H_ + +// TODO(turajs): Add description of function. +double Loudness2Db(double loudness); + +double Linear2Loudness(double rms); + +double Db2Loudness(double db); + +double Dbfs2Loudness(double dbfs); + +#endif // MODULES_AUDIO_PROCESSING_AGC_UTILITY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.cc new file mode 100644 index 0000000000..ca564e8b1d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/gain_controller2.h" + +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +int GainController2::instance_count_ = 0; + +GainController2::GainController2() + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + sample_rate_hz_(AudioProcessing::kSampleRate48kHz), + fixed_gain_(1.f) {} + +GainController2::~GainController2() = default; + +void GainController2::Initialize(int sample_rate_hz) { + RTC_DCHECK(sample_rate_hz == AudioProcessing::kSampleRate8kHz || + sample_rate_hz == AudioProcessing::kSampleRate16kHz || + sample_rate_hz == AudioProcessing::kSampleRate32kHz || + sample_rate_hz == AudioProcessing::kSampleRate48kHz); + sample_rate_hz_ = sample_rate_hz; + data_dumper_->InitiateNewSetOfRecordings(); + data_dumper_->DumpRaw("sample_rate_hz", sample_rate_hz_); + data_dumper_->DumpRaw("fixed_gain_linear", fixed_gain_); +} + +void GainController2::Process(AudioBuffer* audio) { + if (fixed_gain_ == 1.f) + return; + + for (size_t k = 0; k < audio->num_channels(); ++k) { + for (size_t j = 0; j < audio->num_frames(); ++j) { + audio->channels_f()[k][j] = rtc::SafeClamp( + fixed_gain_ * audio->channels_f()[k][j], -32768.f, 32767.f); + } + } +} + +void GainController2::ApplyConfig( + const AudioProcessing::Config::GainController2& config) { + RTC_DCHECK(Validate(config)); + fixed_gain_ = std::pow(10.f, config.fixed_gain_db / 20.f); +} + +bool GainController2::Validate( + const AudioProcessing::Config::GainController2& config) { + return config.fixed_gain_db >= 0.f; +} + +std::string GainController2::ToString( + const AudioProcessing::Config::GainController2& config) { + std::stringstream ss; + ss << "{enabled: " << (config.enabled ? "true" : "false") << ", " + << "fixed_gain_dB: " << config.fixed_gain_db << "}"; + return ss.str(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.h b/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.h new file mode 100644 index 0000000000..11706870f4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_GAIN_CONTROLLER2_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_GAIN_CONTROLLER2_H_ + +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; +class AudioBuffer; + +// Gain Controller 2 aims to automatically adjust levels by acting on the +// microphone gain and/or applying digital gain. +// +// Temporarily implements a fixed gain mode with hard-clipping. +class GainController2 { + public: + GainController2(); + ~GainController2(); + + void Initialize(int sample_rate_hz); + void Process(AudioBuffer* audio); + + void ApplyConfig(const AudioProcessing::Config::GainController2& config); + static bool Validate(const AudioProcessing::Config::GainController2& config); + static std::string ToString( + const AudioProcessing::Config::GainController2& config); + + private: + static int instance_count_; + std::unique_ptr data_dumper_; + int sample_rate_hz_; + float fixed_gain_; + + RTC_DISALLOW_COPY_AND_ASSIGN(GainController2); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_GAIN_CONTROLLER2_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2_unittest.cc new file mode 100644 index 0000000000..46f654db62 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2_unittest.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/agc2/gain_controller2.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { + +constexpr size_t kFrameSizeMs = 10u; +constexpr size_t kStereo = 2u; + +void SetAudioBufferSamples(float value, AudioBuffer* ab) { + // Sets all the samples in |ab| to |value|. + for (size_t k = 0; k < ab->num_channels(); ++k) { + std::fill(ab->channels_f()[k], ab->channels_f()[k] + ab->num_frames(), + value); + } +} + +} // namespace + +TEST(GainController2, CreateApplyConfig) { + // Instances GainController2 and applies different configurations. + std::unique_ptr gain_controller2(new GainController2()); + + // Check that the default config is valid. + AudioProcessing::Config::GainController2 config; + EXPECT_TRUE(GainController2::Validate(config)); + gain_controller2->ApplyConfig(config); + + // Check that attenuation is not allowed. + config.fixed_gain_db = -5.f; + EXPECT_FALSE(GainController2::Validate(config)); + + // Check that valid configurations are applied. + for (const float& fixed_gain_db : {0.f, 5.f, 10.f, 50.f}) { + config.fixed_gain_db = fixed_gain_db; + EXPECT_TRUE(GainController2::Validate(config)); + gain_controller2->ApplyConfig(config); + } +} + +TEST(GainController2, ToString) { + // Tests GainController2::ToString(). + AudioProcessing::Config::GainController2 config; + config.fixed_gain_db = 5.f; + + config.enabled = false; + EXPECT_EQ("{enabled: false, fixed_gain_dB: 5}", + GainController2::ToString(config)); + + config.enabled = true; + EXPECT_EQ("{enabled: true, fixed_gain_dB: 5}", + GainController2::ToString(config)); +} + +TEST(GainController2, Usage) { + // Tests GainController2::Process() on an AudioBuffer instance. + std::unique_ptr gain_controller2(new GainController2()); + gain_controller2->Initialize(AudioProcessing::kSampleRate48kHz); + const size_t num_frames = rtc::CheckedDivExact( + kFrameSizeMs * AudioProcessing::kSampleRate48kHz, 1000); + AudioBuffer ab(num_frames, kStereo, num_frames, kStereo, num_frames); + constexpr float sample_value = 1000.f; + SetAudioBufferSamples(sample_value, &ab); + AudioProcessing::Config::GainController2 config; + + // Check that samples are not modified when the fixed gain is 0 dB. + ASSERT_EQ(config.fixed_gain_db, 0.f); + gain_controller2->ApplyConfig(config); + gain_controller2->Process(&ab); + EXPECT_EQ(ab.channels_f()[0][0], sample_value); + + // Check that samples are amplified when the fixed gain is greater than 0 dB. + config.fixed_gain_db = 5.f; + gain_controller2->ApplyConfig(config); + gain_controller2->Process(&ab); + EXPECT_LT(sample_value, ab.channels_f()[0][0]); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.cc new file mode 100644 index 0000000000..16f11742e3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.cc @@ -0,0 +1,475 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/audio_buffer.h" + +#include "common_audio/channel_buffer.h" +#include "common_audio/include/audio_util.h" +#include "common_audio/resampler/push_sinc_resampler.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/common.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +const size_t kSamplesPer16kHzChannel = 160; +const size_t kSamplesPer32kHzChannel = 320; +const size_t kSamplesPer48kHzChannel = 480; + +int KeyboardChannelIndex(const StreamConfig& stream_config) { + if (!stream_config.has_keyboard()) { + RTC_NOTREACHED(); + return 0; + } + + return stream_config.num_channels(); +} + +size_t NumBandsFromSamplesPerChannel(size_t num_frames) { + size_t num_bands = 1; + if (num_frames == kSamplesPer32kHzChannel || + num_frames == kSamplesPer48kHzChannel) { + num_bands = rtc::CheckedDivExact(num_frames, kSamplesPer16kHzChannel); + } + return num_bands; +} + +} // namespace + +AudioBuffer::AudioBuffer(size_t input_num_frames, + size_t num_input_channels, + size_t process_num_frames, + size_t num_process_channels, + size_t output_num_frames) + : input_num_frames_(input_num_frames), + num_input_channels_(num_input_channels), + proc_num_frames_(process_num_frames), + num_proc_channels_(num_process_channels), + output_num_frames_(output_num_frames), + num_channels_(num_process_channels), + num_bands_(NumBandsFromSamplesPerChannel(proc_num_frames_)), + num_split_frames_(rtc::CheckedDivExact(proc_num_frames_, num_bands_)), + mixed_low_pass_valid_(false), + reference_copied_(false), + activity_(AudioFrame::kVadUnknown), + keyboard_data_(NULL), + data_(new IFChannelBuffer(proc_num_frames_, num_proc_channels_)), + output_buffer_(new IFChannelBuffer(output_num_frames_, num_channels_)) { + RTC_DCHECK_GT(input_num_frames_, 0); + RTC_DCHECK_GT(proc_num_frames_, 0); + RTC_DCHECK_GT(output_num_frames_, 0); + RTC_DCHECK_GT(num_input_channels_, 0); + RTC_DCHECK_GT(num_proc_channels_, 0); + RTC_DCHECK_LE(num_proc_channels_, num_input_channels_); + + if (input_num_frames_ != proc_num_frames_ || + output_num_frames_ != proc_num_frames_) { + // Create an intermediate buffer for resampling. + process_buffer_.reset(new ChannelBuffer(proc_num_frames_, + num_proc_channels_)); + + if (input_num_frames_ != proc_num_frames_) { + for (size_t i = 0; i < num_proc_channels_; ++i) { + input_resamplers_.push_back(std::unique_ptr( + new PushSincResampler(input_num_frames_, proc_num_frames_))); + } + } + + if (output_num_frames_ != proc_num_frames_) { + for (size_t i = 0; i < num_proc_channels_; ++i) { + output_resamplers_.push_back(std::unique_ptr( + new PushSincResampler(proc_num_frames_, output_num_frames_))); + } + } + } + + if (num_bands_ > 1) { + split_data_.reset(new IFChannelBuffer(proc_num_frames_, + num_proc_channels_, + num_bands_)); + splitting_filter_.reset(new SplittingFilter(num_proc_channels_, + num_bands_, + proc_num_frames_)); + } +} + +AudioBuffer::~AudioBuffer() {} + +void AudioBuffer::CopyFrom(const float* const* data, + const StreamConfig& stream_config) { + RTC_DCHECK_EQ(stream_config.num_frames(), input_num_frames_); + RTC_DCHECK_EQ(stream_config.num_channels(), num_input_channels_); + InitForNewData(); + // Initialized lazily because there's a different condition in + // DeinterleaveFrom. + const bool need_to_downmix = + num_input_channels_ > 1 && num_proc_channels_ == 1; + if (need_to_downmix && !input_buffer_) { + input_buffer_.reset( + new IFChannelBuffer(input_num_frames_, num_proc_channels_)); + } + + if (stream_config.has_keyboard()) { + keyboard_data_ = data[KeyboardChannelIndex(stream_config)]; + } + + // Downmix. + const float* const* data_ptr = data; + if (need_to_downmix) { + DownmixToMono(data, input_num_frames_, num_input_channels_, + input_buffer_->fbuf()->channels()[0]); + data_ptr = input_buffer_->fbuf_const()->channels(); + } + + // Resample. + if (input_num_frames_ != proc_num_frames_) { + for (size_t i = 0; i < num_proc_channels_; ++i) { + input_resamplers_[i]->Resample(data_ptr[i], + input_num_frames_, + process_buffer_->channels()[i], + proc_num_frames_); + } + data_ptr = process_buffer_->channels(); + } + + // Convert to the S16 range. + for (size_t i = 0; i < num_proc_channels_; ++i) { + FloatToFloatS16(data_ptr[i], + proc_num_frames_, + data_->fbuf()->channels()[i]); + } +} + +void AudioBuffer::CopyTo(const StreamConfig& stream_config, + float* const* data) { + RTC_DCHECK_EQ(stream_config.num_frames(), output_num_frames_); + RTC_DCHECK(stream_config.num_channels() == num_channels_ || + num_channels_ == 1); + + // Convert to the float range. + float* const* data_ptr = data; + if (output_num_frames_ != proc_num_frames_) { + // Convert to an intermediate buffer for subsequent resampling. + data_ptr = process_buffer_->channels(); + } + for (size_t i = 0; i < num_channels_; ++i) { + FloatS16ToFloat(data_->fbuf()->channels()[i], + proc_num_frames_, + data_ptr[i]); + } + + // Resample. + if (output_num_frames_ != proc_num_frames_) { + for (size_t i = 0; i < num_channels_; ++i) { + output_resamplers_[i]->Resample(data_ptr[i], + proc_num_frames_, + data[i], + output_num_frames_); + } + } + + // Upmix. + for (size_t i = num_channels_; i < stream_config.num_channels(); ++i) { + memcpy(data[i], data[0], output_num_frames_ * sizeof(**data)); + } +} + +void AudioBuffer::InitForNewData() { + keyboard_data_ = NULL; + mixed_low_pass_valid_ = false; + reference_copied_ = false; + activity_ = AudioFrame::kVadUnknown; + num_channels_ = num_proc_channels_; + data_->set_num_channels(num_proc_channels_); + if (split_data_.get()) { + split_data_->set_num_channels(num_proc_channels_); + } +} + +const int16_t* const* AudioBuffer::channels_const() const { + return data_->ibuf_const()->channels(); +} + +int16_t* const* AudioBuffer::channels() { + mixed_low_pass_valid_ = false; + return data_->ibuf()->channels(); +} + +const int16_t* const* AudioBuffer::split_bands_const(size_t channel) const { + return split_data_.get() ? + split_data_->ibuf_const()->bands(channel) : + data_->ibuf_const()->bands(channel); +} + +int16_t* const* AudioBuffer::split_bands(size_t channel) { + mixed_low_pass_valid_ = false; + return split_data_.get() ? + split_data_->ibuf()->bands(channel) : + data_->ibuf()->bands(channel); +} + +const int16_t* const* AudioBuffer::split_channels_const(Band band) const { + if (split_data_.get()) { + return split_data_->ibuf_const()->channels(band); + } else { + return band == kBand0To8kHz ? data_->ibuf_const()->channels() : nullptr; + } +} + +int16_t* const* AudioBuffer::split_channels(Band band) { + mixed_low_pass_valid_ = false; + if (split_data_.get()) { + return split_data_->ibuf()->channels(band); + } else { + return band == kBand0To8kHz ? data_->ibuf()->channels() : nullptr; + } +} + +ChannelBuffer* AudioBuffer::data() { + mixed_low_pass_valid_ = false; + return data_->ibuf(); +} + +const ChannelBuffer* AudioBuffer::data() const { + return data_->ibuf_const(); +} + +ChannelBuffer* AudioBuffer::split_data() { + mixed_low_pass_valid_ = false; + return split_data_.get() ? split_data_->ibuf() : data_->ibuf(); +} + +const ChannelBuffer* AudioBuffer::split_data() const { + return split_data_.get() ? split_data_->ibuf_const() : data_->ibuf_const(); +} + +const float* const* AudioBuffer::channels_const_f() const { + return data_->fbuf_const()->channels(); +} + +float* const* AudioBuffer::channels_f() { + mixed_low_pass_valid_ = false; + return data_->fbuf()->channels(); +} + +const float* const* AudioBuffer::split_bands_const_f(size_t channel) const { + return split_data_.get() ? + split_data_->fbuf_const()->bands(channel) : + data_->fbuf_const()->bands(channel); +} + +float* const* AudioBuffer::split_bands_f(size_t channel) { + mixed_low_pass_valid_ = false; + return split_data_.get() ? + split_data_->fbuf()->bands(channel) : + data_->fbuf()->bands(channel); +} + +const float* const* AudioBuffer::split_channels_const_f(Band band) const { + if (split_data_.get()) { + return split_data_->fbuf_const()->channels(band); + } else { + return band == kBand0To8kHz ? data_->fbuf_const()->channels() : nullptr; + } +} + +float* const* AudioBuffer::split_channels_f(Band band) { + mixed_low_pass_valid_ = false; + if (split_data_.get()) { + return split_data_->fbuf()->channels(band); + } else { + return band == kBand0To8kHz ? data_->fbuf()->channels() : nullptr; + } +} + +ChannelBuffer* AudioBuffer::data_f() { + mixed_low_pass_valid_ = false; + return data_->fbuf(); +} + +const ChannelBuffer* AudioBuffer::data_f() const { + return data_->fbuf_const(); +} + +ChannelBuffer* AudioBuffer::split_data_f() { + mixed_low_pass_valid_ = false; + return split_data_.get() ? split_data_->fbuf() : data_->fbuf(); +} + +const ChannelBuffer* AudioBuffer::split_data_f() const { + return split_data_.get() ? split_data_->fbuf_const() : data_->fbuf_const(); +} + +const int16_t* AudioBuffer::mixed_low_pass_data() { + if (num_proc_channels_ == 1) { + return split_bands_const(0)[kBand0To8kHz]; + } + + if (!mixed_low_pass_valid_) { + if (!mixed_low_pass_channels_.get()) { + mixed_low_pass_channels_.reset( + new ChannelBuffer(num_split_frames_, 1)); + } + + DownmixToMono(split_channels_const(kBand0To8kHz), + num_split_frames_, num_channels_, + mixed_low_pass_channels_->channels()[0]); + mixed_low_pass_valid_ = true; + } + return mixed_low_pass_channels_->channels()[0]; +} + +const int16_t* AudioBuffer::low_pass_reference(int channel) const { + if (!reference_copied_) { + return NULL; + } + + return low_pass_reference_channels_->channels()[channel]; +} + +const float* AudioBuffer::keyboard_data() const { + return keyboard_data_; +} + +void AudioBuffer::set_activity(AudioFrame::VADActivity activity) { + activity_ = activity; +} + +AudioFrame::VADActivity AudioBuffer::activity() const { + return activity_; +} + +size_t AudioBuffer::num_channels() const { + return num_channels_; +} + +void AudioBuffer::set_num_channels(size_t num_channels) { + num_channels_ = num_channels; + data_->set_num_channels(num_channels); + if (split_data_.get()) { + split_data_->set_num_channels(num_channels); + } +} + +size_t AudioBuffer::num_frames() const { + return proc_num_frames_; +} + +size_t AudioBuffer::num_frames_per_band() const { + return num_split_frames_; +} + +size_t AudioBuffer::num_keyboard_frames() const { + // We don't resample the keyboard channel. + return input_num_frames_; +} + +size_t AudioBuffer::num_bands() const { + return num_bands_; +} + +// The resampler is only for supporting 48kHz to 16kHz in the reverse stream. +void AudioBuffer::DeinterleaveFrom(AudioFrame* frame) { + RTC_DCHECK_EQ(frame->num_channels_, num_input_channels_); + RTC_DCHECK_EQ(frame->samples_per_channel_, input_num_frames_); + InitForNewData(); + // Initialized lazily because there's a different condition in CopyFrom. + if ((input_num_frames_ != proc_num_frames_) && !input_buffer_) { + input_buffer_.reset( + new IFChannelBuffer(input_num_frames_, num_proc_channels_)); + } + activity_ = frame->vad_activity_; + + int16_t* const* deinterleaved; + if (input_num_frames_ == proc_num_frames_) { + deinterleaved = data_->ibuf()->channels(); + } else { + deinterleaved = input_buffer_->ibuf()->channels(); + } + // TODO(yujo): handle muted frames more efficiently. + if (num_proc_channels_ == 1) { + // Downmix and deinterleave simultaneously. + DownmixInterleavedToMono(frame->data(), input_num_frames_, + num_input_channels_, deinterleaved[0]); + } else { + RTC_DCHECK_EQ(num_proc_channels_, num_input_channels_); + Deinterleave(frame->data(), + input_num_frames_, + num_proc_channels_, + deinterleaved); + } + + // Resample. + if (input_num_frames_ != proc_num_frames_) { + for (size_t i = 0; i < num_proc_channels_; ++i) { + input_resamplers_[i]->Resample(input_buffer_->fbuf_const()->channels()[i], + input_num_frames_, + data_->fbuf()->channels()[i], + proc_num_frames_); + } + } +} + +void AudioBuffer::InterleaveTo(AudioFrame* frame, bool data_changed) const { + frame->vad_activity_ = activity_; + if (!data_changed) { + return; + } + + RTC_DCHECK(frame->num_channels_ == num_channels_ || num_channels_ == 1); + RTC_DCHECK_EQ(frame->samples_per_channel_, output_num_frames_); + + // Resample if necessary. + IFChannelBuffer* data_ptr = data_.get(); + if (proc_num_frames_ != output_num_frames_) { + for (size_t i = 0; i < num_channels_; ++i) { + output_resamplers_[i]->Resample( + data_->fbuf()->channels()[i], proc_num_frames_, + output_buffer_->fbuf()->channels()[i], output_num_frames_); + } + data_ptr = output_buffer_.get(); + } + + // TODO(yujo): handle muted frames more efficiently. + if (frame->num_channels_ == num_channels_) { + Interleave(data_ptr->ibuf()->channels(), output_num_frames_, num_channels_, + frame->mutable_data()); + } else { + UpmixMonoToInterleaved(data_ptr->ibuf()->channels()[0], output_num_frames_, + frame->num_channels_, frame->mutable_data()); + } +} + +void AudioBuffer::CopyLowPassToReference() { + reference_copied_ = true; + if (!low_pass_reference_channels_.get() || + low_pass_reference_channels_->num_channels() != num_channels_) { + low_pass_reference_channels_.reset( + new ChannelBuffer(num_split_frames_, + num_proc_channels_)); + } + for (size_t i = 0; i < num_proc_channels_; i++) { + memcpy(low_pass_reference_channels_->channels()[i], + split_bands_const(i)[kBand0To8kHz], + low_pass_reference_channels_->num_frames_per_band() * + sizeof(split_bands_const(i)[kBand0To8kHz][0])); + } +} + +void AudioBuffer::SplitIntoFrequencyBands() { + splitting_filter_->Analysis(data_.get(), split_data_.get()); +} + +void AudioBuffer::MergeFrequencyBands() { + splitting_filter_->Synthesis(split_data_.get(), data_.get()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.h new file mode 100644 index 0000000000..8451bdeeaa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.h @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AUDIO_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AUDIO_BUFFER_H_ + +#include +#include + +#include "common_audio/channel_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/splitting_filter.h" +#include "modules/include/module_common_types.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class PushSincResampler; +class IFChannelBuffer; + +enum Band { + kBand0To8kHz = 0, + kBand8To16kHz = 1, + kBand16To24kHz = 2 +}; + +class AudioBuffer { + public: + // TODO(ajm): Switch to take ChannelLayouts. + AudioBuffer(size_t input_num_frames, + size_t num_input_channels, + size_t process_num_frames, + size_t num_process_channels, + size_t output_num_frames); + virtual ~AudioBuffer(); + + size_t num_channels() const; + void set_num_channels(size_t num_channels); + size_t num_frames() const; + size_t num_frames_per_band() const; + size_t num_keyboard_frames() const; + size_t num_bands() const; + + // Returns a pointer array to the full-band channels. + // Usage: + // channels()[channel][sample]. + // Where: + // 0 <= channel < |num_proc_channels_| + // 0 <= sample < |proc_num_frames_| + int16_t* const* channels(); + const int16_t* const* channels_const() const; + float* const* channels_f(); + const float* const* channels_const_f() const; + + // Returns a pointer array to the bands for a specific channel. + // Usage: + // split_bands(channel)[band][sample]. + // Where: + // 0 <= channel < |num_proc_channels_| + // 0 <= band < |num_bands_| + // 0 <= sample < |num_split_frames_| + int16_t* const* split_bands(size_t channel); + const int16_t* const* split_bands_const(size_t channel) const; + float* const* split_bands_f(size_t channel); + const float* const* split_bands_const_f(size_t channel) const; + + // Returns a pointer array to the channels for a specific band. + // Usage: + // split_channels(band)[channel][sample]. + // Where: + // 0 <= band < |num_bands_| + // 0 <= channel < |num_proc_channels_| + // 0 <= sample < |num_split_frames_| + int16_t* const* split_channels(Band band); + const int16_t* const* split_channels_const(Band band) const; + float* const* split_channels_f(Band band); + const float* const* split_channels_const_f(Band band) const; + + // Returns a pointer to the ChannelBuffer that encapsulates the full-band + // data. + ChannelBuffer* data(); + const ChannelBuffer* data() const; + ChannelBuffer* data_f(); + const ChannelBuffer* data_f() const; + + // Returns a pointer to the ChannelBuffer that encapsulates the split data. + ChannelBuffer* split_data(); + const ChannelBuffer* split_data() const; + ChannelBuffer* split_data_f(); + const ChannelBuffer* split_data_f() const; + + // Returns a pointer to the low-pass data downmixed to mono. If this data + // isn't already available it re-calculates it. + const int16_t* mixed_low_pass_data(); + const int16_t* low_pass_reference(int channel) const; + + const float* keyboard_data() const; + + void set_activity(AudioFrame::VADActivity activity); + AudioFrame::VADActivity activity() const; + + // Use for int16 interleaved data. + void DeinterleaveFrom(AudioFrame* audioFrame); + // If |data_changed| is false, only the non-audio data members will be copied + // to |frame|. + void InterleaveTo(AudioFrame* frame, bool data_changed) const; + + // Use for float deinterleaved data. + void CopyFrom(const float* const* data, const StreamConfig& stream_config); + void CopyTo(const StreamConfig& stream_config, float* const* data); + void CopyLowPassToReference(); + + // Splits the signal into different bands. + void SplitIntoFrequencyBands(); + // Recombine the different bands into one signal. + void MergeFrequencyBands(); + + private: + FRIEND_TEST_ALL_PREFIXES(AudioBufferTest, + SetNumChannelsSetsChannelBuffersNumChannels); + // Called from DeinterleaveFrom() and CopyFrom(). + void InitForNewData(); + + // The audio is passed into DeinterleaveFrom() or CopyFrom() with input + // format (samples per channel and number of channels). + const size_t input_num_frames_; + const size_t num_input_channels_; + // The audio is stored by DeinterleaveFrom() or CopyFrom() with processing + // format. + const size_t proc_num_frames_; + const size_t num_proc_channels_; + // The audio is returned by InterleaveTo() and CopyTo() with output samples + // per channels and the current number of channels. This last one can be + // changed at any time using set_num_channels(). + const size_t output_num_frames_; + size_t num_channels_; + + size_t num_bands_; + size_t num_split_frames_; + bool mixed_low_pass_valid_; + bool reference_copied_; + AudioFrame::VADActivity activity_; + + const float* keyboard_data_; + std::unique_ptr data_; + std::unique_ptr split_data_; + std::unique_ptr splitting_filter_; + std::unique_ptr > mixed_low_pass_channels_; + std::unique_ptr > low_pass_reference_channels_; + std::unique_ptr input_buffer_; + std::unique_ptr output_buffer_; + std::unique_ptr > process_buffer_; + std::vector> input_resamplers_; + std::vector> output_resamplers_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AUDIO_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer_unittest.cc new file mode 100644 index 0000000000..4cbb98eb50 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer_unittest.cc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/audio_buffer.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +const size_t kNumFrames = 480u; +const size_t kStereo = 2u; +const size_t kMono = 1u; + +void ExpectNumChannels(const AudioBuffer& ab, size_t num_channels) { + EXPECT_EQ(ab.data()->num_channels(), num_channels); + EXPECT_EQ(ab.data_f()->num_channels(), num_channels); + EXPECT_EQ(ab.split_data()->num_channels(), num_channels); + EXPECT_EQ(ab.split_data_f()->num_channels(), num_channels); + EXPECT_EQ(ab.num_channels(), num_channels); +} + +} // namespace + +TEST(AudioBufferTest, SetNumChannelsSetsChannelBuffersNumChannels) { + AudioBuffer ab(kNumFrames, kStereo, kNumFrames, kStereo, kNumFrames); + ExpectNumChannels(ab, kStereo); + ab.set_num_channels(kMono); + ExpectNumChannels(ab, kMono); + ab.InitForNewData(); + ExpectNumChannels(ab, kStereo); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(AudioBufferTest, SetNumChannelsDeathTest) { + AudioBuffer ab(kNumFrames, kMono, kNumFrames, kMono, kNumFrames); + EXPECT_DEATH(ab.set_num_channels(kStereo), "num_channels"); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_c_gn/moz.build new file mode 100644 index 0000000000..ae9fa47e0b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_c_gn/moz.build @@ -0,0 +1,317 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/analog_agc.c" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/agc/legacy/digital_agc.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_c.c" + ] + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_c.c" + ] + +if CONFIG["CPU_ARCH"] == "mips64": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "ppc64": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "x86": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "WINNT": + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c", + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c" + ] + +Library("audio_processing_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_gn/moz.build new file mode 100644 index 0000000000..63737c372a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_gn/moz.build @@ -0,0 +1,370 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_APM_DEBUG_DUMP"] = "1" +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_INTELLIGIBILITY_ENHANCER"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/", + "/third_party/libwebrtc/webrtc/modules/audio_coding/codecs/isac/main/include/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec/echo_cancellation.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_c.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/echo_control_mobile.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.cc" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_resampler.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_common.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec3_fft.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/aec_state.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_framer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/block_processor_metrics.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/cascaded_biquad_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/comfort_noise_generator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/decimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/downsampled_render_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_canceller3.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_path_variability.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erl_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/erle_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/frame_blocker.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/main_filter_update_gain.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/output_selector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/shadow_filter_update_gain.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/subtractor.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec3/suppression_gain.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/agc/agc_manager_direct.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/agc/loudness_histogram.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/agc/utility.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/agc2/gain_controller2.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/audio_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/include/config.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_NS_FLOAT"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "m", + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + DEFINES["WEBRTC_NS_FIXED"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + DEFINES["WEBRTC_NS_FIXED"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "ppc64": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["WEBRTC_NS_FLOAT"] = True + +Library("audio_processing_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.cc new file mode 100644 index 0000000000..d61ab97c6e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -0,0 +1,2011 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/audio_processing_impl.h" + +#include +#include +#include + +#include "common_audio/audio_converter.h" +#include "common_audio/channel_buffer.h" +#include "common_audio/include/audio_util.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/aec/aec_core.h" +#include "modules/audio_processing/aec3/echo_canceller3.h" +#include "modules/audio_processing/agc/agc_manager_direct.h" +#include "modules/audio_processing/agc2/gain_controller2.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" +#include "modules/audio_processing/common.h" +#include "modules/audio_processing/echo_cancellation_impl.h" +#include "modules/audio_processing/echo_control_mobile_impl.h" +#include "modules/audio_processing/gain_control_for_experimental_agc.h" +#include "modules/audio_processing/gain_control_impl.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/platform_file.h" +#include "rtc_base/refcountedobject.h" +#include "rtc_base/trace_event.h" +#if WEBRTC_INTELLIGIBILITY_ENHANCER +#include "modules/audio_processing/intelligibility/intelligibility_enhancer.h" +#endif +#include "modules/audio_processing/level_controller/level_controller.h" +#include "modules/audio_processing/level_estimator_impl.h" +#include "modules/audio_processing/low_cut_filter.h" +#include "modules/audio_processing/noise_suppression_impl.h" +#include "modules/audio_processing/residual_echo_detector.h" +#include "modules/audio_processing/transient/transient_suppressor.h" +#include "modules/audio_processing/voice_detection_impl.h" +#include "modules/include/module_common_types.h" +#include "system_wrappers/include/file_wrapper.h" +#include "system_wrappers/include/metrics.h" + +// Check to verify that the define for the intelligibility enhancer is properly +// set. +#if !defined(WEBRTC_INTELLIGIBILITY_ENHANCER) || \ + (WEBRTC_INTELLIGIBILITY_ENHANCER != 0 && \ + WEBRTC_INTELLIGIBILITY_ENHANCER != 1) +#error "Set WEBRTC_INTELLIGIBILITY_ENHANCER to either 0 or 1" +#endif + +#define RETURN_ON_ERR(expr) \ + do { \ + int err = (expr); \ + if (err != kNoError) { \ + return err; \ + } \ + } while (0) + +namespace webrtc { + +constexpr int AudioProcessing::kNativeSampleRatesHz[]; + +namespace { + +static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { + switch (layout) { + case AudioProcessing::kMono: + case AudioProcessing::kStereo: + return false; + case AudioProcessing::kMonoAndKeyboard: + case AudioProcessing::kStereoAndKeyboard: + return true; + } + + RTC_NOTREACHED(); + return false; +} + +bool SampleRateSupportsMultiBand(int sample_rate_hz) { + return sample_rate_hz == AudioProcessing::kSampleRate32kHz || + sample_rate_hz == AudioProcessing::kSampleRate48kHz; +} + +int FindNativeProcessRateToUse(int minimum_rate, bool band_splitting_required) { +#ifdef WEBRTC_ARCH_ARM_FAMILY + constexpr int kMaxSplittingNativeProcessRate = + AudioProcessing::kSampleRate32kHz; +#else + constexpr int kMaxSplittingNativeProcessRate = + AudioProcessing::kSampleRate48kHz; +#endif + static_assert( + kMaxSplittingNativeProcessRate <= AudioProcessing::kMaxNativeSampleRateHz, + ""); + const int uppermost_native_rate = band_splitting_required + ? kMaxSplittingNativeProcessRate + : AudioProcessing::kSampleRate48kHz; + + for (auto rate : AudioProcessing::kNativeSampleRatesHz) { + if (rate >= uppermost_native_rate) { + return uppermost_native_rate; + } + if (rate >= minimum_rate) { + return rate; + } + } + RTC_NOTREACHED(); + return uppermost_native_rate; +} + +// Maximum lengths that frame of samples being passed from the render side to +// the capture side can have (does not apply to AEC3). +static const size_t kMaxAllowedValuesOfSamplesPerBand = 160; +static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480; + +// Maximum number of frames to buffer in the render queue. +// TODO(peah): Decrease this once we properly handle hugely unbalanced +// reverse and forward call numbers. +static const size_t kMaxNumFramesToBuffer = 100; + +class HighPassFilterImpl : public HighPassFilter { + public: + explicit HighPassFilterImpl(AudioProcessingImpl* apm) : apm_(apm) {} + ~HighPassFilterImpl() override = default; + + // HighPassFilter implementation. + int Enable(bool enable) override { + apm_->MutateConfig([enable](AudioProcessing::Config* config) { + config->high_pass_filter.enabled = enable; + }); + + return AudioProcessing::kNoError; + } + + bool is_enabled() const override { + return apm_->GetConfig().high_pass_filter.enabled; + } + + private: + AudioProcessingImpl* apm_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(HighPassFilterImpl); +}; + +webrtc::InternalAPMStreamsConfig ToStreamsConfig( + const ProcessingConfig& api_format) { + webrtc::InternalAPMStreamsConfig result; + result.input_sample_rate = api_format.input_stream().sample_rate_hz(); + result.input_num_channels = api_format.input_stream().num_channels(); + result.output_num_channels = api_format.output_stream().num_channels(); + result.render_input_num_channels = + api_format.reverse_input_stream().num_channels(); + result.render_input_sample_rate = + api_format.reverse_input_stream().sample_rate_hz(); + result.output_sample_rate = api_format.output_stream().sample_rate_hz(); + result.render_output_sample_rate = + api_format.reverse_output_stream().sample_rate_hz(); + result.render_output_num_channels = + api_format.reverse_output_stream().num_channels(); + return result; +} +} // namespace + +// Throughout webrtc, it's assumed that success is represented by zero. +static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); + +AudioProcessingImpl::ApmSubmoduleStates::ApmSubmoduleStates( + bool capture_post_processor_enabled) + : capture_post_processor_enabled_(capture_post_processor_enabled) {} + +bool AudioProcessingImpl::ApmSubmoduleStates::Update( + bool low_cut_filter_enabled, + bool echo_canceller_enabled, + bool mobile_echo_controller_enabled, + bool residual_echo_detector_enabled, + bool noise_suppressor_enabled, + bool intelligibility_enhancer_enabled, + bool beamformer_enabled, + bool adaptive_gain_controller_enabled, + bool gain_controller2_enabled, + bool level_controller_enabled, + bool echo_controller_enabled, + bool voice_activity_detector_enabled, + bool level_estimator_enabled, + bool transient_suppressor_enabled) { + bool changed = false; + changed |= (low_cut_filter_enabled != low_cut_filter_enabled_); + changed |= (echo_canceller_enabled != echo_canceller_enabled_); + changed |= + (mobile_echo_controller_enabled != mobile_echo_controller_enabled_); + changed |= + (residual_echo_detector_enabled != residual_echo_detector_enabled_); + changed |= (noise_suppressor_enabled != noise_suppressor_enabled_); + changed |= + (intelligibility_enhancer_enabled != intelligibility_enhancer_enabled_); + changed |= (beamformer_enabled != beamformer_enabled_); + changed |= + (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_); + changed |= + (gain_controller2_enabled != gain_controller2_enabled_); + changed |= (level_controller_enabled != level_controller_enabled_); + changed |= (echo_controller_enabled != echo_controller_enabled_); + changed |= (level_estimator_enabled != level_estimator_enabled_); + changed |= + (voice_activity_detector_enabled != voice_activity_detector_enabled_); + changed |= (transient_suppressor_enabled != transient_suppressor_enabled_); + if (changed) { + low_cut_filter_enabled_ = low_cut_filter_enabled; + echo_canceller_enabled_ = echo_canceller_enabled; + mobile_echo_controller_enabled_ = mobile_echo_controller_enabled; + residual_echo_detector_enabled_ = residual_echo_detector_enabled; + noise_suppressor_enabled_ = noise_suppressor_enabled; + intelligibility_enhancer_enabled_ = intelligibility_enhancer_enabled; + beamformer_enabled_ = beamformer_enabled; + adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled; + gain_controller2_enabled_ = gain_controller2_enabled; + level_controller_enabled_ = level_controller_enabled; + echo_controller_enabled_ = echo_controller_enabled; + level_estimator_enabled_ = level_estimator_enabled; + voice_activity_detector_enabled_ = voice_activity_detector_enabled; + transient_suppressor_enabled_ = transient_suppressor_enabled; + } + + changed |= first_update_; + first_update_ = false; + return changed; +} + +bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandSubModulesActive() + const { +#if WEBRTC_INTELLIGIBILITY_ENHANCER + return CaptureMultiBandProcessingActive() || + intelligibility_enhancer_enabled_ || voice_activity_detector_enabled_; +#else + return CaptureMultiBandProcessingActive() || voice_activity_detector_enabled_; +#endif +} + +bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandProcessingActive() + const { + return low_cut_filter_enabled_ || echo_canceller_enabled_ || + mobile_echo_controller_enabled_ || noise_suppressor_enabled_ || + beamformer_enabled_ || adaptive_gain_controller_enabled_ || + echo_controller_enabled_; +} + +bool AudioProcessingImpl::ApmSubmoduleStates::CaptureFullBandProcessingActive() + const { + return level_controller_enabled_ || gain_controller2_enabled_ || + capture_post_processor_enabled_; +} + +bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive() + const { + return RenderMultiBandProcessingActive() || echo_canceller_enabled_ || + mobile_echo_controller_enabled_ || adaptive_gain_controller_enabled_ || + echo_controller_enabled_; +} + +bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandProcessingActive() + const { +#if WEBRTC_INTELLIGIBILITY_ENHANCER + return intelligibility_enhancer_enabled_; +#else + return false; +#endif +} + +struct AudioProcessingImpl::ApmPublicSubmodules { + ApmPublicSubmodules() {} + // Accessed externally of APM without any lock acquired. + std::unique_ptr echo_cancellation; + std::unique_ptr echo_control_mobile; + std::unique_ptr gain_control; + std::unique_ptr level_estimator; + std::unique_ptr noise_suppression; + std::unique_ptr voice_detection; + std::unique_ptr + gain_control_for_experimental_agc; + + // Accessed internally from both render and capture. + std::unique_ptr transient_suppressor; +#if WEBRTC_INTELLIGIBILITY_ENHANCER + std::unique_ptr intelligibility_enhancer; +#endif +}; + +struct AudioProcessingImpl::ApmPrivateSubmodules { + ApmPrivateSubmodules(NonlinearBeamformer* beamformer, + std::unique_ptr capture_post_processor) + : beamformer(beamformer), + capture_post_processor(std::move(capture_post_processor)) {} + // Accessed internally from capture or during initialization + std::unique_ptr beamformer; + std::unique_ptr agc_manager; + std::unique_ptr gain_controller2; + std::unique_ptr low_cut_filter; + std::unique_ptr level_controller; + std::unique_ptr residual_echo_detector; + std::unique_ptr echo_controller; + std::unique_ptr capture_post_processor; +}; + +AudioProcessing* AudioProcessing::Create() { + webrtc::Config config; + return Create(config, nullptr, nullptr, nullptr); +} + +AudioProcessing* AudioProcessing::Create(const webrtc::Config& config) { + return Create(config, nullptr, nullptr, nullptr); +} + +AudioProcessing* AudioProcessing::Create(const webrtc::Config& config, + NonlinearBeamformer* beamformer) { + return Create(config, nullptr, nullptr, beamformer); +} + +AudioProcessing* AudioProcessing::Create( + const webrtc::Config& config, + std::unique_ptr capture_post_processor, + std::unique_ptr echo_control_factory, + NonlinearBeamformer* beamformer) { + AudioProcessingImpl* apm = new rtc::RefCountedObject( + config, std::move(capture_post_processor), + std::move(echo_control_factory), beamformer); + if (apm->Initialize() != kNoError) { + delete apm; + apm = nullptr; + } + + return apm; +} + +AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config) + : AudioProcessingImpl(config, nullptr, nullptr, nullptr) {} + +AudioProcessingImpl::AudioProcessingImpl( + const webrtc::Config& config, + std::unique_ptr capture_post_processor, + std::unique_ptr echo_control_factory, + NonlinearBeamformer* beamformer) + : high_pass_filter_impl_(new HighPassFilterImpl(this)), + echo_control_factory_(std::move(echo_control_factory)), + submodule_states_(!!capture_post_processor), + public_submodules_(new ApmPublicSubmodules()), + private_submodules_( + new ApmPrivateSubmodules(beamformer, + std::move(capture_post_processor))), + constants_(config.Get().startup_min_volume, + config.Get().clipped_level_min, +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) + false), +#else + config.Get().enabled), +#endif +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) + capture_(false, +#else + capture_(config.Get().enabled, +#endif + config.Get().array_geometry, + config.Get().target_direction), + capture_nonlocked_(config.Get().enabled, + config.Get().enabled) { + { + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + + // Mark Echo Controller enabled if a factory is injected. + capture_nonlocked_.echo_controller_enabled = + static_cast(echo_control_factory_); + + public_submodules_->echo_cancellation.reset( + new EchoCancellationImpl(&crit_render_, &crit_capture_)); + public_submodules_->echo_control_mobile.reset( + new EchoControlMobileImpl(&crit_render_, &crit_capture_)); + public_submodules_->gain_control.reset( + new GainControlImpl(&crit_capture_, &crit_capture_)); + public_submodules_->level_estimator.reset( + new LevelEstimatorImpl(&crit_capture_)); + public_submodules_->noise_suppression.reset( + new NoiseSuppressionImpl(&crit_capture_)); + public_submodules_->voice_detection.reset( + new VoiceDetectionImpl(&crit_capture_)); + public_submodules_->gain_control_for_experimental_agc.reset( + new GainControlForExperimentalAgc( + public_submodules_->gain_control.get(), &crit_capture_)); + private_submodules_->residual_echo_detector.reset( + new ResidualEchoDetector()); + + // TODO(peah): Move this creation to happen only when the level controller + // is enabled. + private_submodules_->level_controller.reset(new LevelController()); + + // TODO(alessiob): Move the injected gain controller once injection is + // implemented. + private_submodules_->gain_controller2.reset(new GainController2()); + + RTC_LOG(LS_INFO) << "Capture post processor activated: " + << !!private_submodules_->capture_post_processor; + } + + SetExtraOptions(config); +} + +AudioProcessingImpl::~AudioProcessingImpl() { + // Depends on gain_control_ and + // public_submodules_->gain_control_for_experimental_agc. + private_submodules_->agc_manager.reset(); + // Depends on gain_control_. + public_submodules_->gain_control_for_experimental_agc.reset(); +} + +int AudioProcessingImpl::Initialize() { + // Run in a single-threaded manner during initialization. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + return InitializeLocked(); +} + +int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz, + int capture_output_sample_rate_hz, + int render_input_sample_rate_hz, + ChannelLayout capture_input_layout, + ChannelLayout capture_output_layout, + ChannelLayout render_input_layout) { + const ProcessingConfig processing_config = { + {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout), + LayoutHasKeyboard(capture_input_layout)}, + {capture_output_sample_rate_hz, + ChannelsFromLayout(capture_output_layout), + LayoutHasKeyboard(capture_output_layout)}, + {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), + LayoutHasKeyboard(render_input_layout)}, + {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), + LayoutHasKeyboard(render_input_layout)}}}; + + return Initialize(processing_config); +} + +int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { + // Run in a single-threaded manner during initialization. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + return InitializeLocked(processing_config); +} + +int AudioProcessingImpl::MaybeInitializeRender( + const ProcessingConfig& processing_config) { + return MaybeInitialize(processing_config, false); +} + +int AudioProcessingImpl::MaybeInitializeCapture( + const ProcessingConfig& processing_config, + bool force_initialization) { + return MaybeInitialize(processing_config, force_initialization); +} + +// Calls InitializeLocked() if any of the audio parameters have changed from +// their current values (needs to be called while holding the crit_render_lock). +int AudioProcessingImpl::MaybeInitialize( + const ProcessingConfig& processing_config, + bool force_initialization) { + // Called from both threads. Thread check is therefore not possible. + if (processing_config == formats_.api_format && !force_initialization) { + return kNoError; + } + + rtc::CritScope cs_capture(&crit_capture_); + return InitializeLocked(processing_config); +} + +int AudioProcessingImpl::InitializeLocked() { + UpdateActiveSubmoduleStates(); + + const int capture_audiobuffer_num_channels = + capture_nonlocked_.beamformer_enabled + ? formats_.api_format.input_stream().num_channels() + : formats_.api_format.output_stream().num_channels(); + + const int render_audiobuffer_num_output_frames = + formats_.api_format.reverse_output_stream().num_frames() == 0 + ? formats_.render_processing_format.num_frames() + : formats_.api_format.reverse_output_stream().num_frames(); + if (formats_.api_format.reverse_input_stream().num_channels() > 0) { + render_.render_audio.reset(new AudioBuffer( + formats_.api_format.reverse_input_stream().num_frames(), + formats_.api_format.reverse_input_stream().num_channels(), + formats_.render_processing_format.num_frames(), + formats_.render_processing_format.num_channels(), + render_audiobuffer_num_output_frames)); + if (formats_.api_format.reverse_input_stream() != + formats_.api_format.reverse_output_stream()) { + render_.render_converter = AudioConverter::Create( + formats_.api_format.reverse_input_stream().num_channels(), + formats_.api_format.reverse_input_stream().num_frames(), + formats_.api_format.reverse_output_stream().num_channels(), + formats_.api_format.reverse_output_stream().num_frames()); + } else { + render_.render_converter.reset(nullptr); + } + } else { + render_.render_audio.reset(nullptr); + render_.render_converter.reset(nullptr); + } + + capture_.capture_audio.reset( + new AudioBuffer(formats_.api_format.input_stream().num_frames(), + formats_.api_format.input_stream().num_channels(), + capture_nonlocked_.capture_processing_format.num_frames(), + capture_audiobuffer_num_channels, + formats_.api_format.output_stream().num_frames())); + + public_submodules_->echo_cancellation->Initialize( + proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), + num_proc_channels()); + AllocateRenderQueue(); + + int success = public_submodules_->echo_cancellation->enable_metrics(true); + RTC_DCHECK_EQ(0, success); + success = public_submodules_->echo_cancellation->enable_delay_logging(true); + RTC_DCHECK_EQ(0, success); + public_submodules_->echo_control_mobile->Initialize( + proc_split_sample_rate_hz(), num_reverse_channels(), + num_output_channels()); + + public_submodules_->gain_control->Initialize(num_proc_channels(), + proc_sample_rate_hz()); + if (constants_.use_experimental_agc) { + if (!private_submodules_->agc_manager.get()) { + private_submodules_->agc_manager.reset(new AgcManagerDirect( + public_submodules_->gain_control.get(), + public_submodules_->gain_control_for_experimental_agc.get(), + constants_.agc_startup_min_volume, constants_.agc_clipped_level_min)); + } + private_submodules_->agc_manager->Initialize(); + private_submodules_->agc_manager->SetCaptureMuted( + capture_.output_will_be_muted); + public_submodules_->gain_control_for_experimental_agc->Initialize(); + } + InitializeTransient(); + InitializeBeamformer(); +#if WEBRTC_INTELLIGIBILITY_ENHANCER + InitializeIntelligibility(); +#endif + InitializeLowCutFilter(); + public_submodules_->noise_suppression->Initialize(num_proc_channels(), + proc_sample_rate_hz()); + public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); + public_submodules_->level_estimator->Initialize(); + InitializeLevelController(); + InitializeResidualEchoDetector(); + InitializeEchoController(); + InitializeGainController2(); + InitializePostProcessor(); + + if (aec_dump_) { + aec_dump_->WriteInitMessage(ToStreamsConfig(formats_.api_format)); + } + return kNoError; +} + +int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { + UpdateActiveSubmoduleStates(); + + for (const auto& stream : config.streams) { + if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { + return kBadSampleRateError; + } + } + + const size_t num_in_channels = config.input_stream().num_channels(); + const size_t num_out_channels = config.output_stream().num_channels(); + + // Need at least one input channel. + // Need either one output channel or as many outputs as there are inputs. + if (num_in_channels == 0 || + !(num_out_channels == 1 || num_out_channels == num_in_channels)) { + return kBadNumberChannelsError; + } + + if (capture_nonlocked_.beamformer_enabled && + num_in_channels != capture_.array_geometry.size()) { + return kBadNumberChannelsError; + } + + formats_.api_format = config; + + int capture_processing_rate = FindNativeProcessRateToUse( + std::min(formats_.api_format.input_stream().sample_rate_hz(), + formats_.api_format.output_stream().sample_rate_hz()), + submodule_states_.CaptureMultiBandSubModulesActive() || + submodule_states_.RenderMultiBandSubModulesActive()); + + capture_nonlocked_.capture_processing_format = + StreamConfig(capture_processing_rate); + + int render_processing_rate; + if (!capture_nonlocked_.echo_controller_enabled) { + render_processing_rate = FindNativeProcessRateToUse( + std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), + formats_.api_format.reverse_output_stream().sample_rate_hz()), + submodule_states_.CaptureMultiBandSubModulesActive() || + submodule_states_.RenderMultiBandSubModulesActive()); + } else { + render_processing_rate = capture_processing_rate; + } + + // TODO(aluebs): Remove this restriction once we figure out why the 3-band + // splitting filter degrades the AEC performance. + if (render_processing_rate > kSampleRate32kHz && + !capture_nonlocked_.echo_controller_enabled) { + render_processing_rate = submodule_states_.RenderMultiBandProcessingActive() + ? kSampleRate32kHz + : kSampleRate16kHz; + } + + // If the forward sample rate is 8 kHz, the render stream is also processed + // at this rate. + if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == + kSampleRate8kHz) { + render_processing_rate = kSampleRate8kHz; + } else { + render_processing_rate = + std::max(render_processing_rate, static_cast(kSampleRate16kHz)); + } + + // Always downmix the render stream to mono for analysis. This has been + // demonstrated to work well for AEC in most practical scenarios. + if (submodule_states_.RenderMultiBandSubModulesActive()) { + formats_.render_processing_format = StreamConfig(render_processing_rate, 1); + } else { + formats_.render_processing_format = StreamConfig( + formats_.api_format.reverse_input_stream().sample_rate_hz(), + formats_.api_format.reverse_input_stream().num_channels()); + } + + if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == + kSampleRate32kHz || + capture_nonlocked_.capture_processing_format.sample_rate_hz() == + kSampleRate48kHz) { + capture_nonlocked_.split_rate = kSampleRate16kHz; + } else { + capture_nonlocked_.split_rate = + capture_nonlocked_.capture_processing_format.sample_rate_hz(); + } + + return InitializeLocked(); +} + +void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) { + config_ = config; + + bool config_ok = LevelController::Validate(config_.level_controller); + if (!config_ok) { + RTC_LOG(LS_ERROR) << "AudioProcessing module config error" << std::endl + << "level_controller: " + << LevelController::ToString(config_.level_controller) + << std::endl + << "Reverting to default parameter set"; + config_.level_controller = AudioProcessing::Config::LevelController(); + } + + // Run in a single-threaded manner when applying the settings. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + + // TODO(peah): Replace the use of capture_nonlocked_.level_controller_enabled + // with the value in config_ everywhere in the code. + if (capture_nonlocked_.level_controller_enabled != + config_.level_controller.enabled) { + capture_nonlocked_.level_controller_enabled = + config_.level_controller.enabled; + // TODO(peah): Remove the conditional initialization to always initialize + // the level controller regardless of whether it is enabled or not. + InitializeLevelController(); + } + RTC_LOG(LS_INFO) << "Level controller activated: " + << capture_nonlocked_.level_controller_enabled; + + private_submodules_->level_controller->ApplyConfig(config_.level_controller); + + InitializeLowCutFilter(); + + RTC_LOG(LS_INFO) << "Highpass filter activated: " + << config_.high_pass_filter.enabled; + + // Deprecated way of activating AEC3. + // TODO(gustaf): Remove when possible. + if (config.echo_canceller3.enabled && !echo_control_factory_) { + capture_nonlocked_.echo_controller_enabled = + config_.echo_canceller3.enabled; + echo_control_factory_ = + std::unique_ptr(new EchoCanceller3Factory()); + InitializeEchoController(); + RTC_LOG(LS_INFO) << "Echo canceller 3 activated: " + << capture_nonlocked_.echo_controller_enabled; + } + + config_ok = GainController2::Validate(config_.gain_controller2); + if (!config_ok) { + RTC_LOG(LS_ERROR) << "AudioProcessing module config error" << std::endl + << "Gain Controller 2: " + << GainController2::ToString(config_.gain_controller2) + << std::endl + << "Reverting to default parameter set"; + config_.gain_controller2 = AudioProcessing::Config::GainController2(); + } + InitializeGainController2(); + private_submodules_->gain_controller2->ApplyConfig(config_.gain_controller2); + RTC_LOG(LS_INFO) << "Gain Controller 2 activated: " + << config_.gain_controller2.enabled; +} + +void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) { + // Run in a single-threaded manner when setting the extra options. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + + public_submodules_->echo_cancellation->SetExtraOptions(config); + + if (capture_.transient_suppressor_enabled != + config.Get().enabled) { + capture_.transient_suppressor_enabled = + config.Get().enabled; + InitializeTransient(); + } + +#if WEBRTC_INTELLIGIBILITY_ENHANCER + if (capture_nonlocked_.intelligibility_enabled != + config.Get().enabled) { + capture_nonlocked_.intelligibility_enabled = + config.Get().enabled; + InitializeIntelligibility(); + } +#endif + +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD + if (capture_nonlocked_.beamformer_enabled != + config.Get().enabled) { + capture_nonlocked_.beamformer_enabled = config.Get().enabled; + if (config.Get().array_geometry.size() > 1) { + capture_.array_geometry = config.Get().array_geometry; + } + capture_.target_direction = config.Get().target_direction; + InitializeBeamformer(); + } +#endif // WEBRTC_ANDROID_PLATFORM_BUILD +} + +int AudioProcessingImpl::proc_sample_rate_hz() const { + // Used as callback from submodules, hence locking is not allowed. + return capture_nonlocked_.capture_processing_format.sample_rate_hz(); +} + +int AudioProcessingImpl::proc_split_sample_rate_hz() const { + // Used as callback from submodules, hence locking is not allowed. + return capture_nonlocked_.split_rate; +} + +size_t AudioProcessingImpl::num_reverse_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return formats_.render_processing_format.num_channels(); +} + +size_t AudioProcessingImpl::num_input_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return formats_.api_format.input_stream().num_channels(); +} + +size_t AudioProcessingImpl::num_proc_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return (capture_nonlocked_.beamformer_enabled || + capture_nonlocked_.echo_controller_enabled) + ? 1 + : num_output_channels(); +} + +size_t AudioProcessingImpl::num_output_channels() const { + // Used as callback from submodules, hence locking is not allowed. + return formats_.api_format.output_stream().num_channels(); +} + +void AudioProcessingImpl::set_output_will_be_muted(bool muted) { + rtc::CritScope cs(&crit_capture_); + capture_.output_will_be_muted = muted; + if (private_submodules_->agc_manager.get()) { + private_submodules_->agc_manager->SetCaptureMuted( + capture_.output_will_be_muted); + } +} + + +int AudioProcessingImpl::ProcessStream(const float* const* src, + size_t samples_per_channel, + int input_sample_rate_hz, + ChannelLayout input_layout, + int output_sample_rate_hz, + ChannelLayout output_layout, + float* const* dest) { + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_ChannelLayout"); + StreamConfig input_stream; + StreamConfig output_stream; + { + // Access the formats_.api_format.input_stream beneath the capture lock. + // The lock must be released as it is later required in the call + // to ProcessStream(,,,); + rtc::CritScope cs(&crit_capture_); + input_stream = formats_.api_format.input_stream(); + output_stream = formats_.api_format.output_stream(); + } + + input_stream.set_sample_rate_hz(input_sample_rate_hz); + input_stream.set_num_channels(ChannelsFromLayout(input_layout)); + input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); + output_stream.set_sample_rate_hz(output_sample_rate_hz); + output_stream.set_num_channels(ChannelsFromLayout(output_layout)); + output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); + + if (samples_per_channel != input_stream.num_frames()) { + return kBadDataLengthError; + } + return ProcessStream(src, input_stream, output_stream, dest); +} + +int AudioProcessingImpl::ProcessStream(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest) { + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig"); + ProcessingConfig processing_config; + bool reinitialization_required = false; + { + // Acquire the capture lock in order to safely call the function + // that retrieves the render side data. This function accesses apm + // getters that need the capture lock held when being called. + rtc::CritScope cs_capture(&crit_capture_); + EmptyQueuedRenderAudio(); + + if (!src || !dest) { + return kNullPointerError; + } + + processing_config = formats_.api_format; + reinitialization_required = UpdateActiveSubmoduleStates(); + } + + processing_config.input_stream() = input_config; + processing_config.output_stream() = output_config; + + { + // Do conditional reinitialization. + rtc::CritScope cs_render(&crit_render_); + RETURN_ON_ERR( + MaybeInitializeCapture(processing_config, reinitialization_required)); + } + rtc::CritScope cs_capture(&crit_capture_); + RTC_DCHECK_EQ(processing_config.input_stream().num_frames(), + formats_.api_format.input_stream().num_frames()); + + if (aec_dump_) { + RecordUnprocessedCaptureStream(src); + } + + capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); + RETURN_ON_ERR(ProcessCaptureStreamLocked()); + capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); + + if (aec_dump_) { + RecordProcessedCaptureStream(dest); + } + return kNoError; +} + +void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) { + EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), + num_reverse_channels(), + &aec_render_queue_buffer_); + + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + + // Insert the samples into the queue. + if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_); + RTC_DCHECK(result); + } + + EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(), + num_reverse_channels(), + &aecm_render_queue_buffer_); + + // Insert the samples into the queue. + if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_); + RTC_DCHECK(result); + } + + if (!constants_.use_experimental_agc) { + GainControlImpl::PackRenderAudioBuffer(audio, &agc_render_queue_buffer_); + // Insert the samples into the queue. + if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_); + RTC_DCHECK(result); + } + } +} + +void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) { + ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_); + + // Insert the samples into the queue. + if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_); + RTC_DCHECK(result); + } +} + +void AudioProcessingImpl::AllocateRenderQueue() { + const size_t new_aec_render_queue_element_max_size = + std::max(static_cast(1), + kMaxAllowedValuesOfSamplesPerBand * + EchoCancellationImpl::NumCancellersRequired( + num_output_channels(), num_reverse_channels())); + + const size_t new_aecm_render_queue_element_max_size = + std::max(static_cast(1), + kMaxAllowedValuesOfSamplesPerBand * + EchoControlMobileImpl::NumCancellersRequired( + num_output_channels(), num_reverse_channels())); + + const size_t new_agc_render_queue_element_max_size = + std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerBand); + + const size_t new_red_render_queue_element_max_size = + std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame); + + // Reallocate the queues if the queue item sizes are too small to fit the + // data to put in the queues. + if (aec_render_queue_element_max_size_ < + new_aec_render_queue_element_max_size) { + aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size; + + std::vector template_queue_element( + aec_render_queue_element_max_size_); + + aec_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + aec_render_queue_element_max_size_))); + + aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_); + aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_); + } else { + aec_render_signal_queue_->Clear(); + } + + if (aecm_render_queue_element_max_size_ < + new_aecm_render_queue_element_max_size) { + aecm_render_queue_element_max_size_ = + new_aecm_render_queue_element_max_size; + + std::vector template_queue_element( + aecm_render_queue_element_max_size_); + + aecm_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + aecm_render_queue_element_max_size_))); + + aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_); + aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_); + } else { + aecm_render_signal_queue_->Clear(); + } + + if (agc_render_queue_element_max_size_ < + new_agc_render_queue_element_max_size) { + agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size; + + std::vector template_queue_element( + agc_render_queue_element_max_size_); + + agc_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + agc_render_queue_element_max_size_))); + + agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_); + agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_); + } else { + agc_render_signal_queue_->Clear(); + } + + if (red_render_queue_element_max_size_ < + new_red_render_queue_element_max_size) { + red_render_queue_element_max_size_ = new_red_render_queue_element_max_size; + + std::vector template_queue_element( + red_render_queue_element_max_size_); + + red_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + red_render_queue_element_max_size_))); + + red_render_queue_buffer_.resize(red_render_queue_element_max_size_); + red_capture_queue_buffer_.resize(red_render_queue_element_max_size_); + } else { + red_render_signal_queue_->Clear(); + } +} + +void AudioProcessingImpl::EmptyQueuedRenderAudio() { + rtc::CritScope cs_capture(&crit_capture_); + while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) { + public_submodules_->echo_cancellation->ProcessRenderAudio( + aec_capture_queue_buffer_); + } + + while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) { + public_submodules_->echo_control_mobile->ProcessRenderAudio( + aecm_capture_queue_buffer_); + } + + while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) { + public_submodules_->gain_control->ProcessRenderAudio( + agc_capture_queue_buffer_); + } + + while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) { + private_submodules_->residual_echo_detector->AnalyzeRenderAudio( + red_capture_queue_buffer_); + } +} + +int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame"); + { + // Acquire the capture lock in order to safely call the function + // that retrieves the render side data. This function accesses apm + // getters that need the capture lock held when being called. + // The lock needs to be released as + // public_submodules_->echo_control_mobile->is_enabled() aquires this lock + // as well. + rtc::CritScope cs_capture(&crit_capture_); + EmptyQueuedRenderAudio(); + } + + if (!frame) { + return kNullPointerError; + } + // Must be a native rate. + if (frame->sample_rate_hz_ != kSampleRate8kHz && + frame->sample_rate_hz_ != kSampleRate16kHz && + frame->sample_rate_hz_ != kSampleRate32kHz && + frame->sample_rate_hz_ != kSampleRate48kHz) { + return kBadSampleRateError; + } + + ProcessingConfig processing_config; + bool reinitialization_required = false; + { + // Aquire lock for the access of api_format. + // The lock is released immediately due to the conditional + // reinitialization. + rtc::CritScope cs_capture(&crit_capture_); + // TODO(ajm): The input and output rates and channels are currently + // constrained to be identical in the int16 interface. + processing_config = formats_.api_format; + + reinitialization_required = UpdateActiveSubmoduleStates(); + } + processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); + processing_config.input_stream().set_num_channels(frame->num_channels_); + processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); + processing_config.output_stream().set_num_channels(frame->num_channels_); + + { + // Do conditional reinitialization. + rtc::CritScope cs_render(&crit_render_); + RETURN_ON_ERR( + MaybeInitializeCapture(processing_config, reinitialization_required)); + } + rtc::CritScope cs_capture(&crit_capture_); + if (frame->samples_per_channel_ != + formats_.api_format.input_stream().num_frames()) { + return kBadDataLengthError; + } + + if (aec_dump_) { + RecordUnprocessedCaptureStream(*frame); + } + + capture_.capture_audio->DeinterleaveFrom(frame); + RETURN_ON_ERR(ProcessCaptureStreamLocked()); + capture_.capture_audio->InterleaveTo( + frame, submodule_states_.CaptureMultiBandProcessingActive() || + submodule_states_.CaptureFullBandProcessingActive()); + + if (aec_dump_) { + RecordProcessedCaptureStream(*frame); + } + + return kNoError; +} + +int AudioProcessingImpl::ProcessCaptureStreamLocked() { + // Ensure that not both the AEC and AECM are active at the same time. + // TODO(peah): Simplify once the public API Enable functions for these + // are moved to APM. + RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() && + public_submodules_->echo_control_mobile->is_enabled())); + + MaybeUpdateHistograms(); + + AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity. + + capture_input_rms_.Analyze(rtc::ArrayView( + capture_buffer->channels_const()[0], + capture_nonlocked_.capture_processing_format.num_frames())); + const bool log_rms = ++capture_rms_interval_counter_ >= 1000; + if (log_rms) { + capture_rms_interval_counter_ = 0; + RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak(); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms", + levels.average, 1, RmsLevel::kMinLevelDb, 64); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms", + levels.peak, 1, RmsLevel::kMinLevelDb, 64); + } + + if (private_submodules_->echo_controller) { + // TODO(peah): Reactivate analogue AGC gain detection once the analogue AGC + // issues have been addressed. + capture_.echo_path_gain_change = false; + private_submodules_->echo_controller->AnalyzeCapture(capture_buffer); + } + + if (constants_.use_experimental_agc && + public_submodules_->gain_control->is_enabled()) { + private_submodules_->agc_manager->AnalyzePreProcess( + capture_buffer->channels()[0], capture_buffer->num_channels(), + capture_nonlocked_.capture_processing_format.num_frames()); + } + + if (submodule_states_.CaptureMultiBandSubModulesActive() && + SampleRateSupportsMultiBand( + capture_nonlocked_.capture_processing_format.sample_rate_hz())) { + capture_buffer->SplitIntoFrequencyBands(); + } + + if (private_submodules_->echo_controller) { + // Force down-mixing of the number of channels after the detection of + // capture signal saturation. + // TODO(peah): Look into ensuring that this kind of tampering with the + // AudioBuffer functionality should not be needed. + capture_buffer->set_num_channels(1); + } + + if (capture_nonlocked_.beamformer_enabled) { + private_submodules_->beamformer->AnalyzeChunk( + *capture_buffer->split_data_f()); + // Discards all channels by the leftmost one. + capture_buffer->set_num_channels(1); + } + + // TODO(peah): Move the AEC3 low-cut filter to this place. + if (private_submodules_->low_cut_filter && + !private_submodules_->echo_controller) { + private_submodules_->low_cut_filter->Process(capture_buffer); + } + RETURN_ON_ERR( + public_submodules_->gain_control->AnalyzeCaptureAudio(capture_buffer)); + public_submodules_->noise_suppression->AnalyzeCaptureAudio(capture_buffer); + + // Ensure that the stream delay was set before the call to the + // AEC ProcessCaptureAudio function. + if (public_submodules_->echo_cancellation->is_enabled() && + !was_stream_delay_set()) { + return AudioProcessing::kStreamParameterNotSetError; + } + + if (private_submodules_->echo_controller) { + private_submodules_->echo_controller->ProcessCapture( + capture_buffer, capture_.echo_path_gain_change); + } else { + RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio( + capture_buffer, stream_delay_ms())); + } + + if (public_submodules_->echo_control_mobile->is_enabled() && + public_submodules_->noise_suppression->is_enabled()) { + capture_buffer->CopyLowPassToReference(); + } + public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer); +#if WEBRTC_INTELLIGIBILITY_ENHANCER + if (capture_nonlocked_.intelligibility_enabled) { + RTC_DCHECK(public_submodules_->noise_suppression->is_enabled()); + int gain_db = public_submodules_->gain_control->is_enabled() ? + public_submodules_->gain_control->compression_gain_db() : + 0; + float gain = std::pow(10.f, gain_db / 20.f); + gain *= capture_nonlocked_.level_controller_enabled ? + private_submodules_->level_controller->GetLastGain() : + 1.f; + public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate( + public_submodules_->noise_suppression->NoiseEstimate(), gain); + } +#endif + + // Ensure that the stream delay was set before the call to the + // AECM ProcessCaptureAudio function. + if (public_submodules_->echo_control_mobile->is_enabled() && + !was_stream_delay_set()) { + return AudioProcessing::kStreamParameterNotSetError; + } + + if (!(private_submodules_->echo_controller || + public_submodules_->echo_cancellation->is_enabled())) { + RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( + capture_buffer, stream_delay_ms())); + } + + if (capture_nonlocked_.beamformer_enabled) { + private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); + } + + public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer); + + if (constants_.use_experimental_agc && + public_submodules_->gain_control->is_enabled() && + (!capture_nonlocked_.beamformer_enabled || + private_submodules_->beamformer->is_target_present())) { + private_submodules_->agc_manager->Process( + capture_buffer->split_bands_const(0)[kBand0To8kHz], + capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate); + } + RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( + capture_buffer, echo_cancellation()->stream_has_echo())); + + if (submodule_states_.CaptureMultiBandProcessingActive() && + SampleRateSupportsMultiBand( + capture_nonlocked_.capture_processing_format.sample_rate_hz())) { + capture_buffer->MergeFrequencyBands(); + } + + if (config_.residual_echo_detector.enabled) { + private_submodules_->residual_echo_detector->AnalyzeCaptureAudio( + rtc::ArrayView(capture_buffer->channels_f()[0], + capture_buffer->num_frames())); + } + + // TODO(aluebs): Investigate if the transient suppression placement should be + // before or after the AGC. + if (capture_.transient_suppressor_enabled) { + float voice_probability = + private_submodules_->agc_manager.get() + ? private_submodules_->agc_manager->voice_probability() + : 1.f; + + public_submodules_->transient_suppressor->Suppress( + capture_buffer->channels_f()[0], capture_buffer->num_frames(), + capture_buffer->num_channels(), + capture_buffer->split_bands_const_f(0)[kBand0To8kHz], + capture_buffer->num_frames_per_band(), capture_buffer->keyboard_data(), + capture_buffer->num_keyboard_frames(), voice_probability, + capture_.key_pressed); + } + + if (config_.gain_controller2.enabled) { + private_submodules_->gain_controller2->Process(capture_buffer); + } + + if (capture_nonlocked_.level_controller_enabled) { + private_submodules_->level_controller->Process(capture_buffer); + } + + if (private_submodules_->capture_post_processor) { + private_submodules_->capture_post_processor->Process(capture_buffer); + } + + // The level estimator operates on the recombined data. + public_submodules_->level_estimator->ProcessStream(capture_buffer); + + capture_output_rms_.Analyze(rtc::ArrayView( + capture_buffer->channels_const()[0], + capture_nonlocked_.capture_processing_format.num_frames())); + if (log_rms) { + RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak(); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelAverageRms", + levels.average, 1, RmsLevel::kMinLevelDb, 64); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms", + levels.peak, 1, RmsLevel::kMinLevelDb, 64); + } + + capture_.was_stream_delay_set = false; + return kNoError; +} + +int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, + size_t samples_per_channel, + int sample_rate_hz, + ChannelLayout layout) { + TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); + rtc::CritScope cs(&crit_render_); + const StreamConfig reverse_config = { + sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), + }; + if (samples_per_channel != reverse_config.num_frames()) { + return kBadDataLengthError; + } + return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); +} + +int AudioProcessingImpl::ProcessReverseStream(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest) { + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); + rtc::CritScope cs(&crit_render_); + RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config)); + if (submodule_states_.RenderMultiBandProcessingActive()) { + render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), + dest); + } else if (formats_.api_format.reverse_input_stream() != + formats_.api_format.reverse_output_stream()) { + render_.render_converter->Convert(src, input_config.num_samples(), dest, + output_config.num_samples()); + } else { + CopyAudioIfNeeded(src, input_config.num_frames(), + input_config.num_channels(), dest); + } + + return kNoError; +} + +int AudioProcessingImpl::AnalyzeReverseStreamLocked( + const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config) { + if (src == nullptr) { + return kNullPointerError; + } + + if (input_config.num_channels() == 0) { + return kBadNumberChannelsError; + } + + ProcessingConfig processing_config = formats_.api_format; + processing_config.reverse_input_stream() = input_config; + processing_config.reverse_output_stream() = output_config; + + RETURN_ON_ERR(MaybeInitializeRender(processing_config)); + assert(input_config.num_frames() == + formats_.api_format.reverse_input_stream().num_frames()); + + if (aec_dump_) { + const size_t channel_size = + formats_.api_format.reverse_input_stream().num_frames(); + const size_t num_channels = + formats_.api_format.reverse_input_stream().num_channels(); + aec_dump_->WriteRenderStreamMessage( + FloatAudioFrame(src, num_channels, channel_size)); + } + render_.render_audio->CopyFrom(src, + formats_.api_format.reverse_input_stream()); + return ProcessRenderStreamLocked(); +} + +int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { + TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); + rtc::CritScope cs(&crit_render_); + if (frame == nullptr) { + return kNullPointerError; + } + // Must be a native rate. + if (frame->sample_rate_hz_ != kSampleRate8kHz && + frame->sample_rate_hz_ != kSampleRate16kHz && + frame->sample_rate_hz_ != kSampleRate32kHz && + frame->sample_rate_hz_ != kSampleRate48kHz) { + return kBadSampleRateError; + } + + if (frame->num_channels_ <= 0) { + return kBadNumberChannelsError; + } + + ProcessingConfig processing_config = formats_.api_format; + processing_config.reverse_input_stream().set_sample_rate_hz( + frame->sample_rate_hz_); + processing_config.reverse_input_stream().set_num_channels( + frame->num_channels_); + processing_config.reverse_output_stream().set_sample_rate_hz( + frame->sample_rate_hz_); + processing_config.reverse_output_stream().set_num_channels( + frame->num_channels_); + + RETURN_ON_ERR(MaybeInitializeRender(processing_config)); + if (frame->samples_per_channel_ != + formats_.api_format.reverse_input_stream().num_frames()) { + return kBadDataLengthError; + } + + if (aec_dump_) { + aec_dump_->WriteRenderStreamMessage(*frame); + } + + render_.render_audio->DeinterleaveFrom(frame); + RETURN_ON_ERR(ProcessRenderStreamLocked()); + render_.render_audio->InterleaveTo( + frame, submodule_states_.RenderMultiBandProcessingActive()); + return kNoError; +} + +int AudioProcessingImpl::ProcessRenderStreamLocked() { + AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. + + QueueNonbandedRenderAudio(render_buffer); + + if (submodule_states_.RenderMultiBandSubModulesActive() && + SampleRateSupportsMultiBand( + formats_.render_processing_format.sample_rate_hz())) { + render_buffer->SplitIntoFrequencyBands(); + } + +#if WEBRTC_INTELLIGIBILITY_ENHANCER + if (capture_nonlocked_.intelligibility_enabled) { + public_submodules_->intelligibility_enhancer->ProcessRenderAudio( + render_buffer); + } +#endif + + if (submodule_states_.RenderMultiBandSubModulesActive()) { + QueueBandedRenderAudio(render_buffer); + } + + // TODO(peah): Perform the queueing ínside QueueRenderAudiuo(). + if (private_submodules_->echo_controller) { + private_submodules_->echo_controller->AnalyzeRender(render_buffer); + } + + if (submodule_states_.RenderMultiBandProcessingActive() && + SampleRateSupportsMultiBand( + formats_.render_processing_format.sample_rate_hz())) { + render_buffer->MergeFrequencyBands(); + } + + return kNoError; +} + +int AudioProcessingImpl::set_stream_delay_ms(int delay) { + rtc::CritScope cs(&crit_capture_); + Error retval = kNoError; + capture_.was_stream_delay_set = true; + delay += capture_.delay_offset_ms; + + if (delay < 0) { + delay = 0; + retval = kBadStreamParameterWarning; + } + + // TODO(ajm): the max is rather arbitrarily chosen; investigate. + if (delay > 500) { + delay = 500; + retval = kBadStreamParameterWarning; + } + + capture_nonlocked_.stream_delay_ms = delay; + return retval; +} + +int AudioProcessingImpl::stream_delay_ms() const { + // Used as callback from submodules, hence locking is not allowed. + return capture_nonlocked_.stream_delay_ms; +} + +bool AudioProcessingImpl::was_stream_delay_set() const { + // Used as callback from submodules, hence locking is not allowed. + return capture_.was_stream_delay_set; +} + +void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { + rtc::CritScope cs(&crit_capture_); + capture_.key_pressed = key_pressed; +} + +void AudioProcessingImpl::set_delay_offset_ms(int offset) { + rtc::CritScope cs(&crit_capture_); + capture_.delay_offset_ms = offset; +} + +int AudioProcessingImpl::delay_offset_ms() const { + rtc::CritScope cs(&crit_capture_); + return capture_.delay_offset_ms; +} + +void AudioProcessingImpl::AttachAecDump(std::unique_ptr aec_dump) { + RTC_DCHECK(aec_dump); + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + + // The previously attached AecDump will be destroyed with the + // 'aec_dump' parameter, which is after locks are released. + aec_dump_.swap(aec_dump); + WriteAecDumpConfigMessage(true); + aec_dump_->WriteInitMessage(ToStreamsConfig(formats_.api_format)); +} + +void AudioProcessingImpl::DetachAecDump() { + // The d-tor of a task-queue based AecDump blocks until all pending + // tasks are done. This construction avoids blocking while holding + // the render and capture locks. + std::unique_ptr aec_dump = nullptr; + { + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + aec_dump = std::move(aec_dump_); + } +} + +AudioProcessing::AudioProcessingStatistics::AudioProcessingStatistics() { + residual_echo_return_loss.Set(-100.0f, -100.0f, -100.0f, -100.0f); + echo_return_loss.Set(-100.0f, -100.0f, -100.0f, -100.0f); + echo_return_loss_enhancement.Set(-100.0f, -100.0f, -100.0f, -100.0f); + a_nlp.Set(-100.0f, -100.0f, -100.0f, -100.0f); +} + +AudioProcessing::AudioProcessingStatistics::AudioProcessingStatistics( + const AudioProcessingStatistics& other) = default; + +AudioProcessing::AudioProcessingStatistics::~AudioProcessingStatistics() = + default; + +// TODO(ivoc): Remove this when GetStatistics() becomes pure virtual. +AudioProcessing::AudioProcessingStatistics AudioProcessing::GetStatistics() + const { + return AudioProcessingStatistics(); +} + +// TODO(ivoc): Remove this when GetStatistics() becomes pure virtual. +AudioProcessingStats AudioProcessing::GetStatistics( + bool has_remote_tracks) const { + return AudioProcessingStats(); +} + +AudioProcessing::AudioProcessingStatistics AudioProcessingImpl::GetStatistics() + const { + AudioProcessingStatistics stats; + EchoCancellation::Metrics metrics; + if (private_submodules_->echo_controller) { + rtc::CritScope cs_capture(&crit_capture_); + auto ec_metrics = private_submodules_->echo_controller->GetMetrics(); + float erl = static_cast(ec_metrics.echo_return_loss); + float erle = static_cast(ec_metrics.echo_return_loss_enhancement); + // Instant value will also be used for min, max and average. + stats.echo_return_loss.Set(erl, erl, erl, erl); + stats.echo_return_loss_enhancement.Set(erle, erle, erle, erle); + } else if (public_submodules_->echo_cancellation->GetMetrics(&metrics) == + Error::kNoError) { + stats.a_nlp.Set(metrics.a_nlp); + stats.divergent_filter_fraction = metrics.divergent_filter_fraction; + stats.echo_return_loss.Set(metrics.echo_return_loss); + stats.echo_return_loss_enhancement.Set( + metrics.echo_return_loss_enhancement); + stats.residual_echo_return_loss.Set(metrics.residual_echo_return_loss); + } + { + rtc::CritScope cs_capture(&crit_capture_); + stats.residual_echo_likelihood = + private_submodules_->residual_echo_detector->echo_likelihood(); + stats.residual_echo_likelihood_recent_max = + private_submodules_->residual_echo_detector + ->echo_likelihood_recent_max(); + } + public_submodules_->echo_cancellation->GetDelayMetrics( + &stats.delay_median, &stats.delay_standard_deviation, + &stats.fraction_poor_delays); + return stats; +} + +AudioProcessingStats AudioProcessingImpl::GetStatistics( + bool has_remote_tracks) const { + AudioProcessingStats stats; + if (has_remote_tracks) { + EchoCancellation::Metrics metrics; + if (private_submodules_->echo_controller) { + rtc::CritScope cs_capture(&crit_capture_); + auto ec_metrics = private_submodules_->echo_controller->GetMetrics(); + stats.echo_return_loss = ec_metrics.echo_return_loss; + stats.echo_return_loss_enhancement = + ec_metrics.echo_return_loss_enhancement; + stats.delay_ms = ec_metrics.delay_ms; + } else if (public_submodules_->echo_cancellation->GetMetrics(&metrics) == + Error::kNoError) { + if (metrics.divergent_filter_fraction != -1.0f) { + stats.divergent_filter_fraction = + rtc::Optional(metrics.divergent_filter_fraction); + } + if (metrics.echo_return_loss.instant != -100) { + stats.echo_return_loss = + rtc::Optional(metrics.echo_return_loss.instant); + } + if (metrics.echo_return_loss_enhancement.instant != -100) { + stats.echo_return_loss_enhancement = + rtc::Optional(metrics.echo_return_loss_enhancement.instant); + } + } + if (config_.residual_echo_detector.enabled) { + rtc::CritScope cs_capture(&crit_capture_); + stats.residual_echo_likelihood = rtc::Optional( + private_submodules_->residual_echo_detector->echo_likelihood()); + stats.residual_echo_likelihood_recent_max = + rtc::Optional(private_submodules_->residual_echo_detector + ->echo_likelihood_recent_max()); + } + int delay_median, delay_std; + float fraction_poor_delays; + if (public_submodules_->echo_cancellation->GetDelayMetrics( + &delay_median, &delay_std, &fraction_poor_delays) == + Error::kNoError) { + if (delay_median >= 0) { + stats.delay_median_ms = rtc::Optional(delay_median); + } + if (delay_std >= 0) { + stats.delay_standard_deviation_ms = rtc::Optional(delay_std); + } + } + } + return stats; +} + +EchoCancellation* AudioProcessingImpl::echo_cancellation() const { + return public_submodules_->echo_cancellation.get(); +} + +EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { + return public_submodules_->echo_control_mobile.get(); +} + +GainControl* AudioProcessingImpl::gain_control() const { + if (constants_.use_experimental_agc) { + return public_submodules_->gain_control_for_experimental_agc.get(); + } + return public_submodules_->gain_control.get(); +} + +HighPassFilter* AudioProcessingImpl::high_pass_filter() const { + return high_pass_filter_impl_.get(); +} + +LevelEstimator* AudioProcessingImpl::level_estimator() const { + return public_submodules_->level_estimator.get(); +} + +NoiseSuppression* AudioProcessingImpl::noise_suppression() const { + return public_submodules_->noise_suppression.get(); +} + +VoiceDetection* AudioProcessingImpl::voice_detection() const { + return public_submodules_->voice_detection.get(); +} + +void AudioProcessingImpl::MutateConfig( + rtc::FunctionView mutator) { + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + mutator(&config_); + ApplyConfig(config_); +} + +AudioProcessing::Config AudioProcessingImpl::GetConfig() const { + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + return config_; +} + +bool AudioProcessingImpl::UpdateActiveSubmoduleStates() { + return submodule_states_.Update( + config_.high_pass_filter.enabled, + public_submodules_->echo_cancellation->is_enabled(), + public_submodules_->echo_control_mobile->is_enabled(), + config_.residual_echo_detector.enabled, + public_submodules_->noise_suppression->is_enabled(), + capture_nonlocked_.intelligibility_enabled, + capture_nonlocked_.beamformer_enabled, + public_submodules_->gain_control->is_enabled(), + config_.gain_controller2.enabled, + capture_nonlocked_.level_controller_enabled, + capture_nonlocked_.echo_controller_enabled, + public_submodules_->voice_detection->is_enabled(), + public_submodules_->level_estimator->is_enabled(), + capture_.transient_suppressor_enabled); +} + + +void AudioProcessingImpl::InitializeTransient() { + if (capture_.transient_suppressor_enabled) { + if (!public_submodules_->transient_suppressor.get()) { + public_submodules_->transient_suppressor.reset(new TransientSuppressor()); + } + public_submodules_->transient_suppressor->Initialize( + capture_nonlocked_.capture_processing_format.sample_rate_hz(), + capture_nonlocked_.split_rate, num_proc_channels()); + } +} + +void AudioProcessingImpl::InitializeBeamformer() { + if (capture_nonlocked_.beamformer_enabled) { + if (!private_submodules_->beamformer) { + private_submodules_->beamformer.reset(new NonlinearBeamformer( + capture_.array_geometry, 1u, capture_.target_direction)); + } + private_submodules_->beamformer->Initialize(kChunkSizeMs, + capture_nonlocked_.split_rate); + } +} + +void AudioProcessingImpl::InitializeIntelligibility() { +#if WEBRTC_INTELLIGIBILITY_ENHANCER + if (capture_nonlocked_.intelligibility_enabled) { + public_submodules_->intelligibility_enhancer.reset( + new IntelligibilityEnhancer(capture_nonlocked_.split_rate, + render_.render_audio->num_channels(), + render_.render_audio->num_bands(), + NoiseSuppressionImpl::num_noise_bins())); + } +#endif +} + +void AudioProcessingImpl::InitializeLowCutFilter() { + if (config_.high_pass_filter.enabled) { + private_submodules_->low_cut_filter.reset( + new LowCutFilter(num_proc_channels(), proc_sample_rate_hz())); + } else { + private_submodules_->low_cut_filter.reset(); + } +} + +void AudioProcessingImpl::InitializeEchoController() { + if (echo_control_factory_) { + private_submodules_->echo_controller = + echo_control_factory_->Create(proc_sample_rate_hz()); + } else { + private_submodules_->echo_controller.reset(); + } +} + +void AudioProcessingImpl::InitializeGainController2() { + if (config_.gain_controller2.enabled) { + private_submodules_->gain_controller2->Initialize(proc_sample_rate_hz()); + } +} + +void AudioProcessingImpl::InitializeLevelController() { + private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); +} + +void AudioProcessingImpl::InitializeResidualEchoDetector() { + private_submodules_->residual_echo_detector->Initialize(); +} + +void AudioProcessingImpl::InitializePostProcessor() { + if (private_submodules_->capture_post_processor) { + private_submodules_->capture_post_processor->Initialize( + proc_sample_rate_hz(), num_proc_channels()); + } +} + +void AudioProcessingImpl::MaybeUpdateHistograms() { + static const int kMinDiffDelayMs = 60; + + if (echo_cancellation()->is_enabled()) { + // Activate delay_jumps_ counters if we know echo_cancellation is running. + // If a stream has echo we know that the echo_cancellation is in process. + if (capture_.stream_delay_jumps == -1 && + echo_cancellation()->stream_has_echo()) { + capture_.stream_delay_jumps = 0; + } + if (capture_.aec_system_delay_jumps == -1 && + echo_cancellation()->stream_has_echo()) { + capture_.aec_system_delay_jumps = 0; + } + + // Detect a jump in platform reported system delay and log the difference. + const int diff_stream_delay_ms = + capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms; + if (diff_stream_delay_ms > kMinDiffDelayMs && + capture_.last_stream_delay_ms != 0) { + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", + diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); + if (capture_.stream_delay_jumps == -1) { + capture_.stream_delay_jumps = 0; // Activate counter if needed. + } + capture_.stream_delay_jumps++; + } + capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms; + + // Detect a jump in AEC system delay and log the difference. + const int samples_per_ms = + rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000); + RTC_DCHECK_LT(0, samples_per_ms); + const int aec_system_delay_ms = + public_submodules_->echo_cancellation->GetSystemDelayInSamples() / + samples_per_ms; + const int diff_aec_system_delay_ms = + aec_system_delay_ms - capture_.last_aec_system_delay_ms; + if (diff_aec_system_delay_ms > kMinDiffDelayMs && + capture_.last_aec_system_delay_ms != 0) { + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", + diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, + 100); + if (capture_.aec_system_delay_jumps == -1) { + capture_.aec_system_delay_jumps = 0; // Activate counter if needed. + } + capture_.aec_system_delay_jumps++; + } + capture_.last_aec_system_delay_ms = aec_system_delay_ms; + } +} + +void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { + // Run in a single-threaded manner. + rtc::CritScope cs_render(&crit_render_); + rtc::CritScope cs_capture(&crit_capture_); + + if (capture_.stream_delay_jumps > -1) { + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", + capture_.stream_delay_jumps, 51); + } + capture_.stream_delay_jumps = -1; + capture_.last_stream_delay_ms = 0; + + if (capture_.aec_system_delay_jumps > -1) { + RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", + capture_.aec_system_delay_jumps, 51); + } + capture_.aec_system_delay_jumps = -1; + capture_.last_aec_system_delay_ms = 0; +} + +void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) { + if (!aec_dump_) { + return; + } + std::string experiments_description = + public_submodules_->echo_cancellation->GetExperimentsDescription(); + // TODO(peah): Add semicolon-separated concatenations of experiment + // descriptions for other submodules. + if (capture_nonlocked_.level_controller_enabled) { + experiments_description += "LevelController;"; + } + if (constants_.agc_clipped_level_min != kClippedLevelMin) { + experiments_description += "AgcClippingLevelExperiment;"; + } + if (capture_nonlocked_.echo_controller_enabled) { + experiments_description += "EchoController;"; + } + if (config_.gain_controller2.enabled) { + experiments_description += "GainController2;"; + } + + InternalAPMConfig apm_config; + + apm_config.aec_enabled = public_submodules_->echo_cancellation->is_enabled(); + apm_config.aec_delay_agnostic_enabled = + public_submodules_->echo_cancellation->is_delay_agnostic_enabled(); + apm_config.aec_drift_compensation_enabled = + public_submodules_->echo_cancellation->is_drift_compensation_enabled(); + apm_config.aec_extended_filter_enabled = + public_submodules_->echo_cancellation->is_extended_filter_enabled(); + apm_config.aec_suppression_level = static_cast( + public_submodules_->echo_cancellation->suppression_level()); + + apm_config.aecm_enabled = + public_submodules_->echo_control_mobile->is_enabled(); + apm_config.aecm_comfort_noise_enabled = + public_submodules_->echo_control_mobile->is_comfort_noise_enabled(); + apm_config.aecm_routing_mode = + static_cast(public_submodules_->echo_control_mobile->routing_mode()); + + apm_config.agc_enabled = public_submodules_->gain_control->is_enabled(); + apm_config.agc_mode = + static_cast(public_submodules_->gain_control->mode()); + apm_config.agc_limiter_enabled = + public_submodules_->gain_control->is_limiter_enabled(); + apm_config.noise_robust_agc_enabled = constants_.use_experimental_agc; + + apm_config.hpf_enabled = config_.high_pass_filter.enabled; + + apm_config.ns_enabled = public_submodules_->noise_suppression->is_enabled(); + apm_config.ns_level = + static_cast(public_submodules_->noise_suppression->level()); + + apm_config.transient_suppression_enabled = + capture_.transient_suppressor_enabled; + apm_config.intelligibility_enhancer_enabled = + capture_nonlocked_.intelligibility_enabled; + apm_config.experiments_description = experiments_description; + + if (!forced && apm_config == apm_config_for_aec_dump_) { + return; + } + aec_dump_->WriteConfig(apm_config); + apm_config_for_aec_dump_ = apm_config; +} + +void AudioProcessingImpl::RecordUnprocessedCaptureStream( + const float* const* src) { + RTC_DCHECK(aec_dump_); + WriteAecDumpConfigMessage(false); + + const size_t channel_size = formats_.api_format.input_stream().num_frames(); + const size_t num_channels = formats_.api_format.input_stream().num_channels(); + aec_dump_->AddCaptureStreamInput( + FloatAudioFrame(src, num_channels, channel_size)); + RecordAudioProcessingState(); +} + +void AudioProcessingImpl::RecordUnprocessedCaptureStream( + const AudioFrame& capture_frame) { + RTC_DCHECK(aec_dump_); + WriteAecDumpConfigMessage(false); + + aec_dump_->AddCaptureStreamInput(capture_frame); + RecordAudioProcessingState(); +} + +void AudioProcessingImpl::RecordProcessedCaptureStream( + const float* const* processed_capture_stream) { + RTC_DCHECK(aec_dump_); + + const size_t channel_size = formats_.api_format.output_stream().num_frames(); + const size_t num_channels = + formats_.api_format.output_stream().num_channels(); + aec_dump_->AddCaptureStreamOutput( + FloatAudioFrame(processed_capture_stream, num_channels, channel_size)); + aec_dump_->WriteCaptureStreamMessage(); +} + +void AudioProcessingImpl::RecordProcessedCaptureStream( + const AudioFrame& processed_capture_frame) { + RTC_DCHECK(aec_dump_); + + aec_dump_->AddCaptureStreamOutput(processed_capture_frame); + aec_dump_->WriteCaptureStreamMessage(); +} + +void AudioProcessingImpl::RecordAudioProcessingState() { + RTC_DCHECK(aec_dump_); + AecDump::AudioProcessingState audio_proc_state; + audio_proc_state.delay = capture_nonlocked_.stream_delay_ms; + audio_proc_state.drift = + public_submodules_->echo_cancellation->stream_drift_samples(); + audio_proc_state.level = gain_control()->stream_analog_level(); + audio_proc_state.keypress = capture_.key_pressed; + aec_dump_->AddAudioProcessingState(audio_proc_state); +} + +AudioProcessingImpl::ApmCaptureState::ApmCaptureState( + bool transient_suppressor_enabled, + const std::vector& array_geometry, + SphericalPointf target_direction) + : aec_system_delay_jumps(-1), + delay_offset_ms(0), + was_stream_delay_set(false), + last_stream_delay_ms(0), + last_aec_system_delay_ms(0), + stream_delay_jumps(-1), + output_will_be_muted(false), + key_pressed(false), + transient_suppressor_enabled(transient_suppressor_enabled), + array_geometry(array_geometry), + target_direction(target_direction), + capture_processing_format(kSampleRate16kHz), + split_rate(kSampleRate16kHz), + echo_path_gain_change(false) {} + +AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; + +AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; + +AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.h new file mode 100644 index 0000000000..021a52037c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl.h @@ -0,0 +1,426 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ + +#include +#include +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/aec_dump.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/render_queue_item_verifier.h" +#include "modules/audio_processing/rms_level.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/function_view.h" +#include "rtc_base/gtest_prod_util.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/protobuf_utils.h" +#include "rtc_base/swap_queue.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/file_wrapper.h" + +namespace webrtc { + +class AudioConverter; +class NonlinearBeamformer; + +class AudioProcessingImpl : public AudioProcessing { + public: + // Methods forcing APM to run in a single-threaded manner. + // Acquires both the render and capture locks. + explicit AudioProcessingImpl(const webrtc::Config& config); + // AudioProcessingImpl takes ownership of capture post processor and + // beamformer. + AudioProcessingImpl(const webrtc::Config& config, + std::unique_ptr capture_post_processor, + std::unique_ptr echo_control_factory, + NonlinearBeamformer* beamformer); + ~AudioProcessingImpl() override; + int Initialize() override; + int Initialize(int capture_input_sample_rate_hz, + int capture_output_sample_rate_hz, + int render_sample_rate_hz, + ChannelLayout capture_input_layout, + ChannelLayout capture_output_layout, + ChannelLayout render_input_layout) override; + int Initialize(const ProcessingConfig& processing_config) override; + void ApplyConfig(const AudioProcessing::Config& config) override; + void SetExtraOptions(const webrtc::Config& config) override; + void UpdateHistogramsOnCallEnd() override; + void AttachAecDump(std::unique_ptr aec_dump) override; + void DetachAecDump() override; + + // Capture-side exclusive methods possibly running APM in a + // multi-threaded manner. Acquire the capture lock. + int ProcessStream(AudioFrame* frame) override; + int ProcessStream(const float* const* src, + size_t samples_per_channel, + int input_sample_rate_hz, + ChannelLayout input_layout, + int output_sample_rate_hz, + ChannelLayout output_layout, + float* const* dest) override; + int ProcessStream(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest) override; + void set_output_will_be_muted(bool muted) override; + int set_stream_delay_ms(int delay) override; + void set_delay_offset_ms(int offset) override; + int delay_offset_ms() const override; + void set_stream_key_pressed(bool key_pressed) override; + + // Render-side exclusive methods possibly running APM in a + // multi-threaded manner. Acquire the render lock. + int ProcessReverseStream(AudioFrame* frame) override; + int AnalyzeReverseStream(const float* const* data, + size_t samples_per_channel, + int sample_rate_hz, + ChannelLayout layout) override; + int ProcessReverseStream(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest) override; + + // Methods only accessed from APM submodules or + // from AudioProcessing tests in a single-threaded manner. + // Hence there is no need for locks in these. + int proc_sample_rate_hz() const override; + int proc_split_sample_rate_hz() const override; + size_t num_input_channels() const override; + size_t num_proc_channels() const override; + size_t num_output_channels() const override; + size_t num_reverse_channels() const override; + int stream_delay_ms() const override; + bool was_stream_delay_set() const override + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + AudioProcessingStatistics GetStatistics() const override; + AudioProcessingStats GetStatistics(bool has_remote_tracks) const override; + + // Methods returning pointers to APM submodules. + // No locks are aquired in those, as those locks + // would offer no protection (the submodules are + // created only once in a single-treaded manner + // during APM creation). + EchoCancellation* echo_cancellation() const override; + EchoControlMobile* echo_control_mobile() const override; + GainControl* gain_control() const override; + // TODO(peah): Deprecate this API call. + HighPassFilter* high_pass_filter() const override; + LevelEstimator* level_estimator() const override; + NoiseSuppression* noise_suppression() const override; + VoiceDetection* voice_detection() const override; + + // TODO(peah): Remove MutateConfig once the new API allows that. + void MutateConfig(rtc::FunctionView mutator); + AudioProcessing::Config GetConfig() const override; + + protected: + // Overridden in a mock. + virtual int InitializeLocked() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + + private: + // TODO(peah): These friend classes should be removed as soon as the new + // parameter setting scheme allows. + FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior); + FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior); + FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior); + struct ApmPublicSubmodules; + struct ApmPrivateSubmodules; + + // Submodule interface implementations. + std::unique_ptr high_pass_filter_impl_; + + // EchoControl factory. + std::unique_ptr echo_control_factory_; + + class ApmSubmoduleStates { + public: + explicit ApmSubmoduleStates(bool capture_post_processor_enabled); + // Updates the submodule state and returns true if it has changed. + bool Update(bool low_cut_filter_enabled, + bool echo_canceller_enabled, + bool mobile_echo_controller_enabled, + bool residual_echo_detector_enabled, + bool noise_suppressor_enabled, + bool intelligibility_enhancer_enabled, + bool beamformer_enabled, + bool adaptive_gain_controller_enabled, + bool gain_controller2_enabled, + bool level_controller_enabled, + bool echo_controller_enabled, + bool voice_activity_detector_enabled, + bool level_estimator_enabled, + bool transient_suppressor_enabled); + bool CaptureMultiBandSubModulesActive() const; + bool CaptureMultiBandProcessingActive() const; + bool CaptureFullBandProcessingActive() const; + bool RenderMultiBandSubModulesActive() const; + bool RenderMultiBandProcessingActive() const; + + private: + const bool capture_post_processor_enabled_ = false; + bool low_cut_filter_enabled_ = false; + bool echo_canceller_enabled_ = false; + bool mobile_echo_controller_enabled_ = false; + bool residual_echo_detector_enabled_ = false; + bool noise_suppressor_enabled_ = false; + bool intelligibility_enhancer_enabled_ = false; + bool beamformer_enabled_ = false; + bool adaptive_gain_controller_enabled_ = false; + bool gain_controller2_enabled_ = false; + bool level_controller_enabled_ = false; + bool echo_controller_enabled_ = false; + bool level_estimator_enabled_ = false; + bool voice_activity_detector_enabled_ = false; + bool transient_suppressor_enabled_ = false; + bool first_update_ = true; + }; + + // Method for modifying the formats struct that are called from both + // the render and capture threads. The check for whether modifications + // are needed is done while holding the render lock only, thereby avoiding + // that the capture thread blocks the render thread. + // The struct is modified in a single-threaded manner by holding both the + // render and capture locks. + int MaybeInitialize(const ProcessingConfig& config, bool force_initialization) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + + int MaybeInitializeRender(const ProcessingConfig& processing_config) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + + int MaybeInitializeCapture(const ProcessingConfig& processing_config, + bool force_initialization) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + + // Method for updating the state keeping track of the active submodules. + // Returns a bool indicating whether the state has changed. + bool UpdateActiveSubmoduleStates() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + // Methods requiring APM running in a single-threaded manner. + // Are called with both the render and capture locks already + // acquired. + void InitializeTransient() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + void InitializeBeamformer() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + void InitializeIntelligibility() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + int InitializeLocked(const ProcessingConfig& config) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + void InitializeLevelController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeResidualEchoDetector() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + void InitializeLowCutFilter() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + void EmptyQueuedRenderAudio(); + void AllocateRenderQueue() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + void QueueBandedRenderAudio(AudioBuffer* audio) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + void QueueNonbandedRenderAudio(AudioBuffer* audio) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + + // Capture-side exclusive methods possibly running APM in a multi-threaded + // manner that are called with the render lock already acquired. + int ProcessCaptureStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void MaybeUpdateHistograms() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + // Render-side exclusive methods possibly running APM in a multi-threaded + // manner that are called with the render lock already acquired. + // TODO(ekm): Remove once all clients updated to new interface. + int AnalyzeReverseStreamLocked(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + int ProcessRenderStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + + // Collects configuration settings from public and private + // submodules to be saved as an audioproc::Config message on the + // AecDump if it is attached. If not |forced|, only writes the current + // config if it is different from the last saved one; if |forced|, + // writes the config regardless of the last saved. + void WriteAecDumpConfigMessage(bool forced) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + // Notifies attached AecDump of current configuration and capture data. + void RecordUnprocessedCaptureStream(const float* const* capture_stream) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + void RecordUnprocessedCaptureStream(const AudioFrame& capture_frame) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + // Notifies attached AecDump of current configuration and + // processed capture data and issues a capture stream recording + // request. + void RecordProcessedCaptureStream( + const float* const* processed_capture_stream) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + void RecordProcessedCaptureStream(const AudioFrame& processed_capture_frame) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + // Notifies attached AecDump about current state (delay, drift, etc). + void RecordAudioProcessingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + + // AecDump instance used for optionally logging APM config, input + // and output to file in the AEC-dump format defined in debug.proto. + std::unique_ptr aec_dump_; + + // Hold the last config written with AecDump for avoiding writing + // the same config twice. + InternalAPMConfig apm_config_for_aec_dump_ RTC_GUARDED_BY(crit_capture_); + + // Critical sections. + rtc::CriticalSection crit_render_ RTC_ACQUIRED_BEFORE(crit_capture_); + rtc::CriticalSection crit_capture_; + + // Struct containing the Config specifying the behavior of APM. + AudioProcessing::Config config_; + + // Class containing information about what submodules are active. + ApmSubmoduleStates submodule_states_; + + // Structs containing the pointers to the submodules. + std::unique_ptr public_submodules_; + std::unique_ptr private_submodules_; + + // State that is written to while holding both the render and capture locks + // but can be read without any lock being held. + // As this is only accessed internally of APM, and all internal methods in APM + // either are holding the render or capture locks, this construct is safe as + // it is not possible to read the variables while writing them. + struct ApmFormatState { + ApmFormatState() + : // Format of processing streams at input/output call sites. + api_format({{{kSampleRate16kHz, 1, false}, + {kSampleRate16kHz, 1, false}, + {kSampleRate16kHz, 1, false}, + {kSampleRate16kHz, 1, false}}}), + render_processing_format(kSampleRate16kHz, 1) {} + ProcessingConfig api_format; + StreamConfig render_processing_format; + } formats_; + + // APM constants. + const struct ApmConstants { + ApmConstants(int agc_startup_min_volume, + int agc_clipped_level_min, + bool use_experimental_agc) + : // Format of processing streams at input/output call sites. + agc_startup_min_volume(agc_startup_min_volume), + agc_clipped_level_min(agc_clipped_level_min), + use_experimental_agc(use_experimental_agc) {} + int agc_startup_min_volume; + int agc_clipped_level_min; + bool use_experimental_agc; + } constants_; + + struct ApmCaptureState { + ApmCaptureState(bool transient_suppressor_enabled, + const std::vector& array_geometry, + SphericalPointf target_direction); + ~ApmCaptureState(); + int aec_system_delay_jumps; + int delay_offset_ms; + bool was_stream_delay_set; + int last_stream_delay_ms; + int last_aec_system_delay_ms; + int stream_delay_jumps; + bool output_will_be_muted; + bool key_pressed; + bool transient_suppressor_enabled; + std::vector array_geometry; + SphericalPointf target_direction; + std::unique_ptr capture_audio; + // Only the rate and samples fields of capture_processing_format_ are used + // because the capture processing number of channels is mutable and is + // tracked by the capture_audio_. + StreamConfig capture_processing_format; + int split_rate; + bool echo_path_gain_change; + } capture_ RTC_GUARDED_BY(crit_capture_); + + struct ApmCaptureNonLockedState { + ApmCaptureNonLockedState(bool beamformer_enabled, + bool intelligibility_enabled) + : capture_processing_format(kSampleRate16kHz), + split_rate(kSampleRate16kHz), + stream_delay_ms(0), + beamformer_enabled(beamformer_enabled), + intelligibility_enabled(intelligibility_enabled) {} + // Only the rate and samples fields of capture_processing_format_ are used + // because the forward processing number of channels is mutable and is + // tracked by the capture_audio_. + StreamConfig capture_processing_format; + int split_rate; + int stream_delay_ms; + bool beamformer_enabled; + bool intelligibility_enabled; + bool level_controller_enabled = false; + bool echo_controller_enabled = false; + } capture_nonlocked_; + + struct ApmRenderState { + ApmRenderState(); + ~ApmRenderState(); + std::unique_ptr render_converter; + std::unique_ptr render_audio; + } render_ RTC_GUARDED_BY(crit_render_); + + size_t aec_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) + RTC_GUARDED_BY(crit_capture_) = 0; + std::vector aec_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); + std::vector aec_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); + + size_t aecm_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) + RTC_GUARDED_BY(crit_capture_) = 0; + std::vector aecm_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); + std::vector aecm_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); + + size_t agc_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) + RTC_GUARDED_BY(crit_capture_) = 0; + std::vector agc_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); + std::vector agc_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); + + size_t red_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) + RTC_GUARDED_BY(crit_capture_) = 0; + std::vector red_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); + std::vector red_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); + + RmsLevel capture_input_rms_ RTC_GUARDED_BY(crit_capture_); + RmsLevel capture_output_rms_ RTC_GUARDED_BY(crit_capture_); + int capture_rms_interval_counter_ RTC_GUARDED_BY(crit_capture_) = 0; + + // Lock protection not needed. + std::unique_ptr, RenderQueueItemVerifier>> + aec_render_signal_queue_; + std::unique_ptr< + SwapQueue, RenderQueueItemVerifier>> + aecm_render_signal_queue_; + std::unique_ptr< + SwapQueue, RenderQueueItemVerifier>> + agc_render_signal_queue_; + std::unique_ptr, RenderQueueItemVerifier>> + red_render_signal_queue_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_locking_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_locking_unittest.cc new file mode 100644 index 0000000000..6c3bad57e6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_locking_unittest.cc @@ -0,0 +1,1135 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/audio_processing_impl.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/test/test_utils.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/event.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/random.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +class AudioProcessingImplLockTest; + +// Type of the render thread APM API call to use in the test. +enum class RenderApiImpl { + ProcessReverseStreamImpl1, + ProcessReverseStreamImpl2, + AnalyzeReverseStreamImpl +}; + +// Type of the capture thread APM API call to use in the test. +enum class CaptureApiImpl { + ProcessStreamImpl1, + ProcessStreamImpl2, + ProcessStreamImpl3 +}; + +// The runtime parameter setting scheme to use in the test. +enum class RuntimeParameterSettingScheme { + SparseStreamMetadataChangeScheme, + ExtremeStreamMetadataChangeScheme, + FixedMonoStreamMetadataScheme, + FixedStereoStreamMetadataScheme +}; + +// Variant of echo canceller settings to use in the test. +enum class AecType { + BasicWebRtcAecSettings, + AecTurnedOff, + BasicWebRtcAecSettingsWithExtentedFilter, + BasicWebRtcAecSettingsWithDelayAgnosticAec, + BasicWebRtcAecSettingsWithAecMobile +}; + +// Thread-safe random number generator wrapper. +class RandomGenerator { + public: + RandomGenerator() : rand_gen_(42U) {} + + int RandInt(int min, int max) { + rtc::CritScope cs(&crit_); + return rand_gen_.Rand(min, max); + } + + int RandInt(int max) { + rtc::CritScope cs(&crit_); + return rand_gen_.Rand(max); + } + + float RandFloat() { + rtc::CritScope cs(&crit_); + return rand_gen_.Rand(); + } + + private: + rtc::CriticalSection crit_; + Random rand_gen_ RTC_GUARDED_BY(crit_); +}; + +// Variables related to the audio data and formats. +struct AudioFrameData { + explicit AudioFrameData(int max_frame_size) { + // Set up the two-dimensional arrays needed for the APM API calls. + input_framechannels.resize(2 * max_frame_size); + input_frame.resize(2); + input_frame[0] = &input_framechannels[0]; + input_frame[1] = &input_framechannels[max_frame_size]; + + output_frame_channels.resize(2 * max_frame_size); + output_frame.resize(2); + output_frame[0] = &output_frame_channels[0]; + output_frame[1] = &output_frame_channels[max_frame_size]; + } + + AudioFrame frame; + std::vector output_frame; + std::vector output_frame_channels; + AudioProcessing::ChannelLayout output_channel_layout = + AudioProcessing::ChannelLayout::kMono; + int input_sample_rate_hz = 16000; + int input_number_of_channels = -1; + std::vector input_frame; + std::vector input_framechannels; + AudioProcessing::ChannelLayout input_channel_layout = + AudioProcessing::ChannelLayout::kMono; + int output_sample_rate_hz = 16000; + int output_number_of_channels = -1; + StreamConfig input_stream_config; + StreamConfig output_stream_config; + int input_samples_per_channel = -1; + int output_samples_per_channel = -1; +}; + +// The configuration for the test. +struct TestConfig { + // Test case generator for the test configurations to use in the brief tests. + static std::vector GenerateBriefTestConfigs() { + std::vector test_configs; + AecType aec_types[] = {AecType::BasicWebRtcAecSettingsWithDelayAgnosticAec, + AecType::BasicWebRtcAecSettingsWithAecMobile}; + for (auto aec_type : aec_types) { + TestConfig test_config; + test_config.aec_type = aec_type; + + test_config.min_number_of_calls = 300; + + // Perform tests only with the extreme runtime parameter setting scheme. + test_config.runtime_parameter_setting_scheme = + RuntimeParameterSettingScheme::ExtremeStreamMetadataChangeScheme; + + // Only test 16 kHz for this test suite. + test_config.initial_sample_rate_hz = 16000; + + // Create test config for the second processing API function set. + test_config.render_api_function = + RenderApiImpl::ProcessReverseStreamImpl2; + test_config.capture_api_function = CaptureApiImpl::ProcessStreamImpl2; + + // Create test config for the first processing API function set. + test_configs.push_back(test_config); + test_config.render_api_function = + RenderApiImpl::AnalyzeReverseStreamImpl; + test_config.capture_api_function = CaptureApiImpl::ProcessStreamImpl3; + test_configs.push_back(test_config); + } + + // Return the created test configurations. + return test_configs; + } + + // Test case generator for the test configurations to use in the extensive + // tests. + static std::vector GenerateExtensiveTestConfigs() { + // Lambda functions for the test config generation. + auto add_processing_apis = [](TestConfig test_config) { + struct AllowedApiCallCombinations { + RenderApiImpl render_api; + CaptureApiImpl capture_api; + }; + + const AllowedApiCallCombinations api_calls[] = { + {RenderApiImpl::ProcessReverseStreamImpl1, + CaptureApiImpl::ProcessStreamImpl1}, + {RenderApiImpl::ProcessReverseStreamImpl2, + CaptureApiImpl::ProcessStreamImpl2}, + {RenderApiImpl::ProcessReverseStreamImpl2, + CaptureApiImpl::ProcessStreamImpl3}, + {RenderApiImpl::AnalyzeReverseStreamImpl, + CaptureApiImpl::ProcessStreamImpl2}, + {RenderApiImpl::AnalyzeReverseStreamImpl, + CaptureApiImpl::ProcessStreamImpl3}}; + std::vector out; + for (auto api_call : api_calls) { + test_config.render_api_function = api_call.render_api; + test_config.capture_api_function = api_call.capture_api; + out.push_back(test_config); + } + return out; + }; + + auto add_aec_settings = [](const std::vector& in) { + std::vector out; + AecType aec_types[] = { + AecType::BasicWebRtcAecSettings, AecType::AecTurnedOff, + AecType::BasicWebRtcAecSettingsWithExtentedFilter, + AecType::BasicWebRtcAecSettingsWithDelayAgnosticAec, + AecType::BasicWebRtcAecSettingsWithAecMobile}; + for (auto test_config : in) { + // Due to a VisualStudio 2015 compiler issue, the internal loop + // variable here cannot override a previously defined name. + // In other words "type" cannot be named "aec_type" here. + // https://connect.microsoft.com/VisualStudio/feedback/details/2291755 + for (auto type : aec_types) { + test_config.aec_type = type; + out.push_back(test_config); + } + } + return out; + }; + + auto add_settings_scheme = [](const std::vector& in) { + std::vector out; + RuntimeParameterSettingScheme schemes[] = { + RuntimeParameterSettingScheme::SparseStreamMetadataChangeScheme, + RuntimeParameterSettingScheme::ExtremeStreamMetadataChangeScheme, + RuntimeParameterSettingScheme::FixedMonoStreamMetadataScheme, + RuntimeParameterSettingScheme::FixedStereoStreamMetadataScheme}; + + for (auto test_config : in) { + for (auto scheme : schemes) { + test_config.runtime_parameter_setting_scheme = scheme; + out.push_back(test_config); + } + } + return out; + }; + + auto add_sample_rates = [](const std::vector& in) { + const int sample_rates[] = {8000, 16000, 32000, 48000}; + + std::vector out; + for (auto test_config : in) { + auto available_rates = + (test_config.aec_type == + AecType::BasicWebRtcAecSettingsWithAecMobile + ? rtc::ArrayView(sample_rates, 2) + : rtc::ArrayView(sample_rates)); + + for (auto rate : available_rates) { + test_config.initial_sample_rate_hz = rate; + out.push_back(test_config); + } + } + return out; + }; + + // Generate test configurations of the relevant combinations of the + // parameters to + // test. + TestConfig test_config; + test_config.min_number_of_calls = 10000; + return add_sample_rates(add_settings_scheme( + add_aec_settings(add_processing_apis(test_config)))); + } + + RenderApiImpl render_api_function = RenderApiImpl::ProcessReverseStreamImpl2; + CaptureApiImpl capture_api_function = CaptureApiImpl::ProcessStreamImpl2; + RuntimeParameterSettingScheme runtime_parameter_setting_scheme = + RuntimeParameterSettingScheme::ExtremeStreamMetadataChangeScheme; + int initial_sample_rate_hz = 16000; + AecType aec_type = AecType::BasicWebRtcAecSettingsWithDelayAgnosticAec; + int min_number_of_calls = 300; +}; + +// Handler for the frame counters. +class FrameCounters { + public: + void IncreaseRenderCounter() { + rtc::CritScope cs(&crit_); + render_count++; + } + + void IncreaseCaptureCounter() { + rtc::CritScope cs(&crit_); + capture_count++; + } + + int GetCaptureCounter() const { + rtc::CritScope cs(&crit_); + return capture_count; + } + + int GetRenderCounter() const { + rtc::CritScope cs(&crit_); + return render_count; + } + + int CaptureMinusRenderCounters() const { + rtc::CritScope cs(&crit_); + return capture_count - render_count; + } + + int RenderMinusCaptureCounters() const { + return -CaptureMinusRenderCounters(); + } + + bool BothCountersExceedeThreshold(int threshold) { + rtc::CritScope cs(&crit_); + return (render_count > threshold && capture_count > threshold); + } + + private: + rtc::CriticalSection crit_; + int render_count RTC_GUARDED_BY(crit_) = 0; + int capture_count RTC_GUARDED_BY(crit_) = 0; +}; + +// Class for handling the capture side processing. +class CaptureProcessor { + public: + CaptureProcessor(int max_frame_size, + RandomGenerator* rand_gen, + rtc::Event* render_call_event, + rtc::Event* capture_call_event, + FrameCounters* shared_counters_state, + AudioProcessingImplLockTest* test_framework, + TestConfig* test_config, + AudioProcessing* apm); + bool Process(); + + private: + static const int kMaxCallDifference = 10; + static const float kCaptureInputFloatLevel; + static const int kCaptureInputFixLevel = 1024; + + void PrepareFrame(); + void CallApmCaptureSide(); + void ApplyRuntimeSettingScheme(); + + RandomGenerator* const rand_gen_ = nullptr; + rtc::Event* const render_call_event_ = nullptr; + rtc::Event* const capture_call_event_ = nullptr; + FrameCounters* const frame_counters_ = nullptr; + AudioProcessingImplLockTest* const test_ = nullptr; + const TestConfig* const test_config_ = nullptr; + AudioProcessing* const apm_ = nullptr; + AudioFrameData frame_data_; +}; + +// Class for handling the stats processing. +class StatsProcessor { + public: + StatsProcessor(RandomGenerator* rand_gen, + TestConfig* test_config, + AudioProcessing* apm); + bool Process(); + + private: + RandomGenerator* rand_gen_ = nullptr; + TestConfig* test_config_ = nullptr; + AudioProcessing* apm_ = nullptr; +}; + +// Class for handling the render side processing. +class RenderProcessor { + public: + RenderProcessor(int max_frame_size, + RandomGenerator* rand_gen, + rtc::Event* render_call_event, + rtc::Event* capture_call_event, + FrameCounters* shared_counters_state, + AudioProcessingImplLockTest* test_framework, + TestConfig* test_config, + AudioProcessing* apm); + bool Process(); + + private: + static const int kMaxCallDifference = 10; + static const int kRenderInputFixLevel = 16384; + static const float kRenderInputFloatLevel; + + void PrepareFrame(); + void CallApmRenderSide(); + void ApplyRuntimeSettingScheme(); + + RandomGenerator* const rand_gen_ = nullptr; + rtc::Event* const render_call_event_ = nullptr; + rtc::Event* const capture_call_event_ = nullptr; + FrameCounters* const frame_counters_ = nullptr; + AudioProcessingImplLockTest* const test_ = nullptr; + const TestConfig* const test_config_ = nullptr; + AudioProcessing* const apm_ = nullptr; + AudioFrameData frame_data_; + bool first_render_call_ = true; +}; + +class AudioProcessingImplLockTest + : public ::testing::TestWithParam { + public: + AudioProcessingImplLockTest(); + bool RunTest(); + bool MaybeEndTest(); + + private: + static const int kTestTimeOutLimit = 10 * 60 * 1000; + static const int kMaxFrameSize = 480; + + // ::testing::TestWithParam<> implementation + void SetUp() override; + void TearDown() override; + + // Thread callback for the render thread + static bool RenderProcessorThreadFunc(void* context) { + return reinterpret_cast(context) + ->render_thread_state_.Process(); + } + + // Thread callback for the capture thread + static bool CaptureProcessorThreadFunc(void* context) { + return reinterpret_cast(context) + ->capture_thread_state_.Process(); + } + + // Thread callback for the stats thread + static bool StatsProcessorThreadFunc(void* context) { + return reinterpret_cast(context) + ->stats_thread_state_.Process(); + } + + // Tests whether all the required render and capture side calls have been + // done. + bool TestDone() { + return frame_counters_.BothCountersExceedeThreshold( + test_config_.min_number_of_calls); + } + + // Start the threads used in the test. + void StartThreads() { + render_thread_.Start(); + render_thread_.SetPriority(rtc::kRealtimePriority); + capture_thread_.Start(); + capture_thread_.SetPriority(rtc::kRealtimePriority); + stats_thread_.Start(); + stats_thread_.SetPriority(rtc::kNormalPriority); + } + + // Event handlers for the test. + rtc::Event test_complete_; + rtc::Event render_call_event_; + rtc::Event capture_call_event_; + + // Thread related variables. + rtc::PlatformThread render_thread_; + rtc::PlatformThread capture_thread_; + rtc::PlatformThread stats_thread_; + mutable RandomGenerator rand_gen_; + + std::unique_ptr apm_; + TestConfig test_config_; + FrameCounters frame_counters_; + RenderProcessor render_thread_state_; + CaptureProcessor capture_thread_state_; + StatsProcessor stats_thread_state_; +}; + +// Sleeps a random time between 0 and max_sleep milliseconds. +void SleepRandomMs(int max_sleep, RandomGenerator* rand_gen) { + int sleeptime = rand_gen->RandInt(0, max_sleep); + SleepMs(sleeptime); +} + +// Populates a float audio frame with random data. +void PopulateAudioFrame(float** frame, + float amplitude, + size_t num_channels, + size_t samples_per_channel, + RandomGenerator* rand_gen) { + for (size_t ch = 0; ch < num_channels; ch++) { + for (size_t k = 0; k < samples_per_channel; k++) { + // Store random 16 bit quantized float number between +-amplitude. + frame[ch][k] = amplitude * (2 * rand_gen->RandFloat() - 1); + } + } +} + +// Populates an audioframe frame of AudioFrame type with random data. +void PopulateAudioFrame(AudioFrame* frame, + int16_t amplitude, + RandomGenerator* rand_gen) { + ASSERT_GT(amplitude, 0); + ASSERT_LE(amplitude, 32767); + int16_t* frame_data = frame->mutable_data(); + for (size_t ch = 0; ch < frame->num_channels_; ch++) { + for (size_t k = 0; k < frame->samples_per_channel_; k++) { + // Store random 16 bit number between -(amplitude+1) and + // amplitude. + frame_data[k * ch] = + rand_gen->RandInt(2 * amplitude + 1) - amplitude - 1; + } + } +} + +AudioProcessingImplLockTest::AudioProcessingImplLockTest() + : test_complete_(false, false), + render_call_event_(false, false), + capture_call_event_(false, false), + render_thread_(RenderProcessorThreadFunc, this, "render"), + capture_thread_(CaptureProcessorThreadFunc, this, "capture"), + stats_thread_(StatsProcessorThreadFunc, this, "stats"), + apm_(AudioProcessingImpl::Create()), + render_thread_state_(kMaxFrameSize, + &rand_gen_, + &render_call_event_, + &capture_call_event_, + &frame_counters_, + this, + &test_config_, + apm_.get()), + capture_thread_state_(kMaxFrameSize, + &rand_gen_, + &render_call_event_, + &capture_call_event_, + &frame_counters_, + this, + &test_config_, + apm_.get()), + stats_thread_state_(&rand_gen_, &test_config_, apm_.get()) {} + +// Run the test with a timeout. +bool AudioProcessingImplLockTest::RunTest() { + StartThreads(); + return test_complete_.Wait(kTestTimeOutLimit); +} + +bool AudioProcessingImplLockTest::MaybeEndTest() { + if (HasFatalFailure() || TestDone()) { + test_complete_.Set(); + return true; + } + return false; +} + +// Setup of test and APM. +void AudioProcessingImplLockTest::SetUp() { + test_config_ = static_cast(GetParam()); + + ASSERT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(true)); + ASSERT_EQ(apm_->kNoError, apm_->gain_control()->Enable(true)); + + ASSERT_EQ(apm_->kNoError, + apm_->gain_control()->set_mode(GainControl::kAdaptiveDigital)); + ASSERT_EQ(apm_->kNoError, apm_->gain_control()->Enable(true)); + + ASSERT_EQ(apm_->kNoError, apm_->noise_suppression()->Enable(true)); + ASSERT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(true)); + + Config config; + if (test_config_.aec_type == AecType::AecTurnedOff) { + ASSERT_EQ(apm_->kNoError, apm_->echo_control_mobile()->Enable(false)); + ASSERT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(false)); + } else if (test_config_.aec_type == + AecType::BasicWebRtcAecSettingsWithAecMobile) { + ASSERT_EQ(apm_->kNoError, apm_->echo_control_mobile()->Enable(true)); + ASSERT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(false)); + } else { + ASSERT_EQ(apm_->kNoError, apm_->echo_control_mobile()->Enable(false)); + ASSERT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + ASSERT_EQ(apm_->kNoError, apm_->echo_cancellation()->enable_metrics(true)); + ASSERT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_delay_logging(true)); + + config.Set( + new ExtendedFilter(test_config_.aec_type == + AecType::BasicWebRtcAecSettingsWithExtentedFilter)); + + config.Set( + new DelayAgnostic(test_config_.aec_type == + AecType::BasicWebRtcAecSettingsWithDelayAgnosticAec)); + + apm_->SetExtraOptions(config); + } +} + +void AudioProcessingImplLockTest::TearDown() { + render_call_event_.Set(); + capture_call_event_.Set(); + render_thread_.Stop(); + capture_thread_.Stop(); + stats_thread_.Stop(); +} + +StatsProcessor::StatsProcessor(RandomGenerator* rand_gen, + TestConfig* test_config, + AudioProcessing* apm) + : rand_gen_(rand_gen), test_config_(test_config), apm_(apm) {} + +// Implements the callback functionality for the statistics +// collection thread. +bool StatsProcessor::Process() { + SleepRandomMs(100, rand_gen_); + + EXPECT_EQ(apm_->echo_cancellation()->is_enabled(), + ((test_config_->aec_type != AecType::AecTurnedOff) && + (test_config_->aec_type != + AecType::BasicWebRtcAecSettingsWithAecMobile))); + apm_->echo_cancellation()->stream_drift_samples(); + EXPECT_EQ(apm_->echo_control_mobile()->is_enabled(), + (test_config_->aec_type != AecType::AecTurnedOff) && + (test_config_->aec_type == + AecType::BasicWebRtcAecSettingsWithAecMobile)); + EXPECT_TRUE(apm_->gain_control()->is_enabled()); + EXPECT_TRUE(apm_->noise_suppression()->is_enabled()); + + // The below return values are not testable. + apm_->noise_suppression()->speech_probability(); + apm_->voice_detection()->is_enabled(); + + return true; +} + +const float CaptureProcessor::kCaptureInputFloatLevel = 0.03125f; + +CaptureProcessor::CaptureProcessor(int max_frame_size, + RandomGenerator* rand_gen, + rtc::Event* render_call_event, + rtc::Event* capture_call_event, + FrameCounters* shared_counters_state, + AudioProcessingImplLockTest* test_framework, + TestConfig* test_config, + AudioProcessing* apm) + : rand_gen_(rand_gen), + render_call_event_(render_call_event), + capture_call_event_(capture_call_event), + frame_counters_(shared_counters_state), + test_(test_framework), + test_config_(test_config), + apm_(apm), + frame_data_(max_frame_size) {} + +// Implements the callback functionality for the capture thread. +bool CaptureProcessor::Process() { + // Sleep a random time to simulate thread jitter. + SleepRandomMs(3, rand_gen_); + + // Check whether the test is done. + if (test_->MaybeEndTest()) { + return false; + } + + // Ensure that the number of render and capture calls do not + // differ too much. + if (frame_counters_->CaptureMinusRenderCounters() > kMaxCallDifference) { + render_call_event_->Wait(rtc::Event::kForever); + } + + // Apply any specified capture side APM non-processing runtime calls. + ApplyRuntimeSettingScheme(); + + // Apply the capture side processing call. + CallApmCaptureSide(); + + // Increase the number of capture-side calls. + frame_counters_->IncreaseCaptureCounter(); + + // Flag to the render thread that another capture API call has occurred + // by triggering this threads call event. + capture_call_event_->Set(); + + return true; +} + +// Prepares a frame with relevant audio data and metadata. +void CaptureProcessor::PrepareFrame() { + // Restrict to a common fixed sample rate if the AudioFrame + // interface is used. + if (test_config_->capture_api_function == + CaptureApiImpl::ProcessStreamImpl1) { + frame_data_.input_sample_rate_hz = test_config_->initial_sample_rate_hz; + frame_data_.output_sample_rate_hz = test_config_->initial_sample_rate_hz; + } + + // Prepare the audioframe data and metadata. + frame_data_.input_samples_per_channel = + frame_data_.input_sample_rate_hz * AudioProcessing::kChunkSizeMs / 1000; + frame_data_.frame.sample_rate_hz_ = frame_data_.input_sample_rate_hz; + frame_data_.frame.num_channels_ = frame_data_.input_number_of_channels; + frame_data_.frame.samples_per_channel_ = + frame_data_.input_samples_per_channel; + PopulateAudioFrame(&frame_data_.frame, kCaptureInputFixLevel, rand_gen_); + + // Prepare the float audio input data and metadata. + frame_data_.input_stream_config.set_sample_rate_hz( + frame_data_.input_sample_rate_hz); + frame_data_.input_stream_config.set_num_channels( + frame_data_.input_number_of_channels); + frame_data_.input_stream_config.set_has_keyboard(false); + PopulateAudioFrame(&frame_data_.input_frame[0], kCaptureInputFloatLevel, + frame_data_.input_number_of_channels, + frame_data_.input_samples_per_channel, rand_gen_); + frame_data_.input_channel_layout = + (frame_data_.input_number_of_channels == 1 + ? AudioProcessing::ChannelLayout::kMono + : AudioProcessing::ChannelLayout::kStereo); + + // Prepare the float audio output data and metadata. + frame_data_.output_samples_per_channel = + frame_data_.output_sample_rate_hz * AudioProcessing::kChunkSizeMs / 1000; + frame_data_.output_stream_config.set_sample_rate_hz( + frame_data_.output_sample_rate_hz); + frame_data_.output_stream_config.set_num_channels( + frame_data_.output_number_of_channels); + frame_data_.output_stream_config.set_has_keyboard(false); + frame_data_.output_channel_layout = + (frame_data_.output_number_of_channels == 1 + ? AudioProcessing::ChannelLayout::kMono + : AudioProcessing::ChannelLayout::kStereo); +} + +// Applies the capture side processing API call. +void CaptureProcessor::CallApmCaptureSide() { + // Prepare a proper capture side processing API call input. + PrepareFrame(); + + // Set the stream delay. + apm_->set_stream_delay_ms(30); + + // Set the analog level. + apm_->gain_control()->set_stream_analog_level(80); + + // Call the specified capture side API processing method. + int result = AudioProcessing::kNoError; + switch (test_config_->capture_api_function) { + case CaptureApiImpl::ProcessStreamImpl1: + result = apm_->ProcessStream(&frame_data_.frame); + break; + case CaptureApiImpl::ProcessStreamImpl2: + result = apm_->ProcessStream( + &frame_data_.input_frame[0], frame_data_.input_samples_per_channel, + frame_data_.input_sample_rate_hz, frame_data_.input_channel_layout, + frame_data_.output_sample_rate_hz, frame_data_.output_channel_layout, + &frame_data_.output_frame[0]); + break; + case CaptureApiImpl::ProcessStreamImpl3: + result = apm_->ProcessStream( + &frame_data_.input_frame[0], frame_data_.input_stream_config, + frame_data_.output_stream_config, &frame_data_.output_frame[0]); + break; + default: + FAIL(); + } + + // Retrieve the new analog level. + apm_->gain_control()->stream_analog_level(); + + // Check the return code for error. + ASSERT_EQ(AudioProcessing::kNoError, result); +} + +// Applies any runtime capture APM API calls and audio stream characteristics +// specified by the scheme for the test. +void CaptureProcessor::ApplyRuntimeSettingScheme() { + const int capture_count_local = frame_counters_->GetCaptureCounter(); + + // Update the number of channels and sample rates for the input and output. + // Note that the counts frequencies for when to set parameters + // are set using prime numbers in order to ensure that the + // permutation scheme in the parameter setting changes. + switch (test_config_->runtime_parameter_setting_scheme) { + case RuntimeParameterSettingScheme::SparseStreamMetadataChangeScheme: + if (capture_count_local == 0) + frame_data_.input_sample_rate_hz = 16000; + else if (capture_count_local % 11 == 0) + frame_data_.input_sample_rate_hz = 32000; + else if (capture_count_local % 73 == 0) + frame_data_.input_sample_rate_hz = 48000; + else if (capture_count_local % 89 == 0) + frame_data_.input_sample_rate_hz = 16000; + else if (capture_count_local % 97 == 0) + frame_data_.input_sample_rate_hz = 8000; + + if (capture_count_local == 0) + frame_data_.input_number_of_channels = 1; + else if (capture_count_local % 4 == 0) + frame_data_.input_number_of_channels = + (frame_data_.input_number_of_channels == 1 ? 2 : 1); + + if (capture_count_local == 0) + frame_data_.output_sample_rate_hz = 16000; + else if (capture_count_local % 5 == 0) + frame_data_.output_sample_rate_hz = 32000; + else if (capture_count_local % 47 == 0) + frame_data_.output_sample_rate_hz = 48000; + else if (capture_count_local % 53 == 0) + frame_data_.output_sample_rate_hz = 16000; + else if (capture_count_local % 71 == 0) + frame_data_.output_sample_rate_hz = 8000; + + if (capture_count_local == 0) + frame_data_.output_number_of_channels = 1; + else if (capture_count_local % 8 == 0) + frame_data_.output_number_of_channels = + (frame_data_.output_number_of_channels == 1 ? 2 : 1); + break; + case RuntimeParameterSettingScheme::ExtremeStreamMetadataChangeScheme: + if (capture_count_local % 2 == 0) { + frame_data_.input_number_of_channels = 1; + frame_data_.input_sample_rate_hz = 16000; + frame_data_.output_number_of_channels = 1; + frame_data_.output_sample_rate_hz = 16000; + } else { + frame_data_.input_number_of_channels = + (frame_data_.input_number_of_channels == 1 ? 2 : 1); + if (frame_data_.input_sample_rate_hz == 8000) + frame_data_.input_sample_rate_hz = 16000; + else if (frame_data_.input_sample_rate_hz == 16000) + frame_data_.input_sample_rate_hz = 32000; + else if (frame_data_.input_sample_rate_hz == 32000) + frame_data_.input_sample_rate_hz = 48000; + else if (frame_data_.input_sample_rate_hz == 48000) + frame_data_.input_sample_rate_hz = 8000; + + frame_data_.output_number_of_channels = + (frame_data_.output_number_of_channels == 1 ? 2 : 1); + if (frame_data_.output_sample_rate_hz == 8000) + frame_data_.output_sample_rate_hz = 16000; + else if (frame_data_.output_sample_rate_hz == 16000) + frame_data_.output_sample_rate_hz = 32000; + else if (frame_data_.output_sample_rate_hz == 32000) + frame_data_.output_sample_rate_hz = 48000; + else if (frame_data_.output_sample_rate_hz == 48000) + frame_data_.output_sample_rate_hz = 8000; + } + break; + case RuntimeParameterSettingScheme::FixedMonoStreamMetadataScheme: + if (capture_count_local == 0) { + frame_data_.input_sample_rate_hz = 16000; + frame_data_.input_number_of_channels = 1; + frame_data_.output_sample_rate_hz = 16000; + frame_data_.output_number_of_channels = 1; + } + break; + case RuntimeParameterSettingScheme::FixedStereoStreamMetadataScheme: + if (capture_count_local == 0) { + frame_data_.input_sample_rate_hz = 16000; + frame_data_.input_number_of_channels = 2; + frame_data_.output_sample_rate_hz = 16000; + frame_data_.output_number_of_channels = 2; + } + break; + default: + FAIL(); + } + + // Call any specified runtime APM setter and + // getter calls. + switch (test_config_->runtime_parameter_setting_scheme) { + case RuntimeParameterSettingScheme::SparseStreamMetadataChangeScheme: + case RuntimeParameterSettingScheme::FixedMonoStreamMetadataScheme: + break; + case RuntimeParameterSettingScheme::ExtremeStreamMetadataChangeScheme: + case RuntimeParameterSettingScheme::FixedStereoStreamMetadataScheme: + if (capture_count_local % 2 == 0) { + ASSERT_EQ(AudioProcessing::Error::kNoError, + apm_->set_stream_delay_ms(30)); + apm_->set_stream_key_pressed(true); + apm_->set_delay_offset_ms(15); + EXPECT_EQ(apm_->delay_offset_ms(), 15); + } else { + ASSERT_EQ(AudioProcessing::Error::kNoError, + apm_->set_stream_delay_ms(50)); + apm_->set_stream_key_pressed(false); + apm_->set_delay_offset_ms(20); + EXPECT_EQ(apm_->delay_offset_ms(), 20); + apm_->delay_offset_ms(); + } + break; + default: + FAIL(); + } + + // Restric the number of output channels not to exceed + // the number of input channels. + frame_data_.output_number_of_channels = + std::min(frame_data_.output_number_of_channels, + frame_data_.input_number_of_channels); +} + +const float RenderProcessor::kRenderInputFloatLevel = 0.5f; + +RenderProcessor::RenderProcessor(int max_frame_size, + RandomGenerator* rand_gen, + rtc::Event* render_call_event, + rtc::Event* capture_call_event, + FrameCounters* shared_counters_state, + AudioProcessingImplLockTest* test_framework, + TestConfig* test_config, + AudioProcessing* apm) + : rand_gen_(rand_gen), + render_call_event_(render_call_event), + capture_call_event_(capture_call_event), + frame_counters_(shared_counters_state), + test_(test_framework), + test_config_(test_config), + apm_(apm), + frame_data_(max_frame_size) {} + +// Implements the callback functionality for the render thread. +bool RenderProcessor::Process() { + // Conditional wait to ensure that a capture call has been done + // before the first render call is performed (implicitly + // required by the APM API). + if (first_render_call_) { + capture_call_event_->Wait(rtc::Event::kForever); + first_render_call_ = false; + } + + // Sleep a random time to simulate thread jitter. + SleepRandomMs(3, rand_gen_); + + // Check whether the test is done. + if (test_->MaybeEndTest()) { + return false; + } + + // Ensure that the number of render and capture calls do not + // differ too much. + if (frame_counters_->RenderMinusCaptureCounters() > kMaxCallDifference) { + capture_call_event_->Wait(rtc::Event::kForever); + } + + // Apply any specified render side APM non-processing runtime calls. + ApplyRuntimeSettingScheme(); + + // Apply the render side processing call. + CallApmRenderSide(); + + // Increase the number of render-side calls. + frame_counters_->IncreaseRenderCounter(); + + // Flag to the capture thread that another render API call has occurred + // by triggering this threads call event. + render_call_event_->Set(); + return true; +} + +// Prepares the render side frame and the accompanying metadata +// with the appropriate information. +void RenderProcessor::PrepareFrame() { + // Restrict to a common fixed sample rate if the AudioFrame interface is + // used. + if ((test_config_->render_api_function == + RenderApiImpl::ProcessReverseStreamImpl1) || + (test_config_->aec_type != + AecType::BasicWebRtcAecSettingsWithAecMobile)) { + frame_data_.input_sample_rate_hz = test_config_->initial_sample_rate_hz; + frame_data_.output_sample_rate_hz = test_config_->initial_sample_rate_hz; + } + + // Prepare the audioframe data and metadata + frame_data_.input_samples_per_channel = + frame_data_.input_sample_rate_hz * AudioProcessing::kChunkSizeMs / 1000; + frame_data_.frame.sample_rate_hz_ = frame_data_.input_sample_rate_hz; + frame_data_.frame.num_channels_ = frame_data_.input_number_of_channels; + frame_data_.frame.samples_per_channel_ = + frame_data_.input_samples_per_channel; + PopulateAudioFrame(&frame_data_.frame, kRenderInputFixLevel, rand_gen_); + + // Prepare the float audio input data and metadata. + frame_data_.input_stream_config.set_sample_rate_hz( + frame_data_.input_sample_rate_hz); + frame_data_.input_stream_config.set_num_channels( + frame_data_.input_number_of_channels); + frame_data_.input_stream_config.set_has_keyboard(false); + PopulateAudioFrame(&frame_data_.input_frame[0], kRenderInputFloatLevel, + frame_data_.input_number_of_channels, + frame_data_.input_samples_per_channel, rand_gen_); + frame_data_.input_channel_layout = + (frame_data_.input_number_of_channels == 1 + ? AudioProcessing::ChannelLayout::kMono + : AudioProcessing::ChannelLayout::kStereo); + + // Prepare the float audio output data and metadata. + frame_data_.output_samples_per_channel = + frame_data_.output_sample_rate_hz * AudioProcessing::kChunkSizeMs / 1000; + frame_data_.output_stream_config.set_sample_rate_hz( + frame_data_.output_sample_rate_hz); + frame_data_.output_stream_config.set_num_channels( + frame_data_.output_number_of_channels); + frame_data_.output_stream_config.set_has_keyboard(false); + frame_data_.output_channel_layout = + (frame_data_.output_number_of_channels == 1 + ? AudioProcessing::ChannelLayout::kMono + : AudioProcessing::ChannelLayout::kStereo); +} + +// Makes the render side processing API call. +void RenderProcessor::CallApmRenderSide() { + // Prepare a proper render side processing API call input. + PrepareFrame(); + + // Call the specified render side API processing method. + int result = AudioProcessing::kNoError; + switch (test_config_->render_api_function) { + case RenderApiImpl::ProcessReverseStreamImpl1: + result = apm_->ProcessReverseStream(&frame_data_.frame); + break; + case RenderApiImpl::ProcessReverseStreamImpl2: + result = apm_->ProcessReverseStream( + &frame_data_.input_frame[0], frame_data_.input_stream_config, + frame_data_.output_stream_config, &frame_data_.output_frame[0]); + break; + case RenderApiImpl::AnalyzeReverseStreamImpl: + result = apm_->AnalyzeReverseStream( + &frame_data_.input_frame[0], frame_data_.input_samples_per_channel, + frame_data_.input_sample_rate_hz, frame_data_.input_channel_layout); + break; + default: + FAIL(); + } + + // Check the return code for error. + ASSERT_EQ(AudioProcessing::kNoError, result); +} + +// Applies any render capture side APM API calls and audio stream +// characteristics +// specified by the scheme for the test. +void RenderProcessor::ApplyRuntimeSettingScheme() { + const int render_count_local = frame_counters_->GetRenderCounter(); + + // Update the number of channels and sample rates for the input and output. + // Note that the counts frequencies for when to set parameters + // are set using prime numbers in order to ensure that the + // permutation scheme in the parameter setting changes. + switch (test_config_->runtime_parameter_setting_scheme) { + case RuntimeParameterSettingScheme::SparseStreamMetadataChangeScheme: + if (render_count_local == 0) + frame_data_.input_sample_rate_hz = 16000; + else if (render_count_local % 47 == 0) + frame_data_.input_sample_rate_hz = 32000; + else if (render_count_local % 71 == 0) + frame_data_.input_sample_rate_hz = 48000; + else if (render_count_local % 79 == 0) + frame_data_.input_sample_rate_hz = 16000; + else if (render_count_local % 83 == 0) + frame_data_.input_sample_rate_hz = 8000; + + if (render_count_local == 0) + frame_data_.input_number_of_channels = 1; + else if (render_count_local % 4 == 0) + frame_data_.input_number_of_channels = + (frame_data_.input_number_of_channels == 1 ? 2 : 1); + + if (render_count_local == 0) + frame_data_.output_sample_rate_hz = 16000; + else if (render_count_local % 17 == 0) + frame_data_.output_sample_rate_hz = 32000; + else if (render_count_local % 19 == 0) + frame_data_.output_sample_rate_hz = 48000; + else if (render_count_local % 29 == 0) + frame_data_.output_sample_rate_hz = 16000; + else if (render_count_local % 61 == 0) + frame_data_.output_sample_rate_hz = 8000; + + if (render_count_local == 0) + frame_data_.output_number_of_channels = 1; + else if (render_count_local % 8 == 0) + frame_data_.output_number_of_channels = + (frame_data_.output_number_of_channels == 1 ? 2 : 1); + break; + case RuntimeParameterSettingScheme::ExtremeStreamMetadataChangeScheme: + if (render_count_local == 0) { + frame_data_.input_number_of_channels = 1; + frame_data_.input_sample_rate_hz = 16000; + frame_data_.output_number_of_channels = 1; + frame_data_.output_sample_rate_hz = 16000; + } else { + frame_data_.input_number_of_channels = + (frame_data_.input_number_of_channels == 1 ? 2 : 1); + if (frame_data_.input_sample_rate_hz == 8000) + frame_data_.input_sample_rate_hz = 16000; + else if (frame_data_.input_sample_rate_hz == 16000) + frame_data_.input_sample_rate_hz = 32000; + else if (frame_data_.input_sample_rate_hz == 32000) + frame_data_.input_sample_rate_hz = 48000; + else if (frame_data_.input_sample_rate_hz == 48000) + frame_data_.input_sample_rate_hz = 8000; + + frame_data_.output_number_of_channels = + (frame_data_.output_number_of_channels == 1 ? 2 : 1); + if (frame_data_.output_sample_rate_hz == 8000) + frame_data_.output_sample_rate_hz = 16000; + else if (frame_data_.output_sample_rate_hz == 16000) + frame_data_.output_sample_rate_hz = 32000; + else if (frame_data_.output_sample_rate_hz == 32000) + frame_data_.output_sample_rate_hz = 48000; + else if (frame_data_.output_sample_rate_hz == 48000) + frame_data_.output_sample_rate_hz = 8000; + } + break; + case RuntimeParameterSettingScheme::FixedMonoStreamMetadataScheme: + if (render_count_local == 0) { + frame_data_.input_sample_rate_hz = 16000; + frame_data_.input_number_of_channels = 1; + frame_data_.output_sample_rate_hz = 16000; + frame_data_.output_number_of_channels = 1; + } + break; + case RuntimeParameterSettingScheme::FixedStereoStreamMetadataScheme: + if (render_count_local == 0) { + frame_data_.input_sample_rate_hz = 16000; + frame_data_.input_number_of_channels = 2; + frame_data_.output_sample_rate_hz = 16000; + frame_data_.output_number_of_channels = 2; + } + break; + default: + FAIL(); + } + + // Restric the number of output channels not to exceed + // the number of input channels. + frame_data_.output_number_of_channels = + std::min(frame_data_.output_number_of_channels, + frame_data_.input_number_of_channels); +} + +} // anonymous namespace + +TEST_P(AudioProcessingImplLockTest, LockTest) { + // Run test and verify that it did not time out. + ASSERT_TRUE(RunTest()); +} + +// Instantiate tests from the extreme test configuration set. +INSTANTIATE_TEST_CASE_P( + DISABLED_AudioProcessingImplLockExtensive, + AudioProcessingImplLockTest, + ::testing::ValuesIn(TestConfig::GenerateExtensiveTestConfigs())); + +INSTANTIATE_TEST_CASE_P( + AudioProcessingImplLockBrief, + AudioProcessingImplLockTest, + ::testing::ValuesIn(TestConfig::GenerateBriefTestConfigs())); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc new file mode 100644 index 0000000000..e152befc5c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/audio_processing_impl.h" + +#include "modules/audio_processing/test/test_utils.h" +#include "modules/include/module_common_types.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::Invoke; + +namespace webrtc { +namespace { + +class MockInitialize : public AudioProcessingImpl { + public: + explicit MockInitialize(const webrtc::Config& config) + : AudioProcessingImpl(config) {} + + MOCK_METHOD0(InitializeLocked, int()); + int RealInitializeLocked() RTC_NO_THREAD_SAFETY_ANALYSIS { + return AudioProcessingImpl::InitializeLocked(); + } + + MOCK_CONST_METHOD0(AddRef, void()); + MOCK_CONST_METHOD0(Release, rtc::RefCountReleaseStatus()); +}; + +} // namespace + +TEST(AudioProcessingImplTest, AudioParameterChangeTriggersInit) { + webrtc::Config config; + MockInitialize mock(config); + ON_CALL(mock, InitializeLocked()) + .WillByDefault(Invoke(&mock, &MockInitialize::RealInitializeLocked)); + + EXPECT_CALL(mock, InitializeLocked()).Times(1); + mock.Initialize(); + + AudioFrame frame; + // Call with the default parameters; there should be an init. + frame.num_channels_ = 1; + SetFrameSampleRate(&frame, 16000); + EXPECT_CALL(mock, InitializeLocked()).Times(0); + EXPECT_NOERR(mock.ProcessStream(&frame)); + EXPECT_NOERR(mock.ProcessReverseStream(&frame)); + + // New sample rate. (Only impacts ProcessStream). + SetFrameSampleRate(&frame, 32000); + EXPECT_CALL(mock, InitializeLocked()) + .Times(1); + EXPECT_NOERR(mock.ProcessStream(&frame)); + + // New number of channels. + // TODO(peah): Investigate why this causes 2 inits. + frame.num_channels_ = 2; + EXPECT_CALL(mock, InitializeLocked()) + .Times(2); + EXPECT_NOERR(mock.ProcessStream(&frame)); + // ProcessStream sets num_channels_ == num_output_channels. + frame.num_channels_ = 2; + EXPECT_NOERR(mock.ProcessReverseStream(&frame)); + + // A new sample rate passed to ProcessReverseStream should cause an init. + SetFrameSampleRate(&frame, 16000); + EXPECT_CALL(mock, InitializeLocked()).Times(1); + EXPECT_NOERR(mock.ProcessReverseStream(&frame)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_c_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_c_gn/moz.build new file mode 100644 index 0000000000..27cfefabd4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_c_gn/moz.build @@ -0,0 +1,183 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_HAS_NEON"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_neon.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +Library("audio_processing_neon_c_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_gn/moz.build new file mode 100644 index 0000000000..1589324a55 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_neon_gn/moz.build @@ -0,0 +1,200 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_APM_DEBUG_DUMP"] = "1" +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_HAS_NEON"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/", + "/third_party/libwebrtc/webrtc/common_audio/resampler/include/", + "/third_party/libwebrtc/webrtc/common_audio/signal_processing/include/", + "/third_party/libwebrtc/webrtc/common_audio/vad/include/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/aecm/aecm_core_neon.cc" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_neon.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_neon.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + +Library("audio_processing_neon_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_performance_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_performance_unittest.cc new file mode 100644 index 0000000000..ea01f24af0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_performance_unittest.cc @@ -0,0 +1,713 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/audio_processing_impl.h" + +#include + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/test/test_utils.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/random.h" +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/event_wrapper.h" +#include "test/gtest.h" +#include "test/testsupport/perf_test.h" + +// Check to verify that the define for the intelligibility enhancer is properly +// set. +#if !defined(WEBRTC_INTELLIGIBILITY_ENHANCER) || \ + (WEBRTC_INTELLIGIBILITY_ENHANCER != 0 && \ + WEBRTC_INTELLIGIBILITY_ENHANCER != 1) +#error "Set WEBRTC_INTELLIGIBILITY_ENHANCER to either 0 or 1" +#endif + +namespace webrtc { + +namespace { + +static const bool kPrintAllDurations = false; + +class CallSimulator; + +// Type of the render thread APM API call to use in the test. +enum class ProcessorType { kRender, kCapture }; + +// Variant of APM processing settings to use in the test. +enum class SettingsType { + kDefaultApmDesktop, + kDefaultApmMobile, + kDefaultApmDesktopAndBeamformer, + kDefaultApmDesktopAndIntelligibilityEnhancer, + kAllSubmodulesTurnedOff, + kDefaultApmDesktopWithoutDelayAgnostic, + kDefaultApmDesktopWithoutExtendedFilter +}; + +// Variables related to the audio data and formats. +struct AudioFrameData { + explicit AudioFrameData(size_t max_frame_size) { + // Set up the two-dimensional arrays needed for the APM API calls. + input_framechannels.resize(2 * max_frame_size); + input_frame.resize(2); + input_frame[0] = &input_framechannels[0]; + input_frame[1] = &input_framechannels[max_frame_size]; + + output_frame_channels.resize(2 * max_frame_size); + output_frame.resize(2); + output_frame[0] = &output_frame_channels[0]; + output_frame[1] = &output_frame_channels[max_frame_size]; + } + + std::vector output_frame_channels; + std::vector output_frame; + std::vector input_framechannels; + std::vector input_frame; + StreamConfig input_stream_config; + StreamConfig output_stream_config; +}; + +// The configuration for the test. +struct SimulationConfig { + SimulationConfig(int sample_rate_hz, SettingsType simulation_settings) + : sample_rate_hz(sample_rate_hz), + simulation_settings(simulation_settings) {} + + static std::vector GenerateSimulationConfigs() { + std::vector simulation_configs; +#ifndef WEBRTC_ANDROID + const SettingsType desktop_settings[] = { + SettingsType::kDefaultApmDesktop, SettingsType::kAllSubmodulesTurnedOff, + SettingsType::kDefaultApmDesktopWithoutDelayAgnostic, + SettingsType::kDefaultApmDesktopWithoutExtendedFilter}; + + const int desktop_sample_rates[] = {8000, 16000, 32000, 48000}; + + for (auto sample_rate : desktop_sample_rates) { + for (auto settings : desktop_settings) { + simulation_configs.push_back(SimulationConfig(sample_rate, settings)); + } + } + +#if WEBRTC_INTELLIGIBILITY_ENHANCER == 1 + const SettingsType intelligibility_enhancer_settings[] = { + SettingsType::kDefaultApmDesktopAndIntelligibilityEnhancer}; + + const int intelligibility_enhancer_sample_rates[] = {8000, 16000, 32000, + 48000}; + + for (auto sample_rate : intelligibility_enhancer_sample_rates) { + for (auto settings : intelligibility_enhancer_settings) { + simulation_configs.push_back(SimulationConfig(sample_rate, settings)); + } + } +#endif + + const SettingsType beamformer_settings[] = { + SettingsType::kDefaultApmDesktopAndBeamformer}; + + const int beamformer_sample_rates[] = {8000, 16000, 32000, 48000}; + + for (auto sample_rate : beamformer_sample_rates) { + for (auto settings : beamformer_settings) { + simulation_configs.push_back(SimulationConfig(sample_rate, settings)); + } + } +#endif + + const SettingsType mobile_settings[] = {SettingsType::kDefaultApmMobile}; + + const int mobile_sample_rates[] = {8000, 16000}; + + for (auto sample_rate : mobile_sample_rates) { + for (auto settings : mobile_settings) { + simulation_configs.push_back(SimulationConfig(sample_rate, settings)); + } + } + + return simulation_configs; + } + + std::string SettingsDescription() const { + std::string description; + switch (simulation_settings) { + case SettingsType::kDefaultApmMobile: + description = "DefaultApmMobile"; + break; + case SettingsType::kDefaultApmDesktop: + description = "DefaultApmDesktop"; + break; + case SettingsType::kDefaultApmDesktopAndBeamformer: + description = "DefaultApmDesktopAndBeamformer"; + break; + case SettingsType::kDefaultApmDesktopAndIntelligibilityEnhancer: + description = "DefaultApmDesktopAndIntelligibilityEnhancer"; + break; + case SettingsType::kAllSubmodulesTurnedOff: + description = "AllSubmodulesOff"; + break; + case SettingsType::kDefaultApmDesktopWithoutDelayAgnostic: + description = "DefaultApmDesktopWithoutDelayAgnostic"; + break; + case SettingsType::kDefaultApmDesktopWithoutExtendedFilter: + description = "DefaultApmDesktopWithoutExtendedFilter"; + break; + } + return description; + } + + int sample_rate_hz = 16000; + SettingsType simulation_settings = SettingsType::kDefaultApmDesktop; +}; + +// Handler for the frame counters. +class FrameCounters { + public: + void IncreaseRenderCounter() { + rtc::AtomicOps::Increment(&render_count_); + } + + void IncreaseCaptureCounter() { + rtc::AtomicOps::Increment(&capture_count_); + } + + int CaptureMinusRenderCounters() const { + // The return value will be approximate, but that's good enough since + // by the time we return the value, it's not guaranteed to be correct + // anyway. + return rtc::AtomicOps::AcquireLoad(&capture_count_) - + rtc::AtomicOps::AcquireLoad(&render_count_); + } + + int RenderMinusCaptureCounters() const { + return -CaptureMinusRenderCounters(); + } + + bool BothCountersExceedeThreshold(int threshold) const { + // TODO(tommi): We could use an event to signal this so that we don't need + // to be polling from the main thread and possibly steal cycles. + const int capture_count = rtc::AtomicOps::AcquireLoad(&capture_count_); + const int render_count = rtc::AtomicOps::AcquireLoad(&render_count_); + return (render_count > threshold && capture_count > threshold); + } + + private: + int render_count_ = 0; + int capture_count_ = 0; +}; + +// Class that represents a flag that can only be raised. +class LockedFlag { + public: + bool get_flag() const { + return rtc::AtomicOps::AcquireLoad(&flag_); + } + + void set_flag() { + if (!get_flag()) // read-only operation to avoid affecting the cache-line. + rtc::AtomicOps::CompareAndSwap(&flag_, 0, 1); + } + + private: + int flag_ = 0; +}; + +// Parent class for the thread processors. +class TimedThreadApiProcessor { + public: + TimedThreadApiProcessor(ProcessorType processor_type, + Random* rand_gen, + FrameCounters* shared_counters_state, + LockedFlag* capture_call_checker, + CallSimulator* test_framework, + const SimulationConfig* simulation_config, + AudioProcessing* apm, + int num_durations_to_store, + float input_level, + int num_channels) + : rand_gen_(rand_gen), + frame_counters_(shared_counters_state), + capture_call_checker_(capture_call_checker), + test_(test_framework), + simulation_config_(simulation_config), + apm_(apm), + frame_data_(kMaxFrameSize), + clock_(webrtc::Clock::GetRealTimeClock()), + num_durations_to_store_(num_durations_to_store), + input_level_(input_level), + processor_type_(processor_type), + num_channels_(num_channels) { + api_call_durations_.reserve(num_durations_to_store_); + } + + // Implements the callback functionality for the threads. + bool Process(); + + // Method for printing out the simulation statistics. + void print_processor_statistics(const std::string& processor_name) const { + const std::string modifier = "_api_call_duration"; + + const std::string sample_rate_name = + "_" + std::to_string(simulation_config_->sample_rate_hz) + "Hz"; + + webrtc::test::PrintResultMeanAndError( + "apm_timing", sample_rate_name, processor_name, + GetDurationAverage(), GetDurationStandardDeviation(), + "us", false); + + if (kPrintAllDurations) { + webrtc::test::PrintResultList("apm_call_durations", sample_rate_name, + processor_name, api_call_durations_, "us", + false); + } + } + + void AddDuration(int64_t duration) { + if (api_call_durations_.size() < num_durations_to_store_) { + api_call_durations_.push_back(duration); + } + } + + private: + static const int kMaxCallDifference = 10; + static const int kMaxFrameSize = 480; + static const int kNumInitializationFrames = 5; + + int64_t GetDurationStandardDeviation() const { + double variance = 0; + const int64_t average_duration = GetDurationAverage(); + for (size_t k = kNumInitializationFrames; k < api_call_durations_.size(); + k++) { + int64_t tmp = api_call_durations_[k] - average_duration; + variance += static_cast(tmp * tmp); + } + const int denominator = rtc::checked_cast(api_call_durations_.size()) - + kNumInitializationFrames; + return (denominator > 0 + ? rtc::checked_cast(sqrt(variance / denominator)) + : -1); + } + + int64_t GetDurationAverage() const { + int64_t average_duration = 0; + for (size_t k = kNumInitializationFrames; k < api_call_durations_.size(); + k++) { + average_duration += api_call_durations_[k]; + } + const int denominator = rtc::checked_cast(api_call_durations_.size()) - + kNumInitializationFrames; + return (denominator > 0 ? average_duration / denominator : -1); + } + + int ProcessCapture() { + // Set the stream delay. + apm_->set_stream_delay_ms(30); + + // Call and time the specified capture side API processing method. + const int64_t start_time = clock_->TimeInMicroseconds(); + const int result = apm_->ProcessStream( + &frame_data_.input_frame[0], frame_data_.input_stream_config, + frame_data_.output_stream_config, &frame_data_.output_frame[0]); + const int64_t end_time = clock_->TimeInMicroseconds(); + + frame_counters_->IncreaseCaptureCounter(); + + AddDuration(end_time - start_time); + + if (first_process_call_) { + // Flag that the capture side has been called at least once + // (needed to ensure that a capture call has been done + // before the first render call is performed (implicitly + // required by the APM API). + capture_call_checker_->set_flag(); + first_process_call_ = false; + } + return result; + } + + bool ReadyToProcessCapture() { + return (frame_counters_->CaptureMinusRenderCounters() <= + kMaxCallDifference); + } + + int ProcessRender() { + // Call and time the specified render side API processing method. + const int64_t start_time = clock_->TimeInMicroseconds(); + const int result = apm_->ProcessReverseStream( + &frame_data_.input_frame[0], frame_data_.input_stream_config, + frame_data_.output_stream_config, &frame_data_.output_frame[0]); + const int64_t end_time = clock_->TimeInMicroseconds(); + frame_counters_->IncreaseRenderCounter(); + + AddDuration(end_time - start_time); + + return result; + } + + bool ReadyToProcessRender() { + // Do not process until at least one capture call has been done. + // (implicitly required by the APM API). + if (first_process_call_ && !capture_call_checker_->get_flag()) { + return false; + } + + // Ensure that the number of render and capture calls do not differ too + // much. + if (frame_counters_->RenderMinusCaptureCounters() > kMaxCallDifference) { + return false; + } + + first_process_call_ = false; + return true; + } + + void PrepareFrame() { + // Lambda function for populating a float multichannel audio frame + // with random data. + auto populate_audio_frame = [](float amplitude, size_t num_channels, + size_t samples_per_channel, Random* rand_gen, + float** frame) { + for (size_t ch = 0; ch < num_channels; ch++) { + for (size_t k = 0; k < samples_per_channel; k++) { + // Store random float number with a value between +-amplitude. + frame[ch][k] = amplitude * (2 * rand_gen->Rand() - 1); + } + } + }; + + // Prepare the audio input data and metadata. + frame_data_.input_stream_config.set_sample_rate_hz( + simulation_config_->sample_rate_hz); + frame_data_.input_stream_config.set_num_channels(num_channels_); + frame_data_.input_stream_config.set_has_keyboard(false); + populate_audio_frame(input_level_, num_channels_, + (simulation_config_->sample_rate_hz * + AudioProcessing::kChunkSizeMs / 1000), + rand_gen_, &frame_data_.input_frame[0]); + + // Prepare the float audio output data and metadata. + frame_data_.output_stream_config.set_sample_rate_hz( + simulation_config_->sample_rate_hz); + frame_data_.output_stream_config.set_num_channels(1); + frame_data_.output_stream_config.set_has_keyboard(false); + } + + bool ReadyToProcess() { + switch (processor_type_) { + case ProcessorType::kRender: + return ReadyToProcessRender(); + + case ProcessorType::kCapture: + return ReadyToProcessCapture(); + } + + // Should not be reached, but the return statement is needed for the code to + // build successfully on Android. + RTC_NOTREACHED(); + return false; + } + + Random* rand_gen_ = nullptr; + FrameCounters* frame_counters_ = nullptr; + LockedFlag* capture_call_checker_ = nullptr; + CallSimulator* test_ = nullptr; + const SimulationConfig* const simulation_config_ = nullptr; + AudioProcessing* apm_ = nullptr; + AudioFrameData frame_data_; + webrtc::Clock* clock_; + const size_t num_durations_to_store_; + std::vector api_call_durations_; + const float input_level_; + bool first_process_call_ = true; + const ProcessorType processor_type_; + const int num_channels_ = 1; +}; + +// Class for managing the test simulation. +class CallSimulator : public ::testing::TestWithParam { + public: + CallSimulator() + : test_complete_(EventWrapper::Create()), + render_thread_( + new rtc::PlatformThread(RenderProcessorThreadFunc, this, "render")), + capture_thread_(new rtc::PlatformThread(CaptureProcessorThreadFunc, + this, + "capture")), + rand_gen_(42U), + simulation_config_(static_cast(GetParam())) {} + + // Run the call simulation with a timeout. + EventTypeWrapper Run() { + StartThreads(); + + EventTypeWrapper result = test_complete_->Wait(kTestTimeout); + + StopThreads(); + + render_thread_state_->print_processor_statistics( + simulation_config_.SettingsDescription() + "_render"); + capture_thread_state_->print_processor_statistics( + simulation_config_.SettingsDescription() + "_capture"); + + return result; + } + + // Tests whether all the required render and capture side calls have been + // done. + bool MaybeEndTest() { + if (frame_counters_.BothCountersExceedeThreshold(kMinNumFramesToProcess)) { + test_complete_->Set(); + return true; + } + return false; + } + + private: + static const float kCaptureInputFloatLevel; + static const float kRenderInputFloatLevel; + static const int kMinNumFramesToProcess = 150; + static const int32_t kTestTimeout = 3 * 10 * kMinNumFramesToProcess; + + // ::testing::TestWithParam<> implementation. + void TearDown() override { StopThreads(); } + + // Stop all running threads. + void StopThreads() { + render_thread_->Stop(); + capture_thread_->Stop(); + } + + // Simulator and APM setup. + void SetUp() override { + // Lambda function for setting the default APM runtime settings for desktop. + auto set_default_desktop_apm_runtime_settings = [](AudioProcessing* apm) { + ASSERT_EQ(apm->kNoError, apm->level_estimator()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->gain_control()->Enable(true)); + ASSERT_EQ(apm->kNoError, + apm->gain_control()->set_mode(GainControl::kAdaptiveDigital)); + ASSERT_EQ(apm->kNoError, apm->gain_control()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->noise_suppression()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->voice_detection()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->echo_control_mobile()->Enable(false)); + ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->enable_metrics(true)); + ASSERT_EQ(apm->kNoError, + apm->echo_cancellation()->enable_delay_logging(true)); + }; + + // Lambda function for setting the default APM runtime settings for mobile. + auto set_default_mobile_apm_runtime_settings = [](AudioProcessing* apm) { + ASSERT_EQ(apm->kNoError, apm->level_estimator()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->gain_control()->Enable(true)); + ASSERT_EQ(apm->kNoError, + apm->gain_control()->set_mode(GainControl::kAdaptiveDigital)); + ASSERT_EQ(apm->kNoError, apm->gain_control()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->noise_suppression()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->voice_detection()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->echo_control_mobile()->Enable(true)); + ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(false)); + }; + + // Lambda function for turning off all of the APM runtime settings + // submodules. + auto turn_off_default_apm_runtime_settings = [](AudioProcessing* apm) { + ASSERT_EQ(apm->kNoError, apm->level_estimator()->Enable(false)); + ASSERT_EQ(apm->kNoError, apm->gain_control()->Enable(false)); + ASSERT_EQ(apm->kNoError, + apm->gain_control()->set_mode(GainControl::kAdaptiveDigital)); + ASSERT_EQ(apm->kNoError, apm->gain_control()->Enable(false)); + ASSERT_EQ(apm->kNoError, apm->noise_suppression()->Enable(false)); + ASSERT_EQ(apm->kNoError, apm->voice_detection()->Enable(false)); + ASSERT_EQ(apm->kNoError, apm->echo_control_mobile()->Enable(false)); + ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(false)); + ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->enable_metrics(false)); + ASSERT_EQ(apm->kNoError, + apm->echo_cancellation()->enable_delay_logging(false)); + }; + + // Lambda function for adding default desktop APM settings to a config. + auto add_default_desktop_config = [](Config* config) { + config->Set(new ExtendedFilter(true)); + config->Set(new DelayAgnostic(true)); + }; + + // Lambda function for adding beamformer settings to a config. + auto add_beamformer_config = [](Config* config) { + const size_t num_mics = 2; + const std::vector array_geometry = + ParseArrayGeometry("0 0 0 0.05 0 0", num_mics); + RTC_CHECK_EQ(array_geometry.size(), num_mics); + + config->Set( + new Beamforming(true, array_geometry, + SphericalPointf(DegreesToRadians(90), 0.f, 1.f))); + }; + + int num_capture_channels = 1; + switch (simulation_config_.simulation_settings) { + case SettingsType::kDefaultApmMobile: { + apm_.reset(AudioProcessingImpl::Create()); + ASSERT_TRUE(!!apm_); + set_default_mobile_apm_runtime_settings(apm_.get()); + break; + } + case SettingsType::kDefaultApmDesktop: { + Config config; + add_default_desktop_config(&config); + apm_.reset(AudioProcessingImpl::Create(config)); + ASSERT_TRUE(!!apm_); + set_default_desktop_apm_runtime_settings(apm_.get()); + apm_->SetExtraOptions(config); + break; + } + case SettingsType::kDefaultApmDesktopAndBeamformer: { + Config config; + add_beamformer_config(&config); + add_default_desktop_config(&config); + apm_.reset(AudioProcessingImpl::Create(config)); + ASSERT_TRUE(!!apm_); + set_default_desktop_apm_runtime_settings(apm_.get()); + apm_->SetExtraOptions(config); + num_capture_channels = 2; + break; + } + case SettingsType::kDefaultApmDesktopAndIntelligibilityEnhancer: { + Config config; + config.Set(new Intelligibility(true)); + add_default_desktop_config(&config); + apm_.reset(AudioProcessingImpl::Create(config)); + ASSERT_TRUE(!!apm_); + set_default_desktop_apm_runtime_settings(apm_.get()); + apm_->SetExtraOptions(config); + break; + } + case SettingsType::kAllSubmodulesTurnedOff: { + apm_.reset(AudioProcessingImpl::Create()); + ASSERT_TRUE(!!apm_); + turn_off_default_apm_runtime_settings(apm_.get()); + break; + } + case SettingsType::kDefaultApmDesktopWithoutDelayAgnostic: { + Config config; + config.Set(new ExtendedFilter(true)); + config.Set(new DelayAgnostic(false)); + apm_.reset(AudioProcessingImpl::Create(config)); + ASSERT_TRUE(!!apm_); + set_default_desktop_apm_runtime_settings(apm_.get()); + apm_->SetExtraOptions(config); + break; + } + case SettingsType::kDefaultApmDesktopWithoutExtendedFilter: { + Config config; + config.Set(new ExtendedFilter(false)); + config.Set(new DelayAgnostic(true)); + apm_.reset(AudioProcessingImpl::Create(config)); + ASSERT_TRUE(!!apm_); + set_default_desktop_apm_runtime_settings(apm_.get()); + apm_->SetExtraOptions(config); + break; + } + } + + render_thread_state_.reset(new TimedThreadApiProcessor( + ProcessorType::kRender, &rand_gen_, &frame_counters_, + &capture_call_checker_, this, &simulation_config_, apm_.get(), + kMinNumFramesToProcess, kRenderInputFloatLevel, 1)); + capture_thread_state_.reset(new TimedThreadApiProcessor( + ProcessorType::kCapture, &rand_gen_, &frame_counters_, + &capture_call_checker_, this, &simulation_config_, apm_.get(), + kMinNumFramesToProcess, kCaptureInputFloatLevel, num_capture_channels)); + } + + // Thread callback for the render thread. + static bool RenderProcessorThreadFunc(void* context) { + return reinterpret_cast(context) + ->render_thread_state_->Process(); + } + + // Thread callback for the capture thread. + static bool CaptureProcessorThreadFunc(void* context) { + return reinterpret_cast(context) + ->capture_thread_state_->Process(); + } + + // Start the threads used in the test. + void StartThreads() { + ASSERT_NO_FATAL_FAILURE(render_thread_->Start()); + render_thread_->SetPriority(rtc::kRealtimePriority); + ASSERT_NO_FATAL_FAILURE(capture_thread_->Start()); + capture_thread_->SetPriority(rtc::kRealtimePriority); + } + + // Event handler for the test. + const std::unique_ptr test_complete_; + + // Thread related variables. + std::unique_ptr render_thread_; + std::unique_ptr capture_thread_; + Random rand_gen_; + + std::unique_ptr apm_; + const SimulationConfig simulation_config_; + FrameCounters frame_counters_; + LockedFlag capture_call_checker_; + std::unique_ptr render_thread_state_; + std::unique_ptr capture_thread_state_; +}; + +// Implements the callback functionality for the threads. +bool TimedThreadApiProcessor::Process() { + PrepareFrame(); + + // Wait in a spinlock manner until it is ok to start processing. + // Note that SleepMs is not applicable since it only allows sleeping + // on a millisecond basis which is too long. + // TODO(tommi): This loop may affect the performance of the test that it's + // meant to measure. See if we could use events instead to signal readiness. + while (!ReadyToProcess()) { + } + + int result = AudioProcessing::kNoError; + switch (processor_type_) { + case ProcessorType::kRender: + result = ProcessRender(); + break; + case ProcessorType::kCapture: + result = ProcessCapture(); + break; + } + + EXPECT_EQ(result, AudioProcessing::kNoError); + + return !test_->MaybeEndTest(); +} + +const float CallSimulator::kRenderInputFloatLevel = 0.5f; +const float CallSimulator::kCaptureInputFloatLevel = 0.03125f; +} // anonymous namespace + +// TODO(peah): Reactivate once issue 7712 has been resolved. +TEST_P(CallSimulator, DISABLED_ApiCallDurationTest) { + // Run test and verify that it did not time out. + EXPECT_EQ(kEventSignaled, Run()); +} + +INSTANTIATE_TEST_CASE_P( + AudioProcessingPerformanceTest, + CallSimulator, + ::testing::ValuesIn(SimulationConfig::GenerateSimulationConfigs())); + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_sse2_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_sse2_gn/moz.build new file mode 100644 index 0000000000..f69cd941e5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_sse2_gn/moz.build @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_APM_DEBUG_DUMP"] = "1" +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/aec/aec_core_sse2.cc", + "/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_sse2.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["CR_XCODE_VERSION"] = "0920" + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + +if CONFIG["OS_TARGET"] == "DragonFly": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2", + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2", + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2", + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2", + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_processing_sse2_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_statistics_gn/moz.build b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_statistics_gn/moz.build new file mode 100644 index 0000000000..848aff350d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_statistics_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("audio_processing_statistics_gn") diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_unittest.cc new file mode 100644 index 0000000000..4da0621abf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/audio_processing_unittest.cc @@ -0,0 +1,3117 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include +#include + +#include +#include +#include +#include + +#include "common_audio/include/audio_util.h" +#include "common_audio/resampler/include/push_resampler.h" +#include "common_audio/resampler/push_sinc_resampler.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "modules/audio_processing/audio_processing_impl.h" +#include "modules/audio_processing/beamformer/mock_nonlinear_beamformer.h" +#include "modules/audio_processing/common.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/include/mock_audio_processing.h" +#include "modules/audio_processing/level_controller/level_controller_constants.h" +#include "modules/audio_processing/test/protobuf_utils.h" +#include "modules/audio_processing/test/test_utils.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/gtest_prod_util.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/protobuf_utils.h" +#include "rtc_base/refcountedobject.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/thread.h" +#include "system_wrappers/include/event_wrapper.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_processing/test/unittest.pb.h" +#else +#include "modules/audio_processing/test/unittest.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() + +namespace webrtc { +namespace { + +// TODO(ekmeyerson): Switch to using StreamConfig and ProcessingConfig where +// applicable. + +// TODO(bjornv): This is not feasible until the functionality has been +// re-implemented; see comment at the bottom of this file. For now, the user has +// to hard code the |write_ref_data| value. +// When false, this will compare the output data with the results stored to +// file. This is the typical case. When the file should be updated, it can +// be set to true with the command-line switch --write_ref_data. +bool write_ref_data = false; +const int32_t kChannels[] = {1, 2}; +const int kSampleRates[] = {8000, 16000, 32000, 48000}; + +#if defined(WEBRTC_AUDIOPROC_FIXED_PROFILE) +// Android doesn't support 48kHz. +const int kProcessSampleRates[] = {8000, 16000, 32000}; +#elif defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) +const int kProcessSampleRates[] = {8000, 16000, 32000, 48000}; +#endif + +enum StreamDirection { kForward = 0, kReverse }; + +void ConvertToFloat(const int16_t* int_data, ChannelBuffer* cb) { + ChannelBuffer cb_int(cb->num_frames(), + cb->num_channels()); + Deinterleave(int_data, + cb->num_frames(), + cb->num_channels(), + cb_int.channels()); + for (size_t i = 0; i < cb->num_channels(); ++i) { + S16ToFloat(cb_int.channels()[i], + cb->num_frames(), + cb->channels()[i]); + } +} + +void ConvertToFloat(const AudioFrame& frame, ChannelBuffer* cb) { + ConvertToFloat(frame.data(), cb); +} + +// Number of channels including the keyboard channel. +size_t TotalChannelsFromLayout(AudioProcessing::ChannelLayout layout) { + switch (layout) { + case AudioProcessing::kMono: + return 1; + case AudioProcessing::kMonoAndKeyboard: + case AudioProcessing::kStereo: + return 2; + case AudioProcessing::kStereoAndKeyboard: + return 3; + } + RTC_NOTREACHED(); + return 0; +} + +int TruncateToMultipleOf10(int value) { + return (value / 10) * 10; +} + +void MixStereoToMono(const float* stereo, float* mono, + size_t samples_per_channel) { + for (size_t i = 0; i < samples_per_channel; ++i) + mono[i] = (stereo[i * 2] + stereo[i * 2 + 1]) / 2; +} + +void MixStereoToMono(const int16_t* stereo, int16_t* mono, + size_t samples_per_channel) { + for (size_t i = 0; i < samples_per_channel; ++i) + mono[i] = (stereo[i * 2] + stereo[i * 2 + 1]) >> 1; +} + +void CopyLeftToRightChannel(int16_t* stereo, size_t samples_per_channel) { + for (size_t i = 0; i < samples_per_channel; i++) { + stereo[i * 2 + 1] = stereo[i * 2]; + } +} + +void VerifyChannelsAreEqual(const int16_t* stereo, size_t samples_per_channel) { + for (size_t i = 0; i < samples_per_channel; i++) { + EXPECT_EQ(stereo[i * 2 + 1], stereo[i * 2]); + } +} + +void SetFrameTo(AudioFrame* frame, int16_t value) { + int16_t* frame_data = frame->mutable_data(); + for (size_t i = 0; i < frame->samples_per_channel_ * frame->num_channels_; + ++i) { + frame_data[i] = value; + } +} + +void SetFrameTo(AudioFrame* frame, int16_t left, int16_t right) { + ASSERT_EQ(2u, frame->num_channels_); + int16_t* frame_data = frame->mutable_data(); + for (size_t i = 0; i < frame->samples_per_channel_ * 2; i += 2) { + frame_data[i] = left; + frame_data[i + 1] = right; + } +} + +void ScaleFrame(AudioFrame* frame, float scale) { + int16_t* frame_data = frame->mutable_data(); + for (size_t i = 0; i < frame->samples_per_channel_ * frame->num_channels_; + ++i) { + frame_data[i] = FloatS16ToS16(frame_data[i] * scale); + } +} + +bool FrameDataAreEqual(const AudioFrame& frame1, const AudioFrame& frame2) { + if (frame1.samples_per_channel_ != frame2.samples_per_channel_) { + return false; + } + if (frame1.num_channels_ != frame2.num_channels_) { + return false; + } + if (memcmp(frame1.data(), frame2.data(), + frame1.samples_per_channel_ * frame1.num_channels_ * + sizeof(int16_t))) { + return false; + } + return true; +} + +void EnableAllAPComponents(AudioProcessing* ap) { +#if defined(WEBRTC_AUDIOPROC_FIXED_PROFILE) + EXPECT_NOERR(ap->echo_control_mobile()->Enable(true)); + + EXPECT_NOERR(ap->gain_control()->set_mode(GainControl::kAdaptiveDigital)); + EXPECT_NOERR(ap->gain_control()->Enable(true)); +#elif defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) + EXPECT_NOERR(ap->echo_cancellation()->enable_drift_compensation(true)); + EXPECT_NOERR(ap->echo_cancellation()->enable_metrics(true)); + EXPECT_NOERR(ap->echo_cancellation()->enable_delay_logging(true)); + EXPECT_NOERR(ap->echo_cancellation()->Enable(true)); + + EXPECT_NOERR(ap->gain_control()->set_mode(GainControl::kAdaptiveAnalog)); + EXPECT_NOERR(ap->gain_control()->set_analog_level_limits(0, 255)); + EXPECT_NOERR(ap->gain_control()->Enable(true)); +#endif + + AudioProcessing::Config apm_config; + apm_config.high_pass_filter.enabled = true; + ap->ApplyConfig(apm_config); + + EXPECT_NOERR(ap->level_estimator()->Enable(true)); + EXPECT_NOERR(ap->noise_suppression()->Enable(true)); + + EXPECT_NOERR(ap->voice_detection()->Enable(true)); +} + +// These functions are only used by ApmTest.Process. +template +T AbsValue(T a) { + return a > 0 ? a: -a; +} + +int16_t MaxAudioFrame(const AudioFrame& frame) { + const size_t length = frame.samples_per_channel_ * frame.num_channels_; + const int16_t* frame_data = frame.data(); + int16_t max_data = AbsValue(frame_data[0]); + for (size_t i = 1; i < length; i++) { + max_data = std::max(max_data, AbsValue(frame_data[i])); + } + + return max_data; +} + +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) +void TestStats(const AudioProcessing::Statistic& test, + const audioproc::Test::Statistic& reference) { + EXPECT_EQ(reference.instant(), test.instant); + EXPECT_EQ(reference.average(), test.average); + EXPECT_EQ(reference.maximum(), test.maximum); + EXPECT_EQ(reference.minimum(), test.minimum); +} + +void WriteStatsMessage(const AudioProcessing::Statistic& output, + audioproc::Test::Statistic* msg) { + msg->set_instant(output.instant); + msg->set_average(output.average); + msg->set_maximum(output.maximum); + msg->set_minimum(output.minimum); +} +#endif + +void OpenFileAndWriteMessage(const std::string& filename, + const MessageLite& msg) { + FILE* file = fopen(filename.c_str(), "wb"); + ASSERT_TRUE(file != NULL); + + int32_t size = msg.ByteSize(); + ASSERT_GT(size, 0); + std::unique_ptr array(new uint8_t[size]); + ASSERT_TRUE(msg.SerializeToArray(array.get(), size)); + + ASSERT_EQ(1u, fwrite(&size, sizeof(size), 1, file)); + ASSERT_EQ(static_cast(size), + fwrite(array.get(), sizeof(array[0]), size, file)); + fclose(file); +} + +std::string ResourceFilePath(const std::string& name, int sample_rate_hz) { + std::ostringstream ss; + // Resource files are all stereo. + ss << name << sample_rate_hz / 1000 << "_stereo"; + return test::ResourcePath(ss.str(), "pcm"); +} + +// Temporary filenames unique to this process. Used to be able to run these +// tests in parallel as each process needs to be running in isolation they can't +// have competing filenames. +std::map temp_filenames; + +std::string OutputFilePath(const std::string& name, + int input_rate, + int output_rate, + int reverse_input_rate, + int reverse_output_rate, + size_t num_input_channels, + size_t num_output_channels, + size_t num_reverse_input_channels, + size_t num_reverse_output_channels, + StreamDirection file_direction) { + std::ostringstream ss; + ss << name << "_i" << num_input_channels << "_" << input_rate / 1000 << "_ir" + << num_reverse_input_channels << "_" << reverse_input_rate / 1000 << "_"; + if (num_output_channels == 1) { + ss << "mono"; + } else if (num_output_channels == 2) { + ss << "stereo"; + } else { + RTC_NOTREACHED(); + } + ss << output_rate / 1000; + if (num_reverse_output_channels == 1) { + ss << "_rmono"; + } else if (num_reverse_output_channels == 2) { + ss << "_rstereo"; + } else { + RTC_NOTREACHED(); + } + ss << reverse_output_rate / 1000; + ss << "_d" << file_direction << "_pcm"; + + std::string filename = ss.str(); + if (temp_filenames[filename].empty()) + temp_filenames[filename] = test::TempFilename(test::OutputPath(), filename); + return temp_filenames[filename]; +} + +void ClearTempFiles() { + for (auto& kv : temp_filenames) + remove(kv.second.c_str()); +} + +// Only remove "out" files. Keep "ref" files. +void ClearTempOutFiles() { + for (auto it = temp_filenames.begin(); it != temp_filenames.end();) { + const std::string& filename = it->first; + if (filename.substr(0, 3).compare("out") == 0) { + remove(it->second.c_str()); + temp_filenames.erase(it++); + } else { + it++; + } + } +} + +void OpenFileAndReadMessage(const std::string& filename, MessageLite* msg) { + FILE* file = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(file != NULL); + ReadMessageFromFile(file, msg); + fclose(file); +} + +// Reads a 10 ms chunk of int16 interleaved audio from the given (assumed +// stereo) file, converts to deinterleaved float (optionally downmixing) and +// returns the result in |cb|. Returns false if the file ended (or on error) and +// true otherwise. +// +// |int_data| and |float_data| are just temporary space that must be +// sufficiently large to hold the 10 ms chunk. +bool ReadChunk(FILE* file, int16_t* int_data, float* float_data, + ChannelBuffer* cb) { + // The files always contain stereo audio. + size_t frame_size = cb->num_frames() * 2; + size_t read_count = fread(int_data, sizeof(int16_t), frame_size, file); + if (read_count != frame_size) { + // Check that the file really ended. + RTC_DCHECK(feof(file)); + return false; // This is expected. + } + + S16ToFloat(int_data, frame_size, float_data); + if (cb->num_channels() == 1) { + MixStereoToMono(float_data, cb->channels()[0], cb->num_frames()); + } else { + Deinterleave(float_data, cb->num_frames(), 2, + cb->channels()); + } + + return true; +} + +class ApmTest : public ::testing::Test { + protected: + ApmTest(); + virtual void SetUp(); + virtual void TearDown(); + + static void SetUpTestCase() { + } + + static void TearDownTestCase() { + ClearTempFiles(); + } + + // Used to select between int and float interface tests. + enum Format { + kIntFormat, + kFloatFormat + }; + + void Init(int sample_rate_hz, + int output_sample_rate_hz, + int reverse_sample_rate_hz, + size_t num_input_channels, + size_t num_output_channels, + size_t num_reverse_channels, + bool open_output_file); + void Init(AudioProcessing* ap); + void EnableAllComponents(); + bool ReadFrame(FILE* file, AudioFrame* frame); + bool ReadFrame(FILE* file, AudioFrame* frame, ChannelBuffer* cb); + void ReadFrameWithRewind(FILE* file, AudioFrame* frame); + void ReadFrameWithRewind(FILE* file, AudioFrame* frame, + ChannelBuffer* cb); + void ProcessWithDefaultStreamParameters(AudioFrame* frame); + void ProcessDelayVerificationTest(int delay_ms, int system_delay_ms, + int delay_min, int delay_max); + void TestChangingChannelsInt16Interface( + size_t num_channels, + AudioProcessing::Error expected_return); + void TestChangingForwardChannels(size_t num_in_channels, + size_t num_out_channels, + AudioProcessing::Error expected_return); + void TestChangingReverseChannels(size_t num_rev_channels, + AudioProcessing::Error expected_return); + void RunQuantizedVolumeDoesNotGetStuckTest(int sample_rate); + void RunManualVolumeChangeIsPossibleTest(int sample_rate); + void StreamParametersTest(Format format); + int ProcessStreamChooser(Format format); + int AnalyzeReverseStreamChooser(Format format); + void ProcessDebugDump(const std::string& in_filename, + const std::string& out_filename, + Format format, + int max_size_bytes); + void VerifyDebugDumpTest(Format format); + + const std::string output_path_; + const std::string ref_filename_; + std::unique_ptr apm_; + AudioFrame* frame_; + AudioFrame* revframe_; + std::unique_ptr > float_cb_; + std::unique_ptr > revfloat_cb_; + int output_sample_rate_hz_; + size_t num_output_channels_; + FILE* far_file_; + FILE* near_file_; + FILE* out_file_; +}; + +ApmTest::ApmTest() + : output_path_(test::OutputPath()), +#if defined(WEBRTC_AUDIOPROC_FIXED_PROFILE) + ref_filename_(test::ResourcePath("audio_processing/output_data_fixed", + "pb")), +#elif defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) +#if defined(WEBRTC_MAC) + // A different file for Mac is needed because on this platform the AEC + // constant |kFixedDelayMs| value is 20 and not 50 as it is on the rest. + ref_filename_(test::ResourcePath("audio_processing/output_data_mac", + "pb")), +#else + ref_filename_(test::ResourcePath("audio_processing/output_data_float", + "pb")), +#endif +#endif + frame_(NULL), + revframe_(NULL), + output_sample_rate_hz_(0), + num_output_channels_(0), + far_file_(NULL), + near_file_(NULL), + out_file_(NULL) { + Config config; + config.Set(new ExperimentalAgc(false)); + apm_.reset(AudioProcessing::Create(config)); +} + +void ApmTest::SetUp() { + ASSERT_TRUE(apm_.get() != NULL); + + frame_ = new AudioFrame(); + revframe_ = new AudioFrame(); + + Init(32000, 32000, 32000, 2, 2, 2, false); +} + +void ApmTest::TearDown() { + if (frame_) { + delete frame_; + } + frame_ = NULL; + + if (revframe_) { + delete revframe_; + } + revframe_ = NULL; + + if (far_file_) { + ASSERT_EQ(0, fclose(far_file_)); + } + far_file_ = NULL; + + if (near_file_) { + ASSERT_EQ(0, fclose(near_file_)); + } + near_file_ = NULL; + + if (out_file_) { + ASSERT_EQ(0, fclose(out_file_)); + } + out_file_ = NULL; +} + +void ApmTest::Init(AudioProcessing* ap) { + ASSERT_EQ(kNoErr, + ap->Initialize( + {{{frame_->sample_rate_hz_, frame_->num_channels_}, + {output_sample_rate_hz_, num_output_channels_}, + {revframe_->sample_rate_hz_, revframe_->num_channels_}, + {revframe_->sample_rate_hz_, revframe_->num_channels_}}})); +} + +void ApmTest::Init(int sample_rate_hz, + int output_sample_rate_hz, + int reverse_sample_rate_hz, + size_t num_input_channels, + size_t num_output_channels, + size_t num_reverse_channels, + bool open_output_file) { + SetContainerFormat(sample_rate_hz, num_input_channels, frame_, &float_cb_); + output_sample_rate_hz_ = output_sample_rate_hz; + num_output_channels_ = num_output_channels; + + SetContainerFormat(reverse_sample_rate_hz, num_reverse_channels, revframe_, + &revfloat_cb_); + Init(apm_.get()); + + if (far_file_) { + ASSERT_EQ(0, fclose(far_file_)); + } + std::string filename = ResourceFilePath("far", sample_rate_hz); + far_file_ = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(far_file_ != NULL) << "Could not open file " << + filename << "\n"; + + if (near_file_) { + ASSERT_EQ(0, fclose(near_file_)); + } + filename = ResourceFilePath("near", sample_rate_hz); + near_file_ = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(near_file_ != NULL) << "Could not open file " << + filename << "\n"; + + if (open_output_file) { + if (out_file_) { + ASSERT_EQ(0, fclose(out_file_)); + } + filename = OutputFilePath( + "out", sample_rate_hz, output_sample_rate_hz, reverse_sample_rate_hz, + reverse_sample_rate_hz, num_input_channels, num_output_channels, + num_reverse_channels, num_reverse_channels, kForward); + out_file_ = fopen(filename.c_str(), "wb"); + ASSERT_TRUE(out_file_ != NULL) << "Could not open file " << + filename << "\n"; + } +} + +void ApmTest::EnableAllComponents() { + EnableAllAPComponents(apm_.get()); +} + +bool ApmTest::ReadFrame(FILE* file, AudioFrame* frame, + ChannelBuffer* cb) { + // The files always contain stereo audio. + size_t frame_size = frame->samples_per_channel_ * 2; + size_t read_count = fread(frame->mutable_data(), + sizeof(int16_t), + frame_size, + file); + if (read_count != frame_size) { + // Check that the file really ended. + EXPECT_NE(0, feof(file)); + return false; // This is expected. + } + + if (frame->num_channels_ == 1) { + MixStereoToMono(frame->data(), frame->mutable_data(), + frame->samples_per_channel_); + } + + if (cb) { + ConvertToFloat(*frame, cb); + } + return true; +} + +bool ApmTest::ReadFrame(FILE* file, AudioFrame* frame) { + return ReadFrame(file, frame, NULL); +} + +// If the end of the file has been reached, rewind it and attempt to read the +// frame again. +void ApmTest::ReadFrameWithRewind(FILE* file, AudioFrame* frame, + ChannelBuffer* cb) { + if (!ReadFrame(near_file_, frame_, cb)) { + rewind(near_file_); + ASSERT_TRUE(ReadFrame(near_file_, frame_, cb)); + } +} + +void ApmTest::ReadFrameWithRewind(FILE* file, AudioFrame* frame) { + ReadFrameWithRewind(file, frame, NULL); +} + +void ApmTest::ProcessWithDefaultStreamParameters(AudioFrame* frame) { + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(0)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(127)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame)); +} + +int ApmTest::ProcessStreamChooser(Format format) { + if (format == kIntFormat) { + return apm_->ProcessStream(frame_); + } + return apm_->ProcessStream(float_cb_->channels(), + frame_->samples_per_channel_, + frame_->sample_rate_hz_, + LayoutFromChannels(frame_->num_channels_), + output_sample_rate_hz_, + LayoutFromChannels(num_output_channels_), + float_cb_->channels()); +} + +int ApmTest::AnalyzeReverseStreamChooser(Format format) { + if (format == kIntFormat) { + return apm_->ProcessReverseStream(revframe_); + } + return apm_->AnalyzeReverseStream( + revfloat_cb_->channels(), + revframe_->samples_per_channel_, + revframe_->sample_rate_hz_, + LayoutFromChannels(revframe_->num_channels_)); +} + +void ApmTest::ProcessDelayVerificationTest(int delay_ms, int system_delay_ms, + int delay_min, int delay_max) { + // The |revframe_| and |frame_| should include the proper frame information, + // hence can be used for extracting information. + AudioFrame tmp_frame; + std::queue frame_queue; + bool causal = true; + + tmp_frame.CopyFrom(*revframe_); + SetFrameTo(&tmp_frame, 0); + + EXPECT_EQ(apm_->kNoError, apm_->Initialize()); + // Initialize the |frame_queue| with empty frames. + int frame_delay = delay_ms / 10; + while (frame_delay < 0) { + AudioFrame* frame = new AudioFrame(); + frame->CopyFrom(tmp_frame); + frame_queue.push(frame); + frame_delay++; + causal = false; + } + while (frame_delay > 0) { + AudioFrame* frame = new AudioFrame(); + frame->CopyFrom(tmp_frame); + frame_queue.push(frame); + frame_delay--; + } + // Run for 4.5 seconds, skipping statistics from the first 2.5 seconds. We + // need enough frames with audio to have reliable estimates, but as few as + // possible to keep processing time down. 4.5 seconds seemed to be a good + // compromise for this recording. + for (int frame_count = 0; frame_count < 450; ++frame_count) { + AudioFrame* frame = new AudioFrame(); + frame->CopyFrom(tmp_frame); + // Use the near end recording, since that has more speech in it. + ASSERT_TRUE(ReadFrame(near_file_, frame)); + frame_queue.push(frame); + AudioFrame* reverse_frame = frame; + AudioFrame* process_frame = frame_queue.front(); + if (!causal) { + reverse_frame = frame_queue.front(); + // When we call ProcessStream() the frame is modified, so we can't use the + // pointer directly when things are non-causal. Use an intermediate frame + // and copy the data. + process_frame = &tmp_frame; + process_frame->CopyFrom(*frame); + } + EXPECT_EQ(apm_->kNoError, apm_->ProcessReverseStream(reverse_frame)); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(system_delay_ms)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(process_frame)); + frame = frame_queue.front(); + frame_queue.pop(); + delete frame; + + if (frame_count == 250) { + int median; + int std; + float poor_fraction; + // Discard the first delay metrics to avoid convergence effects. + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->GetDelayMetrics(&median, &std, + &poor_fraction)); + } + } + + rewind(near_file_); + while (!frame_queue.empty()) { + AudioFrame* frame = frame_queue.front(); + frame_queue.pop(); + delete frame; + } + // Calculate expected delay estimate and acceptable regions. Further, + // limit them w.r.t. AEC delay estimation support. + const size_t samples_per_ms = + rtc::SafeMin(16u, frame_->samples_per_channel_ / 10); + const int expected_median = + rtc::SafeClamp(delay_ms - system_delay_ms, delay_min, delay_max); + const int expected_median_high = rtc::SafeClamp( + expected_median + rtc::dchecked_cast(96 / samples_per_ms), delay_min, + delay_max); + const int expected_median_low = rtc::SafeClamp( + expected_median - rtc::dchecked_cast(96 / samples_per_ms), delay_min, + delay_max); + // Verify delay metrics. + int median; + int std; + float poor_fraction; + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->GetDelayMetrics(&median, &std, + &poor_fraction)); + EXPECT_GE(expected_median_high, median); + EXPECT_LE(expected_median_low, median); +} + +void ApmTest::StreamParametersTest(Format format) { + // No errors when the components are disabled. + EXPECT_EQ(apm_->kNoError, ProcessStreamChooser(format)); + + // -- Missing AGC level -- + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(true)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // Resets after successful ProcessStream(). + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(127)); + EXPECT_EQ(apm_->kNoError, ProcessStreamChooser(format)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // Other stream parameters set correctly. + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_drift_compensation(true)); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(100)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(false)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_drift_compensation(false)); + + // -- Missing delay -- + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + EXPECT_EQ(apm_->kNoError, ProcessStreamChooser(format)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // Resets after successful ProcessStream(). + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(100)); + EXPECT_EQ(apm_->kNoError, ProcessStreamChooser(format)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // Other stream parameters set correctly. + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(true)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_drift_compensation(true)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(127)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(false)); + + // -- Missing drift -- + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // Resets after successful ProcessStream(). + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(100)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, ProcessStreamChooser(format)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // Other stream parameters set correctly. + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(true)); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(100)); + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(127)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // -- No stream parameters -- + EXPECT_EQ(apm_->kNoError, + AnalyzeReverseStreamChooser(format)); + EXPECT_EQ(apm_->kStreamParameterNotSetError, + ProcessStreamChooser(format)); + + // -- All there -- + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(100)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(127)); + EXPECT_EQ(apm_->kNoError, ProcessStreamChooser(format)); +} + +TEST_F(ApmTest, StreamParametersInt) { + StreamParametersTest(kIntFormat); +} + +TEST_F(ApmTest, StreamParametersFloat) { + StreamParametersTest(kFloatFormat); +} + +TEST_F(ApmTest, DefaultDelayOffsetIsZero) { + EXPECT_EQ(0, apm_->delay_offset_ms()); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(50)); + EXPECT_EQ(50, apm_->stream_delay_ms()); +} + +TEST_F(ApmTest, DelayOffsetWithLimitsIsSetProperly) { + // High limit of 500 ms. + apm_->set_delay_offset_ms(100); + EXPECT_EQ(100, apm_->delay_offset_ms()); + EXPECT_EQ(apm_->kBadStreamParameterWarning, apm_->set_stream_delay_ms(450)); + EXPECT_EQ(500, apm_->stream_delay_ms()); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(100)); + EXPECT_EQ(200, apm_->stream_delay_ms()); + + // Low limit of 0 ms. + apm_->set_delay_offset_ms(-50); + EXPECT_EQ(-50, apm_->delay_offset_ms()); + EXPECT_EQ(apm_->kBadStreamParameterWarning, apm_->set_stream_delay_ms(20)); + EXPECT_EQ(0, apm_->stream_delay_ms()); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(100)); + EXPECT_EQ(50, apm_->stream_delay_ms()); +} + +void ApmTest::TestChangingChannelsInt16Interface( + size_t num_channels, + AudioProcessing::Error expected_return) { + frame_->num_channels_ = num_channels; + EXPECT_EQ(expected_return, apm_->ProcessStream(frame_)); + EXPECT_EQ(expected_return, apm_->ProcessReverseStream(frame_)); +} + +void ApmTest::TestChangingForwardChannels( + size_t num_in_channels, + size_t num_out_channels, + AudioProcessing::Error expected_return) { + const StreamConfig input_stream = {frame_->sample_rate_hz_, num_in_channels}; + const StreamConfig output_stream = {output_sample_rate_hz_, num_out_channels}; + + EXPECT_EQ(expected_return, + apm_->ProcessStream(float_cb_->channels(), input_stream, + output_stream, float_cb_->channels())); +} + +void ApmTest::TestChangingReverseChannels( + size_t num_rev_channels, + AudioProcessing::Error expected_return) { + const ProcessingConfig processing_config = { + {{frame_->sample_rate_hz_, apm_->num_input_channels()}, + {output_sample_rate_hz_, apm_->num_output_channels()}, + {frame_->sample_rate_hz_, num_rev_channels}, + {frame_->sample_rate_hz_, num_rev_channels}}}; + + EXPECT_EQ( + expected_return, + apm_->ProcessReverseStream( + float_cb_->channels(), processing_config.reverse_input_stream(), + processing_config.reverse_output_stream(), float_cb_->channels())); +} + +TEST_F(ApmTest, ChannelsInt16Interface) { + // Testing number of invalid and valid channels. + Init(16000, 16000, 16000, 4, 4, 4, false); + + TestChangingChannelsInt16Interface(0, apm_->kBadNumberChannelsError); + + for (size_t i = 1; i < 4; i++) { + TestChangingChannelsInt16Interface(i, kNoErr); + EXPECT_EQ(i, apm_->num_input_channels()); + } +} + +TEST_F(ApmTest, Channels) { + // Testing number of invalid and valid channels. + Init(16000, 16000, 16000, 4, 4, 4, false); + + TestChangingForwardChannels(0, 1, apm_->kBadNumberChannelsError); + TestChangingReverseChannels(0, apm_->kBadNumberChannelsError); + + for (size_t i = 1; i < 4; ++i) { + for (size_t j = 0; j < 1; ++j) { + // Output channels much be one or match input channels. + if (j == 1 || i == j) { + TestChangingForwardChannels(i, j, kNoErr); + TestChangingReverseChannels(i, kNoErr); + + EXPECT_EQ(i, apm_->num_input_channels()); + EXPECT_EQ(j, apm_->num_output_channels()); + // The number of reverse channels used for processing to is always 1. + EXPECT_EQ(1u, apm_->num_reverse_channels()); + } else { + TestChangingForwardChannels(i, j, + AudioProcessing::kBadNumberChannelsError); + } + } + } +} + +TEST_F(ApmTest, SampleRatesInt) { + // Testing invalid sample rates + SetContainerFormat(10000, 2, frame_, &float_cb_); + EXPECT_EQ(apm_->kBadSampleRateError, ProcessStreamChooser(kIntFormat)); + // Testing valid sample rates + int fs[] = {8000, 16000, 32000, 48000}; + for (size_t i = 0; i < arraysize(fs); i++) { + SetContainerFormat(fs[i], 2, frame_, &float_cb_); + EXPECT_NOERR(ProcessStreamChooser(kIntFormat)); + } +} + +TEST_F(ApmTest, EchoCancellation) { + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_drift_compensation(true)); + EXPECT_TRUE(apm_->echo_cancellation()->is_drift_compensation_enabled()); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_drift_compensation(false)); + EXPECT_FALSE(apm_->echo_cancellation()->is_drift_compensation_enabled()); + + EchoCancellation::SuppressionLevel level[] = { + EchoCancellation::kLowSuppression, + EchoCancellation::kModerateSuppression, + EchoCancellation::kHighSuppression, + }; + for (size_t i = 0; i < arraysize(level); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->set_suppression_level(level[i])); + EXPECT_EQ(level[i], + apm_->echo_cancellation()->suppression_level()); + } + + EchoCancellation::Metrics metrics; + EXPECT_EQ(apm_->kNotEnabledError, + apm_->echo_cancellation()->GetMetrics(&metrics)); + + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + EXPECT_TRUE(apm_->echo_cancellation()->is_enabled()); + + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_metrics(true)); + EXPECT_TRUE(apm_->echo_cancellation()->are_metrics_enabled()); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_metrics(false)); + EXPECT_FALSE(apm_->echo_cancellation()->are_metrics_enabled()); + + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_delay_logging(true)); + EXPECT_TRUE(apm_->echo_cancellation()->is_delay_logging_enabled()); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_delay_logging(false)); + EXPECT_FALSE(apm_->echo_cancellation()->is_delay_logging_enabled()); + + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(false)); + EXPECT_FALSE(apm_->echo_cancellation()->is_enabled()); + + int median = 0; + int std = 0; + float poor_fraction = 0; + EXPECT_EQ(apm_->kNotEnabledError, apm_->echo_cancellation()->GetDelayMetrics( + &median, &std, &poor_fraction)); + + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + EXPECT_TRUE(apm_->echo_cancellation()->is_enabled()); + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(false)); + EXPECT_FALSE(apm_->echo_cancellation()->is_enabled()); + + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + EXPECT_TRUE(apm_->echo_cancellation()->is_enabled()); + EXPECT_TRUE(apm_->echo_cancellation()->aec_core() != NULL); + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(false)); + EXPECT_FALSE(apm_->echo_cancellation()->is_enabled()); + EXPECT_FALSE(apm_->echo_cancellation()->aec_core() != NULL); +} + +TEST_F(ApmTest, DISABLED_EchoCancellationReportsCorrectDelays) { + // TODO(bjornv): Fix this test to work with DA-AEC. + // Enable AEC only. + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_drift_compensation(false)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_metrics(false)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_delay_logging(true)); + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + Config config; + config.Set(new DelayAgnostic(false)); + apm_->SetExtraOptions(config); + + // Internally in the AEC the amount of lookahead the delay estimation can + // handle is 15 blocks and the maximum delay is set to 60 blocks. + const int kLookaheadBlocks = 15; + const int kMaxDelayBlocks = 60; + // The AEC has a startup time before it actually starts to process. This + // procedure can flush the internal far-end buffer, which of course affects + // the delay estimation. Therefore, we set a system_delay high enough to + // avoid that. The smallest system_delay you can report without flushing the + // buffer is 66 ms in 8 kHz. + // + // It is known that for 16 kHz (and 32 kHz) sampling frequency there is an + // additional stuffing of 8 ms on the fly, but it seems to have no impact on + // delay estimation. This should be noted though. In case of test failure, + // this could be the cause. + const int kSystemDelayMs = 66; + // Test a couple of corner cases and verify that the estimated delay is + // within a valid region (set to +-1.5 blocks). Note that these cases are + // sampling frequency dependent. + for (size_t i = 0; i < arraysize(kProcessSampleRates); i++) { + Init(kProcessSampleRates[i], + kProcessSampleRates[i], + kProcessSampleRates[i], + 2, + 2, + 2, + false); + // Sampling frequency dependent variables. + const int num_ms_per_block = + std::max(4, static_cast(640 / frame_->samples_per_channel_)); + const int delay_min_ms = -kLookaheadBlocks * num_ms_per_block; + const int delay_max_ms = (kMaxDelayBlocks - 1) * num_ms_per_block; + + // 1) Verify correct delay estimate at lookahead boundary. + int delay_ms = TruncateToMultipleOf10(kSystemDelayMs + delay_min_ms); + ProcessDelayVerificationTest(delay_ms, kSystemDelayMs, delay_min_ms, + delay_max_ms); + // 2) A delay less than maximum lookahead should give an delay estimate at + // the boundary (= -kLookaheadBlocks * num_ms_per_block). + delay_ms -= 20; + ProcessDelayVerificationTest(delay_ms, kSystemDelayMs, delay_min_ms, + delay_max_ms); + // 3) Three values around zero delay. Note that we need to compensate for + // the fake system_delay. + delay_ms = TruncateToMultipleOf10(kSystemDelayMs - 10); + ProcessDelayVerificationTest(delay_ms, kSystemDelayMs, delay_min_ms, + delay_max_ms); + delay_ms = TruncateToMultipleOf10(kSystemDelayMs); + ProcessDelayVerificationTest(delay_ms, kSystemDelayMs, delay_min_ms, + delay_max_ms); + delay_ms = TruncateToMultipleOf10(kSystemDelayMs + 10); + ProcessDelayVerificationTest(delay_ms, kSystemDelayMs, delay_min_ms, + delay_max_ms); + // 4) Verify correct delay estimate at maximum delay boundary. + delay_ms = TruncateToMultipleOf10(kSystemDelayMs + delay_max_ms); + ProcessDelayVerificationTest(delay_ms, kSystemDelayMs, delay_min_ms, + delay_max_ms); + // 5) A delay above the maximum delay should give an estimate at the + // boundary (= (kMaxDelayBlocks - 1) * num_ms_per_block). + delay_ms += 20; + ProcessDelayVerificationTest(delay_ms, kSystemDelayMs, delay_min_ms, + delay_max_ms); + } +} + +TEST_F(ApmTest, EchoControlMobile) { + // Turn AECM on (and AEC off) + Init(16000, 16000, 16000, 2, 2, 2, false); + EXPECT_EQ(apm_->kNoError, apm_->echo_control_mobile()->Enable(true)); + EXPECT_TRUE(apm_->echo_control_mobile()->is_enabled()); + + // Toggle routing modes + EchoControlMobile::RoutingMode mode[] = { + EchoControlMobile::kQuietEarpieceOrHeadset, + EchoControlMobile::kEarpiece, + EchoControlMobile::kLoudEarpiece, + EchoControlMobile::kSpeakerphone, + EchoControlMobile::kLoudSpeakerphone, + }; + for (size_t i = 0; i < arraysize(mode); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->echo_control_mobile()->set_routing_mode(mode[i])); + EXPECT_EQ(mode[i], + apm_->echo_control_mobile()->routing_mode()); + } + // Turn comfort noise off/on + EXPECT_EQ(apm_->kNoError, + apm_->echo_control_mobile()->enable_comfort_noise(false)); + EXPECT_FALSE(apm_->echo_control_mobile()->is_comfort_noise_enabled()); + EXPECT_EQ(apm_->kNoError, + apm_->echo_control_mobile()->enable_comfort_noise(true)); + EXPECT_TRUE(apm_->echo_control_mobile()->is_comfort_noise_enabled()); + // Set and get echo path + const size_t echo_path_size = + apm_->echo_control_mobile()->echo_path_size_bytes(); + std::unique_ptr echo_path_in(new char[echo_path_size]); + std::unique_ptr echo_path_out(new char[echo_path_size]); + EXPECT_EQ(apm_->kNullPointerError, + apm_->echo_control_mobile()->SetEchoPath(NULL, echo_path_size)); + EXPECT_EQ(apm_->kNullPointerError, + apm_->echo_control_mobile()->GetEchoPath(NULL, echo_path_size)); + EXPECT_EQ(apm_->kBadParameterError, + apm_->echo_control_mobile()->GetEchoPath(echo_path_out.get(), 1)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_control_mobile()->GetEchoPath(echo_path_out.get(), + echo_path_size)); + for (size_t i = 0; i < echo_path_size; i++) { + echo_path_in[i] = echo_path_out[i] + 1; + } + EXPECT_EQ(apm_->kBadParameterError, + apm_->echo_control_mobile()->SetEchoPath(echo_path_in.get(), 1)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_control_mobile()->SetEchoPath(echo_path_in.get(), + echo_path_size)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_control_mobile()->GetEchoPath(echo_path_out.get(), + echo_path_size)); + for (size_t i = 0; i < echo_path_size; i++) { + EXPECT_EQ(echo_path_in[i], echo_path_out[i]); + } + + // Process a few frames with NS in the default disabled state. This exercises + // a different codepath than with it enabled. + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(0)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(0)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + + // Turn AECM off + EXPECT_EQ(apm_->kNoError, apm_->echo_control_mobile()->Enable(false)); + EXPECT_FALSE(apm_->echo_control_mobile()->is_enabled()); +} + +TEST_F(ApmTest, GainControl) { + // Testing gain modes + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_mode( + apm_->gain_control()->mode())); + + GainControl::Mode mode[] = { + GainControl::kAdaptiveAnalog, + GainControl::kAdaptiveDigital, + GainControl::kFixedDigital + }; + for (size_t i = 0; i < arraysize(mode); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_mode(mode[i])); + EXPECT_EQ(mode[i], apm_->gain_control()->mode()); + } + // Testing invalid target levels + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_target_level_dbfs(-3)); + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_target_level_dbfs(-40)); + // Testing valid target levels + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_target_level_dbfs( + apm_->gain_control()->target_level_dbfs())); + + int level_dbfs[] = {0, 6, 31}; + for (size_t i = 0; i < arraysize(level_dbfs); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_target_level_dbfs(level_dbfs[i])); + EXPECT_EQ(level_dbfs[i], apm_->gain_control()->target_level_dbfs()); + } + + // Testing invalid compression gains + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_compression_gain_db(-1)); + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_compression_gain_db(100)); + + // Testing valid compression gains + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_compression_gain_db( + apm_->gain_control()->compression_gain_db())); + + int gain_db[] = {0, 10, 90}; + for (size_t i = 0; i < arraysize(gain_db); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_compression_gain_db(gain_db[i])); + EXPECT_EQ(gain_db[i], apm_->gain_control()->compression_gain_db()); + } + + // Testing limiter off/on + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->enable_limiter(false)); + EXPECT_FALSE(apm_->gain_control()->is_limiter_enabled()); + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->enable_limiter(true)); + EXPECT_TRUE(apm_->gain_control()->is_limiter_enabled()); + + // Testing invalid level limits + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_analog_level_limits(-1, 512)); + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_analog_level_limits(100000, 512)); + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_analog_level_limits(512, -1)); + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_analog_level_limits(512, 100000)); + EXPECT_EQ(apm_->kBadParameterError, + apm_->gain_control()->set_analog_level_limits(512, 255)); + + // Testing valid level limits + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_analog_level_limits( + apm_->gain_control()->analog_level_minimum(), + apm_->gain_control()->analog_level_maximum())); + + int min_level[] = {0, 255, 1024}; + for (size_t i = 0; i < arraysize(min_level); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_analog_level_limits(min_level[i], 1024)); + EXPECT_EQ(min_level[i], apm_->gain_control()->analog_level_minimum()); + } + + int max_level[] = {0, 1024, 65535}; + for (size_t i = 0; i < arraysize(min_level); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_analog_level_limits(0, max_level[i])); + EXPECT_EQ(max_level[i], apm_->gain_control()->analog_level_maximum()); + } + + // TODO(ajm): stream_is_saturated() and stream_analog_level() + + // Turn AGC off + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(false)); + EXPECT_FALSE(apm_->gain_control()->is_enabled()); +} + +void ApmTest::RunQuantizedVolumeDoesNotGetStuckTest(int sample_rate) { + Init(sample_rate, sample_rate, sample_rate, 2, 2, 2, false); + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_mode(GainControl::kAdaptiveAnalog)); + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(true)); + + int out_analog_level = 0; + for (int i = 0; i < 2000; ++i) { + ReadFrameWithRewind(near_file_, frame_); + // Ensure the audio is at a low level, so the AGC will try to increase it. + ScaleFrame(frame_, 0.25); + + // Always pass in the same volume. + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(100)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + out_analog_level = apm_->gain_control()->stream_analog_level(); + } + + // Ensure the AGC is still able to reach the maximum. + EXPECT_EQ(255, out_analog_level); +} + +// Verifies that despite volume slider quantization, the AGC can continue to +// increase its volume. +TEST_F(ApmTest, QuantizedVolumeDoesNotGetStuck) { + for (size_t i = 0; i < arraysize(kSampleRates); ++i) { + RunQuantizedVolumeDoesNotGetStuckTest(kSampleRates[i]); + } +} + +void ApmTest::RunManualVolumeChangeIsPossibleTest(int sample_rate) { + Init(sample_rate, sample_rate, sample_rate, 2, 2, 2, false); + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_mode(GainControl::kAdaptiveAnalog)); + EXPECT_EQ(apm_->kNoError, apm_->gain_control()->Enable(true)); + + int out_analog_level = 100; + for (int i = 0; i < 1000; ++i) { + ReadFrameWithRewind(near_file_, frame_); + // Ensure the audio is at a low level, so the AGC will try to increase it. + ScaleFrame(frame_, 0.25); + + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(out_analog_level)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + out_analog_level = apm_->gain_control()->stream_analog_level(); + } + + // Ensure the volume was raised. + EXPECT_GT(out_analog_level, 100); + int highest_level_reached = out_analog_level; + // Simulate a user manual volume change. + out_analog_level = 100; + + for (int i = 0; i < 300; ++i) { + ReadFrameWithRewind(near_file_, frame_); + ScaleFrame(frame_, 0.25); + + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(out_analog_level)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + out_analog_level = apm_->gain_control()->stream_analog_level(); + // Check that AGC respected the manually adjusted volume. + EXPECT_LT(out_analog_level, highest_level_reached); + } + // Check that the volume was still raised. + EXPECT_GT(out_analog_level, 100); +} + +TEST_F(ApmTest, ManualVolumeChangeIsPossible) { + for (size_t i = 0; i < arraysize(kSampleRates); ++i) { + RunManualVolumeChangeIsPossibleTest(kSampleRates[i]); + } +} + +#if !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) +TEST_F(ApmTest, AgcOnlyAdaptsWhenTargetSignalIsPresent) { + const int kSampleRateHz = 16000; + const size_t kSamplesPerChannel = + static_cast(AudioProcessing::kChunkSizeMs * kSampleRateHz / 1000); + const size_t kNumInputChannels = 2; + const size_t kNumOutputChannels = 1; + const size_t kNumChunks = 700; + const float kScaleFactor = 0.25f; + Config config; + std::vector geometry; + geometry.push_back(webrtc::Point(0.f, 0.f, 0.f)); + geometry.push_back(webrtc::Point(0.05f, 0.f, 0.f)); + config.Set(new Beamforming(true, geometry)); + testing::NiceMock* beamformer = + new testing::NiceMock(geometry, 1u); + std::unique_ptr apm( + AudioProcessing::Create(config, nullptr, nullptr, beamformer)); + EXPECT_EQ(kNoErr, apm->gain_control()->Enable(true)); + ChannelBuffer src_buf(kSamplesPerChannel, kNumInputChannels); + ChannelBuffer dest_buf(kSamplesPerChannel, kNumOutputChannels); + const size_t max_length = kSamplesPerChannel * std::max(kNumInputChannels, + kNumOutputChannels); + std::unique_ptr int_data(new int16_t[max_length]); + std::unique_ptr float_data(new float[max_length]); + std::string filename = ResourceFilePath("far", kSampleRateHz); + FILE* far_file = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(far_file != NULL) << "Could not open file " << filename << "\n"; + const int kDefaultVolume = apm->gain_control()->stream_analog_level(); + const int kDefaultCompressionGain = + apm->gain_control()->compression_gain_db(); + bool is_target = false; + EXPECT_CALL(*beamformer, is_target_present()) + .WillRepeatedly(testing::ReturnPointee(&is_target)); + for (size_t i = 0; i < kNumChunks; ++i) { + ASSERT_TRUE(ReadChunk(far_file, + int_data.get(), + float_data.get(), + &src_buf)); + for (size_t j = 0; j < kNumInputChannels; ++j) { + for (size_t k = 0; k < kSamplesPerChannel; ++k) { + src_buf.channels()[j][k] *= kScaleFactor; + } + } + EXPECT_EQ(kNoErr, + apm->ProcessStream(src_buf.channels(), + src_buf.num_frames(), + kSampleRateHz, + LayoutFromChannels(src_buf.num_channels()), + kSampleRateHz, + LayoutFromChannels(dest_buf.num_channels()), + dest_buf.channels())); + } + EXPECT_EQ(kDefaultVolume, + apm->gain_control()->stream_analog_level()); + EXPECT_EQ(kDefaultCompressionGain, + apm->gain_control()->compression_gain_db()); + rewind(far_file); + is_target = true; + for (size_t i = 0; i < kNumChunks; ++i) { + ASSERT_TRUE(ReadChunk(far_file, + int_data.get(), + float_data.get(), + &src_buf)); + for (size_t j = 0; j < kNumInputChannels; ++j) { + for (size_t k = 0; k < kSamplesPerChannel; ++k) { + src_buf.channels()[j][k] *= kScaleFactor; + } + } + EXPECT_EQ(kNoErr, + apm->ProcessStream(src_buf.channels(), + src_buf.num_frames(), + kSampleRateHz, + LayoutFromChannels(src_buf.num_channels()), + kSampleRateHz, + LayoutFromChannels(dest_buf.num_channels()), + dest_buf.channels())); + } + EXPECT_LT(kDefaultVolume, + apm->gain_control()->stream_analog_level()); + EXPECT_LT(kDefaultCompressionGain, + apm->gain_control()->compression_gain_db()); + ASSERT_EQ(0, fclose(far_file)); +} +#endif + +TEST_F(ApmTest, NoiseSuppression) { + // Test valid suppression levels. + NoiseSuppression::Level level[] = { + NoiseSuppression::kLow, + NoiseSuppression::kModerate, + NoiseSuppression::kHigh, + NoiseSuppression::kVeryHigh + }; + for (size_t i = 0; i < arraysize(level); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->noise_suppression()->set_level(level[i])); + EXPECT_EQ(level[i], apm_->noise_suppression()->level()); + } + + // Turn NS on/off + EXPECT_EQ(apm_->kNoError, apm_->noise_suppression()->Enable(true)); + EXPECT_TRUE(apm_->noise_suppression()->is_enabled()); + EXPECT_EQ(apm_->kNoError, apm_->noise_suppression()->Enable(false)); + EXPECT_FALSE(apm_->noise_suppression()->is_enabled()); +} + +TEST_F(ApmTest, HighPassFilter) { + // Turn HP filter on/off + AudioProcessing::Config apm_config; + apm_config.high_pass_filter.enabled = true; + apm_->ApplyConfig(apm_config); + apm_config.high_pass_filter.enabled = false; + apm_->ApplyConfig(apm_config); +} + +TEST_F(ApmTest, LevelEstimator) { + // Turn level estimator on/off + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(false)); + EXPECT_FALSE(apm_->level_estimator()->is_enabled()); + + EXPECT_EQ(apm_->kNotEnabledError, apm_->level_estimator()->RMS()); + + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(true)); + EXPECT_TRUE(apm_->level_estimator()->is_enabled()); + + // Run this test in wideband; in super-wb, the splitting filter distorts the + // audio enough to cause deviation from the expectation for small values. + frame_->samples_per_channel_ = 160; + frame_->num_channels_ = 2; + frame_->sample_rate_hz_ = 16000; + + // Min value if no frames have been processed. + EXPECT_EQ(127, apm_->level_estimator()->RMS()); + + // Min value on zero frames. + SetFrameTo(frame_, 0); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(127, apm_->level_estimator()->RMS()); + + // Try a few RMS values. + // (These also test that the value resets after retrieving it.) + SetFrameTo(frame_, 32767); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(0, apm_->level_estimator()->RMS()); + + SetFrameTo(frame_, 30000); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(1, apm_->level_estimator()->RMS()); + + SetFrameTo(frame_, 10000); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(10, apm_->level_estimator()->RMS()); + + SetFrameTo(frame_, 10); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(70, apm_->level_estimator()->RMS()); + + // Verify reset after enable/disable. + SetFrameTo(frame_, 32767); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(false)); + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(true)); + SetFrameTo(frame_, 1); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(90, apm_->level_estimator()->RMS()); + + // Verify reset after initialize. + SetFrameTo(frame_, 32767); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->Initialize()); + SetFrameTo(frame_, 1); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(90, apm_->level_estimator()->RMS()); +} + +TEST_F(ApmTest, VoiceDetection) { + // Test external VAD + EXPECT_EQ(apm_->kNoError, + apm_->voice_detection()->set_stream_has_voice(true)); + EXPECT_TRUE(apm_->voice_detection()->stream_has_voice()); + EXPECT_EQ(apm_->kNoError, + apm_->voice_detection()->set_stream_has_voice(false)); + EXPECT_FALSE(apm_->voice_detection()->stream_has_voice()); + + // Test valid likelihoods + VoiceDetection::Likelihood likelihood[] = { + VoiceDetection::kVeryLowLikelihood, + VoiceDetection::kLowLikelihood, + VoiceDetection::kModerateLikelihood, + VoiceDetection::kHighLikelihood + }; + for (size_t i = 0; i < arraysize(likelihood); i++) { + EXPECT_EQ(apm_->kNoError, + apm_->voice_detection()->set_likelihood(likelihood[i])); + EXPECT_EQ(likelihood[i], apm_->voice_detection()->likelihood()); + } + + /* TODO(bjornv): Enable once VAD supports other frame lengths than 10 ms + // Test invalid frame sizes + EXPECT_EQ(apm_->kBadParameterError, + apm_->voice_detection()->set_frame_size_ms(12)); + + // Test valid frame sizes + for (int i = 10; i <= 30; i += 10) { + EXPECT_EQ(apm_->kNoError, + apm_->voice_detection()->set_frame_size_ms(i)); + EXPECT_EQ(i, apm_->voice_detection()->frame_size_ms()); + } + */ + + // Turn VAD on/off + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(true)); + EXPECT_TRUE(apm_->voice_detection()->is_enabled()); + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(false)); + EXPECT_FALSE(apm_->voice_detection()->is_enabled()); + + // Test that AudioFrame activity is maintained when VAD is disabled. + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(false)); + AudioFrame::VADActivity activity[] = { + AudioFrame::kVadActive, + AudioFrame::kVadPassive, + AudioFrame::kVadUnknown + }; + for (size_t i = 0; i < arraysize(activity); i++) { + frame_->vad_activity_ = activity[i]; + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(activity[i], frame_->vad_activity_); + } + + // Test that AudioFrame activity is set when VAD is enabled. + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(true)); + frame_->vad_activity_ = AudioFrame::kVadUnknown; + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_NE(AudioFrame::kVadUnknown, frame_->vad_activity_); + + // TODO(bjornv): Add tests for streamed voice; stream_has_voice() +} + +TEST_F(ApmTest, AllProcessingDisabledByDefault) { + EXPECT_FALSE(apm_->echo_cancellation()->is_enabled()); + EXPECT_FALSE(apm_->echo_control_mobile()->is_enabled()); + EXPECT_FALSE(apm_->gain_control()->is_enabled()); + EXPECT_FALSE(apm_->high_pass_filter()->is_enabled()); + EXPECT_FALSE(apm_->level_estimator()->is_enabled()); + EXPECT_FALSE(apm_->noise_suppression()->is_enabled()); + EXPECT_FALSE(apm_->voice_detection()->is_enabled()); +} + +TEST_F(ApmTest, NoProcessingWhenAllComponentsDisabled) { + for (size_t i = 0; i < arraysize(kSampleRates); i++) { + Init(kSampleRates[i], kSampleRates[i], kSampleRates[i], 2, 2, 2, false); + SetFrameTo(frame_, 1000, 2000); + AudioFrame frame_copy; + frame_copy.CopyFrom(*frame_); + for (int j = 0; j < 1000; j++) { + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_TRUE(FrameDataAreEqual(*frame_, frame_copy)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessReverseStream(frame_)); + EXPECT_TRUE(FrameDataAreEqual(*frame_, frame_copy)); + } + } +} + +TEST_F(ApmTest, NoProcessingWhenAllComponentsDisabledFloat) { + // Test that ProcessStream copies input to output even with no processing. + const size_t kSamples = 80; + const int sample_rate = 8000; + const float src[kSamples] = { + -1.0f, 0.0f, 1.0f + }; + float dest[kSamples] = {}; + + auto src_channels = &src[0]; + auto dest_channels = &dest[0]; + + apm_.reset(AudioProcessing::Create()); + EXPECT_NOERR(apm_->ProcessStream( + &src_channels, kSamples, sample_rate, LayoutFromChannels(1), + sample_rate, LayoutFromChannels(1), &dest_channels)); + + for (size_t i = 0; i < kSamples; ++i) { + EXPECT_EQ(src[i], dest[i]); + } + + // Same for ProcessReverseStream. + float rev_dest[kSamples] = {}; + auto rev_dest_channels = &rev_dest[0]; + + StreamConfig input_stream = {sample_rate, 1}; + StreamConfig output_stream = {sample_rate, 1}; + EXPECT_NOERR(apm_->ProcessReverseStream(&src_channels, input_stream, + output_stream, &rev_dest_channels)); + + for (size_t i = 0; i < kSamples; ++i) { + EXPECT_EQ(src[i], rev_dest[i]); + } +} + +TEST_F(ApmTest, IdenticalInputChannelsResultInIdenticalOutputChannels) { + EnableAllComponents(); + + for (size_t i = 0; i < arraysize(kProcessSampleRates); i++) { + Init(kProcessSampleRates[i], + kProcessSampleRates[i], + kProcessSampleRates[i], + 2, + 2, + 2, + false); + int analog_level = 127; + ASSERT_EQ(0, feof(far_file_)); + ASSERT_EQ(0, feof(near_file_)); + while (ReadFrame(far_file_, revframe_) && ReadFrame(near_file_, frame_)) { + CopyLeftToRightChannel(revframe_->mutable_data(), + revframe_->samples_per_channel_); + + ASSERT_EQ(kNoErr, apm_->ProcessReverseStream(revframe_)); + + CopyLeftToRightChannel(frame_->mutable_data(), + frame_->samples_per_channel_); + frame_->vad_activity_ = AudioFrame::kVadUnknown; + + ASSERT_EQ(kNoErr, apm_->set_stream_delay_ms(0)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + ASSERT_EQ(kNoErr, + apm_->gain_control()->set_stream_analog_level(analog_level)); + ASSERT_EQ(kNoErr, apm_->ProcessStream(frame_)); + analog_level = apm_->gain_control()->stream_analog_level(); + + VerifyChannelsAreEqual(frame_->data(), frame_->samples_per_channel_); + } + rewind(far_file_); + rewind(near_file_); + } +} + +TEST_F(ApmTest, SplittingFilter) { + // Verify the filter is not active through undistorted audio when: + // 1. No components are enabled... + SetFrameTo(frame_, 1000); + AudioFrame frame_copy; + frame_copy.CopyFrom(*frame_); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_TRUE(FrameDataAreEqual(*frame_, frame_copy)); + + // 2. Only the level estimator is enabled... + SetFrameTo(frame_, 1000); + frame_copy.CopyFrom(*frame_); + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(true)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_TRUE(FrameDataAreEqual(*frame_, frame_copy)); + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(false)); + + // 3. Only VAD is enabled... + SetFrameTo(frame_, 1000); + frame_copy.CopyFrom(*frame_); + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(true)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_TRUE(FrameDataAreEqual(*frame_, frame_copy)); + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(false)); + + // 4. Both VAD and the level estimator are enabled... + SetFrameTo(frame_, 1000); + frame_copy.CopyFrom(*frame_); + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(true)); + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(true)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_TRUE(FrameDataAreEqual(*frame_, frame_copy)); + EXPECT_EQ(apm_->kNoError, apm_->level_estimator()->Enable(false)); + EXPECT_EQ(apm_->kNoError, apm_->voice_detection()->Enable(false)); + + // 5. Not using super-wb. + frame_->samples_per_channel_ = 160; + frame_->num_channels_ = 2; + frame_->sample_rate_hz_ = 16000; + // Enable AEC, which would require the filter in super-wb. We rely on the + // first few frames of data being unaffected by the AEC. + // TODO(andrew): This test, and the one below, rely rather tenuously on the + // behavior of the AEC. Think of something more robust. + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); + // Make sure we have extended filter enabled. This makes sure nothing is + // touched until we have a farend frame. + Config config; + config.Set(new ExtendedFilter(true)); + apm_->SetExtraOptions(config); + SetFrameTo(frame_, 1000); + frame_copy.CopyFrom(*frame_); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(0)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(0)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_TRUE(FrameDataAreEqual(*frame_, frame_copy)); + + // Check the test is valid. We should have distortion from the filter + // when AEC is enabled (which won't affect the audio). + frame_->samples_per_channel_ = 320; + frame_->num_channels_ = 2; + frame_->sample_rate_hz_ = 32000; + SetFrameTo(frame_, 1000); + frame_copy.CopyFrom(*frame_); + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(0)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_FALSE(FrameDataAreEqual(*frame_, frame_copy)); +} + +#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP +void ApmTest::ProcessDebugDump(const std::string& in_filename, + const std::string& out_filename, + Format format, + int max_size_bytes) { + rtc::TaskQueue worker_queue("ApmTest_worker_queue"); + FILE* in_file = fopen(in_filename.c_str(), "rb"); + ASSERT_TRUE(in_file != NULL); + audioproc::Event event_msg; + bool first_init = true; + + while (ReadMessageFromFile(in_file, &event_msg)) { + if (event_msg.type() == audioproc::Event::INIT) { + const audioproc::Init msg = event_msg.init(); + int reverse_sample_rate = msg.sample_rate(); + if (msg.has_reverse_sample_rate()) { + reverse_sample_rate = msg.reverse_sample_rate(); + } + int output_sample_rate = msg.sample_rate(); + if (msg.has_output_sample_rate()) { + output_sample_rate = msg.output_sample_rate(); + } + + Init(msg.sample_rate(), + output_sample_rate, + reverse_sample_rate, + msg.num_input_channels(), + msg.num_output_channels(), + msg.num_reverse_channels(), + false); + if (first_init) { + // AttachAecDump() writes an additional init message. Don't start + // recording until after the first init to avoid the extra message. + auto aec_dump = + AecDumpFactory::Create(out_filename, max_size_bytes, &worker_queue); + EXPECT_TRUE(aec_dump); + apm_->AttachAecDump(std::move(aec_dump)); + first_init = false; + } + + } else if (event_msg.type() == audioproc::Event::REVERSE_STREAM) { + const audioproc::ReverseStream msg = event_msg.reverse_stream(); + + if (msg.channel_size() > 0) { + ASSERT_EQ(revframe_->num_channels_, + static_cast(msg.channel_size())); + for (int i = 0; i < msg.channel_size(); ++i) { + memcpy(revfloat_cb_->channels()[i], + msg.channel(i).data(), + msg.channel(i).size()); + } + } else { + memcpy(revframe_->mutable_data(), msg.data().data(), msg.data().size()); + if (format == kFloatFormat) { + // We're using an int16 input file; convert to float. + ConvertToFloat(*revframe_, revfloat_cb_.get()); + } + } + AnalyzeReverseStreamChooser(format); + + } else if (event_msg.type() == audioproc::Event::STREAM) { + const audioproc::Stream msg = event_msg.stream(); + // ProcessStream could have changed this for the output frame. + frame_->num_channels_ = apm_->num_input_channels(); + + EXPECT_NOERR(apm_->gain_control()->set_stream_analog_level(msg.level())); + EXPECT_NOERR(apm_->set_stream_delay_ms(msg.delay())); + apm_->echo_cancellation()->set_stream_drift_samples(msg.drift()); + if (msg.has_keypress()) { + apm_->set_stream_key_pressed(msg.keypress()); + } else { + apm_->set_stream_key_pressed(true); + } + + if (msg.input_channel_size() > 0) { + ASSERT_EQ(frame_->num_channels_, + static_cast(msg.input_channel_size())); + for (int i = 0; i < msg.input_channel_size(); ++i) { + memcpy(float_cb_->channels()[i], + msg.input_channel(i).data(), + msg.input_channel(i).size()); + } + } else { + memcpy(frame_->mutable_data(), msg.input_data().data(), + msg.input_data().size()); + if (format == kFloatFormat) { + // We're using an int16 input file; convert to float. + ConvertToFloat(*frame_, float_cb_.get()); + } + } + ProcessStreamChooser(format); + } + } + apm_->DetachAecDump(); + fclose(in_file); +} + +void ApmTest::VerifyDebugDumpTest(Format format) { + const std::string in_filename = test::ResourcePath("ref03", "aecdump"); + std::string format_string; + switch (format) { + case kIntFormat: + format_string = "_int"; + break; + case kFloatFormat: + format_string = "_float"; + break; + } + const std::string ref_filename = test::TempFilename( + test::OutputPath(), std::string("ref") + format_string + "_aecdump"); + const std::string out_filename = test::TempFilename( + test::OutputPath(), std::string("out") + format_string + "_aecdump"); + const std::string limited_filename = test::TempFilename( + test::OutputPath(), std::string("limited") + format_string + "_aecdump"); + const size_t logging_limit_bytes = 100000; + // We expect at least this many bytes in the created logfile. + const size_t logging_expected_bytes = 95000; + EnableAllComponents(); + ProcessDebugDump(in_filename, ref_filename, format, -1); + ProcessDebugDump(ref_filename, out_filename, format, -1); + ProcessDebugDump(ref_filename, limited_filename, format, logging_limit_bytes); + + FILE* ref_file = fopen(ref_filename.c_str(), "rb"); + FILE* out_file = fopen(out_filename.c_str(), "rb"); + FILE* limited_file = fopen(limited_filename.c_str(), "rb"); + ASSERT_TRUE(ref_file != NULL); + ASSERT_TRUE(out_file != NULL); + ASSERT_TRUE(limited_file != NULL); + std::unique_ptr ref_bytes; + std::unique_ptr out_bytes; + std::unique_ptr limited_bytes; + + size_t ref_size = ReadMessageBytesFromFile(ref_file, &ref_bytes); + size_t out_size = ReadMessageBytesFromFile(out_file, &out_bytes); + size_t limited_size = ReadMessageBytesFromFile(limited_file, &limited_bytes); + size_t bytes_read = 0; + size_t bytes_read_limited = 0; + while (ref_size > 0 && out_size > 0) { + bytes_read += ref_size; + bytes_read_limited += limited_size; + EXPECT_EQ(ref_size, out_size); + EXPECT_GE(ref_size, limited_size); + EXPECT_EQ(0, memcmp(ref_bytes.get(), out_bytes.get(), ref_size)); + EXPECT_EQ(0, memcmp(ref_bytes.get(), limited_bytes.get(), limited_size)); + ref_size = ReadMessageBytesFromFile(ref_file, &ref_bytes); + out_size = ReadMessageBytesFromFile(out_file, &out_bytes); + limited_size = ReadMessageBytesFromFile(limited_file, &limited_bytes); + } + EXPECT_GT(bytes_read, 0u); + EXPECT_GT(bytes_read_limited, logging_expected_bytes); + EXPECT_LE(bytes_read_limited, logging_limit_bytes); + EXPECT_NE(0, feof(ref_file)); + EXPECT_NE(0, feof(out_file)); + EXPECT_NE(0, feof(limited_file)); + ASSERT_EQ(0, fclose(ref_file)); + ASSERT_EQ(0, fclose(out_file)); + ASSERT_EQ(0, fclose(limited_file)); + remove(ref_filename.c_str()); + remove(out_filename.c_str()); + remove(limited_filename.c_str()); +} + +TEST_F(ApmTest, VerifyDebugDumpInt) { + VerifyDebugDumpTest(kIntFormat); +} + +TEST_F(ApmTest, VerifyDebugDumpFloat) { + VerifyDebugDumpTest(kFloatFormat); +} +#endif + +// TODO(andrew): expand test to verify output. +TEST_F(ApmTest, DebugDump) { + rtc::TaskQueue worker_queue("ApmTest_worker_queue"); + const std::string filename = + test::TempFilename(test::OutputPath(), "debug_aec"); + { + auto aec_dump = AecDumpFactory::Create("", -1, &worker_queue); + EXPECT_FALSE(aec_dump); + } + +#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP + // Stopping without having started should be OK. + apm_->DetachAecDump(); + + auto aec_dump = AecDumpFactory::Create(filename, -1, &worker_queue); + EXPECT_TRUE(aec_dump); + apm_->AttachAecDump(std::move(aec_dump)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessReverseStream(revframe_)); + apm_->DetachAecDump(); + + // Verify the file has been written. + FILE* fid = fopen(filename.c_str(), "r"); + ASSERT_TRUE(fid != NULL); + + // Clean it up. + ASSERT_EQ(0, fclose(fid)); + ASSERT_EQ(0, remove(filename.c_str())); +#else + // Verify the file has NOT been written. + ASSERT_TRUE(fopen(filename.c_str(), "r") == NULL); +#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP +} + +// TODO(andrew): expand test to verify output. +TEST_F(ApmTest, DebugDumpFromFileHandle) { + rtc::TaskQueue worker_queue("ApmTest_worker_queue"); + + const std::string filename = + test::TempFilename(test::OutputPath(), "debug_aec"); + FILE* fid = fopen(filename.c_str(), "w"); + ASSERT_TRUE(fid); + +#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP + // Stopping without having started should be OK. + apm_->DetachAecDump(); + + auto aec_dump = AecDumpFactory::Create(fid, -1, &worker_queue); + EXPECT_TRUE(aec_dump); + apm_->AttachAecDump(std::move(aec_dump)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessReverseStream(revframe_)); + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + apm_->DetachAecDump(); + + // Verify the file has been written. + fid = fopen(filename.c_str(), "r"); + ASSERT_TRUE(fid != NULL); + + // Clean it up. + ASSERT_EQ(0, fclose(fid)); + ASSERT_EQ(0, remove(filename.c_str())); +#else + ASSERT_EQ(0, fclose(fid)); +#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP +} + +TEST_F(ApmTest, FloatAndIntInterfacesGiveSimilarResults) { + audioproc::OutputData ref_data; + OpenFileAndReadMessage(ref_filename_, &ref_data); + + Config config; + config.Set(new ExperimentalAgc(false)); + std::unique_ptr fapm(AudioProcessing::Create(config)); + EnableAllComponents(); + EnableAllAPComponents(fapm.get()); + for (int i = 0; i < ref_data.test_size(); i++) { + printf("Running test %d of %d...\n", i + 1, ref_data.test_size()); + + audioproc::Test* test = ref_data.mutable_test(i); + // TODO(ajm): Restore downmixing test cases. + if (test->num_input_channels() != test->num_output_channels()) + continue; + + const size_t num_render_channels = + static_cast(test->num_reverse_channels()); + const size_t num_input_channels = + static_cast(test->num_input_channels()); + const size_t num_output_channels = + static_cast(test->num_output_channels()); + const size_t samples_per_channel = static_cast( + test->sample_rate() * AudioProcessing::kChunkSizeMs / 1000); + + Init(test->sample_rate(), test->sample_rate(), test->sample_rate(), + num_input_channels, num_output_channels, num_render_channels, true); + Init(fapm.get()); + + ChannelBuffer output_cb(samples_per_channel, num_input_channels); + ChannelBuffer output_int16(samples_per_channel, + num_input_channels); + + int analog_level = 127; + size_t num_bad_chunks = 0; + while (ReadFrame(far_file_, revframe_, revfloat_cb_.get()) && + ReadFrame(near_file_, frame_, float_cb_.get())) { + frame_->vad_activity_ = AudioFrame::kVadUnknown; + + EXPECT_NOERR(apm_->ProcessReverseStream(revframe_)); + EXPECT_NOERR(fapm->AnalyzeReverseStream( + revfloat_cb_->channels(), + samples_per_channel, + test->sample_rate(), + LayoutFromChannels(num_render_channels))); + + EXPECT_NOERR(apm_->set_stream_delay_ms(0)); + EXPECT_NOERR(fapm->set_stream_delay_ms(0)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + fapm->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_NOERR(apm_->gain_control()->set_stream_analog_level(analog_level)); + EXPECT_NOERR(fapm->gain_control()->set_stream_analog_level(analog_level)); + + EXPECT_NOERR(apm_->ProcessStream(frame_)); + Deinterleave(frame_->data(), samples_per_channel, num_output_channels, + output_int16.channels()); + + EXPECT_NOERR(fapm->ProcessStream( + float_cb_->channels(), + samples_per_channel, + test->sample_rate(), + LayoutFromChannels(num_input_channels), + test->sample_rate(), + LayoutFromChannels(num_output_channels), + float_cb_->channels())); + for (size_t j = 0; j < num_output_channels; ++j) { + FloatToS16(float_cb_->channels()[j], + samples_per_channel, + output_cb.channels()[j]); + float variance = 0; + float snr = ComputeSNR(output_int16.channels()[j], + output_cb.channels()[j], + samples_per_channel, &variance); + + const float kVarianceThreshold = 20; + const float kSNRThreshold = 20; + + // Skip frames with low energy. + if (sqrt(variance) > kVarianceThreshold && snr < kSNRThreshold) { + ++num_bad_chunks; + } + } + + analog_level = fapm->gain_control()->stream_analog_level(); + EXPECT_EQ(apm_->gain_control()->stream_analog_level(), + fapm->gain_control()->stream_analog_level()); + EXPECT_EQ(apm_->echo_cancellation()->stream_has_echo(), + fapm->echo_cancellation()->stream_has_echo()); + EXPECT_NEAR(apm_->noise_suppression()->speech_probability(), + fapm->noise_suppression()->speech_probability(), + 0.01); + + // Reset in case of downmixing. + frame_->num_channels_ = static_cast(test->num_input_channels()); + } + +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) + const size_t kMaxNumBadChunks = 0; +#elif defined(WEBRTC_AUDIOPROC_FIXED_PROFILE) + // There are a few chunks in the fixed-point profile that give low SNR. + // Listening confirmed the difference is acceptable. + const size_t kMaxNumBadChunks = 60; +#endif + EXPECT_LE(num_bad_chunks, kMaxNumBadChunks); + + rewind(far_file_); + rewind(near_file_); + } +} + +// TODO(andrew): Add a test to process a few frames with different combinations +// of enabled components. + +TEST_F(ApmTest, Process) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + audioproc::OutputData ref_data; + + if (!write_ref_data) { + OpenFileAndReadMessage(ref_filename_, &ref_data); + } else { + // Write the desired tests to the protobuf reference file. + for (size_t i = 0; i < arraysize(kChannels); i++) { + for (size_t j = 0; j < arraysize(kChannels); j++) { + for (size_t l = 0; l < arraysize(kProcessSampleRates); l++) { + audioproc::Test* test = ref_data.add_test(); + test->set_num_reverse_channels(kChannels[i]); + test->set_num_input_channels(kChannels[j]); + test->set_num_output_channels(kChannels[j]); + test->set_sample_rate(kProcessSampleRates[l]); + test->set_use_aec_extended_filter(false); + } + } + } +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) + // To test the extended filter mode. + audioproc::Test* test = ref_data.add_test(); + test->set_num_reverse_channels(2); + test->set_num_input_channels(2); + test->set_num_output_channels(2); + test->set_sample_rate(AudioProcessing::kSampleRate32kHz); + test->set_use_aec_extended_filter(true); +#endif + } + + for (int i = 0; i < ref_data.test_size(); i++) { + printf("Running test %d of %d...\n", i + 1, ref_data.test_size()); + + audioproc::Test* test = ref_data.mutable_test(i); + // TODO(ajm): We no longer allow different input and output channels. Skip + // these tests for now, but they should be removed from the set. + if (test->num_input_channels() != test->num_output_channels()) + continue; + + Config config; + config.Set(new ExperimentalAgc(false)); + config.Set( + new ExtendedFilter(test->use_aec_extended_filter())); + apm_.reset(AudioProcessing::Create(config)); + + EnableAllComponents(); + + Init(test->sample_rate(), + test->sample_rate(), + test->sample_rate(), + static_cast(test->num_input_channels()), + static_cast(test->num_output_channels()), + static_cast(test->num_reverse_channels()), + true); + + int frame_count = 0; + int has_echo_count = 0; + int has_voice_count = 0; + int is_saturated_count = 0; + int analog_level = 127; + int analog_level_average = 0; + int max_output_average = 0; + float ns_speech_prob_average = 0.0f; +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) + int stats_index = 0; +#endif + + while (ReadFrame(far_file_, revframe_) && ReadFrame(near_file_, frame_)) { + EXPECT_EQ(apm_->kNoError, apm_->ProcessReverseStream(revframe_)); + + frame_->vad_activity_ = AudioFrame::kVadUnknown; + + EXPECT_EQ(apm_->kNoError, apm_->set_stream_delay_ms(0)); + apm_->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_EQ(apm_->kNoError, + apm_->gain_control()->set_stream_analog_level(analog_level)); + + EXPECT_EQ(apm_->kNoError, apm_->ProcessStream(frame_)); + + // Ensure the frame was downmixed properly. + EXPECT_EQ(static_cast(test->num_output_channels()), + frame_->num_channels_); + + max_output_average += MaxAudioFrame(*frame_); + + if (apm_->echo_cancellation()->stream_has_echo()) { + has_echo_count++; + } + + analog_level = apm_->gain_control()->stream_analog_level(); + analog_level_average += analog_level; + if (apm_->gain_control()->stream_is_saturated()) { + is_saturated_count++; + } + if (apm_->voice_detection()->stream_has_voice()) { + has_voice_count++; + EXPECT_EQ(AudioFrame::kVadActive, frame_->vad_activity_); + } else { + EXPECT_EQ(AudioFrame::kVadPassive, frame_->vad_activity_); + } + + ns_speech_prob_average += apm_->noise_suppression()->speech_probability(); + + size_t frame_size = frame_->samples_per_channel_ * frame_->num_channels_; + size_t write_count = fwrite(frame_->data(), + sizeof(int16_t), + frame_size, + out_file_); + ASSERT_EQ(frame_size, write_count); + + // Reset in case of downmixing. + frame_->num_channels_ = static_cast(test->num_input_channels()); + frame_count++; + +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) + const int kStatsAggregationFrameNum = 100; // 1 second. + if (frame_count % kStatsAggregationFrameNum == 0) { + // Get echo metrics. + EchoCancellation::Metrics echo_metrics; + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->GetMetrics(&echo_metrics)); + + // Get delay metrics. + int median = 0; + int std = 0; + float fraction_poor_delays = 0; + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->GetDelayMetrics( + &median, &std, &fraction_poor_delays)); + + // Get RMS. + int rms_level = apm_->level_estimator()->RMS(); + EXPECT_LE(0, rms_level); + EXPECT_GE(127, rms_level); + + if (!write_ref_data) { + const audioproc::Test::EchoMetrics& reference = + test->echo_metrics(stats_index); + TestStats(echo_metrics.residual_echo_return_loss, + reference.residual_echo_return_loss()); + TestStats(echo_metrics.echo_return_loss, + reference.echo_return_loss()); + TestStats(echo_metrics.echo_return_loss_enhancement, + reference.echo_return_loss_enhancement()); + TestStats(echo_metrics.a_nlp, + reference.a_nlp()); + EXPECT_EQ(echo_metrics.divergent_filter_fraction, + reference.divergent_filter_fraction()); + + const audioproc::Test::DelayMetrics& reference_delay = + test->delay_metrics(stats_index); + EXPECT_EQ(reference_delay.median(), median); + EXPECT_EQ(reference_delay.std(), std); + EXPECT_EQ(reference_delay.fraction_poor_delays(), + fraction_poor_delays); + + EXPECT_EQ(test->rms_level(stats_index), rms_level); + + ++stats_index; + } else { + audioproc::Test::EchoMetrics* message = + test->add_echo_metrics(); + WriteStatsMessage(echo_metrics.residual_echo_return_loss, + message->mutable_residual_echo_return_loss()); + WriteStatsMessage(echo_metrics.echo_return_loss, + message->mutable_echo_return_loss()); + WriteStatsMessage(echo_metrics.echo_return_loss_enhancement, + message->mutable_echo_return_loss_enhancement()); + WriteStatsMessage(echo_metrics.a_nlp, + message->mutable_a_nlp()); + message->set_divergent_filter_fraction( + echo_metrics.divergent_filter_fraction); + + audioproc::Test::DelayMetrics* message_delay = + test->add_delay_metrics(); + message_delay->set_median(median); + message_delay->set_std(std); + message_delay->set_fraction_poor_delays(fraction_poor_delays); + + test->add_rms_level(rms_level); + } + } +#endif // defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE). + } + max_output_average /= frame_count; + analog_level_average /= frame_count; + ns_speech_prob_average /= frame_count; + + if (!write_ref_data) { + const int kIntNear = 1; + // When running the test on a N7 we get a {2, 6} difference of + // |has_voice_count| and |max_output_average| is up to 18 higher. + // All numbers being consistently higher on N7 compare to ref_data. + // TODO(bjornv): If we start getting more of these offsets on Android we + // should consider a different approach. Either using one slack for all, + // or generate a separate android reference. +#if defined(WEBRTC_ANDROID) + const int kHasVoiceCountOffset = 3; + const int kHasVoiceCountNear = 4; + const int kMaxOutputAverageOffset = 9; + const int kMaxOutputAverageNear = 9; +#else + const int kHasVoiceCountOffset = 0; + const int kHasVoiceCountNear = kIntNear; + const int kMaxOutputAverageOffset = 0; + const int kMaxOutputAverageNear = kIntNear; +#endif + EXPECT_NEAR(test->has_echo_count(), has_echo_count, kIntNear); + EXPECT_NEAR(test->has_voice_count(), + has_voice_count - kHasVoiceCountOffset, + kHasVoiceCountNear); + EXPECT_NEAR(test->is_saturated_count(), is_saturated_count, kIntNear); + + EXPECT_NEAR(test->analog_level_average(), analog_level_average, kIntNear); + EXPECT_NEAR(test->max_output_average(), + max_output_average - kMaxOutputAverageOffset, + kMaxOutputAverageNear); +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) + const double kFloatNear = 0.0005; + EXPECT_NEAR(test->ns_speech_probability_average(), + ns_speech_prob_average, + kFloatNear); +#endif + } else { + test->set_has_echo_count(has_echo_count); + test->set_has_voice_count(has_voice_count); + test->set_is_saturated_count(is_saturated_count); + + test->set_analog_level_average(analog_level_average); + test->set_max_output_average(max_output_average); + +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) + EXPECT_LE(0.0f, ns_speech_prob_average); + EXPECT_GE(1.0f, ns_speech_prob_average); + test->set_ns_speech_probability_average(ns_speech_prob_average); +#endif + } + + rewind(far_file_); + rewind(near_file_); + } + + if (write_ref_data) { + OpenFileAndWriteMessage(ref_filename_, ref_data); + } +} + +TEST_F(ApmTest, NoErrorsWithKeyboardChannel) { + struct ChannelFormat { + AudioProcessing::ChannelLayout in_layout; + AudioProcessing::ChannelLayout out_layout; + }; + ChannelFormat cf[] = { + {AudioProcessing::kMonoAndKeyboard, AudioProcessing::kMono}, + {AudioProcessing::kStereoAndKeyboard, AudioProcessing::kMono}, + {AudioProcessing::kStereoAndKeyboard, AudioProcessing::kStereo}, + }; + + std::unique_ptr ap(AudioProcessing::Create()); + // Enable one component just to ensure some processing takes place. + ap->noise_suppression()->Enable(true); + for (size_t i = 0; i < arraysize(cf); ++i) { + const int in_rate = 44100; + const int out_rate = 48000; + ChannelBuffer in_cb(SamplesFromRate(in_rate), + TotalChannelsFromLayout(cf[i].in_layout)); + ChannelBuffer out_cb(SamplesFromRate(out_rate), + ChannelsFromLayout(cf[i].out_layout)); + + // Run over a few chunks. + for (int j = 0; j < 10; ++j) { + EXPECT_NOERR(ap->ProcessStream( + in_cb.channels(), + in_cb.num_frames(), + in_rate, + cf[i].in_layout, + out_rate, + cf[i].out_layout, + out_cb.channels())); + } + } +} + +// Compares the reference and test arrays over a region around the expected +// delay. Finds the highest SNR in that region and adds the variance and squared +// error results to the supplied accumulators. +void UpdateBestSNR(const float* ref, + const float* test, + size_t length, + int expected_delay, + double* variance_acc, + double* sq_error_acc) { + double best_snr = std::numeric_limits::min(); + double best_variance = 0; + double best_sq_error = 0; + // Search over a region of eight samples around the expected delay. + for (int delay = std::max(expected_delay - 4, 0); delay <= expected_delay + 4; + ++delay) { + double sq_error = 0; + double variance = 0; + for (size_t i = 0; i < length - delay; ++i) { + double error = test[i + delay] - ref[i]; + sq_error += error * error; + variance += ref[i] * ref[i]; + } + + if (sq_error == 0) { + *variance_acc += variance; + return; + } + double snr = variance / sq_error; + if (snr > best_snr) { + best_snr = snr; + best_variance = variance; + best_sq_error = sq_error; + } + } + + *variance_acc += best_variance; + *sq_error_acc += best_sq_error; +} + +// Used to test a multitude of sample rate and channel combinations. It works +// by first producing a set of reference files (in SetUpTestCase) that are +// assumed to be correct, as the used parameters are verified by other tests +// in this collection. Primarily the reference files are all produced at +// "native" rates which do not involve any resampling. + +// Each test pass produces an output file with a particular format. The output +// is matched against the reference file closest to its internal processing +// format. If necessary the output is resampled back to its process format. +// Due to the resampling distortion, we don't expect identical results, but +// enforce SNR thresholds which vary depending on the format. 0 is a special +// case SNR which corresponds to inf, or zero error. +typedef std::tuple AudioProcessingTestData; +class AudioProcessingTest + : public testing::TestWithParam { + public: + AudioProcessingTest() + : input_rate_(std::get<0>(GetParam())), + output_rate_(std::get<1>(GetParam())), + reverse_input_rate_(std::get<2>(GetParam())), + reverse_output_rate_(std::get<3>(GetParam())), + expected_snr_(std::get<4>(GetParam())), + expected_reverse_snr_(std::get<5>(GetParam())) {} + + virtual ~AudioProcessingTest() {} + + static void SetUpTestCase() { + // Create all needed output reference files. + const int kNativeRates[] = {8000, 16000, 32000, 48000}; + const size_t kNumChannels[] = {1, 2}; + for (size_t i = 0; i < arraysize(kNativeRates); ++i) { + for (size_t j = 0; j < arraysize(kNumChannels); ++j) { + for (size_t k = 0; k < arraysize(kNumChannels); ++k) { + // The reference files always have matching input and output channels. + ProcessFormat(kNativeRates[i], kNativeRates[i], kNativeRates[i], + kNativeRates[i], kNumChannels[j], kNumChannels[j], + kNumChannels[k], kNumChannels[k], "ref"); + } + } + } + } + + void TearDown() { + // Remove "out" files after each test. + ClearTempOutFiles(); + } + + static void TearDownTestCase() { + ClearTempFiles(); + } + + // Runs a process pass on files with the given parameters and dumps the output + // to a file specified with |output_file_prefix|. Both forward and reverse + // output streams are dumped. + static void ProcessFormat(int input_rate, + int output_rate, + int reverse_input_rate, + int reverse_output_rate, + size_t num_input_channels, + size_t num_output_channels, + size_t num_reverse_input_channels, + size_t num_reverse_output_channels, + const std::string& output_file_prefix) { + Config config; + config.Set(new ExperimentalAgc(false)); + std::unique_ptr ap(AudioProcessing::Create(config)); + EnableAllAPComponents(ap.get()); + + ProcessingConfig processing_config = { + {{input_rate, num_input_channels}, + {output_rate, num_output_channels}, + {reverse_input_rate, num_reverse_input_channels}, + {reverse_output_rate, num_reverse_output_channels}}}; + ap->Initialize(processing_config); + + FILE* far_file = + fopen(ResourceFilePath("far", reverse_input_rate).c_str(), "rb"); + FILE* near_file = fopen(ResourceFilePath("near", input_rate).c_str(), "rb"); + FILE* out_file = + fopen(OutputFilePath(output_file_prefix, input_rate, output_rate, + reverse_input_rate, reverse_output_rate, + num_input_channels, num_output_channels, + num_reverse_input_channels, + num_reverse_output_channels, kForward).c_str(), + "wb"); + FILE* rev_out_file = + fopen(OutputFilePath(output_file_prefix, input_rate, output_rate, + reverse_input_rate, reverse_output_rate, + num_input_channels, num_output_channels, + num_reverse_input_channels, + num_reverse_output_channels, kReverse).c_str(), + "wb"); + ASSERT_TRUE(far_file != NULL); + ASSERT_TRUE(near_file != NULL); + ASSERT_TRUE(out_file != NULL); + ASSERT_TRUE(rev_out_file != NULL); + + ChannelBuffer fwd_cb(SamplesFromRate(input_rate), + num_input_channels); + ChannelBuffer rev_cb(SamplesFromRate(reverse_input_rate), + num_reverse_input_channels); + ChannelBuffer out_cb(SamplesFromRate(output_rate), + num_output_channels); + ChannelBuffer rev_out_cb(SamplesFromRate(reverse_output_rate), + num_reverse_output_channels); + + // Temporary buffers. + const int max_length = + 2 * std::max(std::max(out_cb.num_frames(), rev_out_cb.num_frames()), + std::max(fwd_cb.num_frames(), rev_cb.num_frames())); + std::unique_ptr float_data(new float[max_length]); + std::unique_ptr int_data(new int16_t[max_length]); + + int analog_level = 127; + while (ReadChunk(far_file, int_data.get(), float_data.get(), &rev_cb) && + ReadChunk(near_file, int_data.get(), float_data.get(), &fwd_cb)) { + EXPECT_NOERR(ap->ProcessReverseStream( + rev_cb.channels(), processing_config.reverse_input_stream(), + processing_config.reverse_output_stream(), rev_out_cb.channels())); + + EXPECT_NOERR(ap->set_stream_delay_ms(0)); + ap->echo_cancellation()->set_stream_drift_samples(0); + EXPECT_NOERR(ap->gain_control()->set_stream_analog_level(analog_level)); + + EXPECT_NOERR(ap->ProcessStream( + fwd_cb.channels(), + fwd_cb.num_frames(), + input_rate, + LayoutFromChannels(num_input_channels), + output_rate, + LayoutFromChannels(num_output_channels), + out_cb.channels())); + + // Dump forward output to file. + Interleave(out_cb.channels(), out_cb.num_frames(), out_cb.num_channels(), + float_data.get()); + size_t out_length = out_cb.num_channels() * out_cb.num_frames(); + + ASSERT_EQ(out_length, + fwrite(float_data.get(), sizeof(float_data[0]), + out_length, out_file)); + + // Dump reverse output to file. + Interleave(rev_out_cb.channels(), rev_out_cb.num_frames(), + rev_out_cb.num_channels(), float_data.get()); + size_t rev_out_length = + rev_out_cb.num_channels() * rev_out_cb.num_frames(); + + ASSERT_EQ(rev_out_length, + fwrite(float_data.get(), sizeof(float_data[0]), rev_out_length, + rev_out_file)); + + analog_level = ap->gain_control()->stream_analog_level(); + } + fclose(far_file); + fclose(near_file); + fclose(out_file); + fclose(rev_out_file); + } + + protected: + int input_rate_; + int output_rate_; + int reverse_input_rate_; + int reverse_output_rate_; + double expected_snr_; + double expected_reverse_snr_; +}; + +TEST_P(AudioProcessingTest, Formats) { + struct ChannelFormat { + int num_input; + int num_output; + int num_reverse_input; + int num_reverse_output; + }; + ChannelFormat cf[] = { + {1, 1, 1, 1}, + {1, 1, 2, 1}, + {2, 1, 1, 1}, + {2, 1, 2, 1}, + {2, 2, 1, 1}, + {2, 2, 2, 2}, + }; + + for (size_t i = 0; i < arraysize(cf); ++i) { + ProcessFormat(input_rate_, output_rate_, reverse_input_rate_, + reverse_output_rate_, cf[i].num_input, cf[i].num_output, + cf[i].num_reverse_input, cf[i].num_reverse_output, "out"); + + // Verify output for both directions. + std::vector stream_directions; + stream_directions.push_back(kForward); + stream_directions.push_back(kReverse); + for (StreamDirection file_direction : stream_directions) { + const int in_rate = file_direction ? reverse_input_rate_ : input_rate_; + const int out_rate = file_direction ? reverse_output_rate_ : output_rate_; + const int out_num = + file_direction ? cf[i].num_reverse_output : cf[i].num_output; + const double expected_snr = + file_direction ? expected_reverse_snr_ : expected_snr_; + + const int min_ref_rate = std::min(in_rate, out_rate); + int ref_rate; + + if (min_ref_rate > 32000) { + ref_rate = 48000; + } else if (min_ref_rate > 16000) { + ref_rate = 32000; + } else if (min_ref_rate > 8000) { + ref_rate = 16000; + } else { + ref_rate = 8000; + } +#ifdef WEBRTC_ARCH_ARM_FAMILY + if (file_direction == kForward) { + ref_rate = std::min(ref_rate, 32000); + } +#endif + FILE* out_file = fopen( + OutputFilePath("out", input_rate_, output_rate_, reverse_input_rate_, + reverse_output_rate_, cf[i].num_input, + cf[i].num_output, cf[i].num_reverse_input, + cf[i].num_reverse_output, file_direction).c_str(), + "rb"); + // The reference files always have matching input and output channels. + FILE* ref_file = fopen( + OutputFilePath("ref", ref_rate, ref_rate, ref_rate, ref_rate, + cf[i].num_output, cf[i].num_output, + cf[i].num_reverse_output, cf[i].num_reverse_output, + file_direction).c_str(), + "rb"); + ASSERT_TRUE(out_file != NULL); + ASSERT_TRUE(ref_file != NULL); + + const size_t ref_length = SamplesFromRate(ref_rate) * out_num; + const size_t out_length = SamplesFromRate(out_rate) * out_num; + // Data from the reference file. + std::unique_ptr ref_data(new float[ref_length]); + // Data from the output file. + std::unique_ptr out_data(new float[out_length]); + // Data from the resampled output, in case the reference and output rates + // don't match. + std::unique_ptr cmp_data(new float[ref_length]); + + PushResampler resampler; + resampler.InitializeIfNeeded(out_rate, ref_rate, out_num); + + // Compute the resampling delay of the output relative to the reference, + // to find the region over which we should search for the best SNR. + float expected_delay_sec = 0; + if (in_rate != ref_rate) { + // Input resampling delay. + expected_delay_sec += + PushSincResampler::AlgorithmicDelaySeconds(in_rate); + } + if (out_rate != ref_rate) { + // Output resampling delay. + expected_delay_sec += + PushSincResampler::AlgorithmicDelaySeconds(ref_rate); + // Delay of converting the output back to its processing rate for + // testing. + expected_delay_sec += + PushSincResampler::AlgorithmicDelaySeconds(out_rate); + } + int expected_delay = + floor(expected_delay_sec * ref_rate + 0.5f) * out_num; + + double variance = 0; + double sq_error = 0; + while (fread(out_data.get(), sizeof(out_data[0]), out_length, out_file) && + fread(ref_data.get(), sizeof(ref_data[0]), ref_length, ref_file)) { + float* out_ptr = out_data.get(); + if (out_rate != ref_rate) { + // Resample the output back to its internal processing rate if + // necssary. + ASSERT_EQ(ref_length, + static_cast(resampler.Resample( + out_ptr, out_length, cmp_data.get(), ref_length))); + out_ptr = cmp_data.get(); + } + + // Update the |sq_error| and |variance| accumulators with the highest + // SNR of reference vs output. + UpdateBestSNR(ref_data.get(), out_ptr, ref_length, expected_delay, + &variance, &sq_error); + } + + std::cout << "(" << input_rate_ << ", " << output_rate_ << ", " + << reverse_input_rate_ << ", " << reverse_output_rate_ << ", " + << cf[i].num_input << ", " << cf[i].num_output << ", " + << cf[i].num_reverse_input << ", " << cf[i].num_reverse_output + << ", " << file_direction << "): "; + if (sq_error > 0) { + double snr = 10 * log10(variance / sq_error); + EXPECT_GE(snr, expected_snr); + EXPECT_NE(0, expected_snr); + std::cout << "SNR=" << snr << " dB" << std::endl; + } else { + std::cout << "SNR=inf dB" << std::endl; + } + + fclose(out_file); + fclose(ref_file); + } + } +} + +#if defined(WEBRTC_AUDIOPROC_FLOAT_PROFILE) +INSTANTIATE_TEST_CASE_P( + CommonFormats, + AudioProcessingTest, + testing::Values(std::make_tuple(48000, 48000, 48000, 48000, 0, 0), + std::make_tuple(48000, 48000, 32000, 48000, 40, 30), + std::make_tuple(48000, 48000, 16000, 48000, 40, 20), + std::make_tuple(48000, 44100, 48000, 44100, 20, 20), + std::make_tuple(48000, 44100, 32000, 44100, 20, 15), + std::make_tuple(48000, 44100, 16000, 44100, 20, 15), + std::make_tuple(48000, 32000, 48000, 32000, 30, 35), + std::make_tuple(48000, 32000, 32000, 32000, 30, 0), + std::make_tuple(48000, 32000, 16000, 32000, 30, 20), + std::make_tuple(48000, 16000, 48000, 16000, 25, 20), + std::make_tuple(48000, 16000, 32000, 16000, 25, 20), + std::make_tuple(48000, 16000, 16000, 16000, 25, 0), + + std::make_tuple(44100, 48000, 48000, 48000, 30, 0), + std::make_tuple(44100, 48000, 32000, 48000, 30, 30), + std::make_tuple(44100, 48000, 16000, 48000, 30, 20), + std::make_tuple(44100, 44100, 48000, 44100, 20, 20), + std::make_tuple(44100, 44100, 32000, 44100, 20, 15), + std::make_tuple(44100, 44100, 16000, 44100, 20, 15), + std::make_tuple(44100, 32000, 48000, 32000, 30, 35), + std::make_tuple(44100, 32000, 32000, 32000, 30, 0), + std::make_tuple(44100, 32000, 16000, 32000, 30, 20), + std::make_tuple(44100, 16000, 48000, 16000, 25, 20), + std::make_tuple(44100, 16000, 32000, 16000, 25, 20), + std::make_tuple(44100, 16000, 16000, 16000, 25, 0), + + std::make_tuple(32000, 48000, 48000, 48000, 30, 0), + std::make_tuple(32000, 48000, 32000, 48000, 35, 30), + std::make_tuple(32000, 48000, 16000, 48000, 30, 20), + std::make_tuple(32000, 44100, 48000, 44100, 20, 20), + std::make_tuple(32000, 44100, 32000, 44100, 20, 15), + std::make_tuple(32000, 44100, 16000, 44100, 20, 15), + std::make_tuple(32000, 32000, 48000, 32000, 40, 35), + std::make_tuple(32000, 32000, 32000, 32000, 0, 0), + std::make_tuple(32000, 32000, 16000, 32000, 40, 20), + std::make_tuple(32000, 16000, 48000, 16000, 25, 20), + std::make_tuple(32000, 16000, 32000, 16000, 25, 20), + std::make_tuple(32000, 16000, 16000, 16000, 25, 0), + + std::make_tuple(16000, 48000, 48000, 48000, 25, 0), + std::make_tuple(16000, 48000, 32000, 48000, 25, 30), + std::make_tuple(16000, 48000, 16000, 48000, 25, 20), + std::make_tuple(16000, 44100, 48000, 44100, 15, 20), + std::make_tuple(16000, 44100, 32000, 44100, 15, 15), + std::make_tuple(16000, 44100, 16000, 44100, 15, 15), + std::make_tuple(16000, 32000, 48000, 32000, 25, 35), + std::make_tuple(16000, 32000, 32000, 32000, 25, 0), + std::make_tuple(16000, 32000, 16000, 32000, 25, 20), + std::make_tuple(16000, 16000, 48000, 16000, 40, 20), + std::make_tuple(16000, 16000, 32000, 16000, 40, 20), + std::make_tuple(16000, 16000, 16000, 16000, 0, 0))); + +#elif defined(WEBRTC_AUDIOPROC_FIXED_PROFILE) +INSTANTIATE_TEST_CASE_P( + CommonFormats, + AudioProcessingTest, + testing::Values(std::make_tuple(48000, 48000, 48000, 48000, 20, 0), + std::make_tuple(48000, 48000, 32000, 48000, 20, 30), + std::make_tuple(48000, 48000, 16000, 48000, 20, 20), + std::make_tuple(48000, 44100, 48000, 44100, 15, 20), + std::make_tuple(48000, 44100, 32000, 44100, 15, 15), + std::make_tuple(48000, 44100, 16000, 44100, 15, 15), + std::make_tuple(48000, 32000, 48000, 32000, 20, 35), + std::make_tuple(48000, 32000, 32000, 32000, 20, 0), + std::make_tuple(48000, 32000, 16000, 32000, 20, 20), + std::make_tuple(48000, 16000, 48000, 16000, 20, 20), + std::make_tuple(48000, 16000, 32000, 16000, 20, 20), + std::make_tuple(48000, 16000, 16000, 16000, 20, 0), + + std::make_tuple(44100, 48000, 48000, 48000, 15, 0), + std::make_tuple(44100, 48000, 32000, 48000, 15, 30), + std::make_tuple(44100, 48000, 16000, 48000, 15, 20), + std::make_tuple(44100, 44100, 48000, 44100, 15, 20), + std::make_tuple(44100, 44100, 32000, 44100, 15, 15), + std::make_tuple(44100, 44100, 16000, 44100, 15, 15), + std::make_tuple(44100, 32000, 48000, 32000, 20, 35), + std::make_tuple(44100, 32000, 32000, 32000, 20, 0), + std::make_tuple(44100, 32000, 16000, 32000, 20, 20), + std::make_tuple(44100, 16000, 48000, 16000, 20, 20), + std::make_tuple(44100, 16000, 32000, 16000, 20, 20), + std::make_tuple(44100, 16000, 16000, 16000, 20, 0), + + std::make_tuple(32000, 48000, 48000, 48000, 35, 0), + std::make_tuple(32000, 48000, 32000, 48000, 65, 30), + std::make_tuple(32000, 48000, 16000, 48000, 40, 20), + std::make_tuple(32000, 44100, 48000, 44100, 20, 20), + std::make_tuple(32000, 44100, 32000, 44100, 20, 15), + std::make_tuple(32000, 44100, 16000, 44100, 20, 15), + std::make_tuple(32000, 32000, 48000, 32000, 35, 35), + std::make_tuple(32000, 32000, 32000, 32000, 0, 0), + std::make_tuple(32000, 32000, 16000, 32000, 40, 20), + std::make_tuple(32000, 16000, 48000, 16000, 20, 20), + std::make_tuple(32000, 16000, 32000, 16000, 20, 20), + std::make_tuple(32000, 16000, 16000, 16000, 20, 0), + + std::make_tuple(16000, 48000, 48000, 48000, 25, 0), + std::make_tuple(16000, 48000, 32000, 48000, 25, 30), + std::make_tuple(16000, 48000, 16000, 48000, 25, 20), + std::make_tuple(16000, 44100, 48000, 44100, 15, 20), + std::make_tuple(16000, 44100, 32000, 44100, 15, 15), + std::make_tuple(16000, 44100, 16000, 44100, 15, 15), + std::make_tuple(16000, 32000, 48000, 32000, 25, 35), + std::make_tuple(16000, 32000, 32000, 32000, 25, 0), + std::make_tuple(16000, 32000, 16000, 32000, 25, 20), + std::make_tuple(16000, 16000, 48000, 16000, 35, 20), + std::make_tuple(16000, 16000, 32000, 16000, 35, 20), + std::make_tuple(16000, 16000, 16000, 16000, 0, 0))); +#endif + +} // namespace + +TEST(ApmConfiguration, DefaultBehavior) { + // Verify that the level controller is default off, it can be activated using + // the config, and that the default initial level is maintained after the + // config has been applied. + std::unique_ptr apm( + new rtc::RefCountedObject(webrtc::Config())); + AudioProcessing::Config config; + EXPECT_FALSE(apm->config_.level_controller.enabled); + // TODO(peah): Add test for the existence of the level controller object once + // that is created only when that is specified in the config. + // TODO(peah): Remove the testing for + // apm->capture_nonlocked_.level_controller_enabled once the value in config_ + // is instead used to activate the level controller. + EXPECT_FALSE(apm->capture_nonlocked_.level_controller_enabled); + EXPECT_NEAR(kTargetLcPeakLeveldBFS, + apm->config_.level_controller.initial_peak_level_dbfs, + std::numeric_limits::epsilon()); + config.level_controller.enabled = true; + apm->ApplyConfig(config); + EXPECT_TRUE(apm->config_.level_controller.enabled); + // TODO(peah): Add test for the existence of the level controller object once + // that is created only when the that is specified in the config. + // TODO(peah): Remove the testing for + // apm->capture_nonlocked_.level_controller_enabled once the value in config_ + // is instead used to activate the level controller. + EXPECT_TRUE(apm->capture_nonlocked_.level_controller_enabled); + EXPECT_NEAR(kTargetLcPeakLeveldBFS, + apm->config_.level_controller.initial_peak_level_dbfs, + std::numeric_limits::epsilon()); +} + +TEST(ApmConfiguration, ValidConfigBehavior) { + // Verify that the initial level can be specified and is retained after the + // config has been applied. + std::unique_ptr apm( + new rtc::RefCountedObject(webrtc::Config())); + AudioProcessing::Config config; + config.level_controller.initial_peak_level_dbfs = -50.f; + apm->ApplyConfig(config); + EXPECT_FALSE(apm->config_.level_controller.enabled); + // TODO(peah): Add test for the existence of the level controller object once + // that is created only when the that is specified in the config. + // TODO(peah): Remove the testing for + // apm->capture_nonlocked_.level_controller_enabled once the value in config_ + // is instead used to activate the level controller. + EXPECT_FALSE(apm->capture_nonlocked_.level_controller_enabled); + EXPECT_NEAR(-50.f, apm->config_.level_controller.initial_peak_level_dbfs, + std::numeric_limits::epsilon()); +} + +TEST(ApmConfiguration, InValidConfigBehavior) { + // Verify that the config is properly reset when nonproper values are applied + // for the initial level. + + // Verify that the config is properly reset when the specified initial peak + // level is too low. + std::unique_ptr apm( + new rtc::RefCountedObject(webrtc::Config())); + AudioProcessing::Config config; + config.level_controller.enabled = true; + config.level_controller.initial_peak_level_dbfs = -101.f; + apm->ApplyConfig(config); + EXPECT_FALSE(apm->config_.level_controller.enabled); + // TODO(peah): Add test for the existence of the level controller object once + // that is created only when the that is specified in the config. + // TODO(peah): Remove the testing for + // apm->capture_nonlocked_.level_controller_enabled once the value in config_ + // is instead used to activate the level controller. + EXPECT_FALSE(apm->capture_nonlocked_.level_controller_enabled); + EXPECT_NEAR(kTargetLcPeakLeveldBFS, + apm->config_.level_controller.initial_peak_level_dbfs, + std::numeric_limits::epsilon()); + + // Verify that the config is properly reset when the specified initial peak + // level is too high. + apm.reset(new rtc::RefCountedObject(webrtc::Config())); + config = AudioProcessing::Config(); + config.level_controller.enabled = true; + config.level_controller.initial_peak_level_dbfs = 1.f; + apm->ApplyConfig(config); + EXPECT_FALSE(apm->config_.level_controller.enabled); + // TODO(peah): Add test for the existence of the level controller object once + // that is created only when that is specified in the config. + // TODO(peah): Remove the testing for + // apm->capture_nonlocked_.level_controller_enabled once the value in config_ + // is instead used to activate the level controller. + EXPECT_FALSE(apm->capture_nonlocked_.level_controller_enabled); + EXPECT_NEAR(kTargetLcPeakLeveldBFS, + apm->config_.level_controller.initial_peak_level_dbfs, + std::numeric_limits::epsilon()); +} + +TEST(ApmConfiguration, EnablePostProcessing) { + // Verify that apm uses a capture post processing module if one is provided. + webrtc::Config webrtc_config; + auto mock_post_processor_ptr = + new testing::NiceMock(); + auto mock_post_processor = + std::unique_ptr(mock_post_processor_ptr); + rtc::scoped_refptr apm = AudioProcessing::Create( + webrtc_config, std::move(mock_post_processor), nullptr, nullptr); + + AudioFrame audio; + audio.num_channels_ = 1; + SetFrameSampleRate(&audio, AudioProcessing::NativeRate::kSampleRate16kHz); + + EXPECT_CALL(*mock_post_processor_ptr, Process(testing::_)).Times(1); + apm->ProcessStream(&audio); +} + +class MyEchoControlFactory : public EchoControlFactory { + public: + std::unique_ptr Create(int sample_rate_hz) { + auto ec = new test::MockEchoControl(); + EXPECT_CALL(*ec, AnalyzeRender(testing::_)).Times(1); + EXPECT_CALL(*ec, AnalyzeCapture(testing::_)).Times(2); + EXPECT_CALL(*ec, ProcessCapture(testing::_, testing::_)).Times(2); + return std::unique_ptr(ec); + } +}; + +TEST(ApmConfiguration, EchoControlInjection) { + // Verify that apm uses an injected echo controller if one is provided. + webrtc::Config webrtc_config; + std::unique_ptr echo_control_factory( + new MyEchoControlFactory()); + + rtc::scoped_refptr apm = AudioProcessing::Create( + webrtc_config, nullptr, std::move(echo_control_factory), nullptr); + + AudioFrame audio; + audio.num_channels_ = 1; + SetFrameSampleRate(&audio, AudioProcessing::NativeRate::kSampleRate16kHz); + apm->ProcessStream(&audio); + apm->ProcessReverseStream(&audio); + apm->ProcessStream(&audio); +} + +std::unique_ptr CreateApm(bool use_AEC2) { + Config old_config; + if (use_AEC2) { + old_config.Set(new ExtendedFilter(true)); + old_config.Set(new DelayAgnostic(true)); + } + std::unique_ptr apm(AudioProcessing::Create(old_config)); + if (!apm) { + return apm; + } + + ProcessingConfig processing_config = { + {{32000, 1}, {32000, 1}, {32000, 1}, {32000, 1}}}; + + if (apm->Initialize(processing_config) != 0) { + return nullptr; + } + + // Disable all components except for an AEC and the residual echo detector. + AudioProcessing::Config config; + config.residual_echo_detector.enabled = true; + config.echo_canceller3.enabled = false; + config.high_pass_filter.enabled = false; + config.gain_controller2.enabled = false; + config.level_controller.enabled = false; + apm->ApplyConfig(config); + EXPECT_EQ(apm->gain_control()->Enable(false), 0); + EXPECT_EQ(apm->level_estimator()->Enable(false), 0); + EXPECT_EQ(apm->noise_suppression()->Enable(false), 0); + EXPECT_EQ(apm->voice_detection()->Enable(false), 0); + + if (use_AEC2) { + EXPECT_EQ(apm->echo_control_mobile()->Enable(false), 0); + EXPECT_EQ(apm->echo_cancellation()->enable_metrics(true), 0); + EXPECT_EQ(apm->echo_cancellation()->enable_delay_logging(true), 0); + EXPECT_EQ(apm->echo_cancellation()->Enable(true), 0); + } else { + EXPECT_EQ(apm->echo_cancellation()->Enable(false), 0); + EXPECT_EQ(apm->echo_control_mobile()->Enable(true), 0); + } + return apm; +} + +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_MAC) +#define MAYBE_ApmStatistics DISABLED_ApmStatistics +#else +#define MAYBE_ApmStatistics ApmStatistics +#endif + +TEST(MAYBE_ApmStatistics, AEC2EnabledTest) { + // Set up APM with AEC2 and process some audio. + std::unique_ptr apm = CreateApm(true); + ASSERT_TRUE(apm); + + // Set up an audioframe. + AudioFrame frame; + frame.num_channels_ = 1; + SetFrameSampleRate(&frame, AudioProcessing::NativeRate::kSampleRate48kHz); + + // Fill the audio frame with a sawtooth pattern. + int16_t* ptr = frame.mutable_data(); + for (size_t i = 0; i < frame.kMaxDataSizeSamples; i++) { + ptr[i] = 10000 * ((i % 3) - 1); + } + + // Do some processing. + for (int i = 0; i < 200; i++) { + EXPECT_EQ(apm->ProcessReverseStream(&frame), 0); + EXPECT_EQ(apm->set_stream_delay_ms(0), 0); + EXPECT_EQ(apm->ProcessStream(&frame), 0); + } + + // Test statistics interface. + AudioProcessingStats stats = apm->GetStatistics(true); + // We expect all statistics to be set and have a sensible value. + ASSERT_TRUE(stats.residual_echo_likelihood); + EXPECT_GE(*stats.residual_echo_likelihood, 0.0); + EXPECT_LE(*stats.residual_echo_likelihood, 1.0); + ASSERT_TRUE(stats.residual_echo_likelihood_recent_max); + EXPECT_GE(*stats.residual_echo_likelihood_recent_max, 0.0); + EXPECT_LE(*stats.residual_echo_likelihood_recent_max, 1.0); + ASSERT_TRUE(stats.echo_return_loss); + EXPECT_NE(*stats.echo_return_loss, -100.0); + ASSERT_TRUE(stats.echo_return_loss_enhancement); + EXPECT_NE(*stats.echo_return_loss_enhancement, -100.0); + ASSERT_TRUE(stats.divergent_filter_fraction); + EXPECT_NE(*stats.divergent_filter_fraction, -1.0); + ASSERT_TRUE(stats.delay_standard_deviation_ms); + EXPECT_GE(*stats.delay_standard_deviation_ms, 0); + // We don't check stats.delay_median_ms since it takes too long to settle to a + // value. At least 20 seconds of data need to be processed before it will get + // a value, which would make this test take too much time. + + // If there are no receive streams, we expect the stats not to be set. The + // 'false' argument signals to APM that no receive streams are currently + // active. In that situation the statistics would get stuck at their last + // calculated value (AEC and echo detection need at least one stream in each + // direction), so to avoid that, they should not be set by APM. + stats = apm->GetStatistics(false); + EXPECT_FALSE(stats.residual_echo_likelihood); + EXPECT_FALSE(stats.residual_echo_likelihood_recent_max); + EXPECT_FALSE(stats.echo_return_loss); + EXPECT_FALSE(stats.echo_return_loss_enhancement); + EXPECT_FALSE(stats.divergent_filter_fraction); + EXPECT_FALSE(stats.delay_median_ms); + EXPECT_FALSE(stats.delay_standard_deviation_ms); +} + +TEST(MAYBE_ApmStatistics, AECMEnabledTest) { + // Set up APM with AECM and process some audio. + std::unique_ptr apm = CreateApm(false); + ASSERT_TRUE(apm); + + // Set up an audioframe. + AudioFrame frame; + frame.num_channels_ = 1; + SetFrameSampleRate(&frame, AudioProcessing::NativeRate::kSampleRate48kHz); + + // Fill the audio frame with a sawtooth pattern. + int16_t* ptr = frame.mutable_data(); + for (size_t i = 0; i < frame.kMaxDataSizeSamples; i++) { + ptr[i] = 10000 * ((i % 3) - 1); + } + + // Do some processing. + for (int i = 0; i < 200; i++) { + EXPECT_EQ(apm->ProcessReverseStream(&frame), 0); + EXPECT_EQ(apm->set_stream_delay_ms(0), 0); + EXPECT_EQ(apm->ProcessStream(&frame), 0); + } + + // Test statistics interface. + AudioProcessingStats stats = apm->GetStatistics(true); + // We expect only the residual echo detector statistics to be set and have a + // sensible value. + EXPECT_TRUE(stats.residual_echo_likelihood); + if (stats.residual_echo_likelihood) { + EXPECT_GE(*stats.residual_echo_likelihood, 0.0); + EXPECT_LE(*stats.residual_echo_likelihood, 1.0); + } + EXPECT_TRUE(stats.residual_echo_likelihood_recent_max); + if (stats.residual_echo_likelihood_recent_max) { + EXPECT_GE(*stats.residual_echo_likelihood_recent_max, 0.0); + EXPECT_LE(*stats.residual_echo_likelihood_recent_max, 1.0); + } + EXPECT_FALSE(stats.echo_return_loss); + EXPECT_FALSE(stats.echo_return_loss_enhancement); + EXPECT_FALSE(stats.divergent_filter_fraction); + EXPECT_FALSE(stats.delay_median_ms); + EXPECT_FALSE(stats.delay_standard_deviation_ms); + + // If there are no receive streams, we expect the stats not to be set. + stats = apm->GetStatistics(false); + EXPECT_FALSE(stats.residual_echo_likelihood); + EXPECT_FALSE(stats.residual_echo_likelihood_recent_max); + EXPECT_FALSE(stats.echo_return_loss); + EXPECT_FALSE(stats.echo_return_loss_enhancement); + EXPECT_FALSE(stats.divergent_filter_fraction); + EXPECT_FALSE(stats.delay_median_ms); + EXPECT_FALSE(stats.delay_standard_deviation_ms); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.cc new file mode 100644 index 0000000000..e853559140 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.cc @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/beamformer/array_util.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +const float kMaxDotProduct = 1e-6f; + +} // namespace + +float GetMinimumSpacing(const std::vector& array_geometry) { + RTC_CHECK_GT(array_geometry.size(), 1); + float mic_spacing = std::numeric_limits::max(); + for (size_t i = 0; i < (array_geometry.size() - 1); ++i) { + for (size_t j = i + 1; j < array_geometry.size(); ++j) { + mic_spacing = + std::min(mic_spacing, Distance(array_geometry[i], array_geometry[j])); + } + } + return mic_spacing; +} + +Point PairDirection(const Point& a, const Point& b) { + return {b.x() - a.x(), b.y() - a.y(), b.z() - a.z()}; +} + +float DotProduct(const Point& a, const Point& b) { + return a.x() * b.x() + a.y() * b.y() + a.z() * b.z(); +} + +Point CrossProduct(const Point& a, const Point& b) { + return {a.y() * b.z() - a.z() * b.y(), a.z() * b.x() - a.x() * b.z(), + a.x() * b.y() - a.y() * b.x()}; +} + +bool AreParallel(const Point& a, const Point& b) { + Point cross_product = CrossProduct(a, b); + return DotProduct(cross_product, cross_product) < kMaxDotProduct; +} + +bool ArePerpendicular(const Point& a, const Point& b) { + return std::abs(DotProduct(a, b)) < kMaxDotProduct; +} + +rtc::Optional GetDirectionIfLinear( + const std::vector& array_geometry) { + RTC_DCHECK_GT(array_geometry.size(), 1); + const Point first_pair_direction = + PairDirection(array_geometry[0], array_geometry[1]); + for (size_t i = 2u; i < array_geometry.size(); ++i) { + const Point pair_direction = + PairDirection(array_geometry[i - 1], array_geometry[i]); + if (!AreParallel(first_pair_direction, pair_direction)) { + return rtc::nullopt; + } + } + return first_pair_direction; +} + +rtc::Optional GetNormalIfPlanar( + const std::vector& array_geometry) { + RTC_DCHECK_GT(array_geometry.size(), 1); + const Point first_pair_direction = + PairDirection(array_geometry[0], array_geometry[1]); + Point pair_direction(0.f, 0.f, 0.f); + size_t i = 2u; + bool is_linear = true; + for (; i < array_geometry.size() && is_linear; ++i) { + pair_direction = PairDirection(array_geometry[i - 1], array_geometry[i]); + if (!AreParallel(first_pair_direction, pair_direction)) { + is_linear = false; + } + } + if (is_linear) { + return rtc::nullopt; + } + const Point normal_direction = + CrossProduct(first_pair_direction, pair_direction); + for (; i < array_geometry.size(); ++i) { + pair_direction = PairDirection(array_geometry[i - 1], array_geometry[i]); + if (!ArePerpendicular(normal_direction, pair_direction)) { + return rtc::nullopt; + } + } + return normal_direction; +} + +rtc::Optional GetArrayNormalIfExists( + const std::vector& array_geometry) { + const rtc::Optional direction = GetDirectionIfLinear(array_geometry); + if (direction) { + return Point(direction->y(), -direction->x(), 0.f); + } + const rtc::Optional normal = GetNormalIfPlanar(array_geometry); + if (normal && normal->z() < kMaxDotProduct) { + return normal; + } + return rtc::nullopt; +} + +Point AzimuthToPoint(float azimuth) { + return Point(std::cos(azimuth), std::sin(azimuth), 0.f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.h b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.h new file mode 100644 index 0000000000..f234929693 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_ +#define MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_ + +#include +#include + +#include "api/optional.h" + +namespace webrtc { + +// Coordinates in meters. The convention used is: +// x: the horizontal dimension, with positive to the right from the camera's +// perspective. +// y: the depth dimension, with positive forward from the camera's +// perspective. +// z: the vertical dimension, with positive upwards. +template +struct CartesianPoint { + CartesianPoint() { + c[0] = 0; + c[1] = 0; + c[2] = 0; + } + CartesianPoint(T x, T y, T z) { + c[0] = x; + c[1] = y; + c[2] = z; + } + T x() const { return c[0]; } + T y() const { return c[1]; } + T z() const { return c[2]; } + T c[3]; +}; + +using Point = CartesianPoint; + +// Calculates the direction from a to b. +Point PairDirection(const Point& a, const Point& b); + +float DotProduct(const Point& a, const Point& b); +Point CrossProduct(const Point& a, const Point& b); + +bool AreParallel(const Point& a, const Point& b); +bool ArePerpendicular(const Point& a, const Point& b); + +// Returns the minimum distance between any two Points in the given +// |array_geometry|. +float GetMinimumSpacing(const std::vector& array_geometry); + +// If the given array geometry is linear it returns the direction without +// normalizing. +rtc::Optional GetDirectionIfLinear( + const std::vector& array_geometry); + +// If the given array geometry is planar it returns the normal without +// normalizing. +rtc::Optional GetNormalIfPlanar( + const std::vector& array_geometry); + +// Returns the normal of an array if it has one and it is in the xy-plane. +rtc::Optional GetArrayNormalIfExists( + const std::vector& array_geometry); + +// The resulting Point will be in the xy-plane. +Point AzimuthToPoint(float azimuth); + +template +float Distance(CartesianPoint a, CartesianPoint b) { + return std::sqrt((a.x() - b.x()) * (a.x() - b.x()) + + (a.y() - b.y()) * (a.y() - b.y()) + + (a.z() - b.z()) * (a.z() - b.z())); +} + +// The convention used: +// azimuth: zero is to the right from the camera's perspective, with positive +// angles in radians counter-clockwise. +// elevation: zero is horizontal, with positive angles in radians upwards. +// radius: distance from the camera in meters. +template +struct SphericalPoint { + SphericalPoint(T azimuth, T elevation, T radius) { + s[0] = azimuth; + s[1] = elevation; + s[2] = radius; + } + T azimuth() const { return s[0]; } + T elevation() const { return s[1]; } + T distance() const { return s[2]; } + T s[3]; +}; + +using SphericalPointf = SphericalPoint; + +// Helper functions to transform degrees to radians and the inverse. +template +T DegreesToRadians(T angle_degrees) { + return M_PI * angle_degrees / 180; +} + +template +T RadiansToDegrees(T angle_radians) { + return 180 * angle_radians / M_PI; +} + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util_unittest.cc new file mode 100644 index 0000000000..a5c075ab05 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/array_util_unittest.cc @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// MSVC++ requires this to be set before any other includes to get M_PI. +#define _USE_MATH_DEFINES + +#include "modules/audio_processing/beamformer/array_util.h" + +#include +#include + +#include "test/gtest.h" + +namespace webrtc { + +bool operator==(const Point& lhs, const Point& rhs) { + return lhs.x() == rhs.x() && lhs.y() == rhs.y() && lhs.z() == rhs.z(); +} + +TEST(ArrayUtilTest, PairDirection) { + EXPECT_EQ(Point(1.f, 2.f, 3.f), + PairDirection(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); + EXPECT_EQ(Point(-1.f, -2.f, -3.f), + PairDirection(Point(1.f, 2.f, 3.f), Point(0.f, 0.f, 0.f))); + EXPECT_EQ(Point(0.f, 0.f, 0.f), + PairDirection(Point(1.f, 0.f, 0.f), Point(1.f, 0.f, 0.f))); + EXPECT_EQ(Point(-1.f, 2.f, 0.f), + PairDirection(Point(1.f, 0.f, 0.f), Point(0.f, 2.f, 0.f))); + EXPECT_EQ(Point(-4.f, 4.f, -4.f), + PairDirection(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); +} + +TEST(ArrayUtilTest, DotProduct) { + EXPECT_FLOAT_EQ(0.f, DotProduct(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); + EXPECT_FLOAT_EQ(0.f, DotProduct(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f))); + EXPECT_FLOAT_EQ(0.f, DotProduct(Point(1.f, 1.f, 0.f), Point(1.f, -1.f, 0.f))); + EXPECT_FLOAT_EQ(2.f, DotProduct(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f))); + EXPECT_FLOAT_EQ(-6.f, + DotProduct(Point(-2.f, 0.f, 0.f), Point(3.f, 0.f, 0.f))); + EXPECT_FLOAT_EQ(-10.f, + DotProduct(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); +} + +TEST(ArrayUtilTest, CrossProduct) { + EXPECT_EQ(Point(0.f, 0.f, 0.f), + CrossProduct(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); + EXPECT_EQ(Point(0.f, 0.f, 1.f), + CrossProduct(Point(1.f, 0.f, 0.f), Point(0.f, 1.f, 0.f))); + EXPECT_EQ(Point(1.f, 0.f, 0.f), + CrossProduct(Point(0.f, 1.f, 0.f), Point(0.f, 0.f, 1.f))); + EXPECT_EQ(Point(0.f, -1.f, 0.f), + CrossProduct(Point(1.f, 0.f, 0.f), Point(0.f, 0.f, 1.f))); + EXPECT_EQ(Point(-4.f, -8.f, -4.f), + CrossProduct(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); +} + +TEST(ArrayUtilTest, AreParallel) { + EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); + EXPECT_FALSE(AreParallel(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f))); + EXPECT_FALSE(AreParallel(Point(1.f, 2.f, 0.f), Point(1.f, -0.5f, 0.f))); + EXPECT_FALSE(AreParallel(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); + EXPECT_TRUE(AreParallel(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f))); + EXPECT_TRUE(AreParallel(Point(1.f, 2.f, 3.f), Point(-2.f, -4.f, -6.f))); +} + +TEST(ArrayUtilTest, ArePerpendicular) { + EXPECT_TRUE(ArePerpendicular(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); + EXPECT_TRUE(ArePerpendicular(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f))); + EXPECT_TRUE(ArePerpendicular(Point(1.f, 2.f, 0.f), Point(1.f, -0.5f, 0.f))); + EXPECT_FALSE(ArePerpendicular(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); + EXPECT_FALSE(ArePerpendicular(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f))); + EXPECT_FALSE(ArePerpendicular(Point(1.f, 2.f, 3.f), Point(-2.f, -4.f, -6.f))); +} + +TEST(ArrayUtilTest, GetMinimumSpacing) { + std::vector geometry; + geometry.push_back(Point(0.f, 0.f, 0.f)); + geometry.push_back(Point(0.1f, 0.f, 0.f)); + EXPECT_FLOAT_EQ(0.1f, GetMinimumSpacing(geometry)); + geometry.push_back(Point(0.f, 0.05f, 0.f)); + EXPECT_FLOAT_EQ(0.05f, GetMinimumSpacing(geometry)); + geometry.push_back(Point(0.f, 0.f, 0.02f)); + EXPECT_FLOAT_EQ(0.02f, GetMinimumSpacing(geometry)); + geometry.push_back(Point(-0.003f, -0.004f, 0.02f)); + EXPECT_FLOAT_EQ(0.005f, GetMinimumSpacing(geometry)); +} + +TEST(ArrayUtilTest, GetDirectionIfLinear) { + std::vector geometry; + geometry.push_back(Point(0.f, 0.f, 0.f)); + geometry.push_back(Point(0.1f, 0.f, 0.f)); + EXPECT_TRUE( + AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); + geometry.push_back(Point(0.15f, 0.f, 0.f)); + EXPECT_TRUE( + AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); + geometry.push_back(Point(-0.2f, 0.f, 0.f)); + EXPECT_TRUE( + AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); + geometry.push_back(Point(0.05f, 0.f, 0.f)); + EXPECT_TRUE( + AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); + geometry.push_back(Point(0.1f, 0.1f, 0.f)); + EXPECT_FALSE(GetDirectionIfLinear(geometry)); + geometry.push_back(Point(0.f, 0.f, -0.2f)); + EXPECT_FALSE(GetDirectionIfLinear(geometry)); +} + +TEST(ArrayUtilTest, GetNormalIfPlanar) { + std::vector geometry; + geometry.push_back(Point(0.f, 0.f, 0.f)); + geometry.push_back(Point(0.1f, 0.f, 0.f)); + EXPECT_FALSE(GetNormalIfPlanar(geometry)); + geometry.push_back(Point(0.15f, 0.f, 0.f)); + EXPECT_FALSE(GetNormalIfPlanar(geometry)); + geometry.push_back(Point(0.1f, 0.2f, 0.f)); + EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 1.f), *GetNormalIfPlanar(geometry))); + geometry.push_back(Point(0.f, -0.15f, 0.f)); + EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 1.f), *GetNormalIfPlanar(geometry))); + geometry.push_back(Point(0.f, 0.1f, 0.2f)); + EXPECT_FALSE(GetNormalIfPlanar(geometry)); + geometry.push_back(Point(0.f, 0.f, -0.15f)); + EXPECT_FALSE(GetNormalIfPlanar(geometry)); + geometry.push_back(Point(0.1f, 0.2f, 0.f)); + EXPECT_FALSE(GetNormalIfPlanar(geometry)); +} + +TEST(ArrayUtilTest, GetArrayNormalIfExists) { + std::vector geometry; + geometry.push_back(Point(0.f, 0.f, 0.f)); + geometry.push_back(Point(0.1f, 0.f, 0.f)); + EXPECT_TRUE( + AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); + geometry.push_back(Point(0.15f, 0.f, 0.f)); + EXPECT_TRUE( + AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); + geometry.push_back(Point(0.1f, 0.f, 0.2f)); + EXPECT_TRUE( + AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); + geometry.push_back(Point(0.f, 0.f, -0.1f)); + EXPECT_TRUE( + AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); + geometry.push_back(Point(0.1f, 0.2f, 0.3f)); + EXPECT_FALSE(GetArrayNormalIfExists(geometry)); + geometry.push_back(Point(0.f, -0.1f, 0.f)); + EXPECT_FALSE(GetArrayNormalIfExists(geometry)); + geometry.push_back(Point(1.f, 0.f, -0.2f)); + EXPECT_FALSE(GetArrayNormalIfExists(geometry)); +} + +TEST(ArrayUtilTest, DegreesToRadians) { + EXPECT_FLOAT_EQ(0.f, DegreesToRadians(0.f)); + EXPECT_FLOAT_EQ(static_cast(M_PI) / 6.f, DegreesToRadians(30.f)); + EXPECT_FLOAT_EQ(-static_cast(M_PI) / 4.f, DegreesToRadians(-45.f)); + EXPECT_FLOAT_EQ(static_cast(M_PI) / 3.f, DegreesToRadians(60.f)); + EXPECT_FLOAT_EQ(-static_cast(M_PI) / 2.f, DegreesToRadians(-90.f)); + EXPECT_FLOAT_EQ(2.f * static_cast(M_PI) / 3.f, + DegreesToRadians(120.f)); + EXPECT_FLOAT_EQ(-3.f * static_cast(M_PI) / 4.f, + DegreesToRadians(-135.f)); + EXPECT_FLOAT_EQ(5.f * static_cast(M_PI) / 6.f, + DegreesToRadians(150.f)); + EXPECT_FLOAT_EQ(-static_cast(M_PI), DegreesToRadians(-180.f)); +} + +TEST(ArrayUtilTest, RadiansToDegrees) { + EXPECT_FLOAT_EQ(0.f, RadiansToDegrees(0.f)); + EXPECT_FLOAT_EQ(30.f, RadiansToDegrees(M_PI / 6.f)); + EXPECT_FLOAT_EQ(-45.f, RadiansToDegrees(-M_PI / 4.f)); + EXPECT_FLOAT_EQ(60.f, RadiansToDegrees(M_PI / 3.f)); + EXPECT_FLOAT_EQ(-90.f, RadiansToDegrees(-M_PI / 2.f)); + EXPECT_FLOAT_EQ(120.f, RadiansToDegrees(2.f * M_PI / 3.f)); + EXPECT_FLOAT_EQ(-135.f, RadiansToDegrees(-3.f * M_PI / 4.f)); + EXPECT_FLOAT_EQ(150.f, RadiansToDegrees(5.f * M_PI / 6.f)); + EXPECT_FLOAT_EQ(-180.f, RadiansToDegrees(-M_PI)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix.h b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix.h new file mode 100644 index 0000000000..9960e1dd87 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_ +#define MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_ + +#include + +#include "modules/audio_processing/beamformer/matrix.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +using std::complex; + +// An extension of Matrix for operations that only work on a complex type. +template +class ComplexMatrix : public Matrix > { + public: + ComplexMatrix() : Matrix >() {} + + ComplexMatrix(size_t num_rows, size_t num_columns) + : Matrix >(num_rows, num_columns) {} + + ComplexMatrix(const complex* data, size_t num_rows, size_t num_columns) + : Matrix >(data, num_rows, num_columns) {} + + // Complex Matrix operations. + ComplexMatrix& PointwiseConjugate() { + complex* const data = this->data(); + size_t size = this->num_rows() * this->num_columns(); + for (size_t i = 0; i < size; ++i) { + data[i] = conj(data[i]); + } + + return *this; + } + + ComplexMatrix& PointwiseConjugate(const ComplexMatrix& operand) { + this->CopyFrom(operand); + return PointwiseConjugate(); + } + + ComplexMatrix& ConjugateTranspose() { + this->CopyDataToScratch(); + size_t num_rows = this->num_rows(); + this->SetNumRows(this->num_columns()); + this->SetNumColumns(num_rows); + this->Resize(); + return ConjugateTranspose(this->scratch_elements()); + } + + ComplexMatrix& ConjugateTranspose(const ComplexMatrix& operand) { + RTC_CHECK_EQ(operand.num_rows(), this->num_columns()); + RTC_CHECK_EQ(operand.num_columns(), this->num_rows()); + return ConjugateTranspose(operand.elements()); + } + + ComplexMatrix& ZeroImag() { + complex* const data = this->data(); + size_t size = this->num_rows() * this->num_columns(); + for (size_t i = 0; i < size; ++i) { + data[i] = complex(data[i].real(), 0); + } + + return *this; + } + + ComplexMatrix& ZeroImag(const ComplexMatrix& operand) { + this->CopyFrom(operand); + return ZeroImag(); + } + + private: + ComplexMatrix& ConjugateTranspose(const complex* const* src) { + complex* const* elements = this->elements(); + for (size_t i = 0; i < this->num_rows(); ++i) { + for (size_t j = 0; j < this->num_columns(); ++j) { + elements[i][j] = conj(src[j][i]); + } + } + + return *this; + } +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix_unittest.cc new file mode 100644 index 0000000000..e11dfd2b5c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/complex_matrix_unittest.cc @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/beamformer/complex_matrix.h" +#include "modules/audio_processing/beamformer/matrix_test_helpers.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(ComplexMatrixTest, TestPointwiseConjugate) { + const int kNumRows = 2; + const int kNumCols = 4; + + const complex kValuesInitial[kNumRows][kNumCols] = { + {complex(1.1f, 1.1f), complex(2.2f, -2.2f), + complex(3.3f, 3.3f), complex(4.4f, -4.4f)}, + {complex(5.5f, 5.5f), complex(6.6f, -6.6f), + complex(7.7f, 7.7f), complex(8.8f, -8.8f)}}; + + const complex kValuesExpected[kNumRows][kNumCols] = { + {complex(1.1f, -1.1f), complex(2.2f, 2.2f), + complex(3.3f, -3.3f), complex(4.4f, 4.4f)}, + {complex(5.5f, -5.5f), complex(6.6f, 6.6f), + complex(7.7f, -7.7f), complex(8.8f, 8.8f)}}; + + ComplexMatrix initial_mat(*kValuesInitial, kNumRows, kNumCols); + ComplexMatrix expected_result(*kValuesExpected, kNumRows, kNumCols); + ComplexMatrix actual_result(kNumRows, kNumCols); + + actual_result.PointwiseConjugate(initial_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, + actual_result); + + initial_mat.PointwiseConjugate(); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat, + actual_result); +} + +TEST(ComplexMatrixTest, TestConjugateTranspose) { + const int kNumInitialRows = 2; + const int kNumInitialCols = 4; + const int kNumResultRows = 4; + const int kNumResultCols = 2; + + const complex kValuesInitial[kNumInitialRows][kNumInitialCols] = { + {complex(1.1f, 1.1f), complex(2.2f, 2.2f), + complex(3.3f, 3.3f), complex(4.4f, 4.4f)}, + {complex(5.5f, 5.5f), complex(6.6f, 6.6f), + complex(7.7f, 7.7f), complex(8.8f, 8.8f)}}; + + const complex kValuesExpected[kNumResultRows][kNumResultCols] = { + {complex(1.1f, -1.1f), complex(5.5f, -5.5f)}, + {complex(2.2f, -2.2f), complex(6.6f, -6.6f)}, + {complex(3.3f, -3.3f), complex(7.7f, -7.7f)}, + {complex(4.4f, -4.4f), complex(8.8f, -8.8f)}}; + + ComplexMatrix initial_mat( + *kValuesInitial, kNumInitialRows, kNumInitialCols); + ComplexMatrix expected_result( + *kValuesExpected, kNumResultRows, kNumResultCols); + ComplexMatrix actual_result(kNumResultRows, kNumResultCols); + + actual_result.ConjugateTranspose(initial_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, + actual_result); + + initial_mat.ConjugateTranspose(); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat, + actual_result); +} + +TEST(ComplexMatrixTest, TestZeroImag) { + const int kNumRows = 2; + const int kNumCols = 2; + const complex kValuesInitial[kNumRows][kNumCols] = { + {complex(1.1f, 1.1f), complex(2.2f, 2.2f)}, + {complex(3.3f, 3.3f), complex(4.4f, 4.4f)}}; + const complex kValuesExpected[kNumRows][kNumCols] = { + {complex(1.1f, 0.f), complex(2.2f, 0.f)}, + {complex(3.3f, 0.f), complex(4.4f, 0.f)}}; + + ComplexMatrix initial_mat(*kValuesInitial, kNumRows, kNumCols); + ComplexMatrix expected_result(*kValuesExpected, kNumRows, kNumCols); + ComplexMatrix actual_result; + + actual_result.ZeroImag(initial_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, + actual_result); + + initial_mat.ZeroImag(); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat, + actual_result); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.cc new file mode 100644 index 0000000000..df36d5974f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.cc @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#define _USE_MATH_DEFINES + +#include "modules/audio_processing/beamformer/covariance_matrix_generator.h" + +#include + +namespace webrtc { +namespace { + +float BesselJ0(float x) { +#ifdef WEBRTC_WIN + return _j0(x); +#else + return j0(x); +#endif +} + +// Calculates the Euclidean norm for a row vector. +float Norm(const ComplexMatrix& x) { + RTC_CHECK_EQ(1, x.num_rows()); + const size_t length = x.num_columns(); + const complex* elems = x.elements()[0]; + float result = 0.f; + for (size_t i = 0u; i < length; ++i) { + result += std::norm(elems[i]); + } + return std::sqrt(result); +} + +} // namespace + +void CovarianceMatrixGenerator::UniformCovarianceMatrix( + float wave_number, + const std::vector& geometry, + ComplexMatrix* mat) { + RTC_CHECK_EQ(geometry.size(), mat->num_rows()); + RTC_CHECK_EQ(geometry.size(), mat->num_columns()); + + complex* const* mat_els = mat->elements(); + for (size_t i = 0; i < geometry.size(); ++i) { + for (size_t j = 0; j < geometry.size(); ++j) { + if (wave_number > 0.f) { + mat_els[i][j] = + BesselJ0(wave_number * Distance(geometry[i], geometry[j])); + } else { + mat_els[i][j] = i == j ? 1.f : 0.f; + } + } + } +} + +void CovarianceMatrixGenerator::AngledCovarianceMatrix( + float sound_speed, + float angle, + size_t frequency_bin, + size_t fft_size, + size_t num_freq_bins, + int sample_rate, + const std::vector& geometry, + ComplexMatrix* mat) { + RTC_CHECK_EQ(geometry.size(), mat->num_rows()); + RTC_CHECK_EQ(geometry.size(), mat->num_columns()); + + ComplexMatrix interf_cov_vector(1, geometry.size()); + ComplexMatrix interf_cov_vector_transposed(geometry.size(), 1); + PhaseAlignmentMasks(frequency_bin, + fft_size, + sample_rate, + sound_speed, + geometry, + angle, + &interf_cov_vector); + interf_cov_vector.Scale(1.f / Norm(interf_cov_vector)); + interf_cov_vector_transposed.Transpose(interf_cov_vector); + interf_cov_vector.PointwiseConjugate(); + mat->Multiply(interf_cov_vector_transposed, interf_cov_vector); +} + +void CovarianceMatrixGenerator::PhaseAlignmentMasks( + size_t frequency_bin, + size_t fft_size, + int sample_rate, + float sound_speed, + const std::vector& geometry, + float angle, + ComplexMatrix* mat) { + RTC_CHECK_EQ(1, mat->num_rows()); + RTC_CHECK_EQ(geometry.size(), mat->num_columns()); + + float freq_in_hertz = + (static_cast(frequency_bin) / fft_size) * sample_rate; + + complex* const* mat_els = mat->elements(); + for (size_t c_ix = 0; c_ix < geometry.size(); ++c_ix) { + float distance = std::cos(angle) * geometry[c_ix].x() + + std::sin(angle) * geometry[c_ix].y(); + float phase_shift = -2.f * M_PI * distance * freq_in_hertz / sound_speed; + + // Euler's formula for mat[0][c_ix] = e^(j * phase_shift). + mat_els[0][c_ix] = complex(cos(phase_shift), sin(phase_shift)); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.h new file mode 100644 index 0000000000..6a5841f631 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_COVARIANCE_MATRIX_GENERATOR_H_ +#define MODULES_AUDIO_PROCESSING_BEAMFORMER_COVARIANCE_MATRIX_GENERATOR_H_ + +#include "modules/audio_processing/beamformer/complex_matrix.h" +#include "modules/audio_processing/beamformer/array_util.h" + +namespace webrtc { + +// Helper class for Beamformer in charge of generating covariance matrices. For +// each function, the passed-in ComplexMatrix is expected to be of size +// |num_input_channels| x |num_input_channels|. +class CovarianceMatrixGenerator { + public: + // A uniform covariance matrix with a gap at the target location. WARNING: + // The target angle is assumed to be 0. + static void UniformCovarianceMatrix(float wave_number, + const std::vector& geometry, + ComplexMatrix* mat); + + // The covariance matrix of a source at the given angle. + static void AngledCovarianceMatrix(float sound_speed, + float angle, + size_t frequency_bin, + size_t fft_size, + size_t num_freq_bins, + int sample_rate, + const std::vector& geometry, + ComplexMatrix* mat); + + // Calculates phase shifts that, when applied to a multichannel signal and + // added together, cause constructive interferernce for sources located at + // the given angle. + static void PhaseAlignmentMasks(size_t frequency_bin, + size_t fft_size, + int sample_rate, + float sound_speed, + const std::vector& geometry, + float angle, + ComplexMatrix* mat); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_BF_HELPERS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc new file mode 100644 index 0000000000..a6518e5d95 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#define _USE_MATH_DEFINES + +#include "modules/audio_processing/beamformer/covariance_matrix_generator.h" + +#include + +#include "modules/audio_processing/beamformer/matrix_test_helpers.h" +#include "test/gtest.h" + +namespace webrtc { + +using std::complex; + +TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix2Mics) { + const float kWaveNumber = 0.5775f; + const int kNumberMics = 2; + const float kMicSpacing = 0.05f; + const float kTolerance = 0.0001f; + std::vector geometry; + float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; + for (int i = 0; i < kNumberMics; ++i) { + geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); + } + ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); + CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, + geometry, + &actual_covariance_matrix); + + complex* const* actual_els = actual_covariance_matrix.elements(); + + EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].real(), 0.9998f, kTolerance); + EXPECT_NEAR(actual_els[1][0].real(), 0.9998f, kTolerance); + EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance); + + EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); +} + +TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix3Mics) { + const float kWaveNumber = 10.3861f; + const int kNumberMics = 3; + const float kMicSpacing = 0.04f; + const float kTolerance = 0.0001f; + std::vector geometry; + float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; + for (int i = 0; i < kNumberMics; ++i) { + geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); + } + ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); + CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, + geometry, + &actual_covariance_matrix); + + complex* const* actual_els = actual_covariance_matrix.elements(); + + EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].real(), 0.9573f, kTolerance); + EXPECT_NEAR(actual_els[0][2].real(), 0.8347f, kTolerance); + EXPECT_NEAR(actual_els[1][0].real(), 0.9573f, kTolerance); + EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance); + EXPECT_NEAR(actual_els[1][2].real(), 0.9573f, kTolerance); + EXPECT_NEAR(actual_els[2][0].real(), 0.8347f, kTolerance); + EXPECT_NEAR(actual_els[2][1].real(), 0.9573f, kTolerance); + EXPECT_NEAR(actual_els[2][2].real(), 1.f, kTolerance); + + EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][2].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][2].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[2][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[2][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance); +} + +TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix3DArray) { + const float kWaveNumber = 1.2345f; + const int kNumberMics = 4; + const float kTolerance = 0.0001f; + std::vector geometry; + geometry.push_back(Point(-0.025f, -0.05f, -0.075f)); + geometry.push_back(Point(0.075f, -0.05f, -0.075f)); + geometry.push_back(Point(-0.025f, 0.15f, -0.075f)); + geometry.push_back(Point(-0.025f, -0.05f, 0.225f)); + ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); + CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, + geometry, + &actual_covariance_matrix); + + complex* const* actual_els = actual_covariance_matrix.elements(); + + EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].real(), 0.9962f, kTolerance); + EXPECT_NEAR(actual_els[0][2].real(), 0.9848f, kTolerance); + EXPECT_NEAR(actual_els[0][3].real(), 0.9660f, kTolerance); + EXPECT_NEAR(actual_els[1][0].real(), 0.9962f, kTolerance); + EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance); + EXPECT_NEAR(actual_els[1][2].real(), 0.9810f, kTolerance); + EXPECT_NEAR(actual_els[1][3].real(), 0.9623f, kTolerance); + EXPECT_NEAR(actual_els[2][0].real(), 0.9848f, kTolerance); + EXPECT_NEAR(actual_els[2][1].real(), 0.9810f, kTolerance); + EXPECT_NEAR(actual_els[2][2].real(), 1.f, kTolerance); + EXPECT_NEAR(actual_els[2][3].real(), 0.9511f, kTolerance); + EXPECT_NEAR(actual_els[3][0].real(), 0.9660f, kTolerance); + EXPECT_NEAR(actual_els[3][1].real(), 0.9623f, kTolerance); + EXPECT_NEAR(actual_els[3][2].real(), 0.9511f, kTolerance); + EXPECT_NEAR(actual_els[3][3].real(), 1.f, kTolerance); + + EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][2].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][3].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][2].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][3].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[2][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[2][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[2][3].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[3][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[3][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[3][2].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[3][3].imag(), 0.f, kTolerance); +} + +TEST(CovarianceMatrixGeneratorTest, TestAngledCovarianceMatrix2Mics) { + const float kSpeedOfSound = 340; + const float kAngle = static_cast(M_PI) / 4.f; + const float kFrequencyBin = 6; + const float kFftSize = 512; + const int kNumberFrequencyBins = 257; + const int kSampleRate = 16000; + const int kNumberMics = 2; + const float kMicSpacing = 0.04f; + const float kTolerance = 0.0001f; + std::vector geometry; + float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; + for (int i = 0; i < kNumberMics; ++i) { + geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); + } + ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); + CovarianceMatrixGenerator::AngledCovarianceMatrix(kSpeedOfSound, + kAngle, + kFrequencyBin, + kFftSize, + kNumberFrequencyBins, + kSampleRate, + geometry, + &actual_covariance_matrix); + + complex* const* actual_els = actual_covariance_matrix.elements(); + + EXPECT_NEAR(actual_els[0][0].real(), 0.5f, kTolerance); + EXPECT_NEAR(actual_els[0][1].real(), 0.4976f, kTolerance); + EXPECT_NEAR(actual_els[1][0].real(), 0.4976f, kTolerance); + EXPECT_NEAR(actual_els[1][1].real(), 0.5f, kTolerance); + + EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].imag(), 0.0489f, kTolerance); + EXPECT_NEAR(actual_els[1][0].imag(), -0.0489f, kTolerance); + EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); +} + +TEST(CovarianceMatrixGeneratorTest, TestAngledCovarianceMatrix3Mics) { + const float kSpeedOfSound = 340; + const float kAngle = static_cast(M_PI) / 4.f; + const float kFrequencyBin = 9; + const float kFftSize = 512; + const int kNumberFrequencyBins = 257; + const int kSampleRate = 42000; + const int kNumberMics = 3; + const float kMicSpacing = 0.05f; + const float kTolerance = 0.0001f; + std::vector geometry; + float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; + for (int i = 0; i < kNumberMics; ++i) { + geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); + } + ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); + CovarianceMatrixGenerator::AngledCovarianceMatrix(kSpeedOfSound, + kAngle, + kFrequencyBin, + kFftSize, + kNumberFrequencyBins, + kSampleRate, + geometry, + &actual_covariance_matrix); + + complex* const* actual_els = actual_covariance_matrix.elements(); + + EXPECT_NEAR(actual_els[0][0].real(), 0.3333f, kTolerance); + EXPECT_NEAR(actual_els[0][1].real(), 0.2953f, kTolerance); + EXPECT_NEAR(actual_els[0][2].real(), 0.1899f, kTolerance); + EXPECT_NEAR(actual_els[1][0].real(), 0.2953f, kTolerance); + EXPECT_NEAR(actual_els[1][1].real(), 0.3333f, kTolerance); + EXPECT_NEAR(actual_els[1][2].real(), 0.2953f, kTolerance); + EXPECT_NEAR(actual_els[2][0].real(), 0.1899f, kTolerance); + EXPECT_NEAR(actual_els[2][1].real(), 0.2953f, kTolerance); + EXPECT_NEAR(actual_els[2][2].real(), 0.3333f, kTolerance); + + EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[0][1].imag(), 0.1546f, kTolerance); + EXPECT_NEAR(actual_els[0][2].imag(), 0.274f, kTolerance); + EXPECT_NEAR(actual_els[1][0].imag(), -0.1546f, kTolerance); + EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); + EXPECT_NEAR(actual_els[1][2].imag(), 0.1546f, kTolerance); + EXPECT_NEAR(actual_els[2][0].imag(), -0.274f, kTolerance); + EXPECT_NEAR(actual_els[2][1].imag(), -0.1546f, kTolerance); + EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance); +} + +// PhaseAlignmentMasks is tested by AngledCovarianceMatrix and by +// InitBeamformerWeights in BeamformerUnittest. + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix.h b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix.h new file mode 100644 index 0000000000..bf94c25df0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix.h @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_ +#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_ + +#include +#include +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" + +namespace { + +// Wrappers to get around the compiler warning resulting from the fact that +// there's no std::sqrt overload for ints. We cast all non-complex types to +// a double for the sqrt method. +template +T sqrt_wrapper(T x) { + return sqrt(static_cast(x)); +} + +template +std::complex sqrt_wrapper(std::complex x) { + return sqrt(x); +} +} // namespace + +namespace webrtc { + +// Matrix is a class for doing standard matrix operations on 2 dimensional +// matrices of any size. Results of matrix operations are stored in the +// calling object. Function overloads exist for both in-place (the calling +// object is used as both an operand and the result) and out-of-place (all +// operands are passed in as parameters) operations. If operand dimensions +// mismatch, the program crashes. Out-of-place operations change the size of +// the calling object, if necessary, before operating. +// +// 'In-place' operations that inherently change the size of the matrix (eg. +// Transpose, Multiply on different-sized matrices) must make temporary copies +// (|scratch_elements_| and |scratch_data_|) of existing data to complete the +// operations. +// +// The data is stored contiguously. Data can be accessed internally as a flat +// array, |data_|, or as an array of row pointers, |elements_|, but is +// available to users only as an array of row pointers through |elements()|. +// Memory for storage is allocated when a matrix is resized only if the new +// size overflows capacity. Memory needed temporarily for any operations is +// similarly resized only if the new size overflows capacity. +// +// If you pass in storage through the ctor, that storage is copied into the +// matrix. TODO(claguna): albeit tricky, allow for data to be referenced +// instead of copied, and owned by the user. +template +class Matrix { + public: + Matrix() : num_rows_(0), num_columns_(0) {} + + // Allocates space for the elements and initializes all values to zero. + Matrix(size_t num_rows, size_t num_columns) + : num_rows_(num_rows), num_columns_(num_columns) { + Resize(); + scratch_data_.resize(num_rows_ * num_columns_); + scratch_elements_.resize(num_rows_); + } + + // Copies |data| into the new Matrix. + Matrix(const T* data, size_t num_rows, size_t num_columns) + : num_rows_(0), num_columns_(0) { + CopyFrom(data, num_rows, num_columns); + scratch_data_.resize(num_rows_ * num_columns_); + scratch_elements_.resize(num_rows_); + } + + virtual ~Matrix() {} + + // Deep copy an existing matrix. + void CopyFrom(const Matrix& other) { + CopyFrom(&other.data_[0], other.num_rows_, other.num_columns_); + } + + // Copy |data| into the Matrix. The current data is lost. + void CopyFrom(const T* const data, size_t num_rows, size_t num_columns) { + Resize(num_rows, num_columns); + memcpy(&data_[0], data, num_rows_ * num_columns_ * sizeof(data_[0])); + } + + Matrix& CopyFromColumn(const T* const* src, + size_t column_index, + size_t num_rows) { + Resize(1, num_rows); + for (size_t i = 0; i < num_columns_; ++i) { + data_[i] = src[i][column_index]; + } + + return *this; + } + + void Resize(size_t num_rows, size_t num_columns) { + if (num_rows != num_rows_ || num_columns != num_columns_) { + num_rows_ = num_rows; + num_columns_ = num_columns; + Resize(); + } + } + + // Accessors and mutators. + size_t num_rows() const { return num_rows_; } + size_t num_columns() const { return num_columns_; } + T* const* elements() { return &elements_[0]; } + const T* const* elements() const { return &elements_[0]; } + + T Trace() { + RTC_CHECK_EQ(num_rows_, num_columns_); + + T trace = 0; + for (size_t i = 0; i < num_rows_; ++i) { + trace += elements_[i][i]; + } + return trace; + } + + // Matrix Operations. Returns *this to support method chaining. + Matrix& Transpose() { + CopyDataToScratch(); + Resize(num_columns_, num_rows_); + return Transpose(scratch_elements()); + } + + Matrix& Transpose(const Matrix& operand) { + RTC_CHECK_EQ(operand.num_rows_, num_columns_); + RTC_CHECK_EQ(operand.num_columns_, num_rows_); + + return Transpose(operand.elements()); + } + + template + Matrix& Scale(const S& scalar) { + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] *= scalar; + } + + return *this; + } + + template + Matrix& Scale(const Matrix& operand, const S& scalar) { + CopyFrom(operand); + return Scale(scalar); + } + + Matrix& Add(const Matrix& operand) { + RTC_CHECK_EQ(num_rows_, operand.num_rows_); + RTC_CHECK_EQ(num_columns_, operand.num_columns_); + + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] += operand.data_[i]; + } + + return *this; + } + + Matrix& Add(const Matrix& lhs, const Matrix& rhs) { + CopyFrom(lhs); + return Add(rhs); + } + + Matrix& Subtract(const Matrix& operand) { + RTC_CHECK_EQ(num_rows_, operand.num_rows_); + RTC_CHECK_EQ(num_columns_, operand.num_columns_); + + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] -= operand.data_[i]; + } + + return *this; + } + + Matrix& Subtract(const Matrix& lhs, const Matrix& rhs) { + CopyFrom(lhs); + return Subtract(rhs); + } + + Matrix& PointwiseMultiply(const Matrix& operand) { + RTC_CHECK_EQ(num_rows_, operand.num_rows_); + RTC_CHECK_EQ(num_columns_, operand.num_columns_); + + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] *= operand.data_[i]; + } + + return *this; + } + + Matrix& PointwiseMultiply(const Matrix& lhs, const Matrix& rhs) { + CopyFrom(lhs); + return PointwiseMultiply(rhs); + } + + Matrix& PointwiseDivide(const Matrix& operand) { + RTC_CHECK_EQ(num_rows_, operand.num_rows_); + RTC_CHECK_EQ(num_columns_, operand.num_columns_); + + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] /= operand.data_[i]; + } + + return *this; + } + + Matrix& PointwiseDivide(const Matrix& lhs, const Matrix& rhs) { + CopyFrom(lhs); + return PointwiseDivide(rhs); + } + + Matrix& PointwiseSquareRoot() { + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] = sqrt_wrapper(data_[i]); + } + + return *this; + } + + Matrix& PointwiseSquareRoot(const Matrix& operand) { + CopyFrom(operand); + return PointwiseSquareRoot(); + } + + Matrix& PointwiseAbsoluteValue() { + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] = abs(data_[i]); + } + + return *this; + } + + Matrix& PointwiseAbsoluteValue(const Matrix& operand) { + CopyFrom(operand); + return PointwiseAbsoluteValue(); + } + + Matrix& PointwiseSquare() { + for (size_t i = 0; i < data_.size(); ++i) { + data_[i] *= data_[i]; + } + + return *this; + } + + Matrix& PointwiseSquare(const Matrix& operand) { + CopyFrom(operand); + return PointwiseSquare(); + } + + Matrix& Multiply(const Matrix& lhs, const Matrix& rhs) { + RTC_CHECK_EQ(lhs.num_columns_, rhs.num_rows_); + RTC_CHECK_EQ(num_rows_, lhs.num_rows_); + RTC_CHECK_EQ(num_columns_, rhs.num_columns_); + + return Multiply(lhs.elements(), rhs.num_rows_, rhs.elements()); + } + + Matrix& Multiply(const Matrix& rhs) { + RTC_CHECK_EQ(num_columns_, rhs.num_rows_); + + CopyDataToScratch(); + Resize(num_rows_, rhs.num_columns_); + return Multiply(scratch_elements(), rhs.num_rows_, rhs.elements()); + } + + std::string ToString() const { + std::ostringstream ss; + ss << std::endl << "Matrix" << std::endl; + + for (size_t i = 0; i < num_rows_; ++i) { + for (size_t j = 0; j < num_columns_; ++j) { + ss << elements_[i][j] << " "; + } + ss << std::endl; + } + ss << std::endl; + + return ss.str(); + } + + protected: + void SetNumRows(const size_t num_rows) { num_rows_ = num_rows; } + void SetNumColumns(const size_t num_columns) { num_columns_ = num_columns; } + T* data() { return &data_[0]; } + const T* data() const { return &data_[0]; } + const T* const* scratch_elements() const { return &scratch_elements_[0]; } + + // Resize the matrix. If an increase in capacity is required, the current + // data is lost. + void Resize() { + size_t size = num_rows_ * num_columns_; + data_.resize(size); + elements_.resize(num_rows_); + + for (size_t i = 0; i < num_rows_; ++i) { + elements_[i] = &data_[i * num_columns_]; + } + } + + // Copies data_ into scratch_data_ and updates scratch_elements_ accordingly. + void CopyDataToScratch() { + scratch_data_ = data_; + scratch_elements_.resize(num_rows_); + + for (size_t i = 0; i < num_rows_; ++i) { + scratch_elements_[i] = &scratch_data_[i * num_columns_]; + } + } + + private: + size_t num_rows_; + size_t num_columns_; + std::vector data_; + std::vector elements_; + + // Stores temporary copies of |data_| and |elements_| for in-place operations + // where referring to original data is necessary. + std::vector scratch_data_; + std::vector scratch_elements_; + + // Helpers for Transpose and Multiply operations that unify in-place and + // out-of-place solutions. + Matrix& Transpose(const T* const* src) { + for (size_t i = 0; i < num_rows_; ++i) { + for (size_t j = 0; j < num_columns_; ++j) { + elements_[i][j] = src[j][i]; + } + } + + return *this; + } + + Matrix& Multiply(const T* const* lhs, + size_t num_rows_rhs, + const T* const* rhs) { + for (size_t row = 0; row < num_rows_; ++row) { + for (size_t col = 0; col < num_columns_; ++col) { + T cur_element = 0; + for (size_t i = 0; i < num_rows_rhs; ++i) { + cur_element += lhs[row][i] * rhs[i][col]; + } + + elements_[row][col] = cur_element; + } + } + + return *this; + } + + RTC_DISALLOW_COPY_AND_ASSIGN(Matrix); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_test_helpers.h b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_test_helpers.h new file mode 100644 index 0000000000..62b47082e3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_test_helpers.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_ +#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_ + +#include "modules/audio_processing/beamformer/complex_matrix.h" +#include "modules/audio_processing/beamformer/matrix.h" +#include "test/gtest.h" + +namespace { +const float kTolerance = 0.001f; +} + +namespace webrtc { + +using std::complex; + +// Functions used in both matrix_unittest and complex_matrix_unittest. +class MatrixTestHelpers { + public: + template + static void ValidateMatrixEquality(const Matrix& expected, + const Matrix& actual) { + EXPECT_EQ(expected.num_rows(), actual.num_rows()); + EXPECT_EQ(expected.num_columns(), actual.num_columns()); + + const T* const* expected_elements = expected.elements(); + const T* const* actual_elements = actual.elements(); + for (size_t i = 0; i < expected.num_rows(); ++i) { + for (size_t j = 0; j < expected.num_columns(); ++j) { + EXPECT_EQ(expected_elements[i][j], actual_elements[i][j]); + } + } + } + + static void ValidateMatrixEqualityFloat(const Matrix& expected, + const Matrix& actual) { + EXPECT_EQ(expected.num_rows(), actual.num_rows()); + EXPECT_EQ(expected.num_columns(), actual.num_columns()); + + const float* const* expected_elements = expected.elements(); + const float* const* actual_elements = actual.elements(); + for (size_t i = 0; i < expected.num_rows(); ++i) { + for (size_t j = 0; j < expected.num_columns(); ++j) { + EXPECT_NEAR(expected_elements[i][j], actual_elements[i][j], kTolerance); + } + } + } + + static void ValidateMatrixEqualityComplexFloat( + const Matrix >& expected, + const Matrix >& actual) { + EXPECT_EQ(expected.num_rows(), actual.num_rows()); + EXPECT_EQ(expected.num_columns(), actual.num_columns()); + + const complex* const* expected_elements = expected.elements(); + const complex* const* actual_elements = actual.elements(); + for (size_t i = 0; i < expected.num_rows(); ++i) { + for (size_t j = 0; j < expected.num_columns(); ++j) { + EXPECT_NEAR(expected_elements[i][j].real(), + actual_elements[i][j].real(), + kTolerance); + EXPECT_NEAR(expected_elements[i][j].imag(), + actual_elements[i][j].imag(), + kTolerance); + } + } + } + + static void ValidateMatrixNearEqualityComplexFloat( + const Matrix >& expected, + const Matrix >& actual, + float tolerance) { + EXPECT_EQ(expected.num_rows(), actual.num_rows()); + EXPECT_EQ(expected.num_columns(), actual.num_columns()); + + const complex* const* expected_elements = expected.elements(); + const complex* const* actual_elements = actual.elements(); + for (size_t i = 0; i < expected.num_rows(); ++i) { + for (size_t j = 0; j < expected.num_columns(); ++j) { + EXPECT_NEAR(expected_elements[i][j].real(), + actual_elements[i][j].real(), + tolerance); + EXPECT_NEAR(expected_elements[i][j].imag(), + actual_elements[i][j].imag(), + tolerance); + } + } + } +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_unittest.cc new file mode 100644 index 0000000000..4badfd07ed --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/matrix_unittest.cc @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/beamformer/matrix.h" +#include "modules/audio_processing/beamformer/matrix_test_helpers.h" +#include "test/gtest.h" + +namespace webrtc { + +using std::complex; + +TEST(MatrixTest, TestMultiplySameSize) { + const int kNumRows = 2; + const int kNumCols = 2; + const float kValuesLeft[kNumRows][kNumCols] = {{1.1f, 2.2f}, {3.3f, 4.4f}}; + const float kValuesRight[kNumRows][kNumCols] = {{5.4f, 127.f}, + {4600.f, -555.f}}; + const float kValuesExpected[kNumRows][kNumCols] = {{10125.94f, -1081.3f}, + {20257.82f, -2022.9f}}; + + Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); + Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result(kNumRows, kNumCols); + + actual_result.Multiply(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); + + lh_mat.Multiply(rh_mat); + MatrixTestHelpers::ValidateMatrixEquality(lh_mat, actual_result); +} + +TEST(MatrixTest, TestMultiplyDifferentSize) { + const int kNumRowsLeft = 2; + const int kNumColsLeft = 3; + const int kNumRowsRight = 3; + const int kNumColsRight = 2; + const int kValuesLeft[kNumRowsLeft][kNumColsLeft] = {{35, 466, -15}, + {-3, 3422, 9}}; + const int kValuesRight[kNumRowsRight][kNumColsRight] = { + {765, -42}, {0, 194}, {625, 66321}}; + const int kValuesExpected[kNumRowsLeft][kNumColsRight] = {{17400, -905881}, + {3330, 1260883}}; + + Matrix lh_mat(*kValuesLeft, kNumRowsLeft, kNumColsLeft); + Matrix rh_mat(*kValuesRight, kNumRowsRight, kNumColsRight); + Matrix expected_result(*kValuesExpected, kNumRowsLeft, kNumColsRight); + Matrix actual_result(kNumRowsLeft, kNumColsRight); + + actual_result.Multiply(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); + + lh_mat.Multiply(rh_mat); + MatrixTestHelpers::ValidateMatrixEquality(lh_mat, actual_result); +} + +TEST(MatrixTest, TestTranspose) { + const int kNumInitialRows = 2; + const int kNumInitialCols = 4; + const int kNumResultRows = 4; + const int kNumResultCols = 2; + const float kValuesInitial[kNumInitialRows][kNumInitialCols] = { + {1.1f, 2.2f, 3.3f, 4.4f}, {5.5f, 6.6f, 7.7f, 8.8f}}; + const float kValuesExpected[kNumResultRows][kNumResultCols] = { + {1.1f, 5.5f}, {2.2f, 6.6f}, {3.3f, 7.7f}, {4.4f, 8.8f}}; + + Matrix initial_mat(*kValuesInitial, kNumInitialRows, kNumInitialCols); + Matrix expected_result( + *kValuesExpected, kNumResultRows, kNumResultCols); + Matrix actual_result(kNumResultRows, kNumResultCols); + + actual_result.Transpose(initial_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, + actual_result); + initial_mat.Transpose(); + MatrixTestHelpers::ValidateMatrixEqualityFloat(initial_mat, actual_result); +} + +TEST(MatrixTest, TestScale) { + const int kNumRows = 3; + const int kNumCols = 3; + const int kScaleFactor = -9; + const int kValuesInitial[kNumRows][kNumCols] = { + {1, 20, 5000}, {-3, -29, 66}, {7654, 0, -23455}}; + const int kValuesExpected[kNumRows][kNumCols] = { + {-9, -180, -45000}, {27, 261, -594}, {-68886, 0, 211095}}; + + Matrix initial_mat(*kValuesInitial, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result; + + actual_result.Scale(initial_mat, kScaleFactor); + MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); + + initial_mat.Scale(kScaleFactor); + MatrixTestHelpers::ValidateMatrixEquality(initial_mat, actual_result); +} + +TEST(MatrixTest, TestPointwiseAdd) { + const int kNumRows = 2; + const int kNumCols = 3; + const float kValuesLeft[kNumRows][kNumCols] = {{1.1f, 210.45f, -549.2f}, + {11.876f, 586.7f, -64.35f}}; + const float kValuesRight[kNumRows][kNumCols] = {{-50.4f, 1.f, 0.5f}, + {460.f, -554.2f, 4566.f}}; + const float kValuesExpected[kNumRows][kNumCols] = { + {-49.3f, 211.45f, -548.7f}, {471.876f, 32.5f, 4501.65f}}; + + Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); + Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result; + + actual_result.Add(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, + actual_result); + lh_mat.Add(rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); +} + +TEST(MatrixTest, TestPointwiseSubtract) { + const int kNumRows = 3; + const int kNumCols = 2; + const float kValuesLeft[kNumRows][kNumCols] = { + {1.1f, 210.45f}, {-549.2f, 11.876f}, {586.7f, -64.35f}}; + const float kValuesRight[kNumRows][kNumCols] = { + {-50.4f, 1.f}, {0.5f, 460.f}, {-554.2f, 4566.f}}; + const float kValuesExpected[kNumRows][kNumCols] = { + {51.5f, 209.45f}, {-549.7f, -448.124f}, {1140.9f, -4630.35f}}; + + Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); + Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result; + + actual_result.Subtract(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, + actual_result); + + lh_mat.Subtract(rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); +} + +TEST(MatrixTest, TestPointwiseMultiply) { + const int kNumRows = 1; + const int kNumCols = 5; + const float kValuesLeft[kNumRows][kNumCols] = { + {1.1f, 6.4f, 0.f, -1.f, -88.3f}}; + const float kValuesRight[kNumRows][kNumCols] = { + {53.2f, -210.45f, -549.2f, 99.99f, -45.2f}}; + const float kValuesExpected[kNumRows][kNumCols] = { + {58.52f, -1346.88f, 0.f, -99.99f, 3991.16f}}; + + Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); + Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result; + + actual_result.PointwiseMultiply(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, + actual_result); + + lh_mat.PointwiseMultiply(rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); +} + +TEST(MatrixTest, TestPointwiseDivide) { + const int kNumRows = 5; + const int kNumCols = 1; + const float kValuesLeft[kNumRows][kNumCols] = { + {1.1f}, {6.4f}, {0.f}, {-1.f}, {-88.3f}}; + const float kValuesRight[kNumRows][kNumCols] = { + {53.2f}, {-210.45f}, {-549.2f}, {99.99f}, {-45.2f}}; + const float kValuesExpected[kNumRows][kNumCols] = { + {0.020676691f}, {-0.03041102399f}, {0.f}, {-0.010001f}, {1.9535398f}}; + + Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); + Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result; + + actual_result.PointwiseDivide(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, + actual_result); + + lh_mat.PointwiseDivide(rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); +} + +TEST(MatrixTest, TestPointwiseSquareRoot) { + const int kNumRows = 2; + const int kNumCols = 2; + const int kValues[kNumRows][kNumCols] = {{4, 9}, {16, 0}}; + const int kValuesExpected[kNumRows][kNumCols] = {{2, 3}, {4, 0}}; + + Matrix operand_mat(*kValues, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result; + + actual_result.PointwiseSquareRoot(operand_mat); + MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); + + operand_mat.PointwiseSquareRoot(); + MatrixTestHelpers::ValidateMatrixEquality(operand_mat, actual_result); +} + +TEST(MatrixTest, TestPointwiseSquareRootComplex) { + const int kNumRows = 1; + const int kNumCols = 3; + const complex kValues[kNumRows][kNumCols] = { + {complex(-4.f, 0), complex(0, 9), complex(3, -4)}}; + const complex kValuesExpected[kNumRows][kNumCols] = { + {complex(0.f, 2.f), complex(2.1213202f, 2.1213202f), + complex(2.f, -1.f)}}; + + Matrix > operand_mat(*kValues, kNumRows, kNumCols); + Matrix > expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix > actual_result; + + actual_result.PointwiseSquareRoot(operand_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, + actual_result); + + operand_mat.PointwiseSquareRoot(); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(operand_mat, + actual_result); +} + +TEST(MatrixTest, TestPointwiseAbsoluteValue) { + const int kNumRows = 1; + const int kNumCols = 3; + const complex kValues[kNumRows][kNumCols] = { + {complex(-4.f, 0), complex(0, 9), complex(3, -4)}}; + const complex kValuesExpected[kNumRows][kNumCols] = { + {complex(4.f, 0), complex(9.f, 0), complex(5.f, 0)}}; + + Matrix > operand_mat(*kValues, kNumRows, kNumCols); + Matrix > expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix > actual_result; + + actual_result.PointwiseAbsoluteValue(operand_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, + actual_result); + + operand_mat.PointwiseAbsoluteValue(); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(operand_mat, + actual_result); +} + +TEST(MatrixTest, TestPointwiseSquare) { + const int kNumRows = 1; + const int kNumCols = 3; + const float kValues[kNumRows][kNumCols] = {{2.4f, -4.f, 3.3f}}; + const float kValuesExpected[kNumRows][kNumCols] = {{5.76f, 16.f, 10.89f}}; + + Matrix operand_mat(*kValues, kNumRows, kNumCols); + Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); + Matrix actual_result; + + actual_result.PointwiseSquare(operand_mat); + MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, + actual_result); + + operand_mat.PointwiseSquare(); + MatrixTestHelpers::ValidateMatrixEqualityFloat(operand_mat, actual_result); +} + +TEST(MatrixTest, TestComplexOperations) { + const int kNumRows = 2; + const int kNumCols = 2; + + const complex kValuesLeft[kNumRows][kNumCols] = { + {complex(1.f, 1.f), complex(2.f, 2.f)}, + {complex(3.f, 3.f), complex(4.f, 4.f)}}; + + const complex kValuesRight[kNumRows][kNumCols] = { + {complex(5.f, 5.f), complex(6.f, 6.f)}, + {complex(7.f, 7.f), complex(8.f, 8.f)}}; + + const complex kValuesExpectedAdd[kNumRows][kNumCols] = { + {complex(6.f, 6.f), complex(8.f, 8.f)}, + {complex(10.f, 10.f), complex(12.f, 12.f)}}; + + const complex kValuesExpectedMultiply[kNumRows][kNumCols] = { + {complex(0.f, 38.f), complex(0.f, 44.f)}, + {complex(0.f, 86.f), complex(0.f, 100.f)}}; + + const complex kValuesExpectedPointwiseDivide[kNumRows][kNumCols] = { + {complex(0.2f, 0.f), complex(0.33333333f, 0.f)}, + {complex(0.42857143f, 0.f), complex(0.5f, 0.f)}}; + + Matrix > lh_mat(*kValuesLeft, kNumRows, kNumCols); + Matrix > rh_mat(*kValuesRight, kNumRows, kNumCols); + Matrix > expected_result_add( + *kValuesExpectedAdd, kNumRows, kNumCols); + Matrix > expected_result_multiply( + *kValuesExpectedMultiply, kNumRows, kNumCols); + Matrix > expected_result_pointwise_divide( + *kValuesExpectedPointwiseDivide, kNumRows, kNumCols); + Matrix > actual_result_add; + Matrix > actual_result_multiply(kNumRows, kNumCols); + Matrix > actual_result_pointwise_divide; + + actual_result_add.Add(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result_add, + actual_result_add); + + actual_result_multiply.Multiply(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat( + expected_result_multiply, actual_result_multiply); + + actual_result_pointwise_divide.PointwiseDivide(lh_mat, rh_mat); + MatrixTestHelpers::ValidateMatrixEqualityComplexFloat( + expected_result_pointwise_divide, actual_result_pointwise_divide); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/mock_nonlinear_beamformer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/mock_nonlinear_beamformer.h new file mode 100644 index 0000000000..c4c7358b63 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/mock_nonlinear_beamformer.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_ +#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_ + +#include + +#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockNonlinearBeamformer : public NonlinearBeamformer { + public: + MockNonlinearBeamformer(const std::vector& array_geometry, + size_t num_postfilter_channels) + : NonlinearBeamformer(array_geometry, num_postfilter_channels) {} + + MockNonlinearBeamformer(const std::vector& array_geometry) + : NonlinearBeamformer(array_geometry, 1u) {} + + MOCK_METHOD2(Initialize, void(int chunk_size_ms, int sample_rate_hz)); + MOCK_METHOD1(AnalyzeChunk, void(const ChannelBuffer& data)); + MOCK_METHOD1(PostFilter, void(ChannelBuffer* data)); + MOCK_METHOD1(IsInBeam, bool(const SphericalPointf& spherical_point)); + MOCK_METHOD0(is_target_present, bool()); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.cc new file mode 100644 index 0000000000..12f6d2f465 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.cc @@ -0,0 +1,599 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#define _USE_MATH_DEFINES + +#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" + +#include +#include +#include +#include + +#include "common_audio/window_generator.h" +#include "modules/audio_processing/beamformer/covariance_matrix_generator.h" +#include "rtc_base/arraysize.h" + +namespace webrtc { +namespace { + +// Alpha for the Kaiser Bessel Derived window. +const float kKbdAlpha = 1.5f; + +const float kSpeedOfSoundMeterSeconds = 343; + +// The minimum separation in radians between the target direction and an +// interferer scenario. +const float kMinAwayRadians = 0.2f; + +// The separation between the target direction and the closest interferer +// scenario is proportional to this constant. +const float kAwaySlope = 0.008f; + +// When calculating the interference covariance matrix, this is the weight for +// the weighted average between the uniform covariance matrix and the angled +// covariance matrix. +// Rpsi = Rpsi_angled * kBalance + Rpsi_uniform * (1 - kBalance) +const float kBalance = 0.95f; + +// Alpha coefficients for mask smoothing. +const float kMaskTimeSmoothAlpha = 0.2f; +const float kMaskFrequencySmoothAlpha = 0.6f; + +// The average mask is computed from masks in this mid-frequency range. If these +// ranges are changed |kMaskQuantile| might need to be adjusted. +const int kLowMeanStartHz = 200; +const int kLowMeanEndHz = 400; + +// Range limiter for subtractive terms in the nominator and denominator of the +// postfilter expression. It handles the scenario mismatch between the true and +// model sources (target and interference). +const float kCutOffConstant = 0.9999f; + +// Quantile of mask values which is used to estimate target presence. +const float kMaskQuantile = 0.7f; +// Mask threshold over which the data is considered signal and not interference. +// It has to be updated every time the postfilter calculation is changed +// significantly. +// TODO(aluebs): Write a tool to tune the target threshold automatically based +// on files annotated with target and interference ground truth. +const float kMaskTargetThreshold = 0.01f; +// Time in seconds after which the data is considered interference if the mask +// does not pass |kMaskTargetThreshold|. +const float kHoldTargetSeconds = 0.25f; + +// To compensate for the attenuation this algorithm introduces to the target +// signal. It was estimated empirically from a low-noise low-reverberation +// recording from broadside. +const float kCompensationGain = 2.f; + +// Does conjugate(|norm_mat|) * |mat| * transpose(|norm_mat|). No extra space is +// used; to accomplish this, we compute both multiplications in the same loop. +// The returned norm is clamped to be non-negative. +float Norm(const ComplexMatrix& mat, + const ComplexMatrix& norm_mat) { + RTC_CHECK_EQ(1, norm_mat.num_rows()); + RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_rows()); + RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_columns()); + + complex first_product = complex(0.f, 0.f); + complex second_product = complex(0.f, 0.f); + + const complex* const* mat_els = mat.elements(); + const complex* const* norm_mat_els = norm_mat.elements(); + + for (size_t i = 0; i < norm_mat.num_columns(); ++i) { + for (size_t j = 0; j < norm_mat.num_columns(); ++j) { + first_product += conj(norm_mat_els[0][j]) * mat_els[j][i]; + } + second_product += first_product * norm_mat_els[0][i]; + first_product = 0.f; + } + return std::max(second_product.real(), 0.f); +} + +// Does conjugate(|lhs|) * |rhs| for row vectors |lhs| and |rhs|. +complex ConjugateDotProduct(const ComplexMatrix& lhs, + const ComplexMatrix& rhs) { + RTC_CHECK_EQ(1, lhs.num_rows()); + RTC_CHECK_EQ(1, rhs.num_rows()); + RTC_CHECK_EQ(lhs.num_columns(), rhs.num_columns()); + + const complex* const* lhs_elements = lhs.elements(); + const complex* const* rhs_elements = rhs.elements(); + + complex result = complex(0.f, 0.f); + for (size_t i = 0; i < lhs.num_columns(); ++i) { + result += conj(lhs_elements[0][i]) * rhs_elements[0][i]; + } + + return result; +} + +// Works for positive numbers only. +size_t Round(float x) { + return static_cast(std::floor(x + 0.5f)); +} + +// Calculates the sum of squares of a complex matrix. +float SumSquares(const ComplexMatrix& mat) { + float sum_squares = 0.f; + const complex* const* mat_els = mat.elements(); + for (size_t i = 0; i < mat.num_rows(); ++i) { + for (size_t j = 0; j < mat.num_columns(); ++j) { + float abs_value = std::abs(mat_els[i][j]); + sum_squares += abs_value * abs_value; + } + } + return sum_squares; +} + +// Does |out| = |in|.' * conj(|in|) for row vector |in|. +void TransposedConjugatedProduct(const ComplexMatrix& in, + ComplexMatrix* out) { + RTC_CHECK_EQ(1, in.num_rows()); + RTC_CHECK_EQ(out->num_rows(), in.num_columns()); + RTC_CHECK_EQ(out->num_columns(), in.num_columns()); + const complex* in_elements = in.elements()[0]; + complex* const* out_elements = out->elements(); + for (size_t i = 0; i < out->num_rows(); ++i) { + for (size_t j = 0; j < out->num_columns(); ++j) { + out_elements[i][j] = in_elements[i] * conj(in_elements[j]); + } + } +} + +std::vector GetCenteredArray(std::vector array_geometry) { + for (size_t dim = 0; dim < 3; ++dim) { + float center = 0.f; + for (size_t i = 0; i < array_geometry.size(); ++i) { + center += array_geometry[i].c[dim]; + } + center /= array_geometry.size(); + for (size_t i = 0; i < array_geometry.size(); ++i) { + array_geometry[i].c[dim] -= center; + } + } + return array_geometry; +} + +} // namespace + +const float NonlinearBeamformer::kHalfBeamWidthRadians = DegreesToRadians(20.f); + +// static +const size_t NonlinearBeamformer::kNumFreqBins; + +PostFilterTransform::PostFilterTransform(size_t num_channels, + size_t chunk_length, + float* window, + size_t fft_size) + : transform_(num_channels, + num_channels, + chunk_length, + window, + fft_size, + fft_size / 2, + this), + num_freq_bins_(fft_size / 2 + 1) {} + +void PostFilterTransform::ProcessChunk(float* const* data, float* final_mask) { + final_mask_ = final_mask; + transform_.ProcessChunk(data, data); +} + +void PostFilterTransform::ProcessAudioBlock(const complex* const* input, + size_t num_input_channels, + size_t num_freq_bins, + size_t num_output_channels, + complex* const* output) { + RTC_DCHECK_EQ(num_freq_bins_, num_freq_bins); + RTC_DCHECK_EQ(num_input_channels, num_output_channels); + + for (size_t ch = 0; ch < num_input_channels; ++ch) { + for (size_t f_ix = 0; f_ix < num_freq_bins_; ++f_ix) { + output[ch][f_ix] = + kCompensationGain * final_mask_[f_ix] * input[ch][f_ix]; + } + } +} + +NonlinearBeamformer::NonlinearBeamformer( + const std::vector& array_geometry, + size_t num_postfilter_channels, + SphericalPointf target_direction) + : num_input_channels_(array_geometry.size()), + num_postfilter_channels_(num_postfilter_channels), + array_geometry_(GetCenteredArray(array_geometry)), + array_normal_(GetArrayNormalIfExists(array_geometry)), + min_mic_spacing_(GetMinimumSpacing(array_geometry)), + target_angle_radians_(target_direction.azimuth()), + away_radians_(std::min( + static_cast(M_PI), + std::max(kMinAwayRadians, + kAwaySlope * static_cast(M_PI) / min_mic_spacing_))) { + WindowGenerator::KaiserBesselDerived(kKbdAlpha, kFftSize, window_); +} + +NonlinearBeamformer::~NonlinearBeamformer() = default; + +void NonlinearBeamformer::Initialize(int chunk_size_ms, int sample_rate_hz) { + chunk_length_ = + static_cast(sample_rate_hz / (1000.f / chunk_size_ms)); + sample_rate_hz_ = sample_rate_hz; + + high_pass_postfilter_mask_ = 1.f; + is_target_present_ = false; + hold_target_blocks_ = kHoldTargetSeconds * 2 * sample_rate_hz / kFftSize; + interference_blocks_count_ = hold_target_blocks_; + + process_transform_.reset(new LappedTransform(num_input_channels_, + 0u, + chunk_length_, + window_, + kFftSize, + kFftSize / 2, + this)); + postfilter_transform_.reset(new PostFilterTransform( + num_postfilter_channels_, chunk_length_, window_, kFftSize)); + const float wave_number_step = + (2.f * M_PI * sample_rate_hz_) / (kFftSize * kSpeedOfSoundMeterSeconds); + for (size_t i = 0; i < kNumFreqBins; ++i) { + time_smooth_mask_[i] = 1.f; + final_mask_[i] = 1.f; + wave_numbers_[i] = i * wave_number_step; + } + + InitLowFrequencyCorrectionRanges(); + InitDiffuseCovMats(); + AimAt(SphericalPointf(target_angle_radians_, 0.f, 1.f)); +} + +// These bin indexes determine the regions over which a mean is taken. This is +// applied as a constant value over the adjacent end "frequency correction" +// regions. +// +// low_mean_start_bin_ high_mean_start_bin_ +// v v constant +// |----------------|--------|----------------|-------|----------------| +// constant ^ ^ +// low_mean_end_bin_ high_mean_end_bin_ +// +void NonlinearBeamformer::InitLowFrequencyCorrectionRanges() { + low_mean_start_bin_ = Round(static_cast(kLowMeanStartHz) * + kFftSize / sample_rate_hz_); + low_mean_end_bin_ = Round(static_cast(kLowMeanEndHz) * + kFftSize / sample_rate_hz_); + + RTC_DCHECK_GT(low_mean_start_bin_, 0U); + RTC_DCHECK_LT(low_mean_start_bin_, low_mean_end_bin_); +} + +void NonlinearBeamformer::InitHighFrequencyCorrectionRanges() { + const float kAliasingFreqHz = + kSpeedOfSoundMeterSeconds / + (min_mic_spacing_ * (1.f + std::abs(std::cos(target_angle_radians_)))); + const float kHighMeanStartHz = std::min(0.5f * kAliasingFreqHz, + sample_rate_hz_ / 2.f); + const float kHighMeanEndHz = std::min(0.75f * kAliasingFreqHz, + sample_rate_hz_ / 2.f); + high_mean_start_bin_ = Round(kHighMeanStartHz * kFftSize / sample_rate_hz_); + high_mean_end_bin_ = Round(kHighMeanEndHz * kFftSize / sample_rate_hz_); + + RTC_DCHECK_LT(low_mean_end_bin_, high_mean_end_bin_); + RTC_DCHECK_LT(high_mean_start_bin_, high_mean_end_bin_); + RTC_DCHECK_LT(high_mean_end_bin_, kNumFreqBins - 1); +} + +void NonlinearBeamformer::InitInterfAngles() { + interf_angles_radians_.clear(); + const Point target_direction = AzimuthToPoint(target_angle_radians_); + const Point clockwise_interf_direction = + AzimuthToPoint(target_angle_radians_ - away_radians_); + if (!array_normal_ || + DotProduct(*array_normal_, target_direction) * + DotProduct(*array_normal_, clockwise_interf_direction) >= + 0.f) { + // The target and clockwise interferer are in the same half-plane defined + // by the array. + interf_angles_radians_.push_back(target_angle_radians_ - away_radians_); + } else { + // Otherwise, the interferer will begin reflecting back at the target. + // Instead rotate it away 180 degrees. + interf_angles_radians_.push_back(target_angle_radians_ - away_radians_ + + M_PI); + } + const Point counterclock_interf_direction = + AzimuthToPoint(target_angle_radians_ + away_radians_); + if (!array_normal_ || + DotProduct(*array_normal_, target_direction) * + DotProduct(*array_normal_, counterclock_interf_direction) >= + 0.f) { + // The target and counter-clockwise interferer are in the same half-plane + // defined by the array. + interf_angles_radians_.push_back(target_angle_radians_ + away_radians_); + } else { + // Otherwise, the interferer will begin reflecting back at the target. + // Instead rotate it away 180 degrees. + interf_angles_radians_.push_back(target_angle_radians_ + away_radians_ - + M_PI); + } +} + +void NonlinearBeamformer::InitDelaySumMasks() { + for (size_t f_ix = 0; f_ix < kNumFreqBins; ++f_ix) { + delay_sum_masks_[f_ix].Resize(1, num_input_channels_); + CovarianceMatrixGenerator::PhaseAlignmentMasks( + f_ix, kFftSize, sample_rate_hz_, kSpeedOfSoundMeterSeconds, + array_geometry_, target_angle_radians_, &delay_sum_masks_[f_ix]); + + complex_f norm_factor = sqrt( + ConjugateDotProduct(delay_sum_masks_[f_ix], delay_sum_masks_[f_ix])); + delay_sum_masks_[f_ix].Scale(1.f / norm_factor); + } +} + +void NonlinearBeamformer::InitTargetCovMats() { + for (size_t i = 0; i < kNumFreqBins; ++i) { + target_cov_mats_[i].Resize(num_input_channels_, num_input_channels_); + TransposedConjugatedProduct(delay_sum_masks_[i], &target_cov_mats_[i]); + } +} + +void NonlinearBeamformer::InitDiffuseCovMats() { + for (size_t i = 0; i < kNumFreqBins; ++i) { + uniform_cov_mat_[i].Resize(num_input_channels_, num_input_channels_); + CovarianceMatrixGenerator::UniformCovarianceMatrix( + wave_numbers_[i], array_geometry_, &uniform_cov_mat_[i]); + complex_f normalization_factor = uniform_cov_mat_[i].elements()[0][0]; + uniform_cov_mat_[i].Scale(1.f / normalization_factor); + uniform_cov_mat_[i].Scale(1 - kBalance); + } +} + +void NonlinearBeamformer::InitInterfCovMats() { + for (size_t i = 0; i < kNumFreqBins; ++i) { + interf_cov_mats_[i].clear(); + for (size_t j = 0; j < interf_angles_radians_.size(); ++j) { + interf_cov_mats_[i].push_back(std::unique_ptr( + new ComplexMatrixF(num_input_channels_, num_input_channels_))); + ComplexMatrixF angled_cov_mat(num_input_channels_, num_input_channels_); + CovarianceMatrixGenerator::AngledCovarianceMatrix( + kSpeedOfSoundMeterSeconds, + interf_angles_radians_[j], + i, + kFftSize, + kNumFreqBins, + sample_rate_hz_, + array_geometry_, + &angled_cov_mat); + // Normalize matrices before averaging them. + complex_f normalization_factor = angled_cov_mat.elements()[0][0]; + angled_cov_mat.Scale(1.f / normalization_factor); + // Weighted average of matrices. + angled_cov_mat.Scale(kBalance); + interf_cov_mats_[i][j]->Add(uniform_cov_mat_[i], angled_cov_mat); + } + } +} + +void NonlinearBeamformer::NormalizeCovMats() { + for (size_t i = 0; i < kNumFreqBins; ++i) { + rxiws_[i] = Norm(target_cov_mats_[i], delay_sum_masks_[i]); + rpsiws_[i].clear(); + for (size_t j = 0; j < interf_angles_radians_.size(); ++j) { + rpsiws_[i].push_back(Norm(*interf_cov_mats_[i][j], delay_sum_masks_[i])); + } + } +} + +void NonlinearBeamformer::AnalyzeChunk(const ChannelBuffer& data) { + RTC_DCHECK_EQ(data.num_channels(), num_input_channels_); + RTC_DCHECK_EQ(data.num_frames_per_band(), chunk_length_); + + old_high_pass_mask_ = high_pass_postfilter_mask_; + process_transform_->ProcessChunk(data.channels(0), nullptr); +} + +void NonlinearBeamformer::PostFilter(ChannelBuffer* data) { + RTC_DCHECK_EQ(data->num_frames_per_band(), chunk_length_); + // TODO(aluebs): Change to RTC_CHECK_EQ once the ChannelBuffer is updated. + RTC_DCHECK_GE(data->num_channels(), num_postfilter_channels_); + + postfilter_transform_->ProcessChunk(data->channels(0), final_mask_); + + // Ramp up/down for smoothing is needed in order to avoid discontinuities in + // the transitions between 10 ms frames. + const float ramp_increment = + (high_pass_postfilter_mask_ - old_high_pass_mask_) / + data->num_frames_per_band(); + for (size_t i = 1; i < data->num_bands(); ++i) { + float smoothed_mask = old_high_pass_mask_; + for (size_t j = 0; j < data->num_frames_per_band(); ++j) { + smoothed_mask += ramp_increment; + for (size_t k = 0; k < num_postfilter_channels_; ++k) { + data->channels(i)[k][j] *= smoothed_mask; + } + } + } +} + +void NonlinearBeamformer::AimAt(const SphericalPointf& target_direction) { + target_angle_radians_ = target_direction.azimuth(); + InitHighFrequencyCorrectionRanges(); + InitInterfAngles(); + InitDelaySumMasks(); + InitTargetCovMats(); + InitInterfCovMats(); + NormalizeCovMats(); +} + +bool NonlinearBeamformer::IsInBeam(const SphericalPointf& spherical_point) { + // If more than half-beamwidth degrees away from the beam's center, + // you are out of the beam. + return fabs(spherical_point.azimuth() - target_angle_radians_) < + kHalfBeamWidthRadians; +} + +bool NonlinearBeamformer::is_target_present() { return is_target_present_; } + +void NonlinearBeamformer::ProcessAudioBlock(const complex_f* const* input, + size_t num_input_channels, + size_t num_freq_bins, + size_t num_output_channels, + complex_f* const* output) { + RTC_CHECK_EQ(kNumFreqBins, num_freq_bins); + RTC_CHECK_EQ(num_input_channels_, num_input_channels); + RTC_CHECK_EQ(0, num_output_channels); + + // Calculating the post-filter masks. Note that we need two for each + // frequency bin to account for the positive and negative interferer + // angle. + for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) { + eig_m_.CopyFromColumn(input, i, num_input_channels_); + float eig_m_norm_factor = std::sqrt(SumSquares(eig_m_)); + if (eig_m_norm_factor != 0.f) { + eig_m_.Scale(1.f / eig_m_norm_factor); + } + + float rxim = Norm(target_cov_mats_[i], eig_m_); + float ratio_rxiw_rxim = 0.f; + if (rxim > 0.f) { + ratio_rxiw_rxim = rxiws_[i] / rxim; + } + + complex_f rmw = abs(ConjugateDotProduct(delay_sum_masks_[i], eig_m_)); + rmw *= rmw; + float rmw_r = rmw.real(); + + new_mask_[i] = CalculatePostfilterMask(*interf_cov_mats_[i][0], + rpsiws_[i][0], + ratio_rxiw_rxim, + rmw_r); + for (size_t j = 1; j < interf_angles_radians_.size(); ++j) { + float tmp_mask = CalculatePostfilterMask(*interf_cov_mats_[i][j], + rpsiws_[i][j], + ratio_rxiw_rxim, + rmw_r); + if (tmp_mask < new_mask_[i]) { + new_mask_[i] = tmp_mask; + } + } + } + + ApplyMaskTimeSmoothing(); + EstimateTargetPresence(); + ApplyLowFrequencyCorrection(); + ApplyHighFrequencyCorrection(); + ApplyMaskFrequencySmoothing(); +} + +float NonlinearBeamformer::CalculatePostfilterMask( + const ComplexMatrixF& interf_cov_mat, + float rpsiw, + float ratio_rxiw_rxim, + float rmw_r) { + float rpsim = Norm(interf_cov_mat, eig_m_); + + float ratio = 0.f; + if (rpsim > 0.f) { + ratio = rpsiw / rpsim; + } + + float numerator = 1.f - kCutOffConstant; + if (rmw_r > 0.f) { + numerator = 1.f - std::min(kCutOffConstant, ratio / rmw_r); + } + + float denominator = 1.f - kCutOffConstant; + if (ratio_rxiw_rxim > 0.f) { + denominator = 1.f - std::min(kCutOffConstant, ratio / ratio_rxiw_rxim); + } + + return numerator / denominator; +} + +// Smooth new_mask_ into time_smooth_mask_. +void NonlinearBeamformer::ApplyMaskTimeSmoothing() { + for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) { + time_smooth_mask_[i] = kMaskTimeSmoothAlpha * new_mask_[i] + + (1 - kMaskTimeSmoothAlpha) * time_smooth_mask_[i]; + } +} + +// Copy time_smooth_mask_ to final_mask_ and smooth over frequency. +void NonlinearBeamformer::ApplyMaskFrequencySmoothing() { + // Smooth over frequency in both directions. The "frequency correction" + // regions have constant value, but we enter them to smooth over the jump + // that exists at the boundary. However, this does mean when smoothing "away" + // from the region that we only need to use the last element. + // + // Upward smoothing: + // low_mean_start_bin_ + // v + // |------|------------|------| + // ^------------------>^ + // + // Downward smoothing: + // high_mean_end_bin_ + // v + // |------|------------|------| + // ^<------------------^ + std::copy(time_smooth_mask_, time_smooth_mask_ + kNumFreqBins, final_mask_); + for (size_t i = low_mean_start_bin_; i < kNumFreqBins; ++i) { + final_mask_[i] = kMaskFrequencySmoothAlpha * final_mask_[i] + + (1 - kMaskFrequencySmoothAlpha) * final_mask_[i - 1]; + } + for (size_t i = high_mean_end_bin_ + 1; i > 0; --i) { + final_mask_[i - 1] = kMaskFrequencySmoothAlpha * final_mask_[i - 1] + + (1 - kMaskFrequencySmoothAlpha) * final_mask_[i]; + } +} + +// Apply low frequency correction to time_smooth_mask_. +void NonlinearBeamformer::ApplyLowFrequencyCorrection() { + const float low_frequency_mask = + MaskRangeMean(low_mean_start_bin_, low_mean_end_bin_ + 1); + std::fill(time_smooth_mask_, time_smooth_mask_ + low_mean_start_bin_, + low_frequency_mask); +} + +// Apply high frequency correction to time_smooth_mask_. Update +// high_pass_postfilter_mask_ to use for the high frequency time-domain bands. +void NonlinearBeamformer::ApplyHighFrequencyCorrection() { + high_pass_postfilter_mask_ = + MaskRangeMean(high_mean_start_bin_, high_mean_end_bin_ + 1); + std::fill(time_smooth_mask_ + high_mean_end_bin_ + 1, + time_smooth_mask_ + kNumFreqBins, high_pass_postfilter_mask_); +} + +// Compute mean over the given range of time_smooth_mask_, [first, last). +float NonlinearBeamformer::MaskRangeMean(size_t first, size_t last) { + RTC_DCHECK_GT(last, first); + const float sum = std::accumulate(time_smooth_mask_ + first, + time_smooth_mask_ + last, 0.f); + return sum / (last - first); +} + +void NonlinearBeamformer::EstimateTargetPresence() { + const size_t quantile = static_cast( + (high_mean_end_bin_ - low_mean_start_bin_) * kMaskQuantile + + low_mean_start_bin_); + std::nth_element(new_mask_ + low_mean_start_bin_, new_mask_ + quantile, + new_mask_ + high_mean_end_bin_ + 1); + if (new_mask_[quantile] > kMaskTargetThreshold) { + is_target_present_ = true; + interference_blocks_count_ = 0; + } else { + is_target_present_ = interference_blocks_count_++ < hold_target_blocks_; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h new file mode 100644 index 0000000000..76556e7a45 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_ +#define MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_ + +// MSVC++ requires this to be set before any other includes to get M_PI. +#define _USE_MATH_DEFINES + +#include + +#include +#include + +#include "common_audio/lapped_transform.h" +#include "common_audio/channel_buffer.h" +#include "modules/audio_processing/beamformer/array_util.h" +#include "modules/audio_processing/beamformer/complex_matrix.h" + +namespace webrtc { + +class PostFilterTransform : public LappedTransform::Callback { + public: + PostFilterTransform(size_t num_channels, + size_t chunk_length, + float* window, + size_t fft_size); + + void ProcessChunk(float* const* data, float* final_mask); + + protected: + void ProcessAudioBlock(const complex* const* input, + size_t num_input_channels, + size_t num_freq_bins, + size_t num_output_channels, + complex* const* output) override; + + private: + LappedTransform transform_; + const size_t num_freq_bins_; + float* final_mask_; +}; + +// Enhances sound sources coming directly in front of a uniform linear array +// and suppresses sound sources coming from all other directions. Operates on +// multichannel signals and produces single-channel output. +// +// The implemented nonlinear postfilter algorithm taken from "A Robust Nonlinear +// Beamforming Postprocessor" by Bastiaan Kleijn. +class NonlinearBeamformer : public LappedTransform::Callback { + public: + static const float kHalfBeamWidthRadians; + + explicit NonlinearBeamformer( + const std::vector& array_geometry, + size_t num_postfilter_channels = 1u, + SphericalPointf target_direction = + SphericalPointf(static_cast(M_PI) / 2.f, 0.f, 1.f)); + ~NonlinearBeamformer() override; + + // Sample rate corresponds to the lower band. + // Needs to be called before the NonlinearBeamformer can be used. + virtual void Initialize(int chunk_size_ms, int sample_rate_hz); + + // Analyzes one time-domain chunk of audio. The audio is expected to be split + // into frequency bands inside the ChannelBuffer. The number of frames and + // channels must correspond to the constructor parameters. + virtual void AnalyzeChunk(const ChannelBuffer& data); + + // Applies the postfilter mask to one chunk of audio. The audio is expected to + // be split into frequency bands inside the ChannelBuffer. The number of + // frames and channels must correspond to the constructor parameters. + virtual void PostFilter(ChannelBuffer* data); + + virtual void AimAt(const SphericalPointf& target_direction); + + virtual bool IsInBeam(const SphericalPointf& spherical_point); + + // After processing each block |is_target_present_| is set to true if the + // target signal es present and to false otherwise. This methods can be called + // to know if the data is target signal or interference and process it + // accordingly. + virtual bool is_target_present(); + + protected: + // Process one frequency-domain block of audio. This is where the fun + // happens. Implements LappedTransform::Callback. + void ProcessAudioBlock(const complex* const* input, + size_t num_input_channels, + size_t num_freq_bins, + size_t num_output_channels, + complex* const* output) override; + + private: + FRIEND_TEST_ALL_PREFIXES(NonlinearBeamformerTest, + InterfAnglesTakeAmbiguityIntoAccount); + + typedef Matrix MatrixF; + typedef ComplexMatrix ComplexMatrixF; + typedef complex complex_f; + + void InitLowFrequencyCorrectionRanges(); + void InitHighFrequencyCorrectionRanges(); + void InitInterfAngles(); + void InitDelaySumMasks(); + void InitTargetCovMats(); + void InitDiffuseCovMats(); + void InitInterfCovMats(); + void NormalizeCovMats(); + + // Calculates postfilter masks that minimize the mean squared error of our + // estimation of the desired signal. + float CalculatePostfilterMask(const ComplexMatrixF& interf_cov_mat, + float rpsiw, + float ratio_rxiw_rxim, + float rmxi_r); + + // Prevents the postfilter masks from degenerating too quickly (a cause of + // musical noise). + void ApplyMaskTimeSmoothing(); + void ApplyMaskFrequencySmoothing(); + + // The postfilter masks are unreliable at low frequencies. Calculates a better + // mask by averaging mid-low frequency values. + void ApplyLowFrequencyCorrection(); + + // Postfilter masks are also unreliable at high frequencies. Average mid-high + // frequency masks to calculate a single mask per block which can be applied + // in the time-domain. Further, we average these block-masks over a chunk, + // resulting in one postfilter mask per audio chunk. This allows us to skip + // both transforming and blocking the high-frequency signal. + void ApplyHighFrequencyCorrection(); + + // Compute the means needed for the above frequency correction. + float MaskRangeMean(size_t start_bin, size_t end_bin); + + // Applies post-filter mask to |input| and store in |output|. + void ApplyPostFilter(const complex_f* input, complex_f* output); + + void EstimateTargetPresence(); + + static const size_t kFftSize = 256; + static const size_t kNumFreqBins = kFftSize / 2 + 1; + + // Deals with the fft transform and blocking. + size_t chunk_length_; + std::unique_ptr process_transform_; + std::unique_ptr postfilter_transform_; + float window_[kFftSize]; + + // Parameters exposed to the user. + const size_t num_input_channels_; + const size_t num_postfilter_channels_; + int sample_rate_hz_; + + const std::vector array_geometry_; + // The normal direction of the array if it has one and it is in the xy-plane. + const rtc::Optional array_normal_; + + // Minimum spacing between microphone pairs. + const float min_mic_spacing_; + + // Calculated based on user-input and constants in the .cc file. + size_t low_mean_start_bin_; + size_t low_mean_end_bin_; + size_t high_mean_start_bin_; + size_t high_mean_end_bin_; + + // Quickly varying mask updated every block. + float new_mask_[kNumFreqBins]; + // Time smoothed mask. + float time_smooth_mask_[kNumFreqBins]; + // Time and frequency smoothed mask. + float final_mask_[kNumFreqBins]; + + float target_angle_radians_; + // Angles of the interferer scenarios. + std::vector interf_angles_radians_; + // The angle between the target and the interferer scenarios. + const float away_radians_; + + // Array of length |kNumFreqBins|, Matrix of size |1| x |num_channels_|. + ComplexMatrixF delay_sum_masks_[kNumFreqBins]; + + // Arrays of length |kNumFreqBins|, Matrix of size |num_input_channels_| x + // |num_input_channels_|. + ComplexMatrixF target_cov_mats_[kNumFreqBins]; + ComplexMatrixF uniform_cov_mat_[kNumFreqBins]; + // Array of length |kNumFreqBins|, Matrix of size |num_input_channels_| x + // |num_input_channels_|. The vector has a size equal to the number of + // interferer scenarios. + std::vector> interf_cov_mats_[kNumFreqBins]; + + // Of length |kNumFreqBins|. + float wave_numbers_[kNumFreqBins]; + + // Preallocated for ProcessAudioBlock() + // Of length |kNumFreqBins|. + float rxiws_[kNumFreqBins]; + // The vector has a size equal to the number of interferer scenarios. + std::vector rpsiws_[kNumFreqBins]; + + // The microphone normalization factor. + ComplexMatrixF eig_m_; + + // For processing the high-frequency input signal. + float high_pass_postfilter_mask_; + float old_high_pass_mask_; + + // True when the target signal is present. + bool is_target_present_; + // Number of blocks after which the data is considered interference if the + // mask does not pass |kMaskSignalThreshold|. + size_t hold_target_blocks_; + // Number of blocks since the last mask that passed |kMaskSignalThreshold|. + size_t interference_blocks_count_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_test.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_test.cc new file mode 100644 index 0000000000..296cd6d4ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_test.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "common_audio/channel_buffer.h" +#include "common_audio/wav_file.h" +#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" +#include "modules/audio_processing/test/test_utils.h" +#include "rtc_base/checks.h" +#include "rtc_base/flags.h" +#include "rtc_base/format_macros.h" + +DEFINE_string(i, "", "The name of the input file to read from."); +DEFINE_string(o, "out.wav", "Name of the output file to write to."); +DEFINE_string(mic_positions, "", + "Space delimited cartesian coordinates of microphones in meters. " + "The coordinates of each point are contiguous. " + "For a two element array: \"x1 y1 z1 x2 y2 z2\""); +DEFINE_bool(help, false, "Prints this message."); + +namespace webrtc { +namespace { + +const int kChunksPerSecond = 100; +const int kChunkSizeMs = 1000 / kChunksPerSecond; + +const char kUsage[] = + "Command-line tool to run beamforming on WAV files. The signal is passed\n" + "in as a single band, unlike the audio processing interface which splits\n" + "signals into multiple bands.\n"; + +} // namespace + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || + FLAG_help || argc != 1) { + printf("%s", kUsage); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + + WavReader in_file(FLAG_i); + WavWriter out_file(FLAG_o, in_file.sample_rate(), in_file.num_channels()); + + const size_t num_mics = in_file.num_channels(); + const std::vector array_geometry = + ParseArrayGeometry(FLAG_mic_positions, num_mics); + RTC_CHECK_EQ(array_geometry.size(), num_mics); + + NonlinearBeamformer bf(array_geometry, array_geometry.size()); + bf.Initialize(kChunkSizeMs, in_file.sample_rate()); + + printf("Input file: %s\nChannels: %" PRIuS ", Sample rate: %d Hz\n\n", + FLAG_i, in_file.num_channels(), in_file.sample_rate()); + printf("Output file: %s\nChannels: %" PRIuS ", Sample rate: %d Hz\n\n", + FLAG_o, out_file.num_channels(), out_file.sample_rate()); + + ChannelBuffer buf( + rtc::CheckedDivExact(in_file.sample_rate(), kChunksPerSecond), + in_file.num_channels()); + + std::vector interleaved(buf.size()); + while (in_file.ReadSamples(interleaved.size(), + &interleaved[0]) == interleaved.size()) { + FloatS16ToFloat(&interleaved[0], interleaved.size(), &interleaved[0]); + Deinterleave(&interleaved[0], buf.num_frames(), + buf.num_channels(), buf.channels()); + + bf.AnalyzeChunk(buf); + bf.PostFilter(&buf); + + Interleave(buf.channels(), buf.num_frames(), + buf.num_channels(), &interleaved[0]); + FloatToFloatS16(&interleaved[0], interleaved.size(), &interleaved[0]); + out_file.WriteSamples(&interleaved[0], interleaved.size()); + } + + return 0; +} + +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc new file mode 100644 index 0000000000..78b2f0a68e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// MSVC++ requires this to be set before any other includes to get M_PI. +#define _USE_MATH_DEFINES + +#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kChunkSizeMs = 10; +const int kSampleRateHz = 16000; + +SphericalPointf AzimuthToSphericalPoint(float azimuth_radians) { + return SphericalPointf(azimuth_radians, 0.f, 1.f); +} + +void Verify(NonlinearBeamformer* bf, float target_azimuth_radians) { + EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint(target_azimuth_radians))); + EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint( + target_azimuth_radians - NonlinearBeamformer::kHalfBeamWidthRadians + + 0.001f))); + EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint( + target_azimuth_radians + NonlinearBeamformer::kHalfBeamWidthRadians - + 0.001f))); + EXPECT_FALSE(bf->IsInBeam(AzimuthToSphericalPoint( + target_azimuth_radians - NonlinearBeamformer::kHalfBeamWidthRadians - + 0.001f))); + EXPECT_FALSE(bf->IsInBeam(AzimuthToSphericalPoint( + target_azimuth_radians + NonlinearBeamformer::kHalfBeamWidthRadians + + 0.001f))); +} + +void AimAndVerify(NonlinearBeamformer* bf, float target_azimuth_radians) { + bf->AimAt(AzimuthToSphericalPoint(target_azimuth_radians)); + Verify(bf, target_azimuth_radians); +} + +// Bitexactness test code. +const size_t kNumFramesToProcess = 1000; + +void ProcessOneFrame(int sample_rate_hz, + AudioBuffer* capture_audio_buffer, + NonlinearBeamformer* beamformer) { + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + capture_audio_buffer->SplitIntoFrequencyBands(); + } + + beamformer->AnalyzeChunk(*capture_audio_buffer->split_data_f()); + capture_audio_buffer->set_num_channels(1); + beamformer->PostFilter(capture_audio_buffer->split_data_f()); + + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + capture_audio_buffer->MergeFrequencyBands(); + } +} + +int BeamformerSampleRate(int sample_rate_hz) { + return (sample_rate_hz > AudioProcessing::kSampleRate16kHz + ? AudioProcessing::kSampleRate16kHz + : sample_rate_hz); +} + +void RunBitExactnessTest(int sample_rate_hz, + const std::vector& array_geometry, + const SphericalPointf& target_direction, + rtc::ArrayView output_reference) { + NonlinearBeamformer beamformer(array_geometry, 1u, target_direction); + beamformer.Initialize(AudioProcessing::kChunkSizeMs, + BeamformerSampleRate(sample_rate_hz)); + + const StreamConfig capture_config(sample_rate_hz, array_geometry.size(), + false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(capture_config.num_frames() * + capture_config.num_channels()); + for (size_t frame_no = 0u; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(capture_config.num_frames(), + capture_config.num_channels(), &capture_file, + capture_input); + + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + ProcessOneFrame(sample_rate_hz, &capture_buffer, &beamformer); + } + + // Extract and verify the test results. + std::vector capture_output; + test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer, + &capture_output); + + const float kElementErrorBound = 1.f / static_cast(1 << 15); + + // Compare the output with the reference. Only the first values of the output + // from last frame processed are compared in order not having to specify all + // preceeding frames as testvectors. As the algorithm being tested has a + // memory, testing only the last frame implicitly also tests the preceeding + // frames. + EXPECT_TRUE(test::VerifyDeinterleavedArray( + capture_config.num_frames(), capture_config.num_channels(), + output_reference, capture_output, kElementErrorBound)); +} + +// TODO(peah): Add bitexactness tests for scenarios with more than 2 input +// channels. +std::vector CreateArrayGeometry(int variant) { + std::vector array_geometry; + switch (variant) { + case 1: + array_geometry.push_back(Point(-0.025f, 0.f, 0.f)); + array_geometry.push_back(Point(0.025f, 0.f, 0.f)); + break; + case 2: + array_geometry.push_back(Point(-0.035f, 0.f, 0.f)); + array_geometry.push_back(Point(0.035f, 0.f, 0.f)); + break; + case 3: + array_geometry.push_back(Point(-0.5f, 0.f, 0.f)); + array_geometry.push_back(Point(0.5f, 0.f, 0.f)); + break; + default: + RTC_CHECK(false); + } + return array_geometry; +} + +const SphericalPointf TargetDirection1(0.4f * static_cast(M_PI) / 2.f, + 0.f, + 1.f); +const SphericalPointf TargetDirection2(static_cast(M_PI) / 2.f, + 1.f, + 2.f); + +} // namespace + +TEST(NonlinearBeamformerTest, AimingModifiesBeam) { + std::vector array_geometry; + array_geometry.push_back(Point(-0.025f, 0.f, 0.f)); + array_geometry.push_back(Point(0.025f, 0.f, 0.f)); + NonlinearBeamformer bf(array_geometry, 1u); + bf.Initialize(kChunkSizeMs, kSampleRateHz); + // The default constructor parameter sets the target angle to PI / 2. + Verify(&bf, static_cast(M_PI) / 2.f); + AimAndVerify(&bf, static_cast(M_PI) / 3.f); + AimAndVerify(&bf, 3.f * static_cast(M_PI) / 4.f); + AimAndVerify(&bf, static_cast(M_PI) / 6.f); + AimAndVerify(&bf, static_cast(M_PI)); +} + +TEST(NonlinearBeamformerTest, InterfAnglesTakeAmbiguityIntoAccount) { + { + // For linear arrays there is ambiguity. + std::vector array_geometry; + array_geometry.push_back(Point(-0.1f, 0.f, 0.f)); + array_geometry.push_back(Point(0.f, 0.f, 0.f)); + array_geometry.push_back(Point(0.2f, 0.f, 0.f)); + NonlinearBeamformer bf(array_geometry, 1u); + bf.Initialize(kChunkSizeMs, kSampleRateHz); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, + bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, + bf.interf_angles_radians_[1]); + bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(M_PI - bf.away_radians_ / 2.f, + bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); + } + { + // For planar arrays with normal in the xy-plane there is ambiguity. + std::vector array_geometry; + array_geometry.push_back(Point(-0.1f, 0.f, 0.f)); + array_geometry.push_back(Point(0.f, 0.f, 0.f)); + array_geometry.push_back(Point(0.2f, 0.f, 0.f)); + array_geometry.push_back(Point(0.1f, 0.f, 0.2f)); + array_geometry.push_back(Point(0.f, 0.f, -0.1f)); + NonlinearBeamformer bf(array_geometry, 1u); + bf.Initialize(kChunkSizeMs, kSampleRateHz); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, + bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, + bf.interf_angles_radians_[1]); + bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(M_PI - bf.away_radians_ / 2.f, + bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); + } + { + // For planar arrays with normal not in the xy-plane there is no ambiguity. + std::vector array_geometry; + array_geometry.push_back(Point(0.f, 0.f, 0.f)); + array_geometry.push_back(Point(0.2f, 0.f, 0.f)); + array_geometry.push_back(Point(0.f, 0.1f, -0.2f)); + NonlinearBeamformer bf(array_geometry, 1u); + bf.Initialize(kChunkSizeMs, kSampleRateHz); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, + bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, + bf.interf_angles_radians_[1]); + bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(-bf.away_radians_ / 2.f, bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); + } + { + // For arrays which are not linear or planar there is no ambiguity. + std::vector array_geometry; + array_geometry.push_back(Point(0.f, 0.f, 0.f)); + array_geometry.push_back(Point(0.1f, 0.f, 0.f)); + array_geometry.push_back(Point(0.f, 0.2f, 0.f)); + array_geometry.push_back(Point(0.f, 0.f, 0.3f)); + NonlinearBeamformer bf(array_geometry, 1u); + bf.Initialize(kChunkSizeMs, kSampleRateHz); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, + bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, + bf.interf_angles_radians_[1]); + bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); + EXPECT_EQ(2u, bf.interf_angles_radians_.size()); + EXPECT_FLOAT_EQ(-bf.away_radians_ / 2.f, bf.interf_angles_radians_[0]); + EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); + } +} + +// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in +// this setup. +TEST(BeamformerBitExactnessTest, + DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection1) { + const float kOutputReference[] = {0.001318f, -0.001091f, 0.000990f, + 0.001318f, -0.001091f, 0.000990f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1), + TargetDirection1, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo16kHz_ArrayGeometry1_TargetDirection1) { + const float kOutputReference[] = {-0.000077f, -0.000147f, -0.000138f, + -0.000077f, -0.000147f, -0.000138f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1), + TargetDirection1, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo32kHz_ArrayGeometry1_TargetDirection1) { + const float kOutputReference[] = {-0.000061f, -0.000061f, -0.000061f, + -0.000061f, -0.000061f, -0.000061f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1), + TargetDirection1, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo48kHz_ArrayGeometry1_TargetDirection1) { + const float kOutputReference[] = {0.000450f, 0.000436f, 0.000433f, + 0.000450f, 0.000436f, 0.000433f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1), + TargetDirection1, kOutputReference); +} + +// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in +// this setup. +TEST(BeamformerBitExactnessTest, + DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection2) { + const float kOutputReference[] = {0.001144f, -0.001026f, 0.001074f, + -0.016205f, -0.007324f, -0.015656f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1), + TargetDirection2, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo16kHz_ArrayGeometry1_TargetDirection2) { + const float kOutputReference[] = {0.000221f, -0.000249f, 0.000140f, + 0.000221f, -0.000249f, 0.000140f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1), + TargetDirection2, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo32kHz_ArrayGeometry1_TargetDirection2) { + const float kOutputReference[] = {0.000763f, -0.000336f, 0.000549f, + 0.000763f, -0.000336f, 0.000549f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1), + TargetDirection2, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo48kHz_ArrayGeometry1_TargetDirection2) { + const float kOutputReference[] = {-0.000004f, -0.000494f, 0.000255f, + -0.000004f, -0.000494f, 0.000255f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1), + TargetDirection2, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo8kHz_ArrayGeometry2_TargetDirection2) { + const float kOutputReference[] = {-0.000914f, 0.002170f, -0.002382f, + -0.000914f, 0.002170f, -0.002382f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(2), + TargetDirection2, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo16kHz_ArrayGeometry2_TargetDirection2) { + const float kOutputReference[] = {0.000179f, -0.000179f, 0.000081f, + 0.000179f, -0.000179f, 0.000081f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(2), + TargetDirection2, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo32kHz_ArrayGeometry2_TargetDirection2) { + const float kOutputReference[] = {0.000549f, -0.000214f, 0.000366f, + 0.000549f, -0.000214f, 0.000366f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(2), + TargetDirection2, kOutputReference); +} + +TEST(BeamformerBitExactnessTest, + Stereo48kHz_ArrayGeometry2_TargetDirection2) { + const float kOutputReference[] = {0.000019f, -0.000310f, 0.000182f, + 0.000019f, -0.000310f, 0.000182f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(2), + TargetDirection2, kOutputReference); +} + +// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in +// this setup. +TEST(BeamformerBitExactnessTest, + DISABLED_Stereo16kHz_ArrayGeometry3_TargetDirection1) { + const float kOutputReference[] = {-0.000161f, 0.000171f, -0.000096f, + 0.001007f, 0.000427f, 0.000977f}; + + RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(3), + TargetDirection1, kOutputReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/common.h b/third_party/libwebrtc/webrtc/modules/audio_processing/common.h new file mode 100644 index 0000000000..d8532c5749 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/common.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_COMMON_H_ + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +static inline size_t ChannelsFromLayout(AudioProcessing::ChannelLayout layout) { + switch (layout) { + case AudioProcessing::kMono: + case AudioProcessing::kMonoAndKeyboard: + return 1; + case AudioProcessing::kStereo: + case AudioProcessing::kStereoAndKeyboard: + return 2; + } + RTC_NOTREACHED(); + return 0; +} + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/config_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/config_unittest.cc new file mode 100644 index 0000000000..8776ee3391 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/config_unittest.cc @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/include/config.h" + +#include "test/gtest.h" + +namespace webrtc { +namespace { + +struct MyExperiment { + static const ConfigOptionID identifier = ConfigOptionID::kMyExperimentForTest; + static const int kDefaultFactor; + static const int kDefaultOffset; + + MyExperiment() + : factor(kDefaultFactor), offset(kDefaultOffset) {} + + MyExperiment(int factor, int offset) + : factor(factor), offset(offset) {} + + int factor; + int offset; +}; + +const int MyExperiment::kDefaultFactor = 1; +const int MyExperiment::kDefaultOffset = 2; + +TEST(Config, ReturnsDefaultInstanceIfNotConfigured) { + Config config; + const MyExperiment& my_exp = config.Get(); + EXPECT_EQ(MyExperiment::kDefaultFactor, my_exp.factor); + EXPECT_EQ(MyExperiment::kDefaultOffset, my_exp.offset); +} + +TEST(Config, ReturnOptionWhenSet) { + Config config; + config.Set(new MyExperiment(5, 1)); + const MyExperiment& my_exp = config.Get(); + EXPECT_EQ(5, my_exp.factor); + EXPECT_EQ(1, my_exp.offset); +} + +TEST(Config, SetNullSetsTheOptionBackToDefault) { + Config config; + config.Set(new MyExperiment(5, 1)); + config.Set(NULL); + const MyExperiment& my_exp = config.Get(); + EXPECT_EQ(MyExperiment::kDefaultFactor, my_exp.factor); + EXPECT_EQ(MyExperiment::kDefaultOffset, my_exp.offset); +} + +struct Algo1_CostFunction { + static const ConfigOptionID identifier = + ConfigOptionID::kAlgo1CostFunctionForTest; + Algo1_CostFunction() {} + + virtual int cost(int x) const { + return x; + } + + virtual ~Algo1_CostFunction() {} +}; + +struct SqrCost : Algo1_CostFunction { + virtual int cost(int x) const { + return x*x; + } +}; + +TEST(Config, SupportsPolymorphism) { + Config config; + config.Set(new SqrCost()); + EXPECT_EQ(25, config.Get().cost(5)); +} +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/debug.proto b/third_party/libwebrtc/webrtc/modules/audio_processing/debug.proto new file mode 100644 index 0000000000..44177735e8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/debug.proto @@ -0,0 +1,94 @@ +syntax = "proto2"; +option optimize_for = LITE_RUNTIME; +package webrtc.audioproc; + +// Contains the format of input/output/reverse audio. An Init message is added +// when any of the fields are changed. +message Init { + optional int32 sample_rate = 1; + optional int32 device_sample_rate = 2 [deprecated=true]; + optional int32 num_input_channels = 3; + optional int32 num_output_channels = 4; + optional int32 num_reverse_channels = 5; + optional int32 reverse_sample_rate = 6; + optional int32 output_sample_rate = 7; + optional int32 reverse_output_sample_rate = 8; + optional int32 num_reverse_output_channels = 9; +} + +// May contain interleaved or deinterleaved data, but don't store both formats. +message ReverseStream { + // int16 interleaved data. + optional bytes data = 1; + + // float deinterleaved data, where each repeated element points to a single + // channel buffer of data. + repeated bytes channel = 2; +} + +// May contain interleaved or deinterleaved data, but don't store both formats. +message Stream { + // int16 interleaved data. + optional bytes input_data = 1; + optional bytes output_data = 2; + + optional int32 delay = 3; + optional sint32 drift = 4; + optional int32 level = 5; + optional bool keypress = 6; + + // float deinterleaved data, where each repeated element points to a single + // channel buffer of data. + repeated bytes input_channel = 7; + repeated bytes output_channel = 8; +} + +// Contains the configurations of various APM component. A Config message is +// added when any of the fields are changed. +message Config { + // Next field number 19. + // Acoustic echo canceler. + optional bool aec_enabled = 1; + optional bool aec_delay_agnostic_enabled = 2; + optional bool aec_drift_compensation_enabled = 3; + optional bool aec_extended_filter_enabled = 4; + optional int32 aec_suppression_level = 5; + // Mobile AEC. + optional bool aecm_enabled = 6; + optional bool aecm_comfort_noise_enabled = 7; + optional int32 aecm_routing_mode = 8; + // Automatic gain controller. + optional bool agc_enabled = 9; + optional int32 agc_mode = 10; + optional bool agc_limiter_enabled = 11; + optional bool noise_robust_agc_enabled = 12; + // High pass filter. + optional bool hpf_enabled = 13; + // Noise suppression. + optional bool ns_enabled = 14; + optional int32 ns_level = 15; + // Transient suppression. + optional bool transient_suppression_enabled = 16; + // Semicolon-separated string containing experimental feature + // descriptions. + optional string experiments_description = 17; + // Intelligibility Enhancer + optional bool intelligibility_enhancer_enabled = 18; +} + +message Event { + enum Type { + INIT = 0; + REVERSE_STREAM = 1; + STREAM = 2; + CONFIG = 3; + UNKNOWN_EVENT = 4; + } + + required Type type = 1; + + optional Init init = 2; + optional ReverseStream reverse_stream = 3; + optional Stream stream = 4; + optional Config config = 5; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc new file mode 100644 index 0000000000..857cb1c825 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/echo_cancellation_impl.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kNumFramesToProcess = 100; + +void SetupComponent(int sample_rate_hz, + EchoCancellation::SuppressionLevel suppression_level, + bool drift_compensation_enabled, + EchoCancellationImpl* echo_canceller) { + echo_canceller->Initialize(sample_rate_hz, 1, 1, 1); + EchoCancellation* ec = static_cast(echo_canceller); + ec->Enable(true); + ec->set_suppression_level(suppression_level); + ec->enable_drift_compensation(drift_compensation_enabled); + + Config config; + config.Set(new DelayAgnostic(true)); + config.Set(new ExtendedFilter(true)); + echo_canceller->SetExtraOptions(config); +} + +void ProcessOneFrame(int sample_rate_hz, + int stream_delay_ms, + bool drift_compensation_enabled, + int stream_drift_samples, + AudioBuffer* render_audio_buffer, + AudioBuffer* capture_audio_buffer, + EchoCancellationImpl* echo_canceller) { + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + render_audio_buffer->SplitIntoFrequencyBands(); + capture_audio_buffer->SplitIntoFrequencyBands(); + } + + std::vector render_audio; + EchoCancellationImpl::PackRenderAudioBuffer( + render_audio_buffer, 1, render_audio_buffer->num_channels(), + &render_audio); + echo_canceller->ProcessRenderAudio(render_audio); + + if (drift_compensation_enabled) { + static_cast(echo_canceller) + ->set_stream_drift_samples(stream_drift_samples); + } + + echo_canceller->ProcessCaptureAudio(capture_audio_buffer, stream_delay_ms); + + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + capture_audio_buffer->MergeFrequencyBands(); + } +} + +void RunBitexactnessTest(int sample_rate_hz, + size_t num_channels, + int stream_delay_ms, + bool drift_compensation_enabled, + int stream_drift_samples, + EchoCancellation::SuppressionLevel suppression_level, + bool stream_has_echo_reference, + const rtc::ArrayView& output_reference) { + rtc::CriticalSection crit_render; + rtc::CriticalSection crit_capture; + EchoCancellationImpl echo_canceller(&crit_render, &crit_capture); + SetupComponent(sample_rate_hz, suppression_level, drift_compensation_enabled, + &echo_canceller); + + const int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + const StreamConfig render_config(sample_rate_hz, num_channels, false); + AudioBuffer render_buffer( + render_config.num_frames(), render_config.num_channels(), + render_config.num_frames(), 1, render_config.num_frames()); + test::InputAudioFile render_file( + test::GetApmRenderTestVectorFileName(sample_rate_hz)); + std::vector render_input(samples_per_channel * num_channels); + + const StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), 1, capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(samples_per_channel * num_channels); + + for (int frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &render_file, render_input); + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &capture_file, capture_input); + + test::CopyVectorToAudioBuffer(render_config, render_input, &render_buffer); + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + ProcessOneFrame(sample_rate_hz, stream_delay_ms, drift_compensation_enabled, + stream_drift_samples, &render_buffer, &capture_buffer, + &echo_canceller); + } + + // Extract and verify the test results. + std::vector capture_output; + test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer, + &capture_output); + + EXPECT_EQ(stream_has_echo_reference, + static_cast(&echo_canceller)->stream_has_echo()); + + // Compare the output with the reference. Only the first values of the output + // from last frame processed are compared in order not having to specify all + // preceeding frames as testvectors. As the algorithm being tested has a + // memory, testing only the last frame implicitly also tests the preceeding + // frames. + const float kElementErrorBound = 1.0f / 32768.0f; + EXPECT_TRUE(test::VerifyDeinterleavedArray( + capture_config.num_frames(), capture_config.num_channels(), + output_reference, capture_output, kElementErrorBound)); +} + +const bool kStreamHasEchoReference = true; + +} // namespace + +// TODO(peah): Activate all these tests for ARM and ARM64 once the issue on the +// Chromium ARM and ARM64 boths have been identified. This is tracked in the +// issue https://bugs.chromium.org/p/webrtc/issues/detail?id=5711. + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono8kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono8kHz_HighLevel_NoDrift_StreamDelay0) { +#endif + const float kOutputReference[] = {-0.000646f, -0.001525f, 0.002688f}; + RunBitexactnessTest(8000, 1, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono16kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono16kHz_HighLevel_NoDrift_StreamDelay0) { +#endif + const float kOutputReference[] = {0.000055f, 0.000421f, 0.001149f}; + RunBitexactnessTest(16000, 1, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono32kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono32kHz_HighLevel_NoDrift_StreamDelay0) { +#endif + const float kOutputReference[] = {-0.000671f, 0.000061f, -0.000031f}; + RunBitexactnessTest(32000, 1, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono48kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono48kHz_HighLevel_NoDrift_StreamDelay0) { +#endif + const float kOutputReference[] = {-0.001403f, -0.001411f, -0.000755f}; + RunBitexactnessTest(48000, 1, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono16kHz_LowLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono16kHz_LowLevel_NoDrift_StreamDelay0) { +#endif +#if defined(WEBRTC_MAC) + const float kOutputReference[] = {-0.000145f, 0.000179f, 0.000917f}; +#else + const float kOutputReference[] = {-0.000009f, 0.000363f, 0.001094f}; +#endif + RunBitexactnessTest(16000, 1, 0, false, 0, + EchoCancellation::SuppressionLevel::kLowSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono16kHz_ModerateLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono16kHz_ModerateLevel_NoDrift_StreamDelay0) { +#endif + const float kOutputReference[] = {0.000055f, 0.000421f, 0.001149f}; + RunBitexactnessTest(16000, 1, 0, false, 0, + EchoCancellation::SuppressionLevel::kModerateSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono16kHz_HighLevel_NoDrift_StreamDelay10) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono16kHz_HighLevel_NoDrift_StreamDelay10) { +#endif + const float kOutputReference[] = {0.000055f, 0.000421f, 0.001149f}; + RunBitexactnessTest(16000, 1, 10, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono16kHz_HighLevel_NoDrift_StreamDelay20) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono16kHz_HighLevel_NoDrift_StreamDelay20) { +#endif + const float kOutputReference[] = {0.000055f, 0.000421f, 0.001149f}; + RunBitexactnessTest(16000, 1, 20, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono16kHz_HighLevel_Drift0_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono16kHz_HighLevel_Drift0_StreamDelay0) { +#endif + const float kOutputReference[] = {0.000055f, 0.000421f, 0.001149f}; + RunBitexactnessTest(16000, 1, 0, true, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Mono16kHz_HighLevel_Drift5_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Mono16kHz_HighLevel_Drift5_StreamDelay0) { +#endif + const float kOutputReference[] = {0.000055f, 0.000421f, 0.001149f}; + RunBitexactnessTest(16000, 1, 0, true, 5, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Stereo8kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Stereo8kHz_HighLevel_NoDrift_StreamDelay0) { +#endif +#if defined(WEBRTC_MAC) + const float kOutputReference[] = {-0.000392f, -0.001449f, 0.003004f, + -0.000392f, -0.001449f, 0.003004f}; +#else + const float kOutputReference[] = {-0.000464f, -0.001525f, 0.002933f, + -0.000464f, -0.001525f, 0.002933f}; +#endif + RunBitexactnessTest(8000, 2, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Stereo16kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Stereo16kHz_HighLevel_NoDrift_StreamDelay0) { +#endif + const float kOutputReference[] = {0.000166f, 0.000735f, 0.000841f, + 0.000166f, 0.000735f, 0.000841f}; + RunBitexactnessTest(16000, 2, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Stereo32kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Stereo32kHz_HighLevel_NoDrift_StreamDelay0) { +#endif +#if defined(WEBRTC_MAC) + const float kOutputReference[] = {-0.000458f, 0.000244f, 0.000153f, + -0.000458f, 0.000244f, 0.000153f}; +#else + const float kOutputReference[] = {-0.000427f, 0.000183f, 0.000183f, + -0.000427f, 0.000183f, 0.000183f}; +#endif + RunBitexactnessTest(32000, 2, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(EchoCancellationBitExactnessTest, + Stereo48kHz_HighLevel_NoDrift_StreamDelay0) { +#else +TEST(EchoCancellationBitExactnessTest, + DISABLED_Stereo48kHz_HighLevel_NoDrift_StreamDelay0) { +#endif + const float kOutputReference[] = {-0.001101f, -0.001101f, -0.000449f, + -0.001101f, -0.001101f, -0.000449f}; + RunBitexactnessTest(48000, 2, 0, false, 0, + EchoCancellation::SuppressionLevel::kHighSuppression, + kStreamHasEchoReference, kOutputReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.cc new file mode 100644 index 0000000000..99f676c562 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.cc @@ -0,0 +1,516 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_cancellation_impl.h" + +#include + +#include "modules/audio_processing/aec/aec_core.h" +#include "modules/audio_processing/aec/echo_cancellation.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { +int16_t MapSetting(EchoCancellation::SuppressionLevel level) { + switch (level) { + case EchoCancellation::kLowSuppression: + return kAecNlpConservative; + case EchoCancellation::kModerateSuppression: + return kAecNlpModerate; + case EchoCancellation::kHighSuppression: + return kAecNlpAggressive; + } + RTC_NOTREACHED(); + return -1; +} + +AudioProcessing::Error MapError(int err) { + switch (err) { + case AEC_UNSUPPORTED_FUNCTION_ERROR: + return AudioProcessing::kUnsupportedFunctionError; + case AEC_BAD_PARAMETER_ERROR: + return AudioProcessing::kBadParameterError; + case AEC_BAD_PARAMETER_WARNING: + return AudioProcessing::kBadStreamParameterWarning; + default: + // AEC_UNSPECIFIED_ERROR + // AEC_UNINITIALIZED_ERROR + // AEC_NULL_POINTER_ERROR + return AudioProcessing::kUnspecifiedError; + } +} + +bool EnforceZeroStreamDelay() { +#if defined(CHROMEOS) + return !field_trial::IsEnabled("WebRTC-Aec2ZeroStreamDelayKillSwitch"); +#else + return false; +#endif +} + +} // namespace + +struct EchoCancellationImpl::StreamProperties { + StreamProperties() = delete; + StreamProperties(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels, + size_t num_proc_channels) + : sample_rate_hz(sample_rate_hz), + num_reverse_channels(num_reverse_channels), + num_output_channels(num_output_channels), + num_proc_channels(num_proc_channels) {} + + const int sample_rate_hz; + const size_t num_reverse_channels; + const size_t num_output_channels; + const size_t num_proc_channels; +}; + +class EchoCancellationImpl::Canceller { + public: + Canceller() { + state_ = WebRtcAec_Create(); + RTC_DCHECK(state_); + } + + ~Canceller() { + RTC_CHECK(state_); + WebRtcAec_Free(state_); + } + + void* state() { return state_; } + + void Initialize(int sample_rate_hz) { + // TODO(ajm): Drift compensation is disabled in practice. If restored, it + // should be managed internally and not depend on the hardware sample rate. + // For now, just hardcode a 48 kHz value. + const int error = WebRtcAec_Init(state_, sample_rate_hz, 48000); + RTC_DCHECK_EQ(0, error); + } + + private: + void* state_; +}; + +EchoCancellationImpl::EchoCancellationImpl(rtc::CriticalSection* crit_render, + rtc::CriticalSection* crit_capture) + : crit_render_(crit_render), + crit_capture_(crit_capture), + drift_compensation_enabled_(false), + metrics_enabled_(false), + suppression_level_(kModerateSuppression), + stream_drift_samples_(0), + was_stream_drift_set_(false), + stream_has_echo_(false), + delay_logging_enabled_(false), + extended_filter_enabled_(false), + delay_agnostic_enabled_(false), + enforce_zero_stream_delay_(EnforceZeroStreamDelay()) { + RTC_DCHECK(crit_render); + RTC_DCHECK(crit_capture); +} + +EchoCancellationImpl::~EchoCancellationImpl() = default; + +void EchoCancellationImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { + rtc::CritScope cs_capture(crit_capture_); + if (!enabled_) { + return; + } + + RTC_DCHECK(stream_properties_); + size_t handle_index = 0; + size_t buffer_index = 0; + const size_t num_frames_per_band = + packed_render_audio.size() / (stream_properties_->num_output_channels * + stream_properties_->num_reverse_channels); + for (size_t i = 0; i < stream_properties_->num_output_channels; i++) { + for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) { + WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(), + &packed_render_audio[buffer_index], + num_frames_per_band); + + buffer_index += num_frames_per_band; + } + } +} + + +int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio, + int stream_delay_ms) { + rtc::CritScope cs_capture(crit_capture_); + if (!enabled_) { + return AudioProcessing::kNoError; + } + + const int stream_delay_ms_use = + enforce_zero_stream_delay_ ? 0 : stream_delay_ms; + + if (drift_compensation_enabled_ && !was_stream_drift_set_) { + return AudioProcessing::kStreamParameterNotSetError; + } + + RTC_DCHECK(stream_properties_); + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(audio->num_channels(), stream_properties_->num_proc_channels); + + int err = AudioProcessing::kNoError; + + // The ordering convention must be followed to pass to the correct AEC. + size_t handle_index = 0; + stream_has_echo_ = false; + for (size_t i = 0; i < audio->num_channels(); i++) { + for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) { + err = WebRtcAec_Process(cancellers_[handle_index]->state(), + audio->split_bands_const_f(i), audio->num_bands(), + audio->split_bands_f(i), + audio->num_frames_per_band(), stream_delay_ms_use, + stream_drift_samples_); + + if (err != AudioProcessing::kNoError) { + err = MapError(err); + // TODO(ajm): Figure out how to return warnings properly. + if (err != AudioProcessing::kBadStreamParameterWarning) { + return err; + } + } + + int status = 0; + err = WebRtcAec_get_echo_status(cancellers_[handle_index]->state(), + &status); + if (err != AudioProcessing::kNoError) { + return MapError(err); + } + + if (status == 1) { + stream_has_echo_ = true; + } + + handle_index++; + } + } + + was_stream_drift_set_ = false; + return AudioProcessing::kNoError; +} + +int EchoCancellationImpl::Enable(bool enable) { + // Run in a single-threaded manner. + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + + if (enable && !enabled_) { + enabled_ = enable; // Must be set before Initialize() is called. + + // TODO(peah): Simplify once the Enable function has been removed from + // the public APM API. + RTC_DCHECK(stream_properties_); + Initialize(stream_properties_->sample_rate_hz, + stream_properties_->num_reverse_channels, + stream_properties_->num_output_channels, + stream_properties_->num_proc_channels); + } else { + enabled_ = enable; + } + return AudioProcessing::kNoError; +} + +bool EchoCancellationImpl::is_enabled() const { + rtc::CritScope cs(crit_capture_); + return enabled_; +} + +int EchoCancellationImpl::set_suppression_level(SuppressionLevel level) { + { + if (MapSetting(level) == -1) { + return AudioProcessing::kBadParameterError; + } + rtc::CritScope cs(crit_capture_); + suppression_level_ = level; + } + return Configure(); +} + +EchoCancellation::SuppressionLevel EchoCancellationImpl::suppression_level() + const { + rtc::CritScope cs(crit_capture_); + return suppression_level_; +} + +int EchoCancellationImpl::enable_drift_compensation(bool enable) { + { + rtc::CritScope cs(crit_capture_); + drift_compensation_enabled_ = enable; + } + return Configure(); +} + +bool EchoCancellationImpl::is_drift_compensation_enabled() const { + rtc::CritScope cs(crit_capture_); + return drift_compensation_enabled_; +} + +void EchoCancellationImpl::set_stream_drift_samples(int drift) { + rtc::CritScope cs(crit_capture_); + was_stream_drift_set_ = true; + stream_drift_samples_ = drift; +} + +int EchoCancellationImpl::stream_drift_samples() const { + rtc::CritScope cs(crit_capture_); + return stream_drift_samples_; +} + +int EchoCancellationImpl::enable_metrics(bool enable) { + { + rtc::CritScope cs(crit_capture_); + metrics_enabled_ = enable; + } + return Configure(); +} + +bool EchoCancellationImpl::are_metrics_enabled() const { + rtc::CritScope cs(crit_capture_); + return enabled_ && metrics_enabled_; +} + +// TODO(ajm): we currently just use the metrics from the first AEC. Think more +// aboue the best way to extend this to multi-channel. +int EchoCancellationImpl::GetMetrics(Metrics* metrics) { + rtc::CritScope cs(crit_capture_); + if (metrics == NULL) { + return AudioProcessing::kNullPointerError; + } + + if (!enabled_ || !metrics_enabled_) { + return AudioProcessing::kNotEnabledError; + } + + AecMetrics my_metrics; + memset(&my_metrics, 0, sizeof(my_metrics)); + memset(metrics, 0, sizeof(Metrics)); + + const int err = WebRtcAec_GetMetrics(cancellers_[0]->state(), &my_metrics); + if (err != AudioProcessing::kNoError) { + return MapError(err); + } + + metrics->residual_echo_return_loss.instant = my_metrics.rerl.instant; + metrics->residual_echo_return_loss.average = my_metrics.rerl.average; + metrics->residual_echo_return_loss.maximum = my_metrics.rerl.max; + metrics->residual_echo_return_loss.minimum = my_metrics.rerl.min; + + metrics->echo_return_loss.instant = my_metrics.erl.instant; + metrics->echo_return_loss.average = my_metrics.erl.average; + metrics->echo_return_loss.maximum = my_metrics.erl.max; + metrics->echo_return_loss.minimum = my_metrics.erl.min; + + metrics->echo_return_loss_enhancement.instant = my_metrics.erle.instant; + metrics->echo_return_loss_enhancement.average = my_metrics.erle.average; + metrics->echo_return_loss_enhancement.maximum = my_metrics.erle.max; + metrics->echo_return_loss_enhancement.minimum = my_metrics.erle.min; + + metrics->a_nlp.instant = my_metrics.aNlp.instant; + metrics->a_nlp.average = my_metrics.aNlp.average; + metrics->a_nlp.maximum = my_metrics.aNlp.max; + metrics->a_nlp.minimum = my_metrics.aNlp.min; + + metrics->divergent_filter_fraction = my_metrics.divergent_filter_fraction; + return AudioProcessing::kNoError; +} + +bool EchoCancellationImpl::stream_has_echo() const { + rtc::CritScope cs(crit_capture_); + return stream_has_echo_; +} + +int EchoCancellationImpl::enable_delay_logging(bool enable) { + { + rtc::CritScope cs(crit_capture_); + delay_logging_enabled_ = enable; + } + return Configure(); +} + +bool EchoCancellationImpl::is_delay_logging_enabled() const { + rtc::CritScope cs(crit_capture_); + return enabled_ && delay_logging_enabled_; +} + +bool EchoCancellationImpl::is_delay_agnostic_enabled() const { + rtc::CritScope cs(crit_capture_); + return delay_agnostic_enabled_; +} + +std::string EchoCancellationImpl::GetExperimentsDescription() { + rtc::CritScope cs(crit_capture_); + return refined_adaptive_filter_enabled_ ? "RefinedAdaptiveFilter;" : ""; +} + +bool EchoCancellationImpl::is_refined_adaptive_filter_enabled() const { + rtc::CritScope cs(crit_capture_); + return refined_adaptive_filter_enabled_; +} + +bool EchoCancellationImpl::is_extended_filter_enabled() const { + rtc::CritScope cs(crit_capture_); + return extended_filter_enabled_; +} + +// TODO(bjornv): How should we handle the multi-channel case? +int EchoCancellationImpl::GetDelayMetrics(int* median, int* std) { + rtc::CritScope cs(crit_capture_); + float fraction_poor_delays = 0; + return GetDelayMetrics(median, std, &fraction_poor_delays); +} + +int EchoCancellationImpl::GetDelayMetrics(int* median, int* std, + float* fraction_poor_delays) { + rtc::CritScope cs(crit_capture_); + if (median == NULL) { + return AudioProcessing::kNullPointerError; + } + if (std == NULL) { + return AudioProcessing::kNullPointerError; + } + + if (!enabled_ || !delay_logging_enabled_) { + return AudioProcessing::kNotEnabledError; + } + + const int err = WebRtcAec_GetDelayMetrics(cancellers_[0]->state(), median, + std, fraction_poor_delays); + if (err != AudioProcessing::kNoError) { + return MapError(err); + } + + return AudioProcessing::kNoError; +} + +struct AecCore* EchoCancellationImpl::aec_core() const { + rtc::CritScope cs(crit_capture_); + if (!enabled_) { + return NULL; + } + return WebRtcAec_aec_core(cancellers_[0]->state()); +} + +void EchoCancellationImpl::Initialize(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels, + size_t num_proc_channels) { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + + stream_properties_.reset( + new StreamProperties(sample_rate_hz, num_reverse_channels, + num_output_channels, num_proc_channels)); + + if (!enabled_) { + return; + } + + const size_t num_cancellers_required = + NumCancellersRequired(stream_properties_->num_output_channels, + stream_properties_->num_reverse_channels); + if (num_cancellers_required > cancellers_.size()) { + const size_t cancellers_old_size = cancellers_.size(); + cancellers_.resize(num_cancellers_required); + + for (size_t i = cancellers_old_size; i < cancellers_.size(); ++i) { + cancellers_[i].reset(new Canceller()); + } + } + + for (auto& canceller : cancellers_) { + canceller->Initialize(sample_rate_hz); + } + + Configure(); +} + +int EchoCancellationImpl::GetSystemDelayInSamples() const { + rtc::CritScope cs(crit_capture_); + RTC_DCHECK(enabled_); + // Report the delay for the first AEC component. + return WebRtcAec_system_delay( + WebRtcAec_aec_core(cancellers_[0]->state())); +} + +void EchoCancellationImpl::PackRenderAudioBuffer( + const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(num_channels, audio->num_channels()); + + packed_buffer->clear(); + // The ordering convention must be followed to pass the correct data. + for (size_t i = 0; i < num_output_channels; i++) { + for (size_t j = 0; j < audio->num_channels(); j++) { + // Buffer the samples in the render queue. + packed_buffer->insert(packed_buffer->end(), + audio->split_bands_const_f(j)[kBand0To8kHz], + (audio->split_bands_const_f(j)[kBand0To8kHz] + + audio->num_frames_per_band())); + } + } +} + +void EchoCancellationImpl::SetExtraOptions(const webrtc::Config& config) { + { + rtc::CritScope cs(crit_capture_); + extended_filter_enabled_ = config.Get().enabled; + delay_agnostic_enabled_ = config.Get().enabled; + refined_adaptive_filter_enabled_ = + config.Get().enabled; + } + Configure(); +} + +int EchoCancellationImpl::Configure() { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + AecConfig config; + config.metricsMode = metrics_enabled_; + config.nlpMode = MapSetting(suppression_level_); + config.skewMode = drift_compensation_enabled_; + config.delay_logging = delay_logging_enabled_; + + int error = AudioProcessing::kNoError; + for (auto& canceller : cancellers_) { + WebRtcAec_enable_extended_filter(WebRtcAec_aec_core(canceller->state()), + extended_filter_enabled_ ? 1 : 0); + WebRtcAec_enable_delay_agnostic(WebRtcAec_aec_core(canceller->state()), + delay_agnostic_enabled_ ? 1 : 0); + WebRtcAec_enable_refined_adaptive_filter( + WebRtcAec_aec_core(canceller->state()), + refined_adaptive_filter_enabled_); + const int handle_error = WebRtcAec_set_config(canceller->state(), config); + if (handle_error != AudioProcessing::kNoError) { + error = AudioProcessing::kNoError; + } + } + return error; +} + +size_t EchoCancellationImpl::NumCancellersRequired( + size_t num_output_channels, + size_t num_reverse_channels) { + return num_output_channels * num_reverse_channels; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.h new file mode 100644 index 0000000000..6700249da0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl.h @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_ECHO_CANCELLATION_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_ECHO_CANCELLATION_IMPL_H_ + +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +class AudioBuffer; + +class EchoCancellationImpl : public EchoCancellation { + public: + EchoCancellationImpl(rtc::CriticalSection* crit_render, + rtc::CriticalSection* crit_capture); + ~EchoCancellationImpl() override; + + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); + int ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms); + + // EchoCancellation implementation. + bool is_enabled() const override; + int stream_drift_samples() const override; + SuppressionLevel suppression_level() const override; + bool is_drift_compensation_enabled() const override; + + void Initialize(int sample_rate_hz, + size_t num_reverse_channels_, + size_t num_output_channels_, + size_t num_proc_channels_); + void SetExtraOptions(const webrtc::Config& config); + bool is_delay_agnostic_enabled() const; + bool is_extended_filter_enabled() const; + std::string GetExperimentsDescription(); + bool is_refined_adaptive_filter_enabled() const; + + // Returns the system delay of the first AEC component. + int GetSystemDelayInSamples() const; + + static void PackRenderAudioBuffer(const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer); + static size_t NumCancellersRequired(size_t num_output_channels, + size_t num_reverse_channels); + + // Enable logging of various AEC statistics. + int enable_metrics(bool enable) override; + + // Provides various statistics about the AEC. + int GetMetrics(Metrics* metrics) override; + + // Enable logging of delay metrics. + int enable_delay_logging(bool enable) override; + + // Provides delay metrics. + int GetDelayMetrics(int* median, + int* std, + float* fraction_poor_delays) override; + + private: + class Canceller; + struct StreamProperties; + + // EchoCancellation implementation. + int Enable(bool enable) override; + int enable_drift_compensation(bool enable) override; + void set_stream_drift_samples(int drift) override; + int set_suppression_level(SuppressionLevel level) override; + bool are_metrics_enabled() const override; + bool stream_has_echo() const override; + bool is_delay_logging_enabled() const override; + int GetDelayMetrics(int* median, int* std) override; + + struct AecCore* aec_core() const override; + + void AllocateRenderQueue(); + int Configure(); + + rtc::CriticalSection* const crit_render_ RTC_ACQUIRED_BEFORE(crit_capture_); + rtc::CriticalSection* const crit_capture_; + + bool enabled_ = false; + bool drift_compensation_enabled_ RTC_GUARDED_BY(crit_capture_); + bool metrics_enabled_ RTC_GUARDED_BY(crit_capture_); + SuppressionLevel suppression_level_ RTC_GUARDED_BY(crit_capture_); + int stream_drift_samples_ RTC_GUARDED_BY(crit_capture_); + bool was_stream_drift_set_ RTC_GUARDED_BY(crit_capture_); + bool stream_has_echo_ RTC_GUARDED_BY(crit_capture_); + bool delay_logging_enabled_ RTC_GUARDED_BY(crit_capture_); + bool extended_filter_enabled_ RTC_GUARDED_BY(crit_capture_); + bool delay_agnostic_enabled_ RTC_GUARDED_BY(crit_capture_); + bool refined_adaptive_filter_enabled_ RTC_GUARDED_BY(crit_capture_) = false; + + // Only active on Chrome OS devices. + const bool enforce_zero_stream_delay_ RTC_GUARDED_BY(crit_capture_); + + std::vector> cancellers_; + std::unique_ptr stream_properties_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoCancellationImpl); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_ECHO_CANCELLATION_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc new file mode 100644 index 0000000000..2b9e5c5626 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_cancellation_impl_unittest.cc @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/aec/aec_core.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(EchoCancellationInternalTest, ExtendedFilter) { + std::unique_ptr ap(AudioProcessing::Create()); + EXPECT_TRUE(ap->echo_cancellation()->aec_core() == NULL); + + EXPECT_EQ(ap->kNoError, ap->echo_cancellation()->Enable(true)); + EXPECT_TRUE(ap->echo_cancellation()->is_enabled()); + + AecCore* aec_core = ap->echo_cancellation()->aec_core(); + ASSERT_TRUE(aec_core != NULL); + // Disabled by default. + EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(aec_core)); + + Config config; + config.Set(new ExtendedFilter(true)); + ap->SetExtraOptions(config); + EXPECT_EQ(1, WebRtcAec_extended_filter_enabled(aec_core)); + + // Retains setting after initialization. + EXPECT_EQ(ap->kNoError, ap->Initialize()); + EXPECT_EQ(1, WebRtcAec_extended_filter_enabled(aec_core)); + + config.Set(new ExtendedFilter(false)); + ap->SetExtraOptions(config); + EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(aec_core)); + + // Retains setting after initialization. + EXPECT_EQ(ap->kNoError, ap->Initialize()); + EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(aec_core)); +} + +TEST(EchoCancellationInternalTest, DelayAgnostic) { + std::unique_ptr ap(AudioProcessing::Create()); + EXPECT_TRUE(ap->echo_cancellation()->aec_core() == NULL); + + EXPECT_EQ(ap->kNoError, ap->echo_cancellation()->Enable(true)); + EXPECT_TRUE(ap->echo_cancellation()->is_enabled()); + + AecCore* aec_core = ap->echo_cancellation()->aec_core(); + ASSERT_TRUE(aec_core != NULL); + // Enabled by default. + EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(aec_core)); + + Config config; + config.Set(new DelayAgnostic(true)); + ap->SetExtraOptions(config); + EXPECT_EQ(1, WebRtcAec_delay_agnostic_enabled(aec_core)); + + // Retains setting after initialization. + EXPECT_EQ(ap->kNoError, ap->Initialize()); + EXPECT_EQ(1, WebRtcAec_delay_agnostic_enabled(aec_core)); + + config.Set(new DelayAgnostic(false)); + ap->SetExtraOptions(config); + EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(aec_core)); + + // Retains setting after initialization. + EXPECT_EQ(ap->kNoError, ap->Initialize()); + EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(aec_core)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.cc new file mode 100644 index 0000000000..ecb1810666 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.cc @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_control_mobile_impl.h" + +#include + +#include "modules/audio_processing/aecm/echo_control_mobile.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +namespace { +int16_t MapSetting(EchoControlMobile::RoutingMode mode) { + switch (mode) { + case EchoControlMobile::kQuietEarpieceOrHeadset: + return 0; + case EchoControlMobile::kEarpiece: + return 1; + case EchoControlMobile::kLoudEarpiece: + return 2; + case EchoControlMobile::kSpeakerphone: + return 3; + case EchoControlMobile::kLoudSpeakerphone: + return 4; + } + RTC_NOTREACHED(); + return -1; +} + +AudioProcessing::Error MapError(int err) { + switch (err) { + case AECM_UNSUPPORTED_FUNCTION_ERROR: + return AudioProcessing::kUnsupportedFunctionError; + case AECM_NULL_POINTER_ERROR: + return AudioProcessing::kNullPointerError; + case AECM_BAD_PARAMETER_ERROR: + return AudioProcessing::kBadParameterError; + case AECM_BAD_PARAMETER_WARNING: + return AudioProcessing::kBadStreamParameterWarning; + default: + // AECM_UNSPECIFIED_ERROR + // AECM_UNINITIALIZED_ERROR + return AudioProcessing::kUnspecifiedError; + } +} +} // namespace + +size_t EchoControlMobile::echo_path_size_bytes() { + return WebRtcAecm_echo_path_size_bytes(); +} + +struct EchoControlMobileImpl::StreamProperties { + StreamProperties() = delete; + StreamProperties(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels) + : sample_rate_hz(sample_rate_hz), + num_reverse_channels(num_reverse_channels), + num_output_channels(num_output_channels) {} + + int sample_rate_hz; + size_t num_reverse_channels; + size_t num_output_channels; +}; + +class EchoControlMobileImpl::Canceller { + public: + Canceller() { + state_ = WebRtcAecm_Create(); + RTC_CHECK(state_); + } + + ~Canceller() { + RTC_DCHECK(state_); + WebRtcAecm_Free(state_); + } + + void* state() { + RTC_DCHECK(state_); + return state_; + } + + void Initialize(int sample_rate_hz, + unsigned char* external_echo_path, + size_t echo_path_size_bytes) { + RTC_DCHECK(state_); + int error = WebRtcAecm_Init(state_, sample_rate_hz); + RTC_DCHECK_EQ(AudioProcessing::kNoError, error); + if (external_echo_path != NULL) { + error = WebRtcAecm_InitEchoPath(state_, external_echo_path, + echo_path_size_bytes); + RTC_DCHECK_EQ(AudioProcessing::kNoError, error); + } + } + + private: + void* state_; + RTC_DISALLOW_COPY_AND_ASSIGN(Canceller); +}; + +EchoControlMobileImpl::EchoControlMobileImpl(rtc::CriticalSection* crit_render, + rtc::CriticalSection* crit_capture) + : crit_render_(crit_render), + crit_capture_(crit_capture), + routing_mode_(kSpeakerphone), + comfort_noise_enabled_(true), + external_echo_path_(NULL) { + RTC_DCHECK(crit_render); + RTC_DCHECK(crit_capture); +} + +EchoControlMobileImpl::~EchoControlMobileImpl() { + if (external_echo_path_ != NULL) { + delete [] external_echo_path_; + external_echo_path_ = NULL; + } +} + +void EchoControlMobileImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { + rtc::CritScope cs_capture(crit_capture_); + if (!enabled_) { + return; + } + + RTC_DCHECK(stream_properties_); + + size_t buffer_index = 0; + size_t num_frames_per_band = + packed_render_audio.size() / (stream_properties_->num_output_channels * + stream_properties_->num_reverse_channels); + + for (auto& canceller : cancellers_) { + WebRtcAecm_BufferFarend(canceller->state(), + &packed_render_audio[buffer_index], + num_frames_per_band); + + buffer_index += num_frames_per_band; + } +} + +void EchoControlMobileImpl::PackRenderAudioBuffer( + const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(num_channels, audio->num_channels()); + + // The ordering convention must be followed to pass to the correct AECM. + packed_buffer->clear(); + int render_channel = 0; + for (size_t i = 0; i < num_output_channels; i++) { + for (size_t j = 0; j < audio->num_channels(); j++) { + // Buffer the samples in the render queue. + packed_buffer->insert( + packed_buffer->end(), + audio->split_bands_const(render_channel)[kBand0To8kHz], + (audio->split_bands_const(render_channel)[kBand0To8kHz] + + audio->num_frames_per_band())); + render_channel = (render_channel + 1) % audio->num_channels(); + } + } +} + +size_t EchoControlMobileImpl::NumCancellersRequired( + size_t num_output_channels, + size_t num_reverse_channels) { + return num_output_channels * num_reverse_channels; +} + +int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio, + int stream_delay_ms) { + rtc::CritScope cs_capture(crit_capture_); + if (!enabled_) { + return AudioProcessing::kNoError; + } + + RTC_DCHECK(stream_properties_); + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(audio->num_channels(), stream_properties_->num_output_channels); + RTC_DCHECK_GE(cancellers_.size(), stream_properties_->num_reverse_channels * + audio->num_channels()); + + int err = AudioProcessing::kNoError; + + // The ordering convention must be followed to pass to the correct AECM. + size_t handle_index = 0; + for (size_t capture = 0; capture < audio->num_channels(); ++capture) { + // TODO(ajm): improve how this works, possibly inside AECM. + // This is kind of hacked up. + const int16_t* noisy = audio->low_pass_reference(capture); + const int16_t* clean = audio->split_bands_const(capture)[kBand0To8kHz]; + if (noisy == NULL) { + noisy = clean; + clean = NULL; + } + for (size_t render = 0; render < stream_properties_->num_reverse_channels; + ++render) { + err = WebRtcAecm_Process(cancellers_[handle_index]->state(), noisy, clean, + audio->split_bands(capture)[kBand0To8kHz], + audio->num_frames_per_band(), stream_delay_ms); + + if (err != AudioProcessing::kNoError) { + return MapError(err); + } + + ++handle_index; + } + for (size_t band = 1u; band < audio->num_bands(); ++band) { + memset(audio->split_bands(capture)[band], + 0, + audio->num_frames_per_band() * + sizeof(audio->split_bands(capture)[band][0])); + } + } + return AudioProcessing::kNoError; +} + +int EchoControlMobileImpl::Enable(bool enable) { + // Ensure AEC and AECM are not both enabled. + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + RTC_DCHECK(stream_properties_); + + if (enable && + stream_properties_->sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + return AudioProcessing::kBadSampleRateError; + } + + if (enable && !enabled_) { + enabled_ = enable; // Must be set before Initialize() is called. + + // TODO(peah): Simplify once the Enable function has been removed from + // the public APM API. + Initialize(stream_properties_->sample_rate_hz, + stream_properties_->num_reverse_channels, + stream_properties_->num_output_channels); + } else { + enabled_ = enable; + } + return AudioProcessing::kNoError; +} + +bool EchoControlMobileImpl::is_enabled() const { + rtc::CritScope cs(crit_capture_); + return enabled_; +} + +int EchoControlMobileImpl::set_routing_mode(RoutingMode mode) { + if (MapSetting(mode) == -1) { + return AudioProcessing::kBadParameterError; + } + + { + rtc::CritScope cs(crit_capture_); + routing_mode_ = mode; + } + return Configure(); +} + +EchoControlMobile::RoutingMode EchoControlMobileImpl::routing_mode() + const { + rtc::CritScope cs(crit_capture_); + return routing_mode_; +} + +int EchoControlMobileImpl::enable_comfort_noise(bool enable) { + { + rtc::CritScope cs(crit_capture_); + comfort_noise_enabled_ = enable; + } + return Configure(); +} + +bool EchoControlMobileImpl::is_comfort_noise_enabled() const { + rtc::CritScope cs(crit_capture_); + return comfort_noise_enabled_; +} + +int EchoControlMobileImpl::SetEchoPath(const void* echo_path, + size_t size_bytes) { + { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + if (echo_path == NULL) { + return AudioProcessing::kNullPointerError; + } + if (size_bytes != echo_path_size_bytes()) { + // Size mismatch + return AudioProcessing::kBadParameterError; + } + + if (external_echo_path_ == NULL) { + external_echo_path_ = new unsigned char[size_bytes]; + } + memcpy(external_echo_path_, echo_path, size_bytes); + } + + // TODO(peah): Simplify once the Enable function has been removed from + // the public APM API. + RTC_DCHECK(stream_properties_); + Initialize(stream_properties_->sample_rate_hz, + stream_properties_->num_reverse_channels, + stream_properties_->num_output_channels); + return AudioProcessing::kNoError; +} + +int EchoControlMobileImpl::GetEchoPath(void* echo_path, + size_t size_bytes) const { + rtc::CritScope cs(crit_capture_); + if (echo_path == NULL) { + return AudioProcessing::kNullPointerError; + } + if (size_bytes != echo_path_size_bytes()) { + // Size mismatch + return AudioProcessing::kBadParameterError; + } + if (!enabled_) { + return AudioProcessing::kNotEnabledError; + } + + // Get the echo path from the first channel + int32_t err = + WebRtcAecm_GetEchoPath(cancellers_[0]->state(), echo_path, size_bytes); + if (err != 0) { + return MapError(err); + } + + return AudioProcessing::kNoError; +} + +void EchoControlMobileImpl::Initialize(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels) { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + + stream_properties_.reset(new StreamProperties( + sample_rate_hz, num_reverse_channels, num_output_channels)); + + if (!enabled_) { + return; + } + + if (stream_properties_->sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + RTC_LOG(LS_ERROR) << "AECM only supports 16 kHz or lower sample rates"; + } + + cancellers_.resize( + NumCancellersRequired(stream_properties_->num_output_channels, + stream_properties_->num_reverse_channels)); + + for (auto& canceller : cancellers_) { + if (!canceller) { + canceller.reset(new Canceller()); + } + canceller->Initialize(sample_rate_hz, external_echo_path_, + echo_path_size_bytes()); + } + + Configure(); +} + +int EchoControlMobileImpl::Configure() { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + AecmConfig config; + config.cngMode = comfort_noise_enabled_; + config.echoMode = MapSetting(routing_mode_); + int error = AudioProcessing::kNoError; + for (auto& canceller : cancellers_) { + int handle_error = WebRtcAecm_set_config(canceller->state(), config); + if (handle_error != AudioProcessing::kNoError) { + error = handle_error; + } + } + return error; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.h new file mode 100644 index 0000000000..a03ab4d486 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_impl.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ + +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/render_queue_item_verifier.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/swap_queue.h" + +namespace webrtc { + +class AudioBuffer; + +class EchoControlMobileImpl : public EchoControlMobile { + public: + EchoControlMobileImpl(rtc::CriticalSection* crit_render, + rtc::CriticalSection* crit_capture); + + ~EchoControlMobileImpl() override; + + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); + int ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms); + + // EchoControlMobile implementation. + bool is_enabled() const override; + RoutingMode routing_mode() const override; + bool is_comfort_noise_enabled() const override; + + void Initialize(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels); + + static void PackRenderAudioBuffer(const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer); + + static size_t NumCancellersRequired(size_t num_output_channels, + size_t num_reverse_channels); + + private: + class Canceller; + struct StreamProperties; + + // EchoControlMobile implementation. + int Enable(bool enable) override; + int set_routing_mode(RoutingMode mode) override; + int enable_comfort_noise(bool enable) override; + int SetEchoPath(const void* echo_path, size_t size_bytes) override; + int GetEchoPath(void* echo_path, size_t size_bytes) const override; + + int Configure(); + + rtc::CriticalSection* const crit_render_ RTC_ACQUIRED_BEFORE(crit_capture_); + rtc::CriticalSection* const crit_capture_; + + bool enabled_ = false; + + RoutingMode routing_mode_ RTC_GUARDED_BY(crit_capture_); + bool comfort_noise_enabled_ RTC_GUARDED_BY(crit_capture_); + unsigned char* external_echo_path_ RTC_GUARDED_BY(crit_render_) + RTC_GUARDED_BY(crit_capture_); + + std::vector> cancellers_; + std::unique_ptr stream_properties_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoControlMobileImpl); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc new file mode 100644 index 0000000000..fb58a5b870 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/echo_control_mobile_impl.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +// TODO(peah): Increase the number of frames to proces when the issue of +// non repeatable test results have been found. +const int kNumFramesToProcess = 200; + +void SetupComponent(int sample_rate_hz, + EchoControlMobile::RoutingMode routing_mode, + bool comfort_noise_enabled, + EchoControlMobileImpl* echo_control_mobile) { + echo_control_mobile->Initialize( + sample_rate_hz > 16000 ? 16000 : sample_rate_hz, 1, 1); + EchoControlMobile* ec = static_cast(echo_control_mobile); + ec->Enable(true); + ec->set_routing_mode(routing_mode); + ec->enable_comfort_noise(comfort_noise_enabled); +} + +void ProcessOneFrame(int sample_rate_hz, + int stream_delay_ms, + AudioBuffer* render_audio_buffer, + AudioBuffer* capture_audio_buffer, + EchoControlMobileImpl* echo_control_mobile) { + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + render_audio_buffer->SplitIntoFrequencyBands(); + capture_audio_buffer->SplitIntoFrequencyBands(); + } + + std::vector render_audio; + EchoControlMobileImpl::PackRenderAudioBuffer( + render_audio_buffer, 1, render_audio_buffer->num_channels(), + &render_audio); + echo_control_mobile->ProcessRenderAudio(render_audio); + + echo_control_mobile->ProcessCaptureAudio(capture_audio_buffer, + stream_delay_ms); + + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + capture_audio_buffer->MergeFrequencyBands(); + } +} + +void RunBitexactnessTest(int sample_rate_hz, + size_t num_channels, + int stream_delay_ms, + EchoControlMobile::RoutingMode routing_mode, + bool comfort_noise_enabled, + const rtc::ArrayView& output_reference) { + rtc::CriticalSection crit_render; + rtc::CriticalSection crit_capture; + EchoControlMobileImpl echo_control_mobile(&crit_render, &crit_capture); + SetupComponent(sample_rate_hz, routing_mode, comfort_noise_enabled, + &echo_control_mobile); + + const int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + const StreamConfig render_config(sample_rate_hz, num_channels, false); + AudioBuffer render_buffer( + render_config.num_frames(), render_config.num_channels(), + render_config.num_frames(), 1, render_config.num_frames()); + test::InputAudioFile render_file( + test::GetApmRenderTestVectorFileName(sample_rate_hz)); + std::vector render_input(samples_per_channel * num_channels); + + const StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), 1, capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(samples_per_channel * num_channels); + + for (int frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &render_file, render_input); + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &capture_file, capture_input); + + test::CopyVectorToAudioBuffer(render_config, render_input, &render_buffer); + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + ProcessOneFrame(sample_rate_hz, stream_delay_ms, &render_buffer, + &capture_buffer, &echo_control_mobile); + } + + // Extract and verify the test results. + std::vector capture_output; + test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer, + &capture_output); + + // Compare the output with the reference. Only the first values of the output + // from last frame processed are compared in order not having to specify all + // preceeding frames as testvectors. As the algorithm being tested has a + // memory, testing only the last frame implicitly also tests the preceeding + // frames. + const float kElementErrorBound = 1.0f / 32768.0f; + EXPECT_TRUE(test::VerifyDeinterleavedArray( + capture_config.num_frames(), capture_config.num_channels(), + output_reference, capture_output, kElementErrorBound)); +} + +} // namespace + +// TODO(peah): Renable once the integer overflow issue in aecm_core.c:932:69 +// has been solved. +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono8kHz_LoudSpeakerPhone_CngOn_StreamDelay0) { + const float kOutputReference[] = {0.005280f, 0.002380f, -0.000427f}; + + RunBitexactnessTest(8000, 1, 0, + EchoControlMobile::RoutingMode::kLoudSpeakerphone, true, + kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono16kHz_LoudSpeakerPhone_CngOn_StreamDelay0) { + const float kOutputReference[] = {0.003601f, 0.002991f, 0.001923f}; + RunBitexactnessTest(16000, 1, 0, + EchoControlMobile::RoutingMode::kLoudSpeakerphone, true, + kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono32kHz_LoudSpeakerPhone_CngOn_StreamDelay0) { + const float kOutputReference[] = {0.002258f, 0.002899f, 0.003906f}; + + RunBitexactnessTest(32000, 1, 0, + EchoControlMobile::RoutingMode::kLoudSpeakerphone, true, + kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono48kHz_LoudSpeakerPhone_CngOn_StreamDelay0) { + const float kOutputReference[] = {-0.000046f, 0.000041f, 0.000249f}; + + RunBitexactnessTest(48000, 1, 0, + EchoControlMobile::RoutingMode::kLoudSpeakerphone, true, + kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono16kHz_LoudSpeakerPhone_CngOff_StreamDelay0) { + const float kOutputReference[] = {0.000000f, 0.000000f, 0.000000f}; + + RunBitexactnessTest(16000, 1, 0, + EchoControlMobile::RoutingMode::kLoudSpeakerphone, false, + kOutputReference); +} + +// TODO(peah): Renable once the integer overflow issue in aecm_core.c:932:69 +// has been solved. +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono16kHz_LoudSpeakerPhone_CngOn_StreamDelay5) { + const float kOutputReference[] = {0.003693f, 0.002930f, 0.001801f}; + + RunBitexactnessTest(16000, 1, 5, + EchoControlMobile::RoutingMode::kLoudSpeakerphone, true, + kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + Mono16kHz_LoudSpeakerPhone_CngOn_StreamDelay10) { + const float kOutputReference[] = {-0.002380f, -0.002533f, -0.002563f}; + + RunBitexactnessTest(16000, 1, 10, + EchoControlMobile::RoutingMode::kLoudSpeakerphone, true, + kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono16kHz_QuietEarpieceOrHeadset_CngOn_StreamDelay0) { + const float kOutputReference[] = {0.000397f, 0.000000f, -0.000305f}; + + RunBitexactnessTest(16000, 1, 0, + EchoControlMobile::RoutingMode::kQuietEarpieceOrHeadset, + true, kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono16kHz_Earpiece_CngOn_StreamDelay0) { + const float kOutputReference[] = {0.002167f, 0.001617f, 0.001038f}; + + RunBitexactnessTest(16000, 1, 0, EchoControlMobile::RoutingMode::kEarpiece, + true, kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono16kHz_LoudEarpiece_CngOn_StreamDelay0) { + const float kOutputReference[] = {0.003540f, 0.002899f, 0.001862f}; + + RunBitexactnessTest(16000, 1, 0, + EchoControlMobile::RoutingMode::kLoudEarpiece, true, + kOutputReference); +} + +TEST(EchoControlMobileBitExactnessTest, + DISABLED_Mono16kHz_SpeakerPhone_CngOn_StreamDelay0) { + const float kOutputReference[] = {0.003632f, 0.003052f, 0.001984f}; + + RunBitexactnessTest(16000, 1, 0, + EchoControlMobile::RoutingMode::kSpeakerphone, true, + kOutputReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc new file mode 100644 index 0000000000..0c6cc8a933 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/circular_buffer.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +CircularBuffer::CircularBuffer(size_t size) : buffer_(size) {} +CircularBuffer::~CircularBuffer() = default; + +void CircularBuffer::Push(float value) { + buffer_[next_insertion_index_] = value; + ++next_insertion_index_; + next_insertion_index_ %= buffer_.size(); + RTC_DCHECK_LT(next_insertion_index_, buffer_.size()); + nr_elements_in_buffer_ = std::min(nr_elements_in_buffer_ + 1, buffer_.size()); + RTC_DCHECK_LE(nr_elements_in_buffer_, buffer_.size()); +} + +rtc::Optional CircularBuffer::Pop() { + if (nr_elements_in_buffer_ == 0) { + return rtc::nullopt; + } + const size_t index = + (buffer_.size() + next_insertion_index_ - nr_elements_in_buffer_) % + buffer_.size(); + RTC_DCHECK_LT(index, buffer_.size()); + --nr_elements_in_buffer_; + return buffer_[index]; +} + +void CircularBuffer::Clear() { + std::fill(buffer_.begin(), buffer_.end(), 0.f); + next_insertion_index_ = 0; + nr_elements_in_buffer_ = 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.h new file mode 100644 index 0000000000..53d4afb6d5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_CIRCULAR_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_CIRCULAR_BUFFER_H_ + +#include + +#include "api/optional.h" + +namespace webrtc { + +// Ring buffer containing floating point values. +struct CircularBuffer { + public: + explicit CircularBuffer(size_t size); + ~CircularBuffer(); + + void Push(float value); + rtc::Optional Pop(); + size_t Size() const { return nr_elements_in_buffer_; } + // This function fills the buffer with zeros, but does not change its size. + void Clear(); + + private: + std::vector buffer_; + size_t next_insertion_index_ = 0; + // This is the number of elements that have been pushed into the circular + // buffer, not the allocated buffer size. + size_t nr_elements_in_buffer_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_CIRCULAR_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer_unittest.cc new file mode 100644 index 0000000000..657bd05888 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/circular_buffer_unittest.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/circular_buffer.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(CircularBufferTests, LessThanMaxTest) { + CircularBuffer test_buffer(3); + test_buffer.Push(1.f); + test_buffer.Push(2.f); + EXPECT_EQ(1.f, test_buffer.Pop()); + EXPECT_EQ(2.f, test_buffer.Pop()); +} + +TEST(CircularBufferTests, FillTest) { + CircularBuffer test_buffer(3); + test_buffer.Push(1.f); + test_buffer.Push(2.f); + test_buffer.Push(3.f); + EXPECT_EQ(1.f, test_buffer.Pop()); + EXPECT_EQ(2.f, test_buffer.Pop()); + EXPECT_EQ(3.f, test_buffer.Pop()); +} + +TEST(CircularBufferTests, OverflowTest) { + CircularBuffer test_buffer(3); + test_buffer.Push(1.f); + test_buffer.Push(2.f); + test_buffer.Push(3.f); + test_buffer.Push(4.f); + // Because the circular buffer has a size of 3, the first insert should have + // been forgotten. + EXPECT_EQ(2.f, test_buffer.Pop()); + EXPECT_EQ(3.f, test_buffer.Pop()); + EXPECT_EQ(4.f, test_buffer.Pop()); +} + +TEST(CircularBufferTests, ReadFromEmpty) { + CircularBuffer test_buffer(3); + EXPECT_EQ(rtc::nullopt, test_buffer.Pop()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc new file mode 100644 index 0000000000..10853f3d9c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/mean_variance_estimator.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Parameter controlling the adaptation speed. +constexpr float kAlpha = 0.001f; + +} // namespace + +void MeanVarianceEstimator::Update(float value) { + mean_ = (1.f - kAlpha) * mean_ + kAlpha * value; + variance_ = + (1.f - kAlpha) * variance_ + kAlpha * (value - mean_) * (value - mean_); + RTC_DCHECK(std::isfinite(mean_)); + RTC_DCHECK(std::isfinite(variance_)); +} + +float MeanVarianceEstimator::std_deviation() const { + RTC_DCHECK_GE(variance_, 0.f); + return sqrtf(variance_); +} + +float MeanVarianceEstimator::mean() const { + return mean_; +} + +void MeanVarianceEstimator::Clear() { + mean_ = 0.f; + variance_ = 0.f; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h new file mode 100644 index 0000000000..7f793df1e8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MEAN_VARIANCE_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MEAN_VARIANCE_ESTIMATOR_H_ + +namespace webrtc { + +// This class iteratively estimates the mean and variance of a signal. +class MeanVarianceEstimator { + public: + void Update(float value); + float std_deviation() const; + float mean() const; + void Clear(); + + private: + // Estimate of the expected value of the input values. + float mean_ = 0.f; + // Estimate of the variance of the input values. + float variance_ = 0.f; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MEAN_VARIANCE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator_unittest.cc new file mode 100644 index 0000000000..f8efc3a799 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator_unittest.cc @@ -0,0 +1,64 @@ + +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/mean_variance_estimator.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(MeanVarianceEstimatorTests, InsertTwoValues) { + MeanVarianceEstimator test_estimator; + // Insert two values. + test_estimator.Update(3.f); + test_estimator.Update(5.f); + + EXPECT_GT(test_estimator.mean(), 0.f); + EXPECT_GT(test_estimator.std_deviation(), 0.f); + // Test Clear method + test_estimator.Clear(); + EXPECT_EQ(test_estimator.mean(), 0.f); + EXPECT_EQ(test_estimator.std_deviation(), 0.f); +} + +TEST(MeanVarianceEstimatorTests, InsertZeroes) { + MeanVarianceEstimator test_estimator; + // Insert the same value many times. + for (size_t i = 0; i < 20000; i++) { + test_estimator.Update(0.f); + } + EXPECT_EQ(test_estimator.mean(), 0.f); + EXPECT_EQ(test_estimator.std_deviation(), 0.f); +} + +TEST(MeanVarianceEstimatorTests, ConstantValueTest) { + MeanVarianceEstimator test_estimator; + for (size_t i = 0; i < 20000; i++) { + test_estimator.Update(3.f); + } + // The mean should be close to three, and the standard deviation should be + // close to zero. + EXPECT_NEAR(3.0f, test_estimator.mean(), 0.01f); + EXPECT_NEAR(0.0f, test_estimator.std_deviation(), 0.01f); +} + +TEST(MeanVarianceEstimatorTests, AlternatingValueTest) { + MeanVarianceEstimator test_estimator; + for (size_t i = 0; i < 20000; i++) { + test_estimator.Update(1.f); + test_estimator.Update(-1.f); + } + // The mean should be close to zero, and the standard deviation should be + // close to one. + EXPECT_NEAR(0.0f, test_estimator.mean(), 0.01f); + EXPECT_NEAR(1.0f, test_estimator.std_deviation(), 0.01f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.cc new file mode 100644 index 0000000000..3054e98bd3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/moving_max.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Parameter for controlling how fast the estimated maximum decays after the +// previous maximum is no longer valid. With a value of 0.99, the maximum will +// decay to 1% of its former value after 460 updates. +constexpr float kDecayFactor = 0.99f; + +} // namespace + +MovingMax::MovingMax(size_t window_size) : window_size_(window_size) { + RTC_DCHECK_GT(window_size, 0); +} + +MovingMax::~MovingMax() {} + +void MovingMax::Update(float value) { + if (counter_ >= window_size_ - 1) { + max_value_ *= kDecayFactor; + } else { + ++counter_; + } + if (value > max_value_) { + max_value_ = value; + counter_ = 0; + } +} + +float MovingMax::max() const { + return max_value_; +} + +void MovingMax::Clear() { + max_value_ = 0.f; + counter_ = 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.h b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.h new file mode 100644 index 0000000000..f7d8ee8137 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MOVING_MAX_H_ +#define MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MOVING_MAX_H_ + +#include + +namespace webrtc { + +class MovingMax { + public: + explicit MovingMax(size_t window_size); + ~MovingMax(); + + void Update(float value); + float max() const; + // Reset all of the state in this class. + void Clear(); + + private: + float max_value_ = 0.f; + size_t counter_ = 0; + size_t window_size_ = 1; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MOVING_MAX_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max_unittest.cc new file mode 100644 index 0000000000..b67b86ffc3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/moving_max_unittest.cc @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/moving_max.h" +#include "test/gtest.h" + +namespace webrtc { + +// Test if the maximum is correctly found. +TEST(MovingMaxTests, SimpleTest) { + MovingMax test_moving_max(5); + test_moving_max.Update(1.0f); + test_moving_max.Update(1.1f); + test_moving_max.Update(1.9f); + test_moving_max.Update(1.87f); + test_moving_max.Update(1.89f); + EXPECT_EQ(1.9f, test_moving_max.max()); +} + +// Test if values fall out of the window when expected. +TEST(MovingMaxTests, SlidingWindowTest) { + MovingMax test_moving_max(5); + test_moving_max.Update(1.0f); + test_moving_max.Update(1.9f); + test_moving_max.Update(1.7f); + test_moving_max.Update(1.87f); + test_moving_max.Update(1.89f); + test_moving_max.Update(1.3f); + test_moving_max.Update(1.2f); + EXPECT_LT(test_moving_max.max(), 1.9f); +} + +// Test if Clear() works as expected. +TEST(MovingMaxTests, ClearTest) { + MovingMax test_moving_max(5); + test_moving_max.Update(1.0f); + test_moving_max.Update(1.1f); + test_moving_max.Update(1.9f); + test_moving_max.Update(1.87f); + test_moving_max.Update(1.89f); + EXPECT_EQ(1.9f, test_moving_max.max()); + test_moving_max.Clear(); + EXPECT_EQ(0.f, test_moving_max.max()); +} + +// Test the decay of the estimated maximum. +TEST(MovingMaxTests, DecayTest) { + MovingMax test_moving_max(1); + test_moving_max.Update(1.0f); + float previous_value = 1.0f; + for (int i = 0; i < 500; i++) { + test_moving_max.Update(0.0f); + EXPECT_LT(test_moving_max.max(), previous_value); + EXPECT_GT(test_moving_max.max(), 0.0f); + previous_value = test_moving_max.max(); + } + EXPECT_LT(test_moving_max.max(), 0.01f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc new file mode 100644 index 0000000000..8ec9fe9f0b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/normalized_covariance_estimator.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Parameter controlling the adaptation speed. +constexpr float kAlpha = 0.001f; + +} // namespace + +void NormalizedCovarianceEstimator::Update(float x, + float x_mean, + float x_sigma, + float y, + float y_mean, + float y_sigma) { + covariance_ = + (1.f - kAlpha) * covariance_ + kAlpha * (x - x_mean) * (y - y_mean); + normalized_cross_correlation_ = covariance_ / (x_sigma * y_sigma + .0001f); + RTC_DCHECK(isfinite(covariance_)); + RTC_DCHECK(isfinite(normalized_cross_correlation_)); +} + +void NormalizedCovarianceEstimator::Clear() { + covariance_ = 0.f; + normalized_cross_correlation_ = 0.f; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h new file mode 100644 index 0000000000..e3c36d88ba --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_NORMALIZED_COVARIANCE_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_NORMALIZED_COVARIANCE_ESTIMATOR_H_ + +namespace webrtc { + +// This class iteratively estimates the normalized covariance between two +// signals. +class NormalizedCovarianceEstimator { + public: + void Update(float x, + float x_mean, + float x_var, + float y, + float y_mean, + float y_var); + // This function returns an estimate of the Pearson product-moment correlation + // coefficient of the two signals. + float normalized_cross_correlation() const { + return normalized_cross_correlation_; + } + float covariance() const { return covariance_; } + // This function resets the estimated values to zero. + void Clear(); + + private: + float normalized_cross_correlation_ = 0.f; + // Estimate of the covariance value. + float covariance_ = 0.f; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_NORMALIZED_COVARIANCE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc new file mode 100644 index 0000000000..7e0512ee58 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc @@ -0,0 +1,40 @@ + +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_detector/normalized_covariance_estimator.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(NormalizedCovarianceEstimatorTests, IdenticalSignalTest) { + NormalizedCovarianceEstimator test_estimator; + for (size_t i = 0; i < 10000; i++) { + test_estimator.Update(1.f, 0.f, 1.f, 1.f, 0.f, 1.f); + test_estimator.Update(-1.f, 0.f, 1.f, -1.f, 0.f, 1.f); + } + // A normalized covariance value close to 1 is expected. + EXPECT_NEAR(1.f, test_estimator.normalized_cross_correlation(), 0.01f); + test_estimator.Clear(); + EXPECT_EQ(0.f, test_estimator.normalized_cross_correlation()); +} + +TEST(NormalizedCovarianceEstimatorTests, OppositeSignalTest) { + NormalizedCovarianceEstimator test_estimator; + // Insert the same value many times. + for (size_t i = 0; i < 10000; i++) { + test_estimator.Update(1.f, 0.f, 1.f, -1.f, 0.f, 1.f); + test_estimator.Update(-1.f, 0.f, 1.f, 1.f, 0.f, 1.f); + } + // A normalized covariance value close to -1 is expected. + EXPECT_NEAR(-1.f, test_estimator.normalized_cross_correlation(), 0.01f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.cc new file mode 100644 index 0000000000..d5d978c2e0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.cc @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/gain_control_for_experimental_agc.h" + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +int GainControlForExperimentalAgc::instance_counter_ = 0; + +GainControlForExperimentalAgc::GainControlForExperimentalAgc( + GainControl* gain_control, + rtc::CriticalSection* crit_capture) + : data_dumper_(new ApmDataDumper(instance_counter_)), + real_gain_control_(gain_control), + volume_(0), + crit_capture_(crit_capture) { + instance_counter_++; +} + +GainControlForExperimentalAgc::~GainControlForExperimentalAgc() = default; + +int GainControlForExperimentalAgc::Enable(bool enable) { + return real_gain_control_->Enable(enable); +} + +bool GainControlForExperimentalAgc::is_enabled() const { + return real_gain_control_->is_enabled(); +} + +int GainControlForExperimentalAgc::set_stream_analog_level(int level) { + rtc::CritScope cs_capture(crit_capture_); + data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level", 1, + &level); + volume_ = level; + return AudioProcessing::kNoError; +} + +int GainControlForExperimentalAgc::stream_analog_level() { + rtc::CritScope cs_capture(crit_capture_); + data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1, + &volume_); + return volume_; +} + +int GainControlForExperimentalAgc::set_mode(Mode mode) { + return AudioProcessing::kNoError; +} + +GainControl::Mode GainControlForExperimentalAgc::mode() const { + return GainControl::kAdaptiveAnalog; +} + +int GainControlForExperimentalAgc::set_target_level_dbfs(int level) { + return AudioProcessing::kNoError; +} + +int GainControlForExperimentalAgc::target_level_dbfs() const { + return real_gain_control_->target_level_dbfs(); +} + +int GainControlForExperimentalAgc::set_compression_gain_db(int gain) { + return AudioProcessing::kNoError; +} + +int GainControlForExperimentalAgc::compression_gain_db() const { + return real_gain_control_->compression_gain_db(); +} + +int GainControlForExperimentalAgc::enable_limiter(bool enable) { + return AudioProcessing::kNoError; +} + +bool GainControlForExperimentalAgc::is_limiter_enabled() const { + return real_gain_control_->is_limiter_enabled(); +} + +int GainControlForExperimentalAgc::set_analog_level_limits(int minimum, + int maximum) { + return AudioProcessing::kNoError; +} + +int GainControlForExperimentalAgc::analog_level_minimum() const { + return real_gain_control_->analog_level_minimum(); +} + +int GainControlForExperimentalAgc::analog_level_maximum() const { + return real_gain_control_->analog_level_maximum(); +} + +bool GainControlForExperimentalAgc::stream_is_saturated() const { + return real_gain_control_->stream_is_saturated(); +} + +void GainControlForExperimentalAgc::SetMicVolume(int volume) { + rtc::CritScope cs_capture(crit_capture_); + volume_ = volume; +} + +int GainControlForExperimentalAgc::GetMicVolume() { + rtc::CritScope cs_capture(crit_capture_); + return volume_; +} + +void GainControlForExperimentalAgc::Initialize() { + data_dumper_->InitiateNewSetOfRecordings(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.h b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.h new file mode 100644 index 0000000000..0894a0e30e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_for_experimental_agc.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_GAIN_CONTROL_FOR_EXPERIMENTAL_AGC_H_ +#define MODULES_AUDIO_PROCESSING_GAIN_CONTROL_FOR_EXPERIMENTAL_AGC_H_ + +#include "modules/audio_processing/agc/agc_manager_direct.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +class ApmDataDumper; + +// This class has two main purposes: +// +// 1) It is returned instead of the real GainControl after the new AGC has been +// enabled in order to prevent an outside user from overriding compression +// settings. It doesn't do anything in its implementation, except for +// delegating the const methods and Enable calls to the real GainControl, so +// AGC can still be disabled. +// +// 2) It is injected into AgcManagerDirect and implements volume callbacks for +// getting and setting the volume level. It just caches this value to be used +// in VoiceEngine later. +class GainControlForExperimentalAgc : public GainControl, + public VolumeCallbacks { + public: + GainControlForExperimentalAgc(GainControl* gain_control, + rtc::CriticalSection* crit_capture); + ~GainControlForExperimentalAgc() override; + + // GainControl implementation. + int Enable(bool enable) override; + bool is_enabled() const override; + int set_stream_analog_level(int level) override; + int stream_analog_level() override; + int set_mode(Mode mode) override; + Mode mode() const override; + int set_target_level_dbfs(int level) override; + int target_level_dbfs() const override; + int set_compression_gain_db(int gain) override; + int compression_gain_db() const override; + int enable_limiter(bool enable) override; + bool is_limiter_enabled() const override; + int set_analog_level_limits(int minimum, int maximum) override; + int analog_level_minimum() const override; + int analog_level_maximum() const override; + bool stream_is_saturated() const override; + + // VolumeCallbacks implementation. + void SetMicVolume(int volume) override; + int GetMicVolume() override; + + void Initialize(); + + private: + std::unique_ptr data_dumper_; + GainControl* real_gain_control_; + int volume_; + rtc::CriticalSection* crit_capture_; + static int instance_counter_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(GainControlForExperimentalAgc); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_GAIN_CONTROL_FOR_EXPERIMENTAL_AGC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.cc new file mode 100644 index 0000000000..e550ebbfad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.cc @@ -0,0 +1,439 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/gain_control_impl.h" + +#include "api/optional.h" +#include "modules/audio_processing/agc/legacy/gain_control.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +typedef void Handle; + +namespace { +int16_t MapSetting(GainControl::Mode mode) { + switch (mode) { + case GainControl::kAdaptiveAnalog: + return kAgcModeAdaptiveAnalog; + case GainControl::kAdaptiveDigital: + return kAgcModeAdaptiveDigital; + case GainControl::kFixedDigital: + return kAgcModeFixedDigital; + } + RTC_NOTREACHED(); + return -1; +} + +} // namespace + +class GainControlImpl::GainController { + public: + explicit GainController() { + state_ = WebRtcAgc_Create(); + RTC_CHECK(state_); + } + + ~GainController() { + RTC_DCHECK(state_); + WebRtcAgc_Free(state_); + } + + Handle* state() { + RTC_DCHECK(state_); + return state_; + } + + void Initialize(int minimum_capture_level, + int maximum_capture_level, + Mode mode, + int sample_rate_hz, + int capture_level) { + RTC_DCHECK(state_); + int error = + WebRtcAgc_Init(state_, minimum_capture_level, maximum_capture_level, + MapSetting(mode), sample_rate_hz); + RTC_DCHECK_EQ(0, error); + + set_capture_level(capture_level); + } + + void set_capture_level(int capture_level) { + capture_level_ = capture_level; + } + + int get_capture_level() { + RTC_DCHECK(capture_level_); + return *capture_level_; + } + + private: + Handle* state_; + // TODO(peah): Remove the optional once the initialization is moved into the + // ctor. + rtc::Optional capture_level_; + + RTC_DISALLOW_COPY_AND_ASSIGN(GainController); +}; + +int GainControlImpl::instance_counter_ = 0; + +GainControlImpl::GainControlImpl(rtc::CriticalSection* crit_render, + rtc::CriticalSection* crit_capture) + : crit_render_(crit_render), + crit_capture_(crit_capture), + data_dumper_(new ApmDataDumper(instance_counter_)), + mode_(kAdaptiveAnalog), + minimum_capture_level_(0), + maximum_capture_level_(255), + limiter_enabled_(true), + target_level_dbfs_(3), + compression_gain_db_(9), + analog_capture_level_(0), + was_analog_level_set_(false), + stream_is_saturated_(false) { + RTC_DCHECK(crit_render); + RTC_DCHECK(crit_capture); +} + +GainControlImpl::~GainControlImpl() {} + +void GainControlImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { + rtc::CritScope cs_capture(crit_capture_); + if (!enabled_) { + return; + } + + for (auto& gain_controller : gain_controllers_) { + WebRtcAgc_AddFarend(gain_controller->state(), packed_render_audio.data(), + packed_render_audio.size()); + } +} + +void GainControlImpl::PackRenderAudioBuffer( + AudioBuffer* audio, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + + packed_buffer->clear(); + packed_buffer->insert( + packed_buffer->end(), audio->mixed_low_pass_data(), + (audio->mixed_low_pass_data() + audio->num_frames_per_band())); +} + +int GainControlImpl::AnalyzeCaptureAudio(AudioBuffer* audio) { + rtc::CritScope cs(crit_capture_); + + if (!enabled_) { + return AudioProcessing::kNoError; + } + + RTC_DCHECK(num_proc_channels_); + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(audio->num_channels(), *num_proc_channels_); + RTC_DCHECK_LE(*num_proc_channels_, gain_controllers_.size()); + + if (mode_ == kAdaptiveAnalog) { + int capture_channel = 0; + for (auto& gain_controller : gain_controllers_) { + gain_controller->set_capture_level(analog_capture_level_); + int err = WebRtcAgc_AddMic( + gain_controller->state(), audio->split_bands(capture_channel), + audio->num_bands(), audio->num_frames_per_band()); + + if (err != AudioProcessing::kNoError) { + return AudioProcessing::kUnspecifiedError; + } + ++capture_channel; + } + } else if (mode_ == kAdaptiveDigital) { + int capture_channel = 0; + for (auto& gain_controller : gain_controllers_) { + int32_t capture_level_out = 0; + int err = WebRtcAgc_VirtualMic( + gain_controller->state(), audio->split_bands(capture_channel), + audio->num_bands(), audio->num_frames_per_band(), + analog_capture_level_, &capture_level_out); + + gain_controller->set_capture_level(capture_level_out); + + if (err != AudioProcessing::kNoError) { + return AudioProcessing::kUnspecifiedError; + } + ++capture_channel; + } + } + + return AudioProcessing::kNoError; +} + +int GainControlImpl::ProcessCaptureAudio(AudioBuffer* audio, + bool stream_has_echo) { + rtc::CritScope cs(crit_capture_); + + if (!enabled_) { + return AudioProcessing::kNoError; + } + + if (mode_ == kAdaptiveAnalog && !was_analog_level_set_) { + return AudioProcessing::kStreamParameterNotSetError; + } + + RTC_DCHECK(num_proc_channels_); + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(audio->num_channels(), *num_proc_channels_); + + stream_is_saturated_ = false; + int capture_channel = 0; + for (auto& gain_controller : gain_controllers_) { + int32_t capture_level_out = 0; + uint8_t saturation_warning = 0; + + // The call to stream_has_echo() is ok from a deadlock perspective + // as the capture lock is allready held. + int err = WebRtcAgc_Process( + gain_controller->state(), audio->split_bands_const(capture_channel), + audio->num_bands(), audio->num_frames_per_band(), + audio->split_bands(capture_channel), + gain_controller->get_capture_level(), &capture_level_out, + stream_has_echo, &saturation_warning); + + if (err != AudioProcessing::kNoError) { + return AudioProcessing::kUnspecifiedError; + } + + gain_controller->set_capture_level(capture_level_out); + if (saturation_warning == 1) { + stream_is_saturated_ = true; + } + + ++capture_channel; + } + + RTC_DCHECK_LT(0ul, *num_proc_channels_); + if (mode_ == kAdaptiveAnalog) { + // Take the analog level to be the average across the handles. + analog_capture_level_ = 0; + for (auto& gain_controller : gain_controllers_) { + analog_capture_level_ += gain_controller->get_capture_level(); + } + + analog_capture_level_ /= (*num_proc_channels_); + } + + was_analog_level_set_ = false; + return AudioProcessing::kNoError; +} + +int GainControlImpl::compression_gain_db() const { + rtc::CritScope cs(crit_capture_); + return compression_gain_db_; +} + +// TODO(ajm): ensure this is called under kAdaptiveAnalog. +int GainControlImpl::set_stream_analog_level(int level) { + rtc::CritScope cs(crit_capture_); + data_dumper_->DumpRaw("gain_control_set_stream_analog_level", 1, &level); + + was_analog_level_set_ = true; + if (level < minimum_capture_level_ || level > maximum_capture_level_) { + return AudioProcessing::kBadParameterError; + } + analog_capture_level_ = level; + + return AudioProcessing::kNoError; +} + +int GainControlImpl::stream_analog_level() { + rtc::CritScope cs(crit_capture_); + data_dumper_->DumpRaw("gain_control_stream_analog_level", 1, + &analog_capture_level_); + // TODO(ajm): enable this assertion? + //RTC_DCHECK_EQ(kAdaptiveAnalog, mode_); + + return analog_capture_level_; +} + +int GainControlImpl::Enable(bool enable) { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + if (enable && !enabled_) { + enabled_ = enable; // Must be set before Initialize() is called. + + RTC_DCHECK(num_proc_channels_); + RTC_DCHECK(sample_rate_hz_); + Initialize(*num_proc_channels_, *sample_rate_hz_); + } else { + enabled_ = enable; + } + return AudioProcessing::kNoError; +} + +bool GainControlImpl::is_enabled() const { + rtc::CritScope cs(crit_capture_); + return enabled_; +} + +int GainControlImpl::set_mode(Mode mode) { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + if (MapSetting(mode) == -1) { + return AudioProcessing::kBadParameterError; + } + + mode_ = mode; + RTC_DCHECK(num_proc_channels_); + RTC_DCHECK(sample_rate_hz_); + Initialize(*num_proc_channels_, *sample_rate_hz_); + return AudioProcessing::kNoError; +} + +GainControl::Mode GainControlImpl::mode() const { + rtc::CritScope cs(crit_capture_); + return mode_; +} + +int GainControlImpl::set_analog_level_limits(int minimum, + int maximum) { + if (minimum < 0) { + return AudioProcessing::kBadParameterError; + } + + if (maximum > 65535) { + return AudioProcessing::kBadParameterError; + } + + if (maximum < minimum) { + return AudioProcessing::kBadParameterError; + } + + size_t num_proc_channels_local = 0u; + int sample_rate_hz_local = 0; + { + rtc::CritScope cs(crit_capture_); + + minimum_capture_level_ = minimum; + maximum_capture_level_ = maximum; + + RTC_DCHECK(num_proc_channels_); + RTC_DCHECK(sample_rate_hz_); + num_proc_channels_local = *num_proc_channels_; + sample_rate_hz_local = *sample_rate_hz_; + } + Initialize(num_proc_channels_local, sample_rate_hz_local); + return AudioProcessing::kNoError; +} + +int GainControlImpl::analog_level_minimum() const { + rtc::CritScope cs(crit_capture_); + return minimum_capture_level_; +} + +int GainControlImpl::analog_level_maximum() const { + rtc::CritScope cs(crit_capture_); + return maximum_capture_level_; +} + +bool GainControlImpl::stream_is_saturated() const { + rtc::CritScope cs(crit_capture_); + return stream_is_saturated_; +} + +int GainControlImpl::set_target_level_dbfs(int level) { + if (level > 31 || level < 0) { + return AudioProcessing::kBadParameterError; + } + { + rtc::CritScope cs(crit_capture_); + target_level_dbfs_ = level; + } + return Configure(); +} + +int GainControlImpl::target_level_dbfs() const { + rtc::CritScope cs(crit_capture_); + return target_level_dbfs_; +} + +int GainControlImpl::set_compression_gain_db(int gain) { + if (gain < 0 || gain > 90) { + return AudioProcessing::kBadParameterError; + } + { + rtc::CritScope cs(crit_capture_); + compression_gain_db_ = gain; + } + return Configure(); +} + +int GainControlImpl::enable_limiter(bool enable) { + { + rtc::CritScope cs(crit_capture_); + limiter_enabled_ = enable; + } + return Configure(); +} + +bool GainControlImpl::is_limiter_enabled() const { + rtc::CritScope cs(crit_capture_); + return limiter_enabled_; +} + +void GainControlImpl::Initialize(size_t num_proc_channels, int sample_rate_hz) { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + data_dumper_->InitiateNewSetOfRecordings(); + + num_proc_channels_ = num_proc_channels; + sample_rate_hz_ = sample_rate_hz; + + if (!enabled_) { + return; + } + + gain_controllers_.resize(*num_proc_channels_); + for (auto& gain_controller : gain_controllers_) { + if (!gain_controller) { + gain_controller.reset(new GainController()); + } + gain_controller->Initialize(minimum_capture_level_, maximum_capture_level_, + mode_, *sample_rate_hz_, analog_capture_level_); + } + + Configure(); +} + +int GainControlImpl::Configure() { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + WebRtcAgcConfig config; + // TODO(ajm): Flip the sign here (since AGC expects a positive value) if we + // change the interface. + //RTC_DCHECK_LE(target_level_dbfs_, 0); + //config.targetLevelDbfs = static_cast(-target_level_dbfs_); + config.targetLevelDbfs = static_cast(target_level_dbfs_); + config.compressionGaindB = + static_cast(compression_gain_db_); + config.limiterEnable = limiter_enabled_; + + int error = AudioProcessing::kNoError; + for (auto& gain_controller : gain_controllers_) { + const int handle_error = + WebRtcAgc_set_config(gain_controller->state(), config); + if (handle_error != AudioProcessing::kNoError) { + error = handle_error; + } + } + return error; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.h new file mode 100644 index 0000000000..26745065cf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_impl.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_GAIN_CONTROL_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_GAIN_CONTROL_IMPL_H_ + +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/render_queue_item_verifier.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/swap_queue.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class ApmDataDumper; +class AudioBuffer; + +class GainControlImpl : public GainControl { + public: + GainControlImpl(rtc::CriticalSection* crit_render, + rtc::CriticalSection* crit_capture); + ~GainControlImpl() override; + + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); + int AnalyzeCaptureAudio(AudioBuffer* audio); + int ProcessCaptureAudio(AudioBuffer* audio, bool stream_has_echo); + + void Initialize(size_t num_proc_channels, int sample_rate_hz); + + static void PackRenderAudioBuffer(AudioBuffer* audio, + std::vector* packed_buffer); + + // GainControl implementation. + bool is_enabled() const override; + int stream_analog_level() override; + bool is_limiter_enabled() const override; + Mode mode() const override; + + int compression_gain_db() const override; + + private: + class GainController; + + // GainControl implementation. + int Enable(bool enable) override; + int set_stream_analog_level(int level) override; + int set_mode(Mode mode) override; + int set_target_level_dbfs(int level) override; + int target_level_dbfs() const override; + int set_compression_gain_db(int gain) override; + int enable_limiter(bool enable) override; + int set_analog_level_limits(int minimum, int maximum) override; + int analog_level_minimum() const override; + int analog_level_maximum() const override; + bool stream_is_saturated() const override; + + int Configure(); + + rtc::CriticalSection* const crit_render_ RTC_ACQUIRED_BEFORE(crit_capture_); + rtc::CriticalSection* const crit_capture_; + + std::unique_ptr data_dumper_; + + bool enabled_ = false; + + Mode mode_ RTC_GUARDED_BY(crit_capture_); + int minimum_capture_level_ RTC_GUARDED_BY(crit_capture_); + int maximum_capture_level_ RTC_GUARDED_BY(crit_capture_); + bool limiter_enabled_ RTC_GUARDED_BY(crit_capture_); + int target_level_dbfs_ RTC_GUARDED_BY(crit_capture_); + int compression_gain_db_ RTC_GUARDED_BY(crit_capture_); + int analog_capture_level_ RTC_GUARDED_BY(crit_capture_); + bool was_analog_level_set_ RTC_GUARDED_BY(crit_capture_); + bool stream_is_saturated_ RTC_GUARDED_BY(crit_capture_); + + std::vector> gain_controllers_; + + rtc::Optional num_proc_channels_ RTC_GUARDED_BY(crit_capture_); + rtc::Optional sample_rate_hz_ RTC_GUARDED_BY(crit_capture_); + + static int instance_counter_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(GainControlImpl); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_GAIN_CONTROL_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_unittest.cc new file mode 100644 index 0000000000..62908c7aec --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/gain_control_unittest.cc @@ -0,0 +1,441 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/gain_control_impl.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kNumFramesToProcess = 100; + +void ProcessOneFrame(int sample_rate_hz, + AudioBuffer* render_audio_buffer, + AudioBuffer* capture_audio_buffer, + GainControlImpl* gain_controller) { + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + render_audio_buffer->SplitIntoFrequencyBands(); + capture_audio_buffer->SplitIntoFrequencyBands(); + } + + std::vector render_audio; + GainControlImpl::PackRenderAudioBuffer(render_audio_buffer, &render_audio); + gain_controller->ProcessRenderAudio(render_audio); + gain_controller->AnalyzeCaptureAudio(capture_audio_buffer); + gain_controller->ProcessCaptureAudio(capture_audio_buffer, false); + + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + capture_audio_buffer->MergeFrequencyBands(); + } +} + +void SetupComponent(int sample_rate_hz, + GainControl::Mode mode, + int target_level_dbfs, + int stream_analog_level, + int compression_gain_db, + bool enable_limiter, + int analog_level_min, + int analog_level_max, + GainControlImpl* gain_controller) { + gain_controller->Initialize(1, sample_rate_hz); + GainControl* gc = static_cast(gain_controller); + gc->Enable(true); + gc->set_mode(mode); + gc->set_stream_analog_level(stream_analog_level); + gc->set_target_level_dbfs(target_level_dbfs); + gc->set_compression_gain_db(compression_gain_db); + gc->enable_limiter(enable_limiter); + gc->set_analog_level_limits(analog_level_min, analog_level_max); +} + +void RunBitExactnessTest(int sample_rate_hz, + size_t num_channels, + GainControl::Mode mode, + int target_level_dbfs, + int stream_analog_level, + int compression_gain_db, + bool enable_limiter, + int analog_level_min, + int analog_level_max, + int achieved_stream_analog_level_reference, + rtc::ArrayView output_reference) { + rtc::CriticalSection crit_render; + rtc::CriticalSection crit_capture; + GainControlImpl gain_controller(&crit_render, &crit_capture); + SetupComponent(sample_rate_hz, mode, target_level_dbfs, stream_analog_level, + compression_gain_db, enable_limiter, analog_level_min, + analog_level_max, &gain_controller); + + const int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + const StreamConfig render_config(sample_rate_hz, num_channels, false); + AudioBuffer render_buffer( + render_config.num_frames(), render_config.num_channels(), + render_config.num_frames(), 1, render_config.num_frames()); + test::InputAudioFile render_file( + test::GetApmRenderTestVectorFileName(sample_rate_hz)); + std::vector render_input(samples_per_channel * num_channels); + + const StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), 1, capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(samples_per_channel * num_channels); + + for (int frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &render_file, render_input); + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &capture_file, capture_input); + + test::CopyVectorToAudioBuffer(render_config, render_input, &render_buffer); + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + ProcessOneFrame(sample_rate_hz, &render_buffer, &capture_buffer, + &gain_controller); + } + + // Extract and verify the test results. + std::vector capture_output; + test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer, + &capture_output); + + EXPECT_EQ(achieved_stream_analog_level_reference, + gain_controller.stream_analog_level()); + + // Compare the output with the reference. Only the first values of the output + // from last frame processed are compared in order not having to specify all + // preceeding frames as testvectors. As the algorithm being tested has a + // memory, testing only the last frame implicitly also tests the preceeding + // frames. + const float kElementErrorBound = 1.0f / 32768.0f; + EXPECT_TRUE(test::VerifyDeinterleavedArray( + capture_config.num_frames(), capture_config.num_channels(), + output_reference, capture_output, kElementErrorBound)); +} + +} // namespace + +// TODO(peah): Activate all these tests for ARM and ARM64 once the issue on the +// Chromium ARM and ARM64 boths have been identified. This is tracked in the +// issue https://bugs.chromium.org/p/webrtc/issues/detail?id=5711. + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono8kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono8kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.006622f, -0.002747f, 0.001587f}; + RunBitExactnessTest(8000, 1, GainControl::Mode::kAdaptiveAnalog, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.006561f, -0.004608f, -0.002899f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kAdaptiveAnalog, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Stereo16kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Stereo16kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.027313f, -0.015900f, -0.028107f, + -0.027313f, -0.015900f, -0.028107f}; + RunBitExactnessTest(16000, 2, GainControl::Mode::kAdaptiveAnalog, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono32kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono32kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.010162f, -0.009155f, -0.008301f}; + RunBitExactnessTest(32000, 1, GainControl::Mode::kAdaptiveAnalog, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono48kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono48kHz_AdaptiveAnalog_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.010162f, -0.009155f, -0.008301f}; + RunBitExactnessTest(32000, 1, GainControl::Mode::kAdaptiveAnalog, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono8kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono8kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.004028f, -0.001678f, 0.000946f}; + RunBitExactnessTest(8000, 1, GainControl::Mode::kAdaptiveDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.003967f, -0.002777f, -0.001770f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kAdaptiveDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Stereo16kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Stereo16kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.015411f, -0.008972f, -0.015839f, + -0.015411f, -0.008972f, -0.015839f}; + RunBitExactnessTest(16000, 2, GainControl::Mode::kAdaptiveDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono32kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono32kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.006104f, -0.005524f, -0.004974f}; + RunBitExactnessTest(32000, 1, GainControl::Mode::kAdaptiveDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono48kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono48kHz_AdaptiveDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.006104f, -0.005524f, -0.004974f}; + RunBitExactnessTest(32000, 1, GainControl::Mode::kAdaptiveDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono8kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono8kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.011871f, -0.004944f, 0.002838f}; + RunBitExactnessTest(8000, 1, GainControl::Mode::kFixedDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.011749f, -0.008270f, -0.005219f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kFixedDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Stereo16kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Stereo16kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.048950f, -0.028503f, -0.050354f, + -0.048950f, -0.028503f, -0.050354f}; + RunBitExactnessTest(16000, 2, GainControl::Mode::kFixedDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono32kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono32kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.018188f, -0.016418f, -0.014862f}; + RunBitExactnessTest(32000, 1, GainControl::Mode::kFixedDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono48kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono48kHz_FixedDigital_Tl10_SL50_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 50; + const float kOutputReference[] = {-0.018188f, -0.016418f, -0.014862f}; + RunBitExactnessTest(32000, 1, GainControl::Mode::kFixedDigital, 10, 50, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_AdaptiveAnalog_Tl10_SL10_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_AdaptiveAnalog_Tl10_SL10_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 12; + const float kOutputReference[] = {-0.006561f, -0.004608f, -0.002899f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kAdaptiveAnalog, 10, 10, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_AdaptiveAnalog_Tl10_SL100_CG5_Lim_AL70_80) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_AdaptiveAnalog_Tl10_SL100_CG5_Lim_AL70_80) { +#endif + const int kStreamAnalogLevelReference = 100; + const float kOutputReference[] = {-0.003998f, -0.002808f, -0.001770f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kAdaptiveAnalog, 10, 100, 5, + true, 70, 80, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_AdaptiveDigital_Tl10_SL100_CG5_NoLim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_AdaptiveDigital_Tl10_SL100_CG5_NoLim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 100; + const float kOutputReference[] = {-0.004028f, -0.002838f, -0.001770f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kAdaptiveDigital, 10, 100, 5, + false, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_AdaptiveDigital_Tl40_SL100_CG5_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_AdaptiveDigital_Tl40_SL100_CG5_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 100; + const float kOutputReference[] = {-0.008728f, -0.006134f, -0.003845f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kAdaptiveDigital, 40, 100, 5, + true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) +TEST(GainControlBitExactnessTest, + Mono16kHz_AdaptiveDigital_Tl10_SL100_CG30_Lim_AL0_100) { +#else +TEST(GainControlBitExactnessTest, + DISABLED_Mono16kHz_AdaptiveDigital_Tl10_SL100_CG30_Lim_AL0_100) { +#endif + const int kStreamAnalogLevelReference = 100; + const float kOutputReference[] = {-0.005859f, -0.004120f, -0.002594f}; + RunBitExactnessTest(16000, 1, GainControl::Mode::kAdaptiveDigital, 10, 100, + 30, true, 0, 100, kStreamAnalogLevelReference, + kOutputReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.cc new file mode 100644 index 0000000000..365d01510e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.cc @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/include/aec_dump.h" + +namespace webrtc { +InternalAPMConfig::InternalAPMConfig() = default; +InternalAPMConfig::InternalAPMConfig(const InternalAPMConfig&) = default; +InternalAPMConfig::InternalAPMConfig(InternalAPMConfig&&) = default; +InternalAPMConfig& InternalAPMConfig::operator=(const InternalAPMConfig&) = + default; + +bool InternalAPMConfig::operator==(const InternalAPMConfig& other) { + return aec_enabled == other.aec_enabled && + aec_delay_agnostic_enabled == other.aec_delay_agnostic_enabled && + aec_drift_compensation_enabled == + other.aec_drift_compensation_enabled && + aec_extended_filter_enabled == other.aec_extended_filter_enabled && + aec_suppression_level == other.aec_suppression_level && + aecm_enabled == other.aecm_enabled && + aecm_comfort_noise_enabled == other.aecm_comfort_noise_enabled && + aecm_routing_mode == other.aecm_routing_mode && + agc_enabled == other.agc_enabled && agc_mode == other.agc_mode && + agc_limiter_enabled == other.agc_limiter_enabled && + hpf_enabled == other.hpf_enabled && ns_enabled == other.ns_enabled && + ns_level == other.ns_level && + transient_suppression_enabled == other.transient_suppression_enabled && + intelligibility_enhancer_enabled == + other.intelligibility_enhancer_enabled && + noise_robust_agc_enabled == other.noise_robust_agc_enabled && + experiments_description == other.experiments_description; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.h b/third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.h new file mode 100644 index 0000000000..0c8d2271cc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/aec_dump.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_ + +#include +#include +#include + +#include "api/array_view.h" + +namespace webrtc { + +class AudioFrame; + +// Struct for passing current config from APM without having to +// include protobuf headers. +struct InternalAPMConfig { + InternalAPMConfig(); + InternalAPMConfig(const InternalAPMConfig&); + InternalAPMConfig(InternalAPMConfig&&); + + InternalAPMConfig& operator=(const InternalAPMConfig&); + InternalAPMConfig& operator=(InternalAPMConfig&&) = delete; + + bool operator==(const InternalAPMConfig& other); + + bool aec_enabled = false; + bool aec_delay_agnostic_enabled = false; + bool aec_drift_compensation_enabled = false; + bool aec_extended_filter_enabled = false; + int aec_suppression_level = 0; + bool aecm_enabled = false; + bool aecm_comfort_noise_enabled = false; + int aecm_routing_mode = 0; + bool agc_enabled = false; + int agc_mode = 0; + bool agc_limiter_enabled = false; + bool hpf_enabled = false; + bool ns_enabled = false; + int ns_level = 0; + bool transient_suppression_enabled = false; + bool intelligibility_enhancer_enabled = false; + bool noise_robust_agc_enabled = false; + std::string experiments_description = ""; +}; + +struct InternalAPMStreamsConfig { + int input_sample_rate = 0; + int output_sample_rate = 0; + int render_input_sample_rate = 0; + int render_output_sample_rate = 0; + + size_t input_num_channels = 0; + size_t output_num_channels = 0; + size_t render_input_num_channels = 0; + size_t render_output_num_channels = 0; +}; + +// Class to pass audio data in float** format. This is to avoid +// dependence on AudioBuffer, and avoid problems associated with +// rtc::ArrayView. +class FloatAudioFrame { + public: + // |num_channels| and |channel_size| describe the float** + // |audio_samples|. |audio_samples| is assumed to point to a + // two-dimensional |num_channels * channel_size| array of floats. + FloatAudioFrame(const float* const* audio_samples, + size_t num_channels, + size_t channel_size) + : audio_samples_(audio_samples), + num_channels_(num_channels), + channel_size_(channel_size) {} + + FloatAudioFrame() = delete; + + size_t num_channels() const { return num_channels_; } + + rtc::ArrayView channel(size_t idx) const { + RTC_DCHECK_LE(0, idx); + RTC_DCHECK_LE(idx, num_channels_); + return rtc::ArrayView(audio_samples_[idx], channel_size_); + } + + private: + const float* const* audio_samples_; + size_t num_channels_; + size_t channel_size_; +}; + +// An interface for recording configuration and input/output streams +// of the Audio Processing Module. The recordings are called +// 'aec-dumps' and are stored in a protobuf format defined in +// debug.proto. +// The Write* methods are always safe to call concurrently or +// otherwise for all implementing subclasses. The intended mode of +// operation is to create a protobuf object from the input, and send +// it away to be written to file asynchronously. +class AecDump { + public: + struct AudioProcessingState { + int delay; + int drift; + int level; + bool keypress; + }; + + virtual ~AecDump() = default; + + // Logs Event::Type INIT message. + virtual void WriteInitMessage( + const InternalAPMStreamsConfig& streams_config) = 0; + + // Logs Event::Type STREAM message. To log an input/output pair, + // call the AddCapture* and AddAudioProcessingState methods followed + // by a WriteCaptureStreamMessage call. + virtual void AddCaptureStreamInput(const FloatAudioFrame& src) = 0; + virtual void AddCaptureStreamOutput(const FloatAudioFrame& src) = 0; + virtual void AddCaptureStreamInput(const AudioFrame& frame) = 0; + virtual void AddCaptureStreamOutput(const AudioFrame& frame) = 0; + virtual void AddAudioProcessingState(const AudioProcessingState& state) = 0; + virtual void WriteCaptureStreamMessage() = 0; + + // Logs Event::Type REVERSE_STREAM message. + virtual void WriteRenderStreamMessage(const AudioFrame& frame) = 0; + virtual void WriteRenderStreamMessage(const FloatAudioFrame& src) = 0; + + // Logs Event::Type CONFIG message. + virtual void WriteConfig(const InternalAPMConfig& config) = 0; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.cc new file mode 100644 index 0000000000..8410a3dfbb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.cc @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/include/audio_processing.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +Beamforming::Beamforming() + : enabled(false), + array_geometry(), + target_direction( + SphericalPointf(static_cast(M_PI) / 2.f, 0.f, 1.f)) {} +Beamforming::Beamforming(bool enabled, const std::vector& array_geometry) + : Beamforming(enabled, + array_geometry, + SphericalPointf(static_cast(M_PI) / 2.f, 0.f, 1.f)) {} + +Beamforming::Beamforming(bool enabled, + const std::vector& array_geometry, + SphericalPointf target_direction) + : enabled(enabled), + array_geometry(array_geometry), + target_direction(target_direction) {} + +Beamforming::~Beamforming() {} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.h b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.h new file mode 100644 index 0000000000..df397b1407 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing.h @@ -0,0 +1,1228 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ + +// MSVC++ requires this to be set before any other includes to get M_PI. +// MOZILLA: this is already defined in mozilla-config.h +// #define _USE_MATH_DEFINES + +#include +#include // size_t +#include // FILE +#include +#include + +#include "api/optional.h" +#include "modules/audio_processing/beamformer/array_util.h" +#include "modules/audio_processing/include/audio_processing_statistics.h" +#include "modules/audio_processing/include/config.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/deprecation.h" +#include "rtc_base/platform_file.h" +#include "rtc_base/refcount.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +struct AecCore; + +class AecDump; +class AudioBuffer; +class AudioFrame; + +class NonlinearBeamformer; + +class StreamConfig; +class ProcessingConfig; + +class EchoCancellation; +class EchoControlMobile; +class EchoControlFactory; +class GainControl; +class HighPassFilter; +class LevelEstimator; +class NoiseSuppression; +class PostProcessing; +class VoiceDetection; + +// Use to enable the extended filter mode in the AEC, along with robustness +// measures around the reported system delays. It comes with a significant +// increase in AEC complexity, but is much more robust to unreliable reported +// delays. +// +// Detailed changes to the algorithm: +// - The filter length is changed from 48 to 128 ms. This comes with tuning of +// several parameters: i) filter adaptation stepsize and error threshold; +// ii) non-linear processing smoothing and overdrive. +// - Option to ignore the reported delays on platforms which we deem +// sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c. +// - Faster startup times by removing the excessive "startup phase" processing +// of reported delays. +// - Much more conservative adjustments to the far-end read pointer. We smooth +// the delay difference more heavily, and back off from the difference more. +// Adjustments force a readaptation of the filter, so they should be avoided +// except when really necessary. +struct ExtendedFilter { + ExtendedFilter() : enabled(false) {} + explicit ExtendedFilter(bool enabled) : enabled(enabled) {} + static const ConfigOptionID identifier = ConfigOptionID::kExtendedFilter; + bool enabled; +}; + +// Enables the refined linear filter adaptation in the echo canceller. +// This configuration only applies to EchoCancellation and not +// EchoControlMobile. It can be set in the constructor +// or using AudioProcessing::SetExtraOptions(). +struct RefinedAdaptiveFilter { + RefinedAdaptiveFilter() : enabled(false) {} + explicit RefinedAdaptiveFilter(bool enabled) : enabled(enabled) {} + static const ConfigOptionID identifier = + ConfigOptionID::kAecRefinedAdaptiveFilter; + bool enabled; +}; + +// Enables delay-agnostic echo cancellation. This feature relies on internally +// estimated delays between the process and reverse streams, thus not relying +// on reported system delays. This configuration only applies to +// EchoCancellation and not EchoControlMobile. It can be set in the constructor +// or using AudioProcessing::SetExtraOptions(). +struct DelayAgnostic { + DelayAgnostic() : enabled(false) {} + explicit DelayAgnostic(bool enabled) : enabled(enabled) {} + static const ConfigOptionID identifier = ConfigOptionID::kDelayAgnostic; + bool enabled; +}; + +// Use to enable experimental gain control (AGC). At startup the experimental +// AGC moves the microphone volume up to |startup_min_volume| if the current +// microphone volume is set too low. The value is clamped to its operating range +// [12, 255]. Here, 255 maps to 100%. +// +// Must be provided through AudioProcessing::Create(Confg&). +#if defined(WEBRTC_CHROMIUM_BUILD) +static const int kAgcStartupMinVolume = 85; +#else +static const int kAgcStartupMinVolume = 0; +#endif // defined(WEBRTC_CHROMIUM_BUILD) +static constexpr int kClippedLevelMin = 70; +struct ExperimentalAgc { + ExperimentalAgc() = default; + explicit ExperimentalAgc(bool enabled) : enabled(enabled) {} + ExperimentalAgc(bool enabled, int startup_min_volume) + : enabled(enabled), startup_min_volume(startup_min_volume) {} + ExperimentalAgc(bool enabled, int startup_min_volume, int clipped_level_min) + : enabled(enabled), + startup_min_volume(startup_min_volume), + clipped_level_min(clipped_level_min) {} + static const ConfigOptionID identifier = ConfigOptionID::kExperimentalAgc; + bool enabled = true; + int startup_min_volume = kAgcStartupMinVolume; + // Lowest microphone level that will be applied in response to clipping. + int clipped_level_min = kClippedLevelMin; +}; + +// Use to enable experimental noise suppression. It can be set in the +// constructor or using AudioProcessing::SetExtraOptions(). +struct ExperimentalNs { + ExperimentalNs() : enabled(false) {} + explicit ExperimentalNs(bool enabled) : enabled(enabled) {} + static const ConfigOptionID identifier = ConfigOptionID::kExperimentalNs; + bool enabled; +}; + +// Use to enable beamforming. Must be provided through the constructor. It will +// have no impact if used with AudioProcessing::SetExtraOptions(). +struct Beamforming { + Beamforming(); + Beamforming(bool enabled, const std::vector& array_geometry); + Beamforming(bool enabled, + const std::vector& array_geometry, + SphericalPointf target_direction); + ~Beamforming(); + + static const ConfigOptionID identifier = ConfigOptionID::kBeamforming; + const bool enabled; + const std::vector array_geometry; + const SphericalPointf target_direction; +}; + +// Use to enable intelligibility enhancer in audio processing. +// +// Note: If enabled and the reverse stream has more than one output channel, +// the reverse stream will become an upmixed mono signal. +struct Intelligibility { + Intelligibility() : enabled(false) {} + explicit Intelligibility(bool enabled) : enabled(enabled) {} + static const ConfigOptionID identifier = ConfigOptionID::kIntelligibility; + bool enabled; +}; + +// The Audio Processing Module (APM) provides a collection of voice processing +// components designed for real-time communications software. +// +// APM operates on two audio streams on a frame-by-frame basis. Frames of the +// primary stream, on which all processing is applied, are passed to +// |ProcessStream()|. Frames of the reverse direction stream are passed to +// |ProcessReverseStream()|. On the client-side, this will typically be the +// near-end (capture) and far-end (render) streams, respectively. APM should be +// placed in the signal chain as close to the audio hardware abstraction layer +// (HAL) as possible. +// +// On the server-side, the reverse stream will normally not be used, with +// processing occurring on each incoming stream. +// +// Component interfaces follow a similar pattern and are accessed through +// corresponding getters in APM. All components are disabled at create-time, +// with default settings that are recommended for most situations. New settings +// can be applied without enabling a component. Enabling a component triggers +// memory allocation and initialization to allow it to start processing the +// streams. +// +// Thread safety is provided with the following assumptions to reduce locking +// overhead: +// 1. The stream getters and setters are called from the same thread as +// ProcessStream(). More precisely, stream functions are never called +// concurrently with ProcessStream(). +// 2. Parameter getters are never called concurrently with the corresponding +// setter. +// +// APM accepts only linear PCM audio data in chunks of 10 ms. The int16 +// interfaces use interleaved data, while the float interfaces use deinterleaved +// data. +// +// Usage example, omitting error checking: +// AudioProcessing* apm = AudioProcessing::Create(0); +// +// AudioProcessing::Config config; +// config.level_controller.enabled = true; +// config.high_pass_filter.enabled = true; +// apm->ApplyConfig(config) +// +// apm->echo_cancellation()->enable_drift_compensation(false); +// apm->echo_cancellation()->Enable(true); +// +// apm->noise_reduction()->set_level(kHighSuppression); +// apm->noise_reduction()->Enable(true); +// +// apm->gain_control()->set_analog_level_limits(0, 255); +// apm->gain_control()->set_mode(kAdaptiveAnalog); +// apm->gain_control()->Enable(true); +// +// apm->voice_detection()->Enable(true); +// +// // Start a voice call... +// +// // ... Render frame arrives bound for the audio HAL ... +// apm->ProcessReverseStream(render_frame); +// +// // ... Capture frame arrives from the audio HAL ... +// // Call required set_stream_ functions. +// apm->set_stream_delay_ms(delay_ms); +// apm->gain_control()->set_stream_analog_level(analog_level); +// +// apm->ProcessStream(capture_frame); +// +// // Call required stream_ functions. +// analog_level = apm->gain_control()->stream_analog_level(); +// has_voice = apm->stream_has_voice(); +// +// // Repeate render and capture processing for the duration of the call... +// // Start a new call... +// apm->Initialize(); +// +// // Close the application... +// delete apm; +// +class AudioProcessing : public rtc::RefCountInterface { + public: + // The struct below constitutes the new parameter scheme for the audio + // processing. It is being introduced gradually and until it is fully + // introduced, it is prone to change. + // TODO(peah): Remove this comment once the new config scheme is fully rolled + // out. + // + // The parameters and behavior of the audio processing module are controlled + // by changing the default values in the AudioProcessing::Config struct. + // The config is applied by passing the struct to the ApplyConfig method. + struct Config { + struct LevelController { + bool enabled = false; + + // Sets the initial peak level to use inside the level controller in order + // to compute the signal gain. The unit for the peak level is dBFS and + // the allowed range is [-100, 0]. + float initial_peak_level_dbfs = -6.0206f; + } level_controller; + struct ResidualEchoDetector { + bool enabled = true; + } residual_echo_detector; + + struct HighPassFilter { + bool enabled = false; + } high_pass_filter; + + // Deprecated way of activating AEC3. + // TODO(gustaf): Remove when possible. + struct EchoCanceller3 { + bool enabled = false; + } echo_canceller3; + + // Enables the next generation AGC functionality. This feature replaces the + // standard methods of gain control in the previous AGC. + // The functionality is not yet activated in the code and turning this on + // does not yet have the desired behavior. + struct GainController2 { + bool enabled = false; + float fixed_gain_db = 0.f; + } gain_controller2; + + // Explicit copy assignment implementation to avoid issues with memory + // sanitizer complaints in case of self-assignment. + // TODO(peah): Add buildflag to ensure that this is only included for memory + // sanitizer builds. + Config& operator=(const Config& config) { + if (this != &config) { + memcpy(this, &config, sizeof(*this)); + } + return *this; + } + }; + + // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone. + enum ChannelLayout { + kMono, + // Left, right. + kStereo, + // Mono, keyboard, and mic. + kMonoAndKeyboard, + // Left, right, keyboard, and mic. + kStereoAndKeyboard + }; + + // Creates an APM instance. Use one instance for every primary audio stream + // requiring processing. On the client-side, this would typically be one + // instance for the near-end stream, and additional instances for each far-end + // stream which requires processing. On the server-side, this would typically + // be one instance for every incoming stream. + static AudioProcessing* Create(); + // Allows passing in an optional configuration at create-time. + static AudioProcessing* Create(const webrtc::Config& config); + // Deprecated. Use the Create below, with nullptr PostProcessing. + RTC_DEPRECATED + static AudioProcessing* Create(const webrtc::Config& config, + NonlinearBeamformer* beamformer); + // Allows passing in optional user-defined processing modules. + static AudioProcessing* Create( + const webrtc::Config& config, + std::unique_ptr capture_post_processor, + std::unique_ptr echo_control_factory, + NonlinearBeamformer* beamformer); + ~AudioProcessing() override {} + + // Initializes internal states, while retaining all user settings. This + // should be called before beginning to process a new audio stream. However, + // it is not necessary to call before processing the first stream after + // creation. + // + // It is also not necessary to call if the audio parameters (sample + // rate and number of channels) have changed. Passing updated parameters + // directly to |ProcessStream()| and |ProcessReverseStream()| is permissible. + // If the parameters are known at init-time though, they may be provided. + virtual int Initialize() = 0; + + // The int16 interfaces require: + // - only |NativeRate|s be used + // - that the input, output and reverse rates must match + // - that |processing_config.output_stream()| matches + // |processing_config.input_stream()|. + // + // The float interfaces accept arbitrary rates and support differing input and + // output layouts, but the output must have either one channel or the same + // number of channels as the input. + virtual int Initialize(const ProcessingConfig& processing_config) = 0; + + // Initialize with unpacked parameters. See Initialize() above for details. + // + // TODO(mgraczyk): Remove once clients are updated to use the new interface. + virtual int Initialize(int capture_input_sample_rate_hz, + int capture_output_sample_rate_hz, + int render_sample_rate_hz, + ChannelLayout capture_input_layout, + ChannelLayout capture_output_layout, + ChannelLayout render_input_layout) = 0; + + // TODO(peah): This method is a temporary solution used to take control + // over the parameters in the audio processing module and is likely to change. + virtual void ApplyConfig(const Config& config) = 0; + + // Pass down additional options which don't have explicit setters. This + // ensures the options are applied immediately. + virtual void SetExtraOptions(const webrtc::Config& config) = 0; + + // TODO(ajm): Only intended for internal use. Make private and friend the + // necessary classes? + virtual int proc_sample_rate_hz() const = 0; + virtual int proc_split_sample_rate_hz() const = 0; + virtual size_t num_input_channels() const = 0; + virtual size_t num_proc_channels() const = 0; + virtual size_t num_output_channels() const = 0; + virtual size_t num_reverse_channels() const = 0; + + // Set to true when the output of AudioProcessing will be muted or in some + // other way not used. Ideally, the captured audio would still be processed, + // but some components may change behavior based on this information. + // Default false. + virtual void set_output_will_be_muted(bool muted) = 0; + + // Processes a 10 ms |frame| of the primary audio stream. On the client-side, + // this is the near-end (or captured) audio. + // + // If needed for enabled functionality, any function with the set_stream_ tag + // must be called prior to processing the current frame. Any getter function + // with the stream_ tag which is needed should be called after processing. + // + // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| + // members of |frame| must be valid. If changed from the previous call to this + // method, it will trigger an initialization. + virtual int ProcessStream(AudioFrame* frame) = 0; + + // Accepts deinterleaved float audio with the range [-1, 1]. Each element + // of |src| points to a channel buffer, arranged according to + // |input_layout|. At output, the channels will be arranged according to + // |output_layout| at |output_sample_rate_hz| in |dest|. + // + // The output layout must have one channel or as many channels as the input. + // |src| and |dest| may use the same memory, if desired. + // + // TODO(mgraczyk): Remove once clients are updated to use the new interface. + virtual int ProcessStream(const float* const* src, + size_t samples_per_channel, + int input_sample_rate_hz, + ChannelLayout input_layout, + int output_sample_rate_hz, + ChannelLayout output_layout, + float* const* dest) = 0; + + // Accepts deinterleaved float audio with the range [-1, 1]. Each element of + // |src| points to a channel buffer, arranged according to |input_stream|. At + // output, the channels will be arranged according to |output_stream| in + // |dest|. + // + // The output must have one channel or as many channels as the input. |src| + // and |dest| may use the same memory, if desired. + virtual int ProcessStream(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest) = 0; + + // Processes a 10 ms |frame| of the reverse direction audio stream. The frame + // may be modified. On the client-side, this is the far-end (or to be + // rendered) audio. + // + // It is necessary to provide this if echo processing is enabled, as the + // reverse stream forms the echo reference signal. It is recommended, but not + // necessary, to provide if gain control is enabled. On the server-side this + // typically will not be used. If you're not sure what to pass in here, + // chances are you don't need to use it. + // + // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| + // members of |frame| must be valid. + virtual int ProcessReverseStream(AudioFrame* frame) = 0; + + // Accepts deinterleaved float audio with the range [-1, 1]. Each element + // of |data| points to a channel buffer, arranged according to |layout|. + // TODO(mgraczyk): Remove once clients are updated to use the new interface. + virtual int AnalyzeReverseStream(const float* const* data, + size_t samples_per_channel, + int sample_rate_hz, + ChannelLayout layout) = 0; + + // Accepts deinterleaved float audio with the range [-1, 1]. Each element of + // |data| points to a channel buffer, arranged according to |reverse_config|. + virtual int ProcessReverseStream(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest) = 0; + + // This must be called if and only if echo processing is enabled. + // + // Sets the |delay| in ms between ProcessReverseStream() receiving a far-end + // frame and ProcessStream() receiving a near-end frame containing the + // corresponding echo. On the client-side this can be expressed as + // delay = (t_render - t_analyze) + (t_process - t_capture) + // where, + // - t_analyze is the time a frame is passed to ProcessReverseStream() and + // t_render is the time the first sample of the same frame is rendered by + // the audio hardware. + // - t_capture is the time the first sample of a frame is captured by the + // audio hardware and t_process is the time the same frame is passed to + // ProcessStream(). + virtual int set_stream_delay_ms(int delay) = 0; + virtual int stream_delay_ms() const = 0; + virtual bool was_stream_delay_set() const = 0; + + // Call to signal that a key press occurred (true) or did not occur (false) + // with this chunk of audio. + virtual void set_stream_key_pressed(bool key_pressed) = 0; + + // Sets a delay |offset| in ms to add to the values passed in through + // set_stream_delay_ms(). May be positive or negative. + // + // Note that this could cause an otherwise valid value passed to + // set_stream_delay_ms() to return an error. + virtual void set_delay_offset_ms(int offset) = 0; + virtual int delay_offset_ms() const = 0; + + // Attaches provided webrtc::AecDump for recording debugging + // information. Log file and maximum file size logic is supposed to + // be handled by implementing instance of AecDump. Calling this + // method when another AecDump is attached resets the active AecDump + // with a new one. This causes the d-tor of the earlier AecDump to + // be called. The d-tor call may block until all pending logging + // tasks are completed. + virtual void AttachAecDump(std::unique_ptr aec_dump) = 0; + + // If no AecDump is attached, this has no effect. If an AecDump is + // attached, it's destructor is called. The d-tor may block until + // all pending logging tasks are completed. + virtual void DetachAecDump() = 0; + + // Use to send UMA histograms at end of a call. Note that all histogram + // specific member variables are reset. + virtual void UpdateHistogramsOnCallEnd() = 0; + + // TODO(ivoc): Remove when the calling code no longer uses the old Statistics + // API. + struct Statistic { + int instant = 0; // Instantaneous value. + int average = 0; // Long-term average. + int maximum = 0; // Long-term maximum. + int minimum = 0; // Long-term minimum. + }; + + struct Stat { + void Set(const Statistic& other) { + Set(other.instant, other.average, other.maximum, other.minimum); + } + void Set(float instant, float average, float maximum, float minimum) { + instant_ = instant; + average_ = average; + maximum_ = maximum; + minimum_ = minimum; + } + float instant() const { return instant_; } + float average() const { return average_; } + float maximum() const { return maximum_; } + float minimum() const { return minimum_; } + + private: + float instant_ = 0.0f; // Instantaneous value. + float average_ = 0.0f; // Long-term average. + float maximum_ = 0.0f; // Long-term maximum. + float minimum_ = 0.0f; // Long-term minimum. + }; + + struct AudioProcessingStatistics { + AudioProcessingStatistics(); + AudioProcessingStatistics(const AudioProcessingStatistics& other); + ~AudioProcessingStatistics(); + + // AEC Statistics. + // RERL = ERL + ERLE + Stat residual_echo_return_loss; + // ERL = 10log_10(P_far / P_echo) + Stat echo_return_loss; + // ERLE = 10log_10(P_echo / P_out) + Stat echo_return_loss_enhancement; + // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a) + Stat a_nlp; + // Fraction of time that the AEC linear filter is divergent, in a 1-second + // non-overlapped aggregation window. + float divergent_filter_fraction = -1.0f; + + // The delay metrics consists of the delay median and standard deviation. It + // also consists of the fraction of delay estimates that can make the echo + // cancellation perform poorly. The values are aggregated until the first + // call to |GetStatistics()| and afterwards aggregated and updated every + // second. Note that if there are several clients pulling metrics from + // |GetStatistics()| during a session the first call from any of them will + // change to one second aggregation window for all. + int delay_median = -1; + int delay_standard_deviation = -1; + float fraction_poor_delays = -1.0f; + + // Residual echo detector likelihood. + float residual_echo_likelihood = -1.0f; + // Maximum residual echo likelihood from the last time period. + float residual_echo_likelihood_recent_max = -1.0f; + }; + + // TODO(ivoc): Make this pure virtual when all subclasses have been updated. + virtual AudioProcessingStatistics GetStatistics() const; + + // This returns the stats as optionals and it will replace the regular + // GetStatistics. + virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) const; + + // These provide access to the component interfaces and should never return + // NULL. The pointers will be valid for the lifetime of the APM instance. + // The memory for these objects is entirely managed internally. + virtual EchoCancellation* echo_cancellation() const = 0; + virtual EchoControlMobile* echo_control_mobile() const = 0; + virtual GainControl* gain_control() const = 0; + // TODO(peah): Deprecate this API call. + virtual HighPassFilter* high_pass_filter() const = 0; + virtual LevelEstimator* level_estimator() const = 0; + virtual NoiseSuppression* noise_suppression() const = 0; + virtual VoiceDetection* voice_detection() const = 0; + + // Returns the last applied configuration. + virtual AudioProcessing::Config GetConfig() const = 0; + + enum Error { + // Fatal errors. + kNoError = 0, + kUnspecifiedError = -1, + kCreationFailedError = -2, + kUnsupportedComponentError = -3, + kUnsupportedFunctionError = -4, + kNullPointerError = -5, + kBadParameterError = -6, + kBadSampleRateError = -7, + kBadDataLengthError = -8, + kBadNumberChannelsError = -9, + kFileError = -10, + kStreamParameterNotSetError = -11, + kNotEnabledError = -12, + + // Warnings are non-fatal. + // This results when a set_stream_ parameter is out of range. Processing + // will continue, but the parameter may have been truncated. + kBadStreamParameterWarning = -13 + }; + + enum NativeRate { + kSampleRate8kHz = 8000, + kSampleRate16kHz = 16000, + kSampleRate32kHz = 32000, + kSampleRate44_1kHz = 44100, + kSampleRate48kHz = 48000 + }; + + // TODO(kwiberg): We currently need to support a compiler (Visual C++) that + // complains if we don't explicitly state the size of the array here. Remove + // the size when that's no longer the case. + static constexpr int kNativeSampleRatesHz[4] = { + kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz}; + static constexpr size_t kNumNativeSampleRates = + arraysize(kNativeSampleRatesHz); + static constexpr int kMaxNativeSampleRateHz = + kNativeSampleRatesHz[kNumNativeSampleRates - 1]; + + static const int kChunkSizeMs = 10; +}; + +class StreamConfig { + public: + // sample_rate_hz: The sampling rate of the stream. + // + // num_channels: The number of audio channels in the stream, excluding the + // keyboard channel if it is present. When passing a + // StreamConfig with an array of arrays T*[N], + // + // N == {num_channels + 1 if has_keyboard + // {num_channels if !has_keyboard + // + // has_keyboard: True if the stream has a keyboard channel. When has_keyboard + // is true, the last channel in any corresponding list of + // channels is the keyboard channel. + StreamConfig(int sample_rate_hz = 0, + size_t num_channels = 0, + bool has_keyboard = false) + : sample_rate_hz_(sample_rate_hz), + num_channels_(num_channels), + has_keyboard_(has_keyboard), + num_frames_(calculate_frames(sample_rate_hz)) {} + + void set_sample_rate_hz(int value) { + sample_rate_hz_ = value; + num_frames_ = calculate_frames(value); + } + void set_num_channels(size_t value) { num_channels_ = value; } + void set_has_keyboard(bool value) { has_keyboard_ = value; } + + int sample_rate_hz() const { return sample_rate_hz_; } + + // The number of channels in the stream, not including the keyboard channel if + // present. + size_t num_channels() const { return num_channels_; } + + bool has_keyboard() const { return has_keyboard_; } + size_t num_frames() const { return num_frames_; } + size_t num_samples() const { return num_channels_ * num_frames_; } + + bool operator==(const StreamConfig& other) const { + return sample_rate_hz_ == other.sample_rate_hz_ && + num_channels_ == other.num_channels_ && + has_keyboard_ == other.has_keyboard_; + } + + bool operator!=(const StreamConfig& other) const { return !(*this == other); } + + private: + static size_t calculate_frames(int sample_rate_hz) { + return static_cast( + AudioProcessing::kChunkSizeMs * sample_rate_hz / 1000); + } + + int sample_rate_hz_; + size_t num_channels_; + bool has_keyboard_; + size_t num_frames_; +}; + +class ProcessingConfig { + public: + enum StreamName { + kInputStream, + kOutputStream, + kReverseInputStream, + kReverseOutputStream, + kNumStreamNames, + }; + + const StreamConfig& input_stream() const { + return streams[StreamName::kInputStream]; + } + const StreamConfig& output_stream() const { + return streams[StreamName::kOutputStream]; + } + const StreamConfig& reverse_input_stream() const { + return streams[StreamName::kReverseInputStream]; + } + const StreamConfig& reverse_output_stream() const { + return streams[StreamName::kReverseOutputStream]; + } + + StreamConfig& input_stream() { return streams[StreamName::kInputStream]; } + StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; } + StreamConfig& reverse_input_stream() { + return streams[StreamName::kReverseInputStream]; + } + StreamConfig& reverse_output_stream() { + return streams[StreamName::kReverseOutputStream]; + } + + bool operator==(const ProcessingConfig& other) const { + for (int i = 0; i < StreamName::kNumStreamNames; ++i) { + if (this->streams[i] != other.streams[i]) { + return false; + } + } + return true; + } + + bool operator!=(const ProcessingConfig& other) const { + return !(*this == other); + } + + StreamConfig streams[StreamName::kNumStreamNames]; +}; + +// The acoustic echo cancellation (AEC) component provides better performance +// than AECM but also requires more processing power and is dependent on delay +// stability and reporting accuracy. As such it is well-suited and recommended +// for PC and IP phone applications. +// +// Not recommended to be enabled on the server-side. +class EchoCancellation { + public: + // EchoCancellation and EchoControlMobile may not be enabled simultaneously. + // Enabling one will disable the other. + virtual int Enable(bool enable) = 0; + virtual bool is_enabled() const = 0; + + // Differences in clock speed on the primary and reverse streams can impact + // the AEC performance. On the client-side, this could be seen when different + // render and capture devices are used, particularly with webcams. + // + // This enables a compensation mechanism, and requires that + // set_stream_drift_samples() be called. + virtual int enable_drift_compensation(bool enable) = 0; + virtual bool is_drift_compensation_enabled() const = 0; + + // Sets the difference between the number of samples rendered and captured by + // the audio devices since the last call to |ProcessStream()|. Must be called + // if drift compensation is enabled, prior to |ProcessStream()|. + virtual void set_stream_drift_samples(int drift) = 0; + virtual int stream_drift_samples() const = 0; + + enum SuppressionLevel { + kLowSuppression, + kModerateSuppression, + kHighSuppression + }; + + // Sets the aggressiveness of the suppressor. A higher level trades off + // double-talk performance for increased echo suppression. + virtual int set_suppression_level(SuppressionLevel level) = 0; + virtual SuppressionLevel suppression_level() const = 0; + + // Returns false if the current frame almost certainly contains no echo + // and true if it _might_ contain echo. + virtual bool stream_has_echo() const = 0; + + // Enables the computation of various echo metrics. These are obtained + // through |GetMetrics()|. + virtual int enable_metrics(bool enable) = 0; + virtual bool are_metrics_enabled() const = 0; + + // Each statistic is reported in dB. + // P_far: Far-end (render) signal power. + // P_echo: Near-end (capture) echo signal power. + // P_out: Signal power at the output of the AEC. + // P_a: Internal signal power at the point before the AEC's non-linear + // processor. + struct Metrics { + // RERL = ERL + ERLE + AudioProcessing::Statistic residual_echo_return_loss; + + // ERL = 10log_10(P_far / P_echo) + AudioProcessing::Statistic echo_return_loss; + + // ERLE = 10log_10(P_echo / P_out) + AudioProcessing::Statistic echo_return_loss_enhancement; + + // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a) + AudioProcessing::Statistic a_nlp; + + // Fraction of time that the AEC linear filter is divergent, in a 1-second + // non-overlapped aggregation window. + float divergent_filter_fraction; + }; + + // Deprecated. Use GetStatistics on the AudioProcessing interface instead. + // TODO(ajm): discuss the metrics update period. + virtual int GetMetrics(Metrics* metrics) = 0; + + // Enables computation and logging of delay values. Statistics are obtained + // through |GetDelayMetrics()|. + virtual int enable_delay_logging(bool enable) = 0; + virtual bool is_delay_logging_enabled() const = 0; + + // The delay metrics consists of the delay |median| and the delay standard + // deviation |std|. It also consists of the fraction of delay estimates + // |fraction_poor_delays| that can make the echo cancellation perform poorly. + // The values are aggregated until the first call to |GetDelayMetrics()| and + // afterwards aggregated and updated every second. + // Note that if there are several clients pulling metrics from + // |GetDelayMetrics()| during a session the first call from any of them will + // change to one second aggregation window for all. + // Deprecated. Use GetStatistics on the AudioProcessing interface instead. + virtual int GetDelayMetrics(int* median, int* std) = 0; + // Deprecated. Use GetStatistics on the AudioProcessing interface instead. + virtual int GetDelayMetrics(int* median, int* std, + float* fraction_poor_delays) = 0; + + // Returns a pointer to the low level AEC component. In case of multiple + // channels, the pointer to the first one is returned. A NULL pointer is + // returned when the AEC component is disabled or has not been initialized + // successfully. + virtual struct AecCore* aec_core() const = 0; + + protected: + virtual ~EchoCancellation() {} +}; + +// The acoustic echo control for mobile (AECM) component is a low complexity +// robust option intended for use on mobile devices. +// +// Not recommended to be enabled on the server-side. +class EchoControlMobile { + public: + // EchoCancellation and EchoControlMobile may not be enabled simultaneously. + // Enabling one will disable the other. + virtual int Enable(bool enable) = 0; + virtual bool is_enabled() const = 0; + + // Recommended settings for particular audio routes. In general, the louder + // the echo is expected to be, the higher this value should be set. The + // preferred setting may vary from device to device. + enum RoutingMode { + kQuietEarpieceOrHeadset, + kEarpiece, + kLoudEarpiece, + kSpeakerphone, + kLoudSpeakerphone + }; + + // Sets echo control appropriate for the audio routing |mode| on the device. + // It can and should be updated during a call if the audio routing changes. + virtual int set_routing_mode(RoutingMode mode) = 0; + virtual RoutingMode routing_mode() const = 0; + + // Comfort noise replaces suppressed background noise to maintain a + // consistent signal level. + virtual int enable_comfort_noise(bool enable) = 0; + virtual bool is_comfort_noise_enabled() const = 0; + + // A typical use case is to initialize the component with an echo path from a + // previous call. The echo path is retrieved using |GetEchoPath()|, typically + // at the end of a call. The data can then be stored for later use as an + // initializer before the next call, using |SetEchoPath()|. + // + // Controlling the echo path this way requires the data |size_bytes| to match + // the internal echo path size. This size can be acquired using + // |echo_path_size_bytes()|. |SetEchoPath()| causes an entire reset, worth + // noting if it is to be called during an ongoing call. + // + // It is possible that version incompatibilities may result in a stored echo + // path of the incorrect size. In this case, the stored path should be + // discarded. + virtual int SetEchoPath(const void* echo_path, size_t size_bytes) = 0; + virtual int GetEchoPath(void* echo_path, size_t size_bytes) const = 0; + + // The returned path size is guaranteed not to change for the lifetime of + // the application. + static size_t echo_path_size_bytes(); + + protected: + virtual ~EchoControlMobile() {} +}; + +// Interface for an acoustic echo cancellation (AEC) submodule. +class EchoControl { + public: + // Analysis (not changing) of the render signal. + virtual void AnalyzeRender(AudioBuffer* render) = 0; + + // Analysis (not changing) of the capture signal. + virtual void AnalyzeCapture(AudioBuffer* capture) = 0; + + // Processes the capture signal in order to remove the echo. + virtual void ProcessCapture(AudioBuffer* capture, bool echo_path_change) = 0; + + struct Metrics { + double echo_return_loss; + double echo_return_loss_enhancement; + int delay_ms; + }; + + // Collect current metrics from the echo controller. + virtual Metrics GetMetrics() const = 0; + + virtual ~EchoControl() {} +}; + +// Interface for a factory that creates EchoControllers. +class EchoControlFactory { + public: + virtual std::unique_ptr Create(int sample_rate_hz) = 0; + virtual ~EchoControlFactory() = default; +}; + +// The automatic gain control (AGC) component brings the signal to an +// appropriate range. This is done by applying a digital gain directly and, in +// the analog mode, prescribing an analog gain to be applied at the audio HAL. +// +// Recommended to be enabled on the client-side. +class GainControl { + public: + virtual int Enable(bool enable) = 0; + virtual bool is_enabled() const = 0; + + // When an analog mode is set, this must be called prior to |ProcessStream()| + // to pass the current analog level from the audio HAL. Must be within the + // range provided to |set_analog_level_limits()|. + virtual int set_stream_analog_level(int level) = 0; + + // When an analog mode is set, this should be called after |ProcessStream()| + // to obtain the recommended new analog level for the audio HAL. It is the + // users responsibility to apply this level. + virtual int stream_analog_level() = 0; + + enum Mode { + // Adaptive mode intended for use if an analog volume control is available + // on the capture device. It will require the user to provide coupling + // between the OS mixer controls and AGC through the |stream_analog_level()| + // functions. + // + // It consists of an analog gain prescription for the audio device and a + // digital compression stage. + kAdaptiveAnalog, + + // Adaptive mode intended for situations in which an analog volume control + // is unavailable. It operates in a similar fashion to the adaptive analog + // mode, but with scaling instead applied in the digital domain. As with + // the analog mode, it additionally uses a digital compression stage. + kAdaptiveDigital, + + // Fixed mode which enables only the digital compression stage also used by + // the two adaptive modes. + // + // It is distinguished from the adaptive modes by considering only a + // short time-window of the input signal. It applies a fixed gain through + // most of the input level range, and compresses (gradually reduces gain + // with increasing level) the input signal at higher levels. This mode is + // preferred on embedded devices where the capture signal level is + // predictable, so that a known gain can be applied. + kFixedDigital + }; + + virtual int set_mode(Mode mode) = 0; + virtual Mode mode() const = 0; + + // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels + // from digital full-scale). The convention is to use positive values. For + // instance, passing in a value of 3 corresponds to -3 dBFs, or a target + // level 3 dB below full-scale. Limited to [0, 31]. + // + // TODO(ajm): use a negative value here instead, if/when VoE will similarly + // update its interface. + virtual int set_target_level_dbfs(int level) = 0; + virtual int target_level_dbfs() const = 0; + + // Sets the maximum |gain| the digital compression stage may apply, in dB. A + // higher number corresponds to greater compression, while a value of 0 will + // leave the signal uncompressed. Limited to [0, 90]. + virtual int set_compression_gain_db(int gain) = 0; + virtual int compression_gain_db() const = 0; + + // When enabled, the compression stage will hard limit the signal to the + // target level. Otherwise, the signal will be compressed but not limited + // above the target level. + virtual int enable_limiter(bool enable) = 0; + virtual bool is_limiter_enabled() const = 0; + + // Sets the |minimum| and |maximum| analog levels of the audio capture device. + // Must be set if and only if an analog mode is used. Limited to [0, 65535]. + virtual int set_analog_level_limits(int minimum, + int maximum) = 0; + virtual int analog_level_minimum() const = 0; + virtual int analog_level_maximum() const = 0; + + // Returns true if the AGC has detected a saturation event (period where the + // signal reaches digital full-scale) in the current frame and the analog + // level cannot be reduced. + // + // This could be used as an indicator to reduce or disable analog mic gain at + // the audio HAL. + virtual bool stream_is_saturated() const = 0; + + protected: + virtual ~GainControl() {} +}; +// TODO(peah): Remove this interface. +// A filtering component which removes DC offset and low-frequency noise. +// Recommended to be enabled on the client-side. +class HighPassFilter { + public: + virtual int Enable(bool enable) = 0; + virtual bool is_enabled() const = 0; + + virtual ~HighPassFilter() {} +}; + +// An estimation component used to retrieve level metrics. +class LevelEstimator { + public: + virtual int Enable(bool enable) = 0; + virtual bool is_enabled() const = 0; + + // Returns the root mean square (RMS) level in dBFs (decibels from digital + // full-scale), or alternately dBov. It is computed over all primary stream + // frames since the last call to RMS(). The returned value is positive but + // should be interpreted as negative. It is constrained to [0, 127]. + // + // The computation follows: https://tools.ietf.org/html/rfc6465 + // with the intent that it can provide the RTP audio level indication. + // + // Frames passed to ProcessStream() with an |_energy| of zero are considered + // to have been muted. The RMS of the frame will be interpreted as -127. + virtual int RMS() = 0; + + protected: + virtual ~LevelEstimator() {} +}; + +// The noise suppression (NS) component attempts to remove noise while +// retaining speech. Recommended to be enabled on the client-side. +// +// Recommended to be enabled on the client-side. +class NoiseSuppression { + public: + virtual int Enable(bool enable) = 0; + virtual bool is_enabled() const = 0; + + // Determines the aggressiveness of the suppression. Increasing the level + // will reduce the noise level at the expense of a higher speech distortion. + enum Level { + kLow, + kModerate, + kHigh, + kVeryHigh + }; + + virtual int set_level(Level level) = 0; + virtual Level level() const = 0; + + // Returns the internally computed prior speech probability of current frame + // averaged over output channels. This is not supported in fixed point, for + // which |kUnsupportedFunctionError| is returned. + virtual float speech_probability() const = 0; + + // Returns the noise estimate per frequency bin averaged over all channels. + virtual std::vector NoiseEstimate() = 0; + + protected: + virtual ~NoiseSuppression() {} +}; + +// Interface for a post processing submodule. +class PostProcessing { + public: + // (Re-)Initializes the submodule. + virtual void Initialize(int sample_rate_hz, int num_channels) = 0; + // Processes the given capture or render signal. + virtual void Process(AudioBuffer* audio) = 0; + // Returns a string representation of the module state. + virtual std::string ToString() const = 0; + + virtual ~PostProcessing() {} +}; + +// The voice activity detection (VAD) component analyzes the stream to +// determine if voice is present. A facility is also provided to pass in an +// external VAD decision. +// +// In addition to |stream_has_voice()| the VAD decision is provided through the +// |AudioFrame| passed to |ProcessStream()|. The |vad_activity_| member will be +// modified to reflect the current decision. +class VoiceDetection { + public: + virtual int Enable(bool enable) = 0; + virtual bool is_enabled() const = 0; + + // Returns true if voice is detected in the current frame. Should be called + // after |ProcessStream()|. + virtual bool stream_has_voice() const = 0; + + // Some of the APM functionality requires a VAD decision. In the case that + // a decision is externally available for the current frame, it can be passed + // in here, before |ProcessStream()| is called. + // + // VoiceDetection does _not_ need to be enabled to use this. If it happens to + // be enabled, detection will be skipped for any frame in which an external + // VAD decision is provided. + virtual int set_stream_has_voice(bool has_voice) = 0; + + // Specifies the likelihood that a frame will be declared to contain voice. + // A higher value makes it more likely that speech will not be clipped, at + // the expense of more noise being detected as voice. + enum Likelihood { + kVeryLowLikelihood, + kLowLikelihood, + kModerateLikelihood, + kHighLikelihood + }; + + virtual int set_likelihood(Likelihood likelihood) = 0; + virtual Likelihood likelihood() const = 0; + + // Sets the |size| of the frames in ms on which the VAD will operate. Larger + // frames will improve detection accuracy, but reduce the frequency of + // updates. + // + // This does not impact the size of frames passed to |ProcessStream()|. + virtual int set_frame_size_ms(int size) = 0; + virtual int frame_size_ms() const = 0; + + protected: + virtual ~VoiceDetection() {} +}; + +// Configuration struct for EchoCanceller3 +struct EchoCanceller3Config { + struct Delay { + size_t default_delay = 5; + size_t down_sampling_factor = 4; + size_t num_filters = 4; + } delay; + + struct Erle { + float min = 1.f; + float max_l = 8.f; + float max_h = 1.5f; + } erle; + + struct EpStrength { + float lf = 10.f; + float mf = 10.f; + float hf = 10.f; + float default_len = 0.f; + bool echo_can_saturate = true; + bool bounded_erl = false; + } ep_strength; + + struct Mask { + float m1 = 0.01f; + float m2 = 0.0001f; + float m3 = 0.01f; + float m4 = 0.1f; + float m5 = 0.3f; + float m6 = 0.0001f; + float m7 = 0.01f; + float m8 = 0.0001f; + float m9 = 0.1f; + } gain_mask; + + struct EchoAudibility { + float low_render_limit = 4 * 64.f; + float normal_render_limit = 64.f; + } echo_audibility; + + struct RenderLevels { + float active_render_limit = 100.f; + float poor_excitation_render_limit = 150.f; + } render_levels; + + struct GainUpdates { + struct GainChanges { + float max_inc; + float max_dec; + float rate_inc; + float rate_dec; + float min_inc; + float min_dec; + }; + + GainChanges low_noise = {3.f, 3.f, 1.5f, 1.5f, 1.5f, 1.5f}; + GainChanges normal = {2.f, 2.f, 1.5f, 1.5f, 1.2f, 1.2f}; + GainChanges saturation = {1.2f, 1.2f, 1.5f, 1.5f, 1.f, 1.f}; + GainChanges nonlinear = {1.5f, 1.5f, 1.2f, 1.2f, 1.1f, 1.1f}; + + float floor_first_increase = 0.0001f; + } gain_updates; +}; + +class EchoCanceller3Factory : public EchoControlFactory { + public: + EchoCanceller3Factory(); + EchoCanceller3Factory(const EchoCanceller3Config& config); + std::unique_ptr Create(int sample_rate_hz) override; + + private: + EchoCanceller3Config config_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.cc new file mode 100644 index 0000000000..7139ee502e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.cc @@ -0,0 +1,22 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/include/audio_processing_statistics.h" + +namespace webrtc { + +AudioProcessingStats::AudioProcessingStats() = default; + +AudioProcessingStats::AudioProcessingStats(const AudioProcessingStats& other) = + default; + +AudioProcessingStats::~AudioProcessingStats() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.h b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.h new file mode 100644 index 0000000000..83c9d9932f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/audio_processing_statistics.h @@ -0,0 +1,56 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_ + +#include "api/optional.h" + +namespace webrtc { +// This version of the stats uses Optionals, it will replace the regular +// AudioProcessingStatistics struct. +struct AudioProcessingStats { + AudioProcessingStats(); + AudioProcessingStats(const AudioProcessingStats& other); + ~AudioProcessingStats(); + + // AEC Statistics. + // ERL = 10log_10(P_far / P_echo) + rtc::Optional echo_return_loss; + // ERLE = 10log_10(P_echo / P_out) + rtc::Optional echo_return_loss_enhancement; + // Fraction of time that the AEC linear filter is divergent, in a 1-second + // non-overlapped aggregation window. + rtc::Optional divergent_filter_fraction; + + // The delay metrics consists of the delay median and standard deviation. It + // also consists of the fraction of delay estimates that can make the echo + // cancellation perform poorly. The values are aggregated until the first + // call to |GetStatistics()| and afterwards aggregated and updated every + // second. Note that if there are several clients pulling metrics from + // |GetStatistics()| during a session the first call from any of them will + // change to one second aggregation window for all. + rtc::Optional delay_median_ms; + rtc::Optional delay_standard_deviation_ms; + + // Residual echo detector likelihood. + rtc::Optional residual_echo_likelihood; + // Maximum residual echo likelihood from the last time period. + rtc::Optional residual_echo_likelihood_recent_max; + + // The instantaneous delay estimate produced in the AEC. The unit is in + // milliseconds and the value is the instantaneous value at the time of the + // call to |GetStatistics()|. + int delay_ms; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/config.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/include/config.cc new file mode 100644 index 0000000000..14240db305 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/config.cc @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/include/config.h" + +namespace webrtc { + +Config::Config() {} + +Config::~Config() { + for (OptionMap::iterator it = options_.begin(); it != options_.end(); ++it) { + delete it->second; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/config.h b/third_party/libwebrtc/webrtc/modules/audio_processing/include/config.h new file mode 100644 index 0000000000..16128943b3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/config.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_CONFIG_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_CONFIG_H_ + +#include + +#include "rtc_base/basictypes.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Only add new values to the end of the enumeration and never remove (only +// deprecate) to maintain binary compatibility. +enum class ConfigOptionID { + kMyExperimentForTest, + kAlgo1CostFunctionForTest, + kTemporalLayersFactory, // Deprecated + kNetEqCapacityConfig, // Deprecated + kNetEqFastAccelerate, // Deprecated + kVoicePacing, // Deprecated + kExtendedFilter, + kDelayAgnostic, + kExperimentalAgc, + kExperimentalNs, + kBeamforming, + kIntelligibility, + kEchoCanceller3, // Deprecated + kAecRefinedAdaptiveFilter, + kLevelControl, + kCaptureDeviceInfo +}; + +// Class Config is designed to ease passing a set of options across webrtc code. +// Options are identified by typename in order to avoid incorrect casts. +// +// Usage: +// * declaring an option: +// struct Algo1_CostFunction { +// virtual float cost(int x) const { return x; } +// virtual ~Algo1_CostFunction() {} +// }; +// +// * accessing an option: +// config.Get().cost(value); +// +// * setting an option: +// struct SqrCost : Algo1_CostFunction { +// virtual float cost(int x) const { return x*x; } +// }; +// config.Set(new SqrCost()); +// +// Note: This class is thread-compatible (like STL containers). +class Config { + public: + // Returns the option if set or a default constructed one. + // Callers that access options too often are encouraged to cache the result. + // Returned references are owned by this. + // + // Requires std::is_default_constructible + template const T& Get() const; + + // Set the option, deleting any previous instance of the same. + // This instance gets ownership of the newly set value. + template void Set(T* value); + + Config(); + ~Config(); + + private: + struct BaseOption { + virtual ~BaseOption() {} + }; + + template + struct Option : BaseOption { + explicit Option(T* v): value(v) {} + ~Option() { + delete value; + } + T* value; + }; + + template + static ConfigOptionID identifier() { + return T::identifier; + } + + // Used to instantiate a default constructed object that doesn't needs to be + // owned. This allows Get to be implemented without requiring explicitly + // locks. + template + static const T& default_value() { + RTC_DEFINE_STATIC_LOCAL(const T, def, ()); + return def; + } + + typedef std::map OptionMap; + OptionMap options_; + + // RTC_DISALLOW_COPY_AND_ASSIGN + Config(const Config&); + void operator=(const Config&); +}; + +template +const T& Config::Get() const { + OptionMap::const_iterator it = options_.find(identifier()); + if (it != options_.end()) { + const T* t = static_cast*>(it->second)->value; + if (t) { + return *t; + } + } + return default_value(); +} + +template +void Config::Set(T* value) { + BaseOption*& it = options_[identifier()]; + delete it; + it = new Option(value); +} +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_CONFIG_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/include/mock_audio_processing.h b/third_party/libwebrtc/webrtc/modules/audio_processing/include/mock_audio_processing.h new file mode 100644 index 0000000000..f2bdc2f241 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/include/mock_audio_processing.h @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_MOCK_AUDIO_PROCESSING_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_MOCK_AUDIO_PROCESSING_H_ + +#include + +#include "modules/audio_processing/include/aec_dump.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/include/audio_processing_statistics.h" +#include "test/gmock.h" + +namespace webrtc { + +namespace test { + +class MockEchoCancellation : public EchoCancellation { + public: + virtual ~MockEchoCancellation() {} + MOCK_METHOD1(Enable, int(bool enable)); + MOCK_CONST_METHOD0(is_enabled, bool()); + MOCK_METHOD1(enable_drift_compensation, int(bool enable)); + MOCK_CONST_METHOD0(is_drift_compensation_enabled, bool()); + MOCK_METHOD1(set_stream_drift_samples, void(int drift)); + MOCK_CONST_METHOD0(stream_drift_samples, int()); + MOCK_METHOD1(set_suppression_level, int(SuppressionLevel level)); + MOCK_CONST_METHOD0(suppression_level, SuppressionLevel()); + MOCK_CONST_METHOD0(stream_has_echo, bool()); + MOCK_METHOD1(enable_metrics, int(bool enable)); + MOCK_CONST_METHOD0(are_metrics_enabled, bool()); + MOCK_METHOD1(GetMetrics, int(Metrics* metrics)); + MOCK_METHOD1(enable_delay_logging, int(bool enable)); + MOCK_CONST_METHOD0(is_delay_logging_enabled, bool()); + MOCK_METHOD2(GetDelayMetrics, int(int* median, int* std)); + MOCK_METHOD3(GetDelayMetrics, int(int* median, int* std, + float* fraction_poor_delays)); + MOCK_CONST_METHOD0(aec_core, struct AecCore*()); +}; + +class MockEchoControlMobile : public EchoControlMobile { + public: + virtual ~MockEchoControlMobile() {} + MOCK_METHOD1(Enable, int(bool enable)); + MOCK_CONST_METHOD0(is_enabled, bool()); + MOCK_METHOD1(set_routing_mode, int(RoutingMode mode)); + MOCK_CONST_METHOD0(routing_mode, RoutingMode()); + MOCK_METHOD1(enable_comfort_noise, int(bool enable)); + MOCK_CONST_METHOD0(is_comfort_noise_enabled, bool()); + MOCK_METHOD2(SetEchoPath, int(const void* echo_path, size_t size_bytes)); + MOCK_CONST_METHOD2(GetEchoPath, int(void* echo_path, size_t size_bytes)); +}; + +class MockGainControl : public GainControl { + public: + virtual ~MockGainControl() {} + MOCK_METHOD1(Enable, int(bool enable)); + MOCK_CONST_METHOD0(is_enabled, bool()); + MOCK_METHOD1(set_stream_analog_level, int(int level)); + MOCK_METHOD0(stream_analog_level, int()); + MOCK_METHOD1(set_mode, int(Mode mode)); + MOCK_CONST_METHOD0(mode, Mode()); + MOCK_METHOD1(set_target_level_dbfs, int(int level)); + MOCK_CONST_METHOD0(target_level_dbfs, int()); + MOCK_METHOD1(set_compression_gain_db, int(int gain)); + MOCK_CONST_METHOD0(compression_gain_db, int()); + MOCK_METHOD1(enable_limiter, int(bool enable)); + MOCK_CONST_METHOD0(is_limiter_enabled, bool()); + MOCK_METHOD2(set_analog_level_limits, int(int minimum, int maximum)); + MOCK_CONST_METHOD0(analog_level_minimum, int()); + MOCK_CONST_METHOD0(analog_level_maximum, int()); + MOCK_CONST_METHOD0(stream_is_saturated, bool()); +}; + +class MockHighPassFilter : public HighPassFilter { + public: + virtual ~MockHighPassFilter() {} + MOCK_METHOD1(Enable, int(bool enable)); + MOCK_CONST_METHOD0(is_enabled, bool()); +}; + +class MockLevelEstimator : public LevelEstimator { + public: + virtual ~MockLevelEstimator() {} + MOCK_METHOD1(Enable, int(bool enable)); + MOCK_CONST_METHOD0(is_enabled, bool()); + MOCK_METHOD0(RMS, int()); +}; + +class MockNoiseSuppression : public NoiseSuppression { + public: + virtual ~MockNoiseSuppression() {} + MOCK_METHOD1(Enable, int(bool enable)); + MOCK_CONST_METHOD0(is_enabled, bool()); + MOCK_METHOD1(set_level, int(Level level)); + MOCK_CONST_METHOD0(level, Level()); + MOCK_CONST_METHOD0(speech_probability, float()); + MOCK_METHOD0(NoiseEstimate, std::vector()); +}; + +class MockPostProcessing : public PostProcessing { + public: + virtual ~MockPostProcessing() {} + MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels)); + MOCK_METHOD1(Process, void(AudioBuffer* audio)); + MOCK_CONST_METHOD0(ToString, std::string()); +}; + +class MockEchoControl : public EchoControl { + public: + virtual ~MockEchoControl() {} + MOCK_METHOD1(AnalyzeRender, void(AudioBuffer* render)); + MOCK_METHOD1(AnalyzeCapture, void(AudioBuffer* capture)); + MOCK_METHOD2(ProcessCapture, + void(AudioBuffer* capture, bool echo_path_change)); + MOCK_CONST_METHOD0(GetMetrics, Metrics()); +}; + +class MockVoiceDetection : public VoiceDetection { + public: + virtual ~MockVoiceDetection() {} + MOCK_METHOD1(Enable, int(bool enable)); + MOCK_CONST_METHOD0(is_enabled, bool()); + MOCK_CONST_METHOD0(stream_has_voice, bool()); + MOCK_METHOD1(set_stream_has_voice, int(bool has_voice)); + MOCK_METHOD1(set_likelihood, int(Likelihood likelihood)); + MOCK_CONST_METHOD0(likelihood, Likelihood()); + MOCK_METHOD1(set_frame_size_ms, int(int size)); + MOCK_CONST_METHOD0(frame_size_ms, int()); +}; + +class MockAudioProcessing : public AudioProcessing { + public: + MockAudioProcessing() + : echo_cancellation_(new testing::NiceMock()), + echo_control_mobile_(new testing::NiceMock()), + gain_control_(new testing::NiceMock()), + high_pass_filter_(new testing::NiceMock()), + level_estimator_(new testing::NiceMock()), + noise_suppression_(new testing::NiceMock()), + voice_detection_(new testing::NiceMock()) { + } + + virtual ~MockAudioProcessing() {} + + MOCK_METHOD0(Initialize, int()); + MOCK_METHOD6(Initialize, int(int capture_input_sample_rate_hz, + int capture_output_sample_rate_hz, + int render_sample_rate_hz, + ChannelLayout capture_input_layout, + ChannelLayout capture_output_layout, + ChannelLayout render_input_layout)); + MOCK_METHOD1(Initialize, int(const ProcessingConfig& processing_config)); + MOCK_METHOD1(ApplyConfig, void(const Config& config)); + MOCK_METHOD1(SetExtraOptions, void(const webrtc::Config& config)); + MOCK_CONST_METHOD0(proc_sample_rate_hz, int()); + MOCK_CONST_METHOD0(proc_split_sample_rate_hz, int()); + MOCK_CONST_METHOD0(num_input_channels, size_t()); + MOCK_CONST_METHOD0(num_proc_channels, size_t()); + MOCK_CONST_METHOD0(num_output_channels, size_t()); + MOCK_CONST_METHOD0(num_reverse_channels, size_t()); + MOCK_METHOD1(set_output_will_be_muted, void(bool muted)); + MOCK_METHOD1(ProcessStream, int(AudioFrame* frame)); + MOCK_METHOD7(ProcessStream, int(const float* const* src, + size_t samples_per_channel, + int input_sample_rate_hz, + ChannelLayout input_layout, + int output_sample_rate_hz, + ChannelLayout output_layout, + float* const* dest)); + MOCK_METHOD4(ProcessStream, int(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest)); + MOCK_METHOD1(ProcessReverseStream, int(AudioFrame* frame)); + MOCK_METHOD4(AnalyzeReverseStream, int(const float* const* data, + size_t samples_per_channel, + int sample_rate_hz, + ChannelLayout layout)); + MOCK_METHOD4(ProcessReverseStream, int(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest)); + MOCK_METHOD1(set_stream_delay_ms, int(int delay)); + MOCK_CONST_METHOD0(stream_delay_ms, int()); + MOCK_CONST_METHOD0(was_stream_delay_set, bool()); + MOCK_METHOD1(set_stream_key_pressed, void(bool key_pressed)); + MOCK_METHOD1(set_delay_offset_ms, void(int offset)); + MOCK_CONST_METHOD0(delay_offset_ms, int()); + + virtual void AttachAecDump(std::unique_ptr aec_dump) {} + MOCK_METHOD0(DetachAecDump, void()); + + MOCK_METHOD0(UpdateHistogramsOnCallEnd, void()); + MOCK_CONST_METHOD0(GetStatistics, AudioProcessingStatistics()); + MOCK_CONST_METHOD1(GetStatistics, AudioProcessingStats(bool)); + virtual MockEchoCancellation* echo_cancellation() const { + return echo_cancellation_.get(); + } + virtual MockEchoControlMobile* echo_control_mobile() const { + return echo_control_mobile_.get(); + } + virtual MockGainControl* gain_control() const { + return gain_control_.get(); + } + virtual MockHighPassFilter* high_pass_filter() const { + return high_pass_filter_.get(); + } + virtual MockLevelEstimator* level_estimator() const { + return level_estimator_.get(); + } + virtual MockNoiseSuppression* noise_suppression() const { + return noise_suppression_.get(); + } + virtual MockVoiceDetection* voice_detection() const { + return voice_detection_.get(); + } + + MOCK_CONST_METHOD0(GetConfig, AudioProcessing::Config()); + + private: + std::unique_ptr echo_cancellation_; + std::unique_ptr echo_control_mobile_; + std::unique_ptr gain_control_; + std::unique_ptr high_pass_filter_; + std::unique_ptr level_estimator_; + std::unique_ptr noise_suppression_; + std::unique_ptr voice_detection_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_MOCK_AUDIO_PROCESSING_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc new file mode 100644 index 0000000000..0e696d9fff --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc @@ -0,0 +1,392 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/intelligibility/intelligibility_enhancer.h" + +#include +#include +#include +#include +#include + +#include "common_audio/include/audio_util.h" +#include "common_audio/window_generator.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +namespace { + +const size_t kErbResolution = 2; +const int kWindowSizeMs = 16; +const int kChunkSizeMs = 10; // Size provided by APM. +const float kClipFreqKhz = 0.2f; +const float kKbdAlpha = 1.5f; +const float kLambdaBot = -1.f; // Extreme values in bisection +const float kLambdaTop = -1e-5f; // search for lamda. +const float kVoiceProbabilityThreshold = 0.5f; +// Number of chunks after voice activity which is still considered speech. +const size_t kSpeechOffsetDelay = 10; +const float kDecayRate = 0.995f; // Power estimation decay rate. +const float kMaxRelativeGainChange = 0.005f; +const float kRho = 0.0004f; // Default production and interpretation SNR. +const float kPowerNormalizationFactor = 1.f / (1 << 30); +const float kMaxActiveSNR = 128.f; // 21dB +const float kMinInactiveSNR = 32.f; // 15dB +const size_t kGainUpdatePeriod = 10u; + +// Returns dot product of vectors |a| and |b| with size |length|. +float DotProduct(const float* a, const float* b, size_t length) { + float ret = 0.f; + for (size_t i = 0; i < length; ++i) { + ret += a[i] * b[i]; + } + return ret; +} + +// Computes the power across ERB bands from the power spectral density |pow|. +// Stores it in |result|. +void MapToErbBands(const float* pow, + const std::vector>& filter_bank, + float* result) { + for (size_t i = 0; i < filter_bank.size(); ++i) { + RTC_DCHECK_GT(filter_bank[i].size(), 0); + result[i] = kPowerNormalizationFactor * + DotProduct(filter_bank[i].data(), pow, filter_bank[i].size()); + } +} + +} // namespace + +IntelligibilityEnhancer::IntelligibilityEnhancer(int sample_rate_hz, + size_t num_render_channels, + size_t num_bands, + size_t num_noise_bins) + : freqs_(RealFourier::ComplexLength( + RealFourier::FftOrder(sample_rate_hz * kWindowSizeMs / 1000))), + num_noise_bins_(num_noise_bins), + chunk_length_(static_cast(sample_rate_hz * kChunkSizeMs / 1000)), + bank_size_(GetBankSize(sample_rate_hz, kErbResolution)), + sample_rate_hz_(sample_rate_hz), + num_render_channels_(num_render_channels), + clear_power_estimator_(freqs_, kDecayRate), + noise_power_estimator_(num_noise_bins, kDecayRate), + filtered_clear_pow_(bank_size_, 0.f), + filtered_noise_pow_(num_noise_bins, 0.f), + center_freqs_(bank_size_), + capture_filter_bank_(CreateErbBank(num_noise_bins)), + render_filter_bank_(CreateErbBank(freqs_)), + gains_eq_(bank_size_), + gain_applier_(freqs_, kMaxRelativeGainChange), + audio_s16_(chunk_length_), + chunks_since_voice_(kSpeechOffsetDelay), + is_speech_(false), + snr_(kMaxActiveSNR), + is_active_(false), + num_chunks_(0u), + num_active_chunks_(0u), + noise_estimation_buffer_(num_noise_bins), + noise_estimation_queue_(kMaxNumNoiseEstimatesToBuffer, + std::vector(num_noise_bins), + RenderQueueItemVerifier(num_noise_bins)) { + RTC_DCHECK_LE(kRho, 1.f); + + const size_t erb_index = static_cast( + ceilf(11.17f * logf((kClipFreqKhz + 0.312f) / (kClipFreqKhz + 14.6575f)) + + 43.f)); + start_freq_ = std::max(static_cast(1), erb_index * kErbResolution); + + size_t window_size = static_cast(1) << RealFourier::FftOrder(freqs_); + std::vector kbd_window(window_size); + WindowGenerator::KaiserBesselDerived(kKbdAlpha, window_size, + kbd_window.data()); + render_mangler_.reset(new LappedTransform( + num_render_channels_, num_render_channels_, chunk_length_, + kbd_window.data(), window_size, window_size / 2, this)); + + const size_t initial_delay = render_mangler_->initial_delay(); + for (size_t i = 0u; i < num_bands - 1; ++i) { + high_bands_buffers_.push_back(std::unique_ptr( + new intelligibility::DelayBuffer(initial_delay, num_render_channels_))); + } +} + +IntelligibilityEnhancer::~IntelligibilityEnhancer() { + // Don't rely on this log, since the destructor isn't called when the + // app/tab is killed. + if (num_chunks_ > 0) { + RTC_LOG(LS_INFO) << "Intelligibility Enhancer was active for " + << 100.f * static_cast(num_active_chunks_) / + num_chunks_ + << "% of the call."; + } else { + RTC_LOG(LS_INFO) << "Intelligibility Enhancer processed no chunk."; + } +} + +void IntelligibilityEnhancer::SetCaptureNoiseEstimate( + std::vector noise, float gain) { + RTC_DCHECK_EQ(noise.size(), num_noise_bins_); + for (auto& bin : noise) { + bin *= gain; + } + // Disregarding return value since buffer overflow is acceptable, because it + // is not critical to get each noise estimate. + if (noise_estimation_queue_.Insert(&noise)) { + }; +} + +void IntelligibilityEnhancer::ProcessRenderAudio(AudioBuffer* audio) { + RTC_DCHECK_EQ(num_render_channels_, audio->num_channels()); + while (noise_estimation_queue_.Remove(&noise_estimation_buffer_)) { + noise_power_estimator_.Step(noise_estimation_buffer_.data()); + } + float* const* low_band = audio->split_channels_f(kBand0To8kHz); + is_speech_ = IsSpeech(low_band[0]); + render_mangler_->ProcessChunk(low_band, low_band); + DelayHighBands(audio); +} + +void IntelligibilityEnhancer::ProcessAudioBlock( + const std::complex* const* in_block, + size_t in_channels, + size_t frames, + size_t /* out_channels */, + std::complex* const* out_block) { + RTC_DCHECK_EQ(freqs_, frames); + if (is_speech_) { + clear_power_estimator_.Step(in_block[0]); + } + SnrBasedEffectActivation(); + ++num_chunks_; + if (is_active_) { + ++num_active_chunks_; + if (num_chunks_ % kGainUpdatePeriod == 0) { + MapToErbBands(clear_power_estimator_.power().data(), render_filter_bank_, + filtered_clear_pow_.data()); + MapToErbBands(noise_power_estimator_.power().data(), capture_filter_bank_, + filtered_noise_pow_.data()); + SolveForGainsGivenLambda(kLambdaTop, start_freq_, gains_eq_.data()); + const float power_target = std::accumulate( + filtered_clear_pow_.data(), + filtered_clear_pow_.data() + bank_size_, + 0.f); + const float power_top = + DotProduct(gains_eq_.data(), filtered_clear_pow_.data(), bank_size_); + SolveForGainsGivenLambda(kLambdaBot, start_freq_, gains_eq_.data()); + const float power_bot = + DotProduct(gains_eq_.data(), filtered_clear_pow_.data(), bank_size_); + if (power_target >= power_bot && power_target <= power_top) { + SolveForLambda(power_target); + UpdateErbGains(); + } // Else experiencing power underflow, so do nothing. + } + } + for (size_t i = 0; i < in_channels; ++i) { + gain_applier_.Apply(in_block[i], out_block[i]); + } +} + +void IntelligibilityEnhancer::SnrBasedEffectActivation() { + const float* clear_psd = clear_power_estimator_.power().data(); + const float* noise_psd = noise_power_estimator_.power().data(); + const float clear_power = + std::accumulate(clear_psd, clear_psd + freqs_, 0.f); + const float noise_power = + std::accumulate(noise_psd, noise_psd + freqs_, 0.f); + snr_ = kDecayRate * snr_ + (1.f - kDecayRate) * clear_power / + (noise_power + std::numeric_limits::epsilon()); + if (is_active_) { + if (snr_ > kMaxActiveSNR) { + RTC_LOG(LS_INFO) << "Intelligibility Enhancer was deactivated at chunk " + << num_chunks_; + is_active_ = false; + // Set the target gains to unity. + float* gains = gain_applier_.target(); + for (size_t i = 0; i < freqs_; ++i) { + gains[i] = 1.f; + } + } + } else { + if (snr_ < kMinInactiveSNR) { + RTC_LOG(LS_INFO) << "Intelligibility Enhancer was activated at chunk " + << num_chunks_; + is_active_ = true; + } + } +} + +void IntelligibilityEnhancer::SolveForLambda(float power_target) { + const float kConvergeThresh = 0.001f; // TODO(ekmeyerson): Find best values + const int kMaxIters = 100; // for these, based on experiments. + + const float reciprocal_power_target = + 1.f / (power_target + std::numeric_limits::epsilon()); + float lambda_bot = kLambdaBot; + float lambda_top = kLambdaTop; + float power_ratio = 2.f; // Ratio of achieved power to target power. + int iters = 0; + while (std::fabs(power_ratio - 1.f) > kConvergeThresh && iters <= kMaxIters) { + const float lambda = (lambda_bot + lambda_top) / 2.f; + SolveForGainsGivenLambda(lambda, start_freq_, gains_eq_.data()); + const float power = + DotProduct(gains_eq_.data(), filtered_clear_pow_.data(), bank_size_); + if (power < power_target) { + lambda_bot = lambda; + } else { + lambda_top = lambda; + } + power_ratio = std::fabs(power * reciprocal_power_target); + ++iters; + } +} + +void IntelligibilityEnhancer::UpdateErbGains() { + // (ERB gain) = filterbank' * (freq gain) + float* gains = gain_applier_.target(); + for (size_t i = 0; i < freqs_; ++i) { + gains[i] = 0.f; + for (size_t j = 0; j < bank_size_; ++j) { + gains[i] += render_filter_bank_[j][i] * gains_eq_[j]; + } + } +} + +size_t IntelligibilityEnhancer::GetBankSize(int sample_rate, + size_t erb_resolution) { + float freq_limit = sample_rate / 2000.f; + size_t erb_scale = static_cast(ceilf( + 11.17f * logf((freq_limit + 0.312f) / (freq_limit + 14.6575f)) + 43.f)); + return erb_scale * erb_resolution; +} + +std::vector> IntelligibilityEnhancer::CreateErbBank( + size_t num_freqs) { + std::vector> filter_bank(bank_size_); + size_t lf = 1, rf = 4; + + for (size_t i = 0; i < bank_size_; ++i) { + float abs_temp = fabsf((i + 1.f) / static_cast(kErbResolution)); + center_freqs_[i] = 676170.4f / (47.06538f - expf(0.08950404f * abs_temp)); + center_freqs_[i] -= 14678.49f; + } + float last_center_freq = center_freqs_[bank_size_ - 1]; + for (size_t i = 0; i < bank_size_; ++i) { + center_freqs_[i] *= 0.5f * sample_rate_hz_ / last_center_freq; + } + + for (size_t i = 0; i < bank_size_; ++i) { + filter_bank[i].resize(num_freqs); + } + + for (size_t i = 1; i <= bank_size_; ++i) { + size_t lll = static_cast( + round(center_freqs_[rtc::SafeMax(1, i - lf) - 1] * num_freqs / + (0.5f * sample_rate_hz_))); + size_t ll = static_cast( + round(center_freqs_[rtc::SafeMax(1, i) - 1] * num_freqs / + (0.5f * sample_rate_hz_))); + lll = rtc::SafeClamp(lll, 1, num_freqs) - 1; + ll = rtc::SafeClamp(ll, 1, num_freqs) - 1; + + size_t rrr = static_cast( + round(center_freqs_[rtc::SafeMin(bank_size_, i + rf) - 1] * + num_freqs / (0.5f * sample_rate_hz_))); + size_t rr = static_cast( + round(center_freqs_[rtc::SafeMin(bank_size_, i + 1) - 1] * + num_freqs / (0.5f * sample_rate_hz_))); + rrr = rtc::SafeClamp(rrr, 1, num_freqs) - 1; + rr = rtc::SafeClamp(rr, 1, num_freqs) - 1; + + float step = ll == lll ? 0.f : 1.f / (ll - lll); + float element = 0.f; + for (size_t j = lll; j <= ll; ++j) { + filter_bank[i - 1][j] = element; + element += step; + } + step = rr == rrr ? 0.f : 1.f / (rrr - rr); + element = 1.f; + for (size_t j = rr; j <= rrr; ++j) { + filter_bank[i - 1][j] = element; + element -= step; + } + for (size_t j = ll; j <= rr; ++j) { + filter_bank[i - 1][j] = 1.f; + } + } + + for (size_t i = 0; i < num_freqs; ++i) { + float sum = 0.f; + for (size_t j = 0; j < bank_size_; ++j) { + sum += filter_bank[j][i]; + } + for (size_t j = 0; j < bank_size_; ++j) { + filter_bank[j][i] /= sum; + } + } + return filter_bank; +} + +void IntelligibilityEnhancer::SolveForGainsGivenLambda(float lambda, + size_t start_freq, + float* sols) { + const float kMinPower = 1e-5f; + + const float* pow_x0 = filtered_clear_pow_.data(); + const float* pow_n0 = filtered_noise_pow_.data(); + + for (size_t n = 0; n < start_freq; ++n) { + sols[n] = 1.f; + } + + // Analytic solution for optimal gains. See paper for derivation. + for (size_t n = start_freq; n < bank_size_; ++n) { + if (pow_x0[n] < kMinPower || pow_n0[n] < kMinPower) { + sols[n] = 1.f; + } else { + const float gamma0 = 0.5f * kRho * pow_x0[n] * pow_n0[n] + + lambda * pow_x0[n] * pow_n0[n] * pow_n0[n]; + const float beta0 = + lambda * pow_x0[n] * (2.f - kRho) * pow_x0[n] * pow_n0[n]; + const float alpha0 = + lambda * pow_x0[n] * (1.f - kRho) * pow_x0[n] * pow_x0[n]; + RTC_DCHECK_LT(alpha0, 0.f); + // The quadratic equation should always have real roots, but to guard + // against numerical errors we limit it to a minimum of zero. + sols[n] = std::max( + 0.f, (-beta0 - std::sqrt(std::max( + 0.f, beta0 * beta0 - 4.f * alpha0 * gamma0))) / + (2.f * alpha0)); + } + } +} + +bool IntelligibilityEnhancer::IsSpeech(const float* audio) { + FloatToS16(audio, chunk_length_, audio_s16_.data()); + vad_.ProcessChunk(audio_s16_.data(), chunk_length_, sample_rate_hz_); + if (vad_.last_voice_probability() > kVoiceProbabilityThreshold) { + chunks_since_voice_ = 0; + } else if (chunks_since_voice_ < kSpeechOffsetDelay) { + ++chunks_since_voice_; + } + return chunks_since_voice_ < kSpeechOffsetDelay; +} + +void IntelligibilityEnhancer::DelayHighBands(AudioBuffer* audio) { + RTC_DCHECK_EQ(audio->num_bands(), high_bands_buffers_.size() + 1); + for (size_t i = 0u; i < high_bands_buffers_.size(); ++i) { + Band band = static_cast(i + 1); + high_bands_buffers_[i]->Delay(audio->split_channels_f(band), chunk_length_); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h new file mode 100644 index 0000000000..3e0e269c58 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_ +#define MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_ + +#include +#include +#include + +#include "common_audio/channel_buffer.h" +#include "common_audio/lapped_transform.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/intelligibility/intelligibility_utils.h" +#include "modules/audio_processing/render_queue_item_verifier.h" +#include "modules/audio_processing/vad/voice_activity_detector.h" +#include "rtc_base/swap_queue.h" + +namespace webrtc { + +// Speech intelligibility enhancement module. Reads render and capture +// audio streams and modifies the render stream with a set of gains per +// frequency bin to enhance speech against the noise background. +// Details of the model and algorithm can be found in the original paper: +// http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6882788 +class IntelligibilityEnhancer : public LappedTransform::Callback { + public: + IntelligibilityEnhancer(int sample_rate_hz, + size_t num_render_channels, + size_t num_bands, + size_t num_noise_bins); + + ~IntelligibilityEnhancer() override; + + // Sets the capture noise magnitude spectrum estimate. + void SetCaptureNoiseEstimate(std::vector noise, float gain); + + // Reads chunk of speech in time domain and updates with modified signal. + void ProcessRenderAudio(AudioBuffer* audio); + bool active() const; + + protected: + // All in frequency domain, receives input |in_block|, applies + // intelligibility enhancement, and writes result to |out_block|. + void ProcessAudioBlock(const std::complex* const* in_block, + size_t in_channels, + size_t frames, + size_t out_channels, + std::complex* const* out_block) override; + + private: + FRIEND_TEST_ALL_PREFIXES(IntelligibilityEnhancerTest, TestRenderUpdate); + FRIEND_TEST_ALL_PREFIXES(IntelligibilityEnhancerTest, TestErbCreation); + FRIEND_TEST_ALL_PREFIXES(IntelligibilityEnhancerTest, TestSolveForGains); + FRIEND_TEST_ALL_PREFIXES(IntelligibilityEnhancerTest, + TestNoiseGainHasExpectedResult); + FRIEND_TEST_ALL_PREFIXES(IntelligibilityEnhancerTest, + TestAllBandsHaveSameDelay); + + // Updates the SNR estimation and enables or disables this component using a + // hysteresis. + void SnrBasedEffectActivation(); + + // Bisection search for optimal |lambda|. + void SolveForLambda(float power_target); + + // Transforms freq gains to ERB gains. + void UpdateErbGains(); + + // Returns number of ERB filters. + static size_t GetBankSize(int sample_rate, size_t erb_resolution); + + // Initializes ERB filterbank. + std::vector> CreateErbBank(size_t num_freqs); + + // Analytically solves quadratic for optimal gains given |lambda|. + // Negative gains are set to 0. Stores the results in |sols|. + void SolveForGainsGivenLambda(float lambda, size_t start_freq, float* sols); + + // Returns true if the audio is speech. + bool IsSpeech(const float* audio); + + // Delays the high bands to compensate for the processing delay in the low + // band. + void DelayHighBands(AudioBuffer* audio); + + static const size_t kMaxNumNoiseEstimatesToBuffer = 5; + + const size_t freqs_; // Num frequencies in frequency domain. + const size_t num_noise_bins_; + const size_t chunk_length_; // Chunk size in samples. + const size_t bank_size_; // Num ERB filters. + const int sample_rate_hz_; + const size_t num_render_channels_; + + intelligibility::PowerEstimator> clear_power_estimator_; + intelligibility::PowerEstimator noise_power_estimator_; + std::vector filtered_clear_pow_; + std::vector filtered_noise_pow_; + std::vector center_freqs_; + std::vector> capture_filter_bank_; + std::vector> render_filter_bank_; + size_t start_freq_; + + std::vector gains_eq_; // Pre-filter modified gains. + intelligibility::GainApplier gain_applier_; + + std::unique_ptr render_mangler_; + + VoiceActivityDetector vad_; + std::vector audio_s16_; + size_t chunks_since_voice_; + bool is_speech_; + float snr_; + bool is_active_; + + unsigned long int num_chunks_; + unsigned long int num_active_chunks_; + + std::vector noise_estimation_buffer_; + SwapQueue, RenderQueueItemVerifier> + noise_estimation_queue_; + + std::vector> + high_bands_buffers_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer_unittest.cc new file mode 100644 index 0000000000..98a8dae469 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer_unittest.cc @@ -0,0 +1,536 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include +#include +#include + +#include "api/array_view.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/intelligibility/intelligibility_enhancer.h" +#include "modules/audio_processing/noise_suppression_impl.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "rtc_base/arraysize.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +// Target output for ERB create test. Generated with matlab. +const float kTestCenterFreqs[] = { + 14.5213f, 29.735f, 45.6781f, 62.3884f, 79.9058f, 98.2691f, 117.521f, + 137.708f, 158.879f, 181.084f, 204.378f, 228.816f, 254.459f, 281.371f, + 309.618f, 339.273f, 370.411f, 403.115f, 437.469f, 473.564f, 511.497f, + 551.371f, 593.293f, 637.386f, 683.77f, 732.581f, 783.96f, 838.06f, + 895.046f, 955.09f, 1018.38f, 1085.13f, 1155.54f, 1229.85f, 1308.32f, + 1391.22f, 1478.83f, 1571.5f, 1669.55f, 1773.37f, 1883.37f, 2000.f}; +const float kTestFilterBank[][33] = { + {0.2f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.2f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.2f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.2f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.2f, 0.25f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.25f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.25f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.25f, 0.25f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.25f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.25f, 0.142857f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.25f, 0.285714f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.285714f, 0.142857f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.285714f, 0.285714f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.285714f, 0.142857f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.285714f, 0.285714f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.285714f, 0.142857f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.285714f, 0.285714f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.285714f, 0.142857f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.285714f, 0.285714f, 0.157895f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.285714f, 0.210526f, 0.117647f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.285714f, 0.315789f, 0.176471f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.315789f, 0.352941f, 0.142857f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.352941f, 0.285714f, + 0.157895f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.285714f, + 0.210526f, 0.111111f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.285714f, 0.315789f, 0.222222f, 0.111111f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.315789f, 0.333333f, 0.222222f, 0.111111f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.333333f, 0.333333f, 0.222222f, 0.111111f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.333333f, 0.333333f, 0.222222f, 0.111111f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.333333f, 0.333333f, 0.222222f, 0.111111f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.333333f, 0.333333f, 0.222222f, + 0.108108f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.333333f, 0.333333f, + 0.243243f, 0.153846f, 0.0833333f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.333333f, + 0.324324f, 0.230769f, 0.166667f, 0.0909091f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.324324f, 0.307692f, 0.25f, 0.181818f, 0.0833333f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.307692f, 0.333333f, + 0.363636f, 0.25f, 0.151515f, 0.0793651f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.166667f, 0.363636f, 0.333333f, 0.242424f, + 0.190476f, 0.133333f, 0.0689655f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.333333f, 0.30303f, 0.253968f, 0.2f, 0.137931f, + 0.0714286f, 0.f, 0.f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.30303f, 0.31746f, 0.333333f, 0.275862f, 0.214286f, + 0.125f, 0.0655738f, 0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.15873f, 0.333333f, 0.344828f, 0.357143f, + 0.25f, 0.196721f, 0.137931f, 0.0816327f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.172414f, 0.357143f, + 0.3125f, 0.245902f, 0.172414f, 0.102041f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.3125f, 0.327869f, 0.344828f, 0.204082f, 0.f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.163934f, 0.344828f, 0.408163f, 0.5f}, + {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.204082f, 0.5f}}; +static_assert(arraysize(kTestCenterFreqs) == arraysize(kTestFilterBank), + "Test filterbank badly initialized."); + +// Target output for gain solving test. Generated with matlab. +const size_t kTestStartFreq = 12; // Lowest integral frequency for ERBs. +const float kTestZeroVar = 1.f; +const float kTestNonZeroVarLambdaTop[] = { + 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; +static_assert(arraysize(kTestCenterFreqs) == + arraysize(kTestNonZeroVarLambdaTop), + "Power test data badly initialized."); +const float kMaxTestError = 0.005f; + +// Enhancer initialization parameters. +const int kSamples = 10000; +const int kSampleRate = 4000; +const int kNumChannels = 1; +const int kFragmentSize = kSampleRate / 100; +const size_t kNumNoiseBins = 129; +const size_t kNumBands = 1; + +// Number of frames to process in the bitexactness tests. +const size_t kNumFramesToProcess = 1000; + +int IntelligibilityEnhancerSampleRate(int sample_rate_hz) { + return (sample_rate_hz > AudioProcessing::kSampleRate16kHz + ? AudioProcessing::kSampleRate16kHz + : sample_rate_hz); +} + +// Process one frame of data and produce the output. +void ProcessOneFrame(int sample_rate_hz, + AudioBuffer* render_audio_buffer, + AudioBuffer* capture_audio_buffer, + NoiseSuppressionImpl* noise_suppressor, + IntelligibilityEnhancer* intelligibility_enhancer) { + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + render_audio_buffer->SplitIntoFrequencyBands(); + capture_audio_buffer->SplitIntoFrequencyBands(); + } + + intelligibility_enhancer->ProcessRenderAudio(render_audio_buffer); + + noise_suppressor->AnalyzeCaptureAudio(capture_audio_buffer); + noise_suppressor->ProcessCaptureAudio(capture_audio_buffer); + + intelligibility_enhancer->SetCaptureNoiseEstimate( + noise_suppressor->NoiseEstimate(), 0); + + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + render_audio_buffer->MergeFrequencyBands(); + } +} + +// Processes a specified amount of frames, verifies the results and reports +// any errors. +void RunBitexactnessTest(int sample_rate_hz, + size_t num_channels, + rtc::ArrayView output_reference) { + const StreamConfig render_config(sample_rate_hz, num_channels, false); + AudioBuffer render_buffer( + render_config.num_frames(), render_config.num_channels(), + render_config.num_frames(), render_config.num_channels(), + render_config.num_frames()); + test::InputAudioFile render_file( + test::GetApmRenderTestVectorFileName(sample_rate_hz)); + std::vector render_input(render_buffer.num_frames() * + render_buffer.num_channels()); + + const StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(render_buffer.num_frames() * + capture_buffer.num_channels()); + + rtc::CriticalSection crit_capture; + NoiseSuppressionImpl noise_suppressor(&crit_capture); + noise_suppressor.Initialize(capture_config.num_channels(), sample_rate_hz); + noise_suppressor.Enable(true); + + IntelligibilityEnhancer intelligibility_enhancer( + IntelligibilityEnhancerSampleRate(sample_rate_hz), + render_config.num_channels(), kNumBands, + NoiseSuppressionImpl::num_noise_bins()); + + for (size_t frame_no = 0u; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(render_buffer.num_frames(), + render_buffer.num_channels(), &render_file, + render_input); + ReadFloatSamplesFromStereoFile(capture_buffer.num_frames(), + capture_buffer.num_channels(), &capture_file, + capture_input); + + test::CopyVectorToAudioBuffer(render_config, render_input, &render_buffer); + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + ProcessOneFrame(sample_rate_hz, &render_buffer, &capture_buffer, + &noise_suppressor, &intelligibility_enhancer); + } + + // Extract and verify the test results. + std::vector render_output; + test::ExtractVectorFromAudioBuffer(render_config, &render_buffer, + &render_output); + + const float kElementErrorBound = 1.f / static_cast(1 << 15); + + // Compare the output with the reference. Only the first values of the output + // from last frame processed are compared in order not having to specify all + // preceeding frames as testvectors. As the algorithm being tested has a + // memory, testing only the last frame implicitly also tests the preceeding + // frames. + EXPECT_TRUE(test::VerifyDeinterleavedArray( + render_buffer.num_frames(), render_config.num_channels(), + output_reference, render_output, kElementErrorBound)); +} + +float float_rand() { + return std::rand() * 2.f / RAND_MAX - 1; +} + +} // namespace + +class IntelligibilityEnhancerTest : public ::testing::Test { + protected: + IntelligibilityEnhancerTest() + : clear_buffer_(kFragmentSize, + kNumChannels, + kFragmentSize, + kNumChannels, + kFragmentSize), + stream_config_(kSampleRate, kNumChannels), + clear_data_(kSamples), + noise_data_(kNumNoiseBins), + orig_data_(kSamples) { + std::srand(1); + enh_.reset(new IntelligibilityEnhancer(kSampleRate, kNumChannels, kNumBands, + kNumNoiseBins)); + } + + bool CheckUpdate() { + enh_.reset(new IntelligibilityEnhancer(kSampleRate, kNumChannels, kNumBands, + kNumNoiseBins)); + float* clear_cursor = clear_data_.data(); + for (int i = 0; i < kSamples; i += kFragmentSize) { + enh_->SetCaptureNoiseEstimate(noise_data_, 1); + clear_buffer_.CopyFrom(&clear_cursor, stream_config_); + enh_->ProcessRenderAudio(&clear_buffer_); + clear_buffer_.CopyTo(stream_config_, &clear_cursor); + clear_cursor += kFragmentSize; + } + for (int i = initial_delay_; i < kSamples; i++) { + if (std::fabs(clear_data_[i] - orig_data_[i - initial_delay_]) > + kMaxTestError) { + return true; + } + } + return false; + } + + std::unique_ptr enh_; + // Render clean speech buffer. + AudioBuffer clear_buffer_; + StreamConfig stream_config_; + std::vector clear_data_; + std::vector noise_data_; + std::vector orig_data_; + size_t initial_delay_; +}; + +// For each class of generated data, tests that render stream is updated when +// it should be. +TEST_F(IntelligibilityEnhancerTest, TestRenderUpdate) { + initial_delay_ = enh_->render_mangler_->initial_delay(); + std::fill(noise_data_.begin(), noise_data_.end(), 0.f); + std::fill(orig_data_.begin(), orig_data_.end(), 0.f); + std::fill(clear_data_.begin(), clear_data_.end(), 0.f); + EXPECT_FALSE(CheckUpdate()); + std::generate(clear_data_.begin(), clear_data_.end(), float_rand); + orig_data_ = clear_data_; + EXPECT_FALSE(CheckUpdate()); + std::generate(clear_data_.begin(), clear_data_.end(), float_rand); + orig_data_ = clear_data_; + std::generate(noise_data_.begin(), noise_data_.end(), float_rand); + FloatToFloatS16(noise_data_.data(), noise_data_.size(), noise_data_.data()); + EXPECT_TRUE(CheckUpdate()); +} + +// Tests ERB bank creation, comparing against matlab output. +TEST_F(IntelligibilityEnhancerTest, TestErbCreation) { + ASSERT_EQ(arraysize(kTestCenterFreqs), enh_->bank_size_); + for (size_t i = 0; i < enh_->bank_size_; ++i) { + EXPECT_NEAR(kTestCenterFreqs[i], enh_->center_freqs_[i], kMaxTestError); + ASSERT_EQ(arraysize(kTestFilterBank[0]), enh_->freqs_); + for (size_t j = 0; j < enh_->freqs_; ++j) { + EXPECT_NEAR(kTestFilterBank[i][j], enh_->render_filter_bank_[i][j], + kMaxTestError); + } + } +} + +// Tests analytic solution for optimal gains, comparing +// against matlab output. +TEST_F(IntelligibilityEnhancerTest, TestSolveForGains) { + ASSERT_EQ(kTestStartFreq, enh_->start_freq_); + std::vector sols(enh_->bank_size_); + float lambda = -0.001f; + for (size_t i = 0; i < enh_->bank_size_; i++) { + enh_->filtered_clear_pow_[i] = 0.f; + enh_->filtered_noise_pow_[i] = 0.f; + } + enh_->SolveForGainsGivenLambda(lambda, enh_->start_freq_, sols.data()); + for (size_t i = 0; i < enh_->bank_size_; i++) { + EXPECT_NEAR(kTestZeroVar, sols[i], kMaxTestError); + } + for (size_t i = 0; i < enh_->bank_size_; i++) { + enh_->filtered_clear_pow_[i] = static_cast(i + 1); + enh_->filtered_noise_pow_[i] = static_cast(enh_->bank_size_ - i); + } + enh_->SolveForGainsGivenLambda(lambda, enh_->start_freq_, sols.data()); + for (size_t i = 0; i < enh_->bank_size_; i++) { + EXPECT_NEAR(kTestNonZeroVarLambdaTop[i], sols[i], kMaxTestError); + } + lambda = -1.f; + enh_->SolveForGainsGivenLambda(lambda, enh_->start_freq_, sols.data()); + for (size_t i = 0; i < enh_->bank_size_; i++) { + EXPECT_NEAR(kTestNonZeroVarLambdaTop[i], sols[i], kMaxTestError); + } +} + +TEST_F(IntelligibilityEnhancerTest, TestNoiseGainHasExpectedResult) { + const float kGain = 2.f; + const float kTolerance = 0.007f; + std::vector noise(kNumNoiseBins); + std::vector noise_psd(kNumNoiseBins); + std::generate(noise.begin(), noise.end(), float_rand); + for (size_t i = 0; i < kNumNoiseBins; ++i) { + noise_psd[i] = kGain * kGain * noise[i] * noise[i]; + } + float* clear_cursor = clear_data_.data(); + for (size_t i = 0; i < kNumFramesToProcess; ++i) { + enh_->SetCaptureNoiseEstimate(noise, kGain); + clear_buffer_.CopyFrom(&clear_cursor, stream_config_); + enh_->ProcessRenderAudio(&clear_buffer_); + } + const std::vector& estimated_psd = + enh_->noise_power_estimator_.power(); + for (size_t i = 0; i < kNumNoiseBins; ++i) { + EXPECT_LT(std::abs(estimated_psd[i] - noise_psd[i]) / noise_psd[i], + kTolerance); + } +} + +TEST_F(IntelligibilityEnhancerTest, TestAllBandsHaveSameDelay) { + const int kTestSampleRate = AudioProcessing::kSampleRate32kHz; + const int kTestSplitRate = AudioProcessing::kSampleRate16kHz; + const size_t kTestNumBands = + rtc::CheckedDivExact(kTestSampleRate, kTestSplitRate); + const size_t kTestFragmentSize = rtc::CheckedDivExact(kTestSampleRate, 100); + const size_t kTestSplitFragmentSize = + rtc::CheckedDivExact(kTestSplitRate, 100); + enh_.reset(new IntelligibilityEnhancer(kTestSplitRate, kNumChannels, + kTestNumBands, kNumNoiseBins)); + size_t initial_delay = enh_->render_mangler_->initial_delay(); + std::vector rand_gen_buf(kTestFragmentSize); + AudioBuffer original_buffer(kTestFragmentSize, kNumChannels, + kTestFragmentSize, kNumChannels, + kTestFragmentSize); + AudioBuffer audio_buffer(kTestFragmentSize, kNumChannels, kTestFragmentSize, + kNumChannels, kTestFragmentSize); + for (size_t i = 0u; i < kTestNumBands; ++i) { + std::generate(rand_gen_buf.begin(), rand_gen_buf.end(), float_rand); + original_buffer.split_data_f()->SetDataForTesting(rand_gen_buf.data(), + rand_gen_buf.size()); + audio_buffer.split_data_f()->SetDataForTesting(rand_gen_buf.data(), + rand_gen_buf.size()); + } + enh_->ProcessRenderAudio(&audio_buffer); + for (size_t i = 0u; i < kTestNumBands; ++i) { + const float* original_ptr = original_buffer.split_bands_const_f(0)[i]; + const float* audio_ptr = audio_buffer.split_bands_const_f(0)[i]; + for (size_t j = initial_delay; j < kTestSplitFragmentSize; ++j) { + EXPECT_LT(std::fabs(original_ptr[j - initial_delay] - audio_ptr[j]), + kMaxTestError); + } + } +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Mono8kHz) { + const float kOutputReference[] = {-0.001892f, -0.003296f, -0.001953f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate8kHz, 1, kOutputReference); +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Mono16kHz) { + const float kOutputReference[] = {-0.000977f, -0.003296f, -0.002441f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate16kHz, 1, kOutputReference); +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Mono32kHz) { + const float kOutputReference[] = {0.003021f, -0.011780f, -0.008209f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate32kHz, 1, kOutputReference); +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Mono48kHz) { + const float kOutputReference[] = {-0.027696f, -0.026253f, -0.018001f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate48kHz, 1, kOutputReference); +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Stereo8kHz) { + const float kOutputReference[] = {0.021454f, 0.035919f, 0.026428f, + -0.000641f, 0.000366f, 0.000641f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate8kHz, 2, kOutputReference); +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Stereo16kHz) { + const float kOutputReference[] = {0.021362f, 0.035736f, 0.023895f, + -0.001404f, -0.001465f, 0.000549f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate16kHz, 2, kOutputReference); +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Stereo32kHz) { + const float kOutputReference[] = {0.030641f, 0.027406f, 0.028321f, + -0.001343f, -0.004578f, 0.000977f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate32kHz, 2, kOutputReference); +} + +TEST(IntelligibilityEnhancerBitExactnessTest, DISABLED_Stereo48kHz) { + const float kOutputReference[] = {-0.009276f, -0.001601f, -0.008255f, + -0.012975f, -0.015940f, -0.017820f}; + + RunBitexactnessTest(AudioProcessing::kSampleRate48kHz, 2, kOutputReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.cc new file mode 100644 index 0000000000..b6917f4407 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.cc @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/intelligibility/intelligibility_utils.h" + +#include +#include +#include +#include +#include + +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +namespace intelligibility { + +namespace { + +const float kMinFactor = 0.01f; +const float kMaxFactor = 100.f; + +// Return |current| changed towards |target|, with the relative change being at +// most |limit|. +float UpdateFactor(float target, float current, float limit) { + const float gain = target / (current + std::numeric_limits::epsilon()); + const float clamped_gain = rtc::SafeClamp(gain, 1 - limit, 1 + limit); + return rtc::SafeClamp(current * clamped_gain, kMinFactor, kMaxFactor); +} + +} // namespace + +template +PowerEstimator::PowerEstimator(size_t num_freqs, float decay) + : power_(num_freqs, 0.f), decay_(decay) {} + +template +void PowerEstimator::Step(const T* data) { + for (size_t i = 0; i < power_.size(); ++i) { + power_[i] = decay_ * power_[i] + + (1.f - decay_) * std::abs(data[i]) * std::abs(data[i]); + } +} + +template class PowerEstimator; +template class PowerEstimator>; + +GainApplier::GainApplier(size_t freqs, float relative_change_limit) + : num_freqs_(freqs), + relative_change_limit_(relative_change_limit), + target_(freqs, 1.f), + current_(freqs, 1.f) {} + +GainApplier::~GainApplier() {} + +void GainApplier::Apply(const std::complex* in_block, + std::complex* out_block) { + for (size_t i = 0; i < num_freqs_; ++i) { + current_[i] = UpdateFactor(target_[i], current_[i], relative_change_limit_); + out_block[i] = sqrtf(fabsf(current_[i])) * in_block[i]; + } +} + +DelayBuffer::DelayBuffer(size_t delay, size_t num_channels) + : buffer_(num_channels, std::vector(delay, 0.f)), read_index_(0u) {} + +DelayBuffer::~DelayBuffer() {} + +void DelayBuffer::Delay(float* const* data, size_t length) { + size_t sample_index = read_index_; + for (size_t i = 0u; i < buffer_.size(); ++i) { + sample_index = read_index_; + for (size_t j = 0u; j < length; ++j) { + float swap = data[i][j]; + data[i][j] = buffer_[i][sample_index]; + buffer_[i][sample_index] = swap; + if (++sample_index == buffer_.size()) { + sample_index = 0u; + } + } + } + read_index_ = sample_index; +} + +} // namespace intelligibility + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h new file mode 100644 index 0000000000..4dc17d50b5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_UTILS_H_ +#define MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_UTILS_H_ + +#include +#include + +namespace webrtc { + +namespace intelligibility { + +// Internal helper for computing the power of a stream of arrays. +// The result is an array of power per position: the i-th power is the power of +// the stream of data on the i-th positions in the input arrays. +template +class PowerEstimator { + public: + // Construct an instance for the given input array length (|freqs|), with the + // appropriate parameters. |decay| is the forgetting factor. + PowerEstimator(size_t freqs, float decay); + + // Add a new data point to the series. + void Step(const T* data); + + // The current power array. + const std::vector& power() { return power_; }; + + private: + // The current power array. + std::vector power_; + + const float decay_; +}; + +// Helper class for smoothing gain changes. On each application step, the +// currently used gains are changed towards a set of settable target gains, +// constrained by a limit on the relative changes. +class GainApplier { + public: + GainApplier(size_t freqs, float relative_change_limit); + + ~GainApplier(); + + // Copy |in_block| to |out_block|, multiplied by the current set of gains, + // and step the current set of gains towards the target set. + void Apply(const std::complex* in_block, + std::complex* out_block); + + // Return the current target gain set. Modify this array to set the targets. + float* target() { return target_.data(); } + + private: + const size_t num_freqs_; + const float relative_change_limit_; + std::vector target_; + std::vector current_; +}; + +// Helper class to delay a signal by an integer number of samples. +class DelayBuffer { + public: + DelayBuffer(size_t delay, size_t num_channels); + + ~DelayBuffer(); + + void Delay(float* const* data, size_t length); + + private: + std::vector> buffer_; + size_t read_index_; +}; + +} // namespace intelligibility + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_UTILS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils_unittest.cc new file mode 100644 index 0000000000..fea394c338 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/intelligibility_utils_unittest.cc @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "modules/audio_processing/intelligibility/intelligibility_utils.h" +#include "rtc_base/arraysize.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace intelligibility { + +std::vector>> GenerateTestData(size_t freqs, + size_t samples) { + std::vector>> data(samples); + for (size_t i = 0; i < samples; ++i) { + for (size_t j = 0; j < freqs; ++j) { + const float val = 0.99f / ((i + 1) * (j + 1)); + data[i].push_back(std::complex(val, val)); + } + } + return data; +} + +// Tests PowerEstimator, for all power step types. +TEST(IntelligibilityUtilsTest, TestPowerEstimator) { + const size_t kFreqs = 10; + const size_t kSamples = 100; + const float kDecay = 0.5f; + const std::vector>> test_data( + GenerateTestData(kFreqs, kSamples)); + PowerEstimator> power_estimator(kFreqs, kDecay); + EXPECT_EQ(0, power_estimator.power()[0]); + + // Makes sure Step is doing something. + power_estimator.Step(test_data[0].data()); + for (size_t i = 1; i < kSamples; ++i) { + power_estimator.Step(test_data[i].data()); + for (size_t j = 0; j < kFreqs; ++j) { + EXPECT_GE(power_estimator.power()[j], 0.f); + EXPECT_LE(power_estimator.power()[j], 1.f); + } + } +} + +// Tests gain applier. +TEST(IntelligibilityUtilsTest, TestGainApplier) { + const size_t kFreqs = 10; + const size_t kSamples = 100; + const float kChangeLimit = 0.1f; + GainApplier gain_applier(kFreqs, kChangeLimit); + const std::vector>> in_data( + GenerateTestData(kFreqs, kSamples)); + std::vector>> out_data( + GenerateTestData(kFreqs, kSamples)); + for (size_t i = 0; i < kSamples; ++i) { + gain_applier.Apply(in_data[i].data(), out_data[i].data()); + for (size_t j = 0; j < kFreqs; ++j) { + EXPECT_GT(out_data[i][j].real(), 0.f); + EXPECT_LT(out_data[i][j].real(), 1.f); + EXPECT_GT(out_data[i][j].imag(), 0.f); + EXPECT_LT(out_data[i][j].imag(), 1.f); + } + } +} + +} // namespace intelligibility + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/test/intelligibility_proc.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/test/intelligibility_proc.cc new file mode 100644 index 0000000000..b90449caa3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/intelligibility/test/intelligibility_proc.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "common_audio/channel_buffer.h" +#include "common_audio/include/audio_util.h" +#include "common_audio/wav_file.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/intelligibility/intelligibility_enhancer.h" +#include "modules/audio_processing/noise_suppression_impl.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/flags.h" + +using std::complex; + +namespace webrtc { +namespace { + +DEFINE_string(clear_file, "speech.wav", "Input file with clear speech."); +DEFINE_string(noise_file, "noise.wav", "Input file with noise data."); +DEFINE_string(out_file, "proc_enhanced.wav", "Enhanced output file."); +DEFINE_bool(help, false, "Print this message."); + +int int_main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true)) { + return 1; + } + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + if (argc != 1) { + printf("\n\nInput files must be little-endian 16-bit signed raw PCM.\n"); + return 0; + } + + WavReader in_file(FLAG_clear_file); + WavReader noise_file(FLAG_noise_file); + WavWriter out_file(FLAG_out_file, in_file.sample_rate(), + in_file.num_channels()); + rtc::CriticalSection crit; + NoiseSuppressionImpl ns(&crit); + IntelligibilityEnhancer enh(in_file.sample_rate(), in_file.num_channels(), 1u, + NoiseSuppressionImpl::num_noise_bins()); + ns.Initialize(noise_file.num_channels(), noise_file.sample_rate()); + ns.Enable(true); + const size_t in_samples = noise_file.sample_rate() / 100; + const size_t noise_samples = noise_file.sample_rate() / 100; + std::vector in(in_samples * in_file.num_channels()); + std::vector noise(noise_samples * noise_file.num_channels()); + ChannelBuffer in_buf(in_samples, in_file.num_channels()); + ChannelBuffer noise_buf(noise_samples, noise_file.num_channels()); + AudioBuffer capture_audio(noise_samples, noise_file.num_channels(), + noise_samples, noise_file.num_channels(), + noise_samples); + AudioBuffer render_audio(in_samples, in_file.num_channels(), in_samples, + in_file.num_channels(), in_samples); + StreamConfig noise_config(noise_file.sample_rate(), + noise_file.num_channels()); + StreamConfig in_config(in_file.sample_rate(), in_file.num_channels()); + while (in_file.ReadSamples(in.size(), in.data()) == in.size() && + noise_file.ReadSamples(noise.size(), noise.data()) == noise.size()) { + FloatS16ToFloat(noise.data(), noise.size(), noise.data()); + FloatS16ToFloat(in.data(), in.size(), in.data()); + Deinterleave(in.data(), in_buf.num_frames(), in_buf.num_channels(), + in_buf.channels()); + Deinterleave(noise.data(), noise_buf.num_frames(), noise_buf.num_channels(), + noise_buf.channels()); + capture_audio.CopyFrom(noise_buf.channels(), noise_config); + render_audio.CopyFrom(in_buf.channels(), in_config); + ns.AnalyzeCaptureAudio(&capture_audio); + ns.ProcessCaptureAudio(&capture_audio); + enh.SetCaptureNoiseEstimate(ns.NoiseEstimate(), 1); + enh.ProcessRenderAudio(&render_audio); + render_audio.CopyTo(in_config, in_buf.channels()); + Interleave(in_buf.channels(), in_buf.num_frames(), in_buf.num_channels(), + in.data()); + FloatToFloatS16(in.data(), in.size(), in.data()); + out_file.WriteSamples(in.data(), in.size()); + } + + return 0; +} + +} // namespace +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::int_main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.cc new file mode 100644 index 0000000000..5a4ddc891e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.cc @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/biquad_filter.h" + +namespace webrtc { + +// This method applies a biquad filter to an input signal x to produce an +// output signal y. The biquad coefficients are specified at the construction +// of the object. +void BiQuadFilter::Process(rtc::ArrayView x, + rtc::ArrayView y) { + for (size_t k = 0; k < x.size(); ++k) { + // Use temporary variable for x[k] to allow in-place function call + // (that x and y refer to the same array). + const float tmp = x[k]; + y[k] = coefficients_.b[0] * tmp + coefficients_.b[1] * biquad_state_.b[0] + + coefficients_.b[2] * biquad_state_.b[1] - + coefficients_.a[0] * biquad_state_.a[0] - + coefficients_.a[1] * biquad_state_.a[1]; + biquad_state_.b[1] = biquad_state_.b[0]; + biquad_state_.b[0] = tmp; + biquad_state_.a[1] = biquad_state_.a[0]; + biquad_state_.a[0] = y[k]; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.h new file mode 100644 index 0000000000..dad104d43f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/biquad_filter.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_BIQUAD_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_BIQUAD_FILTER_H_ + +#include + +#include "api/array_view.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class BiQuadFilter { + public: + struct BiQuadCoefficients { + float b[3]; + float a[2]; + }; + + BiQuadFilter() = default; + + void Initialize(const BiQuadCoefficients& coefficients) { + coefficients_ = coefficients; + } + + // Produces a filtered output y of the input x. Both x and y need to + // have the same length. + void Process(rtc::ArrayView x, rtc::ArrayView y); + + private: + struct BiQuadState { + BiQuadState() { + std::fill(b, b + arraysize(b), 0.f); + std::fill(a, a + arraysize(a), 0.f); + } + + float b[2]; + float a[2]; + }; + + BiQuadState biquad_state_; + BiQuadCoefficients coefficients_; + + RTC_DISALLOW_COPY_AND_ASSIGN(BiQuadFilter); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_BIQUAD_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.cc new file mode 100644 index 0000000000..a1702f432c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/down_sampler.h" + +#include +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/level_controller/biquad_filter.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Bandlimiter coefficients computed based on that only +// the first 40 bins of the spectrum for the downsampled +// signal are used. +// [B,A] = butter(2,(41/64*4000)/8000) +const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_16kHz = { + {0.1455f, 0.2911f, 0.1455f}, + {-0.6698f, 0.2520f}}; + +// [B,A] = butter(2,(41/64*4000)/16000) +const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_32kHz = { + {0.0462f, 0.0924f, 0.0462f}, + {-1.3066f, 0.4915f}}; + +// [B,A] = butter(2,(41/64*4000)/24000) +const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_48kHz = { + {0.0226f, 0.0452f, 0.0226f}, + {-1.5320f, 0.6224f}}; + +} // namespace + +DownSampler::DownSampler(ApmDataDumper* data_dumper) + : data_dumper_(data_dumper) { + Initialize(48000); +} +void DownSampler::Initialize(int sample_rate_hz) { + RTC_DCHECK(sample_rate_hz == AudioProcessing::kSampleRate8kHz || + sample_rate_hz == AudioProcessing::kSampleRate16kHz || + sample_rate_hz == AudioProcessing::kSampleRate32kHz || + sample_rate_hz == AudioProcessing::kSampleRate48kHz); + + sample_rate_hz_ = sample_rate_hz; + down_sampling_factor_ = rtc::CheckedDivExact(sample_rate_hz_, 8000); + + /// Note that the down sampling filter is not used if the sample rate is 8 + /// kHz. + if (sample_rate_hz_ == AudioProcessing::kSampleRate16kHz) { + low_pass_filter_.Initialize(kLowPassFilterCoefficients_16kHz); + } else if (sample_rate_hz_ == AudioProcessing::kSampleRate32kHz) { + low_pass_filter_.Initialize(kLowPassFilterCoefficients_32kHz); + } else if (sample_rate_hz_ == AudioProcessing::kSampleRate48kHz) { + low_pass_filter_.Initialize(kLowPassFilterCoefficients_48kHz); + } +} + +void DownSampler::DownSample(rtc::ArrayView in, + rtc::ArrayView out) { + data_dumper_->DumpWav("lc_down_sampler_input", in, sample_rate_hz_, 1); + RTC_DCHECK_EQ(sample_rate_hz_ * AudioProcessing::kChunkSizeMs / 1000, + in.size()); + RTC_DCHECK_EQ( + AudioProcessing::kSampleRate8kHz * AudioProcessing::kChunkSizeMs / 1000, + out.size()); + const size_t kMaxNumFrames = + AudioProcessing::kSampleRate48kHz * AudioProcessing::kChunkSizeMs / 1000; + float x[kMaxNumFrames]; + + // Band-limit the signal to 4 kHz. + if (sample_rate_hz_ != AudioProcessing::kSampleRate8kHz) { + low_pass_filter_.Process(in, rtc::ArrayView(x, in.size())); + + // Downsample the signal. + size_t k = 0; + for (size_t j = 0; j < out.size(); ++j) { + RTC_DCHECK_GT(kMaxNumFrames, k); + out[j] = x[k]; + k += down_sampling_factor_; + } + } else { + std::copy(in.data(), in.data() + in.size(), out.data()); + } + + data_dumper_->DumpWav("lc_down_sampler_output", out, + AudioProcessing::kSampleRate8kHz, 1); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.h new file mode 100644 index 0000000000..d6502425a1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/down_sampler.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_DOWN_SAMPLER_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_DOWN_SAMPLER_H_ + +#include "api/array_view.h" +#include "modules/audio_processing/level_controller/biquad_filter.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; + +class DownSampler { + public: + explicit DownSampler(ApmDataDumper* data_dumper); + void Initialize(int sample_rate_hz); + + void DownSample(rtc::ArrayView in, rtc::ArrayView out); + + private: + ApmDataDumper* data_dumper_; + int sample_rate_hz_; + int down_sampling_factor_; + BiQuadFilter low_pass_filter_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DownSampler); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_DOWN_SAMPLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.cc new file mode 100644 index 0000000000..018f809e01 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/gain_applier.h" + +#include + +#include "api/array_view.h" +#include "rtc_base/checks.h" + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { +namespace { + +const float kMaxSampleValue = 32767.f; +const float kMinSampleValue = -32767.f; + +int CountSaturations(rtc::ArrayView in) { + return std::count_if(in.begin(), in.end(), [](const float& v) { + return v >= kMaxSampleValue || v <= kMinSampleValue; + }); +} + +int CountSaturations(const AudioBuffer& audio) { + int num_saturations = 0; + for (size_t k = 0; k < audio.num_channels(); ++k) { + num_saturations += CountSaturations(rtc::ArrayView( + audio.channels_const_f()[k], audio.num_frames())); + } + return num_saturations; +} + +void LimitToAllowedRange(rtc::ArrayView x) { + for (auto& v : x) { + v = std::max(kMinSampleValue, v); + v = std::min(kMaxSampleValue, v); + } +} + +void LimitToAllowedRange(AudioBuffer* audio) { + for (size_t k = 0; k < audio->num_channels(); ++k) { + LimitToAllowedRange( + rtc::ArrayView(audio->channels_f()[k], audio->num_frames())); + } +} + +float ApplyIncreasingGain(float new_gain, + float old_gain, + float step_size, + rtc::ArrayView x) { + RTC_DCHECK_LT(0.f, step_size); + float gain = old_gain; + for (auto& v : x) { + gain = std::min(new_gain, gain + step_size); + v *= gain; + } + return gain; +} + +float ApplyDecreasingGain(float new_gain, + float old_gain, + float step_size, + rtc::ArrayView x) { + RTC_DCHECK_GT(0.f, step_size); + float gain = old_gain; + for (auto& v : x) { + gain = std::max(new_gain, gain + step_size); + v *= gain; + } + return gain; +} + +float ApplyConstantGain(float gain, rtc::ArrayView x) { + for (auto& v : x) { + v *= gain; + } + + return gain; +} + +float ApplyGain(float new_gain, + float old_gain, + float increase_step_size, + float decrease_step_size, + rtc::ArrayView x) { + RTC_DCHECK_LT(0.f, increase_step_size); + RTC_DCHECK_GT(0.f, decrease_step_size); + if (new_gain == old_gain) { + return ApplyConstantGain(new_gain, x); + } else if (new_gain > old_gain) { + return ApplyIncreasingGain(new_gain, old_gain, increase_step_size, x); + } else { + return ApplyDecreasingGain(new_gain, old_gain, decrease_step_size, x); + } +} + +} // namespace + +GainApplier::GainApplier(ApmDataDumper* data_dumper) + : data_dumper_(data_dumper) {} + +void GainApplier::Initialize(int sample_rate_hz) { + RTC_DCHECK(sample_rate_hz == AudioProcessing::kSampleRate8kHz || + sample_rate_hz == AudioProcessing::kSampleRate16kHz || + sample_rate_hz == AudioProcessing::kSampleRate32kHz || + sample_rate_hz == AudioProcessing::kSampleRate48kHz); + const float kGainIncreaseStepSize48kHz = 0.0001f; + const float kGainDecreaseStepSize48kHz = -0.01f; + const float kGainSaturatedDecreaseStepSize48kHz = -0.05f; + + last_frame_was_saturated_ = false; + old_gain_ = 1.f; + gain_increase_step_size_ = + kGainIncreaseStepSize48kHz * + (static_cast(AudioProcessing::kSampleRate48kHz) / sample_rate_hz); + gain_normal_decrease_step_size_ = + kGainDecreaseStepSize48kHz * + (static_cast(AudioProcessing::kSampleRate48kHz) / sample_rate_hz); + gain_saturated_decrease_step_size_ = + kGainSaturatedDecreaseStepSize48kHz * + (static_cast(AudioProcessing::kSampleRate48kHz) / sample_rate_hz); +} + +int GainApplier::Process(float new_gain, AudioBuffer* audio) { + RTC_CHECK_NE(0.f, gain_increase_step_size_); + RTC_CHECK_NE(0.f, gain_normal_decrease_step_size_); + RTC_CHECK_NE(0.f, gain_saturated_decrease_step_size_); + int num_saturations = 0; + if (new_gain != 1.f) { + float last_applied_gain = 1.f; + float gain_decrease_step_size = last_frame_was_saturated_ + ? gain_saturated_decrease_step_size_ + : gain_normal_decrease_step_size_; + for (size_t k = 0; k < audio->num_channels(); ++k) { + last_applied_gain = ApplyGain( + new_gain, old_gain_, gain_increase_step_size_, + gain_decrease_step_size, + rtc::ArrayView(audio->channels_f()[k], audio->num_frames())); + } + + num_saturations = CountSaturations(*audio); + LimitToAllowedRange(audio); + old_gain_ = last_applied_gain; + } + + data_dumper_->DumpRaw("lc_last_applied_gain", 1, &old_gain_); + + return num_saturations; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.h new file mode 100644 index 0000000000..5669f45bf7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_applier.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_GAIN_APPLIER_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_GAIN_APPLIER_H_ + +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; +class AudioBuffer; + +class GainApplier { + public: + explicit GainApplier(ApmDataDumper* data_dumper); + void Initialize(int sample_rate_hz); + + // Applies the specified gain to the audio frame and returns the resulting + // number of saturated sample values. + int Process(float new_gain, AudioBuffer* audio); + + private: + ApmDataDumper* const data_dumper_; + float old_gain_ = 1.f; + float gain_increase_step_size_ = 0.f; + float gain_normal_decrease_step_size_ = 0.f; + float gain_saturated_decrease_step_size_ = 0.f; + bool last_frame_was_saturated_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(GainApplier); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_GAIN_APPLIER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.cc new file mode 100644 index 0000000000..3ab75b1ce6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/gain_selector.h" + +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/level_controller/level_controller_constants.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +GainSelector::GainSelector() { + Initialize(AudioProcessing::kSampleRate48kHz); +} + +void GainSelector::Initialize(int sample_rate_hz) { + gain_ = 1.f; + frame_length_ = rtc::CheckedDivExact(sample_rate_hz, 100); + highly_nonstationary_signal_hold_counter_ = 0; +} + +// Chooses the gain to apply by the level controller such that +// 1) The level of the stationary noise does not exceed +// a predefined threshold. +// 2) The gain does not exceed the gain that has been found +// to saturate the signal. +// 3) The peak level achieves the target peak level. +// 4) The gain is not below 1. +// 4) The gain is 1 if the signal has been classified as stationary +// for a long time. +// 5) The gain is not above the maximum gain. +float GainSelector::GetNewGain(float peak_level, + float noise_energy, + float saturating_gain, + bool gain_jumpstart, + SignalClassifier::SignalType signal_type) { + RTC_DCHECK_LT(0.f, peak_level); + + if (signal_type == SignalClassifier::SignalType::kHighlyNonStationary || + gain_jumpstart) { + highly_nonstationary_signal_hold_counter_ = 100; + } else { + highly_nonstationary_signal_hold_counter_ = + std::max(0, highly_nonstationary_signal_hold_counter_ - 1); + } + + float desired_gain; + if (highly_nonstationary_signal_hold_counter_ > 0) { + // Compute a desired gain that ensures that the peak level is amplified to + // the target level. + desired_gain = kTargetLcPeakLevel / peak_level; + + // Limit the desired gain so that it does not amplify the noise too much. + float max_noise_energy = kMaxLcNoisePower * frame_length_; + if (noise_energy * desired_gain * desired_gain > max_noise_energy) { + RTC_DCHECK_LE(0.f, noise_energy); + desired_gain = sqrtf(max_noise_energy / noise_energy); + } + } else { + // If the signal has been stationary for a long while, apply a gain of 1 to + // avoid amplifying pure noise. + desired_gain = 1.0f; + } + + // Smootly update the gain towards the desired gain. + gain_ += 0.2f * (desired_gain - gain_); + + // Limit the gain to not exceed the maximum and the saturating gains, and to + // ensure that the lowest possible gain is 1. + gain_ = std::min(gain_, saturating_gain); + gain_ = std::min(gain_, kMaxLcGain); + gain_ = std::max(gain_, 1.f); + + return gain_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.h new file mode 100644 index 0000000000..7966c438d7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/gain_selector.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_GAIN_SELECTOR_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_GAIN_SELECTOR_H_ + +#include "rtc_base/constructormagic.h" + +#include "modules/audio_processing/level_controller/signal_classifier.h" + +namespace webrtc { + +class GainSelector { + public: + GainSelector(); + void Initialize(int sample_rate_hz); + float GetNewGain(float peak_level, + float noise_energy, + float saturating_gain, + bool gain_jumpstart, + SignalClassifier::SignalType signal_type); + + private: + float gain_; + size_t frame_length_; + int highly_nonstationary_signal_hold_counter_; + + RTC_DISALLOW_COPY_AND_ASSIGN(GainSelector); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_GAIN_SELECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.cc new file mode 100644 index 0000000000..521f82e842 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.cc @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/level_controller.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/level_controller/gain_applier.h" +#include "modules/audio_processing/level_controller/gain_selector.h" +#include "modules/audio_processing/level_controller/noise_level_estimator.h" +#include "modules/audio_processing/level_controller/peak_level_estimator.h" +#include "modules/audio_processing/level_controller/saturating_gain_estimator.h" +#include "modules/audio_processing/level_controller/signal_classifier.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { + +void UpdateAndRemoveDcLevel(float forgetting_factor, + float* dc_level, + rtc::ArrayView x) { + RTC_DCHECK(!x.empty()); + float mean = + std::accumulate(x.begin(), x.end(), 0.0f) / static_cast(x.size()); + *dc_level += forgetting_factor * (mean - *dc_level); + + for (float& v : x) { + v -= *dc_level; + } +} + +float FrameEnergy(const AudioBuffer& audio) { + float energy = 0.f; + for (size_t k = 0; k < audio.num_channels(); ++k) { + float channel_energy = + std::accumulate(audio.channels_const_f()[k], + audio.channels_const_f()[k] + audio.num_frames(), 0.f, + [](float a, float b) -> float { return a + b * b; }); + energy = std::max(channel_energy, energy); + } + return energy; +} + +float PeakLevel(const AudioBuffer& audio) { + float peak_level = 0.f; + for (size_t k = 0; k < audio.num_channels(); ++k) { + auto* channel_peak_level = std::max_element( + audio.channels_const_f()[k], + audio.channels_const_f()[k] + audio.num_frames(), + [](float a, float b) { return std::abs(a) < std::abs(b); }); + peak_level = std::max(*channel_peak_level, peak_level); + } + return peak_level; +} + +const int kMetricsFrameInterval = 1000; + +} // namespace + +int LevelController::instance_count_ = 0; + +void LevelController::Metrics::Initialize(int sample_rate_hz) { + RTC_DCHECK(sample_rate_hz == AudioProcessing::kSampleRate8kHz || + sample_rate_hz == AudioProcessing::kSampleRate16kHz || + sample_rate_hz == AudioProcessing::kSampleRate32kHz || + sample_rate_hz == AudioProcessing::kSampleRate48kHz); + + Reset(); + frame_length_ = rtc::CheckedDivExact(sample_rate_hz, 100); +} + +void LevelController::Metrics::Reset() { + metrics_frame_counter_ = 0; + gain_sum_ = 0.f; + peak_level_sum_ = 0.f; + noise_energy_sum_ = 0.f; + max_gain_ = 0.f; + max_peak_level_ = 0.f; + max_noise_energy_ = 0.f; +} + +void LevelController::Metrics::Update(float long_term_peak_level, + float noise_energy, + float gain, + float frame_peak_level) { + const float kdBFSOffset = 90.3090f; + gain_sum_ += gain; + peak_level_sum_ += long_term_peak_level; + noise_energy_sum_ += noise_energy; + max_gain_ = std::max(max_gain_, gain); + max_peak_level_ = std::max(max_peak_level_, long_term_peak_level); + max_noise_energy_ = std::max(max_noise_energy_, noise_energy); + + ++metrics_frame_counter_; + if (metrics_frame_counter_ == kMetricsFrameInterval) { + RTC_DCHECK_LT(0, frame_length_); + RTC_DCHECK_LT(0, kMetricsFrameInterval); + + const int max_noise_power_dbfs = static_cast( + 10 * log10(max_noise_energy_ / frame_length_ + 1e-10f) - kdBFSOffset); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.LevelControl.MaxNoisePower", + max_noise_power_dbfs, -90, 0, 50); + + const int average_noise_power_dbfs = static_cast( + 10 * log10(noise_energy_sum_ / (frame_length_ * kMetricsFrameInterval) + + 1e-10f) - + kdBFSOffset); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.LevelControl.AverageNoisePower", + average_noise_power_dbfs, -90, 0, 50); + + const int max_peak_level_dbfs = static_cast( + 10 * log10(max_peak_level_ * max_peak_level_ + 1e-10f) - kdBFSOffset); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.LevelControl.MaxPeakLevel", + max_peak_level_dbfs, -90, 0, 50); + + const int average_peak_level_dbfs = static_cast( + 10 * log10(peak_level_sum_ * peak_level_sum_ / + (kMetricsFrameInterval * kMetricsFrameInterval) + + 1e-10f) - + kdBFSOffset); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.LevelControl.AveragePeakLevel", + average_peak_level_dbfs, -90, 0, 50); + + RTC_DCHECK_LE(1.f, max_gain_); + RTC_DCHECK_LE(1.f, gain_sum_ / kMetricsFrameInterval); + + const int max_gain_db = static_cast(10 * log10(max_gain_ * max_gain_)); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.LevelControl.MaxGain", max_gain_db, 0, + 33, 30); + + const int average_gain_db = static_cast( + 10 * log10(gain_sum_ * gain_sum_ / + (kMetricsFrameInterval * kMetricsFrameInterval))); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.LevelControl.AverageGain", + average_gain_db, 0, 33, 30); + + const int long_term_peak_level_dbfs = static_cast( + 10 * log10(long_term_peak_level * long_term_peak_level + 1e-10f) - + kdBFSOffset); + + const int frame_peak_level_dbfs = static_cast( + 10 * log10(frame_peak_level * frame_peak_level + 1e-10f) - kdBFSOffset); + + RTC_LOG(LS_INFO) << "Level Controller metrics: {" + << "Max noise power: " << max_noise_power_dbfs << " dBFS, " + << "Average noise power: " << average_noise_power_dbfs + << " dBFS, " + << "Max long term peak level: " << max_peak_level_dbfs + << " dBFS, " + << "Average long term peak level: " + << average_peak_level_dbfs << " dBFS, " + << "Max gain: " << max_gain_db << " dB, " + << "Average gain: " << average_gain_db << " dB, " + << "Long term peak level: " << long_term_peak_level_dbfs + << " dBFS, " + << "Last frame peak level: " << frame_peak_level_dbfs + << " dBFS" + << "}"; + + Reset(); + } +} + +LevelController::LevelController() + : data_dumper_(new ApmDataDumper(instance_count_)), + gain_applier_(data_dumper_.get()), + signal_classifier_(data_dumper_.get()), + peak_level_estimator_(kTargetLcPeakLeveldBFS) { + Initialize(AudioProcessing::kSampleRate48kHz); + ++instance_count_; +} + +LevelController::~LevelController() {} + +void LevelController::Initialize(int sample_rate_hz) { + RTC_DCHECK(sample_rate_hz == AudioProcessing::kSampleRate8kHz || + sample_rate_hz == AudioProcessing::kSampleRate16kHz || + sample_rate_hz == AudioProcessing::kSampleRate32kHz || + sample_rate_hz == AudioProcessing::kSampleRate48kHz); + data_dumper_->InitiateNewSetOfRecordings(); + gain_selector_.Initialize(sample_rate_hz); + gain_applier_.Initialize(sample_rate_hz); + signal_classifier_.Initialize(sample_rate_hz); + noise_level_estimator_.Initialize(sample_rate_hz); + peak_level_estimator_.Initialize(config_.initial_peak_level_dbfs); + saturating_gain_estimator_.Initialize(); + metrics_.Initialize(sample_rate_hz); + + last_gain_ = 1.0f; + sample_rate_hz_ = sample_rate_hz; + dc_forgetting_factor_ = 0.01f * sample_rate_hz / 48000.f; + std::fill(dc_level_, dc_level_ + arraysize(dc_level_), 0.f); +} + +void LevelController::Process(AudioBuffer* audio) { + RTC_DCHECK_LT(0, audio->num_channels()); + RTC_DCHECK_GE(2, audio->num_channels()); + RTC_DCHECK_NE(0.f, dc_forgetting_factor_); + RTC_DCHECK(sample_rate_hz_); + data_dumper_->DumpWav("lc_input", audio->num_frames(), + audio->channels_const_f()[0], *sample_rate_hz_, 1); + + // Remove DC level. + for (size_t k = 0; k < audio->num_channels(); ++k) { + UpdateAndRemoveDcLevel( + dc_forgetting_factor_, &dc_level_[k], + rtc::ArrayView(audio->channels_f()[k], audio->num_frames())); + } + + SignalClassifier::SignalType signal_type; + signal_classifier_.Analyze(*audio, &signal_type); + int tmp = static_cast(signal_type); + data_dumper_->DumpRaw("lc_signal_type", 1, &tmp); + + // Estimate the noise energy. + float noise_energy = + noise_level_estimator_.Analyze(signal_type, FrameEnergy(*audio)); + + // Estimate the overall signal peak level. + const float frame_peak_level = PeakLevel(*audio); + const float long_term_peak_level = + peak_level_estimator_.Analyze(signal_type, frame_peak_level); + + float saturating_gain = saturating_gain_estimator_.GetGain(); + + // Compute the new gain to apply. + last_gain_ = + gain_selector_.GetNewGain(long_term_peak_level, noise_energy, + saturating_gain, gain_jumpstart_, signal_type); + + // Unflag the jumpstart of the gain as it should only happen once. + gain_jumpstart_ = false; + + // Apply the gain to the signal. + int num_saturations = gain_applier_.Process(last_gain_, audio); + + // Estimate the gain that saturates the overall signal. + saturating_gain_estimator_.Update(last_gain_, num_saturations); + + // Update the metrics. + metrics_.Update(long_term_peak_level, noise_energy, last_gain_, + frame_peak_level); + + data_dumper_->DumpRaw("lc_selected_gain", 1, &last_gain_); + data_dumper_->DumpRaw("lc_noise_energy", 1, &noise_energy); + data_dumper_->DumpRaw("lc_peak_level", 1, &long_term_peak_level); + data_dumper_->DumpRaw("lc_saturating_gain", 1, &saturating_gain); + + data_dumper_->DumpWav("lc_output", audio->num_frames(), + audio->channels_f()[0], *sample_rate_hz_, 1); +} + +void LevelController::ApplyConfig( + const AudioProcessing::Config::LevelController& config) { + RTC_DCHECK(Validate(config)); + config_ = config; + peak_level_estimator_.Initialize(config_.initial_peak_level_dbfs); + gain_jumpstart_ = true; +} + +std::string LevelController::ToString( + const AudioProcessing::Config::LevelController& config) { + std::stringstream ss; + ss << "{" + << "enabled: " << (config.enabled ? "true" : "false") << ", " + << "initial_peak_level_dbfs: " << config.initial_peak_level_dbfs << "}"; + return ss.str(); +} + +bool LevelController::Validate( + const AudioProcessing::Config::LevelController& config) { + return (config.initial_peak_level_dbfs < + std::numeric_limits::epsilon() && + config.initial_peak_level_dbfs > + -(100.f + std::numeric_limits::epsilon())); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.h new file mode 100644 index 0000000000..224b886abd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_LEVEL_CONTROLLER_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_LEVEL_CONTROLLER_H_ + +#include +#include + +#include "api/optional.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/level_controller/gain_applier.h" +#include "modules/audio_processing/level_controller/gain_selector.h" +#include "modules/audio_processing/level_controller/noise_level_estimator.h" +#include "modules/audio_processing/level_controller/peak_level_estimator.h" +#include "modules/audio_processing/level_controller/saturating_gain_estimator.h" +#include "modules/audio_processing/level_controller/signal_classifier.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; +class AudioBuffer; + +class LevelController { + public: + LevelController(); + ~LevelController(); + + void Initialize(int sample_rate_hz); + void Process(AudioBuffer* audio); + float GetLastGain() { return last_gain_; } + + // TODO(peah): This method is a temporary solution as the the aim is to + // instead apply the config inside the constructor. Therefore this is likely + // to change. + void ApplyConfig(const AudioProcessing::Config::LevelController& config); + // Validates a config. + static bool Validate(const AudioProcessing::Config::LevelController& config); + // Dumps a config to a string. + static std::string ToString( + const AudioProcessing::Config::LevelController& config); + + private: + class Metrics { + public: + Metrics() { Initialize(AudioProcessing::kSampleRate48kHz); } + void Initialize(int sample_rate_hz); + void Update(float long_term_peak_level, + float noise_level, + float gain, + float frame_peak_level); + + private: + void Reset(); + + size_t metrics_frame_counter_; + float gain_sum_; + float peak_level_sum_; + float noise_energy_sum_; + float max_gain_; + float max_peak_level_; + float max_noise_energy_; + float frame_length_; + }; + + std::unique_ptr data_dumper_; + GainSelector gain_selector_; + GainApplier gain_applier_; + SignalClassifier signal_classifier_; + NoiseLevelEstimator noise_level_estimator_; + PeakLevelEstimator peak_level_estimator_; + SaturatingGainEstimator saturating_gain_estimator_; + Metrics metrics_; + rtc::Optional sample_rate_hz_; + static int instance_count_; + float dc_level_[2]; + float dc_forgetting_factor_; + float last_gain_; + bool gain_jumpstart_ = false; + AudioProcessing::Config::LevelController config_; + + RTC_DISALLOW_COPY_AND_ASSIGN(LevelController); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_LEVEL_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_complexity_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_complexity_unittest.cc new file mode 100644 index 0000000000..7d61cff21d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_complexity_unittest.cc @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/level_controller/level_controller.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "modules/audio_processing/test/performance_timer.h" +#include "modules/audio_processing/test/simulator_buffers.h" +#include "rtc_base/random.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace { + +const size_t kNumFramesToProcess = 300; +const size_t kNumFramesToProcessAtWarmup = 300; +const size_t kToTalNumFrames = + kNumFramesToProcess + kNumFramesToProcessAtWarmup; + +void RunStandaloneSubmodule(int sample_rate_hz, size_t num_channels) { + test::SimulatorBuffers buffers(sample_rate_hz, sample_rate_hz, sample_rate_hz, + sample_rate_hz, num_channels, num_channels, + num_channels, num_channels); + test::PerformanceTimer timer(kNumFramesToProcess); + + LevelController level_controller; + level_controller.Initialize(sample_rate_hz); + + for (size_t frame_no = 0; frame_no < kToTalNumFrames; ++frame_no) { + buffers.UpdateInputBuffers(); + + if (frame_no >= kNumFramesToProcessAtWarmup) { + timer.StartTimer(); + } + level_controller.Process(buffers.capture_input_buffer.get()); + if (frame_no >= kNumFramesToProcessAtWarmup) { + timer.StopTimer(); + } + } + webrtc::test::PrintResultMeanAndError( + "level_controller_call_durations", + "_" + std::to_string(sample_rate_hz) + "Hz_" + + std::to_string(num_channels) + "_channels", + "StandaloneLevelControl", timer.GetDurationAverage(), + timer.GetDurationStandardDeviation(), "us", false); +} + +void RunTogetherWithApm(const std::string& test_description, + int render_input_sample_rate_hz, + int render_output_sample_rate_hz, + int capture_input_sample_rate_hz, + int capture_output_sample_rate_hz, + size_t num_channels, + bool use_mobile_aec, + bool include_default_apm_processing) { + test::SimulatorBuffers buffers( + render_input_sample_rate_hz, capture_input_sample_rate_hz, + render_output_sample_rate_hz, capture_output_sample_rate_hz, num_channels, + num_channels, num_channels, num_channels); + test::PerformanceTimer render_timer(kNumFramesToProcess); + test::PerformanceTimer capture_timer(kNumFramesToProcess); + test::PerformanceTimer total_timer(kNumFramesToProcess); + + webrtc::Config config; + AudioProcessing::Config apm_config; + if (include_default_apm_processing) { + config.Set(new DelayAgnostic(true)); + config.Set(new ExtendedFilter(true)); + } + apm_config.level_controller.enabled = true; + apm_config.residual_echo_detector.enabled = include_default_apm_processing; + + std::unique_ptr apm; + apm.reset(AudioProcessing::Create(config)); + ASSERT_TRUE(apm.get()); + apm->ApplyConfig(apm_config); + + ASSERT_EQ(AudioProcessing::kNoError, + apm->gain_control()->Enable(include_default_apm_processing)); + if (use_mobile_aec) { + ASSERT_EQ(AudioProcessing::kNoError, + apm->echo_cancellation()->Enable(false)); + ASSERT_EQ(AudioProcessing::kNoError, apm->echo_control_mobile()->Enable( + include_default_apm_processing)); + } else { + ASSERT_EQ(AudioProcessing::kNoError, + apm->echo_cancellation()->Enable(include_default_apm_processing)); + ASSERT_EQ(AudioProcessing::kNoError, + apm->echo_control_mobile()->Enable(false)); + } + apm_config.high_pass_filter.enabled = include_default_apm_processing; + ASSERT_EQ(AudioProcessing::kNoError, + apm->noise_suppression()->Enable(include_default_apm_processing)); + ASSERT_EQ(AudioProcessing::kNoError, + apm->voice_detection()->Enable(include_default_apm_processing)); + ASSERT_EQ(AudioProcessing::kNoError, + apm->level_estimator()->Enable(include_default_apm_processing)); + + StreamConfig render_input_config(render_input_sample_rate_hz, num_channels, + false); + StreamConfig render_output_config(render_output_sample_rate_hz, num_channels, + false); + StreamConfig capture_input_config(capture_input_sample_rate_hz, num_channels, + false); + StreamConfig capture_output_config(capture_output_sample_rate_hz, + num_channels, false); + + for (size_t frame_no = 0; frame_no < kToTalNumFrames; ++frame_no) { + buffers.UpdateInputBuffers(); + + if (frame_no >= kNumFramesToProcessAtWarmup) { + total_timer.StartTimer(); + render_timer.StartTimer(); + } + ASSERT_EQ(AudioProcessing::kNoError, + apm->ProcessReverseStream( + &buffers.render_input[0], render_input_config, + render_output_config, &buffers.render_output[0])); + + if (frame_no >= kNumFramesToProcessAtWarmup) { + render_timer.StopTimer(); + + capture_timer.StartTimer(); + } + + ASSERT_EQ(AudioProcessing::kNoError, apm->set_stream_delay_ms(0)); + ASSERT_EQ( + AudioProcessing::kNoError, + apm->ProcessStream(&buffers.capture_input[0], capture_input_config, + capture_output_config, &buffers.capture_output[0])); + + if (frame_no >= kNumFramesToProcessAtWarmup) { + capture_timer.StopTimer(); + total_timer.StopTimer(); + } + } + + webrtc::test::PrintResultMeanAndError( + "level_controller_call_durations", + "_" + std::to_string(render_input_sample_rate_hz) + "_" + + std::to_string(render_output_sample_rate_hz) + "_" + + std::to_string(capture_input_sample_rate_hz) + "_" + + std::to_string(capture_output_sample_rate_hz) + "Hz_" + + std::to_string(num_channels) + "_channels" + "_render", + test_description, render_timer.GetDurationAverage(), + render_timer.GetDurationStandardDeviation(), "us", false); + webrtc::test::PrintResultMeanAndError( + "level_controller_call_durations", + "_" + std::to_string(render_input_sample_rate_hz) + "_" + + std::to_string(render_output_sample_rate_hz) + "_" + + std::to_string(capture_input_sample_rate_hz) + "_" + + std::to_string(capture_output_sample_rate_hz) + "Hz_" + + std::to_string(num_channels) + "_channels" + "_capture", + test_description, capture_timer.GetDurationAverage(), + capture_timer.GetDurationStandardDeviation(), "us", false); + webrtc::test::PrintResultMeanAndError( + "level_controller_call_durations", + "_" + std::to_string(render_input_sample_rate_hz) + "_" + + std::to_string(render_output_sample_rate_hz) + "_" + + std::to_string(capture_input_sample_rate_hz) + "_" + + std::to_string(capture_output_sample_rate_hz) + "Hz_" + + std::to_string(num_channels) + "_channels" + "_total", + test_description, total_timer.GetDurationAverage(), + total_timer.GetDurationStandardDeviation(), "us", false); +} + +} // namespace + +// TODO(peah): Reactivate once issue 7712 has been resolved. +TEST(LevelControllerPerformanceTest, DISABLED_StandaloneProcessing) { + int sample_rates_to_test[] = { + AudioProcessing::kSampleRate8kHz, AudioProcessing::kSampleRate16kHz, + AudioProcessing::kSampleRate32kHz, AudioProcessing::kSampleRate48kHz}; + for (auto sample_rate : sample_rates_to_test) { + for (size_t num_channels = 1; num_channels <= 2; ++num_channels) { + RunStandaloneSubmodule(sample_rate, num_channels); + } + } +} + +void TestSomeSampleRatesWithApm(const std::string& test_name, + bool use_mobile_agc, + bool include_default_apm_processing) { + // Test some stereo combinations first. + size_t num_channels = 2; + RunTogetherWithApm(test_name, 48000, 48000, AudioProcessing::kSampleRate16kHz, + AudioProcessing::kSampleRate32kHz, num_channels, + use_mobile_agc, include_default_apm_processing); + RunTogetherWithApm(test_name, 48000, 48000, AudioProcessing::kSampleRate48kHz, + AudioProcessing::kSampleRate8kHz, num_channels, + use_mobile_agc, include_default_apm_processing); + RunTogetherWithApm(test_name, 48000, 48000, 44100, 44100, num_channels, + use_mobile_agc, include_default_apm_processing); + + // Then test mono combinations. + num_channels = 1; + RunTogetherWithApm(test_name, 48000, 48000, AudioProcessing::kSampleRate48kHz, + AudioProcessing::kSampleRate48kHz, num_channels, + use_mobile_agc, include_default_apm_processing); +} + +// TODO(peah): Reactivate once issue 7712 has been resolved. +#if !defined(WEBRTC_ANDROID) +TEST(LevelControllerPerformanceTest, DISABLED_ProcessingViaApm) { +#else +TEST(LevelControllerPerformanceTest, DISABLED_ProcessingViaApm) { +#endif + // Run without default APM processing and desktop AGC. + TestSomeSampleRatesWithApm("SimpleLevelControlViaApm", false, false); +} + +// TODO(peah): Reactivate once issue 7712 has been resolved. +#if !defined(WEBRTC_ANDROID) +TEST(LevelControllerPerformanceTest, DISABLED_InteractionWithDefaultApm) { +#else +TEST(LevelControllerPerformanceTest, DISABLED_InteractionWithDefaultApm) { +#endif + bool include_default_apm_processing = true; + TestSomeSampleRatesWithApm("LevelControlAndDefaultDesktopApm", false, + include_default_apm_processing); + TestSomeSampleRatesWithApm("LevelControlAndDefaultMobileApm", true, + include_default_apm_processing); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_constants.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_constants.h new file mode 100644 index 0000000000..6cf2cd4c7e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_constants.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_LEVEL_CONTROLLER_CONSTANTS_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_LEVEL_CONTROLLER_CONSTANTS_H_ + +namespace webrtc { + +const float kMaxLcGain = 10; +const float kMaxLcNoisePower = 100.f * 100.f; +const float kTargetLcPeakLevel = 16384.f; +const float kTargetLcPeakLeveldBFS = -6.0206f; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_LEVEL_CONTROLLER_CONSTANTS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_unittest.cc new file mode 100644 index 0000000000..cb36ae08f3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/level_controller_unittest.cc @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/array_view.h" +#include "api/optional.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/level_controller/level_controller.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kNumFramesToProcess = 1000; + +// Processes a specified amount of frames, verifies the results and reports +// any errors. +void RunBitexactnessTest(int sample_rate_hz, + size_t num_channels, + rtc::Optional initial_peak_level_dbfs, + rtc::ArrayView output_reference) { + LevelController level_controller; + level_controller.Initialize(sample_rate_hz); + if (initial_peak_level_dbfs) { + AudioProcessing::Config::LevelController config; + config.initial_peak_level_dbfs = *initial_peak_level_dbfs; + level_controller.ApplyConfig(config); + } + + int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + const StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(samples_per_channel * num_channels); + for (size_t frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &capture_file, capture_input); + + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + level_controller.Process(&capture_buffer); + } + + // Extract test results. + std::vector capture_output; + test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer, + &capture_output); + + // Compare the output with the reference. Only the first values of the output + // from last frame processed are compared in order not having to specify all + // preceding frames as testvectors. As the algorithm being tested has a + // memory, testing only the last frame implicitly also tests the preceeding + // frames. + const float kVectorElementErrorBound = 1.0f / 32768.0f; + EXPECT_TRUE(test::VerifyDeinterleavedArray( + capture_config.num_frames(), capture_config.num_channels(), + output_reference, capture_output, kVectorElementErrorBound)); +} + +} // namespace + +TEST(LevelControllerConfig, ToString) { + AudioProcessing::Config config; + config.level_controller.enabled = true; + config.level_controller.initial_peak_level_dbfs = -6.0206f; + EXPECT_EQ("{enabled: true, initial_peak_level_dbfs: -6.0206}", + LevelController::ToString(config.level_controller)); + + config.level_controller.enabled = false; + config.level_controller.initial_peak_level_dbfs = -50.f; + EXPECT_EQ("{enabled: false, initial_peak_level_dbfs: -50}", + LevelController::ToString(config.level_controller)); +} + +TEST(LevelControlBitExactnessTest, Mono8kHz) { + const float kOutputReference[] = {-0.013939f, -0.012154f, -0.009054f}; + RunBitexactnessTest(AudioProcessing::kSampleRate8kHz, 1, rtc::nullopt, + kOutputReference); +} + +TEST(LevelControlBitExactnessTest, Mono16kHz) { + const float kOutputReference[] = {-0.013706f, -0.013215f, -0.013018f}; + RunBitexactnessTest(AudioProcessing::kSampleRate16kHz, 1, rtc::nullopt, + kOutputReference); +} + +TEST(LevelControlBitExactnessTest, Mono32kHz) { + const float kOutputReference[] = {-0.014495f, -0.016425f, -0.016085f}; + RunBitexactnessTest(AudioProcessing::kSampleRate32kHz, 1, rtc::nullopt, + kOutputReference); +} + +// TODO(peah): Investigate why this particular testcase differ between Android +// and the rest of the platforms. +TEST(LevelControlBitExactnessTest, Mono48kHz) { +#if !(defined(WEBRTC_ARCH_ARM64) || defined(WEBRTC_ARCH_ARM) || \ + defined(WEBRTC_ANDROID)) + const float kOutputReference[] = {-0.014277f, -0.015180f, -0.017437f}; +#else + const float kOutputReference[] = {-0.014306f, -0.015209f, -0.017466f}; +#endif + RunBitexactnessTest(AudioProcessing::kSampleRate48kHz, 1, rtc::nullopt, + kOutputReference); +} + +TEST(LevelControlBitExactnessTest, Stereo8kHz) { + const float kOutputReference[] = {-0.014063f, -0.008450f, -0.012159f, + -0.051967f, -0.023202f, -0.047858f}; + RunBitexactnessTest(AudioProcessing::kSampleRate8kHz, 2, rtc::nullopt, + kOutputReference); +} + +TEST(LevelControlBitExactnessTest, Stereo16kHz) { + const float kOutputReference[] = {-0.012714f, -0.005896f, -0.012220f, + -0.053306f, -0.024549f, -0.051527f}; + RunBitexactnessTest(AudioProcessing::kSampleRate16kHz, 2, rtc::nullopt, + kOutputReference); +} + +TEST(LevelControlBitExactnessTest, Stereo32kHz) { + const float kOutputReference[] = {-0.011764f, -0.007044f, -0.013472f, + -0.053537f, -0.026322f, -0.056253f}; + RunBitexactnessTest(AudioProcessing::kSampleRate32kHz, 2, rtc::nullopt, + kOutputReference); +} + +TEST(LevelControlBitExactnessTest, Stereo48kHz) { + const float kOutputReference[] = {-0.010643f, -0.006334f, -0.011377f, + -0.049088f, -0.023600f, -0.050465f}; + RunBitexactnessTest(AudioProcessing::kSampleRate48kHz, 2, rtc::nullopt, + kOutputReference); +} + +TEST(LevelControlBitExactnessTest, MonoInitial48kHz) { + const float kOutputReference[] = {-0.013884f, -0.014761f, -0.016951f}; + RunBitexactnessTest(AudioProcessing::kSampleRate48kHz, 1, -50, + kOutputReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.cc new file mode 100644 index 0000000000..abf4ea2cb1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.cc @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/noise_level_estimator.h" + +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { + +NoiseLevelEstimator::NoiseLevelEstimator() { + Initialize(AudioProcessing::kSampleRate48kHz); +} + +NoiseLevelEstimator::~NoiseLevelEstimator() {} + +void NoiseLevelEstimator::Initialize(int sample_rate_hz) { + noise_energy_ = 1.f; + first_update_ = true; + min_noise_energy_ = sample_rate_hz * 2.f * 2.f / 100.f; + noise_energy_hold_counter_ = 0; +} + +float NoiseLevelEstimator::Analyze(SignalClassifier::SignalType signal_type, + float frame_energy) { + if (frame_energy <= 0.f) { + return noise_energy_; + } + + if (first_update_) { + // Initialize the noise energy to the frame energy. + first_update_ = false; + return noise_energy_ = std::max(frame_energy, min_noise_energy_); + } + + // Update the noise estimate in a minimum statistics-type manner. + if (signal_type == SignalClassifier::SignalType::kStationary) { + if (frame_energy > noise_energy_) { + // Leak the estimate upwards towards the frame energy if no recent + // downward update. + noise_energy_hold_counter_ = std::max(noise_energy_hold_counter_ - 1, 0); + + if (noise_energy_hold_counter_ == 0) { + noise_energy_ = std::min(noise_energy_ * 1.01f, frame_energy); + } + } else { + // Update smoothly downwards with a limited maximum update magnitude. + noise_energy_ = + std::max(noise_energy_ * 0.9f, + noise_energy_ + 0.05f * (frame_energy - noise_energy_)); + noise_energy_hold_counter_ = 1000; + } + } else { + // For a non-stationary signal, leak the estimate downwards in order to + // avoid estimate locking due to incorrect signal classification. + noise_energy_ = noise_energy_ * 0.99f; + } + + // Ensure a minimum of the estimate. + return noise_energy_ = std::max(noise_energy_, min_noise_energy_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.h new file mode 100644 index 0000000000..94ef6737e7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_level_estimator.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_NOISE_LEVEL_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_NOISE_LEVEL_ESTIMATOR_H_ + +#include "modules/audio_processing/level_controller/signal_classifier.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class NoiseLevelEstimator { + public: + NoiseLevelEstimator(); + ~NoiseLevelEstimator(); + void Initialize(int sample_rate_hz); + float Analyze(SignalClassifier::SignalType signal_type, float frame_energy); + + private: + float min_noise_energy_ = 0.f; + bool first_update_; + float noise_energy_; + int noise_energy_hold_counter_; + + RTC_DISALLOW_COPY_AND_ASSIGN(NoiseLevelEstimator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_NOISE_LEVEL_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.cc new file mode 100644 index 0000000000..6e921c24d1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.cc @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/noise_spectrum_estimator.h" + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/arraysize.h" + +namespace webrtc { +namespace { +constexpr float kMinNoisePower = 100.f; +} // namespace + +NoiseSpectrumEstimator::NoiseSpectrumEstimator(ApmDataDumper* data_dumper) + : data_dumper_(data_dumper) { + Initialize(); +} + +void NoiseSpectrumEstimator::Initialize() { + std::fill(noise_spectrum_, noise_spectrum_ + arraysize(noise_spectrum_), + kMinNoisePower); +} + +void NoiseSpectrumEstimator::Update(rtc::ArrayView spectrum, + bool first_update) { + RTC_DCHECK_EQ(65, spectrum.size()); + + if (first_update) { + // Initialize the noise spectral estimate with the signal spectrum. + std::copy(spectrum.data(), spectrum.data() + spectrum.size(), + noise_spectrum_); + } else { + // Smoothly update the noise spectral estimate towards the signal spectrum + // such that the magnitude of the updates are limited. + for (size_t k = 0; k < spectrum.size(); ++k) { + if (noise_spectrum_[k] < spectrum[k]) { + noise_spectrum_[k] = std::min( + 1.01f * noise_spectrum_[k], + noise_spectrum_[k] + 0.05f * (spectrum[k] - noise_spectrum_[k])); + } else { + noise_spectrum_[k] = std::max( + 0.99f * noise_spectrum_[k], + noise_spectrum_[k] + 0.05f * (spectrum[k] - noise_spectrum_[k])); + } + } + } + + // Ensure that the noise spectal estimate does not become too low. + for (auto& v : noise_spectrum_) { + v = std::max(v, kMinNoisePower); + } + + data_dumper_->DumpRaw("lc_noise_spectrum", 65, noise_spectrum_); + data_dumper_->DumpRaw("lc_signal_spectrum", spectrum); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.h new file mode 100644 index 0000000000..f10933ec96 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/noise_spectrum_estimator.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_NOISE_SPECTRUM_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_NOISE_SPECTRUM_ESTIMATOR_H_ + +#include "api/array_view.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; + +class NoiseSpectrumEstimator { + public: + explicit NoiseSpectrumEstimator(ApmDataDumper* data_dumper); + void Initialize(); + void Update(rtc::ArrayView spectrum, bool first_update); + + rtc::ArrayView GetNoiseSpectrum() const { + return rtc::ArrayView(noise_spectrum_); + } + + private: + ApmDataDumper* data_dumper_; + float noise_spectrum_[65]; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(NoiseSpectrumEstimator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_NOISE_SPECTRUM_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.cc new file mode 100644 index 0000000000..e78d74f514 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.cc @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/peak_level_estimator.h" + +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { +namespace { + +constexpr float kMinLevel = 30.f; + +} // namespace + +PeakLevelEstimator::PeakLevelEstimator(float initial_peak_level_dbfs) { + Initialize(initial_peak_level_dbfs); +} + +PeakLevelEstimator::~PeakLevelEstimator() {} + +void PeakLevelEstimator::Initialize(float initial_peak_level_dbfs) { + RTC_DCHECK_LE(-100.f, initial_peak_level_dbfs); + RTC_DCHECK_GE(0.f, initial_peak_level_dbfs); + + peak_level_ = std::pow(10.f, initial_peak_level_dbfs / 20.f) * 32768.f; + peak_level_ = std::max(peak_level_, kMinLevel); + + hold_counter_ = 0; + initialization_phase_ = true; +} + +float PeakLevelEstimator::Analyze(SignalClassifier::SignalType signal_type, + float frame_peak_level) { + if (frame_peak_level == 0) { + RTC_DCHECK_LE(kMinLevel, peak_level_); + return peak_level_; + } + + if (peak_level_ < frame_peak_level) { + // Smoothly update the estimate upwards when the frame peak level is + // higher than the estimate. + peak_level_ += 0.1f * (frame_peak_level - peak_level_); + hold_counter_ = 100; + initialization_phase_ = false; + } else { + hold_counter_ = std::max(0, hold_counter_ - 1); + + // When the signal is highly non-stationary, update the estimate slowly + // downwards if the estimate is lower than the frame peak level. + if ((signal_type == SignalClassifier::SignalType::kHighlyNonStationary && + hold_counter_ == 0) || + initialization_phase_) { + peak_level_ = + std::max(peak_level_ + 0.01f * (frame_peak_level - peak_level_), + peak_level_ * 0.995f); + } + } + + peak_level_ = std::max(peak_level_, kMinLevel); + + return peak_level_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.h new file mode 100644 index 0000000000..0aa55d2d55 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/peak_level_estimator.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_PEAK_LEVEL_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_PEAK_LEVEL_ESTIMATOR_H_ + +#include "modules/audio_processing/level_controller/level_controller_constants.h" +#include "modules/audio_processing/level_controller/signal_classifier.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class PeakLevelEstimator { + public: + explicit PeakLevelEstimator(float initial_peak_level_dbfs); + ~PeakLevelEstimator(); + void Initialize(float initial_peak_level_dbfs); + float Analyze(SignalClassifier::SignalType signal_type, + float frame_peak_level); + private: + float peak_level_; + int hold_counter_; + bool initialization_phase_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(PeakLevelEstimator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_PEAK_LEVEL_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.cc new file mode 100644 index 0000000000..60110c684b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.cc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/saturating_gain_estimator.h" + +#include +#include + +#include "modules/audio_processing/level_controller/level_controller_constants.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { + +SaturatingGainEstimator::SaturatingGainEstimator() { + Initialize(); +} + +SaturatingGainEstimator::~SaturatingGainEstimator() {} + +void SaturatingGainEstimator::Initialize() { + saturating_gain_ = kMaxLcGain; + saturating_gain_hold_counter_ = 0; +} + +void SaturatingGainEstimator::Update(float gain, int num_saturations) { + bool too_many_saturations = (num_saturations > 2); + + if (too_many_saturations) { + saturating_gain_ = 0.95f * gain; + saturating_gain_hold_counter_ = 1000; + } else { + saturating_gain_hold_counter_ = + std::max(0, saturating_gain_hold_counter_ - 1); + if (saturating_gain_hold_counter_ == 0) { + saturating_gain_ *= 1.001f; + saturating_gain_ = std::min(kMaxLcGain, saturating_gain_); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.h new file mode 100644 index 0000000000..8980f4ef97 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/saturating_gain_estimator.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_SATURATING_GAIN_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_SATURATING_GAIN_ESTIMATOR_H_ + +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; + +class SaturatingGainEstimator { + public: + SaturatingGainEstimator(); + ~SaturatingGainEstimator(); + void Initialize(); + void Update(float gain, int num_saturations); + float GetGain() const { return saturating_gain_; } + + private: + float saturating_gain_; + int saturating_gain_hold_counter_; + + RTC_DISALLOW_COPY_AND_ASSIGN(SaturatingGainEstimator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_SATURATING_GAIN_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.cc new file mode 100644 index 0000000000..d2d5917387 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_controller/signal_classifier.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/level_controller/down_sampler.h" +#include "modules/audio_processing/level_controller/noise_spectrum_estimator.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace { + +void RemoveDcLevel(rtc::ArrayView x) { + RTC_DCHECK_LT(0, x.size()); + float mean = std::accumulate(x.data(), x.data() + x.size(), 0.f); + mean /= x.size(); + + for (float& v : x) { + v -= mean; + } +} + +void PowerSpectrum(const OouraFft* ooura_fft, + rtc::ArrayView x, + rtc::ArrayView spectrum) { + RTC_DCHECK_EQ(65, spectrum.size()); + RTC_DCHECK_EQ(128, x.size()); + float X[128]; + std::copy(x.data(), x.data() + x.size(), X); + ooura_fft->Fft(X); + + float* X_p = X; + RTC_DCHECK_EQ(X_p, &X[0]); + spectrum[0] = (*X_p) * (*X_p); + ++X_p; + RTC_DCHECK_EQ(X_p, &X[1]); + spectrum[64] = (*X_p) * (*X_p); + for (int k = 1; k < 64; ++k) { + ++X_p; + RTC_DCHECK_EQ(X_p, &X[2 * k]); + spectrum[k] = (*X_p) * (*X_p); + ++X_p; + RTC_DCHECK_EQ(X_p, &X[2 * k + 1]); + spectrum[k] += (*X_p) * (*X_p); + } +} + +webrtc::SignalClassifier::SignalType ClassifySignal( + rtc::ArrayView signal_spectrum, + rtc::ArrayView noise_spectrum, + ApmDataDumper* data_dumper) { + int num_stationary_bands = 0; + int num_highly_nonstationary_bands = 0; + + // Detect stationary and highly nonstationary bands. + for (size_t k = 1; k < 40; k++) { + if (signal_spectrum[k] < 3 * noise_spectrum[k] && + signal_spectrum[k] * 3 > noise_spectrum[k]) { + ++num_stationary_bands; + } else if (signal_spectrum[k] > 9 * noise_spectrum[k]) { + ++num_highly_nonstationary_bands; + } + } + + data_dumper->DumpRaw("lc_num_stationary_bands", 1, &num_stationary_bands); + data_dumper->DumpRaw("lc_num_highly_nonstationary_bands", 1, + &num_highly_nonstationary_bands); + + // Use the detected number of bands to classify the overall signal + // stationarity. + if (num_stationary_bands > 15) { + return SignalClassifier::SignalType::kStationary; + } else if (num_highly_nonstationary_bands > 15) { + return SignalClassifier::SignalType::kHighlyNonStationary; + } else { + return SignalClassifier::SignalType::kNonStationary; + } +} + +} // namespace + +SignalClassifier::FrameExtender::FrameExtender(size_t frame_size, + size_t extended_frame_size) + : x_old_(extended_frame_size - frame_size, 0.f) {} + +SignalClassifier::FrameExtender::~FrameExtender() = default; + +void SignalClassifier::FrameExtender::ExtendFrame( + rtc::ArrayView x, + rtc::ArrayView x_extended) { + RTC_DCHECK_EQ(x_old_.size() + x.size(), x_extended.size()); + std::copy(x_old_.data(), x_old_.data() + x_old_.size(), x_extended.data()); + std::copy(x.data(), x.data() + x.size(), x_extended.data() + x_old_.size()); + std::copy(x_extended.data() + x_extended.size() - x_old_.size(), + x_extended.data() + x_extended.size(), x_old_.data()); +} + +SignalClassifier::SignalClassifier(ApmDataDumper* data_dumper) + : data_dumper_(data_dumper), + down_sampler_(data_dumper_), + noise_spectrum_estimator_(data_dumper_) { + Initialize(AudioProcessing::kSampleRate48kHz); +} +SignalClassifier::~SignalClassifier() {} + +void SignalClassifier::Initialize(int sample_rate_hz) { + down_sampler_.Initialize(sample_rate_hz); + noise_spectrum_estimator_.Initialize(); + frame_extender_.reset(new FrameExtender(80, 128)); + sample_rate_hz_ = sample_rate_hz; + initialization_frames_left_ = 2; + consistent_classification_counter_ = 3; + last_signal_type_ = SignalClassifier::SignalType::kNonStationary; +} + +void SignalClassifier::Analyze(const AudioBuffer& audio, + SignalType* signal_type) { + RTC_DCHECK_EQ(audio.num_frames(), sample_rate_hz_ / 100); + + // Compute the signal power spectrum. + float downsampled_frame[80]; + down_sampler_.DownSample(rtc::ArrayView( + audio.channels_const_f()[0], audio.num_frames()), + downsampled_frame); + float extended_frame[128]; + frame_extender_->ExtendFrame(downsampled_frame, extended_frame); + RemoveDcLevel(extended_frame); + float signal_spectrum[65]; + PowerSpectrum(&ooura_fft_, extended_frame, signal_spectrum); + + // Classify the signal based on the estimate of the noise spectrum and the + // signal spectrum estimate. + *signal_type = ClassifySignal(signal_spectrum, + noise_spectrum_estimator_.GetNoiseSpectrum(), + data_dumper_); + + // Update the noise spectrum based on the signal spectrum. + noise_spectrum_estimator_.Update(signal_spectrum, + initialization_frames_left_ > 0); + + // Update the number of frames until a reliable signal spectrum is achieved. + initialization_frames_left_ = std::max(0, initialization_frames_left_ - 1); + + if (last_signal_type_ == *signal_type) { + consistent_classification_counter_ = + std::max(0, consistent_classification_counter_ - 1); + } else { + last_signal_type_ = *signal_type; + consistent_classification_counter_ = 3; + } + + if (consistent_classification_counter_ > 0) { + *signal_type = SignalClassifier::SignalType::kNonStationary; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.h new file mode 100644 index 0000000000..2be13fef7a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_controller/signal_classifier.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_SIGNAL_CLASSIFIER_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_SIGNAL_CLASSIFIER_H_ + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/level_controller/down_sampler.h" +#include "modules/audio_processing/level_controller/noise_spectrum_estimator.h" +#include "modules/audio_processing/utility/ooura_fft.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class ApmDataDumper; +class AudioBuffer; + +class SignalClassifier { + public: + enum class SignalType { kHighlyNonStationary, kNonStationary, kStationary }; + + explicit SignalClassifier(ApmDataDumper* data_dumper); + ~SignalClassifier(); + + void Initialize(int sample_rate_hz); + void Analyze(const AudioBuffer& audio, SignalType* signal_type); + + private: + class FrameExtender { + public: + FrameExtender(size_t frame_size, size_t extended_frame_size); + ~FrameExtender(); + + void ExtendFrame(rtc::ArrayView x, + rtc::ArrayView x_extended); + + private: + std::vector x_old_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameExtender); + }; + + ApmDataDumper* const data_dumper_; + DownSampler down_sampler_; + std::unique_ptr frame_extender_; + NoiseSpectrumEstimator noise_spectrum_estimator_; + int sample_rate_hz_; + int initialization_frames_left_; + int consistent_classification_counter_; + SignalType last_signal_type_; + const OouraFft ooura_fft_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(SignalClassifier); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_CONTROLLER_SIGNAL_CLASSIFIER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.cc new file mode 100644 index 0000000000..c937f84525 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/level_estimator_impl.h" + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/rms_level.h" + +namespace webrtc { + +LevelEstimatorImpl::LevelEstimatorImpl(rtc::CriticalSection* crit) + : crit_(crit), rms_(new RmsLevel()) { + RTC_DCHECK(crit); +} + +LevelEstimatorImpl::~LevelEstimatorImpl() {} + +void LevelEstimatorImpl::Initialize() { + rtc::CritScope cs(crit_); + rms_->Reset(); +} + +void LevelEstimatorImpl::ProcessStream(AudioBuffer* audio) { + RTC_DCHECK(audio); + rtc::CritScope cs(crit_); + if (!enabled_) { + return; + } + + for (size_t i = 0; i < audio->num_channels(); i++) { + rms_->Analyze(rtc::ArrayView(audio->channels_const()[i], + audio->num_frames())); + } +} + +int LevelEstimatorImpl::Enable(bool enable) { + rtc::CritScope cs(crit_); + if (enable && !enabled_) { + rms_->Reset(); + } + enabled_ = enable; + return AudioProcessing::kNoError; +} + +bool LevelEstimatorImpl::is_enabled() const { + rtc::CritScope cs(crit_); + return enabled_; +} + +int LevelEstimatorImpl::RMS() { + rtc::CritScope cs(crit_); + if (!enabled_) { + return AudioProcessing::kNotEnabledError; + } + + return rms_->Average(); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.h new file mode 100644 index 0000000000..901ae4cf58 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_impl.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LEVEL_ESTIMATOR_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_LEVEL_ESTIMATOR_IMPL_H_ + +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +class AudioBuffer; +class RmsLevel; + +class LevelEstimatorImpl : public LevelEstimator { + public: + explicit LevelEstimatorImpl(rtc::CriticalSection* crit); + ~LevelEstimatorImpl() override; + + // TODO(peah): Fold into ctor, once public API is removed. + void Initialize(); + void ProcessStream(AudioBuffer* audio); + + // LevelEstimator implementation. + int Enable(bool enable) override; + bool is_enabled() const override; + int RMS() override; + + private: + rtc::CriticalSection* const crit_ = nullptr; + bool enabled_ RTC_GUARDED_BY(crit_) = false; + std::unique_ptr rms_ RTC_GUARDED_BY(crit_); + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(LevelEstimatorImpl); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LEVEL_ESTIMATOR_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_unittest.cc new file mode 100644 index 0000000000..94b84bbdc3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/level_estimator_unittest.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/level_estimator_impl.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kNumFramesToProcess = 1000; + +// Processes a specified amount of frames, verifies the results and reports +// any errors. +void RunBitexactnessTest(int sample_rate_hz, + size_t num_channels, + int rms_reference) { + rtc::CriticalSection crit_capture; + LevelEstimatorImpl level_estimator(&crit_capture); + level_estimator.Initialize(); + level_estimator.Enable(true); + + int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames()); + + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(samples_per_channel * num_channels); + for (size_t frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &capture_file, capture_input); + + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + level_estimator.ProcessStream(&capture_buffer); + } + + // Extract test results. + int rms = level_estimator.RMS(); + + // Compare the output to the reference. + EXPECT_EQ(rms_reference, rms); +} + +} // namespace + +TEST(LevelEstimatorBitExactnessTest, Mono8kHz) { + const int kRmsReference = 31; + + RunBitexactnessTest(8000, 1, kRmsReference); +} + +TEST(LevelEstimatorBitExactnessTest, Mono16kHz) { + const int kRmsReference = 31; + + RunBitexactnessTest(16000, 1, kRmsReference); +} + +TEST(LevelEstimatorBitExactnessTest, Mono32kHz) { + const int kRmsReference = 31; + + RunBitexactnessTest(32000, 1, kRmsReference); +} + +TEST(LevelEstimatorBitExactnessTest, Mono48kHz) { + const int kRmsReference = 31; + + RunBitexactnessTest(48000, 1, kRmsReference); +} + +TEST(LevelEstimatorBitExactnessTest, Stereo16kHz) { + const int kRmsReference = 30; + + RunBitexactnessTest(16000, 2, kRmsReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.cc new file mode 100644 index 0000000000..fb25b74fff --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/logging/apm_data_dumper.h" + +#include + +#include "rtc_base/stringutils.h" + +// Check to verify that the define is properly set. +#if !defined(WEBRTC_APM_DEBUG_DUMP) || \ + (WEBRTC_APM_DEBUG_DUMP != 0 && WEBRTC_APM_DEBUG_DUMP != 1) +#error "Set WEBRTC_APM_DEBUG_DUMP to either 0 or 1" +#endif + +namespace webrtc { + +namespace { + +#if WEBRTC_APM_DEBUG_DUMP == 1 +std::string FormFileName(const char* name, + int instance_index, + int reinit_index, + const std::string& suffix) { +#ifdef WEBRTC_WIN + char sep = '\\'; +#else + char sep = '/'; +#endif + + std::stringstream ss; + std::string base = rtc::LogMessage::aec_debug_filename(); + ss << base; + + if (base.length() && base.back() != sep) { + ss << sep; + } + + ss << name << "_" << instance_index << "-" << reinit_index << suffix; + return ss.str(); +} +#endif + +} // namespace + +#if WEBRTC_APM_DEBUG_DUMP == 1 +ApmDataDumper::ApmDataDumper(int instance_index) + : instance_index_(instance_index) + , debug_written_(0) {} +#else +ApmDataDumper::ApmDataDumper(int instance_index) {} +#endif + +ApmDataDumper::~ApmDataDumper() {} + +#if WEBRTC_APM_DEBUG_DUMP == 1 +FILE* ApmDataDumper::GetRawFile(const char* name) { + std::string filename = + FormFileName(name, instance_index_, recording_set_index_, ".dat"); + auto& f = raw_files_[filename]; + if (!f) { + f.reset(fopen(filename.c_str(), "wb")); + } + return f.get(); +} + +WavWriter* ApmDataDumper::GetWavFile(const char* name, + int sample_rate_hz, + int num_channels) { + std::string filename = + FormFileName(name, instance_index_, recording_set_index_, ".wav"); + auto& f = wav_files_[filename]; + if (!f) { + f.reset(new WavWriter(filename.c_str(), sample_rate_hz, num_channels)); + } + return f.get(); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.h b/third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.h new file mode 100644 index 0000000000..83ff8b19ad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/logging/apm_data_dumper.h @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LOGGING_APM_DATA_DUMPER_H_ +#define MODULES_AUDIO_PROCESSING_LOGGING_APM_DATA_DUMPER_H_ + +#include + +#include +#include +#include + +#include "api/array_view.h" +#include "common_audio/wav_file.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/logging.h" + +// Check to verify that the define is properly set. +#if !defined(WEBRTC_APM_DEBUG_DUMP) || \ + (WEBRTC_APM_DEBUG_DUMP != 0 && WEBRTC_APM_DEBUG_DUMP != 1) +#error "Set WEBRTC_APM_DEBUG_DUMP to either 0 or 1" +#endif + +namespace webrtc { + +#if WEBRTC_APM_DEBUG_DUMP == 1 +// Functor used to use as a custom deleter in the map of file pointers to raw +// files. +struct RawFileCloseFunctor { + void operator()(FILE* f) const { if (f) fclose(f); } +}; +#endif + +// Class that handles dumping of variables into files. +class ApmDataDumper { + public: + // Constructor that takes an instance index that may + // be used to distinguish data dumped from different + // instances of the code. + explicit ApmDataDumper(int instance_index); + + ~ApmDataDumper(); + + // Reinitializes the data dumping such that new versions + // of all files being dumped to are created. + void InitiateNewSetOfRecordings() { +#if WEBRTC_APM_DEBUG_DUMP == 1 + ++recording_set_index_; + debug_written_ = 0; +#endif + } + + // Methods for performing dumping of data of various types into + // various formats. + void DumpRaw(const char* name, double v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(&v, sizeof(v), 1, file); + } + } +#endif + } + + void DumpRaw(const char* name, size_t v_length, const double* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(v, sizeof(v[0]), v_length, file); + } + } +#endif + } + + void DumpRaw(const char* name, rtc::ArrayView v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, v.size(), v.data()); +#endif + } + + void DumpRaw(const char* name, float v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(&v, sizeof(v), 1, file); + } + } +#endif + } + + void DumpRaw(const char* name, size_t v_length, const float* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(v, sizeof(v[0]), v_length, file); + } + } +#endif + } + + void DumpRaw(const char* name, rtc::ArrayView v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, v.size(), v.data()); +#endif + } + + void DumpRaw(const char* name, bool v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, static_cast(v)); +#endif + } + + void DumpRaw(const char* name, size_t v_length, const bool* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + for (int k = 0; k < v_length; ++k) { + int16_t value = static_cast(v[k]); + fwrite(&value, sizeof(value), 1, file); + } + } + } +#endif + } + + void DumpRaw(const char* name, rtc::ArrayView v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, v.size(), v.data()); +#endif + } + + void DumpRaw(const char* name, int16_t v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(&v, sizeof(v), 1, file); + } + } +#endif + } + + void DumpRaw(const char* name, size_t v_length, const int16_t* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(v, sizeof(v[0]), v_length, file); + } + } +#endif + } + + void DumpRaw(const char* name, rtc::ArrayView v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, v.size(), v.data()); +#endif + } + + void DumpRaw(const char* name, int32_t v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(&v, sizeof(v), 1, file); + } + } +#endif + } + + void DumpRaw(const char* name, size_t v_length, const int32_t* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(v, sizeof(v[0]), v_length, file); + } + } +#endif + } + + void DumpRaw(const char* name, size_t v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(&v, sizeof(v), 1, file); + } + } +#endif + } + + void DumpRaw(const char* name, size_t v_length, const size_t* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + FILE* file = GetRawFile(name); + if (file) { + fwrite(v, sizeof(v[0]), v_length, file); + } + } +#endif + } + + void DumpRaw(const char* name, rtc::ArrayView v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, v.size(), v.data()); +#endif + } + + void DumpWav(const char* name, + size_t v_length, + const float* v, + int sample_rate_hz, + int num_channels) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + if (rtc::LogMessage::aec_debug()) { + WavWriter* file = GetWavFile(name, sample_rate_hz, num_channels); + file->WriteSamples(v, v_length); + // Cheat and use aec_near as a stand-in for "size of the largest file" + // in the dump. We're looking to limit the total time, and that's a + // reasonable stand-in. + if (strcmp(name, "aec_near") == 0) { + updateDebugWritten(v_length * sizeof(float)); + } + } +#endif + } + + void DumpWav(const char* name, + rtc::ArrayView v, + int sample_rate_hz, + int num_channels) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpWav(name, v.size(), v.data(), sample_rate_hz, num_channels); +#endif + } + + private: +#if WEBRTC_APM_DEBUG_DUMP == 1 + const int instance_index_; + int recording_set_index_ = 0; + std::unordered_map> + raw_files_; + std::unordered_map> wav_files_; + + FILE* GetRawFile(const char* name); + WavWriter* GetWavFile(const char* name, int sample_rate_hz, int num_channels); + + uint32_t debug_written_; + + void updateDebugWritten(uint32_t amount) { + debug_written_ += amount; + // Limit largest files to a specific (rough) size, to avoid filling up disk. + if (debug_written_ >= rtc::LogMessage::aec_debug_size()) { + rtc::LogMessage::set_aec_debug(false); + } + } + +#endif + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(ApmDataDumper); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LOGGING_APM_DATA_DUMPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.cc new file mode 100644 index 0000000000..5245c68d02 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/low_cut_filter.h" + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/audio_buffer.h" + +namespace webrtc { +namespace { +const int16_t kFilterCoefficients8kHz[5] = {3798, -7596, 3798, 7807, -3733}; +const int16_t kFilterCoefficients[5] = {4012, -8024, 4012, 8002, -3913}; +} // namespace + +class LowCutFilter::BiquadFilter { + public: + explicit BiquadFilter(int sample_rate_hz) + : ba_(sample_rate_hz == AudioProcessing::kSampleRate8kHz + ? kFilterCoefficients8kHz + : kFilterCoefficients) { + std::memset(x_, 0, sizeof(x_)); + std::memset(y_, 0, sizeof(y_)); + } + + void Process(int16_t* data, size_t length) { + const int16_t* const ba = ba_; + int16_t* x = x_; + int16_t* y = y_; + int32_t tmp_int32 = 0; + + for (size_t i = 0; i < length; i++) { + // y[i] = b[0] * x[i] + b[1] * x[i-1] + b[2] * x[i-2] + // + -a[1] * y[i-1] + -a[2] * y[i-2]; + + tmp_int32 = y[1] * ba[3]; // -a[1] * y[i-1] (low part) + tmp_int32 += y[3] * ba[4]; // -a[2] * y[i-2] (low part) + tmp_int32 = (tmp_int32 >> 15); + tmp_int32 += y[0] * ba[3]; // -a[1] * y[i-1] (high part) + tmp_int32 += y[2] * ba[4]; // -a[2] * y[i-2] (high part) + tmp_int32 *= 2; + + tmp_int32 += data[i] * ba[0]; // b[0] * x[0] + tmp_int32 += x[0] * ba[1]; // b[1] * x[i-1] + tmp_int32 += x[1] * ba[2]; // b[2] * x[i-2] + + // Update state (input part). + x[1] = x[0]; + x[0] = data[i]; + + // Update state (filtered part). + y[2] = y[0]; + y[3] = y[1]; + y[0] = static_cast(tmp_int32 >> 13); + + y[1] = static_cast((tmp_int32 & 0x00001FFF) * 4); + + // Rounding in Q12, i.e. add 2^11. + tmp_int32 += 2048; + + // Saturate (to 2^27) so that the HP filtered signal does not overflow. + tmp_int32 = WEBRTC_SPL_SAT(static_cast(134217727), tmp_int32, + static_cast(-134217728)); + + // Convert back to Q0 and use rounding. + data[i] = static_cast(tmp_int32 >> 12); + } + } + + private: + const int16_t* const ba_ = nullptr; + int16_t x_[2]; + int16_t y_[4]; +}; + +LowCutFilter::LowCutFilter(size_t channels, int sample_rate_hz) { + filters_.resize(channels); + for (size_t i = 0; i < channels; i++) { + filters_[i].reset(new BiquadFilter(sample_rate_hz)); + } +} + +LowCutFilter::~LowCutFilter() {} + +void LowCutFilter::Process(AudioBuffer* audio) { + RTC_DCHECK(audio); + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(filters_.size(), audio->num_channels()); + for (size_t i = 0; i < filters_.size(); i++) { + filters_[i]->Process(audio->split_bands(i)[kBand0To8kHz], + audio->num_frames_per_band()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.h new file mode 100644 index 0000000000..fd4c6f19cb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_LOW_CUT_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_LOW_CUT_FILTER_H_ + +#include +#include + +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class AudioBuffer; + +class LowCutFilter { + public: + LowCutFilter(size_t channels, int sample_rate_hz); + ~LowCutFilter(); + void Process(AudioBuffer* audio); + + private: + class BiquadFilter; + std::vector> filters_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(LowCutFilter); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_LOW_CUT_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter_unittest.cc new file mode 100644 index 0000000000..d98d665d37 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/low_cut_filter_unittest.cc @@ -0,0 +1,682 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/low_cut_filter.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +// Process one frame of data and produce the output. +std::vector ProcessOneFrame(const std::vector& frame_input, + const StreamConfig& stream_config, + LowCutFilter* low_cut_filter) { + AudioBuffer audio_buffer( + stream_config.num_frames(), stream_config.num_channels(), + stream_config.num_frames(), stream_config.num_channels(), + stream_config.num_frames()); + + test::CopyVectorToAudioBuffer(stream_config, frame_input, &audio_buffer); + low_cut_filter->Process(&audio_buffer); + std::vector frame_output; + test::ExtractVectorFromAudioBuffer(stream_config, &audio_buffer, + &frame_output); + return frame_output; +} + +// Processes a specified amount of frames, verifies the results and reports +// any errors. +void RunBitexactnessTest(int sample_rate, + int num_channels, + const std::vector& input, + const std::vector& reference) { + const StreamConfig stream_config(sample_rate, num_channels, false); + LowCutFilter low_cut_filter(num_channels, sample_rate); + + std::vector output; + const size_t num_frames_to_process = + input.size() / + (stream_config.num_frames() * stream_config.num_channels()); + for (size_t frame_no = 0; frame_no < num_frames_to_process; ++frame_no) { + std::vector frame_input( + input.begin() + + stream_config.num_frames() * stream_config.num_channels() * + frame_no, + input.begin() + + stream_config.num_frames() * stream_config.num_channels() * + (frame_no + 1)); + + output = ProcessOneFrame(frame_input, stream_config, &low_cut_filter); + } + + // Form vector to compare the reference to. Only the last frame processed + // is compared in order not having to specify all preceeding frames as + // inputs. As the algorithm being tested has a memory, testing only + // the last frame implicitly also tests the preceeding frames. + const size_t reference_frame_length = + reference.size() / stream_config.num_channels(); + std::vector output_to_verify; + for (size_t channel_no = 0; channel_no < stream_config.num_channels(); + ++channel_no) { + output_to_verify.insert( + output_to_verify.end(), + output.begin() + channel_no * stream_config.num_frames(), + output.begin() + channel_no * stream_config.num_frames() + + reference_frame_length); + } + + const float kElementErrorBound = 1.0f / 32768.0f; + EXPECT_TRUE(test::VerifyDeinterleavedArray( + reference_frame_length, num_channels, reference, output_to_verify, + kElementErrorBound)); +} + +// Method for forming a vector out of an array. +// TODO(peah): Remove once braced initialization is allowed. +std::vector CreateVector(const rtc::ArrayView& array_view) { + std::vector v; + for (auto value : array_view) { + v.push_back(value); + } + return v; +} +} // namespace + +TEST(LowCutFilterBitExactnessTest, Mono8kHzInitial) { + const float kReferenceInput[] = { + 0.153442f, -0.436920f, -0.057602f, -0.141767f, 0.108608f, 0.116834f, + 0.114979f, -0.103151f, -0.169925f, -0.167180f, 0.242024f, -0.525426f, + -0.058781f, 0.076667f, -0.185095f, 0.135319f, -0.020223f, -0.266058f, + 0.045755f, -0.076044f, -0.116221f, -0.201698f, 0.017423f, -0.523475f, + -0.112949f, -0.154125f, -0.258572f, 0.185075f, -0.208205f, 0.153298f, + 0.276703f, -0.044481f, 0.078771f, 0.181337f, -0.022962f, 0.153365f, + -0.358004f, 0.314864f, -0.280593f, -0.518572f, 0.392579f, -0.017786f, + 0.127293f, -0.103003f, -0.289389f, -0.871355f, 0.177583f, -0.081290f, + -0.055957f, 0.115011f, -0.402460f, -0.206836f, 0.325328f, 0.169526f, + -0.363311f, -0.624742f, -0.161979f, 0.060679f, 0.267214f, 0.026576f, + -0.318235f, 0.086812f, -0.332419f, -0.272485f, -0.185369f, -0.348598f, + -0.076833f, -0.255184f, -0.081007f, -0.131121f, -0.116196f, -0.142780f, + 0.349705f, 0.173054f, 0.016750f, -0.415957f, -0.461001f, -0.557111f, + 0.738711f, 0.275720f}; + + const float kReference[] = {0.142277f, -0.418518f, -0.028229f, -0.102112f, + 0.141270f, 0.137791f, 0.124577f, -0.088715f, + -0.142273f, -0.125885f, 0.266640f, -0.468079f}; + + RunBitexactnessTest( + 8000, 1, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} + +TEST(LowCutFilterBitExactnessTest, Mono8kHzConverged) { + const float kReferenceInput[] = { + 0.153442f, -0.436920f, -0.057602f, -0.141767f, 0.108608f, 0.116834f, + 0.114979f, -0.103151f, -0.169925f, -0.167180f, 0.242024f, -0.525426f, + -0.058781f, 0.076667f, -0.185095f, 0.135319f, -0.020223f, -0.266058f, + 0.045755f, -0.076044f, -0.116221f, -0.201698f, 0.017423f, -0.523475f, + -0.112949f, -0.154125f, -0.258572f, 0.185075f, -0.208205f, 0.153298f, + 0.276703f, -0.044481f, 0.078771f, 0.181337f, -0.022962f, 0.153365f, + -0.358004f, 0.314864f, -0.280593f, -0.518572f, 0.392579f, -0.017786f, + 0.127293f, -0.103003f, -0.289389f, -0.871355f, 0.177583f, -0.081290f, + -0.055957f, 0.115011f, -0.402460f, -0.206836f, 0.325328f, 0.169526f, + -0.363311f, -0.624742f, -0.161979f, 0.060679f, 0.267214f, 0.026576f, + -0.318235f, 0.086812f, -0.332419f, -0.272485f, -0.185369f, -0.348598f, + -0.076833f, -0.255184f, -0.081007f, -0.131121f, -0.116196f, -0.142780f, + 0.349705f, 0.173054f, 0.016750f, -0.415957f, -0.461001f, -0.557111f, + 0.738711f, 0.275720f, 0.072868f, -0.276249f, -0.325055f, 0.155285f, + 0.443784f, -0.480153f, -0.127428f, -0.023901f, -0.564837f, 0.238538f, + -0.117578f, 0.542205f, -0.110840f, 0.116025f, -0.323939f, -0.177182f, + -0.331395f, 0.111316f, 0.369140f, -0.168329f, 0.123736f, -0.143013f, + 0.028953f, 0.339200f, 0.034107f, -0.294000f, -0.243034f, -0.048168f, + -0.054348f, -0.245504f, 0.051228f, 0.359128f, -0.071220f, -0.058006f, + -0.624248f, -0.219615f, -0.395067f, -0.109518f, 0.149032f, 0.431928f, + 0.509968f, -0.033143f, -0.090793f, 0.231809f, 0.138986f, 0.216989f, + 0.220683f, -0.419745f, 0.153222f, -0.025956f, -0.215572f, -0.196671f, + 0.363361f, -0.229604f, -0.350704f, 0.060875f, 0.570160f, 0.007246f, + 0.087419f, -0.266043f, 0.474729f, 0.035441f, 0.150312f, -0.269962f, + 0.242166f, 0.110343f, -0.327788f, 0.011268f, -0.127769f, 0.030978f, + -0.071045f, -0.053847f, -0.292886f, -0.091670f, 0.217351f, 0.494707f, + -0.329069f, 0.674122f, 0.432724f, 0.047781f, -0.085408f, -0.198105f, + 0.236135f, -0.196957f, -0.130968f, 0.250552f, 0.123613f, 0.254275f, + 0.143118f, -0.113676f, -0.145703f, 0.225812f, -0.190318f, 0.336481f, + 0.224206f, 0.081584f, 0.000915f, 0.103672f, 1.000000f, -0.031882f, + -0.441377f, 0.543033f, 0.172924f, -0.183717f, 0.742153f, 0.156224f, + 0.083422f, -0.220560f, -0.301964f, -0.501439f, -0.119920f, -0.298610f, + 0.183673f, -0.090064f, 0.501603f, 0.428330f, 0.046506f, -0.080178f, + 0.326700f, -0.325096f, 0.191029f, -0.189729f, -0.113513f, -0.190492f, + 0.163221f, -0.220631f, -0.301576f, 0.156799f, -0.120065f, 0.102529f, + -0.099779f, 0.076429f, -0.727157f, 0.132097f, 0.525583f, 0.294694f, + 0.258287f, -0.067977f, 0.051323f, 0.069258f, 0.027332f, -0.235482f, + -0.099882f, -0.049558f, -0.136291f, 0.237288f, 0.719757f, -0.375235f, + 0.036391f, -0.408991f, 0.369330f, 0.399785f, -0.471419f, 0.551138f, + -0.307569f, 0.064315f, 0.311605f, 0.041736f, 0.650943f, 0.780496f}; + + const float kReference[] = {-0.173553f, -0.265778f, 0.158757f, -0.259399f, + -0.176361f, 0.192877f, 0.056825f, 0.171453f, + 0.050752f, -0.194580f, -0.208679f, 0.153722f}; + + RunBitexactnessTest( + 8000, 1, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} + +TEST(LowCutFilterBitExactnessTest, Stereo8kHzInitial) { + const float kReferenceInput[] = { + 0.790847f, 0.165037f, 0.165494f, 0.709852f, -0.930269f, 0.770840f, + -0.184538f, -0.927236f, 0.492296f, -0.690342f, -0.712183f, 0.211918f, + -0.491038f, -0.351692f, -0.196418f, -0.187253f, -0.227618f, 0.219604f, + -0.666219f, -0.623816f, -0.810742f, -0.353627f, 0.539194f, -0.531764f, + 0.480731f, 0.385637f, 0.648156f, 0.655955f, -0.413264f, -0.381262f, + 0.046060f, -0.349402f, 0.663685f, 0.620590f, 0.113997f, -0.474072f, + 0.361132f, -0.532694f, -0.087149f, -0.230866f, 0.077203f, 0.983407f, + 0.510441f, 0.960910f, -0.530435f, 0.057118f, -0.897128f, 0.513751f, + 0.203960f, 0.714337f, 0.976554f, 0.858969f, -0.180970f, -0.999317f, + 0.081757f, -0.584539f, -0.561433f, -0.348387f, -0.808101f, 0.495067f, + 0.497018f, 0.086599f, -0.323735f, 0.664667f, 0.105144f, 0.915086f, + 0.785667f, -0.286993f, 0.092804f, -0.306636f, 0.245606f, 0.593249f, + 0.491750f, -0.748928f, 0.644788f, -0.949699f, -0.171142f, 0.462815f, + 0.562748f, -0.265428f, 0.489736f, 0.784534f, -0.514793f, -0.740806f, + -0.549864f, -0.299972f, -0.425831f, 0.854976f, -0.897372f, 0.185334f, + -0.674202f, 0.676812f, -0.664878f, 0.004401f, 0.998659f, -0.289186f, + -0.905845f, -0.572679f, -0.204322f, -0.332664f, -0.540795f, 0.872240f, + 0.366378f, 0.924228f, -0.124054f, 0.880673f, -0.988331f, 0.220614f, + 0.602152f, -0.534037f, 0.864937f, 0.526526f, 0.652899f, 0.146927f, + 0.585163f, -0.341918f, -0.553076f, -0.375227f, 0.169047f, 0.659828f, + -0.419075f, -0.194891f, 0.724115f, 0.229479f, 0.982376f, -0.592602f, + 0.654418f, 0.351723f, -0.502101f, -0.048429f, -0.201850f, 0.198876f, + 0.601046f, -0.789862f, 0.642884f, 0.682173f, -0.290988f, -0.139861f, + 0.144478f, 0.401649f, 0.484940f, 0.515768f, -0.221742f, -0.141395f, + 0.912689f, 0.145943f, 0.699444f, -0.447309f, 0.244647f, 0.176723f, + 0.926937f, -0.828195f, 0.000998f, 0.043179f, -0.819668f, 0.809333f, + 0.768778f, -0.122021f, 0.563445f, -0.703070f}; + + const float kReference[] = { + 0.733329f, 0.084109f, 0.072695f, 0.566210f, -1.000000f, 0.652120f, + -0.297424f, -0.964020f, 0.438551f, -0.698364f, -0.654449f, 0.266243f, + 0.454115f, 0.684774f, -0.586823f, -0.747345f, -0.503021f, -0.222961f, + -0.314972f, 0.907224f, -0.796265f, 0.284280f, -0.533417f, 0.773980f}; + + RunBitexactnessTest( + 8000, 2, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} + +TEST(LowCutFilterBitExactnessTest, Stereo8kHzConverged) { + const float kReferenceInput[] = { + -0.502095f, -0.227154f, -0.137133f, 0.661773f, 0.649294f, -0.094003f, + -0.238880f, 0.851737f, 0.481687f, 0.475266f, 0.893832f, 0.020199f, + 0.583758f, -0.095653f, 0.698397f, -0.219138f, 0.476753f, 0.952877f, + 0.046598f, -0.140169f, -0.585684f, -0.353197f, -0.778260f, -0.249580f, + -0.340192f, -0.315790f, 0.634238f, 0.063371f, 0.042244f, 0.548619f, + -0.759474f, 0.250900f, -0.306703f, -0.330761f, 0.149233f, 0.727875f, + -0.602874f, 0.344902f, 0.803663f, -0.601686f, -0.403432f, -0.006959f, + 0.779808f, 0.002829f, -0.446010f, 0.067916f, 0.148499f, -0.174391f, + -0.970473f, 0.405530f, 0.013494f, -0.237468f, -0.870137f, -0.282840f, + -0.531498f, -0.592992f, 0.627559f, -0.213131f, -0.892850f, -0.249897f, + 0.549988f, -0.669405f, 0.824438f, -0.361588f, -0.340441f, -0.591529f, + 0.534429f, -0.860054f, 0.900068f, -0.683580f, -0.427108f, 0.374258f, + -0.717700f, 0.024173f, 0.442654f, 0.857690f, 0.464208f, 0.499696f, + -0.185361f, -0.521017f, 0.041701f, -0.561845f, 0.684776f, 0.325866f, + 0.632471f, 0.587755f, -0.061790f, -0.380950f, 0.375158f, 0.973704f, + 0.539868f, 0.659162f, 0.412170f, 0.190673f, 0.505748f, -0.006556f, + 0.730265f, -0.863945f, 0.937092f, -0.802487f, 0.093954f, -0.194060f, + -0.785920f, 0.448332f, 0.227365f, 0.565936f, 0.133241f, 0.622638f, + 0.153552f, 0.888058f, 0.742904f, 0.015204f, 0.577646f, -0.053939f, + 0.657603f, -0.355037f, 0.952293f, -0.443578f, -0.854338f, 0.502447f, + 0.662377f, 0.844676f, -0.345951f, 0.608139f, 0.076501f, -0.073410f, + 0.641501f, 0.903813f, -0.847454f, 0.417342f, -0.530147f, -0.202209f, + -0.463751f, 0.665027f, 0.990748f, 0.299502f, 0.407906f, 0.864606f, + 0.375305f, 0.136708f, -0.238305f, 0.269159f, -0.273543f, -0.184761f, + -0.262601f, -0.063202f, 0.006828f, 0.821072f, -0.587138f, -0.322793f, + 0.148251f, -0.026135f, -0.475562f, 0.159187f, 0.756655f, -0.878100f, + -0.118247f, -0.831484f, 0.126475f, 0.078621f, 0.536116f, -0.533819f, + 0.174723f, -0.082052f, 0.721963f, 0.321672f, -0.292242f, -0.305627f, + -0.492564f, 0.905056f, -0.403598f, -0.683188f, -0.277406f, 0.483258f, + 0.411800f, 0.401784f, -0.987548f, -0.251309f, 0.802991f, -0.363310f, + 0.194166f, -0.404410f, -0.749971f, -0.223289f, 0.635375f, 0.962351f, + 0.723980f, -0.832358f, -0.324576f, -0.527742f, -0.364389f, 0.968897f, + 0.096502f, 0.498503f, 0.683703f, -0.666221f, 0.806195f, -0.789752f, + 0.490186f, 0.458744f, 0.434939f, -0.733136f, -0.108422f, 0.017574f, + 0.060981f, 0.719434f, 0.355450f, 0.611677f, 0.062486f, 0.911792f, + -0.866646f, 0.083036f, -0.436679f, -0.038199f, 0.369728f, -0.583483f, + 0.216322f, -0.347648f, 0.761694f, -0.733211f, -0.795184f, 0.918234f, + -0.694196f, -0.694924f, -0.688895f, -0.820861f, -0.091151f, 0.337791f, + 0.662603f, 0.580470f, 0.425422f, -0.054805f, 0.417176f, 0.916119f, + 0.011551f, -0.389894f, 0.579622f, -0.527226f, -0.531394f, -0.070601f, + 0.238774f, 0.230659f, -0.754752f, -0.752413f, -0.431082f, 0.471466f, + -0.177384f, 0.657964f, 0.870228f, -0.201867f, -0.895577f, 0.142372f, + 0.495340f, -0.359513f, -0.014131f, -0.556694f, 0.878547f, -0.035389f, + 0.079992f, -0.557886f, -0.808110f, -0.879669f, 0.639018f, 0.542957f, + -0.608609f, 0.790236f, 0.368600f, 0.313693f, 0.980762f, -0.932616f, + -0.151493f, -0.020033f, 0.167009f, -0.833461f, 0.320309f, -0.895390f, + 0.113661f, 0.424050f, -0.024179f, 0.235201f, -0.572445f, 0.291317f, + -0.238715f, -0.792574f, -0.244977f, -0.474278f, -0.517429f, 0.245848f, + 0.045856f, -0.173525f, -0.564416f, 0.717107f, 0.722017f, -0.432122f, + 0.230786f, 0.558979f, 0.909695f, 0.839206f, -0.230369f, -0.674714f, + 0.593503f, -0.772366f, -0.682351f, -0.288344f, 0.695517f, 0.165562f, + 0.172355f, 0.851676f, 0.150157f, -0.980045f, 0.618755f, 0.217617f, + -0.040173f, -0.463120f, -0.483807f, -0.037981f, -0.545317f, -0.902795f, + -0.661516f, -0.483107f, -0.604180f, 0.211386f, 0.647407f, 0.621230f, + 0.604474f, 0.416227f, 0.718756f, 0.562169f, -0.592406f, 0.986686f, + -0.812751f, 0.301237f, -0.569647f, -0.512254f, -0.320624f, -0.604275f, + 0.013667f, 0.901516f, -0.210786f, 0.168930f, 0.213074f, 0.429286f, + -0.196927f, 0.717382f, 0.840970f, 0.501678f, -0.428817f, 0.593632f, + -0.714468f, 0.009100f, 0.221376f, 0.407593f, -0.233320f, 0.457367f, + 0.774569f, -0.888303f, -0.723567f, 0.726130f, -0.156507f, -0.177372f, + 0.918283f, 0.500491f, 0.961994f, -0.532968f, -0.807546f, -0.230836f, + 0.000545f, 0.140512f, 0.953263f, -0.014290f, -0.198234f, 0.989981f, + -0.478004f, 0.330649f, 0.928513f, 0.342302f, -0.401650f, 0.062253f, + -0.997074f, 0.767578f, -0.191232f, -0.397589f, 0.901163f, -0.078704f, + -0.424705f, -0.830747f, 0.164363f, -0.693863f, -0.853811f, 0.161130f, + -0.425970f, -0.276160f, 0.449649f, 0.716623f, -0.304169f, 0.923491f, + 0.907138f, -0.587925f, 0.536490f, 0.231064f, 0.837845f, 0.205075f, + 0.404276f, 0.487350f, -0.229795f, -0.496992f, -0.926481f, -0.055754f, + 0.290145f, -0.442060f, 0.035722f, -0.508667f, -0.404984f, 0.300948f, + 0.782787f, 0.722213f, -0.580170f, -0.201812f, 0.775766f, -0.486944f, + 0.933603f, 0.238315f, -0.669308f, 0.652398f, 0.311386f, 0.092905f, + -0.497341f, -0.919687f, -0.533249f, -0.277774f, 0.266910f, 0.972196f, + -0.585687f, 0.514168f, 0.772656f, -0.055540f, -0.682173f, 0.621842f, + -0.046984f, -0.767425f, 0.751441f, 0.270373f, -0.805437f, 0.816879f, + -0.929968f, -0.920501f, 0.977136f, 0.372363f, -0.246622f, 0.008649f, + 0.526991f, -0.902250f, 0.451855f, 0.402656f, -0.082218f, 0.164590f, + -0.321820f, -0.658749f, -0.201613f, 0.839554f, -0.547909f, -0.277987f, + -0.350876f, -0.832836f, 0.025331f, 0.665730f, 0.809227f, 0.447192f, + -0.234008f, -0.403966f, 0.383423f, 0.760914f, 0.849097f, -0.837494f, + -0.034654f, -0.743470f, -0.494178f, 0.767923f, -0.607446f, -0.757293f}; + + const float kReference[] = { + -0.544495f, 0.264199f, 0.647938f, 0.565569f, 0.496231f, 0.271340f, + 0.519944f, 0.318094f, -0.792999f, 0.733421f, -1.000000f, 0.103977f, + 0.981719f, 0.314859f, 0.476882f, 0.514267f, -0.196381f, -0.425781f, + -0.783081f, 0.101108f, 0.419782f, -0.291718f, 0.183355f, -0.332489f}; + + RunBitexactnessTest( + 8000, 2, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} + +TEST(LowCutFilterBitExactnessTest, Mono16kHzInitial) { + const float kReferenceInput[] = { + 0.150254f, 0.512488f, -0.631245f, 0.240938f, 0.089080f, -0.365440f, + -0.121169f, 0.095748f, 1.000000f, 0.773932f, -0.377232f, 0.848124f, + 0.202718f, -0.017621f, 0.199738f, -0.057279f, -0.034693f, 0.416303f, + 0.393761f, 0.396041f, 0.187653f, -0.337438f, 0.200436f, 0.455577f, + 0.136624f, 0.289150f, 0.203131f, -0.084798f, 0.082124f, -0.220010f, + 0.248266f, -0.320554f, -0.298701f, -0.226218f, -0.822794f, 0.401962f, + 0.090876f, -0.210968f, 0.382936f, -0.478291f, -0.028572f, -0.067474f, + 0.089204f, 0.087430f, -0.241695f, -0.008398f, -0.046076f, 0.175416f, + 0.305518f, 0.309992f, -0.241352f, 0.021618f, -0.339291f, -0.311173f, + -0.001914f, 0.428301f, -0.215087f, 0.103784f, -0.063041f, 0.312250f, + -0.304344f, 0.009098f, 0.154406f, 0.307571f, 0.431537f, 0.024014f, + -0.416832f, -0.207440f, -0.296664f, 0.656846f, -0.172033f, 0.209054f, + -0.053772f, 0.248326f, -0.213741f, -0.391871f, -0.397490f, 0.136428f, + -0.049568f, -0.054788f, 0.396633f, 0.081485f, 0.055279f, 0.443690f, + -0.224812f, 0.194675f, 0.233369f, -0.068107f, 0.060270f, -0.325801f, + -0.320801f, 0.029308f, 0.201837f, 0.722528f, -0.186366f, 0.052351f, + -0.023053f, -0.540192f, -0.122671f, -0.501532f, 0.234847f, -0.248165f, + 0.027971f, -0.152171f, 0.084820f, -0.167764f, 0.136923f, 0.206619f, + 0.478395f, -0.054249f, -0.597574f, -0.234627f, 0.378548f, -0.299619f, + 0.268543f, 0.034666f, 0.401492f, -0.547983f, -0.055248f, -0.337538f, + 0.812657f, 0.230611f, 0.385360f, -0.295713f, -0.130957f, -0.076143f, + 0.306960f, -0.077653f, 0.196049f, -0.573390f, -0.098885f, -0.230155f, + -0.440716f, 0.141956f, 0.078802f, 0.009356f, -0.372703f, 0.315083f, + 0.097859f, -0.083575f, 0.006397f, -0.073216f, -0.489105f, -0.079827f, + -0.232329f, -0.273644f, -0.323162f, -0.149105f, -0.559646f, 0.269458f, + 0.145333f, -0.005597f, -0.009717f, -0.223051f, 0.284676f, -0.037228f, + -0.199679f, 0.377651f, -0.062813f, -0.164607f}; + + const float kReference[] = {0.147160f, 0.495163f, -0.648346f, 0.234931f, + 0.075289f, -0.373779f, -0.117676f, 0.100345f, + 0.981719f, 0.714896f, -0.447357f, 0.770867f}; + + RunBitexactnessTest( + 16000, 1, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} + +TEST(LowCutFilterBitExactnessTest, Mono16kHzConverged) { + const float kReferenceInput[] = { + 0.150254f, 0.512488f, -0.631245f, 0.240938f, 0.089080f, -0.365440f, + -0.121169f, 0.095748f, 1.000000f, 0.773932f, -0.377232f, 0.848124f, + 0.202718f, -0.017621f, 0.199738f, -0.057279f, -0.034693f, 0.416303f, + 0.393761f, 0.396041f, 0.187653f, -0.337438f, 0.200436f, 0.455577f, + 0.136624f, 0.289150f, 0.203131f, -0.084798f, 0.082124f, -0.220010f, + 0.248266f, -0.320554f, -0.298701f, -0.226218f, -0.822794f, 0.401962f, + 0.090876f, -0.210968f, 0.382936f, -0.478291f, -0.028572f, -0.067474f, + 0.089204f, 0.087430f, -0.241695f, -0.008398f, -0.046076f, 0.175416f, + 0.305518f, 0.309992f, -0.241352f, 0.021618f, -0.339291f, -0.311173f, + -0.001914f, 0.428301f, -0.215087f, 0.103784f, -0.063041f, 0.312250f, + -0.304344f, 0.009098f, 0.154406f, 0.307571f, 0.431537f, 0.024014f, + -0.416832f, -0.207440f, -0.296664f, 0.656846f, -0.172033f, 0.209054f, + -0.053772f, 0.248326f, -0.213741f, -0.391871f, -0.397490f, 0.136428f, + -0.049568f, -0.054788f, 0.396633f, 0.081485f, 0.055279f, 0.443690f, + -0.224812f, 0.194675f, 0.233369f, -0.068107f, 0.060270f, -0.325801f, + -0.320801f, 0.029308f, 0.201837f, 0.722528f, -0.186366f, 0.052351f, + -0.023053f, -0.540192f, -0.122671f, -0.501532f, 0.234847f, -0.248165f, + 0.027971f, -0.152171f, 0.084820f, -0.167764f, 0.136923f, 0.206619f, + 0.478395f, -0.054249f, -0.597574f, -0.234627f, 0.378548f, -0.299619f, + 0.268543f, 0.034666f, 0.401492f, -0.547983f, -0.055248f, -0.337538f, + 0.812657f, 0.230611f, 0.385360f, -0.295713f, -0.130957f, -0.076143f, + 0.306960f, -0.077653f, 0.196049f, -0.573390f, -0.098885f, -0.230155f, + -0.440716f, 0.141956f, 0.078802f, 0.009356f, -0.372703f, 0.315083f, + 0.097859f, -0.083575f, 0.006397f, -0.073216f, -0.489105f, -0.079827f, + -0.232329f, -0.273644f, -0.323162f, -0.149105f, -0.559646f, 0.269458f, + 0.145333f, -0.005597f, -0.009717f, -0.223051f, 0.284676f, -0.037228f, + -0.199679f, 0.377651f, -0.062813f, -0.164607f, -0.082091f, -0.236957f, + -0.313025f, 0.705903f, 0.462637f, 0.085942f, -0.351308f, -0.241859f, + -0.049333f, 0.221165f, -0.372235f, -0.651092f, -0.404957f, 0.093201f, + 0.109366f, 0.126224f, -0.036409f, 0.051333f, -0.133063f, 0.240896f, + -0.380532f, 0.127160f, -0.237176f, -0.093586f, 0.154478f, 0.290379f, + -0.312329f, 0.352297f, 0.184480f, -0.018965f, -0.054555f, -0.060811f, + -0.084705f, 0.006440f, 0.014333f, 0.230847f, 0.426721f, 0.130481f, + -0.058605f, 0.174712f, 0.051204f, -0.287773f, 0.265265f, 0.085810f, + 0.037775f, 0.143988f, 0.073051f, -0.263103f, -0.045366f, -0.040816f, + -0.148673f, 0.470072f, -0.244727f, -0.135204f, -0.198973f, -0.328139f, + -0.053722f, -0.076590f, 0.427586f, -0.069591f, -0.297399f, 0.448094f, + 0.345037f, -0.064170f, -0.420903f, -0.124253f, -0.043578f, 0.077149f, + -0.072983f, 0.123916f, 0.109517f, -0.349508f, -0.264912f, -0.207106f, + -0.141912f, -0.089586f, 0.003485f, -0.846518f, -0.127715f, 0.347208f, + -0.298095f, 0.260935f, 0.097899f, -0.008106f, 0.050987f, -0.437362f, + -0.023625f, 0.448230f, 0.027484f, 0.011562f, -0.205167f, -0.008611f, + 0.064930f, 0.119156f, -0.104183f, -0.066078f, 0.565530f, -0.631108f, + 0.623029f, 0.094334f, 0.279472f, -0.465059f, -0.164888f, -0.077706f, + 0.118130f, -0.466746f, 0.131800f, -0.338936f, 0.018497f, 0.182304f, + 0.091398f, 0.302547f, 0.281153f, -0.181899f, 0.071836f, -0.263911f, + -0.369380f, 0.258447f, 0.000014f, -0.015347f, 0.254619f, 0.166159f, + 0.097865f, 0.349389f, 0.259834f, 0.067003f, -0.192925f, -0.182080f, + 0.333139f, -0.450434f, -0.006836f, -0.544615f, 0.285183f, 0.240811f, + 0.000325f, -0.019796f, -0.694804f, 0.162411f, -0.612686f, -0.648134f, + 0.022338f, -0.265058f, 0.114993f, 0.189185f, 0.239697f, -0.193148f, + 0.125581f, 0.028122f, 0.230849f, 0.149832f, 0.250919f, -0.036871f, + -0.041136f, 0.281627f, -0.593466f, -0.141009f, -0.355074f, -0.106915f, + 0.181276f, 0.230753f, -0.283631f, -0.131643f, 0.038292f, -0.081563f, + 0.084345f, 0.111763f, -0.259882f, -0.049416f, -0.595824f, 0.320077f, + -0.175802f, -0.336422f, -0.070966f, -0.399242f, -0.005829f, -0.156680f, + 0.608591f, 0.318150f, -0.697767f, 0.123331f, -0.390716f, -0.071276f, + 0.045943f, 0.208958f, -0.076304f, 0.440505f, -0.134400f, 0.091525f, + 0.185763f, 0.023806f, 0.246186f, 0.090323f, -0.219133f, -0.504520f, + 0.519393f, -0.168939f, 0.028884f, 0.157380f, 0.031745f, -0.252830f, + -0.130705f, -0.034901f, 0.413302f, -0.240559f, 0.219279f, 0.086246f, + -0.065353f, -0.295376f, -0.079405f, -0.024226f, -0.410629f, 0.053706f, + -0.229794f, -0.026336f, 0.093956f, -0.252810f, -0.080555f, 0.097827f, + -0.513040f, 0.289508f, 0.677527f, 0.268109f, -0.088244f, 0.119781f, + -0.289511f, 0.524778f, 0.262884f, 0.220028f, -0.244767f, 0.089411f, + -0.156018f, -0.087030f, -0.159292f, -0.286646f, -0.253953f, -0.058657f, + -0.474756f, 0.169797f, -0.032919f, 0.195384f, 0.075355f, 0.138131f, + -0.414465f, -0.285118f, -0.124915f, 0.030645f, 0.315431f, -0.081032f, + 0.352546f, 0.132860f, 0.328112f, 0.035476f, -0.183550f, -0.413984f, + 0.043452f, 0.228748f, -0.081765f, -0.151125f, -0.086251f, -0.306448f, + -0.137774f, -0.050508f, 0.012811f, -0.017824f, 0.170841f, 0.030549f, + 0.506935f, 0.087197f, 0.504274f, -0.202080f, 0.147146f, -0.072728f, + 0.167713f, 0.165977f, -0.610894f, -0.370849f, -0.402698f, 0.112297f, + 0.410855f, -0.091330f, 0.227008f, 0.152454f, -0.293884f, 0.111074f, + -0.210121f, 0.423728f, -0.009101f, 0.457188f, -0.118785f, 0.164720f, + -0.017547f, -0.565046f, -0.274461f, 0.171169f, -0.015338f, -0.312635f, + -0.175044f, 0.069729f, -0.277504f, 0.272454f, -0.179049f, 0.505495f, + -0.301774f, 0.055664f, -0.425058f, -0.202222f, -0.165787f, 0.112155f, + 0.263284f, 0.083972f, -0.104256f, 0.227892f, 0.223253f, 0.033592f, + 0.159638f, 0.115358f, -0.275811f, 0.212265f, -0.183658f, -0.168768f}; + + const float kReference[] = {-0.248962f, -0.088257f, 0.083041f, -0.037323f, + 0.127659f, 0.149388f, -0.220978f, -0.004242f, + -0.538544f, 0.384289f, -0.117615f, -0.268524f}; + + RunBitexactnessTest( + 16000, 1, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} + +TEST(LowCutFilterBitExactnessTest, Stereo16kHzInitial) { + const float kReferenceInput[] = { + 0.087390f, -0.370759f, -0.235918f, 0.583079f, 0.678359f, 0.360473f, + -0.166156f, 0.285780f, -0.571837f, 0.234542f, 0.350382f, 0.202047f, + -0.307381f, -0.271197f, -0.657038f, 0.590723f, -0.014666f, -0.290754f, + 0.550122f, -0.526390f, 0.689667f, 0.633054f, 0.692457f, -0.259626f, + -0.233541f, 0.722669f, -0.072182f, 0.141096f, 0.390614f, 0.921835f, + 0.092626f, 0.273153f, 0.141785f, 0.854224f, 0.727531f, -0.660321f, + -0.642602f, -0.512991f, 0.503559f, -0.601731f, 0.965881f, 0.419277f, + -0.649128f, 0.716595f, 0.818823f, 0.923326f, 0.141199f, 0.125758f, + -0.646678f, 0.027358f, 0.096944f, -0.669445f, -0.012214f, 0.070235f, + -0.602386f, 0.246338f, -0.947369f, -0.362418f, 0.065999f, -0.346453f, + 0.204381f, -0.276135f, -0.730159f, 0.827627f, 0.281118f, 0.317548f, + 0.350661f, 0.489115f, 0.684355f, 0.033314f, -0.696263f, -0.238671f, + 0.642039f, -0.657271f, -0.340049f, 0.932944f, 0.612585f, -0.555624f, + 0.999546f, -0.872523f, -0.149034f, -0.191324f, -0.199414f, -0.776155f, + -0.151378f, 0.227092f, 0.976123f, -0.560198f, -0.291838f, -0.467516f, + -0.417004f, -0.623221f, -0.954281f, -0.101192f, -0.512720f, 0.737453f, + 0.057222f, 0.828270f, 0.947860f, 0.170852f, -0.762049f, 0.853065f, + 0.187122f, 0.767231f, -0.151048f, 0.214515f, -0.858473f, 0.849545f, + 0.284159f, -0.791001f, 0.400450f, -0.208391f, -0.830190f, -0.571042f, + -0.502402f, -0.546694f, 0.406009f, 0.508305f, 0.094573f, 0.106967f, + 0.261146f, 0.970914f, 0.268556f, 0.200911f, 0.818374f, 0.141673f, + -0.329160f, 0.914278f, -0.120154f, 0.203085f, 0.440525f, 0.357557f, + -0.574482f, -0.836753f, -0.451041f, 0.735037f, 0.118714f, -0.070744f, + -0.139398f, 0.547972f, 0.307841f, 0.315459f, -0.677958f, -0.135246f, + 0.010172f, -0.249335f, -0.039256f, -0.315157f, 0.554293f, -0.232112f, + 0.423113f, -0.038133f, 0.458360f, 0.875118f, 0.034509f, 0.806137f, + -0.563615f, 0.746439f, -0.834614f, -0.069193f, -0.956140f, 0.616561f, + -0.641581f, -0.669216f, -0.636793f, 0.382873f, -0.572473f, -0.403790f, + 0.536670f, 0.002300f, 0.818930f, -0.884294f, -0.126496f, 0.144509f, + 0.130134f, 0.647633f, -0.747802f, -0.399766f, -0.995756f, 0.902215f, + 0.532599f, 0.502608f, -0.722270f, -0.301361f, -0.697319f, -0.006559f, + 0.617305f, 0.265738f, 0.376803f, 0.279140f, 0.458643f, 0.719691f, + 0.253911f, -0.638817f, 0.146613f, -0.672868f, 0.812103f, -0.845314f, + -0.322931f, 0.161235f, -0.049530f, 0.610641f, 0.061556f, -0.545379f, + 0.418970f, -0.702735f, 0.316232f, 0.267965f, -0.541387f, -0.635544f, + -0.667295f, -0.700786f, -0.594505f, 0.909918f, -0.968183f, 0.915029f, + -0.948615f, 0.942221f, -0.404809f, 0.050146f, 0.724678f, 0.792810f, + -0.621979f, 0.321439f, 0.882462f, 0.951414f, -0.784129f, -0.642202f, + 0.493103f, -0.901063f, -0.857430f, -0.021749f, 0.699788f, 0.994083f, + -0.991215f, 0.085215f, 0.722696f, 0.818278f, 0.690701f, 0.757746f, + 0.492364f, -0.765021f, 0.018045f, -0.662336f, 0.662223f, 0.856022f, + -0.661031f, 0.767475f, -0.224274f, -0.234861f, -0.457094f, 0.735766f, + 0.483005f, -0.104255f, 0.419278f, 0.888663f, -0.651764f, -0.510807f, + 0.281858f, 0.617225f, 0.706742f, -0.203765f, -0.769012f, -0.839438f, + -0.279065f, 0.657811f, -0.570781f, 0.582081f, 0.309377f, -0.947707f, + 0.571553f, 0.845126f, -0.015374f, 0.668023f, -0.737293f, 0.519567f, + 0.851472f, 0.665415f, -0.481198f, -0.573956f, 0.044630f, -0.205286f, + -0.041780f, 0.987807f, 0.208957f, 0.889817f, -0.019116f, -0.124107f, + 0.545311f, 0.488133f, -0.114192f, -0.894000f, -0.824356f, 0.595972f, + 0.311165f, -0.935329f, 0.114134f, 0.439603f, -0.779184f, -0.566705f, + 0.622040f, -0.722676f, 0.763798f, 0.847112f, -0.974489f, -0.245681f, + -0.664377f, 0.080446f, -0.796675f, -0.921465f, 0.866458f, 0.943184f, + -0.278144f, 0.288411f, -0.864105f, -0.584176f, -0.920792f, -0.061281f, + -0.699807f, 0.982614f}; + + const float kReference[] = { + 0.085604f, -0.367126f, -0.218170f, 0.594653f, 0.661245f, 0.319041f, + -0.212891f, 0.237800f, -0.614716f, 0.201758f, 0.305032f, 0.144414f, + -0.936523f, 0.647359f, -0.613403f, -0.611542f, -0.549835f, 0.477004f, + -0.477386f, -0.287262f, 0.650746f, 0.101169f, 0.899258f, -0.808014f}; + + RunBitexactnessTest( + 16000, 2, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} + +TEST(LowCutFilterBitExactnessTest, Stereo16kHzConverged) { + const float kReferenceInput[] = { + -0.145875f, 0.910744f, 0.448494f, 0.161783f, 0.080516f, 0.410882f, + -0.989942f, 0.565032f, 0.853719f, -0.983409f, 0.649257f, 0.534672f, + 0.994274f, -0.544694f, 0.839084f, 0.283999f, -0.789360f, -0.463678f, + 0.527688f, 0.611020f, -0.791494f, -0.060482f, -0.561876f, 0.845416f, + -0.359355f, 0.715088f, -0.480307f, 0.756126f, -0.623465f, 0.518388f, + -0.936621f, 0.284678f, 0.133742f, -0.247181f, -0.574903f, 0.584314f, + -0.709113f, -0.021715f, -0.974309f, -0.626776f, -0.029539f, 0.676452f, + -0.717886f, 0.464434f, 0.382134f, -0.931015f, -0.022285f, 0.942781f, + -0.775097f, 0.486428f, 0.277083f, 0.188366f, -0.002755f, 0.135705f, + -0.146991f, -0.847521f, -0.418827f, 0.122670f, 0.266667f, 0.861552f, + 0.955538f, -0.812807f, 0.323470f, 0.205546f, -0.052364f, -0.287487f, + -0.048843f, 0.342044f, 0.919290f, -0.821831f, 0.595485f, 0.181551f, + 0.824394f, -0.797741f, -0.413411f, -0.896824f, 0.008256f, 0.536752f, + -0.434029f, -0.549280f, -0.337421f, -0.093497f, 0.474769f, 0.019771f, + -0.234972f, 0.810966f, 0.930515f, 0.256535f, -0.735938f, 0.236604f, + -0.233960f, 0.982387f, -0.426345f, 0.412383f, 0.070412f, -0.613578f, + 0.378870f, -0.899090f, -0.631132f, -0.908683f, 0.770083f, 0.679589f, + -0.763690f, -0.179170f, -0.759543f, 0.144185f, 0.898780f, -0.487230f, + 0.979731f, -0.300384f, -0.582955f, 0.331654f, 0.946689f, 0.245400f, + -0.872924f, -0.252981f, -0.667497f, -0.537444f, -0.895583f, 0.803513f, + 0.586583f, -0.253971f, 0.664109f, 0.507669f, 0.243726f, -0.211814f, + -0.281444f, -0.822295f, -0.316646f, 0.097341f, -0.078905f, 0.290905f, + 0.027042f, 0.628853f, -0.805634f, -0.072573f, 0.179635f, -0.625656f, + 0.222660f, -0.896116f, 0.151454f, 0.684689f, -0.000548f, -0.121950f, + -0.701886f, -0.943441f, 0.513340f, 0.592212f, -0.412889f, -0.769587f, + -0.249817f, 0.657787f, 0.683553f, 0.330477f, 0.920280f, 0.886236f, + -0.774601f, 0.296575f, -0.038392f, -0.866959f, 0.795542f, -0.005540f, + 0.542607f, -0.879276f, -0.475085f, 0.302139f, -0.732792f, 0.277091f, + -0.230114f, 0.531396f, 0.305831f, -0.237022f, -0.399963f, -0.319721f, + 0.837853f, -0.087466f, -0.115006f, -0.091628f, 0.890564f, -0.561762f, + 0.764806f, -0.960249f, -0.316470f, 0.532055f, -0.314393f, 0.237613f, + -0.093958f, -0.979675f, 0.198162f, 0.203137f, 0.298835f, -0.314559f, + -0.013401f, 0.403548f, 0.775605f, -0.889884f, -0.803276f, 0.299566f, + 0.528142f, 0.975918f, -0.749350f, -0.271046f, 0.352460f, -0.248484f, + 0.726917f, -0.416046f, -0.733050f, 0.345301f, -0.594830f, 0.737030f, + 0.502315f, -0.161241f, -0.999538f, -0.701073f, -0.452331f, 0.744850f, + 0.202502f, -0.357623f, -0.431414f, -0.129368f, 0.807518f, 0.850211f, + 0.010585f, 0.255164f, 0.438528f, -0.952174f, 0.149865f, -0.906931f, + -0.154937f, -0.064531f, -0.954744f, -0.869852f, 0.847913f, 0.068286f, + -0.266407f, -0.272108f, -0.697253f, -0.700783f, -0.298396f, -0.328068f, + 0.568056f, -0.026522f, -0.070404f, -0.737495f, 0.772783f, 0.349115f, + 0.670319f, 0.312976f, 0.967834f, 0.959580f, -0.499694f, 0.249141f, + 0.456485f, -0.003659f, 0.699657f, -0.618164f, -0.751712f, -0.994419f, + -0.694094f, 0.068322f, 0.021267f, -0.229568f, -0.378807f, -0.992889f, + 0.630485f, 0.276837f, -0.103321f, -0.511828f, 0.606770f, 0.647942f, + 0.704381f, -0.065496f, 0.941398f, 0.682488f, -0.842904f, -0.524802f, + 0.635142f, -0.188343f, -0.067376f, 0.903072f, 0.930011f, 0.530570f, + 0.149067f, 0.831850f, -0.009135f, -0.667975f, -0.348005f, -0.407128f, + 0.116597f, -0.865046f, -0.862044f, -0.666431f, 0.894877f, 0.622177f, + 0.420911f, 0.940491f, 0.996854f, 0.974910f, -0.699827f, 0.916958f, + 0.060918f, -0.851827f, -0.376358f, 0.790342f, 0.669537f, -0.995302f, + 0.280420f, 0.606365f, -0.509738f, -0.871756f, -0.473703f, -0.794559f, + -0.032562f, -0.162231f, -0.237422f, 0.773530f, -0.158885f, -0.432304f, + -0.903638f, -0.561668f, -0.521648f, -0.941483f, 0.404622f, -0.984729f, + 0.221841f, -0.183821f, -0.502107f, 0.304919f, -0.359446f, -0.792656f, + 0.071130f, -0.670260f, 0.766877f, 0.332914f, 0.695485f, 0.525322f, + 0.614028f, 0.265905f, 0.420855f, 0.377327f, -0.358104f, 0.063297f, + 0.746388f, -0.890921f, 0.000802f, -0.134474f, 0.808565f, 0.260367f, + 0.966072f, 0.170401f, 0.681273f, -0.062372f, 0.090445f, -0.641792f, + 0.268923f, 0.925918f, 0.068028f, -0.040771f, 0.587332f, -0.814573f, + 0.761599f, -0.992253f, 0.023058f, 0.356927f, 0.131495f, -0.043083f, + -0.358974f, 0.203160f, 0.826305f, 0.365036f, 0.893467f, -0.801822f, + 0.022058f, -0.779743f, 0.090524f, 0.377572f, -0.705166f, 0.555122f, + -0.201898f, 0.796600f, -0.385912f, -0.877898f, -0.561058f, -0.834334f, + 0.900791f, -0.967259f, -0.770663f, -0.975180f, -0.567545f, -0.977145f, + 0.284899f, 0.033982f, -0.508916f, -0.612505f, -0.818259f, -0.263117f, + -0.984414f, 0.205403f, -0.042291f, -0.383765f, 0.488889f, 0.678699f, + -0.475136f, 0.028476f, -0.106452f, -0.317578f, 0.678284f, 0.964985f, + 0.252929f, -0.637450f, -0.753966f, 0.159937f, -0.342928f, -0.463627f, + 0.100478f, -0.638966f, 0.356984f, -0.888623f, -0.931886f, -0.426963f, + -0.845220f, 0.801145f, 0.693212f, -0.208603f, -0.661569f, -0.139095f, + -0.167564f, 0.457527f, -0.187053f, 0.903615f, 0.823970f, 0.902829f, + -0.307998f, -0.419512f, 0.773402f, -0.579938f, -0.738247f, 0.041032f, + 0.810925f, -0.194940f, -0.568477f, -0.842521f, 0.866120f, 0.205743f, + -0.245016f, 0.329863f, 0.584381f, -0.333016f, 0.385318f, -0.592369f, + 0.917427f, 0.423665f, -0.666187f, -0.114446f, 0.265987f, 0.859934f, + 0.058662f, 0.252949f, 0.361638f, 0.846395f, -0.694332f, -0.188558f, + -0.375048f, 0.387798f, 0.781376f, -0.018658f, 0.611647f, -0.347122f, + 0.099758f, -0.222431f, 0.793658f, 0.352240f, 0.656794f, -0.779822f, + -0.441545f, 0.535272f, -0.567887f, -0.931876f, -0.126896f, 0.873727f, + -0.475822f, 0.139491f, -0.280894f, -0.946323f, 0.000838f, 0.654030f, + -0.482035f, -0.908230f, -0.507057f, 0.321464f, -0.341181f, 0.318992f, + -0.973992f, 0.436136f, -0.217762f, -0.932989f, -0.187969f, 0.432615f, + 0.842673f, 0.968031f, 0.966842f, 0.792612f, 0.731406f, 0.601922f, + 0.109958f, -0.162256f, -0.745755f, 0.309241f, 0.727930f, -0.450803f, + 0.680328f, -0.858490f, -0.242416f, -0.463661f, -0.694158f, 0.261999f, + -0.367250f, 0.918224f, -0.002652f, 0.477217f, -0.974489f, 0.210706f, + 0.152903f, 0.614758f, 0.309936f, 0.756457f, 0.804746f, -0.695534f, + -0.614840f, 0.581951f, -0.878590f, -0.220346f, -0.400068f, 0.468360f, + -0.791581f, 0.585151f, 0.565458f, 0.064795f, -0.493295f, -0.858091f, + 0.251607f, -0.950637f, -0.875915f, -0.740776f, -0.098772f, 0.344672f, + 0.712222f, -0.003109f, -0.902431f, -0.372335f, 0.283262f, 0.572773f, + -0.421699f, -0.004264f, 0.636869f, 0.190257f, 0.072849f, -0.338254f, + -0.176620f, 0.588012f, -0.313584f, -0.074787f, -0.264353f, 0.359141f, + 0.135558f, 0.303554f, -0.017773f, -0.203084f, -0.045032f, -0.866825f, + -0.177943f, 0.938184f, 0.561442f, 0.458036f, 0.531301f, 0.513162f, + 0.686541f, 0.540314f, 0.957322f, -0.777281f, -0.207846f, -0.015879f, + -0.483811f, -0.926068f, 0.948763f, 0.452852f, -0.704070f, -0.704211f, + 0.409648f, -0.238013f, -0.847177f, -0.178319f, -0.714019f, 0.597840f, + 0.860496f, -0.990561f, 0.300081f, 0.357065f, -0.492754f, 0.686362f, + -0.412082f, -0.946279f, -0.813386f, 0.595770f, 0.422805f, 0.566814f, + 0.247845f, 0.650831f, -0.929955f, -0.189050f, -0.500662f, -0.038206f, + 0.761678f, -0.438630f, 0.198285f, -0.947548f, -0.689603f, 0.667822f, + -0.610213f, 0.659576f, -0.323850f, 0.342233f, -0.895267f, 0.468618f, + -0.001036f, 0.886600f, -0.420455f, -0.246879f, -0.772489f, 0.929701f, + -0.134977f, -0.830874f, 0.433353f, 0.013575f, -0.343825f, 0.507048f, + 0.672012f, -0.492567f, 0.068850f, -0.129670f, -0.684592f, 0.200962f, + 0.874902f, -0.784483f, 0.799963f, 0.100930f, -0.145287f, -0.695238f, + -0.504908f, -0.105262f, 0.065567f, -0.290698f, 0.546230f, 0.763362f, + 0.468184f, -0.187136f, 0.208357f, 0.282210f, -0.745066f, -0.007616f, + -0.379061f, 0.157149f, 0.887218f, -0.146121f, -0.933743f, 0.858868f, + 0.849965f, -0.283386f, -0.480022f, 0.573719f, 0.023164f, 0.125054f, + 0.369588f, -0.815207f, 0.745158f, 0.885876f, -0.806812f, 0.691765f, + 0.818791f, -0.977318f, 0.047365f, 0.300691f, -0.229709f, 0.298604f, + 0.525707f, 0.151372f, 0.263838f, -0.443592f, 0.679673f, -0.146330f, + 0.263245f, 0.666934f, -0.459629f, -0.198399f, 0.108509f, -0.112269f, + -0.819232f, 0.488763f, -0.934769f, -0.140515f, -0.925475f, 0.951596f, + 0.044680f, 0.819260f, -0.233504f, 0.768904f, -0.489965f, 0.818100f, + 0.789121f, -0.202966f, 0.250040f, 0.135195f, 0.789024f, -0.571668f, + -0.992282f, 0.761163f, -0.529757f, -0.510271f, 0.281834f, -0.390951f, + 0.651242f, 0.767377f, 0.890746f, -0.218409f, 0.602640f, -0.685773f, + 0.250331f, 0.397971f, -0.828262f, 0.062359f, 0.777133f, -0.472668f, + -0.530429f, 0.679314f, -0.008920f, -0.695267f, -0.538464f, 0.315908f, + 0.125897f, -0.416343f, 0.244610f, 0.431811f, -0.438538f, -0.175454f, + -0.275589f, 0.562784f, -0.729026f, 0.804139f, -0.420728f, -0.000884f, + 0.567181f, 0.354124f, -0.700377f, 0.393239f, -0.741974f, 0.891893f, + 0.772824f, 0.030009f, 0.358817f, 0.953587f, -0.749079f, 0.504486f, + 0.654104f, 0.562861f, -0.618235f, -0.142717f, -0.971087f, -0.349429f, + -0.730596f, -0.098965f, 0.144550f, 0.584047f, -0.160527f, 0.065073f, + 0.851409f, 0.798164f, 0.089667f, 0.802248f, -0.896347f, 0.617205f, + -0.330191f, -0.542634f, 0.644804f, -0.303531f, -0.669059f, -0.943733f, + 0.910740f, 0.360581f, 0.721124f, 0.878187f, 0.360388f, 0.834847f, + -0.486617f, 0.771236f, 0.840086f, -0.399873f, -0.853218f, 0.534797f, + -0.830096f, 0.457528f, -0.104221f, 0.302497f, -0.660996f, 0.062898f, + 0.267602f, -0.971808f, -0.059257f, 0.772652f, -0.771943f, -0.114918f, + 0.319096f, -0.410454f, 0.900737f, 0.388572f, -0.586387f, 0.109525f, + 0.758557f, 0.115715f, 0.504668f, 0.789802f, 0.683688f, -0.738287f, + -0.621692f, -0.692720f, -0.942196f, -0.981830f, 0.192903f, 0.218099f, + 0.837847f, 0.467149f, -0.397706f, -0.008851f, -0.483674f, 0.465709f, + -0.766478f, 0.492083f, 0.619578f, 0.490467f, -0.325713f, 0.168650f, + -0.062096f, -0.825470f, 0.657435f, 0.371889f, -0.465350f, 0.938967f, + -0.632452f, -0.400118f, -0.177630f, -0.527022f, -0.609889f, 0.410759f, + -0.638903f, 0.044666f, -0.407656f, -0.074436f, 0.850465f, -0.568222f, + -0.997982f, 0.813212f, 0.360084f, 0.029904f, 0.044138f, -0.794163f, + 0.993761f, -0.282062f, 0.250485f, -0.213267f, -0.984675f, 0.090570f, + 0.018221f, -0.506442f, -0.909209f, 0.683459f, -0.903500f, -0.367359f, + 0.566839f, 0.944800f, 0.172928f, 0.556088f, 0.455395f, 0.301974f, + 0.329230f, 0.877560f, 0.070163f, -0.203120f, 0.340915f, -0.118931f, + -0.734252f, -0.121593f, 0.095285f, -0.209727f, -0.203456f, 0.502697f, + 0.044701f, -0.019134f, -0.822642f, -0.498297f, -0.104882f, 0.275922f, + 0.418891f, 0.985240f, 0.864390f, -0.815541f, 0.907080f, -0.674409f, + 0.940910f, 0.194013f, -0.519546f, -0.859410f, -0.399918f, 0.627090f, + -0.846580f, -0.291054f, -0.735978f, -0.683641f, -0.875706f, 0.403687f, + -0.827037f, 0.233574f, -0.652457f, 0.302802f, -0.002607f, -0.430979f, + 0.661119f, 0.636720f, 0.876339f, -0.999348f, 0.280778f, -0.985289f, + -0.787158f, -0.786411f, -0.265782f, -0.520785f, -0.307720f, -0.500760f, + -0.225871f, -0.157923f, 0.280155f, 0.575106f, -0.460011f, 0.687965f, + 0.480937f, 0.652204f, -0.635616f, -0.869128f, 0.220701f, 0.403106f, + -0.776765f, -0.808353f, 0.195668f, 0.624465f, 0.629156f, -0.821126f, + 0.462557f, 0.807713f, -0.095536f, -0.858625f, -0.517444f, 0.463730f}; + + const float kReference[] = { + -0.816528f, 0.085421f, 0.739647f, -0.922089f, 0.669301f, -0.048187f, + -0.290039f, -0.818085f, -0.596008f, -0.177826f, -0.002197f, -0.350647f, + -0.064301f, 0.337291f, -0.621765f, 0.115909f, 0.311899f, -0.915924f, + 0.020478f, 0.836055f, -0.714020f, -0.037140f, 0.391125f, -0.340118f}; + + RunBitexactnessTest( + 16000, 2, CreateVector(rtc::ArrayView(kReferenceInput)), + CreateVector(rtc::ArrayView(kReference))); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.cc new file mode 100644 index 0000000000..8dd713f4e5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.cc @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/noise_suppression_impl.h" + +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/constructormagic.h" +#if defined(WEBRTC_NS_FLOAT) +#include "modules/audio_processing/ns/noise_suppression.h" +#define NS_CREATE WebRtcNs_Create +#define NS_FREE WebRtcNs_Free +#define NS_INIT WebRtcNs_Init +#define NS_SET_POLICY WebRtcNs_set_policy +typedef NsHandle NsState; +#elif defined(WEBRTC_NS_FIXED) +#include "modules/audio_processing/ns/noise_suppression_x.h" +#define NS_CREATE WebRtcNsx_Create +#define NS_FREE WebRtcNsx_Free +#define NS_INIT WebRtcNsx_Init +#define NS_SET_POLICY WebRtcNsx_set_policy +typedef NsxHandle NsState; +#endif + +namespace webrtc { +class NoiseSuppressionImpl::Suppressor { + public: + explicit Suppressor(int sample_rate_hz) { + state_ = NS_CREATE(); + RTC_CHECK(state_); + int error = NS_INIT(state_, sample_rate_hz); + RTC_DCHECK_EQ(0, error); + } + ~Suppressor() { + NS_FREE(state_); + } + NsState* state() { return state_; } + private: + NsState* state_ = nullptr; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(Suppressor); +}; + +NoiseSuppressionImpl::NoiseSuppressionImpl(rtc::CriticalSection* crit) + : crit_(crit) { + RTC_DCHECK(crit); +} + +NoiseSuppressionImpl::~NoiseSuppressionImpl() {} + +void NoiseSuppressionImpl::Initialize(size_t channels, int sample_rate_hz) { + rtc::CritScope cs(crit_); + channels_ = channels; + sample_rate_hz_ = sample_rate_hz; + std::vector> new_suppressors; + if (enabled_) { + new_suppressors.resize(channels); + for (size_t i = 0; i < channels; i++) { + new_suppressors[i].reset(new Suppressor(sample_rate_hz)); + } + } + suppressors_.swap(new_suppressors); + set_level(level_); +} + +void NoiseSuppressionImpl::AnalyzeCaptureAudio(AudioBuffer* audio) { + RTC_DCHECK(audio); +#if defined(WEBRTC_NS_FLOAT) + rtc::CritScope cs(crit_); + if (!enabled_) { + return; + } + + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(suppressors_.size(), audio->num_channels()); + for (size_t i = 0; i < suppressors_.size(); i++) { + WebRtcNs_Analyze(suppressors_[i]->state(), + audio->split_bands_const_f(i)[kBand0To8kHz]); + } +#endif +} + +void NoiseSuppressionImpl::ProcessCaptureAudio(AudioBuffer* audio) { + RTC_DCHECK(audio); + rtc::CritScope cs(crit_); + if (!enabled_) { + return; + } + + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(suppressors_.size(), audio->num_channels()); + for (size_t i = 0; i < suppressors_.size(); i++) { +#if defined(WEBRTC_NS_FLOAT) + WebRtcNs_Process(suppressors_[i]->state(), + audio->split_bands_const_f(i), + audio->num_bands(), + audio->split_bands_f(i)); +#elif defined(WEBRTC_NS_FIXED) + WebRtcNsx_Process(suppressors_[i]->state(), + audio->split_bands_const(i), + audio->num_bands(), + audio->split_bands(i)); +#endif + } +} + +int NoiseSuppressionImpl::Enable(bool enable) { + rtc::CritScope cs(crit_); + if (enabled_ != enable) { + enabled_ = enable; + Initialize(channels_, sample_rate_hz_); + } + return AudioProcessing::kNoError; +} + +bool NoiseSuppressionImpl::is_enabled() const { + rtc::CritScope cs(crit_); + return enabled_; +} + +int NoiseSuppressionImpl::set_level(Level level) { + int policy = 1; + switch (level) { + case NoiseSuppression::kLow: + policy = 0; + break; + case NoiseSuppression::kModerate: + policy = 1; + break; + case NoiseSuppression::kHigh: + policy = 2; + break; + case NoiseSuppression::kVeryHigh: + policy = 3; + break; + default: + RTC_NOTREACHED(); + } + rtc::CritScope cs(crit_); + level_ = level; + for (auto& suppressor : suppressors_) { + int error = NS_SET_POLICY(suppressor->state(), policy); + RTC_DCHECK_EQ(0, error); + } + return AudioProcessing::kNoError; +} + +NoiseSuppression::Level NoiseSuppressionImpl::level() const { + rtc::CritScope cs(crit_); + return level_; +} + +float NoiseSuppressionImpl::speech_probability() const { + rtc::CritScope cs(crit_); +#if defined(WEBRTC_NS_FLOAT) + float probability_average = 0.0f; + for (auto& suppressor : suppressors_) { + probability_average += + WebRtcNs_prior_speech_probability(suppressor->state()); + } + if (!suppressors_.empty()) { + probability_average /= suppressors_.size(); + } + return probability_average; +#elif defined(WEBRTC_NS_FIXED) + // TODO(peah): Returning error code as a float! Remove this. + // Currently not available for the fixed point implementation. + return AudioProcessing::kUnsupportedFunctionError; +#endif +} + +std::vector NoiseSuppressionImpl::NoiseEstimate() { + rtc::CritScope cs(crit_); + std::vector noise_estimate; +#if defined(WEBRTC_NS_FLOAT) + const float kNumChannelsFraction = 1.f / suppressors_.size(); + noise_estimate.assign(WebRtcNs_num_freq(), 0.f); + for (auto& suppressor : suppressors_) { + const float* noise = WebRtcNs_noise_estimate(suppressor->state()); + for (size_t i = 0; i < noise_estimate.size(); ++i) { + noise_estimate[i] += kNumChannelsFraction * noise[i]; + } + } +#elif defined(WEBRTC_NS_FIXED) + noise_estimate.assign(WebRtcNsx_num_freq(), 0.f); + for (auto& suppressor : suppressors_) { + int q_noise; + const uint32_t* noise = WebRtcNsx_noise_estimate(suppressor->state(), + &q_noise); + const float kNormalizationFactor = + 1.f / ((1 << q_noise) * suppressors_.size()); + for (size_t i = 0; i < noise_estimate.size(); ++i) { + noise_estimate[i] += kNormalizationFactor * noise[i]; + } + } +#endif + return noise_estimate; +} + +size_t NoiseSuppressionImpl::num_noise_bins() { +#if defined(WEBRTC_NS_FLOAT) + return WebRtcNs_num_freq(); +#elif defined(WEBRTC_NS_FIXED) + return WebRtcNsx_num_freq(); +#endif +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.h new file mode 100644 index 0000000000..fba716e253 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_impl.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_ + +#include +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +class AudioBuffer; + +class NoiseSuppressionImpl : public NoiseSuppression { + public: + explicit NoiseSuppressionImpl(rtc::CriticalSection* crit); + ~NoiseSuppressionImpl() override; + + // TODO(peah): Fold into ctor, once public API is removed. + void Initialize(size_t channels, int sample_rate_hz); + void AnalyzeCaptureAudio(AudioBuffer* audio); + void ProcessCaptureAudio(AudioBuffer* audio); + + // NoiseSuppression implementation. + int Enable(bool enable) override; + bool is_enabled() const override; + int set_level(Level level) override; + Level level() const override; + float speech_probability() const override; + std::vector NoiseEstimate() override; + static size_t num_noise_bins(); + + private: + class Suppressor; + rtc::CriticalSection* const crit_; + bool enabled_ RTC_GUARDED_BY(crit_) = false; + Level level_ RTC_GUARDED_BY(crit_) = kModerate; + size_t channels_ RTC_GUARDED_BY(crit_) = 0; + int sample_rate_hz_ RTC_GUARDED_BY(crit_) = 0; + std::vector> suppressors_ RTC_GUARDED_BY(crit_); + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(NoiseSuppressionImpl); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_unittest.cc new file mode 100644 index 0000000000..0b734fdd3f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/noise_suppression_unittest.cc @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/noise_suppression_impl.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kNumFramesToProcess = 1000; + +// Process one frame of data and produce the output. +void ProcessOneFrame(int sample_rate_hz, + AudioBuffer* capture_buffer, + NoiseSuppressionImpl* noise_suppressor) { + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + capture_buffer->SplitIntoFrequencyBands(); + } + + noise_suppressor->AnalyzeCaptureAudio(capture_buffer); + noise_suppressor->ProcessCaptureAudio(capture_buffer); + + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + capture_buffer->MergeFrequencyBands(); + } +} + +// Processes a specified amount of frames, verifies the results and reports +// any errors. +void RunBitexactnessTest(int sample_rate_hz, + size_t num_channels, + NoiseSuppressionImpl::Level level, + float speech_probability_reference, + rtc::ArrayView noise_estimate_reference, + rtc::ArrayView output_reference) { + rtc::CriticalSection crit_capture; + NoiseSuppressionImpl noise_suppressor(&crit_capture); + noise_suppressor.Initialize(num_channels, sample_rate_hz); + noise_suppressor.Enable(true); + noise_suppressor.set_level(level); + + int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + const StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(samples_per_channel * num_channels); + for (size_t frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &capture_file, capture_input); + + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + ProcessOneFrame(sample_rate_hz, &capture_buffer, &noise_suppressor); + } + + // Extract test results. + std::vector capture_output; + test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer, + &capture_output); + float speech_probability = noise_suppressor.speech_probability(); + std::vector noise_estimate = noise_suppressor.NoiseEstimate(); + + const float kVectorElementErrorBound = 1.0f / 32768.0f; + EXPECT_FLOAT_EQ(speech_probability_reference, speech_probability); + EXPECT_TRUE(test::VerifyArray(noise_estimate_reference, noise_estimate, + kVectorElementErrorBound)); + + // Compare the output with the reference. Only the first values of the output + // from last frame processed are compared in order not having to specify all + // preceeding frames as testvectors. As the algorithm being tested has a + // memory, testing only the last frame implicitly also tests the preceeding + // frames. + EXPECT_TRUE(test::VerifyDeinterleavedArray( + capture_config.num_frames(), capture_config.num_channels(), + output_reference, capture_output, kVectorElementErrorBound)); +} + +} // namespace + +TEST(NoiseSuppresionBitExactnessTest, Mono8kHzLow) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {1432.341431f, 3321.919922f, 7677.521973f}; + const float kOutputReference[] = {0.003510f, 0.004517f, 0.004669f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {1432.341431f, 3321.919922f, 7677.521973f}; + const float kOutputReference[] = {0.003510f, 0.004517f, 0.004669f}; +#else + const float kSpeechProbabilityReference = 0.73421317f; + const float kNoiseEstimateReference[] = + {1175.266113f, 3289.305908f, 7532.991211f}; + const float kOutputReference[] = {0.003263f, 0.004402f, 0.004537f}; +#endif + + RunBitexactnessTest(8000, 1, NoiseSuppression::Level::kLow, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +TEST(NoiseSuppresionBitExactnessTest, Mono16kHzLow) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2534.461914f, 6277.638672f, 14367.499023f}; + const float kOutputReference[] = {0.003449f, 0.004334f, 0.004303f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2534.461914f, 6277.638672f, 14367.499023f}; + const float kOutputReference[] = {0.003449f, 0.004334f, 0.004303f}; +#else + const float kSpeechProbabilityReference = 0.71672988f; + const float kNoiseEstimateReference[] = + {2151.313965f, 6509.765137f, 15658.848633f}; + const float kOutputReference[] = {0.003574f, 0.004494f, 0.004499f}; +#endif + + RunBitexactnessTest(16000, 1, NoiseSuppression::Level::kLow, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +TEST(NoiseSuppresionBitExactnessTest, Mono32kHzLow) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2540.059082f, 6317.822754f, 14440.845703f}; + const float kOutputReference[] = {0.001679f, 0.002411f, 0.002594f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2540.059082f, 6317.822754f, 14440.845703f}; + const float kOutputReference[] = {0.001679f, 0.002411f, 0.002594f}; +#else + const float kSpeechProbabilityReference = 0.67999554f; + const float kNoiseEstimateReference[] = + {2149.780518f, 7076.936035f, 14939.945312f}; + const float kOutputReference[] = {0.001221f, 0.001984f, 0.002228f}; +#endif + + RunBitexactnessTest(32000, 1, NoiseSuppression::Level::kLow, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +TEST(NoiseSuppresionBitExactnessTest, Mono48kHzLow) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2564.605713f, 6213.656250f, 13372.284180f}; + const float kOutputReference[] = {-0.013185f, -0.012769f, -0.012023f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2564.605713f, 6213.656250f, 13372.284180f}; + const float kOutputReference[] = {-0.013185f, -0.012769f, -0.012023f}; +#else + const float kSpeechProbabilityReference = 0.70645678f; + const float kNoiseEstimateReference[] = + {2168.783203f, 6902.895508f, 13190.677734f}; + const float kOutputReference[] = {-0.013062f, -0.012657f, -0.011934f}; +#endif + + RunBitexactnessTest(48000, 1, NoiseSuppression::Level::kLow, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +TEST(NoiseSuppresionBitExactnessTest, Stereo16kHzLow) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {9992.127930f, 12689.569336f, 11589.296875f}; + const float kOutputReference[] = {-0.011108f, -0.007904f, -0.012390f, + -0.002441f, 0.000855f, -0.003204f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {10321.353516f, 12133.852539f, 10923.060547f}; + const float kOutputReference[] = {-0.011108f, -0.007904f, -0.012390f, + -0.002472f, 0.000916f, -0.003235f}; +#else + const float kSpeechProbabilityReference = 0.67230678f; + const float kNoiseEstimateReference[] = + {9771.250000f, 11329.377930f, 10503.052734f}; + const float kOutputReference[] = {-0.011459f, -0.008110f, -0.012728f, + -0.002399f, 0.001018f, -0.003189f}; +#endif + + RunBitexactnessTest(16000, 2, NoiseSuppression::Level::kLow, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +TEST(NoiseSuppresionBitExactnessTest, Mono16kHzModerate) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2057.085938f, 7601.055176f, 19666.187500f}; + const float kOutputReference[] = {0.004669f, 0.005524f, 0.005432f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2244.497803f, 6864.164062f, 16726.523438f}; + const float kOutputReference[] = {0.004669f, 0.005615f, 0.005585f}; +#else + const float kSpeechProbabilityReference = 0.70897013f; + const float kNoiseEstimateReference[] = + {2171.490723f, 6553.567871f, 15626.562500f}; + const float kOutputReference[] = {0.004513f, 0.005590f, 0.005614f}; +#endif + + RunBitexactnessTest(16000, 1, NoiseSuppression::Level::kModerate, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +TEST(NoiseSuppresionBitExactnessTest, Mono16kHzHigh) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2095.148193f, 7698.553711f, 19689.533203f}; + const float kOutputReference[] = {0.004639f, 0.005402f, 0.005310f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2282.515625f, 6984.408203f, 16920.960938f}; + const float kOutputReference[] = {0.004547f, 0.005432f, 0.005402f}; +#else + const float kSpeechProbabilityReference = 0.70106733f; + const float kNoiseEstimateReference[] = + {2224.968506f, 6712.025879f, 15785.087891f}; + const float kOutputReference[] = {0.004394f, 0.005406f, 0.005416f}; +#endif + + RunBitexactnessTest(16000, 1, NoiseSuppression::Level::kHigh, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +TEST(NoiseSuppresionBitExactnessTest, Mono16kHzVeryHigh) { +#if defined(WEBRTC_ARCH_ARM64) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2677.733398f, 6186.987305f, 14365.744141f}; + const float kOutputReference[] = {0.004273f, 0.005127f, 0.005188f}; +#elif defined(WEBRTC_ARCH_ARM) + const float kSpeechProbabilityReference = -4.0f; + const float kNoiseEstimateReference[] = + {2677.733398f, 6186.987305f, 14365.744141f}; + const float kOutputReference[] = {0.004273f, 0.005127f, 0.005188f}; +#else + const float kSpeechProbabilityReference = 0.70281971f; + const float kNoiseEstimateReference[] = + {2254.347900f, 6723.699707f, 15771.625977f}; + const float kOutputReference[] = {0.004321f, 0.005247f, 0.005263f}; +#endif + + RunBitexactnessTest(16000, 1, NoiseSuppression::Level::kVeryHigh, + kSpeechProbabilityReference, kNoiseEstimateReference, + kOutputReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/defines.h b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/defines.h new file mode 100644 index 0000000000..66b45a9565 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/defines.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_DEFINES_H_ +#define MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_DEFINES_H_ + +#define BLOCKL_MAX 160 // max processing block length: 160 +#define ANAL_BLOCKL_MAX 256 // max analysis block length: 256 +#define HALF_ANAL_BLOCKL 129 // half max analysis block length + 1 +#define NUM_HIGH_BANDS_MAX 2 // max number of high bands: 2 + +#define QUANTILE (float)0.25 + +#define SIMULT 3 +#define END_STARTUP_LONG 200 +#define END_STARTUP_SHORT 50 +#define FACTOR (float)40.0 +#define WIDTH (float)0.01 + +// Length of fft work arrays. +#define IP_LENGTH (ANAL_BLOCKL_MAX >> 1) // must be at least ceil(2 + sqrt(ANAL_BLOCKL_MAX/2)) +#define W_LENGTH (ANAL_BLOCKL_MAX >> 1) + +//PARAMETERS FOR NEW METHOD +#define DD_PR_SNR (float)0.98 // DD update of prior SNR +#define LRT_TAVG (float)0.50 // tavg parameter for LRT (previously 0.90) +#define SPECT_FL_TAVG (float)0.30 // tavg parameter for spectral flatness measure +#define SPECT_DIFF_TAVG (float)0.30 // tavg parameter for spectral difference measure +#define PRIOR_UPDATE (float)0.10 // update parameter of prior model +#define NOISE_UPDATE (float)0.90 // update parameter for noise +#define SPEECH_UPDATE (float)0.99 // update parameter when likely speech +#define WIDTH_PR_MAP (float)4.0 // width parameter in sigmoid map for prior model +#define LRT_FEATURE_THR (float)0.5 // default threshold for LRT feature +#define SF_FEATURE_THR (float)0.5 // default threshold for Spectral Flatness feature +#define SD_FEATURE_THR (float)0.5 // default threshold for Spectral Difference feature +#define PROB_RANGE (float)0.20 // probability threshold for noise state in + // speech/noise likelihood +#define HIST_PAR_EST 1000 // histogram size for estimation of parameters +#define GAMMA_PAUSE (float)0.05 // update for conservative noise estimate +// +#define B_LIM (float)0.5 // threshold in final energy gain factor calculation +#endif // MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_DEFINES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c new file mode 100644 index 0000000000..e21416f956 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/ns/noise_suppression.h" + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/ns/defines.h" +#include "modules/audio_processing/ns/ns_core.h" + +NsHandle* WebRtcNs_Create() { + NoiseSuppressionC* self = malloc(sizeof(NoiseSuppressionC)); + self->initFlag = 0; + return (NsHandle*)self; +} + +void WebRtcNs_Free(NsHandle* NS_inst) { + free(NS_inst); +} + +int WebRtcNs_Init(NsHandle* NS_inst, uint32_t fs) { + return WebRtcNs_InitCore((NoiseSuppressionC*)NS_inst, fs); +} + +int WebRtcNs_set_policy(NsHandle* NS_inst, int mode) { + return WebRtcNs_set_policy_core((NoiseSuppressionC*)NS_inst, mode); +} + +void WebRtcNs_Analyze(NsHandle* NS_inst, const float* spframe) { + WebRtcNs_AnalyzeCore((NoiseSuppressionC*)NS_inst, spframe); +} + +void WebRtcNs_Process(NsHandle* NS_inst, + const float* const* spframe, + size_t num_bands, + float* const* outframe) { + WebRtcNs_ProcessCore((NoiseSuppressionC*)NS_inst, spframe, num_bands, + outframe); +} + +float WebRtcNs_prior_speech_probability(NsHandle* handle) { + NoiseSuppressionC* self = (NoiseSuppressionC*)handle; + if (handle == NULL) { + return -1; + } + if (self->initFlag == 0) { + return -1; + } + return self->priorSpeechProb; +} + +const float* WebRtcNs_noise_estimate(const NsHandle* handle) { + const NoiseSuppressionC* self = (const NoiseSuppressionC*)handle; + if (handle == NULL || self->initFlag == 0) { + return NULL; + } + return self->noise; +} + +size_t WebRtcNs_num_freq() { + return HALF_ANAL_BLOCKL; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.h b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.h new file mode 100644 index 0000000000..a167142a7f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NS_NOISE_SUPPRESSION_H_ +#define MODULES_AUDIO_PROCESSING_NS_NOISE_SUPPRESSION_H_ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +typedef struct NsHandleT NsHandle; + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This function creates an instance of the floating point Noise Suppression. + */ +NsHandle* WebRtcNs_Create(); + +/* + * This function frees the dynamic memory of a specified noise suppression + * instance. + * + * Input: + * - NS_inst : Pointer to NS instance that should be freed + */ +void WebRtcNs_Free(NsHandle* NS_inst); + +/* + * This function initializes a NS instance and has to be called before any other + * processing is made. + * + * Input: + * - NS_inst : Instance that should be initialized + * - fs : sampling frequency + * + * Output: + * - NS_inst : Initialized instance + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcNs_Init(NsHandle* NS_inst, uint32_t fs); + +/* + * This changes the aggressiveness of the noise suppression method. + * + * Input: + * - NS_inst : Noise suppression instance. + * - mode : 0: Mild, 1: Medium , 2: Aggressive + * + * Output: + * - NS_inst : Updated instance. + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcNs_set_policy(NsHandle* NS_inst, int mode); + +/* + * This functions estimates the background noise for the inserted speech frame. + * The input and output signals should always be 10ms (80 or 160 samples). + * + * Input + * - NS_inst : Noise suppression instance. + * - spframe : Pointer to speech frame buffer for L band + * + * Output: + * - NS_inst : Updated NS instance + */ +void WebRtcNs_Analyze(NsHandle* NS_inst, const float* spframe); + +/* + * This functions does Noise Suppression for the inserted speech frame. The + * input and output signals should always be 10ms (80 or 160 samples). + * + * Input + * - NS_inst : Noise suppression instance. + * - spframe : Pointer to speech frame buffer for each band + * - num_bands : Number of bands + * + * Output: + * - NS_inst : Updated NS instance + * - outframe : Pointer to output frame for each band + */ +void WebRtcNs_Process(NsHandle* NS_inst, + const float* const* spframe, + size_t num_bands, + float* const* outframe); + +/* Returns the internally used prior speech probability of the current frame. + * There is a frequency bin based one as well, with which this should not be + * confused. + * + * Input + * - handle : Noise suppression instance. + * + * Return value : Prior speech probability in interval [0.0, 1.0]. + * -1 - NULL pointer or uninitialized instance. + */ +float WebRtcNs_prior_speech_probability(NsHandle* handle); + +/* Returns a pointer to the noise estimate per frequency bin. The number of + * frequency bins can be provided using WebRtcNs_num_freq(). + * + * Input + * - handle : Noise suppression instance. + * + * Return value : Pointer to the noise estimate per frequency bin. + * Returns NULL if the input is a NULL pointer or an + * uninitialized instance. + */ +const float* WebRtcNs_noise_estimate(const NsHandle* handle); + +/* Returns the number of frequency bins, which is the length of the noise + * estimate for example. + * + * Return value : Number of frequency bins. + */ +size_t WebRtcNs_num_freq(); + +#ifdef __cplusplus +} +#endif + +#endif // MODULES_AUDIO_PROCESSING_NS_NOISE_SUPPRESSION_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.c b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.c new file mode 100644 index 0000000000..1fd3ebc67e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/ns/noise_suppression_x.h" + +#include + +#include "common_audio/signal_processing/include/real_fft.h" +#include "modules/audio_processing/ns/nsx_core.h" +#include "modules/audio_processing/ns/nsx_defines.h" + +NsxHandle* WebRtcNsx_Create() { + NoiseSuppressionFixedC* self = malloc(sizeof(NoiseSuppressionFixedC)); + WebRtcSpl_Init(); + self->real_fft = NULL; + self->initFlag = 0; + return (NsxHandle*)self; +} + +void WebRtcNsx_Free(NsxHandle* nsxInst) { + WebRtcSpl_FreeRealFFT(((NoiseSuppressionFixedC*)nsxInst)->real_fft); + free(nsxInst); +} + +int WebRtcNsx_Init(NsxHandle* nsxInst, uint32_t fs) { + return WebRtcNsx_InitCore((NoiseSuppressionFixedC*)nsxInst, fs); +} + +int WebRtcNsx_set_policy(NsxHandle* nsxInst, int mode) { + return WebRtcNsx_set_policy_core((NoiseSuppressionFixedC*)nsxInst, mode); +} + +void WebRtcNsx_Process(NsxHandle* nsxInst, + const short* const* speechFrame, + int num_bands, + short* const* outFrame) { + WebRtcNsx_ProcessCore((NoiseSuppressionFixedC*)nsxInst, speechFrame, + num_bands, outFrame); +} + +const uint32_t* WebRtcNsx_noise_estimate(const NsxHandle* nsxInst, + int* q_noise) { + *q_noise = 11; + const NoiseSuppressionFixedC* self = (const NoiseSuppressionFixedC*)nsxInst; + if (nsxInst == NULL || self->initFlag == 0) { + return NULL; + } + *q_noise += self->prevQNoise; + return self->prevNoiseU32; +} + +size_t WebRtcNsx_num_freq() { + return HALF_ANAL_BLOCKL; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.h b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.h new file mode 100644 index 0000000000..838861db79 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/noise_suppression_x.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NS_NOISE_SUPPRESSION_X_H_ +#define MODULES_AUDIO_PROCESSING_NS_NOISE_SUPPRESSION_X_H_ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +typedef struct NsxHandleT NsxHandle; + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This function creates an instance of the fixed point Noise Suppression. + */ +NsxHandle* WebRtcNsx_Create(); + +/* + * This function frees the dynamic memory of a specified Noise Suppression + * instance. + * + * Input: + * - nsxInst : Pointer to NS instance that should be freed + */ +void WebRtcNsx_Free(NsxHandle* nsxInst); + +/* + * This function initializes a NS instance + * + * Input: + * - nsxInst : Instance that should be initialized + * - fs : sampling frequency + * + * Output: + * - nsxInst : Initialized instance + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcNsx_Init(NsxHandle* nsxInst, uint32_t fs); + +/* + * This changes the aggressiveness of the noise suppression method. + * + * Input: + * - nsxInst : Instance that should be initialized + * - mode : 0: Mild, 1: Medium , 2: Aggressive + * + * Output: + * - nsxInst : Initialized instance + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcNsx_set_policy(NsxHandle* nsxInst, int mode); + +/* + * This functions does noise suppression for the inserted speech frame. The + * input and output signals should always be 10ms (80 or 160 samples). + * + * Input + * - nsxInst : NSx instance. Needs to be initiated before call. + * - speechFrame : Pointer to speech frame buffer for each band + * - num_bands : Number of bands + * + * Output: + * - nsxInst : Updated NSx instance + * - outFrame : Pointer to output frame for each band + */ +void WebRtcNsx_Process(NsxHandle* nsxInst, + const short* const* speechFrame, + int num_bands, + short* const* outFrame); + +/* Returns a pointer to the noise estimate per frequency bin. The number of + * frequency bins can be provided using WebRtcNsx_num_freq(). + * + * Input + * - nsxInst : NSx instance. Needs to be initiated before call. + * - q_noise : Q value of the noise estimate, which is the number of + * bits that it needs to be right-shifted to be + * normalized. + * + * Return value : Pointer to the noise estimate per frequency bin. + * Returns NULL if the input is a NULL pointer or an + * uninitialized instance. + */ +const uint32_t* WebRtcNsx_noise_estimate(const NsxHandle* nsxInst, + int* q_noise); + +/* Returns the number of frequency bins, which is the length of the noise + * estimate for example. + * + * Return value : Number of frequency bins. + */ +size_t WebRtcNsx_num_freq(); + +#ifdef __cplusplus +} +#endif + +#endif // MODULES_AUDIO_PROCESSING_NS_NOISE_SUPPRESSION_X_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c new file mode 100644 index 0000000000..3345b2451e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.c @@ -0,0 +1,1418 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "rtc_base/checks.h" +#include "common_audio/fft4g.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/ns/noise_suppression.h" +#include "modules/audio_processing/ns/ns_core.h" +#include "modules/audio_processing/ns/windows_private.h" + +// Set Feature Extraction Parameters. +static void set_feature_extraction_parameters(NoiseSuppressionC* self) { + // Bin size of histogram. + self->featureExtractionParams.binSizeLrt = 0.1f; + self->featureExtractionParams.binSizeSpecFlat = 0.05f; + self->featureExtractionParams.binSizeSpecDiff = 0.1f; + + // Range of histogram over which LRT threshold is computed. + self->featureExtractionParams.rangeAvgHistLrt = 1.f; + + // Scale parameters: multiply dominant peaks of the histograms by scale factor + // to obtain thresholds for prior model. + // For LRT and spectral difference. + self->featureExtractionParams.factor1ModelPars = 1.2f; + // For spectral_flatness: used when noise is flatter than speech. + self->featureExtractionParams.factor2ModelPars = 0.9f; + + // Peak limit for spectral flatness (varies between 0 and 1). + self->featureExtractionParams.thresPosSpecFlat = 0.6f; + + // Limit on spacing of two highest peaks in histogram: spacing determined by + // bin size. + self->featureExtractionParams.limitPeakSpacingSpecFlat = + 2 * self->featureExtractionParams.binSizeSpecFlat; + self->featureExtractionParams.limitPeakSpacingSpecDiff = + 2 * self->featureExtractionParams.binSizeSpecDiff; + + // Limit on relevance of second peak. + self->featureExtractionParams.limitPeakWeightsSpecFlat = 0.5f; + self->featureExtractionParams.limitPeakWeightsSpecDiff = 0.5f; + + // Fluctuation limit of LRT feature. + self->featureExtractionParams.thresFluctLrt = 0.05f; + + // Limit on the max and min values for the feature thresholds. + self->featureExtractionParams.maxLrt = 1.f; + self->featureExtractionParams.minLrt = 0.2f; + + self->featureExtractionParams.maxSpecFlat = 0.95f; + self->featureExtractionParams.minSpecFlat = 0.1f; + + self->featureExtractionParams.maxSpecDiff = 1.f; + self->featureExtractionParams.minSpecDiff = 0.16f; + + // Criteria of weight of histogram peak to accept/reject feature. + self->featureExtractionParams.thresWeightSpecFlat = + (int)(0.3 * (self->modelUpdatePars[1])); // For spectral flatness. + self->featureExtractionParams.thresWeightSpecDiff = + (int)(0.3 * (self->modelUpdatePars[1])); // For spectral difference. +} + +// Initialize state. +int WebRtcNs_InitCore(NoiseSuppressionC* self, uint32_t fs) { + int i; + // Check for valid pointer. + if (self == NULL) { + return -1; + } + + // Initialization of struct. + if (fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000) { + self->fs = fs; + } else { + return -1; + } + self->windShift = 0; + // We only support 10ms frames. + if (fs == 8000) { + self->blockLen = 80; + self->anaLen = 128; + self->window = kBlocks80w128; + } else { + self->blockLen = 160; + self->anaLen = 256; + self->window = kBlocks160w256; + } + self->magnLen = self->anaLen / 2 + 1; // Number of frequency bins. + + // Initialize FFT work arrays. + self->ip[0] = 0; // Setting this triggers initialization. + memset(self->dataBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + WebRtc_rdft(self->anaLen, 1, self->dataBuf, self->ip, self->wfft); + + memset(self->analyzeBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + memset(self->dataBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + memset(self->syntBuf, 0, sizeof(float) * ANAL_BLOCKL_MAX); + + // For HB processing. + memset(self->dataBufHB, + 0, + sizeof(float) * NUM_HIGH_BANDS_MAX * ANAL_BLOCKL_MAX); + + // For quantile noise estimation. + memset(self->quantile, 0, sizeof(float) * HALF_ANAL_BLOCKL); + for (i = 0; i < SIMULT * HALF_ANAL_BLOCKL; i++) { + self->lquantile[i] = 8.f; + self->density[i] = 0.3f; + } + + for (i = 0; i < SIMULT; i++) { + self->counter[i] = + (int)floor((float)(END_STARTUP_LONG * (i + 1)) / (float)SIMULT); + } + + self->updates = 0; + + // Wiener filter initialization. + for (i = 0; i < HALF_ANAL_BLOCKL; i++) { + self->smooth[i] = 1.f; + } + + // Set the aggressiveness: default. + self->aggrMode = 0; + + // Initialize variables for new method. + self->priorSpeechProb = 0.5f; // Prior prob for speech/noise. + // Previous analyze mag spectrum. + memset(self->magnPrevAnalyze, 0, sizeof(float) * HALF_ANAL_BLOCKL); + // Previous process mag spectrum. + memset(self->magnPrevProcess, 0, sizeof(float) * HALF_ANAL_BLOCKL); + // Current noise-spectrum. + memset(self->noise, 0, sizeof(float) * HALF_ANAL_BLOCKL); + // Previous noise-spectrum. + memset(self->noisePrev, 0, sizeof(float) * HALF_ANAL_BLOCKL); + // Conservative noise spectrum estimate. + memset(self->magnAvgPause, 0, sizeof(float) * HALF_ANAL_BLOCKL); + // For estimation of HB in second pass. + memset(self->speechProb, 0, sizeof(float) * HALF_ANAL_BLOCKL); + // Initial average magnitude spectrum. + memset(self->initMagnEst, 0, sizeof(float) * HALF_ANAL_BLOCKL); + for (i = 0; i < HALF_ANAL_BLOCKL; i++) { + // Smooth LR (same as threshold). + self->logLrtTimeAvg[i] = LRT_FEATURE_THR; + } + + // Feature quantities. + // Spectral flatness (start on threshold). + self->featureData[0] = SF_FEATURE_THR; + self->featureData[1] = 0.f; // Spectral entropy: not used in this version. + self->featureData[2] = 0.f; // Spectral variance: not used in this version. + // Average LRT factor (start on threshold). + self->featureData[3] = LRT_FEATURE_THR; + // Spectral template diff (start on threshold). + self->featureData[4] = SF_FEATURE_THR; + self->featureData[5] = 0.f; // Normalization for spectral difference. + // Window time-average of input magnitude spectrum. + self->featureData[6] = 0.f; + + memset(self->parametricNoise, 0, sizeof(float) * HALF_ANAL_BLOCKL); + + // Histogram quantities: used to estimate/update thresholds for features. + memset(self->histLrt, 0, sizeof(int) * HIST_PAR_EST); + memset(self->histSpecFlat, 0, sizeof(int) * HIST_PAR_EST); + memset(self->histSpecDiff, 0, sizeof(int) * HIST_PAR_EST); + + + self->blockInd = -1; // Frame counter. + // Default threshold for LRT feature. + self->priorModelPars[0] = LRT_FEATURE_THR; + // Threshold for spectral flatness: determined on-line. + self->priorModelPars[1] = 0.5f; + // sgn_map par for spectral measure: 1 for flatness measure. + self->priorModelPars[2] = 1.f; + // Threshold for template-difference feature: determined on-line. + self->priorModelPars[3] = 0.5f; + // Default weighting parameter for LRT feature. + self->priorModelPars[4] = 1.f; + // Default weighting parameter for spectral flatness feature. + self->priorModelPars[5] = 0.f; + // Default weighting parameter for spectral difference feature. + self->priorModelPars[6] = 0.f; + + // Update flag for parameters: + // 0 no update, 1 = update once, 2 = update every window. + self->modelUpdatePars[0] = 2; + self->modelUpdatePars[1] = 500; // Window for update. + // Counter for update of conservative noise spectrum. + self->modelUpdatePars[2] = 0; + // Counter if the feature thresholds are updated during the sequence. + self->modelUpdatePars[3] = self->modelUpdatePars[1]; + + self->signalEnergy = 0.0; + self->sumMagn = 0.0; + self->whiteNoiseLevel = 0.0; + self->pinkNoiseNumerator = 0.0; + self->pinkNoiseExp = 0.0; + + set_feature_extraction_parameters(self); + + // Default mode. + WebRtcNs_set_policy_core(self, 0); + + self->initFlag = 1; + return 0; +} + +// Estimate noise. +static void NoiseEstimation(NoiseSuppressionC* self, + float* magn, + float* noise) { + size_t i, s, offset; + float lmagn[HALF_ANAL_BLOCKL], delta; + + if (self->updates < END_STARTUP_LONG) { + self->updates++; + } + + for (i = 0; i < self->magnLen; i++) { + lmagn[i] = (float)log(magn[i]); + } + + // Loop over simultaneous estimates. + for (s = 0; s < SIMULT; s++) { + offset = s * self->magnLen; + + // newquantest(...) + for (i = 0; i < self->magnLen; i++) { + // Compute delta. + if (self->density[offset + i] > 1.0) { + delta = FACTOR * 1.f / self->density[offset + i]; + } else { + delta = FACTOR; + } + + // Update log quantile estimate. + if (lmagn[i] > self->lquantile[offset + i]) { + self->lquantile[offset + i] += + QUANTILE * delta / (float)(self->counter[s] + 1); + } else { + self->lquantile[offset + i] -= + (1.f - QUANTILE) * delta / (float)(self->counter[s] + 1); + } + + // Update density estimate. + if (fabs(lmagn[i] - self->lquantile[offset + i]) < WIDTH) { + self->density[offset + i] = + ((float)self->counter[s] * self->density[offset + i] + + 1.f / (2.f * WIDTH)) / + (float)(self->counter[s] + 1); + } + } // End loop over magnitude spectrum. + + if (self->counter[s] >= END_STARTUP_LONG) { + self->counter[s] = 0; + if (self->updates >= END_STARTUP_LONG) { + for (i = 0; i < self->magnLen; i++) { + self->quantile[i] = (float)exp(self->lquantile[offset + i]); + } + } + } + + self->counter[s]++; + } // End loop over simultaneous estimates. + + // Sequentially update the noise during startup. + if (self->updates < END_STARTUP_LONG) { + // Use the last "s" to get noise during startup that differ from zero. + for (i = 0; i < self->magnLen; i++) { + self->quantile[i] = (float)exp(self->lquantile[offset + i]); + } + } + + for (i = 0; i < self->magnLen; i++) { + noise[i] = self->quantile[i]; + } +} + +// Extract thresholds for feature parameters. +// Histograms are computed over some window size (given by +// self->modelUpdatePars[1]). +// Thresholds and weights are extracted every window. +// |flag| = 0 updates histogram only, |flag| = 1 computes the threshold/weights. +// Threshold and weights are returned in: self->priorModelPars. +static void FeatureParameterExtraction(NoiseSuppressionC* self, int flag) { + int i, useFeatureSpecFlat, useFeatureSpecDiff, numHistLrt; + int maxPeak1, maxPeak2; + int weightPeak1SpecFlat, weightPeak2SpecFlat, weightPeak1SpecDiff, + weightPeak2SpecDiff; + + float binMid, featureSum; + float posPeak1SpecFlat, posPeak2SpecFlat, posPeak1SpecDiff, posPeak2SpecDiff; + float fluctLrt, avgHistLrt, avgSquareHistLrt, avgHistLrtCompl; + + // 3 features: LRT, flatness, difference. + // lrt_feature = self->featureData[3]; + // flat_feature = self->featureData[0]; + // diff_feature = self->featureData[4]; + + // Update histograms. + if (flag == 0) { + // LRT + if ((self->featureData[3] < + HIST_PAR_EST * self->featureExtractionParams.binSizeLrt) && + (self->featureData[3] >= 0.0)) { + i = (int)(self->featureData[3] / + self->featureExtractionParams.binSizeLrt); + self->histLrt[i]++; + } + // Spectral flatness. + if ((self->featureData[0] < + HIST_PAR_EST * self->featureExtractionParams.binSizeSpecFlat) && + (self->featureData[0] >= 0.0)) { + i = (int)(self->featureData[0] / + self->featureExtractionParams.binSizeSpecFlat); + self->histSpecFlat[i]++; + } + // Spectral difference. + if ((self->featureData[4] < + HIST_PAR_EST * self->featureExtractionParams.binSizeSpecDiff) && + (self->featureData[4] >= 0.0)) { + i = (int)(self->featureData[4] / + self->featureExtractionParams.binSizeSpecDiff); + self->histSpecDiff[i]++; + } + } + + // Extract parameters for speech/noise probability. + if (flag == 1) { + // LRT feature: compute the average over + // self->featureExtractionParams.rangeAvgHistLrt. + avgHistLrt = 0.0; + avgHistLrtCompl = 0.0; + avgSquareHistLrt = 0.0; + numHistLrt = 0; + for (i = 0; i < HIST_PAR_EST; i++) { + binMid = ((float)i + 0.5f) * self->featureExtractionParams.binSizeLrt; + if (binMid <= self->featureExtractionParams.rangeAvgHistLrt) { + avgHistLrt += self->histLrt[i] * binMid; + numHistLrt += self->histLrt[i]; + } + avgSquareHistLrt += self->histLrt[i] * binMid * binMid; + avgHistLrtCompl += self->histLrt[i] * binMid; + } + if (numHistLrt > 0) { + avgHistLrt = avgHistLrt / ((float)numHistLrt); + } + avgHistLrtCompl = avgHistLrtCompl / ((float)self->modelUpdatePars[1]); + avgSquareHistLrt = avgSquareHistLrt / ((float)self->modelUpdatePars[1]); + fluctLrt = avgSquareHistLrt - avgHistLrt * avgHistLrtCompl; + // Get threshold for LRT feature. + if (fluctLrt < self->featureExtractionParams.thresFluctLrt) { + // Very low fluctuation, so likely noise. + self->priorModelPars[0] = self->featureExtractionParams.maxLrt; + } else { + self->priorModelPars[0] = + self->featureExtractionParams.factor1ModelPars * avgHistLrt; + // Check if value is within min/max range. + if (self->priorModelPars[0] < self->featureExtractionParams.minLrt) { + self->priorModelPars[0] = self->featureExtractionParams.minLrt; + } + if (self->priorModelPars[0] > self->featureExtractionParams.maxLrt) { + self->priorModelPars[0] = self->featureExtractionParams.maxLrt; + } + } + // Done with LRT feature. + + // For spectral flatness and spectral difference: compute the main peaks of + // histogram. + maxPeak1 = 0; + maxPeak2 = 0; + posPeak1SpecFlat = 0.0; + posPeak2SpecFlat = 0.0; + weightPeak1SpecFlat = 0; + weightPeak2SpecFlat = 0; + + // Peaks for flatness. + for (i = 0; i < HIST_PAR_EST; i++) { + binMid = + (i + 0.5f) * self->featureExtractionParams.binSizeSpecFlat; + if (self->histSpecFlat[i] > maxPeak1) { + // Found new "first" peak. + maxPeak2 = maxPeak1; + weightPeak2SpecFlat = weightPeak1SpecFlat; + posPeak2SpecFlat = posPeak1SpecFlat; + + maxPeak1 = self->histSpecFlat[i]; + weightPeak1SpecFlat = self->histSpecFlat[i]; + posPeak1SpecFlat = binMid; + } else if (self->histSpecFlat[i] > maxPeak2) { + // Found new "second" peak. + maxPeak2 = self->histSpecFlat[i]; + weightPeak2SpecFlat = self->histSpecFlat[i]; + posPeak2SpecFlat = binMid; + } + } + + // Compute two peaks for spectral difference. + maxPeak1 = 0; + maxPeak2 = 0; + posPeak1SpecDiff = 0.0; + posPeak2SpecDiff = 0.0; + weightPeak1SpecDiff = 0; + weightPeak2SpecDiff = 0; + // Peaks for spectral difference. + for (i = 0; i < HIST_PAR_EST; i++) { + binMid = + ((float)i + 0.5f) * self->featureExtractionParams.binSizeSpecDiff; + if (self->histSpecDiff[i] > maxPeak1) { + // Found new "first" peak. + maxPeak2 = maxPeak1; + weightPeak2SpecDiff = weightPeak1SpecDiff; + posPeak2SpecDiff = posPeak1SpecDiff; + + maxPeak1 = self->histSpecDiff[i]; + weightPeak1SpecDiff = self->histSpecDiff[i]; + posPeak1SpecDiff = binMid; + } else if (self->histSpecDiff[i] > maxPeak2) { + // Found new "second" peak. + maxPeak2 = self->histSpecDiff[i]; + weightPeak2SpecDiff = self->histSpecDiff[i]; + posPeak2SpecDiff = binMid; + } + } + + // For spectrum flatness feature. + useFeatureSpecFlat = 1; + // Merge the two peaks if they are close. + if ((fabs(posPeak2SpecFlat - posPeak1SpecFlat) < + self->featureExtractionParams.limitPeakSpacingSpecFlat) && + (weightPeak2SpecFlat > + self->featureExtractionParams.limitPeakWeightsSpecFlat * + weightPeak1SpecFlat)) { + weightPeak1SpecFlat += weightPeak2SpecFlat; + posPeak1SpecFlat = 0.5f * (posPeak1SpecFlat + posPeak2SpecFlat); + } + // Reject if weight of peaks is not large enough, or peak value too small. + if (weightPeak1SpecFlat < + self->featureExtractionParams.thresWeightSpecFlat || + posPeak1SpecFlat < self->featureExtractionParams.thresPosSpecFlat) { + useFeatureSpecFlat = 0; + } + // If selected, get the threshold. + if (useFeatureSpecFlat == 1) { + // Compute the threshold. + self->priorModelPars[1] = + self->featureExtractionParams.factor2ModelPars * posPeak1SpecFlat; + // Check if value is within min/max range. + if (self->priorModelPars[1] < self->featureExtractionParams.minSpecFlat) { + self->priorModelPars[1] = self->featureExtractionParams.minSpecFlat; + } + if (self->priorModelPars[1] > self->featureExtractionParams.maxSpecFlat) { + self->priorModelPars[1] = self->featureExtractionParams.maxSpecFlat; + } + } + // Done with flatness feature. + + // For template feature. + useFeatureSpecDiff = 1; + // Merge the two peaks if they are close. + if ((fabs(posPeak2SpecDiff - posPeak1SpecDiff) < + self->featureExtractionParams.limitPeakSpacingSpecDiff) && + (weightPeak2SpecDiff > + self->featureExtractionParams.limitPeakWeightsSpecDiff * + weightPeak1SpecDiff)) { + weightPeak1SpecDiff += weightPeak2SpecDiff; + posPeak1SpecDiff = 0.5f * (posPeak1SpecDiff + posPeak2SpecDiff); + } + // Get the threshold value. + self->priorModelPars[3] = + self->featureExtractionParams.factor1ModelPars * posPeak1SpecDiff; + // Reject if weight of peaks is not large enough. + if (weightPeak1SpecDiff < + self->featureExtractionParams.thresWeightSpecDiff) { + useFeatureSpecDiff = 0; + } + // Check if value is within min/max range. + if (self->priorModelPars[3] < self->featureExtractionParams.minSpecDiff) { + self->priorModelPars[3] = self->featureExtractionParams.minSpecDiff; + } + if (self->priorModelPars[3] > self->featureExtractionParams.maxSpecDiff) { + self->priorModelPars[3] = self->featureExtractionParams.maxSpecDiff; + } + // Done with spectral difference feature. + + // Don't use template feature if fluctuation of LRT feature is very low: + // most likely just noise state. + if (fluctLrt < self->featureExtractionParams.thresFluctLrt) { + useFeatureSpecDiff = 0; + } + + // Select the weights between the features. + // self->priorModelPars[4] is weight for LRT: always selected. + // self->priorModelPars[5] is weight for spectral flatness. + // self->priorModelPars[6] is weight for spectral difference. + featureSum = (float)(1 + useFeatureSpecFlat + useFeatureSpecDiff); + self->priorModelPars[4] = 1.f / featureSum; + self->priorModelPars[5] = ((float)useFeatureSpecFlat) / featureSum; + self->priorModelPars[6] = ((float)useFeatureSpecDiff) / featureSum; + + // Set hists to zero for next update. + if (self->modelUpdatePars[0] >= 1) { + for (i = 0; i < HIST_PAR_EST; i++) { + self->histLrt[i] = 0; + self->histSpecFlat[i] = 0; + self->histSpecDiff[i] = 0; + } + } + } // End of flag == 1. +} + +// Compute spectral flatness on input spectrum. +// |magnIn| is the magnitude spectrum. +// Spectral flatness is returned in self->featureData[0]. +static void ComputeSpectralFlatness(NoiseSuppressionC* self, + const float* magnIn) { + size_t i; + size_t shiftLP = 1; // Option to remove first bin(s) from spectral measures. + float avgSpectralFlatnessNum, avgSpectralFlatnessDen, spectralTmp; + + // Compute spectral measures. + // For flatness. + avgSpectralFlatnessNum = 0.0; + avgSpectralFlatnessDen = self->sumMagn; + for (i = 0; i < shiftLP; i++) { + avgSpectralFlatnessDen -= magnIn[i]; + } + // Compute log of ratio of the geometric to arithmetic mean: check for log(0) + // case. + for (i = shiftLP; i < self->magnLen; i++) { + if (magnIn[i] > 0.0) { + avgSpectralFlatnessNum += (float)log(magnIn[i]); + } else { + self->featureData[0] -= SPECT_FL_TAVG * self->featureData[0]; + return; + } + } + // Normalize. + avgSpectralFlatnessDen = avgSpectralFlatnessDen / self->magnLen; + avgSpectralFlatnessNum = avgSpectralFlatnessNum / self->magnLen; + + // Ratio and inverse log: check for case of log(0). + spectralTmp = (float)exp(avgSpectralFlatnessNum) / avgSpectralFlatnessDen; + + // Time-avg update of spectral flatness feature. + self->featureData[0] += SPECT_FL_TAVG * (spectralTmp - self->featureData[0]); + // Done with flatness feature. +} + +// Compute prior and post SNR based on quantile noise estimation. +// Compute DD estimate of prior SNR. +// Inputs: +// * |magn| is the signal magnitude spectrum estimate. +// * |noise| is the magnitude noise spectrum estimate. +// Outputs: +// * |snrLocPrior| is the computed prior SNR. +// * |snrLocPost| is the computed post SNR. +static void ComputeSnr(const NoiseSuppressionC* self, + const float* magn, + const float* noise, + float* snrLocPrior, + float* snrLocPost) { + size_t i; + + for (i = 0; i < self->magnLen; i++) { + // Previous post SNR. + // Previous estimate: based on previous frame with gain filter. + float previousEstimateStsa = self->magnPrevAnalyze[i] / + (self->noisePrev[i] + 0.0001f) * self->smooth[i]; + // Post SNR. + snrLocPost[i] = 0.f; + if (magn[i] > noise[i]) { + snrLocPost[i] = magn[i] / (noise[i] + 0.0001f) - 1.f; + } + // DD estimate is sum of two terms: current estimate and previous estimate. + // Directed decision update of snrPrior. + snrLocPrior[i] = + DD_PR_SNR * previousEstimateStsa + (1.f - DD_PR_SNR) * snrLocPost[i]; + } // End of loop over frequencies. +} + +// Compute the difference measure between input spectrum and a template/learned +// noise spectrum. +// |magnIn| is the input spectrum. +// The reference/template spectrum is self->magnAvgPause[i]. +// Returns (normalized) spectral difference in self->featureData[4]. +static void ComputeSpectralDifference(NoiseSuppressionC* self, + const float* magnIn) { + // avgDiffNormMagn = var(magnIn) - cov(magnIn, magnAvgPause)^2 / + // var(magnAvgPause) + size_t i; + float avgPause, avgMagn, covMagnPause, varPause, varMagn, avgDiffNormMagn; + + avgPause = 0.0; + avgMagn = self->sumMagn; + // Compute average quantities. + for (i = 0; i < self->magnLen; i++) { + // Conservative smooth noise spectrum from pause frames. + avgPause += self->magnAvgPause[i]; + } + avgPause /= self->magnLen; + avgMagn /= self->magnLen; + + covMagnPause = 0.0; + varPause = 0.0; + varMagn = 0.0; + // Compute variance and covariance quantities. + for (i = 0; i < self->magnLen; i++) { + covMagnPause += (magnIn[i] - avgMagn) * (self->magnAvgPause[i] - avgPause); + varPause += + (self->magnAvgPause[i] - avgPause) * (self->magnAvgPause[i] - avgPause); + varMagn += (magnIn[i] - avgMagn) * (magnIn[i] - avgMagn); + } + covMagnPause /= self->magnLen; + varPause /= self->magnLen; + varMagn /= self->magnLen; + // Update of average magnitude spectrum. + self->featureData[6] += self->signalEnergy; + + avgDiffNormMagn = + varMagn - (covMagnPause * covMagnPause) / (varPause + 0.0001f); + // Normalize and compute time-avg update of difference feature. + avgDiffNormMagn = (float)(avgDiffNormMagn / (self->featureData[5] + 0.0001f)); + self->featureData[4] += + SPECT_DIFF_TAVG * (avgDiffNormMagn - self->featureData[4]); +} + +// Compute speech/noise probability. +// Speech/noise probability is returned in |probSpeechFinal|. +// |magn| is the input magnitude spectrum. +// |noise| is the noise spectrum. +// |snrLocPrior| is the prior SNR for each frequency. +// |snrLocPost| is the post SNR for each frequency. +static void SpeechNoiseProb(NoiseSuppressionC* self, + float* probSpeechFinal, + const float* snrLocPrior, + const float* snrLocPost) { + size_t i; + int sgnMap; + float invLrt, gainPrior, indPrior; + float logLrtTimeAvgKsum, besselTmp; + float indicator0, indicator1, indicator2; + float tmpFloat1, tmpFloat2; + float weightIndPrior0, weightIndPrior1, weightIndPrior2; + float threshPrior0, threshPrior1, threshPrior2; + float widthPrior, widthPrior0, widthPrior1, widthPrior2; + + widthPrior0 = WIDTH_PR_MAP; + // Width for pause region: lower range, so increase width in tanh map. + widthPrior1 = 2.f * WIDTH_PR_MAP; + widthPrior2 = 2.f * WIDTH_PR_MAP; // For spectral-difference measure. + + // Threshold parameters for features. + threshPrior0 = self->priorModelPars[0]; + threshPrior1 = self->priorModelPars[1]; + threshPrior2 = self->priorModelPars[3]; + + // Sign for flatness feature. + sgnMap = (int)(self->priorModelPars[2]); + + // Weight parameters for features. + weightIndPrior0 = self->priorModelPars[4]; + weightIndPrior1 = self->priorModelPars[5]; + weightIndPrior2 = self->priorModelPars[6]; + + // Compute feature based on average LR factor. + // This is the average over all frequencies of the smooth log LRT. + logLrtTimeAvgKsum = 0.0; + for (i = 0; i < self->magnLen; i++) { + tmpFloat1 = 1.f + 2.f * snrLocPrior[i]; + tmpFloat2 = 2.f * snrLocPrior[i] / (tmpFloat1 + 0.0001f); + besselTmp = (snrLocPost[i] + 1.f) * tmpFloat2; + self->logLrtTimeAvg[i] += + LRT_TAVG * (besselTmp - (float)log(tmpFloat1) - self->logLrtTimeAvg[i]); + logLrtTimeAvgKsum += self->logLrtTimeAvg[i]; + } + logLrtTimeAvgKsum = (float)logLrtTimeAvgKsum / (self->magnLen); + self->featureData[3] = logLrtTimeAvgKsum; + // Done with computation of LR factor. + + // Compute the indicator functions. + // Average LRT feature. + widthPrior = widthPrior0; + // Use larger width in tanh map for pause regions. + if (logLrtTimeAvgKsum < threshPrior0) { + widthPrior = widthPrior1; + } + // Compute indicator function: sigmoid map. + indicator0 = + 0.5f * + ((float)tanh(widthPrior * (logLrtTimeAvgKsum - threshPrior0)) + 1.f); + + // Spectral flatness feature. + tmpFloat1 = self->featureData[0]; + widthPrior = widthPrior0; + // Use larger width in tanh map for pause regions. + if (sgnMap == 1 && (tmpFloat1 > threshPrior1)) { + widthPrior = widthPrior1; + } + if (sgnMap == -1 && (tmpFloat1 < threshPrior1)) { + widthPrior = widthPrior1; + } + // Compute indicator function: sigmoid map. + indicator1 = + 0.5f * + ((float)tanh((float)sgnMap * widthPrior * (threshPrior1 - tmpFloat1)) + + 1.f); + + // For template spectrum-difference. + tmpFloat1 = self->featureData[4]; + widthPrior = widthPrior0; + // Use larger width in tanh map for pause regions. + if (tmpFloat1 < threshPrior2) { + widthPrior = widthPrior2; + } + // Compute indicator function: sigmoid map. + indicator2 = + 0.5f * ((float)tanh(widthPrior * (tmpFloat1 - threshPrior2)) + 1.f); + + // Combine the indicator function with the feature weights. + indPrior = weightIndPrior0 * indicator0 + weightIndPrior1 * indicator1 + + weightIndPrior2 * indicator2; + // Done with computing indicator function. + + // Compute the prior probability. + self->priorSpeechProb += PRIOR_UPDATE * (indPrior - self->priorSpeechProb); + // Make sure probabilities are within range: keep floor to 0.01. + if (self->priorSpeechProb > 1.f) { + self->priorSpeechProb = 1.f; + } + if (self->priorSpeechProb < 0.01f) { + self->priorSpeechProb = 0.01f; + } + + // Final speech probability: combine prior model with LR factor:. + gainPrior = (1.f - self->priorSpeechProb) / (self->priorSpeechProb + 0.0001f); + for (i = 0; i < self->magnLen; i++) { + invLrt = (float)exp(-self->logLrtTimeAvg[i]); + invLrt = (float)gainPrior * invLrt; + probSpeechFinal[i] = 1.f / (1.f + invLrt); + } +} + +// Update the noise features. +// Inputs: +// * |magn| is the signal magnitude spectrum estimate. +// * |updateParsFlag| is an update flag for parameters. +static void FeatureUpdate(NoiseSuppressionC* self, + const float* magn, + int updateParsFlag) { + // Compute spectral flatness on input spectrum. + ComputeSpectralFlatness(self, magn); + // Compute difference of input spectrum with learned/estimated noise spectrum. + ComputeSpectralDifference(self, magn); + // Compute histograms for parameter decisions (thresholds and weights for + // features). + // Parameters are extracted once every window time. + // (=self->modelUpdatePars[1]) + if (updateParsFlag >= 1) { + // Counter update. + self->modelUpdatePars[3]--; + // Update histogram. + if (self->modelUpdatePars[3] > 0) { + FeatureParameterExtraction(self, 0); + } + // Compute model parameters. + if (self->modelUpdatePars[3] == 0) { + FeatureParameterExtraction(self, 1); + self->modelUpdatePars[3] = self->modelUpdatePars[1]; + // If wish to update only once, set flag to zero. + if (updateParsFlag == 1) { + self->modelUpdatePars[0] = 0; + } else { + // Update every window: + // Get normalization for spectral difference for next window estimate. + self->featureData[6] = + self->featureData[6] / ((float)self->modelUpdatePars[1]); + self->featureData[5] = + 0.5f * (self->featureData[6] + self->featureData[5]); + self->featureData[6] = 0.f; + } + } + } +} + +// Update the noise estimate. +// Inputs: +// * |magn| is the signal magnitude spectrum estimate. +// * |snrLocPrior| is the prior SNR. +// * |snrLocPost| is the post SNR. +// Output: +// * |noise| is the updated noise magnitude spectrum estimate. +static void UpdateNoiseEstimate(NoiseSuppressionC* self, + const float* magn, + const float* snrLocPrior, + const float* snrLocPost, + float* noise) { + size_t i; + float probSpeech, probNonSpeech; + // Time-avg parameter for noise update. + float gammaNoiseTmp = NOISE_UPDATE; + float gammaNoiseOld; + float noiseUpdateTmp; + + for (i = 0; i < self->magnLen; i++) { + probSpeech = self->speechProb[i]; + probNonSpeech = 1.f - probSpeech; + // Temporary noise update: + // Use it for speech frames if update value is less than previous. + noiseUpdateTmp = gammaNoiseTmp * self->noisePrev[i] + + (1.f - gammaNoiseTmp) * (probNonSpeech * magn[i] + + probSpeech * self->noisePrev[i]); + // Time-constant based on speech/noise state. + gammaNoiseOld = gammaNoiseTmp; + gammaNoiseTmp = NOISE_UPDATE; + // Increase gamma (i.e., less noise update) for frame likely to be speech. + if (probSpeech > PROB_RANGE) { + gammaNoiseTmp = SPEECH_UPDATE; + } + // Conservative noise update. + if (probSpeech < PROB_RANGE) { + self->magnAvgPause[i] += GAMMA_PAUSE * (magn[i] - self->magnAvgPause[i]); + } + // Noise update. + if (gammaNoiseTmp == gammaNoiseOld) { + noise[i] = noiseUpdateTmp; + } else { + noise[i] = gammaNoiseTmp * self->noisePrev[i] + + (1.f - gammaNoiseTmp) * (probNonSpeech * magn[i] + + probSpeech * self->noisePrev[i]); + // Allow for noise update downwards: + // If noise update decreases the noise, it is safe, so allow it to + // happen. + if (noiseUpdateTmp < noise[i]) { + noise[i] = noiseUpdateTmp; + } + } + } // End of freq loop. +} + +// Updates |buffer| with a new |frame|. +// Inputs: +// * |frame| is a new speech frame or NULL for setting to zero. +// * |frame_length| is the length of the new frame. +// * |buffer_length| is the length of the buffer. +// Output: +// * |buffer| is the updated buffer. +static void UpdateBuffer(const float* frame, + size_t frame_length, + size_t buffer_length, + float* buffer) { + RTC_DCHECK_LT(buffer_length, 2 * frame_length); + + memcpy(buffer, + buffer + frame_length, + sizeof(*buffer) * (buffer_length - frame_length)); + if (frame) { + memcpy(buffer + buffer_length - frame_length, + frame, + sizeof(*buffer) * frame_length); + } else { + memset(buffer + buffer_length - frame_length, + 0, + sizeof(*buffer) * frame_length); + } +} + +// Transforms the signal from time to frequency domain. +// Inputs: +// * |time_data| is the signal in the time domain. +// * |time_data_length| is the length of the analysis buffer. +// * |magnitude_length| is the length of the spectrum magnitude, which equals +// the length of both |real| and |imag| (time_data_length / 2 + 1). +// Outputs: +// * |time_data| is the signal in the frequency domain. +// * |real| is the real part of the frequency domain. +// * |imag| is the imaginary part of the frequency domain. +// * |magn| is the calculated signal magnitude in the frequency domain. +static void FFT(NoiseSuppressionC* self, + float* time_data, + size_t time_data_length, + size_t magnitude_length, + float* real, + float* imag, + float* magn) { + size_t i; + + RTC_DCHECK_EQ(magnitude_length, time_data_length / 2 + 1); + + WebRtc_rdft(time_data_length, 1, time_data, self->ip, self->wfft); + + imag[0] = 0; + real[0] = time_data[0]; + magn[0] = fabsf(real[0]) + 1.f; + imag[magnitude_length - 1] = 0; + real[magnitude_length - 1] = time_data[1]; + magn[magnitude_length - 1] = fabsf(real[magnitude_length - 1]) + 1.f; + for (i = 1; i < magnitude_length - 1; ++i) { + real[i] = time_data[2 * i]; + imag[i] = time_data[2 * i + 1]; + // Magnitude spectrum. + magn[i] = sqrtf(real[i] * real[i] + imag[i] * imag[i]) + 1.f; + } +} + +// Transforms the signal from frequency to time domain. +// Inputs: +// * |real| is the real part of the frequency domain. +// * |imag| is the imaginary part of the frequency domain. +// * |magnitude_length| is the length of the spectrum magnitude, which equals +// the length of both |real| and |imag|. +// * |time_data_length| is the length of the analysis buffer +// (2 * (magnitude_length - 1)). +// Output: +// * |time_data| is the signal in the time domain. +static void IFFT(NoiseSuppressionC* self, + const float* real, + const float* imag, + size_t magnitude_length, + size_t time_data_length, + float* time_data) { + size_t i; + + RTC_DCHECK_EQ(time_data_length, 2 * (magnitude_length - 1)); + + time_data[0] = real[0]; + time_data[1] = real[magnitude_length - 1]; + for (i = 1; i < magnitude_length - 1; ++i) { + time_data[2 * i] = real[i]; + time_data[2 * i + 1] = imag[i]; + } + WebRtc_rdft(time_data_length, -1, time_data, self->ip, self->wfft); + + for (i = 0; i < time_data_length; ++i) { + time_data[i] *= 2.f / time_data_length; // FFT scaling. + } +} + +// Calculates the energy of a buffer. +// Inputs: +// * |buffer| is the buffer over which the energy is calculated. +// * |length| is the length of the buffer. +// Returns the calculated energy. +static float Energy(const float* buffer, size_t length) { + size_t i; + float energy = 0.f; + + for (i = 0; i < length; ++i) { + energy += buffer[i] * buffer[i]; + } + + return energy; +} + +// Windows a buffer. +// Inputs: +// * |window| is the window by which to multiply. +// * |data| is the data without windowing. +// * |length| is the length of the window and data. +// Output: +// * |data_windowed| is the windowed data. +static void Windowing(const float* window, + const float* data, + size_t length, + float* data_windowed) { + size_t i; + + for (i = 0; i < length; ++i) { + data_windowed[i] = window[i] * data[i]; + } +} + +// Estimate prior SNR decision-directed and compute DD based Wiener Filter. +// Input: +// * |magn| is the signal magnitude spectrum estimate. +// Output: +// * |theFilter| is the frequency response of the computed Wiener filter. +static void ComputeDdBasedWienerFilter(const NoiseSuppressionC* self, + const float* magn, + float* theFilter) { + size_t i; + float snrPrior, previousEstimateStsa, currentEstimateStsa; + + for (i = 0; i < self->magnLen; i++) { + // Previous estimate: based on previous frame with gain filter. + previousEstimateStsa = self->magnPrevProcess[i] / + (self->noisePrev[i] + 0.0001f) * self->smooth[i]; + // Post and prior SNR. + currentEstimateStsa = 0.f; + if (magn[i] > self->noise[i]) { + currentEstimateStsa = magn[i] / (self->noise[i] + 0.0001f) - 1.f; + } + // DD estimate is sum of two terms: current estimate and previous estimate. + // Directed decision update of |snrPrior|. + snrPrior = DD_PR_SNR * previousEstimateStsa + + (1.f - DD_PR_SNR) * currentEstimateStsa; + // Gain filter. + theFilter[i] = snrPrior / (self->overdrive + snrPrior); + } // End of loop over frequencies. +} + +// Changes the aggressiveness of the noise suppression method. +// |mode| = 0 is mild (6dB), |mode| = 1 is medium (10dB) and |mode| = 2 is +// aggressive (15dB). +// Returns 0 on success and -1 otherwise. +int WebRtcNs_set_policy_core(NoiseSuppressionC* self, int mode) { + // Allow for modes: 0, 1, 2, 3. + if (mode < 0 || mode > 3) { + return (-1); + } + + self->aggrMode = mode; + if (mode == 0) { + self->overdrive = 1.f; + self->denoiseBound = 0.5f; + self->gainmap = 0; + } else if (mode == 1) { + // self->overdrive = 1.25f; + self->overdrive = 1.f; + self->denoiseBound = 0.25f; + self->gainmap = 1; + } else if (mode == 2) { + // self->overdrive = 1.25f; + self->overdrive = 1.1f; + self->denoiseBound = 0.125f; + self->gainmap = 1; + } else if (mode == 3) { + // self->overdrive = 1.3f; + self->overdrive = 1.25f; + self->denoiseBound = 0.09f; + self->gainmap = 1; + } + return 0; +} + +void WebRtcNs_AnalyzeCore(NoiseSuppressionC* self, const float* speechFrame) { + size_t i; + const size_t kStartBand = 5; // Skip first frequency bins during estimation. + int updateParsFlag; + float energy; + float signalEnergy = 0.f; + float sumMagn = 0.f; + float tmpFloat1, tmpFloat2, tmpFloat3; + float winData[ANAL_BLOCKL_MAX]; + float magn[HALF_ANAL_BLOCKL], noise[HALF_ANAL_BLOCKL]; + float snrLocPost[HALF_ANAL_BLOCKL], snrLocPrior[HALF_ANAL_BLOCKL]; + float real[ANAL_BLOCKL_MAX], imag[HALF_ANAL_BLOCKL]; + // Variables during startup. + float sum_log_i = 0.0; + float sum_log_i_square = 0.0; + float sum_log_magn = 0.0; + float sum_log_i_log_magn = 0.0; + float parametric_exp = 0.0; + float parametric_num = 0.0; + + // Check that initiation has been done. + RTC_DCHECK_EQ(1, self->initFlag); + updateParsFlag = self->modelUpdatePars[0]; + + // Update analysis buffer for L band. + UpdateBuffer(speechFrame, self->blockLen, self->anaLen, self->analyzeBuf); + + Windowing(self->window, self->analyzeBuf, self->anaLen, winData); + energy = Energy(winData, self->anaLen); + if (energy == 0.0) { + // We want to avoid updating statistics in this case: + // Updating feature statistics when we have zeros only will cause + // thresholds to move towards zero signal situations. This in turn has the + // effect that once the signal is "turned on" (non-zero values) everything + // will be treated as speech and there is no noise suppression effect. + // Depending on the duration of the inactive signal it takes a + // considerable amount of time for the system to learn what is noise and + // what is speech. + return; + } + + self->blockInd++; // Update the block index only when we process a block. + + FFT(self, winData, self->anaLen, self->magnLen, real, imag, magn); + + for (i = 0; i < self->magnLen; i++) { + signalEnergy += real[i] * real[i] + imag[i] * imag[i]; + sumMagn += magn[i]; + if (self->blockInd < END_STARTUP_SHORT) { + if (i >= kStartBand) { + tmpFloat2 = logf((float)i); + sum_log_i += tmpFloat2; + sum_log_i_square += tmpFloat2 * tmpFloat2; + tmpFloat1 = logf(magn[i]); + sum_log_magn += tmpFloat1; + sum_log_i_log_magn += tmpFloat2 * tmpFloat1; + } + } + } + signalEnergy /= self->magnLen; + self->signalEnergy = signalEnergy; + self->sumMagn = sumMagn; + + // Quantile noise estimate. + NoiseEstimation(self, magn, noise); + // Compute simplified noise model during startup. + if (self->blockInd < END_STARTUP_SHORT) { + // Estimate White noise. + self->whiteNoiseLevel += sumMagn / self->magnLen * self->overdrive; + // Estimate Pink noise parameters. + tmpFloat1 = sum_log_i_square * (self->magnLen - kStartBand); + tmpFloat1 -= (sum_log_i * sum_log_i); + tmpFloat2 = + (sum_log_i_square * sum_log_magn - sum_log_i * sum_log_i_log_magn); + tmpFloat3 = tmpFloat2 / tmpFloat1; + // Constrain the estimated spectrum to be positive. + if (tmpFloat3 < 0.f) { + tmpFloat3 = 0.f; + } + self->pinkNoiseNumerator += tmpFloat3; + tmpFloat2 = (sum_log_i * sum_log_magn); + tmpFloat2 -= (self->magnLen - kStartBand) * sum_log_i_log_magn; + tmpFloat3 = tmpFloat2 / tmpFloat1; + // Constrain the pink noise power to be in the interval [0, 1]. + if (tmpFloat3 < 0.f) { + tmpFloat3 = 0.f; + } + if (tmpFloat3 > 1.f) { + tmpFloat3 = 1.f; + } + self->pinkNoiseExp += tmpFloat3; + + // Calculate frequency independent parts of parametric noise estimate. + if (self->pinkNoiseExp > 0.f) { + // Use pink noise estimate. + parametric_num = + expf(self->pinkNoiseNumerator / (float)(self->blockInd + 1)); + parametric_num *= (float)(self->blockInd + 1); + parametric_exp = self->pinkNoiseExp / (float)(self->blockInd + 1); + } + for (i = 0; i < self->magnLen; i++) { + // Estimate the background noise using the white and pink noise + // parameters. + if (self->pinkNoiseExp == 0.f) { + // Use white noise estimate. + self->parametricNoise[i] = self->whiteNoiseLevel; + } else { + // Use pink noise estimate. + float use_band = (float)(i < kStartBand ? kStartBand : i); + self->parametricNoise[i] = + parametric_num / powf(use_band, parametric_exp); + } + // Weight quantile noise with modeled noise. + noise[i] *= (self->blockInd); + tmpFloat2 = + self->parametricNoise[i] * (END_STARTUP_SHORT - self->blockInd); + noise[i] += (tmpFloat2 / (float)(self->blockInd + 1)); + noise[i] /= END_STARTUP_SHORT; + } + } + // Compute average signal during END_STARTUP_LONG time: + // used to normalize spectral difference measure. + if (self->blockInd < END_STARTUP_LONG) { + self->featureData[5] *= self->blockInd; + self->featureData[5] += signalEnergy; + self->featureData[5] /= (self->blockInd + 1); + } + + // Post and prior SNR needed for SpeechNoiseProb. + ComputeSnr(self, magn, noise, snrLocPrior, snrLocPost); + + FeatureUpdate(self, magn, updateParsFlag); + SpeechNoiseProb(self, self->speechProb, snrLocPrior, snrLocPost); + UpdateNoiseEstimate(self, magn, snrLocPrior, snrLocPost, noise); + + // Keep track of noise spectrum for next frame. + memcpy(self->noise, noise, sizeof(*noise) * self->magnLen); + memcpy(self->magnPrevAnalyze, magn, sizeof(*magn) * self->magnLen); +} + +void WebRtcNs_ProcessCore(NoiseSuppressionC* self, + const float* const* speechFrame, + size_t num_bands, + float* const* outFrame) { + // Main routine for noise reduction. + int flagHB = 0; + size_t i, j; + + float energy1, energy2, gain, factor, factor1, factor2; + float fout[BLOCKL_MAX]; + float winData[ANAL_BLOCKL_MAX]; + float magn[HALF_ANAL_BLOCKL]; + float theFilter[HALF_ANAL_BLOCKL], theFilterTmp[HALF_ANAL_BLOCKL]; + float real[ANAL_BLOCKL_MAX], imag[HALF_ANAL_BLOCKL]; + + // SWB variables. + int deltaBweHB = 1; + int deltaGainHB = 1; + float decayBweHB = 1.0; + float gainMapParHB = 1.0; + float gainTimeDomainHB = 1.0; + float avgProbSpeechHB, avgProbSpeechHBTmp, avgFilterGainHB, gainModHB; + float sumMagnAnalyze, sumMagnProcess; + + // Check that initiation has been done. + RTC_DCHECK_EQ(1, self->initFlag); + RTC_DCHECK_LE(num_bands - 1, NUM_HIGH_BANDS_MAX); + + const float* const* speechFrameHB = NULL; + float* const* outFrameHB = NULL; + size_t num_high_bands = 0; + if (num_bands > 1) { + speechFrameHB = &speechFrame[1]; + outFrameHB = &outFrame[1]; + num_high_bands = num_bands - 1; + flagHB = 1; + // Range for averaging low band quantities for H band gain. + deltaBweHB = (int)self->magnLen / 4; + deltaGainHB = deltaBweHB; + } + + // Update analysis buffer for L band. + UpdateBuffer(speechFrame[0], self->blockLen, self->anaLen, self->dataBuf); + + if (flagHB == 1) { + // Update analysis buffer for H bands. + for (i = 0; i < num_high_bands; ++i) { + UpdateBuffer(speechFrameHB[i], + self->blockLen, + self->anaLen, + self->dataBufHB[i]); + } + } + + Windowing(self->window, self->dataBuf, self->anaLen, winData); + energy1 = Energy(winData, self->anaLen); + if (energy1 == 0.0) { + // Synthesize the special case of zero input. + // Read out fully processed segment. + for (i = self->windShift; i < self->blockLen + self->windShift; i++) { + fout[i - self->windShift] = self->syntBuf[i]; + } + // Update synthesis buffer. + UpdateBuffer(NULL, self->blockLen, self->anaLen, self->syntBuf); + + for (i = 0; i < self->blockLen; ++i) + outFrame[0][i] = + WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, fout[i], WEBRTC_SPL_WORD16_MIN); + + // For time-domain gain of HB. + if (flagHB == 1) { + for (i = 0; i < num_high_bands; ++i) { + for (j = 0; j < self->blockLen; ++j) { + outFrameHB[i][j] = WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, + self->dataBufHB[i][j], + WEBRTC_SPL_WORD16_MIN); + } + } + } + + return; + } + + FFT(self, winData, self->anaLen, self->magnLen, real, imag, magn); + + if (self->blockInd < END_STARTUP_SHORT) { + for (i = 0; i < self->magnLen; i++) { + self->initMagnEst[i] += magn[i]; + } + } + + ComputeDdBasedWienerFilter(self, magn, theFilter); + + for (i = 0; i < self->magnLen; i++) { + // Flooring bottom. + if (theFilter[i] < self->denoiseBound) { + theFilter[i] = self->denoiseBound; + } + // Flooring top. + if (theFilter[i] > 1.f) { + theFilter[i] = 1.f; + } + if (self->blockInd < END_STARTUP_SHORT) { + theFilterTmp[i] = + (self->initMagnEst[i] - self->overdrive * self->parametricNoise[i]); + theFilterTmp[i] /= (self->initMagnEst[i] + 0.0001f); + // Flooring bottom. + if (theFilterTmp[i] < self->denoiseBound) { + theFilterTmp[i] = self->denoiseBound; + } + // Flooring top. + if (theFilterTmp[i] > 1.f) { + theFilterTmp[i] = 1.f; + } + // Weight the two suppression filters. + theFilter[i] *= (self->blockInd); + theFilterTmp[i] *= (END_STARTUP_SHORT - self->blockInd); + theFilter[i] += theFilterTmp[i]; + theFilter[i] /= (END_STARTUP_SHORT); + } + + self->smooth[i] = theFilter[i]; + real[i] *= self->smooth[i]; + imag[i] *= self->smooth[i]; + } + // Keep track of |magn| spectrum for next frame. + memcpy(self->magnPrevProcess, magn, sizeof(*magn) * self->magnLen); + memcpy(self->noisePrev, self->noise, sizeof(self->noise[0]) * self->magnLen); + // Back to time domain. + IFFT(self, real, imag, self->magnLen, self->anaLen, winData); + + // Scale factor: only do it after END_STARTUP_LONG time. + factor = 1.f; + if (self->gainmap == 1 && self->blockInd > END_STARTUP_LONG) { + factor1 = 1.f; + factor2 = 1.f; + + energy2 = Energy(winData, self->anaLen); + gain = (float)sqrt(energy2 / (energy1 + 1.f)); + + // Scaling for new version. + if (gain > B_LIM) { + factor1 = 1.f + 1.3f * (gain - B_LIM); + if (gain * factor1 > 1.f) { + factor1 = 1.f / gain; + } + } + if (gain < B_LIM) { + // Don't reduce scale too much for pause regions: + // attenuation here should be controlled by flooring. + if (gain <= self->denoiseBound) { + gain = self->denoiseBound; + } + factor2 = 1.f - 0.3f * (B_LIM - gain); + } + // Combine both scales with speech/noise prob: + // note prior (priorSpeechProb) is not frequency dependent. + factor = self->priorSpeechProb * factor1 + + (1.f - self->priorSpeechProb) * factor2; + } // Out of self->gainmap == 1. + + Windowing(self->window, winData, self->anaLen, winData); + + // Synthesis. + for (i = 0; i < self->anaLen; i++) { + self->syntBuf[i] += factor * winData[i]; + } + // Read out fully processed segment. + for (i = self->windShift; i < self->blockLen + self->windShift; i++) { + fout[i - self->windShift] = self->syntBuf[i]; + } + // Update synthesis buffer. + UpdateBuffer(NULL, self->blockLen, self->anaLen, self->syntBuf); + + for (i = 0; i < self->blockLen; ++i) + outFrame[0][i] = + WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, fout[i], WEBRTC_SPL_WORD16_MIN); + + // For time-domain gain of HB. + if (flagHB == 1) { + // Average speech prob from low band. + // Average over second half (i.e., 4->8kHz) of frequencies spectrum. + avgProbSpeechHB = 0.0; + for (i = self->magnLen - deltaBweHB - 1; i < self->magnLen - 1; i++) { + avgProbSpeechHB += self->speechProb[i]; + } + avgProbSpeechHB = avgProbSpeechHB / ((float)deltaBweHB); + // If the speech was suppressed by a component between Analyze and + // Process, for example the AEC, then it should not be considered speech + // for high band suppression purposes. + sumMagnAnalyze = 0; + sumMagnProcess = 0; + for (i = 0; i < self->magnLen; ++i) { + sumMagnAnalyze += self->magnPrevAnalyze[i]; + sumMagnProcess += self->magnPrevProcess[i]; + } + avgProbSpeechHB *= sumMagnProcess / sumMagnAnalyze; + // Average filter gain from low band. + // Average over second half (i.e., 4->8kHz) of frequencies spectrum. + avgFilterGainHB = 0.0; + for (i = self->magnLen - deltaGainHB - 1; i < self->magnLen - 1; i++) { + avgFilterGainHB += self->smooth[i]; + } + avgFilterGainHB = avgFilterGainHB / ((float)(deltaGainHB)); + avgProbSpeechHBTmp = 2.f * avgProbSpeechHB - 1.f; + // Gain based on speech probability. + gainModHB = 0.5f * (1.f + (float)tanh(gainMapParHB * avgProbSpeechHBTmp)); + // Combine gain with low band gain. + gainTimeDomainHB = 0.5f * gainModHB + 0.5f * avgFilterGainHB; + if (avgProbSpeechHB >= 0.5f) { + gainTimeDomainHB = 0.25f * gainModHB + 0.75f * avgFilterGainHB; + } + gainTimeDomainHB = gainTimeDomainHB * decayBweHB; + // Make sure gain is within flooring range. + // Flooring bottom. + if (gainTimeDomainHB < self->denoiseBound) { + gainTimeDomainHB = self->denoiseBound; + } + // Flooring top. + if (gainTimeDomainHB > 1.f) { + gainTimeDomainHB = 1.f; + } + // Apply gain. + for (i = 0; i < num_high_bands; ++i) { + for (j = 0; j < self->blockLen; j++) { + outFrameHB[i][j] = + WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, + gainTimeDomainHB * self->dataBufHB[i][j], + WEBRTC_SPL_WORD16_MIN); + } + } + } // End of H band gain computation. +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.h b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.h new file mode 100644 index 0000000000..97f76baf0d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/ns_core.h @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NS_NS_CORE_H_ +#define MODULES_AUDIO_PROCESSING_NS_NS_CORE_H_ + +#include "modules/audio_processing/ns/defines.h" + +typedef struct NSParaExtract_ { + // Bin size of histogram. + float binSizeLrt; + float binSizeSpecFlat; + float binSizeSpecDiff; + // Range of histogram over which LRT threshold is computed. + float rangeAvgHistLrt; + // Scale parameters: multiply dominant peaks of the histograms by scale factor + // to obtain thresholds for prior model. + float factor1ModelPars; // For LRT and spectral difference. + float factor2ModelPars; // For spectral_flatness: used when noise is flatter + // than speech. + // Peak limit for spectral flatness (varies between 0 and 1). + float thresPosSpecFlat; + // Limit on spacing of two highest peaks in histogram: spacing determined by + // bin size. + float limitPeakSpacingSpecFlat; + float limitPeakSpacingSpecDiff; + // Limit on relevance of second peak. + float limitPeakWeightsSpecFlat; + float limitPeakWeightsSpecDiff; + // Limit on fluctuation of LRT feature. + float thresFluctLrt; + // Limit on the max and min values for the feature thresholds. + float maxLrt; + float minLrt; + float maxSpecFlat; + float minSpecFlat; + float maxSpecDiff; + float minSpecDiff; + // Criteria of weight of histogram peak to accept/reject feature. + int thresWeightSpecFlat; + int thresWeightSpecDiff; + +} NSParaExtract; + +typedef struct NoiseSuppressionC_ { + uint32_t fs; + size_t blockLen; + size_t windShift; + size_t anaLen; + size_t magnLen; + int aggrMode; + const float* window; + float analyzeBuf[ANAL_BLOCKL_MAX]; + float dataBuf[ANAL_BLOCKL_MAX]; + float syntBuf[ANAL_BLOCKL_MAX]; + + int initFlag; + // Parameters for quantile noise estimation. + float density[SIMULT * HALF_ANAL_BLOCKL]; + float lquantile[SIMULT * HALF_ANAL_BLOCKL]; + float quantile[HALF_ANAL_BLOCKL]; + int counter[SIMULT]; + int updates; + // Parameters for Wiener filter. + float smooth[HALF_ANAL_BLOCKL]; + float overdrive; + float denoiseBound; + int gainmap; + // FFT work arrays. + size_t ip[IP_LENGTH]; + float wfft[W_LENGTH]; + + // Parameters for new method: some not needed, will reduce/cleanup later. + int32_t blockInd; // Frame index counter. + int modelUpdatePars[4]; // Parameters for updating or estimating. + // Thresholds/weights for prior model. + float priorModelPars[7]; // Parameters for prior model. + float noise[HALF_ANAL_BLOCKL]; // Noise spectrum from current frame. + float noisePrev[HALF_ANAL_BLOCKL]; // Noise spectrum from previous frame. + // Magnitude spectrum of previous analyze frame. + float magnPrevAnalyze[HALF_ANAL_BLOCKL]; + // Magnitude spectrum of previous process frame. + float magnPrevProcess[HALF_ANAL_BLOCKL]; + float logLrtTimeAvg[HALF_ANAL_BLOCKL]; // Log LRT factor with time-smoothing. + float priorSpeechProb; // Prior speech/noise probability. + float featureData[7]; + // Conservative noise spectrum estimate. + float magnAvgPause[HALF_ANAL_BLOCKL]; + float signalEnergy; // Energy of |magn|. + float sumMagn; + float whiteNoiseLevel; // Initial noise estimate. + float initMagnEst[HALF_ANAL_BLOCKL]; // Initial magnitude spectrum estimate. + float pinkNoiseNumerator; // Pink noise parameter: numerator. + float pinkNoiseExp; // Pink noise parameter: power of frequencies. + float parametricNoise[HALF_ANAL_BLOCKL]; + // Parameters for feature extraction. + NSParaExtract featureExtractionParams; + // Histograms for parameter estimation. + int histLrt[HIST_PAR_EST]; + int histSpecFlat[HIST_PAR_EST]; + int histSpecDiff[HIST_PAR_EST]; + // Quantities for high band estimate. + float speechProb[HALF_ANAL_BLOCKL]; // Final speech/noise prob: prior + LRT. + // Buffering data for HB. + float dataBufHB[NUM_HIGH_BANDS_MAX][ANAL_BLOCKL_MAX]; + +} NoiseSuppressionC; + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * WebRtcNs_InitCore(...) + * + * This function initializes a noise suppression instance + * + * Input: + * - self : Instance that should be initialized + * - fs : Sampling frequency + * + * Output: + * - self : Initialized instance + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcNs_InitCore(NoiseSuppressionC* self, uint32_t fs); + +/**************************************************************************** + * WebRtcNs_set_policy_core(...) + * + * This changes the aggressiveness of the noise suppression method. + * + * Input: + * - self : Instance that should be initialized + * - mode : 0: Mild (6dB), 1: Medium (10dB), 2: Aggressive (15dB) + * + * Output: + * - self : Initialized instance + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcNs_set_policy_core(NoiseSuppressionC* self, int mode); + +/**************************************************************************** + * WebRtcNs_AnalyzeCore + * + * Estimate the background noise. + * + * Input: + * - self : Instance that should be initialized + * - speechFrame : Input speech frame for lower band + * + * Output: + * - self : Updated instance + */ +void WebRtcNs_AnalyzeCore(NoiseSuppressionC* self, const float* speechFrame); + +/**************************************************************************** + * WebRtcNs_ProcessCore + * + * Do noise suppression. + * + * Input: + * - self : Instance that should be initialized + * - inFrame : Input speech frame for each band + * - num_bands : Number of bands + * + * Output: + * - self : Updated instance + * - outFrame : Output speech frame for each band + */ +void WebRtcNs_ProcessCore(NoiseSuppressionC* self, + const float* const* inFrame, + size_t num_bands, + float* const* outFrame); + +#ifdef __cplusplus +} +#endif +#endif // MODULES_AUDIO_PROCESSING_NS_NS_CORE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.c b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.c new file mode 100644 index 0000000000..8043656029 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.c @@ -0,0 +1,2107 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/ns/noise_suppression_x.h" + +#include +#include +#include + +#include "rtc_base/checks.h" +#include "common_audio/signal_processing/include/real_fft.h" +#include "modules/audio_processing/ns/nsx_core.h" +#include "system_wrappers/include/cpu_features_wrapper.h" + +#if defined(WEBRTC_HAS_NEON) +/* Tables are defined in ARM assembly files. */ +extern const int16_t WebRtcNsx_kLogTable[9]; +extern const int16_t WebRtcNsx_kCounterDiv[201]; +extern const int16_t WebRtcNsx_kLogTableFrac[256]; +#else +static const int16_t WebRtcNsx_kLogTable[9] = { + 0, 177, 355, 532, 710, 887, 1065, 1242, 1420 +}; + +static const int16_t WebRtcNsx_kCounterDiv[201] = { + 32767, 16384, 10923, 8192, 6554, 5461, 4681, 4096, 3641, 3277, 2979, 2731, + 2521, 2341, 2185, 2048, 1928, 1820, 1725, 1638, 1560, 1489, 1425, 1365, 1311, + 1260, 1214, 1170, 1130, 1092, 1057, 1024, 993, 964, 936, 910, 886, 862, 840, + 819, 799, 780, 762, 745, 728, 712, 697, 683, 669, 655, 643, 630, 618, 607, + 596, 585, 575, 565, 555, 546, 537, 529, 520, 512, 504, 496, 489, 482, 475, + 468, 462, 455, 449, 443, 437, 431, 426, 420, 415, 410, 405, 400, 395, 390, + 386, 381, 377, 372, 368, 364, 360, 356, 352, 349, 345, 341, 338, 334, 331, + 328, 324, 321, 318, 315, 312, 309, 306, 303, 301, 298, 295, 293, 290, 287, + 285, 282, 280, 278, 275, 273, 271, 269, 266, 264, 262, 260, 258, 256, 254, + 252, 250, 248, 246, 245, 243, 241, 239, 237, 236, 234, 232, 231, 229, 228, + 226, 224, 223, 221, 220, 218, 217, 216, 214, 213, 211, 210, 209, 207, 206, + 205, 204, 202, 201, 200, 199, 197, 196, 195, 194, 193, 192, 191, 189, 188, + 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, + 172, 172, 171, 170, 169, 168, 167, 166, 165, 165, 164, 163 +}; + +static const int16_t WebRtcNsx_kLogTableFrac[256] = { + 0, 1, 3, 4, 6, 7, 9, 10, 11, 13, 14, 16, 17, 18, 20, 21, + 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38, 40, 41, 42, + 44, 45, 46, 47, 49, 50, 51, 52, 54, 55, 56, 57, 59, 60, 61, 62, + 63, 65, 66, 67, 68, 69, 71, 72, 73, 74, 75, 77, 78, 79, 80, 81, + 82, 84, 85, 86, 87, 88, 89, 90, 92, 93, 94, 95, 96, 97, 98, 99, + 100, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 116, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, + 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, + 147, 148, 149, 150, 151, 152, 153, 154, 155, 155, 156, 157, 158, 159, 160, + 161, 162, 163, 164, 165, 166, 167, 168, 169, 169, 170, 171, 172, 173, 174, + 175, 176, 177, 178, 178, 179, 180, 181, 182, 183, 184, 185, 185, 186, 187, + 188, 189, 190, 191, 192, 192, 193, 194, 195, 196, 197, 198, 198, 199, 200, + 201, 202, 203, 203, 204, 205, 206, 207, 208, 208, 209, 210, 211, 212, 212, + 213, 214, 215, 216, 216, 217, 218, 219, 220, 220, 221, 222, 223, 224, 224, + 225, 226, 227, 228, 228, 229, 230, 231, 231, 232, 233, 234, 234, 235, 236, + 237, 238, 238, 239, 240, 241, 241, 242, 243, 244, 244, 245, 246, 247, 247, + 248, 249, 249, 250, 251, 252, 252, 253, 254, 255, 255 +}; +#endif // WEBRTC_HAS_NEON + +// Skip first frequency bins during estimation. (0 <= value < 64) +static const size_t kStartBand = 5; + +// hybrib Hanning & flat window +static const int16_t kBlocks80w128x[128] = { + 0, 536, 1072, 1606, 2139, 2669, 3196, 3720, 4240, 4756, 5266, + 5771, 6270, 6762, 7246, 7723, 8192, 8652, 9102, 9543, 9974, 10394, + 10803, 11200, 11585, 11958, 12318, 12665, 12998, 13318, 13623, 13913, 14189, + 14449, 14694, 14924, 15137, 15334, 15515, 15679, 15826, 15956, 16069, 16165, + 16244, 16305, 16349, 16375, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16375, 16349, 16305, 16244, 16165, 16069, 15956, + 15826, 15679, 15515, 15334, 15137, 14924, 14694, 14449, 14189, 13913, 13623, + 13318, 12998, 12665, 12318, 11958, 11585, 11200, 10803, 10394, 9974, 9543, + 9102, 8652, 8192, 7723, 7246, 6762, 6270, 5771, 5266, 4756, 4240, + 3720, 3196, 2669, 2139, 1606, 1072, 536 +}; + +// hybrib Hanning & flat window +static const int16_t kBlocks160w256x[256] = { + 0, 268, 536, 804, 1072, 1339, 1606, 1872, + 2139, 2404, 2669, 2933, 3196, 3459, 3720, 3981, + 4240, 4499, 4756, 5012, 5266, 5520, 5771, 6021, + 6270, 6517, 6762, 7005, 7246, 7486, 7723, 7959, + 8192, 8423, 8652, 8878, 9102, 9324, 9543, 9760, + 9974, 10185, 10394, 10600, 10803, 11003, 11200, 11394, + 11585, 11773, 11958, 12140, 12318, 12493, 12665, 12833, + 12998, 13160, 13318, 13472, 13623, 13770, 13913, 14053, + 14189, 14321, 14449, 14574, 14694, 14811, 14924, 15032, + 15137, 15237, 15334, 15426, 15515, 15599, 15679, 15754, + 15826, 15893, 15956, 16015, 16069, 16119, 16165, 16207, + 16244, 16277, 16305, 16329, 16349, 16364, 16375, 16382, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, + 16384, 16382, 16375, 16364, 16349, 16329, 16305, 16277, + 16244, 16207, 16165, 16119, 16069, 16015, 15956, 15893, + 15826, 15754, 15679, 15599, 15515, 15426, 15334, 15237, + 15137, 15032, 14924, 14811, 14694, 14574, 14449, 14321, + 14189, 14053, 13913, 13770, 13623, 13472, 13318, 13160, + 12998, 12833, 12665, 12493, 12318, 12140, 11958, 11773, + 11585, 11394, 11200, 11003, 10803, 10600, 10394, 10185, + 9974, 9760, 9543, 9324, 9102, 8878, 8652, 8423, + 8192, 7959, 7723, 7486, 7246, 7005, 6762, 6517, + 6270, 6021, 5771, 5520, 5266, 5012, 4756, 4499, + 4240, 3981, 3720, 3459, 3196, 2933, 2669, 2404, + 2139, 1872, 1606, 1339, 1072, 804, 536, 268 +}; + +// Gain factor1 table: Input value in Q8 and output value in Q13 +// original floating point code +// if (gain > blim) { +// factor1 = 1.0 + 1.3 * (gain - blim); +// if (gain * factor1 > 1.0) { +// factor1 = 1.0 / gain; +// } +// } else { +// factor1 = 1.0; +// } +static const int16_t kFactor1Table[257] = { + 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8233, 8274, 8315, 8355, 8396, 8436, 8475, 8515, 8554, 8592, 8631, 8669, + 8707, 8745, 8783, 8820, 8857, 8894, 8931, 8967, 9003, 9039, 9075, 9111, 9146, 9181, + 9216, 9251, 9286, 9320, 9354, 9388, 9422, 9456, 9489, 9523, 9556, 9589, 9622, 9655, + 9687, 9719, 9752, 9784, 9816, 9848, 9879, 9911, 9942, 9973, 10004, 10035, 10066, + 10097, 10128, 10158, 10188, 10218, 10249, 10279, 10308, 10338, 10368, 10397, 10426, + 10456, 10485, 10514, 10543, 10572, 10600, 10629, 10657, 10686, 10714, 10742, 10770, + 10798, 10826, 10854, 10882, 10847, 10810, 10774, 10737, 10701, 10666, 10631, 10596, + 10562, 10527, 10494, 10460, 10427, 10394, 10362, 10329, 10297, 10266, 10235, 10203, + 10173, 10142, 10112, 10082, 10052, 10023, 9994, 9965, 9936, 9908, 9879, 9851, 9824, + 9796, 9769, 9742, 9715, 9689, 9662, 9636, 9610, 9584, 9559, 9534, 9508, 9484, 9459, + 9434, 9410, 9386, 9362, 9338, 9314, 9291, 9268, 9245, 9222, 9199, 9176, 9154, 9132, + 9110, 9088, 9066, 9044, 9023, 9002, 8980, 8959, 8939, 8918, 8897, 8877, 8857, 8836, + 8816, 8796, 8777, 8757, 8738, 8718, 8699, 8680, 8661, 8642, 8623, 8605, 8586, 8568, + 8550, 8532, 8514, 8496, 8478, 8460, 8443, 8425, 8408, 8391, 8373, 8356, 8339, 8323, + 8306, 8289, 8273, 8256, 8240, 8224, 8208, 8192 +}; + +// For Factor2 tables +// original floating point code +// if (gain > blim) { +// factor2 = 1.0; +// } else { +// factor2 = 1.0 - 0.3 * (blim - gain); +// if (gain <= inst->denoiseBound) { +// factor2 = 1.0 - 0.3 * (blim - inst->denoiseBound); +// } +// } +// +// Gain factor table: Input value in Q8 and output value in Q13 +static const int16_t kFactor2Aggressiveness1[257] = { + 7577, 7577, 7577, 7577, 7577, 7577, + 7577, 7577, 7577, 7577, 7577, 7577, 7577, 7577, 7577, 7577, 7577, 7596, 7614, 7632, + 7650, 7667, 7683, 7699, 7715, 7731, 7746, 7761, 7775, 7790, 7804, 7818, 7832, 7845, + 7858, 7871, 7884, 7897, 7910, 7922, 7934, 7946, 7958, 7970, 7982, 7993, 8004, 8016, + 8027, 8038, 8049, 8060, 8070, 8081, 8091, 8102, 8112, 8122, 8132, 8143, 8152, 8162, + 8172, 8182, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192 +}; + +// Gain factor table: Input value in Q8 and output value in Q13 +static const int16_t kFactor2Aggressiveness2[257] = { + 7270, 7270, 7270, 7270, 7270, 7306, + 7339, 7369, 7397, 7424, 7448, 7472, 7495, 7517, 7537, 7558, 7577, 7596, 7614, 7632, + 7650, 7667, 7683, 7699, 7715, 7731, 7746, 7761, 7775, 7790, 7804, 7818, 7832, 7845, + 7858, 7871, 7884, 7897, 7910, 7922, 7934, 7946, 7958, 7970, 7982, 7993, 8004, 8016, + 8027, 8038, 8049, 8060, 8070, 8081, 8091, 8102, 8112, 8122, 8132, 8143, 8152, 8162, + 8172, 8182, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192 +}; + +// Gain factor table: Input value in Q8 and output value in Q13 +static const int16_t kFactor2Aggressiveness3[257] = { + 7184, 7184, 7184, 7229, 7270, 7306, + 7339, 7369, 7397, 7424, 7448, 7472, 7495, 7517, 7537, 7558, 7577, 7596, 7614, 7632, + 7650, 7667, 7683, 7699, 7715, 7731, 7746, 7761, 7775, 7790, 7804, 7818, 7832, 7845, + 7858, 7871, 7884, 7897, 7910, 7922, 7934, 7946, 7958, 7970, 7982, 7993, 8004, 8016, + 8027, 8038, 8049, 8060, 8070, 8081, 8091, 8102, 8112, 8122, 8132, 8143, 8152, 8162, + 8172, 8182, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192 +}; + +// sum of log2(i) from table index to inst->anaLen2 in Q5 +// Note that the first table value is invalid, since log2(0) = -infinity +static const int16_t kSumLogIndex[66] = { + 0, 22917, 22917, 22885, 22834, 22770, 22696, 22613, + 22524, 22428, 22326, 22220, 22109, 21994, 21876, 21754, + 21629, 21501, 21370, 21237, 21101, 20963, 20822, 20679, + 20535, 20388, 20239, 20089, 19937, 19783, 19628, 19470, + 19312, 19152, 18991, 18828, 18664, 18498, 18331, 18164, + 17994, 17824, 17653, 17480, 17306, 17132, 16956, 16779, + 16602, 16423, 16243, 16063, 15881, 15699, 15515, 15331, + 15146, 14960, 14774, 14586, 14398, 14209, 14019, 13829, + 13637, 13445 +}; + +// sum of log2(i)^2 from table index to inst->anaLen2 in Q2 +// Note that the first table value is invalid, since log2(0) = -infinity +static const int16_t kSumSquareLogIndex[66] = { + 0, 16959, 16959, 16955, 16945, 16929, 16908, 16881, + 16850, 16814, 16773, 16729, 16681, 16630, 16575, 16517, + 16456, 16392, 16325, 16256, 16184, 16109, 16032, 15952, + 15870, 15786, 15700, 15612, 15521, 15429, 15334, 15238, + 15140, 15040, 14938, 14834, 14729, 14622, 14514, 14404, + 14292, 14179, 14064, 13947, 13830, 13710, 13590, 13468, + 13344, 13220, 13094, 12966, 12837, 12707, 12576, 12444, + 12310, 12175, 12039, 11902, 11763, 11624, 11483, 11341, + 11198, 11054 +}; + +// log2(table index) in Q12 +// Note that the first table value is invalid, since log2(0) = -infinity +static const int16_t kLogIndex[129] = { + 0, 0, 4096, 6492, 8192, 9511, 10588, 11499, + 12288, 12984, 13607, 14170, 14684, 15157, 15595, 16003, + 16384, 16742, 17080, 17400, 17703, 17991, 18266, 18529, + 18780, 19021, 19253, 19476, 19691, 19898, 20099, 20292, + 20480, 20662, 20838, 21010, 21176, 21338, 21496, 21649, + 21799, 21945, 22087, 22226, 22362, 22495, 22625, 22752, + 22876, 22998, 23117, 23234, 23349, 23462, 23572, 23680, + 23787, 23892, 23994, 24095, 24195, 24292, 24388, 24483, + 24576, 24668, 24758, 24847, 24934, 25021, 25106, 25189, + 25272, 25354, 25434, 25513, 25592, 25669, 25745, 25820, + 25895, 25968, 26041, 26112, 26183, 26253, 26322, 26390, + 26458, 26525, 26591, 26656, 26721, 26784, 26848, 26910, + 26972, 27033, 27094, 27154, 27213, 27272, 27330, 27388, + 27445, 27502, 27558, 27613, 27668, 27722, 27776, 27830, + 27883, 27935, 27988, 28039, 28090, 28141, 28191, 28241, + 28291, 28340, 28388, 28437, 28484, 28532, 28579, 28626, + 28672 +}; + +// determinant of estimation matrix in Q0 corresponding to the log2 tables above +// Note that the first table value is invalid, since log2(0) = -infinity +static const int16_t kDeterminantEstMatrix[66] = { + 0, 29814, 25574, 22640, 20351, 18469, 16873, 15491, + 14277, 13199, 12233, 11362, 10571, 9851, 9192, 8587, + 8030, 7515, 7038, 6596, 6186, 5804, 5448, 5115, + 4805, 4514, 4242, 3988, 3749, 3524, 3314, 3116, + 2930, 2755, 2590, 2435, 2289, 2152, 2022, 1900, + 1785, 1677, 1575, 1478, 1388, 1302, 1221, 1145, + 1073, 1005, 942, 881, 825, 771, 721, 674, + 629, 587, 547, 510, 475, 442, 411, 382, + 355, 330 +}; + +// Update the noise estimation information. +static void UpdateNoiseEstimate(NoiseSuppressionFixedC* inst, int offset) { + int32_t tmp32no1 = 0; + int32_t tmp32no2 = 0; + int16_t tmp16 = 0; + const int16_t kExp2Const = 11819; // Q13 + + size_t i = 0; + + tmp16 = WebRtcSpl_MaxValueW16(inst->noiseEstLogQuantile + offset, + inst->magnLen); + // Guarantee a Q-domain as high as possible and still fit in int16 + inst->qNoise = 14 - (int) WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + kExp2Const, tmp16, 21); + for (i = 0; i < inst->magnLen; i++) { + // inst->quantile[i]=exp(inst->lquantile[offset+i]); + // in Q21 + tmp32no2 = kExp2Const * inst->noiseEstLogQuantile[offset + i]; + tmp32no1 = (0x00200000 | (tmp32no2 & 0x001FFFFF)); // 2^21 + frac + tmp16 = (int16_t)(tmp32no2 >> 21); + tmp16 -= 21;// shift 21 to get result in Q0 + tmp16 += (int16_t) inst->qNoise; //shift to get result in Q(qNoise) + if (tmp16 < 0) { + tmp32no1 >>= -tmp16; + } else { + tmp32no1 <<= tmp16; + } + inst->noiseEstQuantile[i] = WebRtcSpl_SatW32ToW16(tmp32no1); + } +} + +// Noise Estimation +static void NoiseEstimationC(NoiseSuppressionFixedC* inst, + uint16_t* magn, + uint32_t* noise, + int16_t* q_noise) { + int16_t lmagn[HALF_ANAL_BLOCKL], counter, countDiv; + int16_t countProd, delta, zeros, frac; + int16_t log2, tabind, logval, tmp16, tmp16no1, tmp16no2; + const int16_t log2_const = 22713; // Q15 + const int16_t width_factor = 21845; + + size_t i, s, offset; + + tabind = inst->stages - inst->normData; + RTC_DCHECK_LT(tabind, 9); + RTC_DCHECK_GT(tabind, -9); + if (tabind < 0) { + logval = -WebRtcNsx_kLogTable[-tabind]; + } else { + logval = WebRtcNsx_kLogTable[tabind]; + } + + // lmagn(i)=log(magn(i))=log(2)*log2(magn(i)) + // magn is in Q(-stages), and the real lmagn values are: + // real_lmagn(i)=log(magn(i)*2^stages)=log(magn(i))+log(2^stages) + // lmagn in Q8 + for (i = 0; i < inst->magnLen; i++) { + if (magn[i]) { + zeros = WebRtcSpl_NormU32((uint32_t)magn[i]); + frac = (int16_t)((((uint32_t)magn[i] << zeros) + & 0x7FFFFFFF) >> 23); + // log2(magn(i)) + RTC_DCHECK_LT(frac, 256); + log2 = (int16_t)(((31 - zeros) << 8) + + WebRtcNsx_kLogTableFrac[frac]); + // log2(magn(i))*log(2) + lmagn[i] = (int16_t)((log2 * log2_const) >> 15); + // + log(2^stages) + lmagn[i] += logval; + } else { + lmagn[i] = logval;//0; + } + } + + // loop over simultaneous estimates + for (s = 0; s < SIMULT; s++) { + offset = s * inst->magnLen; + + // Get counter values from state + counter = inst->noiseEstCounter[s]; + RTC_DCHECK_LT(counter, 201); + countDiv = WebRtcNsx_kCounterDiv[counter]; + countProd = (int16_t)(counter * countDiv); + + // quant_est(...) + for (i = 0; i < inst->magnLen; i++) { + // compute delta + if (inst->noiseEstDensity[offset + i] > 512) { + // Get the value for delta by shifting intead of dividing. + int factor = WebRtcSpl_NormW16(inst->noiseEstDensity[offset + i]); + delta = (int16_t)(FACTOR_Q16 >> (14 - factor)); + } else { + delta = FACTOR_Q7; + if (inst->blockIndex < END_STARTUP_LONG) { + // Smaller step size during startup. This prevents from using + // unrealistic values causing overflow. + delta = FACTOR_Q7_STARTUP; + } + } + + // update log quantile estimate + tmp16 = (int16_t)((delta * countDiv) >> 14); + if (lmagn[i] > inst->noiseEstLogQuantile[offset + i]) { + // +=QUANTILE*delta/(inst->counter[s]+1) QUANTILE=0.25, =1 in Q2 + // CounterDiv=1/(inst->counter[s]+1) in Q15 + tmp16 += 2; + inst->noiseEstLogQuantile[offset + i] += tmp16 / 4; + } else { + tmp16 += 1; + // *(1-QUANTILE), in Q2 QUANTILE=0.25, 1-0.25=0.75=3 in Q2 + // TODO(bjornv): investigate why we need to truncate twice. + tmp16no2 = (int16_t)((tmp16 / 2) * 3 / 2); + inst->noiseEstLogQuantile[offset + i] -= tmp16no2; + if (inst->noiseEstLogQuantile[offset + i] < logval) { + // This is the smallest fixed point representation we can + // have, hence we limit the output. + inst->noiseEstLogQuantile[offset + i] = logval; + } + } + + // update density estimate + if (WEBRTC_SPL_ABS_W16(lmagn[i] - inst->noiseEstLogQuantile[offset + i]) + < WIDTH_Q8) { + tmp16no1 = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + inst->noiseEstDensity[offset + i], countProd, 15); + tmp16no2 = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + width_factor, countDiv, 15); + inst->noiseEstDensity[offset + i] = tmp16no1 + tmp16no2; + } + } // end loop over magnitude spectrum + + if (counter >= END_STARTUP_LONG) { + inst->noiseEstCounter[s] = 0; + if (inst->blockIndex >= END_STARTUP_LONG) { + UpdateNoiseEstimate(inst, offset); + } + } + inst->noiseEstCounter[s]++; + + } // end loop over simultaneous estimates + + // Sequentially update the noise during startup + if (inst->blockIndex < END_STARTUP_LONG) { + UpdateNoiseEstimate(inst, offset); + } + + for (i = 0; i < inst->magnLen; i++) { + noise[i] = (uint32_t)(inst->noiseEstQuantile[i]); // Q(qNoise) + } + (*q_noise) = (int16_t)inst->qNoise; +} + +// Filter the data in the frequency domain, and create spectrum. +static void PrepareSpectrumC(NoiseSuppressionFixedC* inst, int16_t* freq_buf) { + size_t i = 0, j = 0; + + for (i = 0; i < inst->magnLen; i++) { + inst->real[i] = (int16_t)((inst->real[i] * + (int16_t)(inst->noiseSupFilter[i])) >> 14); // Q(normData-stages) + inst->imag[i] = (int16_t)((inst->imag[i] * + (int16_t)(inst->noiseSupFilter[i])) >> 14); // Q(normData-stages) + } + + freq_buf[0] = inst->real[0]; + freq_buf[1] = -inst->imag[0]; + for (i = 1, j = 2; i < inst->anaLen2; i += 1, j += 2) { + freq_buf[j] = inst->real[i]; + freq_buf[j + 1] = -inst->imag[i]; + } + freq_buf[inst->anaLen] = inst->real[inst->anaLen2]; + freq_buf[inst->anaLen + 1] = -inst->imag[inst->anaLen2]; +} + +// Denormalize the real-valued signal |in|, the output from inverse FFT. +static void DenormalizeC(NoiseSuppressionFixedC* inst, + int16_t* in, + int factor) { + size_t i = 0; + int32_t tmp32 = 0; + for (i = 0; i < inst->anaLen; i += 1) { + tmp32 = WEBRTC_SPL_SHIFT_W32((int32_t)in[i], + factor - inst->normData); + inst->real[i] = WebRtcSpl_SatW32ToW16(tmp32); // Q0 + } +} + +// For the noise supression process, synthesis, read out fully processed +// segment, and update synthesis buffer. +static void SynthesisUpdateC(NoiseSuppressionFixedC* inst, + int16_t* out_frame, + int16_t gain_factor) { + size_t i = 0; + int16_t tmp16a = 0; + int16_t tmp16b = 0; + int32_t tmp32 = 0; + + // synthesis + for (i = 0; i < inst->anaLen; i++) { + tmp16a = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + inst->window[i], inst->real[i], 14); // Q0, window in Q14 + tmp32 = WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(tmp16a, gain_factor, 13); // Q0 + // Down shift with rounding + tmp16b = WebRtcSpl_SatW32ToW16(tmp32); // Q0 + inst->synthesisBuffer[i] = WebRtcSpl_AddSatW16(inst->synthesisBuffer[i], + tmp16b); // Q0 + } + + // read out fully processed segment + for (i = 0; i < inst->blockLen10ms; i++) { + out_frame[i] = inst->synthesisBuffer[i]; // Q0 + } + + // update synthesis buffer + memcpy(inst->synthesisBuffer, inst->synthesisBuffer + inst->blockLen10ms, + (inst->anaLen - inst->blockLen10ms) * sizeof(*inst->synthesisBuffer)); + WebRtcSpl_ZerosArrayW16(inst->synthesisBuffer + + inst->anaLen - inst->blockLen10ms, inst->blockLen10ms); +} + +// Update analysis buffer for lower band, and window data before FFT. +static void AnalysisUpdateC(NoiseSuppressionFixedC* inst, + int16_t* out, + int16_t* new_speech) { + size_t i = 0; + + // For lower band update analysis buffer. + memcpy(inst->analysisBuffer, inst->analysisBuffer + inst->blockLen10ms, + (inst->anaLen - inst->blockLen10ms) * sizeof(*inst->analysisBuffer)); + memcpy(inst->analysisBuffer + inst->anaLen - inst->blockLen10ms, new_speech, + inst->blockLen10ms * sizeof(*inst->analysisBuffer)); + + // Window data before FFT. + for (i = 0; i < inst->anaLen; i++) { + out[i] = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + inst->window[i], inst->analysisBuffer[i], 14); // Q0 + } +} + +// Normalize the real-valued signal |in|, the input to forward FFT. +static void NormalizeRealBufferC(NoiseSuppressionFixedC* inst, + const int16_t* in, + int16_t* out) { + size_t i = 0; + RTC_DCHECK_GE(inst->normData, 0); + for (i = 0; i < inst->anaLen; ++i) { + out[i] = in[i] << inst->normData; // Q(normData) + } +} + +// Declare function pointers. +NoiseEstimation WebRtcNsx_NoiseEstimation; +PrepareSpectrum WebRtcNsx_PrepareSpectrum; +SynthesisUpdate WebRtcNsx_SynthesisUpdate; +AnalysisUpdate WebRtcNsx_AnalysisUpdate; +Denormalize WebRtcNsx_Denormalize; +NormalizeRealBuffer WebRtcNsx_NormalizeRealBuffer; + +#if defined(WEBRTC_HAS_NEON) +// Initialize function pointers for ARM Neon platform. +static void WebRtcNsx_InitNeon(void) { + WebRtcNsx_NoiseEstimation = WebRtcNsx_NoiseEstimationNeon; + WebRtcNsx_PrepareSpectrum = WebRtcNsx_PrepareSpectrumNeon; + WebRtcNsx_SynthesisUpdate = WebRtcNsx_SynthesisUpdateNeon; + WebRtcNsx_AnalysisUpdate = WebRtcNsx_AnalysisUpdateNeon; +} +#endif + +#if defined(MIPS32_LE) +// Initialize function pointers for MIPS platform. +static void WebRtcNsx_InitMips(void) { + WebRtcNsx_PrepareSpectrum = WebRtcNsx_PrepareSpectrum_mips; + WebRtcNsx_SynthesisUpdate = WebRtcNsx_SynthesisUpdate_mips; + WebRtcNsx_AnalysisUpdate = WebRtcNsx_AnalysisUpdate_mips; + WebRtcNsx_NormalizeRealBuffer = WebRtcNsx_NormalizeRealBuffer_mips; +#if defined(MIPS_DSP_R1_LE) + WebRtcNsx_Denormalize = WebRtcNsx_Denormalize_mips; +#endif +} +#endif + +void WebRtcNsx_CalcParametricNoiseEstimate(NoiseSuppressionFixedC* inst, + int16_t pink_noise_exp_avg, + int32_t pink_noise_num_avg, + int freq_index, + uint32_t* noise_estimate, + uint32_t* noise_estimate_avg) { + int32_t tmp32no1 = 0; + int32_t tmp32no2 = 0; + + int16_t int_part = 0; + int16_t frac_part = 0; + + // Use pink noise estimate + // noise_estimate = 2^(pinkNoiseNumerator + pinkNoiseExp * log2(j)) + RTC_DCHECK_GE(freq_index, 0); + RTC_DCHECK_LT(freq_index, 129); + tmp32no2 = (pink_noise_exp_avg * kLogIndex[freq_index]) >> 15; // Q11 + tmp32no1 = pink_noise_num_avg - tmp32no2; // Q11 + + // Calculate output: 2^tmp32no1 + // Output in Q(minNorm-stages) + tmp32no1 += (inst->minNorm - inst->stages) << 11; + if (tmp32no1 > 0) { + int_part = (int16_t)(tmp32no1 >> 11); + frac_part = (int16_t)(tmp32no1 & 0x000007ff); // Q11 + // Piecewise linear approximation of 'b' in + // 2^(int_part+frac_part) = 2^int_part * (1 + b) + // 'b' is given in Q11 and below stored in frac_part. + if (frac_part >> 10) { + // Upper fractional part + tmp32no2 = (2048 - frac_part) * 1244; // Q21 + tmp32no2 = 2048 - (tmp32no2 >> 10); + } else { + // Lower fractional part + tmp32no2 = (frac_part * 804) >> 10; + } + // Shift fractional part to Q(minNorm-stages) + tmp32no2 = WEBRTC_SPL_SHIFT_W32(tmp32no2, int_part - 11); + *noise_estimate_avg = (1 << int_part) + (uint32_t)tmp32no2; + // Scale up to initMagnEst, which is not block averaged + *noise_estimate = (*noise_estimate_avg) * (uint32_t)(inst->blockIndex + 1); + } +} + +// Initialize state +int32_t WebRtcNsx_InitCore(NoiseSuppressionFixedC* inst, uint32_t fs) { + int i; + + //check for valid pointer + if (inst == NULL) { + return -1; + } + // + + // Initialization of struct + if (fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000) { + inst->fs = fs; + } else { + return -1; + } + + if (fs == 8000) { + inst->blockLen10ms = 80; + inst->anaLen = 128; + inst->stages = 7; + inst->window = kBlocks80w128x; + inst->thresholdLogLrt = 131072; //default threshold for LRT feature + inst->maxLrt = 0x0040000; + inst->minLrt = 52429; + } else { + inst->blockLen10ms = 160; + inst->anaLen = 256; + inst->stages = 8; + inst->window = kBlocks160w256x; + inst->thresholdLogLrt = 212644; //default threshold for LRT feature + inst->maxLrt = 0x0080000; + inst->minLrt = 104858; + } + inst->anaLen2 = inst->anaLen / 2; + inst->magnLen = inst->anaLen2 + 1; + + if (inst->real_fft != NULL) { + WebRtcSpl_FreeRealFFT(inst->real_fft); + } + inst->real_fft = WebRtcSpl_CreateRealFFT(inst->stages); + if (inst->real_fft == NULL) { + return -1; + } + + WebRtcSpl_ZerosArrayW16(inst->analysisBuffer, ANAL_BLOCKL_MAX); + WebRtcSpl_ZerosArrayW16(inst->synthesisBuffer, ANAL_BLOCKL_MAX); + + // for HB processing + WebRtcSpl_ZerosArrayW16(inst->dataBufHBFX[0], + NUM_HIGH_BANDS_MAX * ANAL_BLOCKL_MAX); + // for quantile noise estimation + WebRtcSpl_ZerosArrayW16(inst->noiseEstQuantile, HALF_ANAL_BLOCKL); + for (i = 0; i < SIMULT * HALF_ANAL_BLOCKL; i++) { + inst->noiseEstLogQuantile[i] = 2048; // Q8 + inst->noiseEstDensity[i] = 153; // Q9 + } + for (i = 0; i < SIMULT; i++) { + inst->noiseEstCounter[i] = (int16_t)(END_STARTUP_LONG * (i + 1)) / SIMULT; + } + + // Initialize suppression filter with ones + WebRtcSpl_MemSetW16((int16_t*)inst->noiseSupFilter, 16384, HALF_ANAL_BLOCKL); + + // Set the aggressiveness: default + inst->aggrMode = 0; + + //initialize variables for new method + inst->priorNonSpeechProb = 8192; // Q14(0.5) prior probability for speech/noise + for (i = 0; i < HALF_ANAL_BLOCKL; i++) { + inst->prevMagnU16[i] = 0; + inst->prevNoiseU32[i] = 0; //previous noise-spectrum + inst->logLrtTimeAvgW32[i] = 0; //smooth LR ratio + inst->avgMagnPause[i] = 0; //conservative noise spectrum estimate + inst->initMagnEst[i] = 0; //initial average magnitude spectrum + } + + //feature quantities + inst->thresholdSpecDiff = 50; //threshold for difference feature: determined on-line + inst->thresholdSpecFlat = 20480; //threshold for flatness: determined on-line + inst->featureLogLrt = inst->thresholdLogLrt; //average LRT factor (= threshold) + inst->featureSpecFlat = inst->thresholdSpecFlat; //spectral flatness (= threshold) + inst->featureSpecDiff = inst->thresholdSpecDiff; //spectral difference (= threshold) + inst->weightLogLrt = 6; //default weighting par for LRT feature + inst->weightSpecFlat = 0; //default weighting par for spectral flatness feature + inst->weightSpecDiff = 0; //default weighting par for spectral difference feature + + inst->curAvgMagnEnergy = 0; //window time-average of input magnitude spectrum + inst->timeAvgMagnEnergy = 0; //normalization for spectral difference + inst->timeAvgMagnEnergyTmp = 0; //normalization for spectral difference + + //histogram quantities: used to estimate/update thresholds for features + WebRtcSpl_ZerosArrayW16(inst->histLrt, HIST_PAR_EST); + WebRtcSpl_ZerosArrayW16(inst->histSpecDiff, HIST_PAR_EST); + WebRtcSpl_ZerosArrayW16(inst->histSpecFlat, HIST_PAR_EST); + + inst->blockIndex = -1; //frame counter + + //inst->modelUpdate = 500; //window for update + inst->modelUpdate = (1 << STAT_UPDATES); //window for update + inst->cntThresUpdate = 0; //counter feature thresholds updates + + inst->sumMagn = 0; + inst->magnEnergy = 0; + inst->prevQMagn = 0; + inst->qNoise = 0; + inst->prevQNoise = 0; + + inst->energyIn = 0; + inst->scaleEnergyIn = 0; + + inst->whiteNoiseLevel = 0; + inst->pinkNoiseNumerator = 0; + inst->pinkNoiseExp = 0; + inst->minNorm = 15; // Start with full scale + inst->zeroInputSignal = 0; + + //default mode + WebRtcNsx_set_policy_core(inst, 0); + +#ifdef NS_FILEDEBUG + inst->infile = fopen("indebug.pcm", "wb"); + inst->outfile = fopen("outdebug.pcm", "wb"); + inst->file1 = fopen("file1.pcm", "wb"); + inst->file2 = fopen("file2.pcm", "wb"); + inst->file3 = fopen("file3.pcm", "wb"); + inst->file4 = fopen("file4.pcm", "wb"); + inst->file5 = fopen("file5.pcm", "wb"); +#endif + + // Initialize function pointers. + WebRtcNsx_NoiseEstimation = NoiseEstimationC; + WebRtcNsx_PrepareSpectrum = PrepareSpectrumC; + WebRtcNsx_SynthesisUpdate = SynthesisUpdateC; + WebRtcNsx_AnalysisUpdate = AnalysisUpdateC; + WebRtcNsx_Denormalize = DenormalizeC; + WebRtcNsx_NormalizeRealBuffer = NormalizeRealBufferC; + +#if defined(WEBRTC_HAS_NEON) + WebRtcNsx_InitNeon(); +#endif + +#if defined(MIPS32_LE) + WebRtcNsx_InitMips(); +#endif + + inst->initFlag = 1; + + return 0; +} + +int WebRtcNsx_set_policy_core(NoiseSuppressionFixedC* inst, int mode) { + // allow for modes:0,1,2,3 + if (mode < 0 || mode > 3) { + return -1; + } + + inst->aggrMode = mode; + if (mode == 0) { + inst->overdrive = 256; // Q8(1.0) + inst->denoiseBound = 8192; // Q14(0.5) + inst->gainMap = 0; // No gain compensation + } else if (mode == 1) { + inst->overdrive = 256; // Q8(1.0) + inst->denoiseBound = 4096; // Q14(0.25) + inst->factor2Table = kFactor2Aggressiveness1; + inst->gainMap = 1; + } else if (mode == 2) { + inst->overdrive = 282; // ~= Q8(1.1) + inst->denoiseBound = 2048; // Q14(0.125) + inst->factor2Table = kFactor2Aggressiveness2; + inst->gainMap = 1; + } else if (mode == 3) { + inst->overdrive = 320; // Q8(1.25) + inst->denoiseBound = 1475; // ~= Q14(0.09) + inst->factor2Table = kFactor2Aggressiveness3; + inst->gainMap = 1; + } + return 0; +} + +// Extract thresholds for feature parameters +// histograms are computed over some window_size (given by window_pars) +// thresholds and weights are extracted every window +// flag 0 means update histogram only, flag 1 means compute the thresholds/weights +// threshold and weights are returned in: inst->priorModelPars +void WebRtcNsx_FeatureParameterExtraction(NoiseSuppressionFixedC* inst, + int flag) { + uint32_t tmpU32; + uint32_t histIndex; + uint32_t posPeak1SpecFlatFX, posPeak2SpecFlatFX; + uint32_t posPeak1SpecDiffFX, posPeak2SpecDiffFX; + + int32_t tmp32; + int32_t fluctLrtFX, thresFluctLrtFX; + int32_t avgHistLrtFX, avgSquareHistLrtFX, avgHistLrtComplFX; + + int16_t j; + int16_t numHistLrt; + + int i; + int useFeatureSpecFlat, useFeatureSpecDiff, featureSum; + int maxPeak1, maxPeak2; + int weightPeak1SpecFlat, weightPeak2SpecFlat; + int weightPeak1SpecDiff, weightPeak2SpecDiff; + + //update histograms + if (!flag) { + // LRT + // Type casting to UWord32 is safe since negative values will not be wrapped to larger + // values than HIST_PAR_EST + histIndex = (uint32_t)(inst->featureLogLrt); + if (histIndex < HIST_PAR_EST) { + inst->histLrt[histIndex]++; + } + // Spectral flatness + // (inst->featureSpecFlat*20)>>10 = (inst->featureSpecFlat*5)>>8 + histIndex = (inst->featureSpecFlat * 5) >> 8; + if (histIndex < HIST_PAR_EST) { + inst->histSpecFlat[histIndex]++; + } + // Spectral difference + histIndex = HIST_PAR_EST; + if (inst->timeAvgMagnEnergy > 0) { + // Guard against division by zero + // If timeAvgMagnEnergy == 0 we have no normalizing statistics and + // therefore can't update the histogram + histIndex = ((inst->featureSpecDiff * 5) >> inst->stages) / + inst->timeAvgMagnEnergy; + } + if (histIndex < HIST_PAR_EST) { + inst->histSpecDiff[histIndex]++; + } + } + + // extract parameters for speech/noise probability + if (flag) { + useFeatureSpecDiff = 1; + //for LRT feature: + // compute the average over inst->featureExtractionParams.rangeAvgHistLrt + avgHistLrtFX = 0; + avgSquareHistLrtFX = 0; + numHistLrt = 0; + for (i = 0; i < BIN_SIZE_LRT; i++) { + j = (2 * i + 1); + tmp32 = inst->histLrt[i] * j; + avgHistLrtFX += tmp32; + numHistLrt += inst->histLrt[i]; + avgSquareHistLrtFX += tmp32 * j; + } + avgHistLrtComplFX = avgHistLrtFX; + for (; i < HIST_PAR_EST; i++) { + j = (2 * i + 1); + tmp32 = inst->histLrt[i] * j; + avgHistLrtComplFX += tmp32; + avgSquareHistLrtFX += tmp32 * j; + } + fluctLrtFX = avgSquareHistLrtFX * numHistLrt - + avgHistLrtFX * avgHistLrtComplFX; + thresFluctLrtFX = THRES_FLUCT_LRT * numHistLrt; + // get threshold for LRT feature: + tmpU32 = (FACTOR_1_LRT_DIFF * (uint32_t)avgHistLrtFX); + if ((fluctLrtFX < thresFluctLrtFX) || (numHistLrt == 0) || + (tmpU32 > (uint32_t)(100 * numHistLrt))) { + //very low fluctuation, so likely noise + inst->thresholdLogLrt = inst->maxLrt; + } else { + tmp32 = (int32_t)((tmpU32 << (9 + inst->stages)) / numHistLrt / + 25); + // check if value is within min/max range + inst->thresholdLogLrt = WEBRTC_SPL_SAT(inst->maxLrt, + tmp32, + inst->minLrt); + } + if (fluctLrtFX < thresFluctLrtFX) { + // Do not use difference feature if fluctuation of LRT feature is very low: + // most likely just noise state + useFeatureSpecDiff = 0; + } + + // for spectral flatness and spectral difference: compute the main peaks of histogram + maxPeak1 = 0; + maxPeak2 = 0; + posPeak1SpecFlatFX = 0; + posPeak2SpecFlatFX = 0; + weightPeak1SpecFlat = 0; + weightPeak2SpecFlat = 0; + + // peaks for flatness + for (i = 0; i < HIST_PAR_EST; i++) { + if (inst->histSpecFlat[i] > maxPeak1) { + // Found new "first" peak + maxPeak2 = maxPeak1; + weightPeak2SpecFlat = weightPeak1SpecFlat; + posPeak2SpecFlatFX = posPeak1SpecFlatFX; + + maxPeak1 = inst->histSpecFlat[i]; + weightPeak1SpecFlat = inst->histSpecFlat[i]; + posPeak1SpecFlatFX = (uint32_t)(2 * i + 1); + } else if (inst->histSpecFlat[i] > maxPeak2) { + // Found new "second" peak + maxPeak2 = inst->histSpecFlat[i]; + weightPeak2SpecFlat = inst->histSpecFlat[i]; + posPeak2SpecFlatFX = (uint32_t)(2 * i + 1); + } + } + + // for spectral flatness feature + useFeatureSpecFlat = 1; + // merge the two peaks if they are close + if ((posPeak1SpecFlatFX - posPeak2SpecFlatFX < LIM_PEAK_SPACE_FLAT_DIFF) + && (weightPeak2SpecFlat * LIM_PEAK_WEIGHT_FLAT_DIFF > weightPeak1SpecFlat)) { + weightPeak1SpecFlat += weightPeak2SpecFlat; + posPeak1SpecFlatFX = (posPeak1SpecFlatFX + posPeak2SpecFlatFX) >> 1; + } + //reject if weight of peaks is not large enough, or peak value too small + if (weightPeak1SpecFlat < THRES_WEIGHT_FLAT_DIFF || posPeak1SpecFlatFX + < THRES_PEAK_FLAT) { + useFeatureSpecFlat = 0; + } else { // if selected, get the threshold + // compute the threshold and check if value is within min/max range + inst->thresholdSpecFlat = WEBRTC_SPL_SAT(MAX_FLAT_Q10, FACTOR_2_FLAT_Q10 + * posPeak1SpecFlatFX, MIN_FLAT_Q10); //Q10 + } + // done with flatness feature + + if (useFeatureSpecDiff) { + //compute two peaks for spectral difference + maxPeak1 = 0; + maxPeak2 = 0; + posPeak1SpecDiffFX = 0; + posPeak2SpecDiffFX = 0; + weightPeak1SpecDiff = 0; + weightPeak2SpecDiff = 0; + // peaks for spectral difference + for (i = 0; i < HIST_PAR_EST; i++) { + if (inst->histSpecDiff[i] > maxPeak1) { + // Found new "first" peak + maxPeak2 = maxPeak1; + weightPeak2SpecDiff = weightPeak1SpecDiff; + posPeak2SpecDiffFX = posPeak1SpecDiffFX; + + maxPeak1 = inst->histSpecDiff[i]; + weightPeak1SpecDiff = inst->histSpecDiff[i]; + posPeak1SpecDiffFX = (uint32_t)(2 * i + 1); + } else if (inst->histSpecDiff[i] > maxPeak2) { + // Found new "second" peak + maxPeak2 = inst->histSpecDiff[i]; + weightPeak2SpecDiff = inst->histSpecDiff[i]; + posPeak2SpecDiffFX = (uint32_t)(2 * i + 1); + } + } + + // merge the two peaks if they are close + if ((posPeak1SpecDiffFX - posPeak2SpecDiffFX < LIM_PEAK_SPACE_FLAT_DIFF) + && (weightPeak2SpecDiff * LIM_PEAK_WEIGHT_FLAT_DIFF > weightPeak1SpecDiff)) { + weightPeak1SpecDiff += weightPeak2SpecDiff; + posPeak1SpecDiffFX = (posPeak1SpecDiffFX + posPeak2SpecDiffFX) >> 1; + } + // get the threshold value and check if value is within min/max range + inst->thresholdSpecDiff = WEBRTC_SPL_SAT(MAX_DIFF, FACTOR_1_LRT_DIFF + * posPeak1SpecDiffFX, MIN_DIFF); //5x bigger + //reject if weight of peaks is not large enough + if (weightPeak1SpecDiff < THRES_WEIGHT_FLAT_DIFF) { + useFeatureSpecDiff = 0; + } + // done with spectral difference feature + } + + // select the weights between the features + // inst->priorModelPars[4] is weight for LRT: always selected + featureSum = 6 / (1 + useFeatureSpecFlat + useFeatureSpecDiff); + inst->weightLogLrt = featureSum; + inst->weightSpecFlat = useFeatureSpecFlat * featureSum; + inst->weightSpecDiff = useFeatureSpecDiff * featureSum; + + // set histograms to zero for next update + WebRtcSpl_ZerosArrayW16(inst->histLrt, HIST_PAR_EST); + WebRtcSpl_ZerosArrayW16(inst->histSpecDiff, HIST_PAR_EST); + WebRtcSpl_ZerosArrayW16(inst->histSpecFlat, HIST_PAR_EST); + } // end of flag == 1 +} + + +// Compute spectral flatness on input spectrum +// magn is the magnitude spectrum +// spectral flatness is returned in inst->featureSpecFlat +void WebRtcNsx_ComputeSpectralFlatness(NoiseSuppressionFixedC* inst, + uint16_t* magn) { + uint32_t tmpU32; + uint32_t avgSpectralFlatnessNum, avgSpectralFlatnessDen; + + int32_t tmp32; + int32_t currentSpectralFlatness, logCurSpectralFlatness; + + int16_t zeros, frac, intPart; + + size_t i; + + // for flatness + avgSpectralFlatnessNum = 0; + avgSpectralFlatnessDen = inst->sumMagn - (uint32_t)magn[0]; // Q(normData-stages) + + // compute log of ratio of the geometric to arithmetic mean: check for log(0) case + // flatness = exp( sum(log(magn[i]))/N - log(sum(magn[i])/N) ) + // = exp( sum(log(magn[i]))/N ) * N / sum(magn[i]) + // = 2^( sum(log2(magn[i]))/N - (log2(sum(magn[i])) - log2(N)) ) [This is used] + for (i = 1; i < inst->magnLen; i++) { + // First bin is excluded from spectrum measures. Number of bins is now a power of 2 + if (magn[i]) { + zeros = WebRtcSpl_NormU32((uint32_t)magn[i]); + frac = (int16_t)(((uint32_t)((uint32_t)(magn[i]) << zeros) + & 0x7FFFFFFF) >> 23); + // log2(magn(i)) + RTC_DCHECK_LT(frac, 256); + tmpU32 = (uint32_t)(((31 - zeros) << 8) + + WebRtcNsx_kLogTableFrac[frac]); // Q8 + avgSpectralFlatnessNum += tmpU32; // Q8 + } else { + //if at least one frequency component is zero, treat separately + tmpU32 = WEBRTC_SPL_UMUL_32_16(inst->featureSpecFlat, SPECT_FLAT_TAVG_Q14); // Q24 + inst->featureSpecFlat -= tmpU32 >> 14; // Q10 + return; + } + } + //ratio and inverse log: check for case of log(0) + zeros = WebRtcSpl_NormU32(avgSpectralFlatnessDen); + frac = (int16_t)(((avgSpectralFlatnessDen << zeros) & 0x7FFFFFFF) >> 23); + // log2(avgSpectralFlatnessDen) + RTC_DCHECK_LT(frac, 256); + tmp32 = (int32_t)(((31 - zeros) << 8) + WebRtcNsx_kLogTableFrac[frac]); // Q8 + logCurSpectralFlatness = (int32_t)avgSpectralFlatnessNum; + logCurSpectralFlatness += ((int32_t)(inst->stages - 1) << (inst->stages + 7)); // Q(8+stages-1) + logCurSpectralFlatness -= (tmp32 << (inst->stages - 1)); + logCurSpectralFlatness <<= (10 - inst->stages); // Q17 + tmp32 = (int32_t)(0x00020000 | (WEBRTC_SPL_ABS_W32(logCurSpectralFlatness) + & 0x0001FFFF)); //Q17 + intPart = 7 - (logCurSpectralFlatness >> 17); // Add 7 for output in Q10. + if (intPart > 0) { + currentSpectralFlatness = tmp32 >> intPart; + } else { + currentSpectralFlatness = tmp32 << -intPart; + } + + //time average update of spectral flatness feature + tmp32 = currentSpectralFlatness - (int32_t)inst->featureSpecFlat; // Q10 + tmp32 *= SPECT_FLAT_TAVG_Q14; // Q24 + inst->featureSpecFlat += tmp32 >> 14; // Q10 + // done with flatness feature +} + + +// Compute the difference measure between input spectrum and a template/learned noise spectrum +// magn_tmp is the input spectrum +// the reference/template spectrum is inst->magn_avg_pause[i] +// returns (normalized) spectral difference in inst->featureSpecDiff +void WebRtcNsx_ComputeSpectralDifference(NoiseSuppressionFixedC* inst, + uint16_t* magnIn) { + // This is to be calculated: + // avgDiffNormMagn = var(magnIn) - cov(magnIn, magnAvgPause)^2 / var(magnAvgPause) + + uint32_t tmpU32no1, tmpU32no2; + uint32_t varMagnUFX, varPauseUFX, avgDiffNormMagnUFX; + + int32_t tmp32no1, tmp32no2; + int32_t avgPauseFX, avgMagnFX, covMagnPauseFX; + int32_t maxPause, minPause; + + int16_t tmp16no1; + + size_t i; + int norm32, nShifts; + + avgPauseFX = 0; + maxPause = 0; + minPause = inst->avgMagnPause[0]; // Q(prevQMagn) + // compute average quantities + for (i = 0; i < inst->magnLen; i++) { + // Compute mean of magn_pause + avgPauseFX += inst->avgMagnPause[i]; // in Q(prevQMagn) + maxPause = WEBRTC_SPL_MAX(maxPause, inst->avgMagnPause[i]); + minPause = WEBRTC_SPL_MIN(minPause, inst->avgMagnPause[i]); + } + // normalize by replacing div of "inst->magnLen" with "inst->stages-1" shifts + avgPauseFX >>= inst->stages - 1; + avgMagnFX = inst->sumMagn >> (inst->stages - 1); + // Largest possible deviation in magnPause for (co)var calculations + tmp32no1 = WEBRTC_SPL_MAX(maxPause - avgPauseFX, avgPauseFX - minPause); + // Get number of shifts to make sure we don't get wrap around in varPause + nShifts = WEBRTC_SPL_MAX(0, 10 + inst->stages - WebRtcSpl_NormW32(tmp32no1)); + + varMagnUFX = 0; + varPauseUFX = 0; + covMagnPauseFX = 0; + for (i = 0; i < inst->magnLen; i++) { + // Compute var and cov of magn and magn_pause + tmp16no1 = (int16_t)((int32_t)magnIn[i] - avgMagnFX); + tmp32no2 = inst->avgMagnPause[i] - avgPauseFX; + varMagnUFX += (uint32_t)(tmp16no1 * tmp16no1); // Q(2*qMagn) + tmp32no1 = tmp32no2 * tmp16no1; // Q(prevQMagn+qMagn) + covMagnPauseFX += tmp32no1; // Q(prevQMagn+qMagn) + tmp32no1 = tmp32no2 >> nShifts; // Q(prevQMagn-minPause). + varPauseUFX += tmp32no1 * tmp32no1; // Q(2*(prevQMagn-minPause)) + } + //update of average magnitude spectrum: Q(-2*stages) and averaging replaced by shifts + inst->curAvgMagnEnergy += + inst->magnEnergy >> (2 * inst->normData + inst->stages - 1); + + avgDiffNormMagnUFX = varMagnUFX; // Q(2*qMagn) + if ((varPauseUFX) && (covMagnPauseFX)) { + tmpU32no1 = (uint32_t)WEBRTC_SPL_ABS_W32(covMagnPauseFX); // Q(prevQMagn+qMagn) + norm32 = WebRtcSpl_NormU32(tmpU32no1) - 16; + if (norm32 > 0) { + tmpU32no1 <<= norm32; // Q(prevQMagn+qMagn+norm32) + } else { + tmpU32no1 >>= -norm32; // Q(prevQMagn+qMagn+norm32) + } + tmpU32no2 = WEBRTC_SPL_UMUL(tmpU32no1, tmpU32no1); // Q(2*(prevQMagn+qMagn-norm32)) + + nShifts += norm32; + nShifts <<= 1; + if (nShifts < 0) { + varPauseUFX >>= (-nShifts); // Q(2*(qMagn+norm32+minPause)) + nShifts = 0; + } + if (varPauseUFX > 0) { + // Q(2*(qMagn+norm32-16+minPause)) + tmpU32no1 = tmpU32no2 / varPauseUFX; + tmpU32no1 >>= nShifts; + + // Q(2*qMagn) + avgDiffNormMagnUFX -= WEBRTC_SPL_MIN(avgDiffNormMagnUFX, tmpU32no1); + } else { + avgDiffNormMagnUFX = 0; + } + } + //normalize and compute time average update of difference feature + tmpU32no1 = avgDiffNormMagnUFX >> (2 * inst->normData); + if (inst->featureSpecDiff > tmpU32no1) { + tmpU32no2 = WEBRTC_SPL_UMUL_32_16(inst->featureSpecDiff - tmpU32no1, + SPECT_DIFF_TAVG_Q8); // Q(8-2*stages) + inst->featureSpecDiff -= tmpU32no2 >> 8; // Q(-2*stages) + } else { + tmpU32no2 = WEBRTC_SPL_UMUL_32_16(tmpU32no1 - inst->featureSpecDiff, + SPECT_DIFF_TAVG_Q8); // Q(8-2*stages) + inst->featureSpecDiff += tmpU32no2 >> 8; // Q(-2*stages) + } +} + +// Transform input (speechFrame) to frequency domain magnitude (magnU16) +void WebRtcNsx_DataAnalysis(NoiseSuppressionFixedC* inst, + short* speechFrame, + uint16_t* magnU16) { + uint32_t tmpU32no1; + + int32_t tmp_1_w32 = 0; + int32_t tmp_2_w32 = 0; + int32_t sum_log_magn = 0; + int32_t sum_log_i_log_magn = 0; + + uint16_t sum_log_magn_u16 = 0; + uint16_t tmp_u16 = 0; + + int16_t sum_log_i = 0; + int16_t sum_log_i_square = 0; + int16_t frac = 0; + int16_t log2 = 0; + int16_t matrix_determinant = 0; + int16_t maxWinData; + + size_t i, j; + int zeros; + int net_norm = 0; + int right_shifts_in_magnU16 = 0; + int right_shifts_in_initMagnEst = 0; + + int16_t winData_buff[ANAL_BLOCKL_MAX * 2 + 16]; + int16_t realImag_buff[ANAL_BLOCKL_MAX * 2 + 16]; + + // Align the structures to 32-byte boundary for the FFT function. + int16_t* winData = (int16_t*) (((uintptr_t)winData_buff + 31) & ~31); + int16_t* realImag = (int16_t*) (((uintptr_t) realImag_buff + 31) & ~31); + + // Update analysis buffer for lower band, and window data before FFT. + WebRtcNsx_AnalysisUpdate(inst, winData, speechFrame); + + // Get input energy + inst->energyIn = + WebRtcSpl_Energy(winData, inst->anaLen, &inst->scaleEnergyIn); + + // Reset zero input flag + inst->zeroInputSignal = 0; + // Acquire norm for winData + maxWinData = WebRtcSpl_MaxAbsValueW16(winData, inst->anaLen); + inst->normData = WebRtcSpl_NormW16(maxWinData); + if (maxWinData == 0) { + // Treat zero input separately. + inst->zeroInputSignal = 1; + return; + } + + // Determine the net normalization in the frequency domain + net_norm = inst->stages - inst->normData; + // Track lowest normalization factor and use it to prevent wrap around in shifting + right_shifts_in_magnU16 = inst->normData - inst->minNorm; + right_shifts_in_initMagnEst = WEBRTC_SPL_MAX(-right_shifts_in_magnU16, 0); + inst->minNorm -= right_shifts_in_initMagnEst; + right_shifts_in_magnU16 = WEBRTC_SPL_MAX(right_shifts_in_magnU16, 0); + + // create realImag as winData interleaved with zeros (= imag. part), normalize it + WebRtcNsx_NormalizeRealBuffer(inst, winData, realImag); + + // FFT output will be in winData[]. + WebRtcSpl_RealForwardFFT(inst->real_fft, realImag, winData); + + inst->imag[0] = 0; // Q(normData-stages) + inst->imag[inst->anaLen2] = 0; + inst->real[0] = winData[0]; // Q(normData-stages) + inst->real[inst->anaLen2] = winData[inst->anaLen]; + // Q(2*(normData-stages)) + inst->magnEnergy = (uint32_t)(inst->real[0] * inst->real[0]); + inst->magnEnergy += (uint32_t)(inst->real[inst->anaLen2] * + inst->real[inst->anaLen2]); + magnU16[0] = (uint16_t)WEBRTC_SPL_ABS_W16(inst->real[0]); // Q(normData-stages) + magnU16[inst->anaLen2] = (uint16_t)WEBRTC_SPL_ABS_W16(inst->real[inst->anaLen2]); + inst->sumMagn = (uint32_t)magnU16[0]; // Q(normData-stages) + inst->sumMagn += (uint32_t)magnU16[inst->anaLen2]; + + if (inst->blockIndex >= END_STARTUP_SHORT) { + for (i = 1, j = 2; i < inst->anaLen2; i += 1, j += 2) { + inst->real[i] = winData[j]; + inst->imag[i] = -winData[j + 1]; + // magnitude spectrum + // energy in Q(2*(normData-stages)) + tmpU32no1 = (uint32_t)(winData[j] * winData[j]); + tmpU32no1 += (uint32_t)(winData[j + 1] * winData[j + 1]); + inst->magnEnergy += tmpU32no1; // Q(2*(normData-stages)) + + magnU16[i] = (uint16_t)WebRtcSpl_SqrtFloor(tmpU32no1); // Q(normData-stages) + inst->sumMagn += (uint32_t)magnU16[i]; // Q(normData-stages) + } + } else { + // + // Gather information during startup for noise parameter estimation + // + + // Switch initMagnEst to Q(minNorm-stages) + inst->initMagnEst[0] >>= right_shifts_in_initMagnEst; + inst->initMagnEst[inst->anaLen2] >>= right_shifts_in_initMagnEst; + + // Update initMagnEst with magnU16 in Q(minNorm-stages). + inst->initMagnEst[0] += magnU16[0] >> right_shifts_in_magnU16; + inst->initMagnEst[inst->anaLen2] += + magnU16[inst->anaLen2] >> right_shifts_in_magnU16; + + log2 = 0; + if (magnU16[inst->anaLen2]) { + // Calculate log2(magnU16[inst->anaLen2]) + zeros = WebRtcSpl_NormU32((uint32_t)magnU16[inst->anaLen2]); + frac = (int16_t)((((uint32_t)magnU16[inst->anaLen2] << zeros) & + 0x7FFFFFFF) >> 23); // Q8 + // log2(magnU16(i)) in Q8 + RTC_DCHECK_LT(frac, 256); + log2 = (int16_t)(((31 - zeros) << 8) + WebRtcNsx_kLogTableFrac[frac]); + } + + sum_log_magn = (int32_t)log2; // Q8 + // sum_log_i_log_magn in Q17 + sum_log_i_log_magn = (kLogIndex[inst->anaLen2] * log2) >> 3; + + for (i = 1, j = 2; i < inst->anaLen2; i += 1, j += 2) { + inst->real[i] = winData[j]; + inst->imag[i] = -winData[j + 1]; + // magnitude spectrum + // energy in Q(2*(normData-stages)) + tmpU32no1 = (uint32_t)(winData[j] * winData[j]); + tmpU32no1 += (uint32_t)(winData[j + 1] * winData[j + 1]); + inst->magnEnergy += tmpU32no1; // Q(2*(normData-stages)) + + magnU16[i] = (uint16_t)WebRtcSpl_SqrtFloor(tmpU32no1); // Q(normData-stages) + inst->sumMagn += (uint32_t)magnU16[i]; // Q(normData-stages) + + // Switch initMagnEst to Q(minNorm-stages) + inst->initMagnEst[i] >>= right_shifts_in_initMagnEst; + + // Update initMagnEst with magnU16 in Q(minNorm-stages). + inst->initMagnEst[i] += magnU16[i] >> right_shifts_in_magnU16; + + if (i >= kStartBand) { + // For pink noise estimation. Collect data neglecting lower frequency band + log2 = 0; + if (magnU16[i]) { + zeros = WebRtcSpl_NormU32((uint32_t)magnU16[i]); + frac = (int16_t)((((uint32_t)magnU16[i] << zeros) & + 0x7FFFFFFF) >> 23); + // log2(magnU16(i)) in Q8 + RTC_DCHECK_LT(frac, 256); + log2 = (int16_t)(((31 - zeros) << 8) + + WebRtcNsx_kLogTableFrac[frac]); + } + sum_log_magn += (int32_t)log2; // Q8 + // sum_log_i_log_magn in Q17 + sum_log_i_log_magn += (kLogIndex[i] * log2) >> 3; + } + } + + // + //compute simplified noise model during startup + // + + // Estimate White noise + + // Switch whiteNoiseLevel to Q(minNorm-stages) + inst->whiteNoiseLevel >>= right_shifts_in_initMagnEst; + + // Update the average magnitude spectrum, used as noise estimate. + tmpU32no1 = WEBRTC_SPL_UMUL_32_16(inst->sumMagn, inst->overdrive); + tmpU32no1 >>= inst->stages + 8; + + // Replacing division above with 'stages' shifts + // Shift to same Q-domain as whiteNoiseLevel + tmpU32no1 >>= right_shifts_in_magnU16; + // This operation is safe from wrap around as long as END_STARTUP_SHORT < 128 + RTC_DCHECK_LT(END_STARTUP_SHORT, 128); + inst->whiteNoiseLevel += tmpU32no1; // Q(minNorm-stages) + + // Estimate Pink noise parameters + // Denominator used in both parameter estimates. + // The value is only dependent on the size of the frequency band (kStartBand) + // and to reduce computational complexity stored in a table (kDeterminantEstMatrix[]) + RTC_DCHECK_LT(kStartBand, 66); + matrix_determinant = kDeterminantEstMatrix[kStartBand]; // Q0 + sum_log_i = kSumLogIndex[kStartBand]; // Q5 + sum_log_i_square = kSumSquareLogIndex[kStartBand]; // Q2 + if (inst->fs == 8000) { + // Adjust values to shorter blocks in narrow band. + tmp_1_w32 = (int32_t)matrix_determinant; + tmp_1_w32 += (kSumLogIndex[65] * sum_log_i) >> 9; + tmp_1_w32 -= (kSumLogIndex[65] * kSumLogIndex[65]) >> 10; + tmp_1_w32 -= (int32_t)sum_log_i_square << 4; + tmp_1_w32 -= ((inst->magnLen - kStartBand) * kSumSquareLogIndex[65]) >> 2; + matrix_determinant = (int16_t)tmp_1_w32; + sum_log_i -= kSumLogIndex[65]; // Q5 + sum_log_i_square -= kSumSquareLogIndex[65]; // Q2 + } + + // Necessary number of shifts to fit sum_log_magn in a word16 + zeros = 16 - WebRtcSpl_NormW32(sum_log_magn); + if (zeros < 0) { + zeros = 0; + } + tmp_1_w32 = sum_log_magn << 1; // Q9 + sum_log_magn_u16 = (uint16_t)(tmp_1_w32 >> zeros); // Q(9-zeros). + + // Calculate and update pinkNoiseNumerator. Result in Q11. + tmp_2_w32 = WEBRTC_SPL_MUL_16_U16(sum_log_i_square, sum_log_magn_u16); // Q(11-zeros) + tmpU32no1 = sum_log_i_log_magn >> 12; // Q5 + + // Shift the largest value of sum_log_i and tmp32no3 before multiplication + tmp_u16 = ((uint16_t)sum_log_i << 1); // Q6 + if ((uint32_t)sum_log_i > tmpU32no1) { + tmp_u16 >>= zeros; + } else { + tmpU32no1 >>= zeros; + } + tmp_2_w32 -= (int32_t)WEBRTC_SPL_UMUL_32_16(tmpU32no1, tmp_u16); // Q(11-zeros) + matrix_determinant >>= zeros; // Q(-zeros) + tmp_2_w32 = WebRtcSpl_DivW32W16(tmp_2_w32, matrix_determinant); // Q11 + tmp_2_w32 += (int32_t)net_norm << 11; // Q11 + if (tmp_2_w32 < 0) { + tmp_2_w32 = 0; + } + inst->pinkNoiseNumerator += tmp_2_w32; // Q11 + + // Calculate and update pinkNoiseExp. Result in Q14. + tmp_2_w32 = WEBRTC_SPL_MUL_16_U16(sum_log_i, sum_log_magn_u16); // Q(14-zeros) + tmp_1_w32 = sum_log_i_log_magn >> (3 + zeros); + tmp_1_w32 *= inst->magnLen - kStartBand; + tmp_2_w32 -= tmp_1_w32; // Q(14-zeros) + if (tmp_2_w32 > 0) { + // If the exponential parameter is negative force it to zero, which means a + // flat spectrum. + tmp_1_w32 = WebRtcSpl_DivW32W16(tmp_2_w32, matrix_determinant); // Q14 + inst->pinkNoiseExp += WEBRTC_SPL_SAT(16384, tmp_1_w32, 0); // Q14 + } + } +} + +void WebRtcNsx_DataSynthesis(NoiseSuppressionFixedC* inst, short* outFrame) { + int32_t energyOut; + + int16_t realImag_buff[ANAL_BLOCKL_MAX * 2 + 16]; + int16_t rfft_out_buff[ANAL_BLOCKL_MAX * 2 + 16]; + + // Align the structures to 32-byte boundary for the FFT function. + int16_t* realImag = (int16_t*) (((uintptr_t)realImag_buff + 31) & ~31); + int16_t* rfft_out = (int16_t*) (((uintptr_t) rfft_out_buff + 31) & ~31); + + int16_t tmp16no1, tmp16no2; + int16_t energyRatio; + int16_t gainFactor, gainFactor1, gainFactor2; + + size_t i; + int outCIFFT; + int scaleEnergyOut = 0; + + if (inst->zeroInputSignal) { + // synthesize the special case of zero input + // read out fully processed segment + for (i = 0; i < inst->blockLen10ms; i++) { + outFrame[i] = inst->synthesisBuffer[i]; // Q0 + } + // update synthesis buffer + memcpy(inst->synthesisBuffer, inst->synthesisBuffer + inst->blockLen10ms, + (inst->anaLen - inst->blockLen10ms) * sizeof(*inst->synthesisBuffer)); + WebRtcSpl_ZerosArrayW16(inst->synthesisBuffer + inst->anaLen - inst->blockLen10ms, + inst->blockLen10ms); + return; + } + + // Filter the data in the frequency domain, and create spectrum. + WebRtcNsx_PrepareSpectrum(inst, realImag); + + // Inverse FFT output will be in rfft_out[]. + outCIFFT = WebRtcSpl_RealInverseFFT(inst->real_fft, realImag, rfft_out); + + WebRtcNsx_Denormalize(inst, rfft_out, outCIFFT); + + //scale factor: only do it after END_STARTUP_LONG time + gainFactor = 8192; // 8192 = Q13(1.0) + if (inst->gainMap == 1 && + inst->blockIndex > END_STARTUP_LONG && + inst->energyIn > 0) { + // Q(-scaleEnergyOut) + energyOut = WebRtcSpl_Energy(inst->real, inst->anaLen, &scaleEnergyOut); + if (scaleEnergyOut == 0 && !(energyOut & 0x7f800000)) { + energyOut = WEBRTC_SPL_SHIFT_W32(energyOut, 8 + scaleEnergyOut + - inst->scaleEnergyIn); + } else { + // |energyIn| is currently in Q(|scaleEnergyIn|), but to later on end up + // with an |energyRatio| in Q8 we need to change the Q-domain to + // Q(-8-scaleEnergyOut). + inst->energyIn >>= 8 + scaleEnergyOut - inst->scaleEnergyIn; + } + + RTC_DCHECK_GT(inst->energyIn, 0); + energyRatio = (energyOut + inst->energyIn / 2) / inst->energyIn; // Q8 + // Limit the ratio to [0, 1] in Q8, i.e., [0, 256] + energyRatio = WEBRTC_SPL_SAT(256, energyRatio, 0); + + // all done in lookup tables now + RTC_DCHECK_LT(energyRatio, 257); + gainFactor1 = kFactor1Table[energyRatio]; // Q8 + gainFactor2 = inst->factor2Table[energyRatio]; // Q8 + + //combine both scales with speech/noise prob: note prior (priorSpeechProb) is not frequency dependent + + // factor = inst->priorSpeechProb*factor1 + (1.0-inst->priorSpeechProb)*factor2; // original code + tmp16no1 = (int16_t)(((16384 - inst->priorNonSpeechProb) * gainFactor1) >> + 14); // in Q13, where 16384 = Q14(1.0) + tmp16no2 = (int16_t)((inst->priorNonSpeechProb * gainFactor2) >> 14); + gainFactor = tmp16no1 + tmp16no2; // Q13 + } // out of flag_gain_map==1 + + // Synthesis, read out fully processed segment, and update synthesis buffer. + WebRtcNsx_SynthesisUpdate(inst, outFrame, gainFactor); +} + +void WebRtcNsx_ProcessCore(NoiseSuppressionFixedC* inst, + const short* const* speechFrame, + int num_bands, + short* const* outFrame) { + // main routine for noise suppression + + uint32_t tmpU32no1, tmpU32no2, tmpU32no3; + uint32_t satMax, maxNoiseU32; + uint32_t tmpMagnU32, tmpNoiseU32; + uint32_t nearMagnEst; + uint32_t noiseUpdateU32; + uint32_t noiseU32[HALF_ANAL_BLOCKL]; + uint32_t postLocSnr[HALF_ANAL_BLOCKL]; + uint32_t priorLocSnr[HALF_ANAL_BLOCKL]; + uint32_t prevNearSnr[HALF_ANAL_BLOCKL]; + uint32_t curNearSnr; + uint32_t priorSnr; + uint32_t noise_estimate = 0; + uint32_t noise_estimate_avg = 0; + uint32_t numerator = 0; + + int32_t tmp32no1, tmp32no2; + int32_t pink_noise_num_avg = 0; + + uint16_t tmpU16no1; + uint16_t magnU16[HALF_ANAL_BLOCKL]; + uint16_t prevNoiseU16[HALF_ANAL_BLOCKL]; + uint16_t nonSpeechProbFinal[HALF_ANAL_BLOCKL]; + uint16_t gammaNoise, prevGammaNoise; + uint16_t noiseSupFilterTmp[HALF_ANAL_BLOCKL]; + + int16_t qMagn, qNoise; + int16_t avgProbSpeechHB, gainModHB, avgFilterGainHB, gainTimeDomainHB; + int16_t pink_noise_exp_avg = 0; + + size_t i, j; + int nShifts, postShifts; + int norm32no1, norm32no2; + int flag, sign; + int q_domain_to_use = 0; + + // Code for ARMv7-Neon platform assumes the following: + RTC_DCHECK_GT(inst->anaLen, 0); + RTC_DCHECK_GT(inst->anaLen2, 0); + RTC_DCHECK_EQ(0, inst->anaLen % 16); + RTC_DCHECK_EQ(0, inst->anaLen2 % 8); + RTC_DCHECK_GT(inst->blockLen10ms, 0); + RTC_DCHECK_EQ(0, inst->blockLen10ms % 16); + RTC_DCHECK_EQ(inst->magnLen, inst->anaLen2 + 1); + +#ifdef NS_FILEDEBUG + if (fwrite(spframe, sizeof(short), + inst->blockLen10ms, inst->infile) != inst->blockLen10ms) { + RTC_NOTREACHED(); + } +#endif + + // Check that initialization has been done + RTC_DCHECK_EQ(1, inst->initFlag); + RTC_DCHECK_LE(num_bands - 1, NUM_HIGH_BANDS_MAX); + + const short* const* speechFrameHB = NULL; + short* const* outFrameHB = NULL; + size_t num_high_bands = 0; + if (num_bands > 1) { + speechFrameHB = &speechFrame[1]; + outFrameHB = &outFrame[1]; + num_high_bands = (size_t)(num_bands - 1); + } + + // Store speechFrame and transform to frequency domain + WebRtcNsx_DataAnalysis(inst, (short*)speechFrame[0], magnU16); + + if (inst->zeroInputSignal) { + WebRtcNsx_DataSynthesis(inst, outFrame[0]); + + if (num_bands > 1) { + // update analysis buffer for H band + // append new data to buffer FX + for (i = 0; i < num_high_bands; ++i) { + int block_shift = inst->anaLen - inst->blockLen10ms; + memcpy(inst->dataBufHBFX[i], inst->dataBufHBFX[i] + inst->blockLen10ms, + block_shift * sizeof(*inst->dataBufHBFX[i])); + memcpy(inst->dataBufHBFX[i] + block_shift, speechFrameHB[i], + inst->blockLen10ms * sizeof(*inst->dataBufHBFX[i])); + for (j = 0; j < inst->blockLen10ms; j++) { + outFrameHB[i][j] = inst->dataBufHBFX[i][j]; // Q0 + } + } + } // end of H band gain computation + return; + } + + // Update block index when we have something to process + inst->blockIndex++; + // + + // Norm of magn + qMagn = inst->normData - inst->stages; + + // Compute spectral flatness on input spectrum + WebRtcNsx_ComputeSpectralFlatness(inst, magnU16); + + // quantile noise estimate + WebRtcNsx_NoiseEstimation(inst, magnU16, noiseU32, &qNoise); + + //noise estimate from previous frame + for (i = 0; i < inst->magnLen; i++) { + prevNoiseU16[i] = (uint16_t)(inst->prevNoiseU32[i] >> 11); // Q(prevQNoise) + } + + if (inst->blockIndex < END_STARTUP_SHORT) { + // Noise Q-domain to be used later; see description at end of section. + q_domain_to_use = WEBRTC_SPL_MIN((int)qNoise, inst->minNorm - inst->stages); + + // Calculate frequency independent parts in parametric noise estimate and calculate + // the estimate for the lower frequency band (same values for all frequency bins) + if (inst->pinkNoiseExp) { + pink_noise_exp_avg = (int16_t)WebRtcSpl_DivW32W16(inst->pinkNoiseExp, + (int16_t)(inst->blockIndex + 1)); // Q14 + pink_noise_num_avg = WebRtcSpl_DivW32W16(inst->pinkNoiseNumerator, + (int16_t)(inst->blockIndex + 1)); // Q11 + WebRtcNsx_CalcParametricNoiseEstimate(inst, + pink_noise_exp_avg, + pink_noise_num_avg, + kStartBand, + &noise_estimate, + &noise_estimate_avg); + } else { + // Use white noise estimate if we have poor pink noise parameter estimates + noise_estimate = inst->whiteNoiseLevel; // Q(minNorm-stages) + noise_estimate_avg = noise_estimate / (inst->blockIndex + 1); // Q(minNorm-stages) + } + for (i = 0; i < inst->magnLen; i++) { + // Estimate the background noise using the pink noise parameters if permitted + if ((inst->pinkNoiseExp) && (i >= kStartBand)) { + // Reset noise_estimate + noise_estimate = 0; + noise_estimate_avg = 0; + // Calculate the parametric noise estimate for current frequency bin + WebRtcNsx_CalcParametricNoiseEstimate(inst, + pink_noise_exp_avg, + pink_noise_num_avg, + i, + &noise_estimate, + &noise_estimate_avg); + } + // Calculate parametric Wiener filter + noiseSupFilterTmp[i] = inst->denoiseBound; + if (inst->initMagnEst[i]) { + // numerator = (initMagnEst - noise_estimate * overdrive) + // Result in Q(8+minNorm-stages) + tmpU32no1 = WEBRTC_SPL_UMUL_32_16(noise_estimate, inst->overdrive); + numerator = inst->initMagnEst[i] << 8; + if (numerator > tmpU32no1) { + // Suppression filter coefficient larger than zero, so calculate. + numerator -= tmpU32no1; + + // Determine number of left shifts in numerator for best accuracy after + // division + nShifts = WebRtcSpl_NormU32(numerator); + nShifts = WEBRTC_SPL_SAT(6, nShifts, 0); + + // Shift numerator to Q(nShifts+8+minNorm-stages) + numerator <<= nShifts; + + // Shift denominator to Q(nShifts-6+minNorm-stages) + tmpU32no1 = inst->initMagnEst[i] >> (6 - nShifts); + if (tmpU32no1 == 0) { + // This is only possible if numerator = 0, in which case + // we don't need any division. + tmpU32no1 = 1; + } + tmpU32no2 = numerator / tmpU32no1; // Q14 + noiseSupFilterTmp[i] = (uint16_t)WEBRTC_SPL_SAT(16384, tmpU32no2, + (uint32_t)(inst->denoiseBound)); // Q14 + } + } + // Weight quantile noise 'noiseU32' with modeled noise 'noise_estimate_avg' + // 'noiseU32 is in Q(qNoise) and 'noise_estimate' in Q(minNorm-stages) + // To guarantee that we do not get wrap around when shifting to the same domain + // we use the lowest one. Furthermore, we need to save 6 bits for the weighting. + // 'noise_estimate_avg' can handle this operation by construction, but 'noiseU32' + // may not. + + // Shift 'noiseU32' to 'q_domain_to_use' + tmpU32no1 = noiseU32[i] >> (qNoise - q_domain_to_use); + // Shift 'noise_estimate_avg' to 'q_domain_to_use' + tmpU32no2 = noise_estimate_avg >> + (inst->minNorm - inst->stages - q_domain_to_use); + // Make a simple check to see if we have enough room for weighting 'tmpU32no1' + // without wrap around + nShifts = 0; + if (tmpU32no1 & 0xfc000000) { + tmpU32no1 >>= 6; + tmpU32no2 >>= 6; + nShifts = 6; + } + tmpU32no1 *= inst->blockIndex; + tmpU32no2 *= (END_STARTUP_SHORT - inst->blockIndex); + // Add them together and divide by startup length + noiseU32[i] = WebRtcSpl_DivU32U16(tmpU32no1 + tmpU32no2, END_STARTUP_SHORT); + // Shift back if necessary + noiseU32[i] <<= nShifts; + } + // Update new Q-domain for 'noiseU32' + qNoise = q_domain_to_use; + } + // compute average signal during END_STARTUP_LONG time: + // used to normalize spectral difference measure + if (inst->blockIndex < END_STARTUP_LONG) { + // substituting division with shift ending up in Q(-2*stages) + inst->timeAvgMagnEnergyTmp += + inst->magnEnergy >> (2 * inst->normData + inst->stages - 1); + inst->timeAvgMagnEnergy = WebRtcSpl_DivU32U16(inst->timeAvgMagnEnergyTmp, + inst->blockIndex + 1); + } + + //start processing at frames == converged+1 + // STEP 1: compute prior and post SNR based on quantile noise estimates + + // compute direct decision (DD) estimate of prior SNR: needed for new method + satMax = (uint32_t)1048575;// Largest possible value without getting overflow despite shifting 12 steps + postShifts = 6 + qMagn - qNoise; + nShifts = 5 - inst->prevQMagn + inst->prevQNoise; + for (i = 0; i < inst->magnLen; i++) { + // FLOAT: + // post SNR + // postLocSnr[i] = 0.0; + // if (magn[i] > noise[i]) + // { + // postLocSnr[i] = magn[i] / (noise[i] + 0.0001); + // } + // // previous post SNR + // // previous estimate: based on previous frame with gain filter (smooth is previous filter) + // + // prevNearSnr[i] = inst->prevMagnU16[i] / (inst->noisePrev[i] + 0.0001) * (inst->smooth[i]); + // + // // DD estimate is sum of two terms: current estimate and previous estimate + // // directed decision update of priorSnr (or we actually store [2*priorSnr+1]) + // + // priorLocSnr[i] = DD_PR_SNR * prevNearSnr[i] + (1.0 - DD_PR_SNR) * (postLocSnr[i] - 1.0); + + // calculate post SNR: output in Q11 + postLocSnr[i] = 2048; // 1.0 in Q11 + tmpU32no1 = (uint32_t)magnU16[i] << 6; // Q(6+qMagn) + if (postShifts < 0) { + tmpU32no2 = noiseU32[i] >> -postShifts; // Q(6+qMagn) + } else { + tmpU32no2 = noiseU32[i] << postShifts; // Q(6+qMagn) + } + if (tmpU32no1 > tmpU32no2) { + // Current magnitude larger than noise + tmpU32no1 <<= 11; // Q(17+qMagn) + if (tmpU32no2 > 0) { + tmpU32no1 /= tmpU32no2; // Q11 + postLocSnr[i] = WEBRTC_SPL_MIN(satMax, tmpU32no1); // Q11 + } else { + postLocSnr[i] = satMax; + } + } + + // calculate prevNearSnr[i] and save for later instead of recalculating it later + // |nearMagnEst| in Q(prevQMagn + 14) + nearMagnEst = inst->prevMagnU16[i] * inst->noiseSupFilter[i]; + tmpU32no1 = nearMagnEst << 3; // Q(prevQMagn+17) + tmpU32no2 = inst->prevNoiseU32[i] >> nShifts; // Q(prevQMagn+6) + + if (tmpU32no2 > 0) { + tmpU32no1 /= tmpU32no2; // Q11 + tmpU32no1 = WEBRTC_SPL_MIN(satMax, tmpU32no1); // Q11 + } else { + tmpU32no1 = satMax; // Q11 + } + prevNearSnr[i] = tmpU32no1; // Q11 + + //directed decision update of priorSnr + tmpU32no1 = WEBRTC_SPL_UMUL_32_16(prevNearSnr[i], DD_PR_SNR_Q11); // Q22 + tmpU32no2 = WEBRTC_SPL_UMUL_32_16(postLocSnr[i] - 2048, ONE_MINUS_DD_PR_SNR_Q11); // Q22 + priorSnr = tmpU32no1 + tmpU32no2 + 512; // Q22 (added 512 for rounding) + // priorLocSnr = 1 + 2*priorSnr + priorLocSnr[i] = 2048 + (priorSnr >> 10); // Q11 + } // end of loop over frequencies + // done with step 1: DD computation of prior and post SNR + + // STEP 2: compute speech/noise likelihood + + //compute difference of input spectrum with learned/estimated noise spectrum + WebRtcNsx_ComputeSpectralDifference(inst, magnU16); + //compute histograms for determination of parameters (thresholds and weights for features) + //parameters are extracted once every window time (=inst->modelUpdate) + //counter update + inst->cntThresUpdate++; + flag = (int)(inst->cntThresUpdate == inst->modelUpdate); + //update histogram + WebRtcNsx_FeatureParameterExtraction(inst, flag); + //compute model parameters + if (flag) { + inst->cntThresUpdate = 0; // Reset counter + //update every window: + // get normalization for spectral difference for next window estimate + + // Shift to Q(-2*stages) + inst->curAvgMagnEnergy >>= STAT_UPDATES; + + tmpU32no1 = (inst->curAvgMagnEnergy + inst->timeAvgMagnEnergy + 1) >> 1; //Q(-2*stages) + // Update featureSpecDiff + if ((tmpU32no1 != inst->timeAvgMagnEnergy) && (inst->featureSpecDiff) && + (inst->timeAvgMagnEnergy > 0)) { + norm32no1 = 0; + tmpU32no3 = tmpU32no1; + while (0xFFFF0000 & tmpU32no3) { + tmpU32no3 >>= 1; + norm32no1++; + } + tmpU32no2 = inst->featureSpecDiff; + while (0xFFFF0000 & tmpU32no2) { + tmpU32no2 >>= 1; + norm32no1++; + } + tmpU32no3 = WEBRTC_SPL_UMUL(tmpU32no3, tmpU32no2); + tmpU32no3 /= inst->timeAvgMagnEnergy; + if (WebRtcSpl_NormU32(tmpU32no3) < norm32no1) { + inst->featureSpecDiff = 0x007FFFFF; + } else { + inst->featureSpecDiff = WEBRTC_SPL_MIN(0x007FFFFF, + tmpU32no3 << norm32no1); + } + } + + inst->timeAvgMagnEnergy = tmpU32no1; // Q(-2*stages) + inst->curAvgMagnEnergy = 0; + } + + //compute speech/noise probability + WebRtcNsx_SpeechNoiseProb(inst, nonSpeechProbFinal, priorLocSnr, postLocSnr); + + //time-avg parameter for noise update + gammaNoise = NOISE_UPDATE_Q8; // Q8 + + maxNoiseU32 = 0; + postShifts = inst->prevQNoise - qMagn; + nShifts = inst->prevQMagn - qMagn; + for (i = 0; i < inst->magnLen; i++) { + // temporary noise update: use it for speech frames if update value is less than previous + // the formula has been rewritten into: + // noiseUpdate = noisePrev[i] + (1 - gammaNoise) * nonSpeechProb * (magn[i] - noisePrev[i]) + + if (postShifts < 0) { + tmpU32no2 = magnU16[i] >> -postShifts; // Q(prevQNoise) + } else { + tmpU32no2 = (uint32_t)magnU16[i] << postShifts; // Q(prevQNoise) + } + if (prevNoiseU16[i] > tmpU32no2) { + sign = -1; + tmpU32no1 = prevNoiseU16[i] - tmpU32no2; + } else { + sign = 1; + tmpU32no1 = tmpU32no2 - prevNoiseU16[i]; + } + noiseUpdateU32 = inst->prevNoiseU32[i]; // Q(prevQNoise+11) + tmpU32no3 = 0; + if ((tmpU32no1) && (nonSpeechProbFinal[i])) { + // This value will be used later, if gammaNoise changes + tmpU32no3 = WEBRTC_SPL_UMUL_32_16(tmpU32no1, nonSpeechProbFinal[i]); // Q(prevQNoise+8) + if (0x7c000000 & tmpU32no3) { + // Shifting required before multiplication + tmpU32no2 = (tmpU32no3 >> 5) * gammaNoise; // Q(prevQNoise+11) + } else { + // We can do shifting after multiplication + tmpU32no2 = (tmpU32no3 * gammaNoise) >> 5; // Q(prevQNoise+11) + } + if (sign > 0) { + noiseUpdateU32 += tmpU32no2; // Q(prevQNoise+11) + } else { + // This operation is safe. We can never get wrap around, since worst + // case scenario means magnU16 = 0 + noiseUpdateU32 -= tmpU32no2; // Q(prevQNoise+11) + } + } + + //increase gamma (i.e., less noise update) for frame likely to be speech + prevGammaNoise = gammaNoise; + gammaNoise = NOISE_UPDATE_Q8; + //time-constant based on speech/noise state + //increase gamma (i.e., less noise update) for frames likely to be speech + if (nonSpeechProbFinal[i] < ONE_MINUS_PROB_RANGE_Q8) { + gammaNoise = GAMMA_NOISE_TRANS_AND_SPEECH_Q8; + } + + if (prevGammaNoise != gammaNoise) { + // new noise update + // this line is the same as above, only that the result is stored in a different variable and the gammaNoise + // has changed + // + // noiseUpdate = noisePrev[i] + (1 - gammaNoise) * nonSpeechProb * (magn[i] - noisePrev[i]) + + if (0x7c000000 & tmpU32no3) { + // Shifting required before multiplication + tmpU32no2 = (tmpU32no3 >> 5) * gammaNoise; // Q(prevQNoise+11) + } else { + // We can do shifting after multiplication + tmpU32no2 = (tmpU32no3 * gammaNoise) >> 5; // Q(prevQNoise+11) + } + if (sign > 0) { + tmpU32no1 = inst->prevNoiseU32[i] + tmpU32no2; // Q(prevQNoise+11) + } else { + tmpU32no1 = inst->prevNoiseU32[i] - tmpU32no2; // Q(prevQNoise+11) + } + if (noiseUpdateU32 > tmpU32no1) { + noiseUpdateU32 = tmpU32no1; // Q(prevQNoise+11) + } + } + noiseU32[i] = noiseUpdateU32; // Q(prevQNoise+11) + if (noiseUpdateU32 > maxNoiseU32) { + maxNoiseU32 = noiseUpdateU32; + } + + // conservative noise update + // // original FLOAT code + // if (prob_speech < PROB_RANGE) { + // inst->avgMagnPause[i] = inst->avgMagnPause[i] + (1.0 - gamma_pause)*(magn[i] - inst->avgMagnPause[i]); + // } + + tmp32no2 = WEBRTC_SPL_SHIFT_W32(inst->avgMagnPause[i], -nShifts); + if (nonSpeechProbFinal[i] > ONE_MINUS_PROB_RANGE_Q8) { + if (nShifts < 0) { + tmp32no1 = (int32_t)magnU16[i] - tmp32no2; // Q(qMagn) + tmp32no1 *= ONE_MINUS_GAMMA_PAUSE_Q8; // Q(8+prevQMagn+nShifts) + tmp32no1 = (tmp32no1 + 128) >> 8; // Q(qMagn). + } else { + // In Q(qMagn+nShifts) + tmp32no1 = ((int32_t)magnU16[i] << nShifts) - inst->avgMagnPause[i]; + tmp32no1 *= ONE_MINUS_GAMMA_PAUSE_Q8; // Q(8+prevQMagn+nShifts) + tmp32no1 = (tmp32no1 + (128 << nShifts)) >> (8 + nShifts); // Q(qMagn). + } + tmp32no2 += tmp32no1; // Q(qMagn) + } + inst->avgMagnPause[i] = tmp32no2; + } // end of frequency loop + + norm32no1 = WebRtcSpl_NormU32(maxNoiseU32); + qNoise = inst->prevQNoise + norm32no1 - 5; + // done with step 2: noise update + + // STEP 3: compute dd update of prior snr and post snr based on new noise estimate + nShifts = inst->prevQNoise + 11 - qMagn; + for (i = 0; i < inst->magnLen; i++) { + // FLOAT code + // // post and prior SNR + // curNearSnr = 0.0; + // if (magn[i] > noise[i]) + // { + // curNearSnr = magn[i] / (noise[i] + 0.0001) - 1.0; + // } + // // DD estimate is sum of two terms: current estimate and previous estimate + // // directed decision update of snrPrior + // snrPrior = DD_PR_SNR * prevNearSnr[i] + (1.0 - DD_PR_SNR) * curNearSnr; + // // gain filter + // tmpFloat1 = inst->overdrive + snrPrior; + // tmpFloat2 = snrPrior / tmpFloat1; + // theFilter[i] = tmpFloat2; + + // calculate curNearSnr again, this is necessary because a new noise estimate has been made since then. for the original + curNearSnr = 0; // Q11 + if (nShifts < 0) { + // This case is equivalent with magn < noise which implies curNearSnr = 0; + tmpMagnU32 = (uint32_t)magnU16[i]; // Q(qMagn) + tmpNoiseU32 = noiseU32[i] << -nShifts; // Q(qMagn) + } else if (nShifts > 17) { + tmpMagnU32 = (uint32_t)magnU16[i] << 17; // Q(qMagn+17) + tmpNoiseU32 = noiseU32[i] >> (nShifts - 17); // Q(qMagn+17) + } else { + tmpMagnU32 = (uint32_t)magnU16[i] << nShifts; // Q(qNoise_prev+11) + tmpNoiseU32 = noiseU32[i]; // Q(qNoise_prev+11) + } + if (tmpMagnU32 > tmpNoiseU32) { + tmpU32no1 = tmpMagnU32 - tmpNoiseU32; // Q(qCur) + norm32no2 = WEBRTC_SPL_MIN(11, WebRtcSpl_NormU32(tmpU32no1)); + tmpU32no1 <<= norm32no2; // Q(qCur+norm32no2) + tmpU32no2 = tmpNoiseU32 >> (11 - norm32no2); // Q(qCur+norm32no2-11) + if (tmpU32no2 > 0) { + tmpU32no1 /= tmpU32no2; // Q11 + } + curNearSnr = WEBRTC_SPL_MIN(satMax, tmpU32no1); // Q11 + } + + //directed decision update of priorSnr + // FLOAT + // priorSnr = DD_PR_SNR * prevNearSnr + (1.0-DD_PR_SNR) * curNearSnr; + + tmpU32no1 = WEBRTC_SPL_UMUL_32_16(prevNearSnr[i], DD_PR_SNR_Q11); // Q22 + tmpU32no2 = WEBRTC_SPL_UMUL_32_16(curNearSnr, ONE_MINUS_DD_PR_SNR_Q11); // Q22 + priorSnr = tmpU32no1 + tmpU32no2; // Q22 + + //gain filter + tmpU32no1 = inst->overdrive + ((priorSnr + 8192) >> 14); // Q8 + RTC_DCHECK_GT(inst->overdrive, 0); + tmpU16no1 = (priorSnr + tmpU32no1 / 2) / tmpU32no1; // Q14 + inst->noiseSupFilter[i] = WEBRTC_SPL_SAT(16384, tmpU16no1, inst->denoiseBound); // 16384 = Q14(1.0) // Q14 + + // Weight in the parametric Wiener filter during startup + if (inst->blockIndex < END_STARTUP_SHORT) { + // Weight the two suppression filters + tmpU32no1 = inst->noiseSupFilter[i] * inst->blockIndex; + tmpU32no2 = noiseSupFilterTmp[i] * + (END_STARTUP_SHORT - inst->blockIndex); + tmpU32no1 += tmpU32no2; + inst->noiseSupFilter[i] = (uint16_t)WebRtcSpl_DivU32U16(tmpU32no1, + END_STARTUP_SHORT); + } + } // end of loop over frequencies + //done with step3 + + // save noise and magnitude spectrum for next frame + inst->prevQNoise = qNoise; + inst->prevQMagn = qMagn; + if (norm32no1 > 5) { + for (i = 0; i < inst->magnLen; i++) { + inst->prevNoiseU32[i] = noiseU32[i] << (norm32no1 - 5); // Q(qNoise+11) + inst->prevMagnU16[i] = magnU16[i]; // Q(qMagn) + } + } else { + for (i = 0; i < inst->magnLen; i++) { + inst->prevNoiseU32[i] = noiseU32[i] >> (5 - norm32no1); // Q(qNoise+11) + inst->prevMagnU16[i] = magnU16[i]; // Q(qMagn) + } + } + + WebRtcNsx_DataSynthesis(inst, outFrame[0]); +#ifdef NS_FILEDEBUG + if (fwrite(outframe, sizeof(short), + inst->blockLen10ms, inst->outfile) != inst->blockLen10ms) { + RTC_NOTREACHED(); + } +#endif + + //for H band: + // only update data buffer, then apply time-domain gain is applied derived from L band + if (num_bands > 1) { + // update analysis buffer for H band + // append new data to buffer FX + for (i = 0; i < num_high_bands; ++i) { + memcpy(inst->dataBufHBFX[i], inst->dataBufHBFX[i] + inst->blockLen10ms, + (inst->anaLen - inst->blockLen10ms) * sizeof(*inst->dataBufHBFX[i])); + memcpy(inst->dataBufHBFX[i] + inst->anaLen - inst->blockLen10ms, + speechFrameHB[i], inst->blockLen10ms * sizeof(*inst->dataBufHBFX[i])); + } + // range for averaging low band quantities for H band gain + + gainTimeDomainHB = 16384; // 16384 = Q14(1.0) + //average speech prob from low band + //average filter gain from low band + //avg over second half (i.e., 4->8kHz) of freq. spectrum + tmpU32no1 = 0; // Q12 + tmpU16no1 = 0; // Q8 + for (i = inst->anaLen2 - (inst->anaLen2 >> 2); i < inst->anaLen2; i++) { + tmpU16no1 += nonSpeechProbFinal[i]; // Q8 + tmpU32no1 += (uint32_t)(inst->noiseSupFilter[i]); // Q14 + } + RTC_DCHECK_GE(inst->stages, 7); + avgProbSpeechHB = (4096 - (tmpU16no1 >> (inst->stages - 7))); // Q12 + avgFilterGainHB = (int16_t)(tmpU32no1 >> (inst->stages - 3)); // Q14 + + // // original FLOAT code + // // gain based on speech probability: + // avg_prob_speech_tt=(float)2.0*avg_prob_speech-(float)1.0; + // gain_mod=(float)0.5*((float)1.0+(float)tanh(avg_prob_speech_tt)); // between 0 and 1 + + // gain based on speech probability: + // original expression: "0.5 * (1 + tanh(2x-1))" + // avgProbSpeechHB has been anyway saturated to a value between 0 and 1 so the other cases don't have to be dealt with + // avgProbSpeechHB and gainModHB are in Q12, 3607 = Q12(0.880615234375) which is a zero point of + // |0.5 * (1 + tanh(2x-1)) - x| - |0.5 * (1 + tanh(2x-1)) - 0.880615234375| meaning that from that point the error of approximating + // the expression with f(x) = x would be greater than the error of approximating the expression with f(x) = 0.880615234375 + // error: "|0.5 * (1 + tanh(2x-1)) - x| from x=0 to 0.880615234375" -> http://www.wolframalpha.com/input/?i=|0.5+*+(1+%2B+tanh(2x-1))+-+x|+from+x%3D0+to+0.880615234375 + // and: "|0.5 * (1 + tanh(2x-1)) - 0.880615234375| from x=0.880615234375 to 1" -> http://www.wolframalpha.com/input/?i=+|0.5+*+(1+%2B+tanh(2x-1))+-+0.880615234375|+from+x%3D0.880615234375+to+1 + gainModHB = WEBRTC_SPL_MIN(avgProbSpeechHB, 3607); + + // // original FLOAT code + // //combine gain with low band gain + // if (avg_prob_speech < (float)0.5) { + // gain_time_domain_HB=(float)0.5*gain_mod+(float)0.5*avg_filter_gain; + // } + // else { + // gain_time_domain_HB=(float)0.25*gain_mod+(float)0.75*avg_filter_gain; + // } + + + //combine gain with low band gain + if (avgProbSpeechHB < 2048) { + // 2048 = Q12(0.5) + // the next two lines in float are "gain_time_domain = 0.5 * gain_mod + 0.5 * avg_filter_gain"; Q2(0.5) = 2 equals one left shift + gainTimeDomainHB = (gainModHB << 1) + (avgFilterGainHB >> 1); // Q14 + } else { + // "gain_time_domain = 0.25 * gain_mod + 0.75 * agv_filter_gain;" + gainTimeDomainHB = (int16_t)((3 * avgFilterGainHB) >> 2); // 3 = Q2(0.75) + gainTimeDomainHB += gainModHB; // Q14 + } + //make sure gain is within flooring range + gainTimeDomainHB + = WEBRTC_SPL_SAT(16384, gainTimeDomainHB, (int16_t)(inst->denoiseBound)); // 16384 = Q14(1.0) + + + //apply gain + for (i = 0; i < num_high_bands; ++i) { + for (j = 0; j < inst->blockLen10ms; j++) { + outFrameHB[i][j] = (int16_t)((gainTimeDomainHB * + inst->dataBufHBFX[i][j]) >> 14); // Q0 + } + } + } // end of H band gain computation +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.h b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.h new file mode 100644 index 0000000000..c8097f7946 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core.h @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_NSX_CORE_H_ +#define MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_NSX_CORE_H_ + +#ifdef NS_FILEDEBUG +#include +#endif + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/ns/nsx_defines.h" +#include "typedefs.h" // NOLINT(build/include) + +typedef struct NoiseSuppressionFixedC_ { + uint32_t fs; + + const int16_t* window; + int16_t analysisBuffer[ANAL_BLOCKL_MAX]; + int16_t synthesisBuffer[ANAL_BLOCKL_MAX]; + uint16_t noiseSupFilter[HALF_ANAL_BLOCKL]; + uint16_t overdrive; /* Q8 */ + uint16_t denoiseBound; /* Q14 */ + const int16_t* factor2Table; + int16_t noiseEstLogQuantile[SIMULT* HALF_ANAL_BLOCKL]; + int16_t noiseEstDensity[SIMULT* HALF_ANAL_BLOCKL]; + int16_t noiseEstCounter[SIMULT]; + int16_t noiseEstQuantile[HALF_ANAL_BLOCKL]; + + size_t anaLen; + size_t anaLen2; + size_t magnLen; + int aggrMode; + int stages; + int initFlag; + int gainMap; + + int32_t maxLrt; + int32_t minLrt; + // Log LRT factor with time-smoothing in Q8. + int32_t logLrtTimeAvgW32[HALF_ANAL_BLOCKL]; + int32_t featureLogLrt; + int32_t thresholdLogLrt; + int16_t weightLogLrt; + + uint32_t featureSpecDiff; + uint32_t thresholdSpecDiff; + int16_t weightSpecDiff; + + uint32_t featureSpecFlat; + uint32_t thresholdSpecFlat; + int16_t weightSpecFlat; + + // Conservative estimate of noise spectrum. + int32_t avgMagnPause[HALF_ANAL_BLOCKL]; + uint32_t magnEnergy; + uint32_t sumMagn; + uint32_t curAvgMagnEnergy; + uint32_t timeAvgMagnEnergy; + uint32_t timeAvgMagnEnergyTmp; + + uint32_t whiteNoiseLevel; // Initial noise estimate. + // Initial magnitude spectrum estimate. + uint32_t initMagnEst[HALF_ANAL_BLOCKL]; + // Pink noise parameters: + int32_t pinkNoiseNumerator; // Numerator. + int32_t pinkNoiseExp; // Power of freq. + int minNorm; // Smallest normalization factor. + int zeroInputSignal; // Zero input signal flag. + + // Noise spectrum from previous frame. + uint32_t prevNoiseU32[HALF_ANAL_BLOCKL]; + // Magnitude spectrum from previous frame. + uint16_t prevMagnU16[HALF_ANAL_BLOCKL]; + // Prior speech/noise probability in Q14. + int16_t priorNonSpeechProb; + + int blockIndex; // Frame index counter. + // Parameter for updating or estimating thresholds/weights for prior model. + int modelUpdate; + int cntThresUpdate; + + // Histograms for parameter estimation. + int16_t histLrt[HIST_PAR_EST]; + int16_t histSpecFlat[HIST_PAR_EST]; + int16_t histSpecDiff[HIST_PAR_EST]; + + // Quantities for high band estimate. + int16_t dataBufHBFX[NUM_HIGH_BANDS_MAX][ANAL_BLOCKL_MAX]; + + int qNoise; + int prevQNoise; + int prevQMagn; + size_t blockLen10ms; + + int16_t real[ANAL_BLOCKL_MAX]; + int16_t imag[ANAL_BLOCKL_MAX]; + int32_t energyIn; + int scaleEnergyIn; + int normData; + + struct RealFFT* real_fft; +} NoiseSuppressionFixedC; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/**************************************************************************** + * WebRtcNsx_InitCore(...) + * + * This function initializes a noise suppression instance + * + * Input: + * - inst : Instance that should be initialized + * - fs : Sampling frequency + * + * Output: + * - inst : Initialized instance + * + * Return value : 0 - Ok + * -1 - Error + */ +int32_t WebRtcNsx_InitCore(NoiseSuppressionFixedC* inst, uint32_t fs); + +/**************************************************************************** + * WebRtcNsx_set_policy_core(...) + * + * This changes the aggressiveness of the noise suppression method. + * + * Input: + * - inst : Instance that should be initialized + * - mode : 0: Mild (6 dB), 1: Medium (10 dB), 2: Aggressive (15 dB) + * + * Output: + * - inst : Initialized instance + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcNsx_set_policy_core(NoiseSuppressionFixedC* inst, int mode); + +/**************************************************************************** + * WebRtcNsx_ProcessCore + * + * Do noise suppression. + * + * Input: + * - inst : Instance that should be initialized + * - inFrame : Input speech frame for each band + * - num_bands : Number of bands + * + * Output: + * - inst : Updated instance + * - outFrame : Output speech frame for each band + */ +void WebRtcNsx_ProcessCore(NoiseSuppressionFixedC* inst, + const short* const* inFrame, + int num_bands, + short* const* outFrame); + +/**************************************************************************** + * Some function pointers, for internal functions shared by ARM NEON and + * generic C code. + */ +// Noise Estimation. +typedef void (*NoiseEstimation)(NoiseSuppressionFixedC* inst, + uint16_t* magn, + uint32_t* noise, + int16_t* q_noise); +extern NoiseEstimation WebRtcNsx_NoiseEstimation; + +// Filter the data in the frequency domain, and create spectrum. +typedef void (*PrepareSpectrum)(NoiseSuppressionFixedC* inst, + int16_t* freq_buff); +extern PrepareSpectrum WebRtcNsx_PrepareSpectrum; + +// For the noise supression process, synthesis, read out fully processed +// segment, and update synthesis buffer. +typedef void (*SynthesisUpdate)(NoiseSuppressionFixedC* inst, + int16_t* out_frame, + int16_t gain_factor); +extern SynthesisUpdate WebRtcNsx_SynthesisUpdate; + +// Update analysis buffer for lower band, and window data before FFT. +typedef void (*AnalysisUpdate)(NoiseSuppressionFixedC* inst, + int16_t* out, + int16_t* new_speech); +extern AnalysisUpdate WebRtcNsx_AnalysisUpdate; + +// Denormalize the real-valued signal |in|, the output from inverse FFT. +typedef void (*Denormalize)(NoiseSuppressionFixedC* inst, + int16_t* in, + int factor); +extern Denormalize WebRtcNsx_Denormalize; + +// Normalize the real-valued signal |in|, the input to forward FFT. +typedef void (*NormalizeRealBuffer)(NoiseSuppressionFixedC* inst, + const int16_t* in, + int16_t* out); +extern NormalizeRealBuffer WebRtcNsx_NormalizeRealBuffer; + +// Compute speech/noise probability. +// Intended to be private. +void WebRtcNsx_SpeechNoiseProb(NoiseSuppressionFixedC* inst, + uint16_t* nonSpeechProbFinal, + uint32_t* priorLocSnr, + uint32_t* postLocSnr); + +#if defined(WEBRTC_HAS_NEON) +// For the above function pointers, functions for generic platforms are declared +// and defined as static in file nsx_core.c, while those for ARM Neon platforms +// are declared below and defined in file nsx_core_neon.c. +void WebRtcNsx_NoiseEstimationNeon(NoiseSuppressionFixedC* inst, + uint16_t* magn, + uint32_t* noise, + int16_t* q_noise); +void WebRtcNsx_SynthesisUpdateNeon(NoiseSuppressionFixedC* inst, + int16_t* out_frame, + int16_t gain_factor); +void WebRtcNsx_AnalysisUpdateNeon(NoiseSuppressionFixedC* inst, + int16_t* out, + int16_t* new_speech); +void WebRtcNsx_PrepareSpectrumNeon(NoiseSuppressionFixedC* inst, + int16_t* freq_buff); +#endif + +#if defined(MIPS32_LE) +// For the above function pointers, functions for generic platforms are declared +// and defined as static in file nsx_core.c, while those for MIPS platforms +// are declared below and defined in file nsx_core_mips.c. +void WebRtcNsx_SynthesisUpdate_mips(NoiseSuppressionFixedC* inst, + int16_t* out_frame, + int16_t gain_factor); +void WebRtcNsx_AnalysisUpdate_mips(NoiseSuppressionFixedC* inst, + int16_t* out, + int16_t* new_speech); +void WebRtcNsx_PrepareSpectrum_mips(NoiseSuppressionFixedC* inst, + int16_t* freq_buff); +void WebRtcNsx_NormalizeRealBuffer_mips(NoiseSuppressionFixedC* inst, + const int16_t* in, + int16_t* out); +#if defined(MIPS_DSP_R1_LE) +void WebRtcNsx_Denormalize_mips(NoiseSuppressionFixedC* inst, + int16_t* in, + int factor); +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif // MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_NSX_CORE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_c.c b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_c.c new file mode 100644 index 0000000000..162fb1990a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_c.c @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/checks.h" +#include "modules/audio_processing/ns/noise_suppression_x.h" +#include "modules/audio_processing/ns/nsx_core.h" +#include "modules/audio_processing/ns/nsx_defines.h" + +static const int16_t kIndicatorTable[17] = { + 0, 2017, 3809, 5227, 6258, 6963, 7424, 7718, + 7901, 8014, 8084, 8126, 8152, 8168, 8177, 8183, 8187 +}; + +// Compute speech/noise probability +// speech/noise probability is returned in: probSpeechFinal +//snrLocPrior is the prior SNR for each frequency (in Q11) +//snrLocPost is the post SNR for each frequency (in Q11) +void WebRtcNsx_SpeechNoiseProb(NoiseSuppressionFixedC* inst, + uint16_t* nonSpeechProbFinal, + uint32_t* priorLocSnr, + uint32_t* postLocSnr) { + uint32_t zeros, num, den, tmpU32no1, tmpU32no2, tmpU32no3; + int32_t invLrtFX, indPriorFX, tmp32, tmp32no1, tmp32no2, besselTmpFX32; + int32_t frac32, logTmp; + int32_t logLrtTimeAvgKsumFX; + int16_t indPriorFX16; + int16_t tmp16, tmp16no1, tmp16no2, tmpIndFX, tableIndex, frac, intPart; + size_t i; + int normTmp, normTmp2, nShifts; + + // compute feature based on average LR factor + // this is the average over all frequencies of the smooth log LRT + logLrtTimeAvgKsumFX = 0; + for (i = 0; i < inst->magnLen; i++) { + besselTmpFX32 = (int32_t)postLocSnr[i]; // Q11 + normTmp = WebRtcSpl_NormU32(postLocSnr[i]); + num = postLocSnr[i] << normTmp; // Q(11+normTmp) + if (normTmp > 10) { + den = priorLocSnr[i] << (normTmp - 11); // Q(normTmp) + } else { + den = priorLocSnr[i] >> (11 - normTmp); // Q(normTmp) + } + if (den > 0) { + besselTmpFX32 -= num / den; // Q11 + } else { + besselTmpFX32 = 0; + } + + // inst->logLrtTimeAvg[i] += LRT_TAVG * (besselTmp - log(snrLocPrior) + // - inst->logLrtTimeAvg[i]); + // Here, LRT_TAVG = 0.5 + zeros = WebRtcSpl_NormU32(priorLocSnr[i]); + frac32 = (int32_t)(((priorLocSnr[i] << zeros) & 0x7FFFFFFF) >> 19); + tmp32 = (frac32 * frac32 * -43) >> 19; + tmp32 += ((int16_t)frac32 * 5412) >> 12; + frac32 = tmp32 + 37; + // tmp32 = log2(priorLocSnr[i]) + tmp32 = (int32_t)(((31 - zeros) << 12) + frac32) - (11 << 12); // Q12 + logTmp = (tmp32 * 178) >> 8; // log2(priorLocSnr[i])*log(2) + // tmp32no1 = LRT_TAVG * (log(snrLocPrior) + inst->logLrtTimeAvg[i]) in Q12. + tmp32no1 = (logTmp + inst->logLrtTimeAvgW32[i]) / 2; + inst->logLrtTimeAvgW32[i] += (besselTmpFX32 - tmp32no1); // Q12 + + logLrtTimeAvgKsumFX += inst->logLrtTimeAvgW32[i]; // Q12 + } + inst->featureLogLrt = (logLrtTimeAvgKsumFX * BIN_SIZE_LRT) >> + (inst->stages + 11); + + // done with computation of LR factor + + // + //compute the indicator functions + // + + // average LRT feature + // FLOAT code + // indicator0 = 0.5 * (tanh(widthPrior * + // (logLrtTimeAvgKsum - threshPrior0)) + 1.0); + tmpIndFX = 16384; // Q14(1.0) + tmp32no1 = logLrtTimeAvgKsumFX - inst->thresholdLogLrt; // Q12 + nShifts = 7 - inst->stages; // WIDTH_PR_MAP_SHIFT - inst->stages + 5; + //use larger width in tanh map for pause regions + if (tmp32no1 < 0) { + tmpIndFX = 0; + tmp32no1 = -tmp32no1; + //widthPrior = widthPrior * 2.0; + nShifts++; + } + tmp32no1 = WEBRTC_SPL_SHIFT_W32(tmp32no1, nShifts); // Q14 + // compute indicator function: sigmoid map + if (tmp32no1 < (16 << 14) && tmp32no1 >= 0) { + tableIndex = (int16_t)(tmp32no1 >> 14); + tmp16no2 = kIndicatorTable[tableIndex]; + tmp16no1 = kIndicatorTable[tableIndex + 1] - kIndicatorTable[tableIndex]; + frac = (int16_t)(tmp32no1 & 0x00003fff); // Q14 + tmp16no2 += (int16_t)((tmp16no1 * frac) >> 14); + if (tmpIndFX == 0) { + tmpIndFX = 8192 - tmp16no2; // Q14 + } else { + tmpIndFX = 8192 + tmp16no2; // Q14 + } + } + indPriorFX = inst->weightLogLrt * tmpIndFX; // 6*Q14 + + //spectral flatness feature + if (inst->weightSpecFlat) { + tmpU32no1 = WEBRTC_SPL_UMUL(inst->featureSpecFlat, 400); // Q10 + tmpIndFX = 16384; // Q14(1.0) + //use larger width in tanh map for pause regions + tmpU32no2 = inst->thresholdSpecFlat - tmpU32no1; //Q10 + nShifts = 4; + if (inst->thresholdSpecFlat < tmpU32no1) { + tmpIndFX = 0; + tmpU32no2 = tmpU32no1 - inst->thresholdSpecFlat; + //widthPrior = widthPrior * 2.0; + nShifts++; + } + tmpU32no1 = WebRtcSpl_DivU32U16(tmpU32no2 << nShifts, 25); // Q14 + // compute indicator function: sigmoid map + // FLOAT code + // indicator1 = 0.5 * (tanh(sgnMap * widthPrior * + // (threshPrior1 - tmpFloat1)) + 1.0); + if (tmpU32no1 < (16 << 14)) { + tableIndex = (int16_t)(tmpU32no1 >> 14); + tmp16no2 = kIndicatorTable[tableIndex]; + tmp16no1 = kIndicatorTable[tableIndex + 1] - kIndicatorTable[tableIndex]; + frac = (int16_t)(tmpU32no1 & 0x00003fff); // Q14 + tmp16no2 += (int16_t)((tmp16no1 * frac) >> 14); + if (tmpIndFX) { + tmpIndFX = 8192 + tmp16no2; // Q14 + } else { + tmpIndFX = 8192 - tmp16no2; // Q14 + } + } + indPriorFX += inst->weightSpecFlat * tmpIndFX; // 6*Q14 + } + + //for template spectral-difference + if (inst->weightSpecDiff) { + tmpU32no1 = 0; + if (inst->featureSpecDiff) { + normTmp = WEBRTC_SPL_MIN(20 - inst->stages, + WebRtcSpl_NormU32(inst->featureSpecDiff)); + RTC_DCHECK_GE(normTmp, 0); + tmpU32no1 = inst->featureSpecDiff << normTmp; // Q(normTmp-2*stages) + tmpU32no2 = inst->timeAvgMagnEnergy >> (20 - inst->stages - normTmp); + if (tmpU32no2 > 0) { + // Q(20 - inst->stages) + tmpU32no1 /= tmpU32no2; + } else { + tmpU32no1 = (uint32_t)(0x7fffffff); + } + } + tmpU32no3 = (inst->thresholdSpecDiff << 17) / 25; + tmpU32no2 = tmpU32no1 - tmpU32no3; + nShifts = 1; + tmpIndFX = 16384; // Q14(1.0) + //use larger width in tanh map for pause regions + if (tmpU32no2 & 0x80000000) { + tmpIndFX = 0; + tmpU32no2 = tmpU32no3 - tmpU32no1; + //widthPrior = widthPrior * 2.0; + nShifts--; + } + tmpU32no1 = tmpU32no2 >> nShifts; + // compute indicator function: sigmoid map + /* FLOAT code + indicator2 = 0.5 * (tanh(widthPrior * (tmpFloat1 - threshPrior2)) + 1.0); + */ + if (tmpU32no1 < (16 << 14)) { + tableIndex = (int16_t)(tmpU32no1 >> 14); + tmp16no2 = kIndicatorTable[tableIndex]; + tmp16no1 = kIndicatorTable[tableIndex + 1] - kIndicatorTable[tableIndex]; + frac = (int16_t)(tmpU32no1 & 0x00003fff); // Q14 + tmp16no2 += (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + tmp16no1, frac, 14); + if (tmpIndFX) { + tmpIndFX = 8192 + tmp16no2; + } else { + tmpIndFX = 8192 - tmp16no2; + } + } + indPriorFX += inst->weightSpecDiff * tmpIndFX; // 6*Q14 + } + + //combine the indicator function with the feature weights + // FLOAT code + // indPrior = 1 - (weightIndPrior0 * indicator0 + weightIndPrior1 * + // indicator1 + weightIndPrior2 * indicator2); + indPriorFX16 = WebRtcSpl_DivW32W16ResW16(98307 - indPriorFX, 6); // Q14 + // done with computing indicator function + + //compute the prior probability + // FLOAT code + // inst->priorNonSpeechProb += PRIOR_UPDATE * + // (indPriorNonSpeech - inst->priorNonSpeechProb); + tmp16 = indPriorFX16 - inst->priorNonSpeechProb; // Q14 + inst->priorNonSpeechProb += (int16_t)((PRIOR_UPDATE_Q14 * tmp16) >> 14); + + //final speech probability: combine prior model with LR factor: + + memset(nonSpeechProbFinal, 0, sizeof(uint16_t) * inst->magnLen); + + if (inst->priorNonSpeechProb > 0) { + for (i = 0; i < inst->magnLen; i++) { + // FLOAT code + // invLrt = exp(inst->logLrtTimeAvg[i]); + // invLrt = inst->priorSpeechProb * invLrt; + // nonSpeechProbFinal[i] = (1.0 - inst->priorSpeechProb) / + // (1.0 - inst->priorSpeechProb + invLrt); + // invLrt = (1.0 - inst->priorNonSpeechProb) * invLrt; + // nonSpeechProbFinal[i] = inst->priorNonSpeechProb / + // (inst->priorNonSpeechProb + invLrt); + if (inst->logLrtTimeAvgW32[i] < 65300) { + tmp32no1 = (inst->logLrtTimeAvgW32[i] * 23637) >> 14; // Q12 + intPart = (int16_t)(tmp32no1 >> 12); + if (intPart < -8) { + intPart = -8; + } + frac = (int16_t)(tmp32no1 & 0x00000fff); // Q12 + + // Quadratic approximation of 2^frac + tmp32no2 = (frac * frac * 44) >> 19; // Q12. + tmp32no2 += (frac * 84) >> 7; // Q12 + invLrtFX = (1 << (8 + intPart)) + + WEBRTC_SPL_SHIFT_W32(tmp32no2, intPart - 4); // Q8 + + normTmp = WebRtcSpl_NormW32(invLrtFX); + normTmp2 = WebRtcSpl_NormW16((16384 - inst->priorNonSpeechProb)); + if (normTmp + normTmp2 >= 7) { + if (normTmp + normTmp2 < 15) { + invLrtFX >>= 15 - normTmp2 - normTmp; + // Q(normTmp+normTmp2-7) + tmp32no1 = invLrtFX * (16384 - inst->priorNonSpeechProb); + // Q(normTmp+normTmp2+7) + invLrtFX = WEBRTC_SPL_SHIFT_W32(tmp32no1, 7 - normTmp - normTmp2); + // Q14 + } else { + tmp32no1 = invLrtFX * (16384 - inst->priorNonSpeechProb); + // Q22 + invLrtFX = tmp32no1 >> 8; // Q14. + } + + tmp32no1 = (int32_t)inst->priorNonSpeechProb << 8; // Q22 + + nonSpeechProbFinal[i] = tmp32no1 / + (inst->priorNonSpeechProb + invLrtFX); // Q8 + } + } + } + } +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_mips.c b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_mips.c new file mode 100644 index 0000000000..d58a9b2347 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_mips.c @@ -0,0 +1,1002 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "rtc_base/checks.h" +#include "modules/audio_processing/ns/noise_suppression_x.h" +#include "modules/audio_processing/ns/nsx_core.h" + +static const int16_t kIndicatorTable[17] = { + 0, 2017, 3809, 5227, 6258, 6963, 7424, 7718, + 7901, 8014, 8084, 8126, 8152, 8168, 8177, 8183, 8187 +}; + +// Compute speech/noise probability +// speech/noise probability is returned in: probSpeechFinal +//snrLocPrior is the prior SNR for each frequency (in Q11) +//snrLocPost is the post SNR for each frequency (in Q11) +void WebRtcNsx_SpeechNoiseProb(NoiseSuppressionFixedC* inst, + uint16_t* nonSpeechProbFinal, + uint32_t* priorLocSnr, + uint32_t* postLocSnr) { + uint32_t tmpU32no1, tmpU32no2, tmpU32no3; + int32_t indPriorFX, tmp32no1; + int32_t logLrtTimeAvgKsumFX; + int16_t indPriorFX16; + int16_t tmp16, tmp16no1, tmp16no2, tmpIndFX, tableIndex, frac; + size_t i; + int normTmp, nShifts; + + int32_t r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + int32_t const_max = 0x7fffffff; + int32_t const_neg43 = -43; + int32_t const_5412 = 5412; + int32_t const_11rsh12 = (11 << 12); + int32_t const_178 = 178; + + + // compute feature based on average LR factor + // this is the average over all frequencies of the smooth log LRT + logLrtTimeAvgKsumFX = 0; + for (i = 0; i < inst->magnLen; i++) { + r0 = postLocSnr[i]; // Q11 + r1 = priorLocSnr[i]; + r2 = inst->logLrtTimeAvgW32[i]; + + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "clz %[r3], %[r0] \n\t" + "clz %[r5], %[r1] \n\t" + "slti %[r4], %[r3], 32 \n\t" + "slti %[r6], %[r5], 32 \n\t" + "movz %[r3], $0, %[r4] \n\t" + "movz %[r5], $0, %[r6] \n\t" + "slti %[r4], %[r3], 11 \n\t" + "addiu %[r6], %[r3], -11 \n\t" + "neg %[r7], %[r6] \n\t" + "sllv %[r6], %[r1], %[r6] \n\t" + "srav %[r7], %[r1], %[r7] \n\t" + "movn %[r6], %[r7], %[r4] \n\t" + "sllv %[r1], %[r1], %[r5] \n\t" + "and %[r1], %[r1], %[const_max] \n\t" + "sra %[r1], %[r1], 19 \n\t" + "mul %[r7], %[r1], %[r1] \n\t" + "sllv %[r3], %[r0], %[r3] \n\t" + "divu %[r8], %[r3], %[r6] \n\t" + "slti %[r6], %[r6], 1 \n\t" + "mul %[r7], %[r7], %[const_neg43] \n\t" + "sra %[r7], %[r7], 19 \n\t" + "movz %[r3], %[r8], %[r6] \n\t" + "subu %[r0], %[r0], %[r3] \n\t" + "movn %[r0], $0, %[r6] \n\t" + "mul %[r1], %[r1], %[const_5412] \n\t" + "sra %[r1], %[r1], 12 \n\t" + "addu %[r7], %[r7], %[r1] \n\t" + "addiu %[r1], %[r7], 37 \n\t" + "addiu %[r5], %[r5], -31 \n\t" + "neg %[r5], %[r5] \n\t" + "sll %[r5], %[r5], 12 \n\t" + "addu %[r5], %[r5], %[r1] \n\t" + "subu %[r7], %[r5], %[const_11rsh12] \n\t" + "mul %[r7], %[r7], %[const_178] \n\t" + "sra %[r7], %[r7], 8 \n\t" + "addu %[r7], %[r7], %[r2] \n\t" + "sra %[r7], %[r7], 1 \n\t" + "subu %[r2], %[r2], %[r7] \n\t" + "addu %[r2], %[r2], %[r0] \n\t" + ".set pop \n\t" + : [r0] "+r" (r0), [r1] "+r" (r1), [r2] "+r" (r2), + [r3] "=&r" (r3), [r4] "=&r" (r4), [r5] "=&r" (r5), + [r6] "=&r" (r6), [r7] "=&r" (r7), [r8] "=&r" (r8) + : [const_max] "r" (const_max), [const_neg43] "r" (const_neg43), + [const_5412] "r" (const_5412), [const_11rsh12] "r" (const_11rsh12), + [const_178] "r" (const_178) + : "hi", "lo" + ); + inst->logLrtTimeAvgW32[i] = r2; + logLrtTimeAvgKsumFX += r2; + } + + inst->featureLogLrt = (logLrtTimeAvgKsumFX * BIN_SIZE_LRT) >> + (inst->stages + 11); + + // done with computation of LR factor + + // + // compute the indicator functions + // + + // average LRT feature + // FLOAT code + // indicator0 = 0.5 * (tanh(widthPrior * + // (logLrtTimeAvgKsum - threshPrior0)) + 1.0); + tmpIndFX = 16384; // Q14(1.0) + tmp32no1 = logLrtTimeAvgKsumFX - inst->thresholdLogLrt; // Q12 + nShifts = 7 - inst->stages; // WIDTH_PR_MAP_SHIFT - inst->stages + 5; + //use larger width in tanh map for pause regions + if (tmp32no1 < 0) { + tmpIndFX = 0; + tmp32no1 = -tmp32no1; + //widthPrior = widthPrior * 2.0; + nShifts++; + } + tmp32no1 = WEBRTC_SPL_SHIFT_W32(tmp32no1, nShifts); // Q14 + // compute indicator function: sigmoid map + if (tmp32no1 < (16 << 14) && tmp32no1 >= 0) { + tableIndex = (int16_t)(tmp32no1 >> 14); + tmp16no2 = kIndicatorTable[tableIndex]; + tmp16no1 = kIndicatorTable[tableIndex + 1] - kIndicatorTable[tableIndex]; + frac = (int16_t)(tmp32no1 & 0x00003fff); // Q14 + tmp16no2 += (int16_t)((tmp16no1 * frac) >> 14); + if (tmpIndFX == 0) { + tmpIndFX = 8192 - tmp16no2; // Q14 + } else { + tmpIndFX = 8192 + tmp16no2; // Q14 + } + } + indPriorFX = inst->weightLogLrt * tmpIndFX; // 6*Q14 + + //spectral flatness feature + if (inst->weightSpecFlat) { + tmpU32no1 = WEBRTC_SPL_UMUL(inst->featureSpecFlat, 400); // Q10 + tmpIndFX = 16384; // Q14(1.0) + //use larger width in tanh map for pause regions + tmpU32no2 = inst->thresholdSpecFlat - tmpU32no1; //Q10 + nShifts = 4; + if (inst->thresholdSpecFlat < tmpU32no1) { + tmpIndFX = 0; + tmpU32no2 = tmpU32no1 - inst->thresholdSpecFlat; + //widthPrior = widthPrior * 2.0; + nShifts++; + } + tmpU32no1 = WebRtcSpl_DivU32U16(tmpU32no2 << nShifts, 25); //Q14 + // compute indicator function: sigmoid map + // FLOAT code + // indicator1 = 0.5 * (tanh(sgnMap * widthPrior * + // (threshPrior1 - tmpFloat1)) + 1.0); + if (tmpU32no1 < (16 << 14)) { + tableIndex = (int16_t)(tmpU32no1 >> 14); + tmp16no2 = kIndicatorTable[tableIndex]; + tmp16no1 = kIndicatorTable[tableIndex + 1] - kIndicatorTable[tableIndex]; + frac = (int16_t)(tmpU32no1 & 0x00003fff); // Q14 + tmp16no2 += (int16_t)((tmp16no1 * frac) >> 14); + if (tmpIndFX) { + tmpIndFX = 8192 + tmp16no2; // Q14 + } else { + tmpIndFX = 8192 - tmp16no2; // Q14 + } + } + indPriorFX += inst->weightSpecFlat * tmpIndFX; // 6*Q14 + } + + //for template spectral-difference + if (inst->weightSpecDiff) { + tmpU32no1 = 0; + if (inst->featureSpecDiff) { + normTmp = WEBRTC_SPL_MIN(20 - inst->stages, + WebRtcSpl_NormU32(inst->featureSpecDiff)); + RTC_DCHECK_GE(normTmp, 0); + tmpU32no1 = inst->featureSpecDiff << normTmp; // Q(normTmp-2*stages) + tmpU32no2 = inst->timeAvgMagnEnergy >> (20 - inst->stages - normTmp); + if (tmpU32no2 > 0) { + // Q(20 - inst->stages) + tmpU32no1 /= tmpU32no2; + } else { + tmpU32no1 = (uint32_t)(0x7fffffff); + } + } + tmpU32no3 = (inst->thresholdSpecDiff << 17) / 25; + tmpU32no2 = tmpU32no1 - tmpU32no3; + nShifts = 1; + tmpIndFX = 16384; // Q14(1.0) + //use larger width in tanh map for pause regions + if (tmpU32no2 & 0x80000000) { + tmpIndFX = 0; + tmpU32no2 = tmpU32no3 - tmpU32no1; + //widthPrior = widthPrior * 2.0; + nShifts--; + } + tmpU32no1 = tmpU32no2 >> nShifts; + // compute indicator function: sigmoid map + /* FLOAT code + indicator2 = 0.5 * (tanh(widthPrior * (tmpFloat1 - threshPrior2)) + 1.0); + */ + if (tmpU32no1 < (16 << 14)) { + tableIndex = (int16_t)(tmpU32no1 >> 14); + tmp16no2 = kIndicatorTable[tableIndex]; + tmp16no1 = kIndicatorTable[tableIndex + 1] - kIndicatorTable[tableIndex]; + frac = (int16_t)(tmpU32no1 & 0x00003fff); // Q14 + tmp16no2 += (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + tmp16no1, frac, 14); + if (tmpIndFX) { + tmpIndFX = 8192 + tmp16no2; + } else { + tmpIndFX = 8192 - tmp16no2; + } + } + indPriorFX += inst->weightSpecDiff * tmpIndFX; // 6*Q14 + } + + //combine the indicator function with the feature weights + // FLOAT code + // indPrior = 1 - (weightIndPrior0 * indicator0 + weightIndPrior1 * + // indicator1 + weightIndPrior2 * indicator2); + indPriorFX16 = WebRtcSpl_DivW32W16ResW16(98307 - indPriorFX, 6); // Q14 + // done with computing indicator function + + //compute the prior probability + // FLOAT code + // inst->priorNonSpeechProb += PRIOR_UPDATE * + // (indPriorNonSpeech - inst->priorNonSpeechProb); + tmp16 = indPriorFX16 - inst->priorNonSpeechProb; // Q14 + inst->priorNonSpeechProb += (int16_t)((PRIOR_UPDATE_Q14 * tmp16) >> 14); + + //final speech probability: combine prior model with LR factor: + + memset(nonSpeechProbFinal, 0, sizeof(uint16_t) * inst->magnLen); + + if (inst->priorNonSpeechProb > 0) { + r0 = inst->priorNonSpeechProb; + r1 = 16384 - r0; + int32_t const_23637 = 23637; + int32_t const_44 = 44; + int32_t const_84 = 84; + int32_t const_1 = 1; + int32_t const_neg8 = -8; + for (i = 0; i < inst->magnLen; i++) { + r2 = inst->logLrtTimeAvgW32[i]; + if (r2 < 65300) { + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "mul %[r2], %[r2], %[const_23637] \n\t" + "sll %[r6], %[r1], 16 \n\t" + "clz %[r7], %[r6] \n\t" + "clo %[r8], %[r6] \n\t" + "slt %[r9], %[r6], $0 \n\t" + "movn %[r7], %[r8], %[r9] \n\t" + "sra %[r2], %[r2], 14 \n\t" + "andi %[r3], %[r2], 0xfff \n\t" + "mul %[r4], %[r3], %[r3] \n\t" + "mul %[r3], %[r3], %[const_84] \n\t" + "sra %[r2], %[r2], 12 \n\t" + "slt %[r5], %[r2], %[const_neg8] \n\t" + "movn %[r2], %[const_neg8], %[r5] \n\t" + "mul %[r4], %[r4], %[const_44] \n\t" + "sra %[r3], %[r3], 7 \n\t" + "addiu %[r7], %[r7], -1 \n\t" + "slti %[r9], %[r7], 31 \n\t" + "movz %[r7], $0, %[r9] \n\t" + "sra %[r4], %[r4], 19 \n\t" + "addu %[r4], %[r4], %[r3] \n\t" + "addiu %[r3], %[r2], 8 \n\t" + "addiu %[r2], %[r2], -4 \n\t" + "neg %[r5], %[r2] \n\t" + "sllv %[r6], %[r4], %[r2] \n\t" + "srav %[r5], %[r4], %[r5] \n\t" + "slt %[r2], %[r2], $0 \n\t" + "movn %[r6], %[r5], %[r2] \n\t" + "sllv %[r3], %[const_1], %[r3] \n\t" + "addu %[r2], %[r3], %[r6] \n\t" + "clz %[r4], %[r2] \n\t" + "clo %[r5], %[r2] \n\t" + "slt %[r8], %[r2], $0 \n\t" + "movn %[r4], %[r5], %[r8] \n\t" + "addiu %[r4], %[r4], -1 \n\t" + "slt %[r5], $0, %[r2] \n\t" + "or %[r5], %[r5], %[r7] \n\t" + "movz %[r4], $0, %[r5] \n\t" + "addiu %[r6], %[r7], -7 \n\t" + "addu %[r6], %[r6], %[r4] \n\t" + "bltz %[r6], 1f \n\t" + " nop \n\t" + "addiu %[r4], %[r6], -8 \n\t" + "neg %[r3], %[r4] \n\t" + "srav %[r5], %[r2], %[r3] \n\t" + "mul %[r5], %[r5], %[r1] \n\t" + "mul %[r2], %[r2], %[r1] \n\t" + "slt %[r4], %[r4], $0 \n\t" + "srav %[r5], %[r5], %[r6] \n\t" + "sra %[r2], %[r2], 8 \n\t" + "movn %[r2], %[r5], %[r4] \n\t" + "sll %[r3], %[r0], 8 \n\t" + "addu %[r2], %[r0], %[r2] \n\t" + "divu %[r3], %[r3], %[r2] \n\t" + "1: \n\t" + ".set pop \n\t" + : [r2] "+r" (r2), [r3] "=&r" (r3), [r4] "=&r" (r4), + [r5] "=&r" (r5), [r6] "=&r" (r6), [r7] "=&r" (r7), + [r8] "=&r" (r8), [r9] "=&r" (r9) + : [r0] "r" (r0), [r1] "r" (r1), [const_23637] "r" (const_23637), + [const_neg8] "r" (const_neg8), [const_84] "r" (const_84), + [const_1] "r" (const_1), [const_44] "r" (const_44) + : "hi", "lo" + ); + nonSpeechProbFinal[i] = r3; + } + } + } +} + +// Update analysis buffer for lower band, and window data before FFT. +void WebRtcNsx_AnalysisUpdate_mips(NoiseSuppressionFixedC* inst, + int16_t* out, + int16_t* new_speech) { + int iters, after; + int anaLen = (int)inst->anaLen; + int *window = (int*)inst->window; + int *anaBuf = (int*)inst->analysisBuffer; + int *outBuf = (int*)out; + int r0, r1, r2, r3, r4, r5, r6, r7; +#if defined(MIPS_DSP_R1_LE) + int r8; +#endif + + // For lower band update analysis buffer. + memcpy(inst->analysisBuffer, inst->analysisBuffer + inst->blockLen10ms, + (inst->anaLen - inst->blockLen10ms) * sizeof(*inst->analysisBuffer)); + memcpy(inst->analysisBuffer + inst->anaLen - inst->blockLen10ms, new_speech, + inst->blockLen10ms * sizeof(*inst->analysisBuffer)); + + // Window data before FFT. +#if defined(MIPS_DSP_R1_LE) + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "sra %[iters], %[anaLen], 3 \n\t" + "1: \n\t" + "blez %[iters], 2f \n\t" + " nop \n\t" + "lw %[r0], 0(%[window]) \n\t" + "lw %[r1], 0(%[anaBuf]) \n\t" + "lw %[r2], 4(%[window]) \n\t" + "lw %[r3], 4(%[anaBuf]) \n\t" + "lw %[r4], 8(%[window]) \n\t" + "lw %[r5], 8(%[anaBuf]) \n\t" + "lw %[r6], 12(%[window]) \n\t" + "lw %[r7], 12(%[anaBuf]) \n\t" + "muleq_s.w.phl %[r8], %[r0], %[r1] \n\t" + "muleq_s.w.phr %[r0], %[r0], %[r1] \n\t" + "muleq_s.w.phl %[r1], %[r2], %[r3] \n\t" + "muleq_s.w.phr %[r2], %[r2], %[r3] \n\t" + "muleq_s.w.phl %[r3], %[r4], %[r5] \n\t" + "muleq_s.w.phr %[r4], %[r4], %[r5] \n\t" + "muleq_s.w.phl %[r5], %[r6], %[r7] \n\t" + "muleq_s.w.phr %[r6], %[r6], %[r7] \n\t" +#if defined(MIPS_DSP_R2_LE) + "precr_sra_r.ph.w %[r8], %[r0], 15 \n\t" + "precr_sra_r.ph.w %[r1], %[r2], 15 \n\t" + "precr_sra_r.ph.w %[r3], %[r4], 15 \n\t" + "precr_sra_r.ph.w %[r5], %[r6], 15 \n\t" + "sw %[r8], 0(%[outBuf]) \n\t" + "sw %[r1], 4(%[outBuf]) \n\t" + "sw %[r3], 8(%[outBuf]) \n\t" + "sw %[r5], 12(%[outBuf]) \n\t" +#else + "shra_r.w %[r8], %[r8], 15 \n\t" + "shra_r.w %[r0], %[r0], 15 \n\t" + "shra_r.w %[r1], %[r1], 15 \n\t" + "shra_r.w %[r2], %[r2], 15 \n\t" + "shra_r.w %[r3], %[r3], 15 \n\t" + "shra_r.w %[r4], %[r4], 15 \n\t" + "shra_r.w %[r5], %[r5], 15 \n\t" + "shra_r.w %[r6], %[r6], 15 \n\t" + "sll %[r0], %[r0], 16 \n\t" + "sll %[r2], %[r2], 16 \n\t" + "sll %[r4], %[r4], 16 \n\t" + "sll %[r6], %[r6], 16 \n\t" + "packrl.ph %[r0], %[r8], %[r0] \n\t" + "packrl.ph %[r2], %[r1], %[r2] \n\t" + "packrl.ph %[r4], %[r3], %[r4] \n\t" + "packrl.ph %[r6], %[r5], %[r6] \n\t" + "sw %[r0], 0(%[outBuf]) \n\t" + "sw %[r2], 4(%[outBuf]) \n\t" + "sw %[r4], 8(%[outBuf]) \n\t" + "sw %[r6], 12(%[outBuf]) \n\t" +#endif + "addiu %[window], %[window], 16 \n\t" + "addiu %[anaBuf], %[anaBuf], 16 \n\t" + "addiu %[outBuf], %[outBuf], 16 \n\t" + "b 1b \n\t" + " addiu %[iters], %[iters], -1 \n\t" + "2: \n\t" + "andi %[after], %[anaLen], 7 \n\t" + "3: \n\t" + "blez %[after], 4f \n\t" + " nop \n\t" + "lh %[r0], 0(%[window]) \n\t" + "lh %[r1], 0(%[anaBuf]) \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "addiu %[window], %[window], 2 \n\t" + "addiu %[anaBuf], %[anaBuf], 2 \n\t" + "addiu %[outBuf], %[outBuf], 2 \n\t" + "shra_r.w %[r0], %[r0], 14 \n\t" + "sh %[r0], -2(%[outBuf]) \n\t" + "b 3b \n\t" + " addiu %[after], %[after], -1 \n\t" + "4: \n\t" + ".set pop \n\t" + : [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), + [r3] "=&r" (r3), [r4] "=&r" (r4), [r5] "=&r" (r5), + [r6] "=&r" (r6), [r7] "=&r" (r7), [r8] "=&r" (r8), + [iters] "=&r" (iters), [after] "=&r" (after), + [window] "+r" (window),[anaBuf] "+r" (anaBuf), + [outBuf] "+r" (outBuf) + : [anaLen] "r" (anaLen) + : "memory", "hi", "lo" + ); +#else + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "sra %[iters], %[anaLen], 2 \n\t" + "1: \n\t" + "blez %[iters], 2f \n\t" + " nop \n\t" + "lh %[r0], 0(%[window]) \n\t" + "lh %[r1], 0(%[anaBuf]) \n\t" + "lh %[r2], 2(%[window]) \n\t" + "lh %[r3], 2(%[anaBuf]) \n\t" + "lh %[r4], 4(%[window]) \n\t" + "lh %[r5], 4(%[anaBuf]) \n\t" + "lh %[r6], 6(%[window]) \n\t" + "lh %[r7], 6(%[anaBuf]) \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "mul %[r2], %[r2], %[r3] \n\t" + "mul %[r4], %[r4], %[r5] \n\t" + "mul %[r6], %[r6], %[r7] \n\t" + "addiu %[window], %[window], 8 \n\t" + "addiu %[anaBuf], %[anaBuf], 8 \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "addiu %[r2], %[r2], 0x2000 \n\t" + "addiu %[r4], %[r4], 0x2000 \n\t" + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r0], %[r0], 14 \n\t" + "sra %[r2], %[r2], 14 \n\t" + "sra %[r4], %[r4], 14 \n\t" + "sra %[r6], %[r6], 14 \n\t" + "sh %[r0], 0(%[outBuf]) \n\t" + "sh %[r2], 2(%[outBuf]) \n\t" + "sh %[r4], 4(%[outBuf]) \n\t" + "sh %[r6], 6(%[outBuf]) \n\t" + "addiu %[outBuf], %[outBuf], 8 \n\t" + "b 1b \n\t" + " addiu %[iters], %[iters], -1 \n\t" + "2: \n\t" + "andi %[after], %[anaLen], 3 \n\t" + "3: \n\t" + "blez %[after], 4f \n\t" + " nop \n\t" + "lh %[r0], 0(%[window]) \n\t" + "lh %[r1], 0(%[anaBuf]) \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "addiu %[window], %[window], 2 \n\t" + "addiu %[anaBuf], %[anaBuf], 2 \n\t" + "addiu %[outBuf], %[outBuf], 2 \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "sra %[r0], %[r0], 14 \n\t" + "sh %[r0], -2(%[outBuf]) \n\t" + "b 3b \n\t" + " addiu %[after], %[after], -1 \n\t" + "4: \n\t" + ".set pop \n\t" + : [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), + [r3] "=&r" (r3), [r4] "=&r" (r4), [r5] "=&r" (r5), + [r6] "=&r" (r6), [r7] "=&r" (r7), [iters] "=&r" (iters), + [after] "=&r" (after), [window] "+r" (window), + [anaBuf] "+r" (anaBuf), [outBuf] "+r" (outBuf) + : [anaLen] "r" (anaLen) + : "memory", "hi", "lo" + ); +#endif +} + +// For the noise supression process, synthesis, read out fully processed +// segment, and update synthesis buffer. +void WebRtcNsx_SynthesisUpdate_mips(NoiseSuppressionFixedC* inst, + int16_t* out_frame, + int16_t gain_factor) { + int iters = (int)inst->blockLen10ms >> 2; + int after = inst->blockLen10ms & 3; + int r0, r1, r2, r3, r4, r5, r6, r7; + int16_t *window = (int16_t*)inst->window; + int16_t *real = inst->real; + int16_t *synthBuf = inst->synthesisBuffer; + int16_t *out = out_frame; + int sat_pos = 0x7fff; + int sat_neg = 0xffff8000; + int block10 = (int)inst->blockLen10ms; + int anaLen = (int)inst->anaLen; + + __asm __volatile( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "blez %[iters], 2f \n\t" + " nop \n\t" + "lh %[r0], 0(%[window]) \n\t" + "lh %[r1], 0(%[real]) \n\t" + "lh %[r2], 2(%[window]) \n\t" + "lh %[r3], 2(%[real]) \n\t" + "lh %[r4], 4(%[window]) \n\t" + "lh %[r5], 4(%[real]) \n\t" + "lh %[r6], 6(%[window]) \n\t" + "lh %[r7], 6(%[real]) \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "mul %[r2], %[r2], %[r3] \n\t" + "mul %[r4], %[r4], %[r5] \n\t" + "mul %[r6], %[r6], %[r7] \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "addiu %[r2], %[r2], 0x2000 \n\t" + "addiu %[r4], %[r4], 0x2000 \n\t" + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r0], %[r0], 14 \n\t" + "sra %[r2], %[r2], 14 \n\t" + "sra %[r4], %[r4], 14 \n\t" + "sra %[r6], %[r6], 14 \n\t" + "mul %[r0], %[r0], %[gain_factor] \n\t" + "mul %[r2], %[r2], %[gain_factor] \n\t" + "mul %[r4], %[r4], %[gain_factor] \n\t" + "mul %[r6], %[r6], %[gain_factor] \n\t" + "addiu %[r0], %[r0], 0x1000 \n\t" + "addiu %[r2], %[r2], 0x1000 \n\t" + "addiu %[r4], %[r4], 0x1000 \n\t" + "addiu %[r6], %[r6], 0x1000 \n\t" + "sra %[r0], %[r0], 13 \n\t" + "sra %[r2], %[r2], 13 \n\t" + "sra %[r4], %[r4], 13 \n\t" + "sra %[r6], %[r6], 13 \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "slt %[r3], %[r2], %[sat_pos] \n\t" + "slt %[r5], %[r4], %[sat_pos] \n\t" + "slt %[r7], %[r6], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "movz %[r2], %[sat_pos], %[r3] \n\t" + "movz %[r4], %[sat_pos], %[r5] \n\t" + "movz %[r6], %[sat_pos], %[r7] \n\t" + "lh %[r1], 0(%[synthBuf]) \n\t" + "lh %[r3], 2(%[synthBuf]) \n\t" + "lh %[r5], 4(%[synthBuf]) \n\t" + "lh %[r7], 6(%[synthBuf]) \n\t" + "addu %[r0], %[r0], %[r1] \n\t" + "addu %[r2], %[r2], %[r3] \n\t" + "addu %[r4], %[r4], %[r5] \n\t" + "addu %[r6], %[r6], %[r7] \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "slt %[r3], %[r2], %[sat_pos] \n\t" + "slt %[r5], %[r4], %[sat_pos] \n\t" + "slt %[r7], %[r6], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "movz %[r2], %[sat_pos], %[r3] \n\t" + "movz %[r4], %[sat_pos], %[r5] \n\t" + "movz %[r6], %[sat_pos], %[r7] \n\t" + "slt %[r1], %[r0], %[sat_neg] \n\t" + "slt %[r3], %[r2], %[sat_neg] \n\t" + "slt %[r5], %[r4], %[sat_neg] \n\t" + "slt %[r7], %[r6], %[sat_neg] \n\t" + "movn %[r0], %[sat_neg], %[r1] \n\t" + "movn %[r2], %[sat_neg], %[r3] \n\t" + "movn %[r4], %[sat_neg], %[r5] \n\t" + "movn %[r6], %[sat_neg], %[r7] \n\t" + "sh %[r0], 0(%[synthBuf]) \n\t" + "sh %[r2], 2(%[synthBuf]) \n\t" + "sh %[r4], 4(%[synthBuf]) \n\t" + "sh %[r6], 6(%[synthBuf]) \n\t" + "sh %[r0], 0(%[out]) \n\t" + "sh %[r2], 2(%[out]) \n\t" + "sh %[r4], 4(%[out]) \n\t" + "sh %[r6], 6(%[out]) \n\t" + "addiu %[window], %[window], 8 \n\t" + "addiu %[real], %[real], 8 \n\t" + "addiu %[synthBuf],%[synthBuf], 8 \n\t" + "addiu %[out], %[out], 8 \n\t" + "b 1b \n\t" + " addiu %[iters], %[iters], -1 \n\t" + "2: \n\t" + "blez %[after], 3f \n\t" + " subu %[block10], %[anaLen], %[block10] \n\t" + "lh %[r0], 0(%[window]) \n\t" + "lh %[r1], 0(%[real]) \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "addiu %[window], %[window], 2 \n\t" + "addiu %[real], %[real], 2 \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "sra %[r0], %[r0], 14 \n\t" + "mul %[r0], %[r0], %[gain_factor] \n\t" + "addiu %[r0], %[r0], 0x1000 \n\t" + "sra %[r0], %[r0], 13 \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "lh %[r1], 0(%[synthBuf]) \n\t" + "addu %[r0], %[r0], %[r1] \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "slt %[r1], %[r0], %[sat_neg] \n\t" + "movn %[r0], %[sat_neg], %[r1] \n\t" + "sh %[r0], 0(%[synthBuf]) \n\t" + "sh %[r0], 0(%[out]) \n\t" + "addiu %[synthBuf],%[synthBuf], 2 \n\t" + "addiu %[out], %[out], 2 \n\t" + "b 2b \n\t" + " addiu %[after], %[after], -1 \n\t" + "3: \n\t" + "sra %[iters], %[block10], 2 \n\t" + "4: \n\t" + "blez %[iters], 5f \n\t" + " andi %[after], %[block10], 3 \n\t" + "lh %[r0], 0(%[window]) \n\t" + "lh %[r1], 0(%[real]) \n\t" + "lh %[r2], 2(%[window]) \n\t" + "lh %[r3], 2(%[real]) \n\t" + "lh %[r4], 4(%[window]) \n\t" + "lh %[r5], 4(%[real]) \n\t" + "lh %[r6], 6(%[window]) \n\t" + "lh %[r7], 6(%[real]) \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "mul %[r2], %[r2], %[r3] \n\t" + "mul %[r4], %[r4], %[r5] \n\t" + "mul %[r6], %[r6], %[r7] \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "addiu %[r2], %[r2], 0x2000 \n\t" + "addiu %[r4], %[r4], 0x2000 \n\t" + "addiu %[r6], %[r6], 0x2000 \n\t" + "sra %[r0], %[r0], 14 \n\t" + "sra %[r2], %[r2], 14 \n\t" + "sra %[r4], %[r4], 14 \n\t" + "sra %[r6], %[r6], 14 \n\t" + "mul %[r0], %[r0], %[gain_factor] \n\t" + "mul %[r2], %[r2], %[gain_factor] \n\t" + "mul %[r4], %[r4], %[gain_factor] \n\t" + "mul %[r6], %[r6], %[gain_factor] \n\t" + "addiu %[r0], %[r0], 0x1000 \n\t" + "addiu %[r2], %[r2], 0x1000 \n\t" + "addiu %[r4], %[r4], 0x1000 \n\t" + "addiu %[r6], %[r6], 0x1000 \n\t" + "sra %[r0], %[r0], 13 \n\t" + "sra %[r2], %[r2], 13 \n\t" + "sra %[r4], %[r4], 13 \n\t" + "sra %[r6], %[r6], 13 \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "slt %[r3], %[r2], %[sat_pos] \n\t" + "slt %[r5], %[r4], %[sat_pos] \n\t" + "slt %[r7], %[r6], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "movz %[r2], %[sat_pos], %[r3] \n\t" + "movz %[r4], %[sat_pos], %[r5] \n\t" + "movz %[r6], %[sat_pos], %[r7] \n\t" + "lh %[r1], 0(%[synthBuf]) \n\t" + "lh %[r3], 2(%[synthBuf]) \n\t" + "lh %[r5], 4(%[synthBuf]) \n\t" + "lh %[r7], 6(%[synthBuf]) \n\t" + "addu %[r0], %[r0], %[r1] \n\t" + "addu %[r2], %[r2], %[r3] \n\t" + "addu %[r4], %[r4], %[r5] \n\t" + "addu %[r6], %[r6], %[r7] \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "slt %[r3], %[r2], %[sat_pos] \n\t" + "slt %[r5], %[r4], %[sat_pos] \n\t" + "slt %[r7], %[r6], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "movz %[r2], %[sat_pos], %[r3] \n\t" + "movz %[r4], %[sat_pos], %[r5] \n\t" + "movz %[r6], %[sat_pos], %[r7] \n\t" + "slt %[r1], %[r0], %[sat_neg] \n\t" + "slt %[r3], %[r2], %[sat_neg] \n\t" + "slt %[r5], %[r4], %[sat_neg] \n\t" + "slt %[r7], %[r6], %[sat_neg] \n\t" + "movn %[r0], %[sat_neg], %[r1] \n\t" + "movn %[r2], %[sat_neg], %[r3] \n\t" + "movn %[r4], %[sat_neg], %[r5] \n\t" + "movn %[r6], %[sat_neg], %[r7] \n\t" + "sh %[r0], 0(%[synthBuf]) \n\t" + "sh %[r2], 2(%[synthBuf]) \n\t" + "sh %[r4], 4(%[synthBuf]) \n\t" + "sh %[r6], 6(%[synthBuf]) \n\t" + "addiu %[window], %[window], 8 \n\t" + "addiu %[real], %[real], 8 \n\t" + "addiu %[synthBuf],%[synthBuf], 8 \n\t" + "b 4b \n\t" + " addiu %[iters], %[iters], -1 \n\t" + "5: \n\t" + "blez %[after], 6f \n\t" + " nop \n\t" + "lh %[r0], 0(%[window]) \n\t" + "lh %[r1], 0(%[real]) \n\t" + "mul %[r0], %[r0], %[r1] \n\t" + "addiu %[window], %[window], 2 \n\t" + "addiu %[real], %[real], 2 \n\t" + "addiu %[r0], %[r0], 0x2000 \n\t" + "sra %[r0], %[r0], 14 \n\t" + "mul %[r0], %[r0], %[gain_factor] \n\t" + "addiu %[r0], %[r0], 0x1000 \n\t" + "sra %[r0], %[r0], 13 \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "lh %[r1], 0(%[synthBuf]) \n\t" + "addu %[r0], %[r0], %[r1] \n\t" + "slt %[r1], %[r0], %[sat_pos] \n\t" + "movz %[r0], %[sat_pos], %[r1] \n\t" + "slt %[r1], %[r0], %[sat_neg] \n\t" + "movn %[r0], %[sat_neg], %[r1] \n\t" + "sh %[r0], 0(%[synthBuf]) \n\t" + "addiu %[synthBuf],%[synthBuf], 2 \n\t" + "b 2b \n\t" + " addiu %[after], %[after], -1 \n\t" + "6: \n\t" + ".set pop \n\t" + : [r0] "=&r" (r0), [r1] "=&r" (r1), [r2] "=&r" (r2), + [r3] "=&r" (r3), [r4] "=&r" (r4), [r5] "=&r" (r5), + [r6] "=&r" (r6), [r7] "=&r" (r7), [iters] "+r" (iters), + [after] "+r" (after), [block10] "+r" (block10), + [window] "+r" (window), [real] "+r" (real), + [synthBuf] "+r" (synthBuf), [out] "+r" (out) + : [gain_factor] "r" (gain_factor), [sat_pos] "r" (sat_pos), + [sat_neg] "r" (sat_neg), [anaLen] "r" (anaLen) + : "memory", "hi", "lo" + ); + + // update synthesis buffer + memcpy(inst->synthesisBuffer, inst->synthesisBuffer + inst->blockLen10ms, + (inst->anaLen - inst->blockLen10ms) * sizeof(*inst->synthesisBuffer)); + WebRtcSpl_ZerosArrayW16(inst->synthesisBuffer + + inst->anaLen - inst->blockLen10ms, inst->blockLen10ms); +} + +// Filter the data in the frequency domain, and create spectrum. +void WebRtcNsx_PrepareSpectrum_mips(NoiseSuppressionFixedC* inst, + int16_t* freq_buf) { + uint16_t *noiseSupFilter = inst->noiseSupFilter; + int16_t *real = inst->real; + int16_t *imag = inst->imag; + int32_t loop_count = 2; + int16_t tmp_1, tmp_2, tmp_3, tmp_4, tmp_5, tmp_6; + int16_t tmp16 = (int16_t)(inst->anaLen << 1) - 4; + int16_t* freq_buf_f = freq_buf; + int16_t* freq_buf_s = &freq_buf[tmp16]; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + //first sample + "lh %[tmp_1], 0(%[noiseSupFilter]) \n\t" + "lh %[tmp_2], 0(%[real]) \n\t" + "lh %[tmp_3], 0(%[imag]) \n\t" + "mul %[tmp_2], %[tmp_2], %[tmp_1] \n\t" + "mul %[tmp_3], %[tmp_3], %[tmp_1] \n\t" + "sra %[tmp_2], %[tmp_2], 14 \n\t" + "sra %[tmp_3], %[tmp_3], 14 \n\t" + "sh %[tmp_2], 0(%[real]) \n\t" + "sh %[tmp_3], 0(%[imag]) \n\t" + "negu %[tmp_3], %[tmp_3] \n\t" + "sh %[tmp_2], 0(%[freq_buf_f]) \n\t" + "sh %[tmp_3], 2(%[freq_buf_f]) \n\t" + "addiu %[real], %[real], 2 \n\t" + "addiu %[imag], %[imag], 2 \n\t" + "addiu %[noiseSupFilter], %[noiseSupFilter], 2 \n\t" + "addiu %[freq_buf_f], %[freq_buf_f], 4 \n\t" + "1: \n\t" + "lh %[tmp_1], 0(%[noiseSupFilter]) \n\t" + "lh %[tmp_2], 0(%[real]) \n\t" + "lh %[tmp_3], 0(%[imag]) \n\t" + "lh %[tmp_4], 2(%[noiseSupFilter]) \n\t" + "lh %[tmp_5], 2(%[real]) \n\t" + "lh %[tmp_6], 2(%[imag]) \n\t" + "mul %[tmp_2], %[tmp_2], %[tmp_1] \n\t" + "mul %[tmp_3], %[tmp_3], %[tmp_1] \n\t" + "mul %[tmp_5], %[tmp_5], %[tmp_4] \n\t" + "mul %[tmp_6], %[tmp_6], %[tmp_4] \n\t" + "addiu %[loop_count], %[loop_count], 2 \n\t" + "sra %[tmp_2], %[tmp_2], 14 \n\t" + "sra %[tmp_3], %[tmp_3], 14 \n\t" + "sra %[tmp_5], %[tmp_5], 14 \n\t" + "sra %[tmp_6], %[tmp_6], 14 \n\t" + "addiu %[noiseSupFilter], %[noiseSupFilter], 4 \n\t" + "sh %[tmp_2], 0(%[real]) \n\t" + "sh %[tmp_2], 4(%[freq_buf_s]) \n\t" + "sh %[tmp_3], 0(%[imag]) \n\t" + "sh %[tmp_3], 6(%[freq_buf_s]) \n\t" + "negu %[tmp_3], %[tmp_3] \n\t" + "sh %[tmp_5], 2(%[real]) \n\t" + "sh %[tmp_5], 0(%[freq_buf_s]) \n\t" + "sh %[tmp_6], 2(%[imag]) \n\t" + "sh %[tmp_6], 2(%[freq_buf_s]) \n\t" + "negu %[tmp_6], %[tmp_6] \n\t" + "addiu %[freq_buf_s], %[freq_buf_s], -8 \n\t" + "addiu %[real], %[real], 4 \n\t" + "addiu %[imag], %[imag], 4 \n\t" + "sh %[tmp_2], 0(%[freq_buf_f]) \n\t" + "sh %[tmp_3], 2(%[freq_buf_f]) \n\t" + "sh %[tmp_5], 4(%[freq_buf_f]) \n\t" + "sh %[tmp_6], 6(%[freq_buf_f]) \n\t" + "blt %[loop_count], %[loop_size], 1b \n\t" + " addiu %[freq_buf_f], %[freq_buf_f], 8 \n\t" + //last two samples: + "lh %[tmp_1], 0(%[noiseSupFilter]) \n\t" + "lh %[tmp_2], 0(%[real]) \n\t" + "lh %[tmp_3], 0(%[imag]) \n\t" + "lh %[tmp_4], 2(%[noiseSupFilter]) \n\t" + "lh %[tmp_5], 2(%[real]) \n\t" + "lh %[tmp_6], 2(%[imag]) \n\t" + "mul %[tmp_2], %[tmp_2], %[tmp_1] \n\t" + "mul %[tmp_3], %[tmp_3], %[tmp_1] \n\t" + "mul %[tmp_5], %[tmp_5], %[tmp_4] \n\t" + "mul %[tmp_6], %[tmp_6], %[tmp_4] \n\t" + "sra %[tmp_2], %[tmp_2], 14 \n\t" + "sra %[tmp_3], %[tmp_3], 14 \n\t" + "sra %[tmp_5], %[tmp_5], 14 \n\t" + "sra %[tmp_6], %[tmp_6], 14 \n\t" + "sh %[tmp_2], 0(%[real]) \n\t" + "sh %[tmp_2], 4(%[freq_buf_s]) \n\t" + "sh %[tmp_3], 0(%[imag]) \n\t" + "sh %[tmp_3], 6(%[freq_buf_s]) \n\t" + "negu %[tmp_3], %[tmp_3] \n\t" + "sh %[tmp_2], 0(%[freq_buf_f]) \n\t" + "sh %[tmp_3], 2(%[freq_buf_f]) \n\t" + "sh %[tmp_5], 4(%[freq_buf_f]) \n\t" + "sh %[tmp_6], 6(%[freq_buf_f]) \n\t" + "sh %[tmp_5], 2(%[real]) \n\t" + "sh %[tmp_6], 2(%[imag]) \n\t" + ".set pop \n\t" + : [real] "+r" (real), [imag] "+r" (imag), + [freq_buf_f] "+r" (freq_buf_f), [freq_buf_s] "+r" (freq_buf_s), + [loop_count] "+r" (loop_count), [noiseSupFilter] "+r" (noiseSupFilter), + [tmp_1] "=&r" (tmp_1), [tmp_2] "=&r" (tmp_2), [tmp_3] "=&r" (tmp_3), + [tmp_4] "=&r" (tmp_4), [tmp_5] "=&r" (tmp_5), [tmp_6] "=&r" (tmp_6) + : [loop_size] "r" (inst->anaLen2) + : "memory", "hi", "lo" + ); +} + +#if defined(MIPS_DSP_R1_LE) +// Denormalize the real-valued signal |in|, the output from inverse FFT. +void WebRtcNsx_Denormalize_mips(NoiseSuppressionFixedC* inst, + int16_t* in, + int factor) { + int32_t r0, r1, r2, r3, t0; + int len = (int)inst->anaLen; + int16_t *out = &inst->real[0]; + int shift = factor - inst->normData; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beqz %[len], 8f \n\t" + " nop \n\t" + "bltz %[shift], 4f \n\t" + " sra %[t0], %[len], 2 \n\t" + "beqz %[t0], 2f \n\t" + " andi %[len], %[len], 3 \n\t" + "1: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 2(%[in]) \n\t" + "lh %[r2], 4(%[in]) \n\t" + "lh %[r3], 6(%[in]) \n\t" + "shllv_s.ph %[r0], %[r0], %[shift] \n\t" + "shllv_s.ph %[r1], %[r1], %[shift] \n\t" + "shllv_s.ph %[r2], %[r2], %[shift] \n\t" + "shllv_s.ph %[r3], %[r3], %[shift] \n\t" + "addiu %[in], %[in], 8 \n\t" + "addiu %[t0], %[t0], -1 \n\t" + "sh %[r0], 0(%[out]) \n\t" + "sh %[r1], 2(%[out]) \n\t" + "sh %[r2], 4(%[out]) \n\t" + "sh %[r3], 6(%[out]) \n\t" + "bgtz %[t0], 1b \n\t" + " addiu %[out], %[out], 8 \n\t" + "2: \n\t" + "beqz %[len], 8f \n\t" + " nop \n\t" + "3: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "addiu %[in], %[in], 2 \n\t" + "addiu %[len], %[len], -1 \n\t" + "shllv_s.ph %[r0], %[r0], %[shift] \n\t" + "addiu %[out], %[out], 2 \n\t" + "bgtz %[len], 3b \n\t" + " sh %[r0], -2(%[out]) \n\t" + "b 8f \n\t" + "4: \n\t" + "negu %[shift], %[shift] \n\t" + "beqz %[t0], 6f \n\t" + " andi %[len], %[len], 3 \n\t" + "5: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 2(%[in]) \n\t" + "lh %[r2], 4(%[in]) \n\t" + "lh %[r3], 6(%[in]) \n\t" + "srav %[r0], %[r0], %[shift] \n\t" + "srav %[r1], %[r1], %[shift] \n\t" + "srav %[r2], %[r2], %[shift] \n\t" + "srav %[r3], %[r3], %[shift] \n\t" + "addiu %[in], %[in], 8 \n\t" + "addiu %[t0], %[t0], -1 \n\t" + "sh %[r0], 0(%[out]) \n\t" + "sh %[r1], 2(%[out]) \n\t" + "sh %[r2], 4(%[out]) \n\t" + "sh %[r3], 6(%[out]) \n\t" + "bgtz %[t0], 5b \n\t" + " addiu %[out], %[out], 8 \n\t" + "6: \n\t" + "beqz %[len], 8f \n\t" + " nop \n\t" + "7: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "addiu %[in], %[in], 2 \n\t" + "addiu %[len], %[len], -1 \n\t" + "srav %[r0], %[r0], %[shift] \n\t" + "addiu %[out], %[out], 2 \n\t" + "bgtz %[len], 7b \n\t" + " sh %[r0], -2(%[out]) \n\t" + "8: \n\t" + ".set pop \n\t" + : [t0] "=&r" (t0), [r0] "=&r" (r0), [r1] "=&r" (r1), + [r2] "=&r" (r2), [r3] "=&r" (r3) + : [len] "r" (len), [shift] "r" (shift), [in] "r" (in), + [out] "r" (out) + : "memory" + ); +} +#endif + +// Normalize the real-valued signal |in|, the input to forward FFT. +void WebRtcNsx_NormalizeRealBuffer_mips(NoiseSuppressionFixedC* inst, + const int16_t* in, + int16_t* out) { + int32_t r0, r1, r2, r3, t0; + int len = (int)inst->anaLen; + int shift = inst->normData; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beqz %[len], 4f \n\t" + " sra %[t0], %[len], 2 \n\t" + "beqz %[t0], 2f \n\t" + " andi %[len], %[len], 3 \n\t" + "1: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "lh %[r1], 2(%[in]) \n\t" + "lh %[r2], 4(%[in]) \n\t" + "lh %[r3], 6(%[in]) \n\t" + "sllv %[r0], %[r0], %[shift] \n\t" + "sllv %[r1], %[r1], %[shift] \n\t" + "sllv %[r2], %[r2], %[shift] \n\t" + "sllv %[r3], %[r3], %[shift] \n\t" + "addiu %[in], %[in], 8 \n\t" + "addiu %[t0], %[t0], -1 \n\t" + "sh %[r0], 0(%[out]) \n\t" + "sh %[r1], 2(%[out]) \n\t" + "sh %[r2], 4(%[out]) \n\t" + "sh %[r3], 6(%[out]) \n\t" + "bgtz %[t0], 1b \n\t" + " addiu %[out], %[out], 8 \n\t" + "2: \n\t" + "beqz %[len], 4f \n\t" + " nop \n\t" + "3: \n\t" + "lh %[r0], 0(%[in]) \n\t" + "addiu %[in], %[in], 2 \n\t" + "addiu %[len], %[len], -1 \n\t" + "sllv %[r0], %[r0], %[shift] \n\t" + "addiu %[out], %[out], 2 \n\t" + "bgtz %[len], 3b \n\t" + " sh %[r0], -2(%[out]) \n\t" + "4: \n\t" + ".set pop \n\t" + : [t0] "=&r" (t0), [r0] "=&r" (r0), [r1] "=&r" (r1), + [r2] "=&r" (r2), [r3] "=&r" (r3) + : [len] "r" (len), [shift] "r" (shift), [in] "r" (in), + [out] "r" (out) + : "memory" + ); +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_neon.c b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_neon.c new file mode 100644 index 0000000000..64ce99c32c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_core_neon.c @@ -0,0 +1,606 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/ns/nsx_core.h" + +#include + +#include "rtc_base/checks.h" + +// Constants to compensate for shifting signal log(2^shifts). +const int16_t WebRtcNsx_kLogTable[9] = { + 0, 177, 355, 532, 710, 887, 1065, 1242, 1420 +}; + +const int16_t WebRtcNsx_kCounterDiv[201] = { + 32767, 16384, 10923, 8192, 6554, 5461, 4681, 4096, 3641, 3277, 2979, 2731, + 2521, 2341, 2185, 2048, 1928, 1820, 1725, 1638, 1560, 1489, 1425, 1365, 1311, + 1260, 1214, 1170, 1130, 1092, 1057, 1024, 993, 964, 936, 910, 886, 862, 840, + 819, 799, 780, 762, 745, 728, 712, 697, 683, 669, 655, 643, 630, 618, 607, + 596, 585, 575, 565, 555, 546, 537, 529, 520, 512, 504, 496, 489, 482, 475, + 468, 462, 455, 449, 443, 437, 431, 426, 420, 415, 410, 405, 400, 395, 390, + 386, 381, 377, 372, 368, 364, 360, 356, 352, 349, 345, 341, 338, 334, 331, + 328, 324, 321, 318, 315, 312, 309, 306, 303, 301, 298, 295, 293, 290, 287, + 285, 282, 280, 278, 275, 273, 271, 269, 266, 264, 262, 260, 258, 256, 254, + 252, 250, 248, 246, 245, 243, 241, 239, 237, 236, 234, 232, 231, 229, 228, + 226, 224, 223, 221, 220, 218, 217, 216, 214, 213, 211, 210, 209, 207, 206, + 205, 204, 202, 201, 200, 199, 197, 196, 195, 194, 193, 192, 191, 189, 188, + 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, + 172, 172, 171, 170, 169, 168, 167, 166, 165, 165, 164, 163 +}; + +const int16_t WebRtcNsx_kLogTableFrac[256] = { + 0, 1, 3, 4, 6, 7, 9, 10, 11, 13, 14, 16, 17, 18, 20, 21, + 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38, 40, 41, 42, + 44, 45, 46, 47, 49, 50, 51, 52, 54, 55, 56, 57, 59, 60, 61, 62, + 63, 65, 66, 67, 68, 69, 71, 72, 73, 74, 75, 77, 78, 79, 80, 81, + 82, 84, 85, 86, 87, 88, 89, 90, 92, 93, 94, 95, 96, 97, 98, 99, + 100, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 116, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, + 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, + 147, 148, 149, 150, 151, 152, 153, 154, 155, 155, 156, 157, 158, 159, 160, + 161, 162, 163, 164, 165, 166, 167, 168, 169, 169, 170, 171, 172, 173, 174, + 175, 176, 177, 178, 178, 179, 180, 181, 182, 183, 184, 185, 185, 186, 187, + 188, 189, 190, 191, 192, 192, 193, 194, 195, 196, 197, 198, 198, 199, 200, + 201, 202, 203, 203, 204, 205, 206, 207, 208, 208, 209, 210, 211, 212, 212, + 213, 214, 215, 216, 216, 217, 218, 219, 220, 220, 221, 222, 223, 224, 224, + 225, 226, 227, 228, 228, 229, 230, 231, 231, 232, 233, 234, 234, 235, 236, + 237, 238, 238, 239, 240, 241, 241, 242, 243, 244, 244, 245, 246, 247, 247, + 248, 249, 249, 250, 251, 252, 252, 253, 254, 255, 255 +}; + +// Update the noise estimation information. +static void UpdateNoiseEstimateNeon(NoiseSuppressionFixedC* inst, int offset) { + const int16_t kExp2Const = 11819; // Q13 + int16_t* ptr_noiseEstLogQuantile = NULL; + int16_t* ptr_noiseEstQuantile = NULL; + int16x4_t kExp2Const16x4 = vdup_n_s16(kExp2Const); + int32x4_t twentyOne32x4 = vdupq_n_s32(21); + int32x4_t constA32x4 = vdupq_n_s32(0x1fffff); + int32x4_t constB32x4 = vdupq_n_s32(0x200000); + + int16_t tmp16 = WebRtcSpl_MaxValueW16(inst->noiseEstLogQuantile + offset, + inst->magnLen); + + // Guarantee a Q-domain as high as possible and still fit in int16 + inst->qNoise = 14 - (int) WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(kExp2Const, + tmp16, + 21); + + int32x4_t qNoise32x4 = vdupq_n_s32(inst->qNoise); + + for (ptr_noiseEstLogQuantile = &inst->noiseEstLogQuantile[offset], + ptr_noiseEstQuantile = &inst->noiseEstQuantile[0]; + ptr_noiseEstQuantile < &inst->noiseEstQuantile[inst->magnLen - 3]; + ptr_noiseEstQuantile += 4, ptr_noiseEstLogQuantile += 4) { + + // tmp32no2 = kExp2Const * inst->noiseEstLogQuantile[offset + i]; + int16x4_t v16x4 = vld1_s16(ptr_noiseEstLogQuantile); + int32x4_t v32x4B = vmull_s16(v16x4, kExp2Const16x4); + + // tmp32no1 = (0x00200000 | (tmp32no2 & 0x001FFFFF)); // 2^21 + frac + int32x4_t v32x4A = vandq_s32(v32x4B, constA32x4); + v32x4A = vorrq_s32(v32x4A, constB32x4); + + // tmp16 = (int16_t)(tmp32no2 >> 21); + v32x4B = vshrq_n_s32(v32x4B, 21); + + // tmp16 -= 21;// shift 21 to get result in Q0 + v32x4B = vsubq_s32(v32x4B, twentyOne32x4); + + // tmp16 += (int16_t) inst->qNoise; + // shift to get result in Q(qNoise) + v32x4B = vaddq_s32(v32x4B, qNoise32x4); + + // if (tmp16 < 0) { + // tmp32no1 >>= -tmp16; + // } else { + // tmp32no1 <<= tmp16; + // } + v32x4B = vshlq_s32(v32x4A, v32x4B); + + // tmp16 = WebRtcSpl_SatW32ToW16(tmp32no1); + v16x4 = vqmovn_s32(v32x4B); + + //inst->noiseEstQuantile[i] = tmp16; + vst1_s16(ptr_noiseEstQuantile, v16x4); + } + + // Last iteration: + + // inst->quantile[i]=exp(inst->lquantile[offset+i]); + // in Q21 + int32_t tmp32no2 = kExp2Const * *ptr_noiseEstLogQuantile; + int32_t tmp32no1 = (0x00200000 | (tmp32no2 & 0x001FFFFF)); // 2^21 + frac + + tmp16 = (int16_t)(tmp32no2 >> 21); + tmp16 -= 21;// shift 21 to get result in Q0 + tmp16 += (int16_t) inst->qNoise; //shift to get result in Q(qNoise) + if (tmp16 < 0) { + tmp32no1 >>= -tmp16; + } else { + tmp32no1 <<= tmp16; + } + *ptr_noiseEstQuantile = WebRtcSpl_SatW32ToW16(tmp32no1); +} + +// Noise Estimation +void WebRtcNsx_NoiseEstimationNeon(NoiseSuppressionFixedC* inst, + uint16_t* magn, + uint32_t* noise, + int16_t* q_noise) { + int16_t lmagn[HALF_ANAL_BLOCKL], counter, countDiv; + int16_t countProd, delta, zeros, frac; + int16_t log2, tabind, logval, tmp16, tmp16no1, tmp16no2; + const int16_t log2_const = 22713; + const int16_t width_factor = 21845; + + size_t i, s, offset; + + tabind = inst->stages - inst->normData; + RTC_DCHECK_LT(tabind, 9); + RTC_DCHECK_GT(tabind, -9); + if (tabind < 0) { + logval = -WebRtcNsx_kLogTable[-tabind]; + } else { + logval = WebRtcNsx_kLogTable[tabind]; + } + + int16x8_t logval_16x8 = vdupq_n_s16(logval); + + // lmagn(i)=log(magn(i))=log(2)*log2(magn(i)) + // magn is in Q(-stages), and the real lmagn values are: + // real_lmagn(i)=log(magn(i)*2^stages)=log(magn(i))+log(2^stages) + // lmagn in Q8 + for (i = 0; i < inst->magnLen; i++) { + if (magn[i]) { + zeros = WebRtcSpl_NormU32((uint32_t)magn[i]); + frac = (int16_t)((((uint32_t)magn[i] << zeros) + & 0x7FFFFFFF) >> 23); + RTC_DCHECK_LT(frac, 256); + // log2(magn(i)) + log2 = (int16_t)(((31 - zeros) << 8) + + WebRtcNsx_kLogTableFrac[frac]); + // log2(magn(i))*log(2) + lmagn[i] = (int16_t)((log2 * log2_const) >> 15); + // + log(2^stages) + lmagn[i] += logval; + } else { + lmagn[i] = logval; + } + } + + int16x4_t Q3_16x4 = vdup_n_s16(3); + int16x8_t WIDTHQ8_16x8 = vdupq_n_s16(WIDTH_Q8); + int16x8_t WIDTHFACTOR_16x8 = vdupq_n_s16(width_factor); + + int16_t factor = FACTOR_Q7; + if (inst->blockIndex < END_STARTUP_LONG) + factor = FACTOR_Q7_STARTUP; + + // Loop over simultaneous estimates + for (s = 0; s < SIMULT; s++) { + offset = s * inst->magnLen; + + // Get counter values from state + counter = inst->noiseEstCounter[s]; + RTC_DCHECK_LT(counter, 201); + countDiv = WebRtcNsx_kCounterDiv[counter]; + countProd = (int16_t)(counter * countDiv); + + // quant_est(...) + int16_t deltaBuff[8]; + int16x4_t tmp16x4_0; + int16x4_t tmp16x4_1; + int16x4_t countDiv_16x4 = vdup_n_s16(countDiv); + int16x8_t countProd_16x8 = vdupq_n_s16(countProd); + int16x8_t tmp16x8_0 = vdupq_n_s16(countDiv); + int16x8_t prod16x8 = vqrdmulhq_s16(WIDTHFACTOR_16x8, tmp16x8_0); + int16x8_t tmp16x8_1; + int16x8_t tmp16x8_2; + int16x8_t tmp16x8_3; + uint16x8_t tmp16x8_4; + int32x4_t tmp32x4; + + for (i = 0; i + 7 < inst->magnLen; i += 8) { + // Compute delta. + // Smaller step size during startup. This prevents from using + // unrealistic values causing overflow. + tmp16x8_0 = vdupq_n_s16(factor); + vst1q_s16(deltaBuff, tmp16x8_0); + + int j; + for (j = 0; j < 8; j++) { + if (inst->noiseEstDensity[offset + i + j] > 512) { + // Get values for deltaBuff by shifting intead of dividing. + int factor = WebRtcSpl_NormW16(inst->noiseEstDensity[offset + i + j]); + deltaBuff[j] = (int16_t)(FACTOR_Q16 >> (14 - factor)); + } + } + + // Update log quantile estimate + + // tmp16 = (int16_t)((delta * countDiv) >> 14); + tmp32x4 = vmull_s16(vld1_s16(&deltaBuff[0]), countDiv_16x4); + tmp16x4_1 = vshrn_n_s32(tmp32x4, 14); + tmp32x4 = vmull_s16(vld1_s16(&deltaBuff[4]), countDiv_16x4); + tmp16x4_0 = vshrn_n_s32(tmp32x4, 14); + tmp16x8_0 = vcombine_s16(tmp16x4_1, tmp16x4_0); // Keep for several lines. + + // prepare for the "if" branch + // tmp16 += 2; + // tmp16_1 = (Word16)(tmp16>>2); + tmp16x8_1 = vrshrq_n_s16(tmp16x8_0, 2); + + // inst->noiseEstLogQuantile[offset+i] + tmp16_1; + tmp16x8_2 = vld1q_s16(&inst->noiseEstLogQuantile[offset + i]); // Keep + tmp16x8_1 = vaddq_s16(tmp16x8_2, tmp16x8_1); // Keep for several lines + + // Prepare for the "else" branch + // tmp16 += 1; + // tmp16_1 = (Word16)(tmp16>>1); + tmp16x8_0 = vrshrq_n_s16(tmp16x8_0, 1); + + // tmp16_2 = (int16_t)((tmp16_1 * 3) >> 1); + tmp32x4 = vmull_s16(vget_low_s16(tmp16x8_0), Q3_16x4); + tmp16x4_1 = vshrn_n_s32(tmp32x4, 1); + + // tmp16_2 = (int16_t)((tmp16_1 * 3) >> 1); + tmp32x4 = vmull_s16(vget_high_s16(tmp16x8_0), Q3_16x4); + tmp16x4_0 = vshrn_n_s32(tmp32x4, 1); + + // inst->noiseEstLogQuantile[offset + i] - tmp16_2; + tmp16x8_0 = vcombine_s16(tmp16x4_1, tmp16x4_0); // keep + tmp16x8_0 = vsubq_s16(tmp16x8_2, tmp16x8_0); + + // logval is the smallest fixed point representation we can have. Values + // below that will correspond to values in the interval [0, 1], which + // can't possibly occur. + tmp16x8_0 = vmaxq_s16(tmp16x8_0, logval_16x8); + + // Do the if-else branches: + tmp16x8_3 = vld1q_s16(&lmagn[i]); // keep for several lines + tmp16x8_4 = vcgtq_s16(tmp16x8_3, tmp16x8_2); + tmp16x8_2 = vbslq_s16(tmp16x8_4, tmp16x8_1, tmp16x8_0); + vst1q_s16(&inst->noiseEstLogQuantile[offset + i], tmp16x8_2); + + // Update density estimate + // tmp16_1 + tmp16_2 + tmp16x8_1 = vld1q_s16(&inst->noiseEstDensity[offset + i]); + tmp16x8_0 = vqrdmulhq_s16(tmp16x8_1, countProd_16x8); + tmp16x8_0 = vaddq_s16(tmp16x8_0, prod16x8); + + // lmagn[i] - inst->noiseEstLogQuantile[offset + i] + tmp16x8_3 = vsubq_s16(tmp16x8_3, tmp16x8_2); + tmp16x8_3 = vabsq_s16(tmp16x8_3); + tmp16x8_4 = vcgtq_s16(WIDTHQ8_16x8, tmp16x8_3); + tmp16x8_1 = vbslq_s16(tmp16x8_4, tmp16x8_0, tmp16x8_1); + vst1q_s16(&inst->noiseEstDensity[offset + i], tmp16x8_1); + } // End loop over magnitude spectrum + + // Last iteration over magnitude spectrum: + // compute delta + if (inst->noiseEstDensity[offset + i] > 512) { + // Get values for deltaBuff by shifting intead of dividing. + int factor = WebRtcSpl_NormW16(inst->noiseEstDensity[offset + i]); + delta = (int16_t)(FACTOR_Q16 >> (14 - factor)); + } else { + delta = FACTOR_Q7; + if (inst->blockIndex < END_STARTUP_LONG) { + // Smaller step size during startup. This prevents from using + // unrealistic values causing overflow. + delta = FACTOR_Q7_STARTUP; + } + } + // update log quantile estimate + tmp16 = (int16_t)((delta * countDiv) >> 14); + if (lmagn[i] > inst->noiseEstLogQuantile[offset + i]) { + // +=QUANTILE*delta/(inst->counter[s]+1) QUANTILE=0.25, =1 in Q2 + // CounterDiv=1/(inst->counter[s]+1) in Q15 + tmp16 += 2; + inst->noiseEstLogQuantile[offset + i] += tmp16 / 4; + } else { + tmp16 += 1; + // *(1-QUANTILE), in Q2 QUANTILE=0.25, 1-0.25=0.75=3 in Q2 + // TODO(bjornv): investigate why we need to truncate twice. + tmp16no2 = (int16_t)((tmp16 / 2) * 3 / 2); + inst->noiseEstLogQuantile[offset + i] -= tmp16no2; + if (inst->noiseEstLogQuantile[offset + i] < logval) { + // logval is the smallest fixed point representation we can have. + // Values below that will correspond to values in the interval + // [0, 1], which can't possibly occur. + inst->noiseEstLogQuantile[offset + i] = logval; + } + } + + // update density estimate + if (WEBRTC_SPL_ABS_W16(lmagn[i] - inst->noiseEstLogQuantile[offset + i]) + < WIDTH_Q8) { + tmp16no1 = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + inst->noiseEstDensity[offset + i], countProd, 15); + tmp16no2 = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND( + width_factor, countDiv, 15); + inst->noiseEstDensity[offset + i] = tmp16no1 + tmp16no2; + } + + + if (counter >= END_STARTUP_LONG) { + inst->noiseEstCounter[s] = 0; + if (inst->blockIndex >= END_STARTUP_LONG) { + UpdateNoiseEstimateNeon(inst, offset); + } + } + inst->noiseEstCounter[s]++; + + } // end loop over simultaneous estimates + + // Sequentially update the noise during startup + if (inst->blockIndex < END_STARTUP_LONG) { + UpdateNoiseEstimateNeon(inst, offset); + } + + for (i = 0; i < inst->magnLen; i++) { + noise[i] = (uint32_t)(inst->noiseEstQuantile[i]); // Q(qNoise) + } + (*q_noise) = (int16_t)inst->qNoise; +} + +// Filter the data in the frequency domain, and create spectrum. +void WebRtcNsx_PrepareSpectrumNeon(NoiseSuppressionFixedC* inst, + int16_t* freq_buf) { + RTC_DCHECK_EQ(1, inst->magnLen % 8); + RTC_DCHECK_EQ(0, inst->anaLen2 % 16); + + // (1) Filtering. + + // Fixed point C code for the next block is as follows: + // for (i = 0; i < inst->magnLen; i++) { + // inst->real[i] = (int16_t)((inst->real[i] * + // (int16_t)(inst->noiseSupFilter[i])) >> 14); // Q(normData-stages) + // inst->imag[i] = (int16_t)((inst->imag[i] * + // (int16_t)(inst->noiseSupFilter[i])) >> 14); // Q(normData-stages) + // } + + int16_t* preal = &inst->real[0]; + int16_t* pimag = &inst->imag[0]; + int16_t* pns_filter = (int16_t*)&inst->noiseSupFilter[0]; + int16_t* pimag_end = pimag + inst->magnLen - 4; + + while (pimag < pimag_end) { + int16x8_t real = vld1q_s16(preal); + int16x8_t imag = vld1q_s16(pimag); + int16x8_t ns_filter = vld1q_s16(pns_filter); + + int32x4_t tmp_r_0 = vmull_s16(vget_low_s16(real), vget_low_s16(ns_filter)); + int32x4_t tmp_i_0 = vmull_s16(vget_low_s16(imag), vget_low_s16(ns_filter)); + int32x4_t tmp_r_1 = vmull_s16(vget_high_s16(real), + vget_high_s16(ns_filter)); + int32x4_t tmp_i_1 = vmull_s16(vget_high_s16(imag), + vget_high_s16(ns_filter)); + + int16x4_t result_r_0 = vshrn_n_s32(tmp_r_0, 14); + int16x4_t result_i_0 = vshrn_n_s32(tmp_i_0, 14); + int16x4_t result_r_1 = vshrn_n_s32(tmp_r_1, 14); + int16x4_t result_i_1 = vshrn_n_s32(tmp_i_1, 14); + + vst1q_s16(preal, vcombine_s16(result_r_0, result_r_1)); + vst1q_s16(pimag, vcombine_s16(result_i_0, result_i_1)); + preal += 8; + pimag += 8; + pns_filter += 8; + } + + // Filter the last element + *preal = (int16_t)((*preal * *pns_filter) >> 14); + *pimag = (int16_t)((*pimag * *pns_filter) >> 14); + + // (2) Create spectrum. + + // Fixed point C code for the rest of the function is as follows: + // freq_buf[0] = inst->real[0]; + // freq_buf[1] = -inst->imag[0]; + // for (i = 1, j = 2; i < inst->anaLen2; i += 1, j += 2) { + // freq_buf[j] = inst->real[i]; + // freq_buf[j + 1] = -inst->imag[i]; + // } + // freq_buf[inst->anaLen] = inst->real[inst->anaLen2]; + // freq_buf[inst->anaLen + 1] = -inst->imag[inst->anaLen2]; + + preal = &inst->real[0]; + pimag = &inst->imag[0]; + pimag_end = pimag + inst->anaLen2; + int16_t * freq_buf_start = freq_buf; + while (pimag < pimag_end) { + // loop unroll + int16x8x2_t real_imag_0; + int16x8x2_t real_imag_1; + real_imag_0.val[1] = vld1q_s16(pimag); + real_imag_0.val[0] = vld1q_s16(preal); + preal += 8; + pimag += 8; + real_imag_1.val[1] = vld1q_s16(pimag); + real_imag_1.val[0] = vld1q_s16(preal); + preal += 8; + pimag += 8; + + real_imag_0.val[1] = vnegq_s16(real_imag_0.val[1]); + real_imag_1.val[1] = vnegq_s16(real_imag_1.val[1]); + vst2q_s16(freq_buf_start, real_imag_0); + freq_buf_start += 16; + vst2q_s16(freq_buf_start, real_imag_1); + freq_buf_start += 16; + } + freq_buf[inst->anaLen] = inst->real[inst->anaLen2]; + freq_buf[inst->anaLen + 1] = -inst->imag[inst->anaLen2]; +} + +// For the noise supress process, synthesis, read out fully processed segment, +// and update synthesis buffer. +void WebRtcNsx_SynthesisUpdateNeon(NoiseSuppressionFixedC* inst, + int16_t* out_frame, + int16_t gain_factor) { + RTC_DCHECK_EQ(0, inst->anaLen % 16); + RTC_DCHECK_EQ(0, inst->blockLen10ms % 16); + + int16_t* preal_start = inst->real; + const int16_t* pwindow = inst->window; + int16_t* preal_end = preal_start + inst->anaLen; + int16_t* psynthesis_buffer = inst->synthesisBuffer; + + while (preal_start < preal_end) { + // Loop unroll. + int16x8_t window_0 = vld1q_s16(pwindow); + int16x8_t real_0 = vld1q_s16(preal_start); + int16x8_t synthesis_buffer_0 = vld1q_s16(psynthesis_buffer); + + int16x8_t window_1 = vld1q_s16(pwindow + 8); + int16x8_t real_1 = vld1q_s16(preal_start + 8); + int16x8_t synthesis_buffer_1 = vld1q_s16(psynthesis_buffer + 8); + + int32x4_t tmp32a_0_low = vmull_s16(vget_low_s16(real_0), + vget_low_s16(window_0)); + int32x4_t tmp32a_0_high = vmull_s16(vget_high_s16(real_0), + vget_high_s16(window_0)); + + int32x4_t tmp32a_1_low = vmull_s16(vget_low_s16(real_1), + vget_low_s16(window_1)); + int32x4_t tmp32a_1_high = vmull_s16(vget_high_s16(real_1), + vget_high_s16(window_1)); + + int16x4_t tmp16a_0_low = vqrshrn_n_s32(tmp32a_0_low, 14); + int16x4_t tmp16a_0_high = vqrshrn_n_s32(tmp32a_0_high, 14); + + int16x4_t tmp16a_1_low = vqrshrn_n_s32(tmp32a_1_low, 14); + int16x4_t tmp16a_1_high = vqrshrn_n_s32(tmp32a_1_high, 14); + + int32x4_t tmp32b_0_low = vmull_n_s16(tmp16a_0_low, gain_factor); + int32x4_t tmp32b_0_high = vmull_n_s16(tmp16a_0_high, gain_factor); + + int32x4_t tmp32b_1_low = vmull_n_s16(tmp16a_1_low, gain_factor); + int32x4_t tmp32b_1_high = vmull_n_s16(tmp16a_1_high, gain_factor); + + int16x4_t tmp16b_0_low = vqrshrn_n_s32(tmp32b_0_low, 13); + int16x4_t tmp16b_0_high = vqrshrn_n_s32(tmp32b_0_high, 13); + + int16x4_t tmp16b_1_low = vqrshrn_n_s32(tmp32b_1_low, 13); + int16x4_t tmp16b_1_high = vqrshrn_n_s32(tmp32b_1_high, 13); + + synthesis_buffer_0 = vqaddq_s16(vcombine_s16(tmp16b_0_low, tmp16b_0_high), + synthesis_buffer_0); + synthesis_buffer_1 = vqaddq_s16(vcombine_s16(tmp16b_1_low, tmp16b_1_high), + synthesis_buffer_1); + vst1q_s16(psynthesis_buffer, synthesis_buffer_0); + vst1q_s16(psynthesis_buffer + 8, synthesis_buffer_1); + + pwindow += 16; + preal_start += 16; + psynthesis_buffer += 16; + } + + // Read out fully processed segment. + int16_t * p_start = inst->synthesisBuffer; + int16_t * p_end = inst->synthesisBuffer + inst->blockLen10ms; + int16_t * p_frame = out_frame; + while (p_start < p_end) { + int16x8_t frame_0 = vld1q_s16(p_start); + vst1q_s16(p_frame, frame_0); + p_start += 8; + p_frame += 8; + } + + // Update synthesis buffer. + int16_t* p_start_src = inst->synthesisBuffer + inst->blockLen10ms; + int16_t* p_end_src = inst->synthesisBuffer + inst->anaLen; + int16_t* p_start_dst = inst->synthesisBuffer; + while (p_start_src < p_end_src) { + int16x8_t frame = vld1q_s16(p_start_src); + vst1q_s16(p_start_dst, frame); + p_start_src += 8; + p_start_dst += 8; + } + + p_start = inst->synthesisBuffer + inst->anaLen - inst->blockLen10ms; + p_end = p_start + inst->blockLen10ms; + int16x8_t zero = vdupq_n_s16(0); + for (;p_start < p_end; p_start += 8) { + vst1q_s16(p_start, zero); + } +} + +// Update analysis buffer for lower band, and window data before FFT. +void WebRtcNsx_AnalysisUpdateNeon(NoiseSuppressionFixedC* inst, + int16_t* out, + int16_t* new_speech) { + RTC_DCHECK_EQ(0, inst->blockLen10ms % 16); + RTC_DCHECK_EQ(0, inst->anaLen % 16); + + // For lower band update analysis buffer. + // memcpy(inst->analysisBuffer, inst->analysisBuffer + inst->blockLen10ms, + // (inst->anaLen - inst->blockLen10ms) * sizeof(*inst->analysisBuffer)); + int16_t* p_start_src = inst->analysisBuffer + inst->blockLen10ms; + int16_t* p_end_src = inst->analysisBuffer + inst->anaLen; + int16_t* p_start_dst = inst->analysisBuffer; + while (p_start_src < p_end_src) { + int16x8_t frame = vld1q_s16(p_start_src); + vst1q_s16(p_start_dst, frame); + + p_start_src += 8; + p_start_dst += 8; + } + + // memcpy(inst->analysisBuffer + inst->anaLen - inst->blockLen10ms, + // new_speech, inst->blockLen10ms * sizeof(*inst->analysisBuffer)); + p_start_src = new_speech; + p_end_src = new_speech + inst->blockLen10ms; + p_start_dst = inst->analysisBuffer + inst->anaLen - inst->blockLen10ms; + while (p_start_src < p_end_src) { + int16x8_t frame = vld1q_s16(p_start_src); + vst1q_s16(p_start_dst, frame); + + p_start_src += 8; + p_start_dst += 8; + } + + // Window data before FFT. + int16_t* p_start_window = (int16_t*) inst->window; + int16_t* p_start_buffer = inst->analysisBuffer; + int16_t* p_end_buffer = inst->analysisBuffer + inst->anaLen; + int16_t* p_start_out = out; + + // Load the first element to reduce pipeline bubble. + int16x8_t window = vld1q_s16(p_start_window); + int16x8_t buffer = vld1q_s16(p_start_buffer); + p_start_window += 8; + p_start_buffer += 8; + + while (p_start_buffer < p_end_buffer) { + // Unroll loop. + int32x4_t tmp32_low = vmull_s16(vget_low_s16(window), vget_low_s16(buffer)); + int32x4_t tmp32_high = vmull_s16(vget_high_s16(window), + vget_high_s16(buffer)); + window = vld1q_s16(p_start_window); + buffer = vld1q_s16(p_start_buffer); + + int16x4_t result_low = vrshrn_n_s32(tmp32_low, 14); + int16x4_t result_high = vrshrn_n_s32(tmp32_high, 14); + vst1q_s16(p_start_out, vcombine_s16(result_low, result_high)); + + p_start_buffer += 8; + p_start_window += 8; + p_start_out += 8; + } + int32x4_t tmp32_low = vmull_s16(vget_low_s16(window), vget_low_s16(buffer)); + int32x4_t tmp32_high = vmull_s16(vget_high_s16(window), + vget_high_s16(buffer)); + + int16x4_t result_low = vrshrn_n_s32(tmp32_low, 14); + int16x4_t result_high = vrshrn_n_s32(tmp32_high, 14); + vst1q_s16(p_start_out, vcombine_s16(result_low, result_high)); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_defines.h b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_defines.h new file mode 100644 index 0000000000..12869b3cf5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/nsx_defines.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_NSX_DEFINES_H_ +#define MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_NSX_DEFINES_H_ + +#define ANAL_BLOCKL_MAX 256 /* Max analysis block length */ +#define HALF_ANAL_BLOCKL 129 /* Half max analysis block length + 1 */ +#define NUM_HIGH_BANDS_MAX 2 /* Max number of high bands */ +#define SIMULT 3 +#define END_STARTUP_LONG 200 +#define END_STARTUP_SHORT 50 +#define FACTOR_Q16 2621440 /* 40 in Q16 */ +#define FACTOR_Q7 5120 /* 40 in Q7 */ +#define FACTOR_Q7_STARTUP 1024 /* 8 in Q7 */ +#define WIDTH_Q8 3 /* 0.01 in Q8 (or 25 ) */ + +/* PARAMETERS FOR NEW METHOD */ +#define DD_PR_SNR_Q11 2007 /* ~= Q11(0.98) DD update of prior SNR */ +#define ONE_MINUS_DD_PR_SNR_Q11 41 /* DD update of prior SNR */ +#define SPECT_FLAT_TAVG_Q14 4915 /* (0.30) tavg parameter for spectral flatness measure */ +#define SPECT_DIFF_TAVG_Q8 77 /* (0.30) tavg parameter for spectral flatness measure */ +#define PRIOR_UPDATE_Q14 1638 /* Q14(0.1) Update parameter of prior model */ +#define NOISE_UPDATE_Q8 26 /* 26 ~= Q8(0.1) Update parameter for noise */ + +/* Probability threshold for noise state in speech/noise likelihood. */ +#define ONE_MINUS_PROB_RANGE_Q8 205 /* 205 ~= Q8(0.8) */ +#define HIST_PAR_EST 1000 /* Histogram size for estimation of parameters */ + +/* FEATURE EXTRACTION CONFIG */ +/* Bin size of histogram */ +#define BIN_SIZE_LRT 10 +/* Scale parameters: multiply dominant peaks of the histograms by scale factor to obtain. */ +/* Thresholds for prior model */ +#define FACTOR_1_LRT_DIFF 6 /* For LRT and spectral difference (5 times bigger) */ +/* For spectral_flatness: used when noise is flatter than speech (10 times bigger). */ +#define FACTOR_2_FLAT_Q10 922 +/* Peak limit for spectral flatness (varies between 0 and 1) */ +#define THRES_PEAK_FLAT 24 /* * 2 * BIN_SIZE_FLAT_FX */ +/* Limit on spacing of two highest peaks in histogram: spacing determined by bin size. */ +#define LIM_PEAK_SPACE_FLAT_DIFF 4 /* * 2 * BIN_SIZE_DIFF_FX */ +/* Limit on relevance of second peak */ +#define LIM_PEAK_WEIGHT_FLAT_DIFF 2 +#define THRES_FLUCT_LRT 10240 /* = 20 * inst->modelUpdate; fluctuation limit of LRT feat. */ +/* Limit on the max and min values for the feature thresholds */ +#define MAX_FLAT_Q10 38912 /* * 2 * BIN_SIZE_FLAT_FX */ +#define MIN_FLAT_Q10 4096 /* * 2 * BIN_SIZE_FLAT_FX */ +#define MAX_DIFF 100 /* * 2 * BIN_SIZE_DIFF_FX */ +#define MIN_DIFF 16 /* * 2 * BIN_SIZE_DIFF_FX */ +/* Criteria of weight of histogram peak to accept/reject feature */ +#define THRES_WEIGHT_FLAT_DIFF 154 /*(int)(0.3*(inst->modelUpdate)) for flatness and difference */ + +#define STAT_UPDATES 9 /* Update every 512 = 1 << 9 block */ +#define ONE_MINUS_GAMMA_PAUSE_Q8 13 /* ~= Q8(0.05) Update for conservative noise estimate */ +#define GAMMA_NOISE_TRANS_AND_SPEECH_Q8 3 /* ~= Q8(0.01) Update for transition and noise region */ + +#endif /* MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_NSX_DEFINES_H_ */ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/ns/windows_private.h b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/windows_private.h new file mode 100644 index 0000000000..2ffd693d8d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/ns/windows_private.h @@ -0,0 +1,574 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_WINDOWS_PRIVATE_H_ +#define MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_WINDOWS_PRIVATE_H_ + +// Hanning window for 4ms 16kHz +static const float kHanning64w128[128] = { + 0.00000000000000f, 0.02454122852291f, 0.04906767432742f, + 0.07356456359967f, 0.09801714032956f, 0.12241067519922f, + 0.14673047445536f, 0.17096188876030f, 0.19509032201613f, + 0.21910124015687f, 0.24298017990326f, 0.26671275747490f, + 0.29028467725446f, 0.31368174039889f, 0.33688985339222f, + 0.35989503653499f, 0.38268343236509f, 0.40524131400499f, + 0.42755509343028f, 0.44961132965461f, 0.47139673682600f, + 0.49289819222978f, 0.51410274419322f, 0.53499761988710f, + 0.55557023301960f, 0.57580819141785f, 0.59569930449243f, + 0.61523159058063f, 0.63439328416365f, 0.65317284295378f, + 0.67155895484702f, 0.68954054473707f, 0.70710678118655f, + 0.72424708295147f, 0.74095112535496f, 0.75720884650648f, + 0.77301045336274f, 0.78834642762661f, 0.80320753148064f, + 0.81758481315158f, 0.83146961230255f, 0.84485356524971f, + 0.85772861000027f, 0.87008699110871f, 0.88192126434835f, + 0.89322430119552f, 0.90398929312344f, 0.91420975570353f, + 0.92387953251129f, 0.93299279883474f, 0.94154406518302f, + 0.94952818059304f, 0.95694033573221f, 0.96377606579544f, + 0.97003125319454f, 0.97570213003853f, 0.98078528040323f, + 0.98527764238894f, 0.98917650996478f, 0.99247953459871f, + 0.99518472667220f, 0.99729045667869f, 0.99879545620517f, + 0.99969881869620f, 1.00000000000000f, + 0.99969881869620f, 0.99879545620517f, 0.99729045667869f, + 0.99518472667220f, 0.99247953459871f, 0.98917650996478f, + 0.98527764238894f, 0.98078528040323f, 0.97570213003853f, + 0.97003125319454f, 0.96377606579544f, 0.95694033573221f, + 0.94952818059304f, 0.94154406518302f, 0.93299279883474f, + 0.92387953251129f, 0.91420975570353f, 0.90398929312344f, + 0.89322430119552f, 0.88192126434835f, 0.87008699110871f, + 0.85772861000027f, 0.84485356524971f, 0.83146961230255f, + 0.81758481315158f, 0.80320753148064f, 0.78834642762661f, + 0.77301045336274f, 0.75720884650648f, 0.74095112535496f, + 0.72424708295147f, 0.70710678118655f, 0.68954054473707f, + 0.67155895484702f, 0.65317284295378f, 0.63439328416365f, + 0.61523159058063f, 0.59569930449243f, 0.57580819141785f, + 0.55557023301960f, 0.53499761988710f, 0.51410274419322f, + 0.49289819222978f, 0.47139673682600f, 0.44961132965461f, + 0.42755509343028f, 0.40524131400499f, 0.38268343236509f, + 0.35989503653499f, 0.33688985339222f, 0.31368174039889f, + 0.29028467725446f, 0.26671275747490f, 0.24298017990326f, + 0.21910124015687f, 0.19509032201613f, 0.17096188876030f, + 0.14673047445536f, 0.12241067519922f, 0.09801714032956f, + 0.07356456359967f, 0.04906767432742f, 0.02454122852291f +}; + + + +// hybrib Hanning & flat window +static const float kBlocks80w128[128] = { + (float)0.00000000, (float)0.03271908, (float)0.06540313, (float)0.09801714, (float)0.13052619, + (float)0.16289547, (float)0.19509032, (float)0.22707626, (float)0.25881905, (float)0.29028468, + (float)0.32143947, (float)0.35225005, (float)0.38268343, (float)0.41270703, (float)0.44228869, + (float)0.47139674, (float)0.50000000, (float)0.52806785, (float)0.55557023, (float)0.58247770, + (float)0.60876143, (float)0.63439328, (float)0.65934582, (float)0.68359230, (float)0.70710678, + (float)0.72986407, (float)0.75183981, (float)0.77301045, (float)0.79335334, (float)0.81284668, + (float)0.83146961, (float)0.84920218, (float)0.86602540, (float)0.88192126, (float)0.89687274, + (float)0.91086382, (float)0.92387953, (float)0.93590593, (float)0.94693013, (float)0.95694034, + (float)0.96592583, (float)0.97387698, (float)0.98078528, (float)0.98664333, (float)0.99144486, + (float)0.99518473, (float)0.99785892, (float)0.99946459, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)0.99946459, (float)0.99785892, (float)0.99518473, (float)0.99144486, + (float)0.98664333, (float)0.98078528, (float)0.97387698, (float)0.96592583, (float)0.95694034, + (float)0.94693013, (float)0.93590593, (float)0.92387953, (float)0.91086382, (float)0.89687274, + (float)0.88192126, (float)0.86602540, (float)0.84920218, (float)0.83146961, (float)0.81284668, + (float)0.79335334, (float)0.77301045, (float)0.75183981, (float)0.72986407, (float)0.70710678, + (float)0.68359230, (float)0.65934582, (float)0.63439328, (float)0.60876143, (float)0.58247770, + (float)0.55557023, (float)0.52806785, (float)0.50000000, (float)0.47139674, (float)0.44228869, + (float)0.41270703, (float)0.38268343, (float)0.35225005, (float)0.32143947, (float)0.29028468, + (float)0.25881905, (float)0.22707626, (float)0.19509032, (float)0.16289547, (float)0.13052619, + (float)0.09801714, (float)0.06540313, (float)0.03271908 +}; + +// hybrib Hanning & flat window +static const float kBlocks160w256[256] = { + (float)0.00000000, (float)0.01636173, (float)0.03271908, (float)0.04906767, (float)0.06540313, + (float)0.08172107, (float)0.09801714, (float)0.11428696, (float)0.13052619, (float)0.14673047, + (float)0.16289547, (float)0.17901686, (float)0.19509032, (float)0.21111155, (float)0.22707626, + (float)0.24298018, (float)0.25881905, (float)0.27458862, (float)0.29028468, (float)0.30590302, + (float)0.32143947, (float)0.33688985, (float)0.35225005, (float)0.36751594, (float)0.38268343, + (float)0.39774847, (float)0.41270703, (float)0.42755509, (float)0.44228869, (float)0.45690388, + (float)0.47139674, (float)0.48576339, (float)0.50000000, (float)0.51410274, (float)0.52806785, + (float)0.54189158, (float)0.55557023, (float)0.56910015, (float)0.58247770, (float)0.59569930, + (float)0.60876143, (float)0.62166057, (float)0.63439328, (float)0.64695615, (float)0.65934582, + (float)0.67155895, (float)0.68359230, (float)0.69544264, (float)0.70710678, (float)0.71858162, + (float)0.72986407, (float)0.74095113, (float)0.75183981, (float)0.76252720, (float)0.77301045, + (float)0.78328675, (float)0.79335334, (float)0.80320753, (float)0.81284668, (float)0.82226822, + (float)0.83146961, (float)0.84044840, (float)0.84920218, (float)0.85772861, (float)0.86602540, + (float)0.87409034, (float)0.88192126, (float)0.88951608, (float)0.89687274, (float)0.90398929, + (float)0.91086382, (float)0.91749450, (float)0.92387953, (float)0.93001722, (float)0.93590593, + (float)0.94154407, (float)0.94693013, (float)0.95206268, (float)0.95694034, (float)0.96156180, + (float)0.96592583, (float)0.97003125, (float)0.97387698, (float)0.97746197, (float)0.98078528, + (float)0.98384601, (float)0.98664333, (float)0.98917651, (float)0.99144486, (float)0.99344778, + (float)0.99518473, (float)0.99665524, (float)0.99785892, (float)0.99879546, (float)0.99946459, + (float)0.99986614, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)0.99986614, (float)0.99946459, (float)0.99879546, (float)0.99785892, + (float)0.99665524, (float)0.99518473, (float)0.99344778, (float)0.99144486, (float)0.98917651, + (float)0.98664333, (float)0.98384601, (float)0.98078528, (float)0.97746197, (float)0.97387698, + (float)0.97003125, (float)0.96592583, (float)0.96156180, (float)0.95694034, (float)0.95206268, + (float)0.94693013, (float)0.94154407, (float)0.93590593, (float)0.93001722, (float)0.92387953, + (float)0.91749450, (float)0.91086382, (float)0.90398929, (float)0.89687274, (float)0.88951608, + (float)0.88192126, (float)0.87409034, (float)0.86602540, (float)0.85772861, (float)0.84920218, + (float)0.84044840, (float)0.83146961, (float)0.82226822, (float)0.81284668, (float)0.80320753, + (float)0.79335334, (float)0.78328675, (float)0.77301045, (float)0.76252720, (float)0.75183981, + (float)0.74095113, (float)0.72986407, (float)0.71858162, (float)0.70710678, (float)0.69544264, + (float)0.68359230, (float)0.67155895, (float)0.65934582, (float)0.64695615, (float)0.63439328, + (float)0.62166057, (float)0.60876143, (float)0.59569930, (float)0.58247770, (float)0.56910015, + (float)0.55557023, (float)0.54189158, (float)0.52806785, (float)0.51410274, (float)0.50000000, + (float)0.48576339, (float)0.47139674, (float)0.45690388, (float)0.44228869, (float)0.42755509, + (float)0.41270703, (float)0.39774847, (float)0.38268343, (float)0.36751594, (float)0.35225005, + (float)0.33688985, (float)0.32143947, (float)0.30590302, (float)0.29028468, (float)0.27458862, + (float)0.25881905, (float)0.24298018, (float)0.22707626, (float)0.21111155, (float)0.19509032, + (float)0.17901686, (float)0.16289547, (float)0.14673047, (float)0.13052619, (float)0.11428696, + (float)0.09801714, (float)0.08172107, (float)0.06540313, (float)0.04906767, (float)0.03271908, + (float)0.01636173 +}; + +// hybrib Hanning & flat window: for 20ms +static const float kBlocks320w512[512] = { + (float)0.00000000, (float)0.00818114, (float)0.01636173, (float)0.02454123, (float)0.03271908, + (float)0.04089475, (float)0.04906767, (float)0.05723732, (float)0.06540313, (float)0.07356456, + (float)0.08172107, (float)0.08987211, (float)0.09801714, (float)0.10615561, (float)0.11428696, + (float)0.12241068, (float)0.13052619, (float)0.13863297, (float)0.14673047, (float)0.15481816, + (float)0.16289547, (float)0.17096189, (float)0.17901686, (float)0.18705985, (float)0.19509032, + (float)0.20310773, (float)0.21111155, (float)0.21910124, (float)0.22707626, (float)0.23503609, + (float)0.24298018, (float)0.25090801, (float)0.25881905, (float)0.26671276, (float)0.27458862, + (float)0.28244610, (float)0.29028468, (float)0.29810383, (float)0.30590302, (float)0.31368174, + (float)0.32143947, (float)0.32917568, (float)0.33688985, (float)0.34458148, (float)0.35225005, + (float)0.35989504, (float)0.36751594, (float)0.37511224, (float)0.38268343, (float)0.39022901, + (float)0.39774847, (float)0.40524131, (float)0.41270703, (float)0.42014512, (float)0.42755509, + (float)0.43493645, (float)0.44228869, (float)0.44961133, (float)0.45690388, (float)0.46416584, + (float)0.47139674, (float)0.47859608, (float)0.48576339, (float)0.49289819, (float)0.50000000, + (float)0.50706834, (float)0.51410274, (float)0.52110274, (float)0.52806785, (float)0.53499762, + (float)0.54189158, (float)0.54874927, (float)0.55557023, (float)0.56235401, (float)0.56910015, + (float)0.57580819, (float)0.58247770, (float)0.58910822, (float)0.59569930, (float)0.60225052, + (float)0.60876143, (float)0.61523159, (float)0.62166057, (float)0.62804795, (float)0.63439328, + (float)0.64069616, (float)0.64695615, (float)0.65317284, (float)0.65934582, (float)0.66547466, + (float)0.67155895, (float)0.67759830, (float)0.68359230, (float)0.68954054, (float)0.69544264, + (float)0.70129818, (float)0.70710678, (float)0.71286806, (float)0.71858162, (float)0.72424708, + (float)0.72986407, (float)0.73543221, (float)0.74095113, (float)0.74642045, (float)0.75183981, + (float)0.75720885, (float)0.76252720, (float)0.76779452, (float)0.77301045, (float)0.77817464, + (float)0.78328675, (float)0.78834643, (float)0.79335334, (float)0.79830715, (float)0.80320753, + (float)0.80805415, (float)0.81284668, (float)0.81758481, (float)0.82226822, (float)0.82689659, + (float)0.83146961, (float)0.83598698, (float)0.84044840, (float)0.84485357, (float)0.84920218, + (float)0.85349396, (float)0.85772861, (float)0.86190585, (float)0.86602540, (float)0.87008699, + (float)0.87409034, (float)0.87803519, (float)0.88192126, (float)0.88574831, (float)0.88951608, + (float)0.89322430, (float)0.89687274, (float)0.90046115, (float)0.90398929, (float)0.90745693, + (float)0.91086382, (float)0.91420976, (float)0.91749450, (float)0.92071783, (float)0.92387953, + (float)0.92697940, (float)0.93001722, (float)0.93299280, (float)0.93590593, (float)0.93875641, + (float)0.94154407, (float)0.94426870, (float)0.94693013, (float)0.94952818, (float)0.95206268, + (float)0.95453345, (float)0.95694034, (float)0.95928317, (float)0.96156180, (float)0.96377607, + (float)0.96592583, (float)0.96801094, (float)0.97003125, (float)0.97198664, (float)0.97387698, + (float)0.97570213, (float)0.97746197, (float)0.97915640, (float)0.98078528, (float)0.98234852, + (float)0.98384601, (float)0.98527764, (float)0.98664333, (float)0.98794298, (float)0.98917651, + (float)0.99034383, (float)0.99144486, (float)0.99247953, (float)0.99344778, (float)0.99434953, + (float)0.99518473, (float)0.99595331, (float)0.99665524, (float)0.99729046, (float)0.99785892, + (float)0.99836060, (float)0.99879546, (float)0.99916346, (float)0.99946459, (float)0.99969882, + (float)0.99986614, (float)0.99996653, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, (float)1.00000000, + (float)1.00000000, (float)0.99996653, (float)0.99986614, (float)0.99969882, (float)0.99946459, + (float)0.99916346, (float)0.99879546, (float)0.99836060, (float)0.99785892, (float)0.99729046, + (float)0.99665524, (float)0.99595331, (float)0.99518473, (float)0.99434953, (float)0.99344778, + (float)0.99247953, (float)0.99144486, (float)0.99034383, (float)0.98917651, (float)0.98794298, + (float)0.98664333, (float)0.98527764, (float)0.98384601, (float)0.98234852, (float)0.98078528, + (float)0.97915640, (float)0.97746197, (float)0.97570213, (float)0.97387698, (float)0.97198664, + (float)0.97003125, (float)0.96801094, (float)0.96592583, (float)0.96377607, (float)0.96156180, + (float)0.95928317, (float)0.95694034, (float)0.95453345, (float)0.95206268, (float)0.94952818, + (float)0.94693013, (float)0.94426870, (float)0.94154407, (float)0.93875641, (float)0.93590593, + (float)0.93299280, (float)0.93001722, (float)0.92697940, (float)0.92387953, (float)0.92071783, + (float)0.91749450, (float)0.91420976, (float)0.91086382, (float)0.90745693, (float)0.90398929, + (float)0.90046115, (float)0.89687274, (float)0.89322430, (float)0.88951608, (float)0.88574831, + (float)0.88192126, (float)0.87803519, (float)0.87409034, (float)0.87008699, (float)0.86602540, + (float)0.86190585, (float)0.85772861, (float)0.85349396, (float)0.84920218, (float)0.84485357, + (float)0.84044840, (float)0.83598698, (float)0.83146961, (float)0.82689659, (float)0.82226822, + (float)0.81758481, (float)0.81284668, (float)0.80805415, (float)0.80320753, (float)0.79830715, + (float)0.79335334, (float)0.78834643, (float)0.78328675, (float)0.77817464, (float)0.77301045, + (float)0.76779452, (float)0.76252720, (float)0.75720885, (float)0.75183981, (float)0.74642045, + (float)0.74095113, (float)0.73543221, (float)0.72986407, (float)0.72424708, (float)0.71858162, + (float)0.71286806, (float)0.70710678, (float)0.70129818, (float)0.69544264, (float)0.68954054, + (float)0.68359230, (float)0.67759830, (float)0.67155895, (float)0.66547466, (float)0.65934582, + (float)0.65317284, (float)0.64695615, (float)0.64069616, (float)0.63439328, (float)0.62804795, + (float)0.62166057, (float)0.61523159, (float)0.60876143, (float)0.60225052, (float)0.59569930, + (float)0.58910822, (float)0.58247770, (float)0.57580819, (float)0.56910015, (float)0.56235401, + (float)0.55557023, (float)0.54874927, (float)0.54189158, (float)0.53499762, (float)0.52806785, + (float)0.52110274, (float)0.51410274, (float)0.50706834, (float)0.50000000, (float)0.49289819, + (float)0.48576339, (float)0.47859608, (float)0.47139674, (float)0.46416584, (float)0.45690388, + (float)0.44961133, (float)0.44228869, (float)0.43493645, (float)0.42755509, (float)0.42014512, + (float)0.41270703, (float)0.40524131, (float)0.39774847, (float)0.39022901, (float)0.38268343, + (float)0.37511224, (float)0.36751594, (float)0.35989504, (float)0.35225005, (float)0.34458148, + (float)0.33688985, (float)0.32917568, (float)0.32143947, (float)0.31368174, (float)0.30590302, + (float)0.29810383, (float)0.29028468, (float)0.28244610, (float)0.27458862, (float)0.26671276, + (float)0.25881905, (float)0.25090801, (float)0.24298018, (float)0.23503609, (float)0.22707626, + (float)0.21910124, (float)0.21111155, (float)0.20310773, (float)0.19509032, (float)0.18705985, + (float)0.17901686, (float)0.17096189, (float)0.16289547, (float)0.15481816, (float)0.14673047, + (float)0.13863297, (float)0.13052619, (float)0.12241068, (float)0.11428696, (float)0.10615561, + (float)0.09801714, (float)0.08987211, (float)0.08172107, (float)0.07356456, (float)0.06540313, + (float)0.05723732, (float)0.04906767, (float)0.04089475, (float)0.03271908, (float)0.02454123, + (float)0.01636173, (float)0.00818114 +}; + + +// Hanning window: for 15ms at 16kHz with symmetric zeros +static const float kBlocks240w512[512] = { + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00654494, (float)0.01308960, (float)0.01963369, + (float)0.02617695, (float)0.03271908, (float)0.03925982, (float)0.04579887, (float)0.05233596, + (float)0.05887080, (float)0.06540313, (float)0.07193266, (float)0.07845910, (float)0.08498218, + (float)0.09150162, (float)0.09801714, (float)0.10452846, (float)0.11103531, (float)0.11753740, + (float)0.12403446, (float)0.13052620, (float)0.13701233, (float)0.14349262, (float)0.14996676, + (float)0.15643448, (float)0.16289547, (float)0.16934951, (float)0.17579629, (float)0.18223552, + (float)0.18866697, (float)0.19509032, (float)0.20150533, (float)0.20791170, (float)0.21430916, + (float)0.22069745, (float)0.22707628, (float)0.23344538, (float)0.23980446, (float)0.24615330, + (float)0.25249159, (float)0.25881904, (float)0.26513544, (float)0.27144045, (float)0.27773386, + (float)0.28401536, (float)0.29028466, (float)0.29654160, (float)0.30278578, (float)0.30901700, + (float)0.31523499, (float)0.32143945, (float)0.32763019, (float)0.33380687, (float)0.33996925, + (float)0.34611708, (float)0.35225007, (float)0.35836795, (float)0.36447051, (float)0.37055743, + (float)0.37662852, (float)0.38268346, (float)0.38872197, (float)0.39474389, (float)0.40074885, + (float)0.40673664, (float)0.41270703, (float)0.41865975, (float)0.42459452, (float)0.43051112, + (float)0.43640924, (float)0.44228873, (float)0.44814920, (float)0.45399052, (float)0.45981237, + (float)0.46561453, (float)0.47139674, (float)0.47715878, (float)0.48290035, (float)0.48862126, + (float)0.49432120, (float)0.50000000, (float)0.50565743, (float)0.51129311, (float)0.51690692, + (float)0.52249855, (float)0.52806789, (float)0.53361452, (float)0.53913832, (float)0.54463905, + (float)0.55011642, (float)0.55557024, (float)0.56100029, (float)0.56640625, (float)0.57178795, + (float)0.57714522, (float)0.58247769, (float)0.58778524, (float)0.59306765, (float)0.59832460, + (float)0.60355598, (float)0.60876143, (float)0.61394083, (float)0.61909395, (float)0.62422055, + (float)0.62932038, (float)0.63439333, (float)0.63943899, (float)0.64445734, (float)0.64944810, + (float)0.65441096, (float)0.65934587, (float)0.66425246, (float)0.66913062, (float)0.67398012, + (float)0.67880076, (float)0.68359232, (float)0.68835455, (float)0.69308740, (float)0.69779050, + (float)0.70246369, (float)0.70710677, (float)0.71171963, (float)0.71630198, (float)0.72085363, + (float)0.72537440, (float)0.72986406, (float)0.73432255, (float)0.73874950, (float)0.74314487, + (float)0.74750835, (float)0.75183982, (float)0.75613910, (float)0.76040596, (float)0.76464027, + (float)0.76884186, (float)0.77301043, (float)0.77714598, (float)0.78124821, (float)0.78531694, + (float)0.78935206, (float)0.79335338, (float)0.79732066, (float)0.80125386, (float)0.80515265, + (float)0.80901700, (float)0.81284672, (float)0.81664157, (float)0.82040149, (float)0.82412618, + (float)0.82781565, (float)0.83146966, (float)0.83508795, (float)0.83867061, (float)0.84221727, + (float)0.84572780, (float)0.84920216, (float)0.85264021, (float)0.85604161, (float)0.85940641, + (float)0.86273444, (float)0.86602545, (float)0.86927933, (float)0.87249607, (float)0.87567532, + (float)0.87881714, (float)0.88192129, (float)0.88498765, (float)0.88801610, (float)0.89100653, + (float)0.89395881, (float)0.89687276, (float)0.89974827, (float)0.90258533, (float)0.90538365, + (float)0.90814316, (float)0.91086388, (float)0.91354549, (float)0.91618794, (float)0.91879123, + (float)0.92135513, (float)0.92387950, (float)0.92636442, (float)0.92880958, (float)0.93121493, + (float)0.93358046, (float)0.93590593, (float)0.93819135, (float)0.94043654, (float)0.94264150, + (float)0.94480604, (float)0.94693011, (float)0.94901365, (float)0.95105654, (float)0.95305866, + (float)0.95501995, (float)0.95694035, (float)0.95881975, (float)0.96065807, (float)0.96245527, + (float)0.96421117, (float)0.96592581, (float)0.96759909, (float)0.96923089, (float)0.97082120, + (float)0.97236991, (float)0.97387701, (float)0.97534233, (float)0.97676587, (float)0.97814763, + (float)0.97948742, (float)0.98078531, (float)0.98204112, (float)0.98325491, (float)0.98442656, + (float)0.98555607, (float)0.98664331, (float)0.98768836, (float)0.98869103, (float)0.98965138, + (float)0.99056935, (float)0.99144489, (float)0.99227792, (float)0.99306846, (float)0.99381649, + (float)0.99452192, (float)0.99518472, (float)0.99580491, (float)0.99638247, (float)0.99691731, + (float)0.99740952, (float)0.99785894, (float)0.99826562, (float)0.99862951, (float)0.99895066, + (float)0.99922901, (float)0.99946457, (float)0.99965733, (float)0.99980724, (float)0.99991435, + (float)0.99997860, (float)1.00000000, (float)0.99997860, (float)0.99991435, (float)0.99980724, + (float)0.99965733, (float)0.99946457, (float)0.99922901, (float)0.99895066, (float)0.99862951, + (float)0.99826562, (float)0.99785894, (float)0.99740946, (float)0.99691731, (float)0.99638247, + (float)0.99580491, (float)0.99518472, (float)0.99452192, (float)0.99381644, (float)0.99306846, + (float)0.99227792, (float)0.99144489, (float)0.99056935, (float)0.98965138, (float)0.98869103, + (float)0.98768836, (float)0.98664331, (float)0.98555607, (float)0.98442656, (float)0.98325491, + (float)0.98204112, (float)0.98078525, (float)0.97948742, (float)0.97814757, (float)0.97676587, + (float)0.97534227, (float)0.97387695, (float)0.97236991, (float)0.97082120, (float)0.96923089, + (float)0.96759909, (float)0.96592581, (float)0.96421117, (float)0.96245521, (float)0.96065807, + (float)0.95881969, (float)0.95694029, (float)0.95501995, (float)0.95305860, (float)0.95105648, + (float)0.94901365, (float)0.94693011, (float)0.94480604, (float)0.94264150, (float)0.94043654, + (float)0.93819129, (float)0.93590593, (float)0.93358046, (float)0.93121493, (float)0.92880952, + (float)0.92636436, (float)0.92387950, (float)0.92135507, (float)0.91879123, (float)0.91618794, + (float)0.91354543, (float)0.91086382, (float)0.90814310, (float)0.90538365, (float)0.90258527, + (float)0.89974827, (float)0.89687276, (float)0.89395875, (float)0.89100647, (float)0.88801610, + (float)0.88498759, (float)0.88192123, (float)0.87881714, (float)0.87567532, (float)0.87249595, + (float)0.86927933, (float)0.86602539, (float)0.86273432, (float)0.85940641, (float)0.85604161, + (float)0.85264009, (float)0.84920216, (float)0.84572780, (float)0.84221715, (float)0.83867055, + (float)0.83508795, (float)0.83146954, (float)0.82781565, (float)0.82412612, (float)0.82040137, + (float)0.81664157, (float)0.81284660, (float)0.80901700, (float)0.80515265, (float)0.80125374, + (float)0.79732066, (float)0.79335332, (float)0.78935200, (float)0.78531694, (float)0.78124815, + (float)0.77714586, (float)0.77301049, (float)0.76884180, (float)0.76464021, (float)0.76040596, + (float)0.75613904, (float)0.75183970, (float)0.74750835, (float)0.74314481, (float)0.73874938, + (float)0.73432249, (float)0.72986400, (float)0.72537428, (float)0.72085363, (float)0.71630186, + (float)0.71171951, (float)0.70710677, (float)0.70246363, (float)0.69779032, (float)0.69308734, + (float)0.68835449, (float)0.68359220, (float)0.67880070, (float)0.67398006, (float)0.66913044, + (float)0.66425240, (float)0.65934575, (float)0.65441096, (float)0.64944804, (float)0.64445722, + (float)0.63943905, (float)0.63439327, (float)0.62932026, (float)0.62422055, (float)0.61909389, + (float)0.61394072, (float)0.60876143, (float)0.60355592, (float)0.59832448, (float)0.59306765, + (float)0.58778518, (float)0.58247757, (float)0.57714522, (float)0.57178789, (float)0.56640613, + (float)0.56100023, (float)0.55557019, (float)0.55011630, (float)0.54463905, (float)0.53913826, + (float)0.53361434, (float)0.52806783, (float)0.52249849, (float)0.51690674, (float)0.51129305, + (float)0.50565726, (float)0.50000006, (float)0.49432117, (float)0.48862115, (float)0.48290038, + (float)0.47715873, (float)0.47139663, (float)0.46561456, (float)0.45981231, (float)0.45399037, + (float)0.44814920, (float)0.44228864, (float)0.43640912, (float)0.43051112, (float)0.42459446, + (float)0.41865960, (float)0.41270703, (float)0.40673658, (float)0.40074870, (float)0.39474386, + (float)0.38872188, (float)0.38268328, (float)0.37662849, (float)0.37055734, (float)0.36447033, + (float)0.35836792, (float)0.35224995, (float)0.34611690, (float)0.33996922, (float)0.33380675, + (float)0.32763001, (float)0.32143945, (float)0.31523487, (float)0.30901679, (float)0.30278572, + (float)0.29654145, (float)0.29028472, (float)0.28401530, (float)0.27773371, (float)0.27144048, + (float)0.26513538, (float)0.25881892, (float)0.25249159, (float)0.24615324, (float)0.23980433, + (float)0.23344538, (float)0.22707619, (float)0.22069728, (float)0.21430916, (float)0.20791161, + (float)0.20150517, (float)0.19509031, (float)0.18866688, (float)0.18223536, (float)0.17579627, + (float)0.16934940, (float)0.16289529, (float)0.15643445, (float)0.14996666, (float)0.14349243, + (float)0.13701232, (float)0.13052608, (float)0.12403426, (float)0.11753736, (float)0.11103519, + (float)0.10452849, (float)0.09801710, (float)0.09150149, (float)0.08498220, (float)0.07845904, + (float)0.07193252, (float)0.06540315, (float)0.05887074, (float)0.05233581, (float)0.04579888, + (float)0.03925974, (float)0.03271893, (float)0.02617695, (float)0.01963361, (float)0.01308943, + (float)0.00654493, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000 +}; + + +// Hanning window: for 30ms with 1024 fft with symmetric zeros at 16kHz +static const float kBlocks480w1024[1024] = { + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00327249, (float)0.00654494, + (float)0.00981732, (float)0.01308960, (float)0.01636173, (float)0.01963369, (float)0.02290544, + (float)0.02617695, (float)0.02944817, (float)0.03271908, (float)0.03598964, (float)0.03925982, + (float)0.04252957, (float)0.04579887, (float)0.04906768, (float)0.05233596, (float)0.05560368, + (float)0.05887080, (float)0.06213730, (float)0.06540313, (float)0.06866825, (float)0.07193266, + (float)0.07519628, (float)0.07845910, (float)0.08172107, (float)0.08498218, (float)0.08824237, + (float)0.09150162, (float)0.09475989, (float)0.09801714, (float)0.10127335, (float)0.10452846, + (float)0.10778246, (float)0.11103531, (float)0.11428697, (float)0.11753740, (float)0.12078657, + (float)0.12403446, (float)0.12728101, (float)0.13052620, (float)0.13376999, (float)0.13701233, + (float)0.14025325, (float)0.14349262, (float)0.14673047, (float)0.14996676, (float)0.15320145, + (float)0.15643448, (float)0.15966582, (float)0.16289547, (float)0.16612339, (float)0.16934951, + (float)0.17257382, (float)0.17579629, (float)0.17901687, (float)0.18223552, (float)0.18545224, + (float)0.18866697, (float)0.19187967, (float)0.19509032, (float)0.19829889, (float)0.20150533, + (float)0.20470962, (float)0.20791170, (float)0.21111156, (float)0.21430916, (float)0.21750447, + (float)0.22069745, (float)0.22388805, (float)0.22707628, (float)0.23026206, (float)0.23344538, + (float)0.23662618, (float)0.23980446, (float)0.24298020, (float)0.24615330, (float)0.24932377, + (float)0.25249159, (float)0.25565669, (float)0.25881904, (float)0.26197866, (float)0.26513544, + (float)0.26828939, (float)0.27144045, (float)0.27458861, (float)0.27773386, (float)0.28087610, + (float)0.28401536, (float)0.28715158, (float)0.29028466, (float)0.29341471, (float)0.29654160, + (float)0.29966527, (float)0.30278578, (float)0.30590302, (float)0.30901700, (float)0.31212768, + (float)0.31523499, (float)0.31833893, (float)0.32143945, (float)0.32453656, (float)0.32763019, + (float)0.33072028, (float)0.33380687, (float)0.33688986, (float)0.33996925, (float)0.34304500, + (float)0.34611708, (float)0.34918544, (float)0.35225007, (float)0.35531089, (float)0.35836795, + (float)0.36142117, (float)0.36447051, (float)0.36751595, (float)0.37055743, (float)0.37359497, + (float)0.37662852, (float)0.37965801, (float)0.38268346, (float)0.38570479, (float)0.38872197, + (float)0.39173502, (float)0.39474389, (float)0.39774847, (float)0.40074885, (float)0.40374491, + (float)0.40673664, (float)0.40972406, (float)0.41270703, (float)0.41568562, (float)0.41865975, + (float)0.42162940, (float)0.42459452, (float)0.42755508, (float)0.43051112, (float)0.43346250, + (float)0.43640924, (float)0.43935132, (float)0.44228873, (float)0.44522133, (float)0.44814920, + (float)0.45107228, (float)0.45399052, (float)0.45690390, (float)0.45981237, (float)0.46271592, + (float)0.46561453, (float)0.46850815, (float)0.47139674, (float)0.47428030, (float)0.47715878, + (float)0.48003215, (float)0.48290035, (float)0.48576337, (float)0.48862126, (float)0.49147385, + (float)0.49432120, (float)0.49716330, (float)0.50000000, (float)0.50283140, (float)0.50565743, + (float)0.50847799, (float)0.51129311, (float)0.51410276, (float)0.51690692, (float)0.51970553, + (float)0.52249855, (float)0.52528602, (float)0.52806789, (float)0.53084403, (float)0.53361452, + (float)0.53637928, (float)0.53913832, (float)0.54189163, (float)0.54463905, (float)0.54738063, + (float)0.55011642, (float)0.55284631, (float)0.55557024, (float)0.55828828, (float)0.56100029, + (float)0.56370628, (float)0.56640625, (float)0.56910014, (float)0.57178795, (float)0.57446963, + (float)0.57714522, (float)0.57981455, (float)0.58247769, (float)0.58513463, (float)0.58778524, + (float)0.59042960, (float)0.59306765, (float)0.59569931, (float)0.59832460, (float)0.60094351, + (float)0.60355598, (float)0.60616195, (float)0.60876143, (float)0.61135441, (float)0.61394083, + (float)0.61652070, (float)0.61909395, (float)0.62166059, (float)0.62422055, (float)0.62677383, + (float)0.62932038, (float)0.63186020, (float)0.63439333, (float)0.63691956, (float)0.63943899, + (float)0.64195162, (float)0.64445734, (float)0.64695615, (float)0.64944810, (float)0.65193301, + (float)0.65441096, (float)0.65688187, (float)0.65934587, (float)0.66180271, (float)0.66425246, + (float)0.66669512, (float)0.66913062, (float)0.67155898, (float)0.67398012, (float)0.67639405, + (float)0.67880076, (float)0.68120021, (float)0.68359232, (float)0.68597710, (float)0.68835455, + (float)0.69072467, (float)0.69308740, (float)0.69544262, (float)0.69779050, (float)0.70013082, + (float)0.70246369, (float)0.70478904, (float)0.70710677, (float)0.70941699, (float)0.71171963, + (float)0.71401459, (float)0.71630198, (float)0.71858168, (float)0.72085363, (float)0.72311789, + (float)0.72537440, (float)0.72762316, (float)0.72986406, (float)0.73209721, (float)0.73432255, + (float)0.73653996, (float)0.73874950, (float)0.74095118, (float)0.74314487, (float)0.74533057, + (float)0.74750835, (float)0.74967808, (float)0.75183982, (float)0.75399351, (float)0.75613910, + (float)0.75827658, (float)0.76040596, (float)0.76252723, (float)0.76464027, (float)0.76674515, + (float)0.76884186, (float)0.77093029, (float)0.77301043, (float)0.77508241, (float)0.77714598, + (float)0.77920127, (float)0.78124821, (float)0.78328675, (float)0.78531694, (float)0.78733873, + (float)0.78935206, (float)0.79135692, (float)0.79335338, (float)0.79534125, (float)0.79732066, + (float)0.79929149, (float)0.80125386, (float)0.80320752, (float)0.80515265, (float)0.80708915, + (float)0.80901700, (float)0.81093621, (float)0.81284672, (float)0.81474853, (float)0.81664157, + (float)0.81852591, (float)0.82040149, (float)0.82226825, (float)0.82412618, (float)0.82597536, + (float)0.82781565, (float)0.82964706, (float)0.83146966, (float)0.83328325, (float)0.83508795, + (float)0.83688378, (float)0.83867061, (float)0.84044838, (float)0.84221727, (float)0.84397703, + (float)0.84572780, (float)0.84746957, (float)0.84920216, (float)0.85092574, (float)0.85264021, + (float)0.85434544, (float)0.85604161, (float)0.85772866, (float)0.85940641, (float)0.86107504, + (float)0.86273444, (float)0.86438453, (float)0.86602545, (float)0.86765707, (float)0.86927933, + (float)0.87089235, (float)0.87249607, (float)0.87409031, (float)0.87567532, (float)0.87725097, + (float)0.87881714, (float)0.88037390, (float)0.88192129, (float)0.88345921, (float)0.88498765, + (float)0.88650668, (float)0.88801610, (float)0.88951612, (float)0.89100653, (float)0.89248741, + (float)0.89395881, (float)0.89542055, (float)0.89687276, (float)0.89831537, (float)0.89974827, + (float)0.90117162, (float)0.90258533, (float)0.90398932, (float)0.90538365, (float)0.90676826, + (float)0.90814316, (float)0.90950841, (float)0.91086388, (float)0.91220951, (float)0.91354549, + (float)0.91487163, (float)0.91618794, (float)0.91749454, (float)0.91879123, (float)0.92007810, + (float)0.92135513, (float)0.92262226, (float)0.92387950, (float)0.92512691, (float)0.92636442, + (float)0.92759192, (float)0.92880958, (float)0.93001723, (float)0.93121493, (float)0.93240267, + (float)0.93358046, (float)0.93474817, (float)0.93590593, (float)0.93705362, (float)0.93819135, + (float)0.93931901, (float)0.94043654, (float)0.94154406, (float)0.94264150, (float)0.94372880, + (float)0.94480604, (float)0.94587320, (float)0.94693011, (float)0.94797695, (float)0.94901365, + (float)0.95004016, (float)0.95105654, (float)0.95206273, (float)0.95305866, (float)0.95404440, + (float)0.95501995, (float)0.95598525, (float)0.95694035, (float)0.95788521, (float)0.95881975, + (float)0.95974404, (float)0.96065807, (float)0.96156180, (float)0.96245527, (float)0.96333838, + (float)0.96421117, (float)0.96507370, (float)0.96592581, (float)0.96676767, (float)0.96759909, + (float)0.96842021, (float)0.96923089, (float)0.97003126, (float)0.97082120, (float)0.97160077, + (float)0.97236991, (float)0.97312868, (float)0.97387701, (float)0.97461486, (float)0.97534233, + (float)0.97605932, (float)0.97676587, (float)0.97746199, (float)0.97814763, (float)0.97882277, + (float)0.97948742, (float)0.98014158, (float)0.98078531, (float)0.98141843, (float)0.98204112, + (float)0.98265332, (float)0.98325491, (float)0.98384601, (float)0.98442656, (float)0.98499662, + (float)0.98555607, (float)0.98610497, (float)0.98664331, (float)0.98717111, (float)0.98768836, + (float)0.98819500, (float)0.98869103, (float)0.98917651, (float)0.98965138, (float)0.99011570, + (float)0.99056935, (float)0.99101239, (float)0.99144489, (float)0.99186671, (float)0.99227792, + (float)0.99267852, (float)0.99306846, (float)0.99344778, (float)0.99381649, (float)0.99417448, + (float)0.99452192, (float)0.99485862, (float)0.99518472, (float)0.99550015, (float)0.99580491, + (float)0.99609905, (float)0.99638247, (float)0.99665523, (float)0.99691731, (float)0.99716878, + (float)0.99740952, (float)0.99763954, (float)0.99785894, (float)0.99806762, (float)0.99826562, + (float)0.99845290, (float)0.99862951, (float)0.99879545, (float)0.99895066, (float)0.99909520, + (float)0.99922901, (float)0.99935216, (float)0.99946457, (float)0.99956632, (float)0.99965733, + (float)0.99973762, (float)0.99980724, (float)0.99986613, (float)0.99991435, (float)0.99995178, + (float)0.99997860, (float)0.99999464, (float)1.00000000, (float)0.99999464, (float)0.99997860, + (float)0.99995178, (float)0.99991435, (float)0.99986613, (float)0.99980724, (float)0.99973762, + (float)0.99965733, (float)0.99956632, (float)0.99946457, (float)0.99935216, (float)0.99922901, + (float)0.99909520, (float)0.99895066, (float)0.99879545, (float)0.99862951, (float)0.99845290, + (float)0.99826562, (float)0.99806762, (float)0.99785894, (float)0.99763954, (float)0.99740946, + (float)0.99716872, (float)0.99691731, (float)0.99665523, (float)0.99638247, (float)0.99609905, + (float)0.99580491, (float)0.99550015, (float)0.99518472, (float)0.99485862, (float)0.99452192, + (float)0.99417448, (float)0.99381644, (float)0.99344778, (float)0.99306846, (float)0.99267852, + (float)0.99227792, (float)0.99186671, (float)0.99144489, (float)0.99101239, (float)0.99056935, + (float)0.99011564, (float)0.98965138, (float)0.98917651, (float)0.98869103, (float)0.98819494, + (float)0.98768836, (float)0.98717111, (float)0.98664331, (float)0.98610497, (float)0.98555607, + (float)0.98499656, (float)0.98442656, (float)0.98384601, (float)0.98325491, (float)0.98265326, + (float)0.98204112, (float)0.98141843, (float)0.98078525, (float)0.98014158, (float)0.97948742, + (float)0.97882277, (float)0.97814757, (float)0.97746193, (float)0.97676587, (float)0.97605932, + (float)0.97534227, (float)0.97461486, (float)0.97387695, (float)0.97312862, (float)0.97236991, + (float)0.97160077, (float)0.97082120, (float)0.97003126, (float)0.96923089, (float)0.96842015, + (float)0.96759909, (float)0.96676761, (float)0.96592581, (float)0.96507365, (float)0.96421117, + (float)0.96333838, (float)0.96245521, (float)0.96156180, (float)0.96065807, (float)0.95974404, + (float)0.95881969, (float)0.95788515, (float)0.95694029, (float)0.95598525, (float)0.95501995, + (float)0.95404440, (float)0.95305860, (float)0.95206267, (float)0.95105648, (float)0.95004016, + (float)0.94901365, (float)0.94797695, (float)0.94693011, (float)0.94587314, (float)0.94480604, + (float)0.94372880, (float)0.94264150, (float)0.94154406, (float)0.94043654, (float)0.93931895, + (float)0.93819129, (float)0.93705362, (float)0.93590593, (float)0.93474817, (float)0.93358046, + (float)0.93240267, (float)0.93121493, (float)0.93001723, (float)0.92880952, (float)0.92759192, + (float)0.92636436, (float)0.92512691, (float)0.92387950, (float)0.92262226, (float)0.92135507, + (float)0.92007804, (float)0.91879123, (float)0.91749448, (float)0.91618794, (float)0.91487157, + (float)0.91354543, (float)0.91220951, (float)0.91086382, (float)0.90950835, (float)0.90814310, + (float)0.90676820, (float)0.90538365, (float)0.90398932, (float)0.90258527, (float)0.90117157, + (float)0.89974827, (float)0.89831525, (float)0.89687276, (float)0.89542055, (float)0.89395875, + (float)0.89248741, (float)0.89100647, (float)0.88951600, (float)0.88801610, (float)0.88650662, + (float)0.88498759, (float)0.88345915, (float)0.88192123, (float)0.88037384, (float)0.87881714, + (float)0.87725091, (float)0.87567532, (float)0.87409031, (float)0.87249595, (float)0.87089223, + (float)0.86927933, (float)0.86765701, (float)0.86602539, (float)0.86438447, (float)0.86273432, + (float)0.86107504, (float)0.85940641, (float)0.85772860, (float)0.85604161, (float)0.85434544, + (float)0.85264009, (float)0.85092574, (float)0.84920216, (float)0.84746951, (float)0.84572780, + (float)0.84397697, (float)0.84221715, (float)0.84044844, (float)0.83867055, (float)0.83688372, + (float)0.83508795, (float)0.83328319, (float)0.83146954, (float)0.82964706, (float)0.82781565, + (float)0.82597530, (float)0.82412612, (float)0.82226813, (float)0.82040137, (float)0.81852591, + (float)0.81664157, (float)0.81474847, (float)0.81284660, (float)0.81093609, (float)0.80901700, + (float)0.80708915, (float)0.80515265, (float)0.80320752, (float)0.80125374, (float)0.79929143, + (float)0.79732066, (float)0.79534125, (float)0.79335332, (float)0.79135686, (float)0.78935200, + (float)0.78733861, (float)0.78531694, (float)0.78328675, (float)0.78124815, (float)0.77920121, + (float)0.77714586, (float)0.77508223, (float)0.77301049, (float)0.77093029, (float)0.76884180, + (float)0.76674509, (float)0.76464021, (float)0.76252711, (float)0.76040596, (float)0.75827658, + (float)0.75613904, (float)0.75399339, (float)0.75183970, (float)0.74967796, (float)0.74750835, + (float)0.74533057, (float)0.74314481, (float)0.74095106, (float)0.73874938, (float)0.73653996, + (float)0.73432249, (float)0.73209721, (float)0.72986400, (float)0.72762305, (float)0.72537428, + (float)0.72311789, (float)0.72085363, (float)0.71858162, (float)0.71630186, (float)0.71401453, + (float)0.71171951, (float)0.70941705, (float)0.70710677, (float)0.70478898, (float)0.70246363, + (float)0.70013070, (float)0.69779032, (float)0.69544268, (float)0.69308734, (float)0.69072461, + (float)0.68835449, (float)0.68597704, (float)0.68359220, (float)0.68120021, (float)0.67880070, + (float)0.67639399, (float)0.67398006, (float)0.67155886, (float)0.66913044, (float)0.66669512, + (float)0.66425240, (float)0.66180259, (float)0.65934575, (float)0.65688181, (float)0.65441096, + (float)0.65193301, (float)0.64944804, (float)0.64695609, (float)0.64445722, (float)0.64195150, + (float)0.63943905, (float)0.63691956, (float)0.63439327, (float)0.63186014, (float)0.62932026, + (float)0.62677372, (float)0.62422055, (float)0.62166059, (float)0.61909389, (float)0.61652064, + (float)0.61394072, (float)0.61135429, (float)0.60876143, (float)0.60616189, (float)0.60355592, + (float)0.60094339, (float)0.59832448, (float)0.59569913, (float)0.59306765, (float)0.59042960, + (float)0.58778518, (float)0.58513451, (float)0.58247757, (float)0.57981461, (float)0.57714522, + (float)0.57446963, (float)0.57178789, (float)0.56910002, (float)0.56640613, (float)0.56370628, + (float)0.56100023, (float)0.55828822, (float)0.55557019, (float)0.55284619, (float)0.55011630, + (float)0.54738069, (float)0.54463905, (float)0.54189152, (float)0.53913826, (float)0.53637916, + (float)0.53361434, (float)0.53084403, (float)0.52806783, (float)0.52528596, (float)0.52249849, + (float)0.51970541, (float)0.51690674, (float)0.51410276, (float)0.51129305, (float)0.50847787, + (float)0.50565726, (float)0.50283122, (float)0.50000006, (float)0.49716327, (float)0.49432117, + (float)0.49147379, (float)0.48862115, (float)0.48576325, (float)0.48290038, (float)0.48003212, + (float)0.47715873, (float)0.47428021, (float)0.47139663, (float)0.46850798, (float)0.46561456, + (float)0.46271589, (float)0.45981231, (float)0.45690379, (float)0.45399037, (float)0.45107210, + (float)0.44814920, (float)0.44522130, (float)0.44228864, (float)0.43935123, (float)0.43640912, + (float)0.43346232, (float)0.43051112, (float)0.42755505, (float)0.42459446, (float)0.42162928, + (float)0.41865960, (float)0.41568545, (float)0.41270703, (float)0.40972400, (float)0.40673658, + (float)0.40374479, (float)0.40074870, (float)0.39774850, (float)0.39474386, (float)0.39173496, + (float)0.38872188, (float)0.38570464, (float)0.38268328, (float)0.37965804, (float)0.37662849, + (float)0.37359491, (float)0.37055734, (float)0.36751580, (float)0.36447033, (float)0.36142117, + (float)0.35836792, (float)0.35531086, (float)0.35224995, (float)0.34918529, (float)0.34611690, + (float)0.34304500, (float)0.33996922, (float)0.33688980, (float)0.33380675, (float)0.33072016, + (float)0.32763001, (float)0.32453656, (float)0.32143945, (float)0.31833887, (float)0.31523487, + (float)0.31212750, (float)0.30901679, (float)0.30590302, (float)0.30278572, (float)0.29966521, + (float)0.29654145, (float)0.29341453, (float)0.29028472, (float)0.28715155, (float)0.28401530, + (float)0.28087601, (float)0.27773371, (float)0.27458847, (float)0.27144048, (float)0.26828936, + (float)0.26513538, (float)0.26197854, (float)0.25881892, (float)0.25565651, (float)0.25249159, + (float)0.24932374, (float)0.24615324, (float)0.24298008, (float)0.23980433, (float)0.23662600, + (float)0.23344538, (float)0.23026201, (float)0.22707619, (float)0.22388794, (float)0.22069728, + (float)0.21750426, (float)0.21430916, (float)0.21111152, (float)0.20791161, (float)0.20470949, + (float)0.20150517, (float)0.19829892, (float)0.19509031, (float)0.19187963, (float)0.18866688, + (float)0.18545210, (float)0.18223536, (float)0.17901689, (float)0.17579627, (float)0.17257376, + (float)0.16934940, (float)0.16612324, (float)0.16289529, (float)0.15966584, (float)0.15643445, + (float)0.15320137, (float)0.14996666, (float)0.14673033, (float)0.14349243, (float)0.14025325, + (float)0.13701232, (float)0.13376991, (float)0.13052608, (float)0.12728085, (float)0.12403426, + (float)0.12078657, (float)0.11753736, (float)0.11428688, (float)0.11103519, (float)0.10778230, + (float)0.10452849, (float)0.10127334, (float)0.09801710, (float)0.09475980, (float)0.09150149, + (float)0.08824220, (float)0.08498220, (float)0.08172106, (float)0.07845904, (float)0.07519618, + (float)0.07193252, (float)0.06866808, (float)0.06540315, (float)0.06213728, (float)0.05887074, + (float)0.05560357, (float)0.05233581, (float)0.04906749, (float)0.04579888, (float)0.04252954, + (float)0.03925974, (float)0.03598953, (float)0.03271893, (float)0.02944798, (float)0.02617695, + (float)0.02290541, (float)0.01963361, (float)0.01636161, (float)0.01308943, (float)0.00981712, + (float)0.00654493, (float)0.00327244, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000, + (float)0.00000000, (float)0.00000000, (float)0.00000000, (float)0.00000000 +}; + +#endif // MODULES_AUDIO_PROCESSING_NS_MAIN_SOURCE_WINDOWS_PRIVATE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/render_queue_item_verifier.h b/third_party/libwebrtc/webrtc/modules/audio_processing/render_queue_item_verifier.h new file mode 100644 index 0000000000..b8aff4a107 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/render_queue_item_verifier.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_RENDER_QUEUE_ITEM_VERIFIER_H_ +#define MODULES_AUDIO_PROCESSING_RENDER_QUEUE_ITEM_VERIFIER_H_ + +#include + +namespace webrtc { + +// Functor to use when supplying a verifier function for the queue item +// verifcation. +template +class RenderQueueItemVerifier { + public: + explicit RenderQueueItemVerifier(size_t minimum_capacity) + : minimum_capacity_(minimum_capacity) {} + + bool operator()(const std::vector& v) const { + return v.capacity() >= minimum_capacity_; + } + + private: + size_t minimum_capacity_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_RENDER_QUEUE_ITEM_VERIFIER_H__ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.cc new file mode 100644 index 0000000000..b35c1558c7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.cc @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/residual_echo_detector.h" + +#include +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/atomicops.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/metrics.h" + +namespace { + +float Power(rtc::ArrayView input) { + if (input.empty()) { + return 0.f; + } + return std::inner_product(input.begin(), input.end(), input.begin(), 0.f) / + input.size(); +} + +constexpr size_t kLookbackFrames = 650; +// TODO(ivoc): Verify the size of this buffer. +constexpr size_t kRenderBufferSize = 30; +constexpr float kAlpha = 0.001f; +// 10 seconds of data, updated every 10 ms. +constexpr size_t kAggregationBufferSize = 10 * 100; + +} // namespace + +namespace webrtc { + +int ResidualEchoDetector::instance_count_ = 0; + +ResidualEchoDetector::ResidualEchoDetector() + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + render_buffer_(kRenderBufferSize), + render_power_(kLookbackFrames), + render_power_mean_(kLookbackFrames), + render_power_std_dev_(kLookbackFrames), + covariances_(kLookbackFrames), + recent_likelihood_max_(kAggregationBufferSize) {} + +ResidualEchoDetector::~ResidualEchoDetector() = default; + +void ResidualEchoDetector::AnalyzeRenderAudio( + rtc::ArrayView render_audio) { + // Dump debug data assuming 48 kHz sample rate (if this assumption is not + // valid the dumped audio will need to be converted offline accordingly). + data_dumper_->DumpWav("ed_render", render_audio.size(), render_audio.data(), + 48000, 1); + + if (render_buffer_.Size() == 0) { + frames_since_zero_buffer_size_ = 0; + } else if (frames_since_zero_buffer_size_ >= kRenderBufferSize) { + // This can happen in a few cases: at the start of a call, due to a glitch + // or due to clock drift. The excess capture value will be ignored. + // TODO(ivoc): Include how often this happens in APM stats. + render_buffer_.Pop(); + frames_since_zero_buffer_size_ = 0; + } + ++frames_since_zero_buffer_size_; + float power = Power(render_audio); + render_buffer_.Push(power); +} + +void ResidualEchoDetector::AnalyzeCaptureAudio( + rtc::ArrayView capture_audio) { + // Dump debug data assuming 48 kHz sample rate (if this assumption is not + // valid the dumped audio will need to be converted offline accordingly). + data_dumper_->DumpWav("ed_capture", capture_audio.size(), + capture_audio.data(), 48000, 1); + + if (first_process_call_) { + // On the first process call (so the start of a call), we must flush the + // render buffer, otherwise the render data will be delayed. + render_buffer_.Clear(); + first_process_call_ = false; + } + + // Get the next render value. + const rtc::Optional buffered_render_power = render_buffer_.Pop(); + if (!buffered_render_power) { + // This can happen in a few cases: at the start of a call, due to a glitch + // or due to clock drift. The excess capture value will be ignored. + // TODO(ivoc): Include how often this happens in APM stats. + return; + } + // Update the render statistics, and store the statistics in circular buffers. + render_statistics_.Update(*buffered_render_power); + RTC_DCHECK_LT(next_insertion_index_, kLookbackFrames); + render_power_[next_insertion_index_] = *buffered_render_power; + render_power_mean_[next_insertion_index_] = render_statistics_.mean(); + render_power_std_dev_[next_insertion_index_] = + render_statistics_.std_deviation(); + + // Get the next capture value, update capture statistics and add the relevant + // values to the buffers. + const float capture_power = Power(capture_audio); + capture_statistics_.Update(capture_power); + const float capture_mean = capture_statistics_.mean(); + const float capture_std_deviation = capture_statistics_.std_deviation(); + + // Update the covariance values and determine the new echo likelihood. + echo_likelihood_ = 0.f; + size_t read_index = next_insertion_index_; + + int best_delay = -1; + for (size_t delay = 0; delay < covariances_.size(); ++delay) { + RTC_DCHECK_LT(read_index, render_power_.size()); + covariances_[delay].Update(capture_power, capture_mean, + capture_std_deviation, render_power_[read_index], + render_power_mean_[read_index], + render_power_std_dev_[read_index]); + read_index = read_index > 0 ? read_index - 1 : kLookbackFrames - 1; + + if (covariances_[delay].normalized_cross_correlation() > echo_likelihood_) { + echo_likelihood_ = covariances_[delay].normalized_cross_correlation(); + best_delay = static_cast(delay); + } + } + // This is a temporary log message to help find the underlying cause for echo + // likelihoods > 1.0. + // TODO(ivoc): Remove once the issue is resolved. + if (echo_likelihood_ > 1.1f) { + // Make sure we don't spam the log. + if (log_counter_ < 5 && best_delay != -1) { + size_t read_index = kLookbackFrames + next_insertion_index_ - best_delay; + if (read_index >= kLookbackFrames) { + read_index -= kLookbackFrames; + } + RTC_DCHECK_LT(read_index, render_power_.size()); + RTC_LOG_F(LS_ERROR) << "Echo detector internal state: {" + << "Echo likelihood: " << echo_likelihood_ + << ", Best Delay: " << best_delay << ", Covariance: " + << covariances_[best_delay].covariance() + << ", Last capture power: " << capture_power + << ", Capture mean: " << capture_mean + << ", Capture_standard deviation: " + << capture_std_deviation << ", Last render power: " + << render_power_[read_index] + << ", Render mean: " << render_power_mean_[read_index] + << ", Render standard deviation: " + << render_power_std_dev_[read_index] + << ", Reliability: " << reliability_ << "}"; + log_counter_++; + } + } + RTC_DCHECK_LT(echo_likelihood_, 1.1f); + + reliability_ = (1.0f - kAlpha) * reliability_ + kAlpha * 1.0f; + echo_likelihood_ *= reliability_; + // This is a temporary fix to prevent echo likelihood values > 1.0. + // TODO(ivoc): Find the root cause of this issue and fix it. + echo_likelihood_ = std::min(echo_likelihood_, 1.0f); + int echo_percentage = static_cast(echo_likelihood_ * 100); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.ResidualEchoDetector.EchoLikelihood", + echo_percentage, 0, 100, 100 /* number of bins */); + + // Update the buffer of recent likelihood values. + recent_likelihood_max_.Update(echo_likelihood_); + + // Update the next insertion index. + next_insertion_index_ = next_insertion_index_ < (kLookbackFrames - 1) + ? next_insertion_index_ + 1 + : 0; +} + +void ResidualEchoDetector::Initialize() { + render_buffer_.Clear(); + std::fill(render_power_.begin(), render_power_.end(), 0.f); + std::fill(render_power_mean_.begin(), render_power_mean_.end(), 0.f); + std::fill(render_power_std_dev_.begin(), render_power_std_dev_.end(), 0.f); + render_statistics_.Clear(); + capture_statistics_.Clear(); + recent_likelihood_max_.Clear(); + for (auto& cov : covariances_) { + cov.Clear(); + } + echo_likelihood_ = 0.f; + next_insertion_index_ = 0; + reliability_ = 0.f; +} + +void ResidualEchoDetector::PackRenderAudioBuffer( + AudioBuffer* audio, + std::vector* packed_buffer) { + packed_buffer->clear(); + packed_buffer->insert(packed_buffer->end(), audio->channels_f()[0], + audio->channels_f()[0] + audio->num_frames()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.h b/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.h new file mode 100644 index 0000000000..de1b989110 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ +#define MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/echo_detector/circular_buffer.h" +#include "modules/audio_processing/echo_detector/mean_variance_estimator.h" +#include "modules/audio_processing/echo_detector/moving_max.h" +#include "modules/audio_processing/echo_detector/normalized_covariance_estimator.h" + +namespace webrtc { + +class ApmDataDumper; +class AudioBuffer; +class EchoDetector; + +class ResidualEchoDetector { + public: + ResidualEchoDetector(); + ~ResidualEchoDetector(); + + // This function should be called while holding the render lock. + void AnalyzeRenderAudio(rtc::ArrayView render_audio); + + // This function should be called while holding the capture lock. + void AnalyzeCaptureAudio(rtc::ArrayView capture_audio); + + // This function should be called while holding the capture lock. + void Initialize(); + + // This function is for testing purposes only. + void SetReliabilityForTest(float value) { reliability_ = value; } + + static void PackRenderAudioBuffer(AudioBuffer* audio, + std::vector* packed_buffer); + + // This function should be called while holding the capture lock. + float echo_likelihood() const { return echo_likelihood_; } + + float echo_likelihood_recent_max() const { + return recent_likelihood_max_.max(); + } + + private: + static int instance_count_; + std::unique_ptr data_dumper_; + // Keep track if the |Process| function has been previously called. + bool first_process_call_ = true; + // Buffer for storing the power of incoming farend buffers. This is needed for + // cases where calls to BufferFarend and Process are jittery. + CircularBuffer render_buffer_; + // Count how long ago it was that the size of |render_buffer_| was zero. This + // value is also reset to zero when clock drift is detected and a value from + // the renderbuffer is discarded, even though the buffer is not actually zero + // at that point. This is done to avoid repeatedly removing elements in this + // situation. + size_t frames_since_zero_buffer_size_ = 0; + + // Circular buffers containing delayed versions of the power, mean and + // standard deviation, for calculating the delayed covariance values. + std::vector render_power_; + std::vector render_power_mean_; + std::vector render_power_std_dev_; + // Covariance estimates for different delay values. + std::vector covariances_; + // Index where next element should be inserted in all of the above circular + // buffers. + size_t next_insertion_index_ = 0; + + MeanVarianceEstimator render_statistics_; + MeanVarianceEstimator capture_statistics_; + // Current echo likelihood. + float echo_likelihood_ = 0.f; + // Reliability of the current likelihood. + float reliability_ = 0.f; + MovingMax recent_likelihood_max_; + + int log_counter_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector_unittest.cc new file mode 100644 index 0000000000..baf83ba4aa --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/residual_echo_detector_unittest.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/residual_echo_detector.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(ResidualEchoDetectorTests, Echo) { + ResidualEchoDetector echo_detector; + echo_detector.SetReliabilityForTest(1.0f); + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal has a delay of 10 frames w.r.t. the render + // signal, but is otherwise identical. Both signals are periodic with a 20 + // frame interval. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + echo_detector.AnalyzeCaptureAudio(zeros); + } else if (i % 20 == 10) { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(zeros); + } + } + // We expect to detect echo with near certain likelihood. + EXPECT_NEAR(1.f, echo_detector.echo_likelihood(), 0.01f); +} + +TEST(ResidualEchoDetectorTests, NoEcho) { + ResidualEchoDetector echo_detector; + echo_detector.SetReliabilityForTest(1.0f); + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal is always zero, so no echo should be + // detected. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + } + echo_detector.AnalyzeCaptureAudio(zeros); + } + // We expect to not detect any echo. + EXPECT_NEAR(0.f, echo_detector.echo_likelihood(), 0.01f); +} + +TEST(ResidualEchoDetectorTests, EchoWithRenderClockDrift) { + ResidualEchoDetector echo_detector; + echo_detector.SetReliabilityForTest(1.0f); + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal has a delay of 10 frames w.r.t. the render + // signal, but is otherwise identical. Both signals are periodic with a 20 + // frame interval. There is a simulated clock drift of 1% in this test, with + // the render side producing data slightly faster. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + echo_detector.AnalyzeCaptureAudio(zeros); + } else if (i % 20 == 10) { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(zeros); + } + if (i % 100 == 0) { + // This is causing the simulated clock drift. + echo_detector.AnalyzeRenderAudio(zeros); + } + } + // We expect to detect echo with high likelihood. Clock drift is harder to + // correct on the render side than on the capture side. This is due to the + // render buffer, clock drift can only be discovered after a certain delay. + // A growing buffer can be caused by jitter or clock drift and it's not + // possible to make this decision right away. For this reason we only expect + // an echo likelihood of 75% in this test. + EXPECT_GT(echo_detector.echo_likelihood(), 0.75f); +} + +TEST(ResidualEchoDetectorTests, EchoWithCaptureClockDrift) { + ResidualEchoDetector echo_detector; + echo_detector.SetReliabilityForTest(1.0f); + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal has a delay of 10 frames w.r.t. the render + // signal, but is otherwise identical. Both signals are periodic with a 20 + // frame interval. There is a simulated clock drift of 1% in this test, with + // the capture side producing data slightly faster. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + echo_detector.AnalyzeCaptureAudio(zeros); + } else if (i % 20 == 10) { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(zeros); + } + if (i % 100 == 0) { + // This is causing the simulated clock drift. + echo_detector.AnalyzeCaptureAudio(zeros); + } + } + // We expect to detect echo with near certain likelihood. + EXPECT_NEAR(1.f, echo_detector.echo_likelihood(), 0.01f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.cc new file mode 100644 index 0000000000..55db226af2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/rms_level.h" + +#include +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { +static constexpr float kMaxSquaredLevel = 32768 * 32768; +// kMinLevel is the level corresponding to kMinLevelDb, that is 10^(-127/10). +static constexpr float kMinLevel = 1.995262314968883e-13f; + +// Calculates the normalized RMS value from a mean square value. The input +// should be the sum of squared samples divided by the number of samples. The +// value will be normalized to full range before computing the RMS, wich is +// returned as a negated dBfs. That is, 0 is full amplitude while 127 is very +// faint. +int ComputeRms(float mean_square) { + if (mean_square <= kMinLevel * kMaxSquaredLevel) { + // Very faint; simply return the minimum value. + return RmsLevel::kMinLevelDb; + } + // Normalize by the max level. + const float mean_square_norm = mean_square / kMaxSquaredLevel; + RTC_DCHECK_GT(mean_square_norm, kMinLevel); + // 20log_10(x^0.5) = 10log_10(x) + const float rms = 10.f * log10(mean_square_norm); + RTC_DCHECK_LE(rms, 0.f); + RTC_DCHECK_GT(rms, -RmsLevel::kMinLevelDb); + // Return the negated value. + return static_cast(-rms + 0.5f); +} +} // namespace + +RmsLevel::RmsLevel() { + Reset(); +} + +RmsLevel::~RmsLevel() = default; + +void RmsLevel::Reset() { + sum_square_ = 0.f; + sample_count_ = 0; + max_sum_square_ = 0.f; + block_size_ = rtc::nullopt; +} + +void RmsLevel::Analyze(rtc::ArrayView data) { + if (data.empty()) { + return; + } + + CheckBlockSize(data.size()); + + const float sum_square = + std::accumulate(data.begin(), data.end(), 0.f, + [](float a, int16_t b) { return a + b * b; }); + RTC_DCHECK_GE(sum_square, 0.f); + sum_square_ += sum_square; + sample_count_ += data.size(); + + max_sum_square_ = std::max(max_sum_square_, sum_square); +} + +void RmsLevel::AnalyzeMuted(size_t length) { + CheckBlockSize(length); + sample_count_ += length; +} + +int RmsLevel::Average() { + int rms = (sample_count_ == 0) ? RmsLevel::kMinLevelDb + : ComputeRms(sum_square_ / sample_count_); + Reset(); + return rms; +} + +RmsLevel::Levels RmsLevel::AverageAndPeak() { + // Note that block_size_ should by design always be non-empty when + // sample_count_ != 0. Also, the * operator of rtc::Optional enforces this + // with a DCHECK. + Levels levels = (sample_count_ == 0) + ? Levels{RmsLevel::kMinLevelDb, RmsLevel::kMinLevelDb} + : Levels{ComputeRms(sum_square_ / sample_count_), + ComputeRms(max_sum_square_ / *block_size_)}; + Reset(); + return levels; +} + +void RmsLevel::CheckBlockSize(size_t block_size) { + if (block_size_ != block_size) { + Reset(); + block_size_ = block_size; + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.h b/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.h new file mode 100644 index 0000000000..6fe22fd120 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_RMS_LEVEL_H_ +#define MODULES_AUDIO_PROCESSING_RMS_LEVEL_H_ + +#include "api/array_view.h" +#include "api/optional.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Computes the root mean square (RMS) level in dBFs (decibels from digital +// full-scale) of audio data. The computation follows RFC 6465: +// https://tools.ietf.org/html/rfc6465 +// with the intent that it can provide the RTP audio level indication. +// +// The expected approach is to provide constant-sized chunks of audio to +// Analyze(). When enough chunks have been accumulated to form a packet, call +// Average() to get the audio level indicator for the RTP header. +class RmsLevel { + public: + struct Levels { + int average; + int peak; + }; + + static constexpr int kMinLevelDb = 127; + + RmsLevel(); + ~RmsLevel(); + + // Can be called to reset internal states, but is not required during normal + // operation. + void Reset(); + + // Pass each chunk of audio to Analyze() to accumulate the level. + void Analyze(rtc::ArrayView data); + + // If all samples with the given |length| have a magnitude of zero, this is + // a shortcut to avoid some computation. + void AnalyzeMuted(size_t length); + + // Computes the RMS level over all data passed to Analyze() since the last + // call to Average(). The returned value is positive but should be interpreted + // as negative as per the RFC. It is constrained to [0, 127]. Resets the + // internal state to start a new measurement period. + int Average(); + + // Like Average() above, but also returns the RMS peak value. Resets the + // internal state to start a new measurement period. + Levels AverageAndPeak(); + + private: + // Compares |block_size| with |block_size_|. If they are different, calls + // Reset() and stores the new size. + void CheckBlockSize(size_t block_size); + + float sum_square_; + size_t sample_count_; + float max_sum_square_; + rtc::Optional block_size_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_RMS_LEVEL_H_ + diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level_unittest.cc new file mode 100644 index 0000000000..cf7683d1c8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/rms_level_unittest.cc @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/rms_level.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/mathutils.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { +constexpr int kSampleRateHz = 48000; +constexpr size_t kBlockSizeSamples = kSampleRateHz / 100; + +std::unique_ptr RunTest(rtc::ArrayView input) { + std::unique_ptr level(new RmsLevel); + for (size_t n = 0; n + kBlockSizeSamples <= input.size(); + n += kBlockSizeSamples) { + level->Analyze(input.subview(n, kBlockSizeSamples)); + } + return level; +} + +std::vector CreateSinusoid(int frequency_hz, + int amplitude, + size_t num_samples) { + std::vector x(num_samples); + for (size_t n = 0; n < num_samples; ++n) { + x[n] = rtc::saturated_cast( + amplitude * std::sin(2 * M_PI * n * frequency_hz / kSampleRateHz)); + } + return x; +} +} // namespace + +TEST(RmsLevelTest, Run1000HzFullScale) { + auto x = CreateSinusoid(1000, INT16_MAX, kSampleRateHz); + auto level = RunTest(x); + EXPECT_EQ(3, level->Average()); // -3 dBFS +} + +TEST(RmsLevelTest, Run1000HzFullScaleAverageAndPeak) { + auto x = CreateSinusoid(1000, INT16_MAX, kSampleRateHz); + auto level = RunTest(x); + auto stats = level->AverageAndPeak(); + EXPECT_EQ(3, stats.average); // -3 dBFS + EXPECT_EQ(3, stats.peak); +} + +TEST(RmsLevelTest, Run1000HzHalfScale) { + auto x = CreateSinusoid(1000, INT16_MAX / 2, kSampleRateHz); + auto level = RunTest(x); + EXPECT_EQ(9, level->Average()); // -9 dBFS +} + +TEST(RmsLevelTest, RunZeros) { + std::vector x(kSampleRateHz, 0); // 1 second of pure silence. + auto level = RunTest(x); + EXPECT_EQ(127, level->Average()); +} + +TEST(RmsLevelTest, RunZerosAverageAndPeak) { + std::vector x(kSampleRateHz, 0); // 1 second of pure silence. + auto level = RunTest(x); + auto stats = level->AverageAndPeak(); + EXPECT_EQ(127, stats.average); + EXPECT_EQ(127, stats.peak); +} + +TEST(RmsLevelTest, NoSamples) { + RmsLevel level; + EXPECT_EQ(127, level.Average()); // Return minimum if no samples are given. +} + +TEST(RmsLevelTest, NoSamplesAverageAndPeak) { + RmsLevel level; + auto stats = level.AverageAndPeak(); + EXPECT_EQ(127, stats.average); + EXPECT_EQ(127, stats.peak); +} + +TEST(RmsLevelTest, PollTwice) { + auto x = CreateSinusoid(1000, INT16_MAX, kSampleRateHz); + auto level = RunTest(x); + level->Average(); + EXPECT_EQ(127, level->Average()); // Stats should be reset at this point. +} + +TEST(RmsLevelTest, Reset) { + auto x = CreateSinusoid(1000, INT16_MAX, kSampleRateHz); + auto level = RunTest(x); + level->Reset(); + EXPECT_EQ(127, level->Average()); // Stats should be reset at this point. +} + +// Inserts 1 second of full-scale sinusoid, followed by 1 second of muted. +TEST(RmsLevelTest, ProcessMuted) { + auto x = CreateSinusoid(1000, INT16_MAX, kSampleRateHz); + auto level = RunTest(x); + const size_t kBlocksPerSecond = rtc::CheckedDivExact( + static_cast(kSampleRateHz), kBlockSizeSamples); + for (size_t i = 0; i < kBlocksPerSecond; ++i) { + level->AnalyzeMuted(kBlockSizeSamples); + } + EXPECT_EQ(6, level->Average()); // Average RMS halved due to the silence. +} + +// Inserts 1 second of half-scale sinusoid, follwed by 10 ms of full-scale, and +// finally 1 second of half-scale again. Expect the average to be -9 dBFS due +// to the vast majority of the signal being half-scale, and the peak to be +// -3 dBFS. +TEST(RmsLevelTest, RunHalfScaleAndInsertFullScale) { + auto half_scale = CreateSinusoid(1000, INT16_MAX / 2, kSampleRateHz); + auto full_scale = CreateSinusoid(1000, INT16_MAX, kSampleRateHz / 100); + auto x = half_scale; + x.insert(x.end(), full_scale.begin(), full_scale.end()); + x.insert(x.end(), half_scale.begin(), half_scale.end()); + ASSERT_EQ(static_cast(2 * kSampleRateHz + kSampleRateHz / 100), + x.size()); + auto level = RunTest(x); + auto stats = level->AverageAndPeak(); + EXPECT_EQ(9, stats.average); + EXPECT_EQ(3, stats.peak); +} + +TEST(RmsLevelTest, ResetOnBlockSizeChange) { + auto x = CreateSinusoid(1000, INT16_MAX, kSampleRateHz); + auto level = RunTest(x); + // Create a new signal with half amplitude, but double block length. + auto y = CreateSinusoid(1000, INT16_MAX / 2, kBlockSizeSamples * 2); + level->Analyze(y); + auto stats = level->AverageAndPeak(); + // Expect all stats to only be influenced by the last signal (y), since the + // changed block size should reset the stats. + EXPECT_EQ(9, stats.average); + EXPECT_EQ(9, stats.peak); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.cc new file mode 100644 index 0000000000..e2b8f82c36 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/splitting_filter.h" + +#include "common_audio/channel_buffer.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +SplittingFilter::SplittingFilter(size_t num_channels, + size_t num_bands, + size_t num_frames) + : num_bands_(num_bands) { + RTC_CHECK(num_bands_ == 2 || num_bands_ == 3); + if (num_bands_ == 2) { + two_bands_states_.resize(num_channels); + } else if (num_bands_ == 3) { + for (size_t i = 0; i < num_channels; ++i) { + three_band_filter_banks_.push_back(std::unique_ptr( + new ThreeBandFilterBank(num_frames))); + } + } +} + +SplittingFilter::~SplittingFilter() = default; + +void SplittingFilter::Analysis(const IFChannelBuffer* data, + IFChannelBuffer* bands) { + RTC_DCHECK_EQ(num_bands_, bands->num_bands()); + RTC_DCHECK_EQ(data->num_channels(), bands->num_channels()); + RTC_DCHECK_EQ(data->num_frames(), + bands->num_frames_per_band() * bands->num_bands()); + if (bands->num_bands() == 2) { + TwoBandsAnalysis(data, bands); + } else if (bands->num_bands() == 3) { + ThreeBandsAnalysis(data, bands); + } +} + +void SplittingFilter::Synthesis(const IFChannelBuffer* bands, + IFChannelBuffer* data) { + RTC_DCHECK_EQ(num_bands_, bands->num_bands()); + RTC_DCHECK_EQ(data->num_channels(), bands->num_channels()); + RTC_DCHECK_EQ(data->num_frames(), + bands->num_frames_per_band() * bands->num_bands()); + if (bands->num_bands() == 2) { + TwoBandsSynthesis(bands, data); + } else if (bands->num_bands() == 3) { + ThreeBandsSynthesis(bands, data); + } +} + +void SplittingFilter::TwoBandsAnalysis(const IFChannelBuffer* data, + IFChannelBuffer* bands) { + RTC_DCHECK_EQ(two_bands_states_.size(), data->num_channels()); + for (size_t i = 0; i < two_bands_states_.size(); ++i) { + WebRtcSpl_AnalysisQMF(data->ibuf_const()->channels()[i], + data->num_frames(), + bands->ibuf()->channels(0)[i], + bands->ibuf()->channels(1)[i], + two_bands_states_[i].analysis_state1, + two_bands_states_[i].analysis_state2); + } +} + +void SplittingFilter::TwoBandsSynthesis(const IFChannelBuffer* bands, + IFChannelBuffer* data) { + RTC_DCHECK_LE(data->num_channels(), two_bands_states_.size()); + for (size_t i = 0; i < data->num_channels(); ++i) { + WebRtcSpl_SynthesisQMF(bands->ibuf_const()->channels(0)[i], + bands->ibuf_const()->channels(1)[i], + bands->num_frames_per_band(), + data->ibuf()->channels()[i], + two_bands_states_[i].synthesis_state1, + two_bands_states_[i].synthesis_state2); + } +} + +void SplittingFilter::ThreeBandsAnalysis(const IFChannelBuffer* data, + IFChannelBuffer* bands) { + RTC_DCHECK_EQ(three_band_filter_banks_.size(), data->num_channels()); + for (size_t i = 0; i < three_band_filter_banks_.size(); ++i) { + three_band_filter_banks_[i]->Analysis(data->fbuf_const()->channels()[i], + data->num_frames(), + bands->fbuf()->bands(i)); + } +} + +void SplittingFilter::ThreeBandsSynthesis(const IFChannelBuffer* bands, + IFChannelBuffer* data) { + RTC_DCHECK_LE(data->num_channels(), three_band_filter_banks_.size()); + for (size_t i = 0; i < data->num_channels(); ++i) { + three_band_filter_banks_[i]->Synthesis(bands->fbuf_const()->bands(i), + bands->num_frames_per_band(), + data->fbuf()->channels()[i]); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.h new file mode 100644 index 0000000000..7d60c82ff6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_SPLITTING_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_SPLITTING_FILTER_H_ + +#include +#include +#include + +#include "modules/audio_processing/three_band_filter_bank.h" + +namespace webrtc { + +class IFChannelBuffer; + +struct TwoBandsStates { + TwoBandsStates() { + memset(analysis_state1, 0, sizeof(analysis_state1)); + memset(analysis_state2, 0, sizeof(analysis_state2)); + memset(synthesis_state1, 0, sizeof(synthesis_state1)); + memset(synthesis_state2, 0, sizeof(synthesis_state2)); + } + + static const int kStateSize = 6; + int analysis_state1[kStateSize]; + int analysis_state2[kStateSize]; + int synthesis_state1[kStateSize]; + int synthesis_state2[kStateSize]; +}; + +// Splitting filter which is able to split into and merge from 2 or 3 frequency +// bands. The number of channels needs to be provided at construction time. +// +// For each block, Analysis() is called to split into bands and then Synthesis() +// to merge these bands again. The input and output signals are contained in +// IFChannelBuffers and for the different bands an array of IFChannelBuffers is +// used. +class SplittingFilter { + public: + SplittingFilter(size_t num_channels, size_t num_bands, size_t num_frames); + ~SplittingFilter(); + + void Analysis(const IFChannelBuffer* data, IFChannelBuffer* bands); + void Synthesis(const IFChannelBuffer* bands, IFChannelBuffer* data); + + private: + // Two-band analysis and synthesis work for 640 samples or less. + void TwoBandsAnalysis(const IFChannelBuffer* data, IFChannelBuffer* bands); + void TwoBandsSynthesis(const IFChannelBuffer* bands, IFChannelBuffer* data); + void ThreeBandsAnalysis(const IFChannelBuffer* data, IFChannelBuffer* bands); + void ThreeBandsSynthesis(const IFChannelBuffer* bands, IFChannelBuffer* data); + void InitBuffers(); + + const size_t num_bands_; + std::vector two_bands_states_; + std::vector> three_band_filter_banks_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_SPLITTING_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter_unittest.cc new file mode 100644 index 0000000000..3e0dbb9b9c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/splitting_filter_unittest.cc @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// MSVC++ requires this to be set before any other includes to get M_PI. +#define _USE_MATH_DEFINES + +#include + +#include "common_audio/channel_buffer.h" +#include "modules/audio_processing/splitting_filter.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const size_t kSamplesPer16kHzChannel = 160; +const size_t kSamplesPer48kHzChannel = 480; + +} // namespace + +// Generates a signal from presence or absence of sine waves of different +// frequencies. +// Splits into 3 bands and checks their presence or absence. +// Recombines the bands. +// Calculates the delay. +// Checks that the cross correlation of input and output is high enough at the +// calculated delay. +TEST(SplittingFilterTest, SplitsIntoThreeBandsAndReconstructs) { + static const int kChannels = 1; + static const int kSampleRateHz = 48000; + static const size_t kNumBands = 3; + static const int kFrequenciesHz[kNumBands] = {1000, 12000, 18000}; + static const float kAmplitude = 8192.f; + static const size_t kChunks = 8; + SplittingFilter splitting_filter(kChannels, + kNumBands, + kSamplesPer48kHzChannel); + IFChannelBuffer in_data(kSamplesPer48kHzChannel, kChannels, kNumBands); + IFChannelBuffer bands(kSamplesPer48kHzChannel, kChannels, kNumBands); + IFChannelBuffer out_data(kSamplesPer48kHzChannel, kChannels, kNumBands); + for (size_t i = 0; i < kChunks; ++i) { + // Input signal generation. + bool is_present[kNumBands]; + memset(in_data.fbuf()->channels()[0], + 0, + kSamplesPer48kHzChannel * sizeof(in_data.fbuf()->channels()[0][0])); + for (size_t j = 0; j < kNumBands; ++j) { + is_present[j] = i & (static_cast(1) << j); + float amplitude = is_present[j] ? kAmplitude : 0.f; + for (size_t k = 0; k < kSamplesPer48kHzChannel; ++k) { + in_data.fbuf()->channels()[0][k] += + amplitude * sin(2.f * M_PI * kFrequenciesHz[j] * + (i * kSamplesPer48kHzChannel + k) / kSampleRateHz); + } + } + // Three band splitting filter. + splitting_filter.Analysis(&in_data, &bands); + // Energy calculation. + float energy[kNumBands]; + for (size_t j = 0; j < kNumBands; ++j) { + energy[j] = 0.f; + for (size_t k = 0; k < kSamplesPer16kHzChannel; ++k) { + energy[j] += bands.fbuf_const()->channels(j)[0][k] * + bands.fbuf_const()->channels(j)[0][k]; + } + energy[j] /= kSamplesPer16kHzChannel; + if (is_present[j]) { + EXPECT_GT(energy[j], kAmplitude * kAmplitude / 4); + } else { + EXPECT_LT(energy[j], kAmplitude * kAmplitude / 4); + } + } + // Three band merge. + splitting_filter.Synthesis(&bands, &out_data); + // Delay and cross correlation estimation. + float xcorr = 0.f; + for (size_t delay = 0; delay < kSamplesPer48kHzChannel; ++delay) { + float tmpcorr = 0.f; + for (size_t j = delay; j < kSamplesPer48kHzChannel; ++j) { + tmpcorr += in_data.fbuf_const()->channels()[0][j - delay] * + out_data.fbuf_const()->channels()[0][j]; + } + tmpcorr /= kSamplesPer48kHzChannel; + if (tmpcorr > xcorr) { + xcorr = tmpcorr; + } + } + // High cross correlation check. + bool any_present = false; + for (size_t j = 0; j < kNumBands; ++j) { + any_present |= is_present[j]; + } + if (any_present) { + EXPECT_GT(xcorr, kAmplitude * kAmplitude / 4); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc new file mode 100644 index 0000000000..6d0b07c7ed --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc @@ -0,0 +1,557 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/test/aec_dump_based_simulator.h" + +#include "modules/audio_processing/test/protobuf_utils.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { +namespace { + +// Verify output bitexactness for the fixed interface. +// TODO(peah): Check whether it would make sense to add a threshold +// to use for checking the bitexactness in a soft manner. +bool VerifyFixedBitExactness(const webrtc::audioproc::Stream& msg, + const AudioFrame& frame) { + if ((sizeof(int16_t) * frame.samples_per_channel_ * frame.num_channels_) != + msg.output_data().size()) { + return false; + } else { + const int16_t* frame_data = frame.data(); + for (size_t k = 0; k < frame.num_channels_ * frame.samples_per_channel_; + ++k) { + if (msg.output_data().data()[k] != frame_data[k]) { + return false; + } + } + } + return true; +} + +// Verify output bitexactness for the float interface. +bool VerifyFloatBitExactness(const webrtc::audioproc::Stream& msg, + const StreamConfig& out_config, + const ChannelBuffer& out_buf) { + if (static_cast(msg.output_channel_size()) != + out_config.num_channels() || + msg.output_channel(0).size() != out_config.num_frames()) { + return false; + } else { + for (int ch = 0; ch < msg.output_channel_size(); ++ch) { + for (size_t sample = 0; sample < out_config.num_frames(); ++sample) { + if (msg.output_channel(ch).data()[sample] != + out_buf.channels()[ch][sample]) { + return false; + } + } + } + } + return true; +} + +} // namespace + +AecDumpBasedSimulator::AecDumpBasedSimulator(const SimulationSettings& settings) + : AudioProcessingSimulator(settings) {} + +AecDumpBasedSimulator::~AecDumpBasedSimulator() = default; + +void AecDumpBasedSimulator::PrepareProcessStreamCall( + const webrtc::audioproc::Stream& msg) { + if (msg.has_input_data()) { + // Fixed interface processing. + // Verify interface invariance. + RTC_CHECK(interface_used_ == InterfaceType::kFixedInterface || + interface_used_ == InterfaceType::kNotSpecified); + interface_used_ = InterfaceType::kFixedInterface; + + // Populate input buffer. + RTC_CHECK_EQ(sizeof(*fwd_frame_.data()) * fwd_frame_.samples_per_channel_ * + fwd_frame_.num_channels_, + msg.input_data().size()); + memcpy(fwd_frame_.mutable_data(), msg.input_data().data(), + msg.input_data().size()); + } else { + // Float interface processing. + // Verify interface invariance. + RTC_CHECK(interface_used_ == InterfaceType::kFloatInterface || + interface_used_ == InterfaceType::kNotSpecified); + interface_used_ = InterfaceType::kFloatInterface; + + RTC_CHECK_EQ(in_buf_->num_channels(), + static_cast(msg.input_channel_size())); + + // Populate input buffer. + for (size_t i = 0; i < in_buf_->num_channels(); ++i) { + RTC_CHECK_EQ(in_buf_->num_frames() * sizeof(*in_buf_->channels()[i]), + msg.input_channel(i).size()); + std::memcpy(in_buf_->channels()[i], msg.input_channel(i).data(), + msg.input_channel(i).size()); + } + } + + if (artificial_nearend_buffer_reader_) { + if (artificial_nearend_buffer_reader_->Read( + artificial_nearend_buf_.get())) { + if (msg.has_input_data()) { + int16_t* fwd_frame_data = fwd_frame_.mutable_data(); + for (size_t k = 0; k < in_buf_->num_frames(); ++k) { + fwd_frame_data[k] = rtc::saturated_cast( + fwd_frame_data[k] + + static_cast(32767 * + artificial_nearend_buf_->channels()[0][k])); + } + } else { + for (int i = 0; i < msg.input_channel_size(); ++i) { + for (size_t k = 0; k < in_buf_->num_frames(); ++k) { + in_buf_->channels()[i][k] += + artificial_nearend_buf_->channels()[0][k]; + in_buf_->channels()[i][k] = std::min( + 32767.f, std::max(-32768.f, in_buf_->channels()[i][k])); + } + } + } + } else { + if (!artificial_nearend_eof_reported_) { + std::cout << "The artificial nearend file ended before the recording."; + artificial_nearend_eof_reported_ = true; + } + } + } + + if (!settings_.stream_delay) { + if (msg.has_delay()) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->set_stream_delay_ms(msg.delay())); + } + } else { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->set_stream_delay_ms(*settings_.stream_delay)); + } + + if (!settings_.stream_drift_samples) { + if (msg.has_drift()) { + ap_->echo_cancellation()->set_stream_drift_samples(msg.drift()); + } + } else { + ap_->echo_cancellation()->set_stream_drift_samples( + *settings_.stream_drift_samples); + } + + if (!settings_.use_ts) { + if (msg.has_keypress()) { + ap_->set_stream_key_pressed(msg.keypress()); + } + } else { + ap_->set_stream_key_pressed(*settings_.use_ts); + } + + // Level is always logged in AEC dumps. + RTC_CHECK(msg.has_level()); + aec_dump_mic_level_ = msg.level(); +} + +void AecDumpBasedSimulator::VerifyProcessStreamBitExactness( + const webrtc::audioproc::Stream& msg) { + if (bitexact_output_) { + if (interface_used_ == InterfaceType::kFixedInterface) { + bitexact_output_ = VerifyFixedBitExactness(msg, fwd_frame_); + } else { + bitexact_output_ = VerifyFloatBitExactness(msg, out_config_, *out_buf_); + } + } +} + +void AecDumpBasedSimulator::PrepareReverseProcessStreamCall( + const webrtc::audioproc::ReverseStream& msg) { + if (msg.has_data()) { + // Fixed interface processing. + // Verify interface invariance. + RTC_CHECK(interface_used_ == InterfaceType::kFixedInterface || + interface_used_ == InterfaceType::kNotSpecified); + interface_used_ = InterfaceType::kFixedInterface; + + // Populate input buffer. + RTC_CHECK_EQ(sizeof(int16_t) * rev_frame_.samples_per_channel_ * + rev_frame_.num_channels_, + msg.data().size()); + memcpy(rev_frame_.mutable_data(), msg.data().data(), msg.data().size()); + } else { + // Float interface processing. + // Verify interface invariance. + RTC_CHECK(interface_used_ == InterfaceType::kFloatInterface || + interface_used_ == InterfaceType::kNotSpecified); + interface_used_ = InterfaceType::kFloatInterface; + + RTC_CHECK_EQ(reverse_in_buf_->num_channels(), + static_cast(msg.channel_size())); + + // Populate input buffer. + for (int i = 0; i < msg.channel_size(); ++i) { + RTC_CHECK_EQ(reverse_in_buf_->num_frames() * + sizeof(*reverse_in_buf_->channels()[i]), + msg.channel(i).size()); + std::memcpy(reverse_in_buf_->channels()[i], msg.channel(i).data(), + msg.channel(i).size()); + } + } +} + +void AecDumpBasedSimulator::Process() { + CreateAudioProcessor(); + dump_input_file_ = OpenFile(settings_.aec_dump_input_filename->c_str(), "rb"); + + if (settings_.artificial_nearend_filename) { + std::unique_ptr artificial_nearend_file( + new WavReader(settings_.artificial_nearend_filename->c_str())); + + RTC_CHECK_EQ(1, artificial_nearend_file->num_channels()) + << "Only mono files for the artificial nearend are supported, " + "reverted to not using the artificial nearend file"; + + const int sample_rate_hz = artificial_nearend_file->sample_rate(); + artificial_nearend_buffer_reader_.reset( + new ChannelBufferWavReader(std::move(artificial_nearend_file))); + artificial_nearend_buf_.reset(new ChannelBuffer( + rtc::CheckedDivExact(sample_rate_hz, kChunksPerSecond), 1)); + } + + webrtc::audioproc::Event event_msg; + int num_forward_chunks_processed = 0; + while (ReadMessageFromFile(dump_input_file_, &event_msg)) { + switch (event_msg.type()) { + case webrtc::audioproc::Event::INIT: + RTC_CHECK(event_msg.has_init()); + HandleMessage(event_msg.init()); + break; + case webrtc::audioproc::Event::STREAM: + RTC_CHECK(event_msg.has_stream()); + HandleMessage(event_msg.stream()); + ++num_forward_chunks_processed; + break; + case webrtc::audioproc::Event::REVERSE_STREAM: + RTC_CHECK(event_msg.has_reverse_stream()); + HandleMessage(event_msg.reverse_stream()); + break; + case webrtc::audioproc::Event::CONFIG: + RTC_CHECK(event_msg.has_config()); + HandleMessage(event_msg.config()); + break; + default: + RTC_CHECK(false); + } + } + + fclose(dump_input_file_); + + DestroyAudioProcessor(); +} + +void AecDumpBasedSimulator::HandleMessage( + const webrtc::audioproc::Config& msg) { + if (settings_.use_verbose_logging) { + std::cout << "Config at frame:" << std::endl; + std::cout << " Forward: " << get_num_process_stream_calls() << std::endl; + std::cout << " Reverse: " << get_num_reverse_process_stream_calls() + << std::endl; + } + + if (!settings_.discard_all_settings_in_aecdump) { + if (settings_.use_verbose_logging) { + std::cout << "Setting used in config:" << std::endl; + } + Config config; + AudioProcessing::Config apm_config; + + if (msg.has_aec_enabled() || settings_.use_aec) { + bool enable = settings_.use_aec ? *settings_.use_aec : msg.aec_enabled(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_cancellation()->Enable(enable)); + if (settings_.use_verbose_logging) { + std::cout << " aec_enabled: " << (enable ? "true" : "false") + << std::endl; + } + } + + if (msg.has_aec_delay_agnostic_enabled() || settings_.use_delay_agnostic) { + bool enable = settings_.use_delay_agnostic + ? *settings_.use_delay_agnostic + : msg.aec_delay_agnostic_enabled(); + config.Set(new DelayAgnostic(enable)); + if (settings_.use_verbose_logging) { + std::cout << " aec_delay_agnostic_enabled: " + << (enable ? "true" : "false") << std::endl; + } + } + + if (msg.has_aec_drift_compensation_enabled() || + settings_.use_drift_compensation) { + bool enable = settings_.use_drift_compensation + ? *settings_.use_drift_compensation + : msg.aec_drift_compensation_enabled(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_cancellation()->enable_drift_compensation(enable)); + if (settings_.use_verbose_logging) { + std::cout << " aec_drift_compensation_enabled: " + << (enable ? "true" : "false") << std::endl; + } + } + + if (msg.has_aec_extended_filter_enabled() || + settings_.use_extended_filter) { + bool enable = settings_.use_extended_filter + ? *settings_.use_extended_filter + : msg.aec_extended_filter_enabled(); + config.Set(new ExtendedFilter(enable)); + if (settings_.use_verbose_logging) { + std::cout << " aec_extended_filter_enabled: " + << (enable ? "true" : "false") << std::endl; + } + } + + if (msg.has_aec_suppression_level() || settings_.aec_suppression_level) { + int level = settings_.aec_suppression_level + ? *settings_.aec_suppression_level + : msg.aec_suppression_level(); + RTC_CHECK_EQ( + AudioProcessing::kNoError, + ap_->echo_cancellation()->set_suppression_level( + static_cast(level))); + if (settings_.use_verbose_logging) { + std::cout << " aec_suppression_level: " << level << std::endl; + } + } + + if (msg.has_aecm_enabled() || settings_.use_aecm) { + bool enable = + settings_.use_aecm ? *settings_.use_aecm : msg.aecm_enabled(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_control_mobile()->Enable(enable)); + if (settings_.use_verbose_logging) { + std::cout << " aecm_enabled: " << (enable ? "true" : "false") + << std::endl; + } + } + + if (msg.has_aecm_comfort_noise_enabled() || + settings_.use_aecm_comfort_noise) { + bool enable = settings_.use_aecm_comfort_noise + ? *settings_.use_aecm_comfort_noise + : msg.aecm_comfort_noise_enabled(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_control_mobile()->enable_comfort_noise(enable)); + if (settings_.use_verbose_logging) { + std::cout << " aecm_comfort_noise_enabled: " + << (enable ? "true" : "false") << std::endl; + } + } + + if (msg.has_aecm_routing_mode() || settings_.aecm_routing_mode) { + int routing_mode = settings_.aecm_routing_mode + ? *settings_.aecm_routing_mode + : msg.aecm_routing_mode(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_control_mobile()->set_routing_mode( + static_cast( + routing_mode))); + if (settings_.use_verbose_logging) { + std::cout << " aecm_routing_mode: " << routing_mode << std::endl; + } + } + + if (msg.has_agc_enabled() || settings_.use_agc) { + bool enable = settings_.use_agc ? *settings_.use_agc : msg.agc_enabled(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->Enable(enable)); + if (settings_.use_verbose_logging) { + std::cout << " agc_enabled: " << (enable ? "true" : "false") + << std::endl; + } + } + + if (msg.has_agc_mode() || settings_.agc_mode) { + int mode = settings_.agc_mode ? *settings_.agc_mode : msg.agc_mode(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->set_mode( + static_cast(mode))); + if (settings_.use_verbose_logging) { + std::cout << " agc_mode: " << mode << std::endl; + } + } + + if (msg.has_agc_limiter_enabled() || settings_.use_agc_limiter) { + bool enable = settings_.use_agc_limiter ? *settings_.use_agc_limiter + : msg.agc_limiter_enabled(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->enable_limiter(enable)); + if (settings_.use_verbose_logging) { + std::cout << " agc_limiter_enabled: " << (enable ? "true" : "false") + << std::endl; + } + } + + // TODO(peah): Add support for controlling the Experimental AGC from the + // command line. + if (msg.has_noise_robust_agc_enabled()) { + config.Set( + new ExperimentalAgc(msg.noise_robust_agc_enabled())); + if (settings_.use_verbose_logging) { + std::cout << " noise_robust_agc_enabled: " + << (msg.noise_robust_agc_enabled() ? "true" : "false") + << std::endl; + } + } + + if (msg.has_transient_suppression_enabled() || settings_.use_ts) { + bool enable = settings_.use_ts ? *settings_.use_ts + : msg.transient_suppression_enabled(); + config.Set(new ExperimentalNs(enable)); + if (settings_.use_verbose_logging) { + std::cout << " transient_suppression_enabled: " + << (enable ? "true" : "false") << std::endl; + } + } + + if (msg.has_intelligibility_enhancer_enabled() || settings_.use_ie) { + bool enable = settings_.use_ie ? *settings_.use_ie + : msg.intelligibility_enhancer_enabled(); + config.Set(new Intelligibility(enable)); + if (settings_.use_verbose_logging) { + std::cout << " intelligibility_enhancer_enabled: " + << (enable ? "true" : "false") << std::endl; + } + } + + if (msg.has_hpf_enabled() || settings_.use_hpf) { + bool enable = settings_.use_hpf ? *settings_.use_hpf : msg.hpf_enabled(); + apm_config.high_pass_filter.enabled = enable; + if (settings_.use_verbose_logging) { + std::cout << " hpf_enabled: " << (enable ? "true" : "false") + << std::endl; + } + } + + if (msg.has_ns_enabled() || settings_.use_ns) { + bool enable = settings_.use_ns ? *settings_.use_ns : msg.ns_enabled(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->noise_suppression()->Enable(enable)); + if (settings_.use_verbose_logging) { + std::cout << " ns_enabled: " << (enable ? "true" : "false") + << std::endl; + } + } + + if (msg.has_ns_level() || settings_.ns_level) { + int level = settings_.ns_level ? *settings_.ns_level : msg.ns_level(); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->noise_suppression()->set_level( + static_cast(level))); + if (settings_.use_verbose_logging) { + std::cout << " ns_level: " << level << std::endl; + } + } + + if (settings_.use_verbose_logging && msg.has_experiments_description() && + !msg.experiments_description().empty()) { + std::cout << " experiments not included by default in the simulation: " + << msg.experiments_description() << std::endl; + } + + if (settings_.use_refined_adaptive_filter) { + config.Set( + new RefinedAdaptiveFilter(*settings_.use_refined_adaptive_filter)); + } + + if (settings_.use_lc) { + apm_config.level_controller.enabled = *settings_.use_lc; + } + + if (settings_.use_ed) { + apm_config.residual_echo_detector.enabled = *settings_.use_ed; + } + + ap_->ApplyConfig(apm_config); + ap_->SetExtraOptions(config); + } +} + +void AecDumpBasedSimulator::HandleMessage(const webrtc::audioproc::Init& msg) { + RTC_CHECK(msg.has_sample_rate()); + RTC_CHECK(msg.has_num_input_channels()); + RTC_CHECK(msg.has_num_reverse_channels()); + RTC_CHECK(msg.has_reverse_sample_rate()); + + if (settings_.use_verbose_logging) { + std::cout << "Init at frame:" << std::endl; + std::cout << " Forward: " << get_num_process_stream_calls() << std::endl; + std::cout << " Reverse: " << get_num_reverse_process_stream_calls() + << std::endl; + } + + int num_output_channels; + if (settings_.output_num_channels) { + num_output_channels = *settings_.output_num_channels; + } else { + num_output_channels = msg.has_num_output_channels() + ? msg.num_output_channels() + : msg.num_input_channels(); + } + + int output_sample_rate; + if (settings_.output_sample_rate_hz) { + output_sample_rate = *settings_.output_sample_rate_hz; + } else { + output_sample_rate = msg.has_output_sample_rate() ? msg.output_sample_rate() + : msg.sample_rate(); + } + + int num_reverse_output_channels; + if (settings_.reverse_output_num_channels) { + num_reverse_output_channels = *settings_.reverse_output_num_channels; + } else { + num_reverse_output_channels = msg.has_num_reverse_output_channels() + ? msg.num_reverse_output_channels() + : msg.num_reverse_channels(); + } + + int reverse_output_sample_rate; + if (settings_.reverse_output_sample_rate_hz) { + reverse_output_sample_rate = *settings_.reverse_output_sample_rate_hz; + } else { + reverse_output_sample_rate = msg.has_reverse_output_sample_rate() + ? msg.reverse_output_sample_rate() + : msg.reverse_sample_rate(); + } + + SetupBuffersConfigsOutputs( + msg.sample_rate(), output_sample_rate, msg.reverse_sample_rate(), + reverse_output_sample_rate, msg.num_input_channels(), num_output_channels, + msg.num_reverse_channels(), num_reverse_output_channels); +} + +void AecDumpBasedSimulator::HandleMessage( + const webrtc::audioproc::Stream& msg) { + PrepareProcessStreamCall(msg); + ProcessStream(interface_used_ == InterfaceType::kFixedInterface); + VerifyProcessStreamBitExactness(msg); +} + +void AecDumpBasedSimulator::HandleMessage( + const webrtc::audioproc::ReverseStream& msg) { + PrepareReverseProcessStreamCall(msg); + ProcessReverseStream(interface_used_ == InterfaceType::kFixedInterface); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h new file mode 100644 index 0000000000..4c29bf7315 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_AEC_DUMP_BASED_SIMULATOR_H_ +#define MODULES_AUDIO_PROCESSING_TEST_AEC_DUMP_BASED_SIMULATOR_H_ + +#include "modules/audio_processing/test/audio_processing_simulator.h" + +#include "rtc_base/constructormagic.h" +#include "rtc_base/ignore_wundef.h" + +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" +#else +#include "modules/audio_processing/debug.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() + +namespace webrtc { +namespace test { + +// Used to perform an audio processing simulation from an aec dump. +class AecDumpBasedSimulator final : public AudioProcessingSimulator { + public: + explicit AecDumpBasedSimulator(const SimulationSettings& settings); + ~AecDumpBasedSimulator() override; + + // Processes the messages in the aecdump file. + void Process() override; + + private: + void HandleMessage(const webrtc::audioproc::Init& msg); + void HandleMessage(const webrtc::audioproc::Stream& msg); + void HandleMessage(const webrtc::audioproc::ReverseStream& msg); + void HandleMessage(const webrtc::audioproc::Config& msg); + void PrepareProcessStreamCall(const webrtc::audioproc::Stream& msg); + void PrepareReverseProcessStreamCall( + const webrtc::audioproc::ReverseStream& msg); + void VerifyProcessStreamBitExactness(const webrtc::audioproc::Stream& msg); + + enum InterfaceType { + kFixedInterface, + kFloatInterface, + kNotSpecified, + }; + + FILE* dump_input_file_; + std::unique_ptr> artificial_nearend_buf_; + std::unique_ptr artificial_nearend_buffer_reader_; + bool artificial_nearend_eof_reported_ = false; + InterfaceType interface_used_ = InterfaceType::kNotSpecified; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AecDumpBasedSimulator); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_AEC_DUMP_BASED_SIMULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/AndroidManifest.xml b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/AndroidManifest.xml new file mode 100644 index 0000000000..c6063b3d76 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/AndroidManifest.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/default.properties b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/default.properties new file mode 100644 index 0000000000..9a2c9f6c88 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/default.properties @@ -0,0 +1,11 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system use, +# "build.properties", and override values to adapt the script to your +# project structure. + +# Project target. +target=android-9 diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/jni/main.c b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/jni/main.c new file mode 100644 index 0000000000..2e19635683 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/jni/main.c @@ -0,0 +1,307 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +//BEGIN_INCLUDE(all) +#include +#include + +#include +#include + +#include +#include +#include + +#define LOGI(...) ((void)__android_log_print(ANDROID_LOG_INFO, "native-activity", __VA_ARGS__)) +#define LOGW(...) ((void)__android_log_print(ANDROID_LOG_WARN, "native-activity", __VA_ARGS__)) + +/** + * Our saved state data. + */ +struct saved_state { + float angle; + int32_t x; + int32_t y; +}; + +/** + * Shared state for our app. + */ +struct engine { + struct android_app* app; + + ASensorManager* sensorManager; + const ASensor* accelerometerSensor; + ASensorEventQueue* sensorEventQueue; + + int animating; + EGLDisplay display; + EGLSurface surface; + EGLContext context; + int32_t width; + int32_t height; + struct saved_state state; +}; + +/** + * Initialize an EGL context for the current display. + */ +static int engine_init_display(struct engine* engine) { + // initialize OpenGL ES and EGL + + /* + * Here specify the attributes of the desired configuration. + * Below, we select an EGLConfig with at least 8 bits per color + * component compatible with on-screen windows + */ + const EGLint attribs[] = { + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_BLUE_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_RED_SIZE, 8, + EGL_NONE + }; + EGLint w, h, dummy, format; + EGLint numConfigs; + EGLConfig config; + EGLSurface surface; + EGLContext context; + + EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY); + + eglInitialize(display, 0, 0); + + /* Here, the application chooses the configuration it desires. In this + * sample, we have a very simplified selection process, where we pick + * the first EGLConfig that matches our criteria */ + eglChooseConfig(display, attribs, &config, 1, &numConfigs); + + /* EGL_NATIVE_VISUAL_ID is an attribute of the EGLConfig that is + * guaranteed to be accepted by ANativeWindow_setBuffersGeometry(). + * As soon as we picked a EGLConfig, we can safely reconfigure the + * ANativeWindow buffers to match, using EGL_NATIVE_VISUAL_ID. */ + eglGetConfigAttrib(display, config, EGL_NATIVE_VISUAL_ID, &format); + + ANativeWindow_setBuffersGeometry(engine->app->window, 0, 0, format); + + surface = eglCreateWindowSurface(display, config, engine->app->window, NULL); + context = eglCreateContext(display, config, NULL, NULL); + + if (eglMakeCurrent(display, surface, surface, context) == EGL_FALSE) { + LOGW("Unable to eglMakeCurrent"); + return -1; + } + + eglQuerySurface(display, surface, EGL_WIDTH, &w); + eglQuerySurface(display, surface, EGL_HEIGHT, &h); + + engine->display = display; + engine->context = context; + engine->surface = surface; + engine->width = w; + engine->height = h; + engine->state.angle = 0; + + // Initialize GL state. + glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_FASTEST); + glEnable(GL_CULL_FACE); + glShadeModel(GL_SMOOTH); + glDisable(GL_DEPTH_TEST); + + return 0; +} + +/** + * Just the current frame in the display. + */ +static void engine_draw_frame(struct engine* engine) { + if (engine->display == NULL) { + // No display. + return; + } + + // Just fill the screen with a color. + glClearColor(((float)engine->state.x)/engine->width, engine->state.angle, + ((float)engine->state.y)/engine->height, 1); + glClear(GL_COLOR_BUFFER_BIT); + + eglSwapBuffers(engine->display, engine->surface); +} + +/** + * Tear down the EGL context currently associated with the display. + */ +static void engine_term_display(struct engine* engine) { + if (engine->display != EGL_NO_DISPLAY) { + eglMakeCurrent(engine->display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); + if (engine->context != EGL_NO_CONTEXT) { + eglDestroyContext(engine->display, engine->context); + } + if (engine->surface != EGL_NO_SURFACE) { + eglDestroySurface(engine->display, engine->surface); + } + eglTerminate(engine->display); + } + engine->animating = 0; + engine->display = EGL_NO_DISPLAY; + engine->context = EGL_NO_CONTEXT; + engine->surface = EGL_NO_SURFACE; +} + +/** + * Process the next input event. + */ +static int32_t engine_handle_input(struct android_app* app, AInputEvent* event) { + struct engine* engine = (struct engine*)app->userData; + if (AInputEvent_getType(event) == AINPUT_EVENT_TYPE_MOTION) { + engine->animating = 1; + engine->state.x = AMotionEvent_getX(event, 0); + engine->state.y = AMotionEvent_getY(event, 0); + return 1; + } + return 0; +} + +/** + * Process the next main command. + */ +static void engine_handle_cmd(struct android_app* app, int32_t cmd) { + struct engine* engine = (struct engine*)app->userData; + switch (cmd) { + case APP_CMD_SAVE_STATE: + // The system has asked us to save our current state. Do so. + engine->app->savedState = malloc(sizeof(struct saved_state)); + *((struct saved_state*)engine->app->savedState) = engine->state; + engine->app->savedStateSize = sizeof(struct saved_state); + break; + case APP_CMD_INIT_WINDOW: + // The window is being shown, get it ready. + if (engine->app->window != NULL) { + engine_init_display(engine); + engine_draw_frame(engine); + } + break; + case APP_CMD_TERM_WINDOW: + // The window is being hidden or closed, clean it up. + engine_term_display(engine); + break; + case APP_CMD_GAINED_FOCUS: + // When our app gains focus, we start monitoring the accelerometer. + if (engine->accelerometerSensor != NULL) { + ASensorEventQueue_enableSensor(engine->sensorEventQueue, + engine->accelerometerSensor); + // We'd like to get 60 events per second (in us). + ASensorEventQueue_setEventRate(engine->sensorEventQueue, + engine->accelerometerSensor, (1000L/60)*1000); + } + break; + case APP_CMD_LOST_FOCUS: + // When our app loses focus, we stop monitoring the accelerometer. + // This is to avoid consuming battery while not being used. + if (engine->accelerometerSensor != NULL) { + ASensorEventQueue_disableSensor(engine->sensorEventQueue, + engine->accelerometerSensor); + } + // Also stop animating. + engine->animating = 0; + engine_draw_frame(engine); + break; + } +} + +/** + * This is the main entry point of a native application that is using + * android_native_app_glue. It runs in its own thread, with its own + * event loop for receiving input events and doing other things. + */ +void android_main(struct android_app* state) { + struct engine engine; + + // Make sure glue isn't stripped. + app_dummy(); + + memset(&engine, 0, sizeof(engine)); + state->userData = &engine; + state->onAppCmd = engine_handle_cmd; + state->onInputEvent = engine_handle_input; + engine.app = state; + + // Prepare to monitor accelerometer + engine.sensorManager = ASensorManager_getInstance(); + engine.accelerometerSensor = ASensorManager_getDefaultSensor(engine.sensorManager, + ASENSOR_TYPE_ACCELEROMETER); + engine.sensorEventQueue = ASensorManager_createEventQueue(engine.sensorManager, + state->looper, LOOPER_ID_USER, NULL, NULL); + + if (state->savedState != NULL) { + // We are starting with a previous saved state; restore from it. + engine.state = *(struct saved_state*)state->savedState; + } + + // loop waiting for stuff to do. + + while (1) { + // Read all pending events. + int ident; + int events; + struct android_poll_source* source; + + // If not animating, we will block forever waiting for events. + // If animating, we loop until all events are read, then continue + // to draw the next frame of animation. + while ((ident=ALooper_pollAll(engine.animating ? 0 : -1, NULL, &events, + (void**)&source)) >= 0) { + + // Process this event. + if (source != NULL) { + source->process(state, source); + } + + // If a sensor has data, process it now. + if (ident == LOOPER_ID_USER) { + if (engine.accelerometerSensor != NULL) { + ASensorEvent event; + while (ASensorEventQueue_getEvents(engine.sensorEventQueue, + &event, 1) > 0) { + LOGI("accelerometer: x=%f y=%f z=%f", + event.acceleration.x, event.acceleration.y, + event.acceleration.z); + } + } + } + + // Check if we are exiting. + if (state->destroyRequested != 0) { + engine_term_display(&engine); + return; + } + } + + if (engine.animating) { + // Done with events; draw next animation frame. + engine.state.angle += .01f; + if (engine.state.angle > 1) { + engine.state.angle = 0; + } + + // Drawing is throttled to the screen update rate, so there + // is no need to do timing here. + engine_draw_frame(&engine); + } + } +} +//END_INCLUDE(all) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/res/values/strings.xml b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/res/values/strings.xml new file mode 100644 index 0000000000..d0bd0f3051 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/android/apmtest/res/values/strings.xml @@ -0,0 +1,4 @@ + + + apmtest + diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/apmtest.m b/third_party/libwebrtc/webrtc/modules/audio_processing/test/apmtest.m new file mode 100644 index 0000000000..1c8183c3ec --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/apmtest.m @@ -0,0 +1,365 @@ +% +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function apmtest(task, testname, filepath, casenumber, legacy) +%APMTEST is a tool to process APM file sets and easily display the output. +% APMTEST(TASK, TESTNAME, CASENUMBER) performs one of several TASKs: +% 'test' Processes the files to produce test output. +% 'list' Prints a list of cases in the test set, preceded by their +% CASENUMBERs. +% 'show' Uses spclab to show the test case specified by the +% CASENUMBER parameter. +% +% using a set of test files determined by TESTNAME: +% 'all' All tests. +% 'apm' The standard APM test set (default). +% 'apmm' The mobile APM test set. +% 'aec' The AEC test set. +% 'aecm' The AECM test set. +% 'agc' The AGC test set. +% 'ns' The NS test set. +% 'vad' The VAD test set. +% +% FILEPATH specifies the path to the test data files. +% +% CASENUMBER can be used to select a single test case. Omit CASENUMBER, +% or set to zero, to use all test cases. +% + +if nargin < 5 || isempty(legacy) + % Set to true to run old VQE recordings. + legacy = false; +end + +if nargin < 4 || isempty(casenumber) + casenumber = 0; +end + +if nargin < 3 || isempty(filepath) + filepath = 'data/'; +end + +if nargin < 2 || isempty(testname) + testname = 'all'; +end + +if nargin < 1 || isempty(task) + task = 'test'; +end + +if ~strcmp(task, 'test') && ~strcmp(task, 'list') && ~strcmp(task, 'show') + error(['TASK ' task ' is not recognized']); +end + +if casenumber == 0 && strcmp(task, 'show') + error(['CASENUMBER must be specified for TASK ' task]); +end + +inpath = [filepath 'input/']; +outpath = [filepath 'output/']; +refpath = [filepath 'reference/']; + +if strcmp(testname, 'all') + tests = {'apm','apmm','aec','aecm','agc','ns','vad'}; +else + tests = {testname}; +end + +if legacy + progname = './test'; +else + progname = './process_test'; +end + +global farFile; +global nearFile; +global eventFile; +global delayFile; +global driftFile; + +if legacy + farFile = 'vqeFar.pcm'; + nearFile = 'vqeNear.pcm'; + eventFile = 'vqeEvent.dat'; + delayFile = 'vqeBuf.dat'; + driftFile = 'vqeDrift.dat'; +else + farFile = 'apm_far.pcm'; + nearFile = 'apm_near.pcm'; + eventFile = 'apm_event.dat'; + delayFile = 'apm_delay.dat'; + driftFile = 'apm_drift.dat'; +end + +simulateMode = false; +nErr = 0; +nCases = 0; +for i=1:length(tests) + simulateMode = false; + + if strcmp(tests{i}, 'apm') + testdir = ['apm/']; + outfile = ['out']; + if legacy + opt = ['-ec 1 -agc 2 -nc 2 -vad 3']; + else + opt = ['--no_progress -hpf' ... + ' -aec --drift_compensation -agc --fixed_digital' ... + ' -ns --ns_moderate -vad']; + end + + elseif strcmp(tests{i}, 'apm-swb') + simulateMode = true; + testdir = ['apm-swb/']; + outfile = ['out']; + if legacy + opt = ['-fs 32000 -ec 1 -agc 2 -nc 2']; + else + opt = ['--no_progress -fs 32000 -hpf' ... + ' -aec --drift_compensation -agc --adaptive_digital' ... + ' -ns --ns_moderate -vad']; + end + elseif strcmp(tests{i}, 'apmm') + testdir = ['apmm/']; + outfile = ['out']; + opt = ['-aec --drift_compensation -agc --fixed_digital -hpf -ns ' ... + '--ns_moderate']; + + else + error(['TESTNAME ' tests{i} ' is not recognized']); + end + + inpathtest = [inpath testdir]; + outpathtest = [outpath testdir]; + refpathtest = [refpath testdir]; + + if ~exist(inpathtest,'dir') + error(['Input directory ' inpathtest ' does not exist']); + end + + if ~exist(refpathtest,'dir') + warning(['Reference directory ' refpathtest ' does not exist']); + end + + [status, errMsg] = mkdir(outpathtest); + if (status == 0) + error(errMsg); + end + + [nErr, nCases] = recurseDir(inpathtest, outpathtest, refpathtest, outfile, ... + progname, opt, simulateMode, nErr, nCases, task, casenumber, legacy); + + if strcmp(task, 'test') || strcmp(task, 'show') + system(['rm ' farFile]); + system(['rm ' nearFile]); + if simulateMode == false + system(['rm ' eventFile]); + system(['rm ' delayFile]); + system(['rm ' driftFile]); + end + end +end + +if ~strcmp(task, 'list') + if nErr == 0 + fprintf(1, '\nAll files are bit-exact to reference\n', nErr); + else + fprintf(1, '\n%d files are NOT bit-exact to reference\n', nErr); + end +end + + +function [nErrOut, nCases] = recurseDir(inpath, outpath, refpath, ... + outfile, progname, opt, simulateMode, nErr, nCases, task, casenumber, ... + legacy) + +global farFile; +global nearFile; +global eventFile; +global delayFile; +global driftFile; + +dirs = dir(inpath); +nDirs = 0; +nErrOut = nErr; +for i=3:length(dirs) % skip . and .. + nDirs = nDirs + dirs(i).isdir; +end + + +if nDirs == 0 + nCases = nCases + 1; + + if casenumber == nCases || casenumber == 0 + + if strcmp(task, 'list') + fprintf([num2str(nCases) '. ' outfile '\n']) + else + vadoutfile = ['vad_' outfile '.dat']; + outfile = [outfile '.pcm']; + + % Check for VAD test + vadTest = 0; + if ~isempty(findstr(opt, '-vad')) + vadTest = 1; + if legacy + opt = [opt ' ' outpath vadoutfile]; + else + opt = [opt ' --vad_out_file ' outpath vadoutfile]; + end + end + + if exist([inpath 'vqeFar.pcm']) + system(['ln -s -f ' inpath 'vqeFar.pcm ' farFile]); + elseif exist([inpath 'apm_far.pcm']) + system(['ln -s -f ' inpath 'apm_far.pcm ' farFile]); + end + + if exist([inpath 'vqeNear.pcm']) + system(['ln -s -f ' inpath 'vqeNear.pcm ' nearFile]); + elseif exist([inpath 'apm_near.pcm']) + system(['ln -s -f ' inpath 'apm_near.pcm ' nearFile]); + end + + if exist([inpath 'vqeEvent.dat']) + system(['ln -s -f ' inpath 'vqeEvent.dat ' eventFile]); + elseif exist([inpath 'apm_event.dat']) + system(['ln -s -f ' inpath 'apm_event.dat ' eventFile]); + end + + if exist([inpath 'vqeBuf.dat']) + system(['ln -s -f ' inpath 'vqeBuf.dat ' delayFile]); + elseif exist([inpath 'apm_delay.dat']) + system(['ln -s -f ' inpath 'apm_delay.dat ' delayFile]); + end + + if exist([inpath 'vqeSkew.dat']) + system(['ln -s -f ' inpath 'vqeSkew.dat ' driftFile]); + elseif exist([inpath 'vqeDrift.dat']) + system(['ln -s -f ' inpath 'vqeDrift.dat ' driftFile]); + elseif exist([inpath 'apm_drift.dat']) + system(['ln -s -f ' inpath 'apm_drift.dat ' driftFile]); + end + + if simulateMode == false + command = [progname ' -o ' outpath outfile ' ' opt]; + else + if legacy + inputCmd = [' -in ' nearFile]; + else + inputCmd = [' -i ' nearFile]; + end + + if exist([farFile]) + if legacy + inputCmd = [' -if ' farFile inputCmd]; + else + inputCmd = [' -ir ' farFile inputCmd]; + end + end + command = [progname inputCmd ' -o ' outpath outfile ' ' opt]; + end + % This prevents MATLAB from using its own C libraries. + shellcmd = ['bash -c "unset LD_LIBRARY_PATH;']; + fprintf([command '\n']); + [status, result] = system([shellcmd command '"']); + fprintf(result); + + fprintf(['Reference file: ' refpath outfile '\n']); + + if vadTest == 1 + equal_to_ref = are_files_equal([outpath vadoutfile], ... + [refpath vadoutfile], ... + 'int8'); + if ~equal_to_ref + nErr = nErr + 1; + end + end + + [equal_to_ref, diffvector] = are_files_equal([outpath outfile], ... + [refpath outfile], ... + 'int16'); + if ~equal_to_ref + nErr = nErr + 1; + end + + if strcmp(task, 'show') + % Assume the last init gives the sample rate of interest. + str_idx = strfind(result, 'Sample rate:'); + fs = str2num(result(str_idx(end) + 13:str_idx(end) + 17)); + fprintf('Using %d Hz\n', fs); + + if exist([farFile]) + spclab(fs, farFile, nearFile, [refpath outfile], ... + [outpath outfile], diffvector); + %spclab(fs, diffvector); + else + spclab(fs, nearFile, [refpath outfile], [outpath outfile], ... + diffvector); + %spclab(fs, diffvector); + end + end + end + end +else + + for i=3:length(dirs) + if dirs(i).isdir + [nErr, nCases] = recurseDir([inpath dirs(i).name '/'], outpath, ... + refpath,[outfile '_' dirs(i).name], progname, opt, ... + simulateMode, nErr, nCases, task, casenumber, legacy); + end + end +end +nErrOut = nErr; + +function [are_equal, diffvector] = ... + are_files_equal(newfile, reffile, precision, diffvector) + +are_equal = false; +diffvector = 0; +if ~exist(newfile,'file') + warning(['Output file ' newfile ' does not exist']); + return +end + +if ~exist(reffile,'file') + warning(['Reference file ' reffile ' does not exist']); + return +end + +fid = fopen(newfile,'rb'); +new = fread(fid,inf,precision); +fclose(fid); + +fid = fopen(reffile,'rb'); +ref = fread(fid,inf,precision); +fclose(fid); + +if length(new) ~= length(ref) + warning('Reference is not the same length as output'); + minlength = min(length(new), length(ref)); + new = new(1:minlength); + ref = ref(1:minlength); +end +diffvector = new - ref; + +if isequal(new, ref) + fprintf([newfile ' is bit-exact to reference\n']); + are_equal = true; +else + if isempty(new) + warning([newfile ' is empty']); + return + end + snr = snrseg(new,ref,80); + fprintf('\n'); + are_equal = false; +end diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.cc new file mode 100644 index 0000000000..0f0e5cd520 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/audio_buffer_tools.h" + +#include + +namespace webrtc { +namespace test { + +void SetupFrame(const StreamConfig& stream_config, + std::vector* frame, + std::vector* frame_samples) { + frame_samples->resize(stream_config.num_channels() * + stream_config.num_frames()); + frame->resize(stream_config.num_channels()); + for (size_t ch = 0; ch < stream_config.num_channels(); ++ch) { + (*frame)[ch] = &(*frame_samples)[ch * stream_config.num_frames()]; + } +} + +void CopyVectorToAudioBuffer(const StreamConfig& stream_config, + rtc::ArrayView source, + AudioBuffer* destination) { + std::vector input; + std::vector input_samples; + + SetupFrame(stream_config, &input, &input_samples); + + RTC_CHECK_EQ(input_samples.size(), source.size()); + memcpy(input_samples.data(), source.data(), + source.size() * sizeof(source[0])); + + destination->CopyFrom(&input[0], stream_config); +} + +void ExtractVectorFromAudioBuffer(const StreamConfig& stream_config, + AudioBuffer* source, + std::vector* destination) { + std::vector output; + + SetupFrame(stream_config, &output, destination); + + source->CopyTo(stream_config, &output[0]); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.h new file mode 100644 index 0000000000..dc53e4f6a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_buffer_tools.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_AUDIO_BUFFER_TOOLS_H_ +#define MODULES_AUDIO_PROCESSING_TEST_AUDIO_BUFFER_TOOLS_H_ + +#include +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { +namespace test { + +// Copies a vector into an audiobuffer. +void CopyVectorToAudioBuffer(const StreamConfig& stream_config, + rtc::ArrayView source, + AudioBuffer* destination); + +// Extracts a vector from an audiobuffer. +void ExtractVectorFromAudioBuffer(const StreamConfig& stream_config, + AudioBuffer* source, + std::vector* destination); + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_AUDIO_BUFFER_TOOLS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.cc new file mode 100644 index 0000000000..416fd98849 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.cc @@ -0,0 +1,451 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/audio_processing_simulator.h" + +#include +#include +#include +#include +#include +#include + +#include "common_audio/include/audio_util.h" +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/test/fake_recording_device.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/stringutils.h" + +namespace webrtc { +namespace test { +namespace { + +void CopyFromAudioFrame(const AudioFrame& src, ChannelBuffer* dest) { + RTC_CHECK_EQ(src.num_channels_, dest->num_channels()); + RTC_CHECK_EQ(src.samples_per_channel_, dest->num_frames()); + // Copy the data from the input buffer. + std::vector tmp(src.samples_per_channel_ * src.num_channels_); + S16ToFloat(src.data(), tmp.size(), tmp.data()); + Deinterleave(tmp.data(), src.samples_per_channel_, src.num_channels_, + dest->channels()); +} + +std::string GetIndexedOutputWavFilename(const std::string& wav_name, + int counter) { + std::stringstream ss; + ss << wav_name.substr(0, wav_name.size() - 4) << "_" << counter + << wav_name.substr(wav_name.size() - 4); + return ss.str(); +} + +void WriteEchoLikelihoodGraphFileHeader(std::ofstream* output_file) { + (*output_file) << "import numpy as np" << std::endl + << "import matplotlib.pyplot as plt" << std::endl + << "y = np.array(["; +} + +void WriteEchoLikelihoodGraphFileFooter(std::ofstream* output_file) { + (*output_file) << "])" << std::endl + << "x = np.arange(len(y))*.01" << std::endl + << "plt.plot(x, y)" << std::endl + << "plt.ylabel('Echo likelihood')" << std::endl + << "plt.xlabel('Time (s)')" << std::endl + << "plt.ylim([0,1])" << std::endl + << "plt.show()" << std::endl; +} + +} // namespace + +SimulationSettings::SimulationSettings() = default; +SimulationSettings::SimulationSettings(const SimulationSettings&) = default; +SimulationSettings::~SimulationSettings() = default; + +void CopyToAudioFrame(const ChannelBuffer& src, AudioFrame* dest) { + RTC_CHECK_EQ(src.num_channels(), dest->num_channels_); + RTC_CHECK_EQ(src.num_frames(), dest->samples_per_channel_); + int16_t* dest_data = dest->mutable_data(); + for (size_t ch = 0; ch < dest->num_channels_; ++ch) { + for (size_t sample = 0; sample < dest->samples_per_channel_; ++sample) { + dest_data[sample * dest->num_channels_ + ch] = + src.channels()[ch][sample] * 32767; + } + } +} + +AudioProcessingSimulator::AudioProcessingSimulator( + const SimulationSettings& settings) + : settings_(settings), + analog_mic_level_(settings.initial_mic_level), + fake_recording_device_( + settings.initial_mic_level, + settings_.simulate_mic_gain ? *settings.simulated_mic_kind : 0), + worker_queue_("file_writer_task_queue") { + if (settings_.ed_graph_output_filename && + !settings_.ed_graph_output_filename->empty()) { + residual_echo_likelihood_graph_writer_.open( + *settings_.ed_graph_output_filename); + RTC_CHECK(residual_echo_likelihood_graph_writer_.is_open()); + WriteEchoLikelihoodGraphFileHeader(&residual_echo_likelihood_graph_writer_); + } + + if (settings_.simulate_mic_gain) + RTC_LOG(LS_VERBOSE) << "Simulating analog mic gain"; +} + +AudioProcessingSimulator::~AudioProcessingSimulator() { + if (residual_echo_likelihood_graph_writer_.is_open()) { + WriteEchoLikelihoodGraphFileFooter(&residual_echo_likelihood_graph_writer_); + residual_echo_likelihood_graph_writer_.close(); + } +} + +AudioProcessingSimulator::ScopedTimer::~ScopedTimer() { + int64_t interval = rtc::TimeNanos() - start_time_; + proc_time_->sum += interval; + proc_time_->max = std::max(proc_time_->max, interval); + proc_time_->min = std::min(proc_time_->min, interval); +} + +void AudioProcessingSimulator::ProcessStream(bool fixed_interface) { + // Optionally use the fake recording device to simulate analog gain. + if (settings_.simulate_mic_gain) { + if (settings_.aec_dump_input_filename) { + // When the analog gain is simulated and an AEC dump is used as input, set + // the undo level to |aec_dump_mic_level_| to virtually restore the + // unmodified microphone signal level. + fake_recording_device_.SetUndoMicLevel(aec_dump_mic_level_); + } + + if (fixed_interface) { + fake_recording_device_.SimulateAnalogGain(&fwd_frame_); + } else { + fake_recording_device_.SimulateAnalogGain(in_buf_.get()); + } + + // Notify the current mic level to AGC. + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->set_stream_analog_level( + fake_recording_device_.MicLevel())); + } else { + // Notify the current mic level to AGC. + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->set_stream_analog_level( + settings_.aec_dump_input_filename ? aec_dump_mic_level_ + : analog_mic_level_)); + } + + // Process the current audio frame. + if (fixed_interface) { + { + const auto st = ScopedTimer(mutable_proc_time()); + RTC_CHECK_EQ(AudioProcessing::kNoError, ap_->ProcessStream(&fwd_frame_)); + } + CopyFromAudioFrame(fwd_frame_, out_buf_.get()); + } else { + const auto st = ScopedTimer(mutable_proc_time()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->ProcessStream(in_buf_->channels(), in_config_, + out_config_, out_buf_->channels())); + } + + // Store the mic level suggested by AGC. + // Note that when the analog gain is simulated and an AEC dump is used as + // input, |analog_mic_level_| will not be used with set_stream_analog_level(). + analog_mic_level_ = ap_->gain_control()->stream_analog_level(); + if (settings_.simulate_mic_gain) { + fake_recording_device_.SetMicLevel(analog_mic_level_); + } + + if (buffer_writer_) { + buffer_writer_->Write(*out_buf_); + } + + if (residual_echo_likelihood_graph_writer_.is_open()) { + auto stats = ap_->GetStatistics(); + residual_echo_likelihood_graph_writer_ << stats.residual_echo_likelihood + << ", "; + } + + ++num_process_stream_calls_; +} + +void AudioProcessingSimulator::ProcessReverseStream(bool fixed_interface) { + if (fixed_interface) { + const auto st = ScopedTimer(mutable_proc_time()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->ProcessReverseStream(&rev_frame_)); + CopyFromAudioFrame(rev_frame_, reverse_out_buf_.get()); + + } else { + const auto st = ScopedTimer(mutable_proc_time()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->ProcessReverseStream( + reverse_in_buf_->channels(), reverse_in_config_, + reverse_out_config_, reverse_out_buf_->channels())); + } + + if (reverse_buffer_writer_) { + reverse_buffer_writer_->Write(*reverse_out_buf_); + } + + ++num_reverse_process_stream_calls_; +} + +void AudioProcessingSimulator::SetupBuffersConfigsOutputs( + int input_sample_rate_hz, + int output_sample_rate_hz, + int reverse_input_sample_rate_hz, + int reverse_output_sample_rate_hz, + int input_num_channels, + int output_num_channels, + int reverse_input_num_channels, + int reverse_output_num_channels) { + in_config_ = StreamConfig(input_sample_rate_hz, input_num_channels); + in_buf_.reset(new ChannelBuffer( + rtc::CheckedDivExact(input_sample_rate_hz, kChunksPerSecond), + input_num_channels)); + + reverse_in_config_ = + StreamConfig(reverse_input_sample_rate_hz, reverse_input_num_channels); + reverse_in_buf_.reset(new ChannelBuffer( + rtc::CheckedDivExact(reverse_input_sample_rate_hz, kChunksPerSecond), + reverse_input_num_channels)); + + out_config_ = StreamConfig(output_sample_rate_hz, output_num_channels); + out_buf_.reset(new ChannelBuffer( + rtc::CheckedDivExact(output_sample_rate_hz, kChunksPerSecond), + output_num_channels)); + + reverse_out_config_ = + StreamConfig(reverse_output_sample_rate_hz, reverse_output_num_channels); + reverse_out_buf_.reset(new ChannelBuffer( + rtc::CheckedDivExact(reverse_output_sample_rate_hz, kChunksPerSecond), + reverse_output_num_channels)); + + fwd_frame_.sample_rate_hz_ = input_sample_rate_hz; + fwd_frame_.samples_per_channel_ = + rtc::CheckedDivExact(fwd_frame_.sample_rate_hz_, kChunksPerSecond); + fwd_frame_.num_channels_ = input_num_channels; + + rev_frame_.sample_rate_hz_ = reverse_input_sample_rate_hz; + rev_frame_.samples_per_channel_ = + rtc::CheckedDivExact(rev_frame_.sample_rate_hz_, kChunksPerSecond); + rev_frame_.num_channels_ = reverse_input_num_channels; + + if (settings_.use_verbose_logging) { + rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE); + + std::cout << "Sample rates:" << std::endl; + std::cout << " Forward input: " << input_sample_rate_hz << std::endl; + std::cout << " Forward output: " << output_sample_rate_hz << std::endl; + std::cout << " Reverse input: " << reverse_input_sample_rate_hz + << std::endl; + std::cout << " Reverse output: " << reverse_output_sample_rate_hz + << std::endl; + std::cout << "Number of channels: " << std::endl; + std::cout << " Forward input: " << input_num_channels << std::endl; + std::cout << " Forward output: " << output_num_channels << std::endl; + std::cout << " Reverse input: " << reverse_input_num_channels << std::endl; + std::cout << " Reverse output: " << reverse_output_num_channels + << std::endl; + } + + SetupOutput(); +} + +void AudioProcessingSimulator::SetupOutput() { + if (settings_.output_filename) { + std::string filename; + if (settings_.store_intermediate_output) { + filename = GetIndexedOutputWavFilename(*settings_.output_filename, + output_reset_counter_); + } else { + filename = *settings_.output_filename; + } + + std::unique_ptr out_file( + new WavWriter(filename, out_config_.sample_rate_hz(), + static_cast(out_config_.num_channels()))); + buffer_writer_.reset(new ChannelBufferWavWriter(std::move(out_file))); + } + + if (settings_.reverse_output_filename) { + std::string filename; + if (settings_.store_intermediate_output) { + filename = GetIndexedOutputWavFilename(*settings_.reverse_output_filename, + output_reset_counter_); + } else { + filename = *settings_.reverse_output_filename; + } + + std::unique_ptr reverse_out_file( + new WavWriter(filename, reverse_out_config_.sample_rate_hz(), + static_cast(reverse_out_config_.num_channels()))); + reverse_buffer_writer_.reset( + new ChannelBufferWavWriter(std::move(reverse_out_file))); + } + + ++output_reset_counter_; +} + +void AudioProcessingSimulator::DestroyAudioProcessor() { + if (settings_.aec_dump_output_filename) { + ap_->DetachAecDump(); + } +} + +void AudioProcessingSimulator::CreateAudioProcessor() { + Config config; + AudioProcessing::Config apm_config; + std::unique_ptr echo_control_factory; + if (settings_.use_bf && *settings_.use_bf) { + config.Set(new Beamforming( + true, ParseArrayGeometry(*settings_.microphone_positions), + SphericalPointf(DegreesToRadians(settings_.target_angle_degrees), 0.f, + 1.f))); + } + if (settings_.use_ts) { + config.Set(new ExperimentalNs(*settings_.use_ts)); + } + if (settings_.use_ie) { + config.Set(new Intelligibility(*settings_.use_ie)); + } + if (settings_.use_agc2) { + apm_config.gain_controller2.enabled = *settings_.use_agc2; + apm_config.gain_controller2.fixed_gain_db = settings_.agc2_fixed_gain_db; + } + if (settings_.use_aec3 && *settings_.use_aec3) { + echo_control_factory.reset(new EchoCanceller3Factory()); + } + if (settings_.use_lc) { + apm_config.level_controller.enabled = *settings_.use_lc; + } + if (settings_.use_hpf) { + apm_config.high_pass_filter.enabled = *settings_.use_hpf; + } + + if (settings_.use_refined_adaptive_filter) { + config.Set( + new RefinedAdaptiveFilter(*settings_.use_refined_adaptive_filter)); + } + config.Set(new ExtendedFilter( + !settings_.use_extended_filter || *settings_.use_extended_filter)); + config.Set(new DelayAgnostic(!settings_.use_delay_agnostic || + *settings_.use_delay_agnostic)); + config.Set(new ExperimentalAgc( + !settings_.use_experimental_agc || *settings_.use_experimental_agc)); + if (settings_.use_ed) { + apm_config.residual_echo_detector.enabled = *settings_.use_ed; + } + + ap_.reset(AudioProcessing::Create(config, nullptr, + std::move(echo_control_factory), nullptr)); + RTC_CHECK(ap_); + + ap_->ApplyConfig(apm_config); + + if (settings_.use_aec) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_cancellation()->Enable(*settings_.use_aec)); + } + if (settings_.use_aecm) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_control_mobile()->Enable(*settings_.use_aecm)); + } + if (settings_.use_agc) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->Enable(*settings_.use_agc)); + } + if (settings_.use_ns) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->noise_suppression()->Enable(*settings_.use_ns)); + } + if (settings_.use_le) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->level_estimator()->Enable(*settings_.use_le)); + } + if (settings_.use_vad) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->voice_detection()->Enable(*settings_.use_vad)); + } + if (settings_.use_agc_limiter) { + RTC_CHECK_EQ(AudioProcessing::kNoError, ap_->gain_control()->enable_limiter( + *settings_.use_agc_limiter)); + } + if (settings_.agc_target_level) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->set_target_level_dbfs( + *settings_.agc_target_level)); + } + if (settings_.agc_compression_gain) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->gain_control()->set_compression_gain_db( + *settings_.agc_compression_gain)); + } + if (settings_.agc_mode) { + RTC_CHECK_EQ( + AudioProcessing::kNoError, + ap_->gain_control()->set_mode( + static_cast(*settings_.agc_mode))); + } + + if (settings_.use_drift_compensation) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_cancellation()->enable_drift_compensation( + *settings_.use_drift_compensation)); + } + + if (settings_.aec_suppression_level) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_cancellation()->set_suppression_level( + static_cast( + *settings_.aec_suppression_level))); + } + + if (settings_.aecm_routing_mode) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_control_mobile()->set_routing_mode( + static_cast( + *settings_.aecm_routing_mode))); + } + + if (settings_.use_aecm_comfort_noise) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->echo_control_mobile()->enable_comfort_noise( + *settings_.use_aecm_comfort_noise)); + } + + if (settings_.vad_likelihood) { + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->voice_detection()->set_likelihood( + static_cast( + *settings_.vad_likelihood))); + } + if (settings_.ns_level) { + RTC_CHECK_EQ( + AudioProcessing::kNoError, + ap_->noise_suppression()->set_level( + static_cast(*settings_.ns_level))); + } + + if (settings_.use_ts) { + ap_->set_stream_key_pressed(*settings_.use_ts); + } + + if (settings_.aec_dump_output_filename) { + ap_->AttachAecDump(AecDumpFactory::Create( + *settings_.aec_dump_output_filename, -1, &worker_queue_)); + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.h new file mode 100644 index 0000000000..41a3f45106 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audio_processing_simulator.h @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_AUDIO_PROCESSING_SIMULATOR_H_ +#define MODULES_AUDIO_PROCESSING_TEST_AUDIO_PROCESSING_SIMULATOR_H_ + +#include +#include +#include +#include +#include + +#include "api/optional.h" +#include "common_audio/channel_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/test/fake_recording_device.h" +#include "modules/audio_processing/test/test_utils.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/timeutils.h" + +namespace webrtc { +namespace test { + +// Holds all the parameters available for controlling the simulation. +struct SimulationSettings { + SimulationSettings(); + SimulationSettings(const SimulationSettings&); + ~SimulationSettings(); + rtc::Optional stream_delay; + rtc::Optional stream_drift_samples; + rtc::Optional output_sample_rate_hz; + rtc::Optional output_num_channels; + rtc::Optional reverse_output_sample_rate_hz; + rtc::Optional reverse_output_num_channels; + rtc::Optional microphone_positions; + int target_angle_degrees = 90; + rtc::Optional output_filename; + rtc::Optional reverse_output_filename; + rtc::Optional input_filename; + rtc::Optional reverse_input_filename; + rtc::Optional artificial_nearend_filename; + rtc::Optional use_aec; + rtc::Optional use_aecm; + rtc::Optional use_ed; // Residual Echo Detector. + rtc::Optional ed_graph_output_filename; + rtc::Optional use_agc; + rtc::Optional use_agc2; + rtc::Optional use_hpf; + rtc::Optional use_ns; + rtc::Optional use_ts; + rtc::Optional use_bf; + rtc::Optional use_ie; + rtc::Optional use_vad; + rtc::Optional use_le; + rtc::Optional use_all; + rtc::Optional aec_suppression_level; + rtc::Optional use_delay_agnostic; + rtc::Optional use_extended_filter; + rtc::Optional use_drift_compensation; + rtc::Optional use_aec3; + rtc::Optional use_lc; + rtc::Optional use_experimental_agc; + rtc::Optional aecm_routing_mode; + rtc::Optional use_aecm_comfort_noise; + rtc::Optional agc_mode; + rtc::Optional agc_target_level; + rtc::Optional use_agc_limiter; + rtc::Optional agc_compression_gain; + float agc2_fixed_gain_db; + rtc::Optional vad_likelihood; + rtc::Optional ns_level; + rtc::Optional use_refined_adaptive_filter; + int initial_mic_level; + bool simulate_mic_gain = false; + rtc::Optional simulated_mic_kind; + bool report_performance = false; + bool report_bitexactness = false; + bool use_verbose_logging = false; + bool discard_all_settings_in_aecdump = true; + rtc::Optional aec_dump_input_filename; + rtc::Optional aec_dump_output_filename; + bool fixed_interface = false; + bool store_intermediate_output = false; + rtc::Optional custom_call_order_filename; +}; + +// Holds a few statistics about a series of TickIntervals. +struct TickIntervalStats { + TickIntervalStats() : min(std::numeric_limits::max()) {} + int64_t sum; + int64_t max; + int64_t min; +}; + +// Copies samples present in a ChannelBuffer into an AudioFrame. +void CopyToAudioFrame(const ChannelBuffer& src, AudioFrame* dest); + +// Provides common functionality for performing audioprocessing simulations. +class AudioProcessingSimulator { + public: + static const int kChunksPerSecond = 1000 / AudioProcessing::kChunkSizeMs; + + explicit AudioProcessingSimulator(const SimulationSettings& settings); + virtual ~AudioProcessingSimulator(); + + // Processes the data in the input. + virtual void Process() = 0; + + // Returns the execution time of all AudioProcessing calls. + const TickIntervalStats& proc_time() const { return proc_time_; } + + // Reports whether the processed recording was bitexact. + bool OutputWasBitexact() { return bitexact_output_; } + + size_t get_num_process_stream_calls() { return num_process_stream_calls_; } + size_t get_num_reverse_process_stream_calls() { + return num_reverse_process_stream_calls_; + } + + protected: + // RAII class for execution time measurement. Updates the provided + // TickIntervalStats based on the time between ScopedTimer creation and + // leaving the enclosing scope. + class ScopedTimer { + public: + explicit ScopedTimer(TickIntervalStats* proc_time) + : proc_time_(proc_time), start_time_(rtc::TimeNanos()) {} + + ~ScopedTimer(); + + private: + TickIntervalStats* const proc_time_; + int64_t start_time_; + }; + + TickIntervalStats* mutable_proc_time() { return &proc_time_; } + void ProcessStream(bool fixed_interface); + void ProcessReverseStream(bool fixed_interface); + void CreateAudioProcessor(); + void DestroyAudioProcessor(); + void SetupBuffersConfigsOutputs(int input_sample_rate_hz, + int output_sample_rate_hz, + int reverse_input_sample_rate_hz, + int reverse_output_sample_rate_hz, + int input_num_channels, + int output_num_channels, + int reverse_input_num_channels, + int reverse_output_num_channels); + + const SimulationSettings settings_; + std::unique_ptr ap_; + + std::unique_ptr> in_buf_; + std::unique_ptr> out_buf_; + std::unique_ptr> reverse_in_buf_; + std::unique_ptr> reverse_out_buf_; + StreamConfig in_config_; + StreamConfig out_config_; + StreamConfig reverse_in_config_; + StreamConfig reverse_out_config_; + std::unique_ptr buffer_reader_; + std::unique_ptr reverse_buffer_reader_; + AudioFrame rev_frame_; + AudioFrame fwd_frame_; + bool bitexact_output_ = true; + int aec_dump_mic_level_ = 0; + + private: + void SetupOutput(); + + size_t num_process_stream_calls_ = 0; + size_t num_reverse_process_stream_calls_ = 0; + size_t output_reset_counter_ = 0; + std::unique_ptr buffer_writer_; + std::unique_ptr reverse_buffer_writer_; + TickIntervalStats proc_time_; + std::ofstream residual_echo_likelihood_graph_writer_; + int analog_mic_level_; + FakeRecordingDevice fake_recording_device_; + + rtc::TaskQueue worker_queue_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioProcessingSimulator); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_AUDIO_PROCESSING_SIMULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/audioproc_float.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audioproc_float.cc new file mode 100644 index 0000000000..c5229a4e10 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/audioproc_float.cc @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/test/aec_dump_based_simulator.h" +#include "modules/audio_processing/test/audio_processing_simulator.h" +#include "modules/audio_processing/test/wav_based_simulator.h" +#include "rtc_base/flags.h" + +namespace webrtc { +namespace test { +namespace { + +const int kParameterNotSpecifiedValue = -10000; + +const char kUsageDescription[] = + "Usage: audioproc_f [options] -i \n" + " or\n" + " audioproc_f [options] -dump_input \n" + "\n\n" + "Command-line tool to simulate a call using the audio " + "processing module, either based on wav files or " + "protobuf debug dump recordings.\n"; + +DEFINE_string(dump_input, "", "Aec dump input filename"); +DEFINE_string(dump_output, "", "Aec dump output filename"); +DEFINE_string(i, "", "Forward stream input wav filename"); +DEFINE_string(o, "", "Forward stream output wav filename"); +DEFINE_string(ri, "", "Reverse stream input wav filename"); +DEFINE_string(ro, "", "Reverse stream output wav filename"); +DEFINE_string(artificial_nearend, "", "Artificial nearend wav filename"); +DEFINE_int(output_num_channels, + kParameterNotSpecifiedValue, + "Number of forward stream output channels"); +DEFINE_int(reverse_output_num_channels, + kParameterNotSpecifiedValue, + "Number of Reverse stream output channels"); +DEFINE_int(output_sample_rate_hz, + kParameterNotSpecifiedValue, + "Forward stream output sample rate in Hz"); +DEFINE_int(reverse_output_sample_rate_hz, + kParameterNotSpecifiedValue, + "Reverse stream output sample rate in Hz"); +DEFINE_string(mic_positions, + "", + "Space delimited cartesian coordinates of microphones in " + "meters. The coordinates of each point are contiguous. For a " + "two element array: \"x1 y1 z1 x2 y2 z2\""); +DEFINE_int(target_angle_degrees, + 90, + "The azimuth of the target in degrees (0-359). Only applies to " + "beamforming."); +DEFINE_bool(fixed_interface, + false, + "Use the fixed interface when operating on wav files"); +DEFINE_int(aec, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the echo canceller"); +DEFINE_int(aecm, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the mobile echo controller"); +DEFINE_int(ed, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate (0) the residual echo detector"); +DEFINE_string(ed_graph, "", "Output filename for graph of echo likelihood"); +DEFINE_int(agc, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the AGC"); +DEFINE_int(agc2, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the AGC2"); +DEFINE_int(hpf, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the high-pass filter"); +DEFINE_int(ns, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the noise suppressor"); +DEFINE_int(ts, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the transient suppressor"); +DEFINE_int(bf, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the beamformer"); +DEFINE_int(ie, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the intelligibility enhancer"); +DEFINE_int(vad, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the voice activity detector"); +DEFINE_int(le, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the level estimator"); +DEFINE_bool(all_default, + false, + "Activate all of the default components (will be overridden by any " + "other settings)"); +DEFINE_int(aec_suppression_level, + kParameterNotSpecifiedValue, + "Set the aec suppression level (0-2)"); +DEFINE_int(delay_agnostic, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the AEC delay agnostic mode"); +DEFINE_int(extended_filter, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the AEC extended filter mode"); +DEFINE_int(drift_compensation, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the drift compensation"); +DEFINE_int(aec3, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the experimental AEC mode AEC3"); +DEFINE_int(lc, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the level control"); +DEFINE_int(experimental_agc, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the experimental AGC"); +DEFINE_int( + refined_adaptive_filter, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the refined adaptive filter functionality"); +DEFINE_int(aecm_routing_mode, + kParameterNotSpecifiedValue, + "Specify the AECM routing mode (0-4)"); +DEFINE_int(aecm_comfort_noise, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the AECM comfort noise"); +DEFINE_int(agc_mode, + kParameterNotSpecifiedValue, + "Specify the AGC mode (0-2)"); +DEFINE_int(agc_target_level, + kParameterNotSpecifiedValue, + "Specify the AGC target level (0-31)"); +DEFINE_int(agc_limiter, + kParameterNotSpecifiedValue, + "Activate (1) or deactivate(0) the level estimator"); +DEFINE_int(agc_compression_gain, + kParameterNotSpecifiedValue, + "Specify the AGC compression gain (0-90)"); +DEFINE_float(agc2_fixed_gain_db, 0.f, "AGC2 fixed gain (dB) to apply"); +DEFINE_int(vad_likelihood, + kParameterNotSpecifiedValue, + "Specify the VAD likelihood (0-3)"); +DEFINE_int(ns_level, + kParameterNotSpecifiedValue, + "Specify the NS level (0-3)"); +DEFINE_int(stream_delay, + kParameterNotSpecifiedValue, + "Specify the stream delay in ms to use"); +DEFINE_int(stream_drift_samples, + kParameterNotSpecifiedValue, + "Specify the number of stream drift samples to use"); +DEFINE_int(initial_mic_level, 100, "Initial mic level (0-255)"); +DEFINE_int(simulate_mic_gain, + 0, + "Activate (1) or deactivate(0) the analog mic gain simulation"); +DEFINE_int(simulated_mic_kind, + kParameterNotSpecifiedValue, + "Specify which microphone kind to use for microphone simulation"); +DEFINE_bool(performance_report, false, "Report the APM performance "); +DEFINE_bool(verbose, false, "Produce verbose output"); +DEFINE_bool(bitexactness_report, + false, + "Report bitexactness for aec dump result reproduction"); +DEFINE_bool(discard_settings_in_aecdump, + false, + "Discard any config settings specified in the aec dump"); +DEFINE_bool(store_intermediate_output, + false, + "Creates new output files after each init"); +DEFINE_string(custom_call_order_file, "", "Custom process API call order file"); +DEFINE_bool(help, false, "Print this message"); + +void SetSettingIfSpecified(const std::string& value, + rtc::Optional* parameter) { + if (value.compare("") != 0) { + *parameter = value; + } +} + +void SetSettingIfSpecified(int value, rtc::Optional* parameter) { + if (value != kParameterNotSpecifiedValue) { + *parameter = value; + } +} + +void SetSettingIfFlagSet(int32_t flag, rtc::Optional* parameter) { + if (flag == 0) { + *parameter = false; + } else if (flag == 1) { + *parameter = true; + } +} + +SimulationSettings CreateSettings() { + SimulationSettings settings; + if (FLAG_all_default) { + settings.use_le = true; + settings.use_vad = true; + settings.use_ie = false; + settings.use_bf = false; + settings.use_ts = true; + settings.use_ns = true; + settings.use_hpf = true; + settings.use_agc = true; + settings.use_agc2 = false; + settings.use_aec = true; + settings.use_aecm = false; + settings.use_ed = false; + } + SetSettingIfSpecified(FLAG_dump_input, &settings.aec_dump_input_filename); + SetSettingIfSpecified(FLAG_dump_output, &settings.aec_dump_output_filename); + SetSettingIfSpecified(FLAG_i, &settings.input_filename); + SetSettingIfSpecified(FLAG_o, &settings.output_filename); + SetSettingIfSpecified(FLAG_ri, &settings.reverse_input_filename); + SetSettingIfSpecified(FLAG_ro, &settings.reverse_output_filename); + SetSettingIfSpecified(FLAG_artificial_nearend, + &settings.artificial_nearend_filename); + SetSettingIfSpecified(FLAG_output_num_channels, + &settings.output_num_channels); + SetSettingIfSpecified(FLAG_reverse_output_num_channels, + &settings.reverse_output_num_channels); + SetSettingIfSpecified(FLAG_output_sample_rate_hz, + &settings.output_sample_rate_hz); + SetSettingIfSpecified(FLAG_reverse_output_sample_rate_hz, + &settings.reverse_output_sample_rate_hz); + SetSettingIfSpecified(FLAG_mic_positions, &settings.microphone_positions); + settings.target_angle_degrees = FLAG_target_angle_degrees; + SetSettingIfFlagSet(FLAG_aec, &settings.use_aec); + SetSettingIfFlagSet(FLAG_aecm, &settings.use_aecm); + SetSettingIfFlagSet(FLAG_ed, &settings.use_ed); + SetSettingIfSpecified(FLAG_ed_graph, &settings.ed_graph_output_filename); + SetSettingIfFlagSet(FLAG_agc, &settings.use_agc); + SetSettingIfFlagSet(FLAG_agc2, &settings.use_agc2); + SetSettingIfFlagSet(FLAG_hpf, &settings.use_hpf); + SetSettingIfFlagSet(FLAG_ns, &settings.use_ns); + SetSettingIfFlagSet(FLAG_ts, &settings.use_ts); + SetSettingIfFlagSet(FLAG_bf, &settings.use_bf); + SetSettingIfFlagSet(FLAG_ie, &settings.use_ie); + SetSettingIfFlagSet(FLAG_vad, &settings.use_vad); + SetSettingIfFlagSet(FLAG_le, &settings.use_le); + SetSettingIfSpecified(FLAG_aec_suppression_level, + &settings.aec_suppression_level); + SetSettingIfFlagSet(FLAG_delay_agnostic, &settings.use_delay_agnostic); + SetSettingIfFlagSet(FLAG_extended_filter, &settings.use_extended_filter); + SetSettingIfFlagSet(FLAG_drift_compensation, + &settings.use_drift_compensation); + SetSettingIfFlagSet(FLAG_refined_adaptive_filter, + &settings.use_refined_adaptive_filter); + + SetSettingIfFlagSet(FLAG_aec3, &settings.use_aec3); + SetSettingIfFlagSet(FLAG_lc, &settings.use_lc); + SetSettingIfFlagSet(FLAG_experimental_agc, &settings.use_experimental_agc); + SetSettingIfSpecified(FLAG_aecm_routing_mode, &settings.aecm_routing_mode); + SetSettingIfFlagSet(FLAG_aecm_comfort_noise, + &settings.use_aecm_comfort_noise); + SetSettingIfSpecified(FLAG_agc_mode, &settings.agc_mode); + SetSettingIfSpecified(FLAG_agc_target_level, &settings.agc_target_level); + SetSettingIfFlagSet(FLAG_agc_limiter, &settings.use_agc_limiter); + SetSettingIfSpecified(FLAG_agc_compression_gain, + &settings.agc_compression_gain); + settings.agc2_fixed_gain_db = FLAG_agc2_fixed_gain_db; + SetSettingIfSpecified(FLAG_vad_likelihood, &settings.vad_likelihood); + SetSettingIfSpecified(FLAG_ns_level, &settings.ns_level); + SetSettingIfSpecified(FLAG_stream_delay, &settings.stream_delay); + SetSettingIfSpecified(FLAG_stream_drift_samples, + &settings.stream_drift_samples); + SetSettingIfSpecified(FLAG_custom_call_order_file, + &settings.custom_call_order_filename); + settings.initial_mic_level = FLAG_initial_mic_level; + settings.simulate_mic_gain = FLAG_simulate_mic_gain; + SetSettingIfSpecified(FLAG_simulated_mic_kind, &settings.simulated_mic_kind); + settings.report_performance = FLAG_performance_report; + settings.use_verbose_logging = FLAG_verbose; + settings.report_bitexactness = FLAG_bitexactness_report; + settings.discard_all_settings_in_aecdump = FLAG_discard_settings_in_aecdump; + settings.fixed_interface = FLAG_fixed_interface; + settings.store_intermediate_output = FLAG_store_intermediate_output; + + return settings; +} + +void ReportConditionalErrorAndExit(bool condition, const std::string& message) { + if (condition) { + std::cerr << message << std::endl; + exit(1); + } +} + +void PerformBasicParameterSanityChecks(const SimulationSettings& settings) { + if (settings.input_filename || settings.reverse_input_filename) { + ReportConditionalErrorAndExit(!!settings.aec_dump_input_filename, + "Error: The aec dump cannot be specified " + "together with input wav files!\n"); + + ReportConditionalErrorAndExit(!!settings.artificial_nearend_filename, + "Error: The artificial nearend cannot be " + "specified together with input wav files!\n"); + + ReportConditionalErrorAndExit(!settings.input_filename, + "Error: When operating at wav files, the " + "input wav filename must be " + "specified!\n"); + + ReportConditionalErrorAndExit( + settings.reverse_output_filename && !settings.reverse_input_filename, + "Error: When operating at wav files, the reverse input wav filename " + "must be specified if the reverse output wav filename is specified!\n"); + } else { + ReportConditionalErrorAndExit(!settings.aec_dump_input_filename, + "Error: Either the aec dump or the wav " + "input files must be specified!\n"); + } + + ReportConditionalErrorAndExit( + settings.use_aec && *settings.use_aec && settings.use_aecm && + *settings.use_aecm, + "Error: The AEC and the AECM cannot be activated at the same time!\n"); + + ReportConditionalErrorAndExit( + settings.output_sample_rate_hz && *settings.output_sample_rate_hz <= 0, + "Error: --output_sample_rate_hz must be positive!\n"); + + ReportConditionalErrorAndExit( + settings.reverse_output_sample_rate_hz && + settings.output_sample_rate_hz && + *settings.output_sample_rate_hz <= 0, + "Error: --reverse_output_sample_rate_hz must be positive!\n"); + + ReportConditionalErrorAndExit( + settings.output_num_channels && *settings.output_num_channels <= 0, + "Error: --output_num_channels must be positive!\n"); + + ReportConditionalErrorAndExit( + settings.reverse_output_num_channels && + *settings.reverse_output_num_channels <= 0, + "Error: --reverse_output_num_channels must be positive!\n"); + + ReportConditionalErrorAndExit( + settings.use_bf && *settings.use_bf && !settings.microphone_positions, + "Error: --mic_positions must be specified when the beamformer is " + "activated.\n"); + + ReportConditionalErrorAndExit( + settings.target_angle_degrees < 0 || settings.target_angle_degrees > 359, + "Error: -target_angle_degrees must be specified between 0 and 359.\n"); + + ReportConditionalErrorAndExit( + settings.aec_suppression_level && + ((*settings.aec_suppression_level) < 0 || + (*settings.aec_suppression_level) > 2), + "Error: --aec_suppression_level must be specified between 0 and 2.\n"); + + ReportConditionalErrorAndExit( + settings.aecm_routing_mode && ((*settings.aecm_routing_mode) < 0 || + (*settings.aecm_routing_mode) > 4), + "Error: --aecm_routing_mode must be specified between 0 and 4.\n"); + + ReportConditionalErrorAndExit( + settings.agc_target_level && ((*settings.agc_target_level) < 0 || + (*settings.agc_target_level) > 31), + "Error: --agc_target_level must be specified between 0 and 31.\n"); + + ReportConditionalErrorAndExit( + settings.agc_compression_gain && ((*settings.agc_compression_gain) < 0 || + (*settings.agc_compression_gain) > 90), + "Error: --agc_compression_gain must be specified between 0 and 90.\n"); + + ReportConditionalErrorAndExit( + settings.use_agc && *settings.use_agc && settings.use_agc2 && + *settings.use_agc2, + "Error: --agc and --agc2 cannot be both active.\n"); + + ReportConditionalErrorAndExit( + settings.use_agc2 && *settings.use_agc2 && + ((settings.agc2_fixed_gain_db) < 0 || + (settings.agc2_fixed_gain_db) > 90), + "Error: --agc2_fixed_gain_db must be specified between 0 and 90.\n"); + + ReportConditionalErrorAndExit( + settings.vad_likelihood && + ((*settings.vad_likelihood) < 0 || (*settings.vad_likelihood) > 3), + "Error: --vad_likelihood must be specified between 0 and 3.\n"); + + ReportConditionalErrorAndExit( + settings.ns_level && + ((*settings.ns_level) < 0 || (*settings.ns_level) > 3), + "Error: --ns_level must be specified between 0 and 3.\n"); + + ReportConditionalErrorAndExit( + settings.report_bitexactness && !settings.aec_dump_input_filename, + "Error: --bitexactness_report can only be used when operating on an " + "aecdump\n"); + + ReportConditionalErrorAndExit( + settings.custom_call_order_filename && settings.aec_dump_input_filename, + "Error: --custom_call_order_file cannot be used when operating on an " + "aecdump\n"); + + ReportConditionalErrorAndExit( + (settings.initial_mic_level < 0 || settings.initial_mic_level > 255), + "Error: --initial_mic_level must be specified between 0 and 255.\n"); + + ReportConditionalErrorAndExit( + settings.simulated_mic_kind && !settings.simulate_mic_gain, + "Error: --simulated_mic_kind cannot be specified when mic simulation is " + "disabled\n"); + + ReportConditionalErrorAndExit( + !settings.simulated_mic_kind && settings.simulate_mic_gain, + "Error: --simulated_mic_kind must be specified when mic simulation is " + "enabled\n"); + + auto valid_wav_name = [](const std::string& wav_file_name) { + if (wav_file_name.size() < 5) { + return false; + } + if ((wav_file_name.compare(wav_file_name.size() - 4, 4, ".wav") == 0) || + (wav_file_name.compare(wav_file_name.size() - 4, 4, ".WAV") == 0)) { + return true; + } + return false; + }; + + ReportConditionalErrorAndExit( + settings.input_filename && (!valid_wav_name(*settings.input_filename)), + "Error: --i must be a valid .wav file name.\n"); + + ReportConditionalErrorAndExit( + settings.output_filename && (!valid_wav_name(*settings.output_filename)), + "Error: --o must be a valid .wav file name.\n"); + + ReportConditionalErrorAndExit( + settings.reverse_input_filename && + (!valid_wav_name(*settings.reverse_input_filename)), + "Error: --ri must be a valid .wav file name.\n"); + + ReportConditionalErrorAndExit( + settings.reverse_output_filename && + (!valid_wav_name(*settings.reverse_output_filename)), + "Error: --ro must be a valid .wav file name.\n"); + + ReportConditionalErrorAndExit( + settings.artificial_nearend_filename && + !valid_wav_name(*settings.artificial_nearend_filename), + "Error: --artifical_nearend must be a valid .wav file name.\n"); +} + +} // namespace + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || + FLAG_help || argc != 1) { + printf("%s", kUsageDescription); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + + SimulationSettings settings = CreateSettings(); + PerformBasicParameterSanityChecks(settings); + std::unique_ptr processor; + + if (settings.aec_dump_input_filename) { + processor.reset(new AecDumpBasedSimulator(settings)); + } else { + processor.reset(new WavBasedSimulator(settings)); + } + + processor->Process(); + + if (settings.report_performance) { + const auto& proc_time = processor->proc_time(); + int64_t exec_time_us = proc_time.sum / rtc::kNumNanosecsPerMicrosec; + std::cout << std::endl + << "Execution time: " << exec_time_us * 1e-6 << " s, File time: " + << processor->get_num_process_stream_calls() * 1.f / + AudioProcessingSimulator::kChunksPerSecond + << std::endl + << "Time per fwd stream chunk (mean, max, min): " << std::endl + << exec_time_us * 1.f / processor->get_num_process_stream_calls() + << " us, " << 1.f * proc_time.max / rtc::kNumNanosecsPerMicrosec + << " us, " << 1.f * proc_time.min / rtc::kNumNanosecsPerMicrosec + << " us" << std::endl; + } + + if (settings.report_bitexactness && settings.aec_dump_input_filename) { + if (processor->OutputWasBitexact()) { + std::cout << "The processing was bitexact."; + } else { + std::cout << "The processing was not bitexact."; + } + } + + return 0; +} + +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.cc new file mode 100644 index 0000000000..c49bb5099f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.cc @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/bitexactness_tools.h" + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +std::string GetApmRenderTestVectorFileName(int sample_rate_hz) { + switch (sample_rate_hz) { + case 8000: + return ResourcePath("far8_stereo", "pcm"); + case 16000: + return ResourcePath("far16_stereo", "pcm"); + case 32000: + return ResourcePath("far32_stereo", "pcm"); + case 48000: + return ResourcePath("far48_stereo", "pcm"); + default: + RTC_NOTREACHED(); + } + return ""; +} + +std::string GetApmCaptureTestVectorFileName(int sample_rate_hz) { + switch (sample_rate_hz) { + case 8000: + return ResourcePath("near8_stereo", "pcm"); + case 16000: + return ResourcePath("near16_stereo", "pcm"); + case 32000: + return ResourcePath("near32_stereo", "pcm"); + case 48000: + return ResourcePath("near48_stereo", "pcm"); + default: + RTC_NOTREACHED(); + } + return ""; +} + +void ReadFloatSamplesFromStereoFile(size_t samples_per_channel, + size_t num_channels, + InputAudioFile* stereo_pcm_file, + rtc::ArrayView data) { + RTC_DCHECK_EQ(data.size(), samples_per_channel * num_channels); + std::vector read_samples(samples_per_channel * 2); + stereo_pcm_file->Read(samples_per_channel * 2, read_samples.data()); + + // Convert samples to float and discard any channels not needed. + for (size_t sample = 0; sample < samples_per_channel; ++sample) { + for (size_t channel = 0; channel < num_channels; ++channel) { + data[sample * num_channels + channel] = + read_samples[sample * 2 + channel] / 32768.0f; + } + } +} + +::testing::AssertionResult VerifyDeinterleavedArray( + size_t samples_per_channel, + size_t num_channels, + rtc::ArrayView reference, + rtc::ArrayView output, + float element_error_bound) { + // Form vectors to compare the reference to. Only the first values of the + // outputs are compared in order not having to specify all preceeding frames + // as testvectors. + const size_t reference_frame_length = + rtc::CheckedDivExact(reference.size(), num_channels); + + std::vector output_to_verify; + for (size_t channel_no = 0; channel_no < num_channels; ++channel_no) { + output_to_verify.insert(output_to_verify.end(), + output.begin() + channel_no * samples_per_channel, + output.begin() + channel_no * samples_per_channel + + reference_frame_length); + } + + return VerifyArray(reference, output_to_verify, element_error_bound); +} + +::testing::AssertionResult VerifyArray(rtc::ArrayView reference, + rtc::ArrayView output, + float element_error_bound) { + // The vectors are deemed to be bitexact only if + // a) output have a size at least as long as the reference. + // b) the samples in the reference are bitexact with the corresponding samples + // in the output. + + bool equal = true; + if (output.size() < reference.size()) { + equal = false; + } else { + // Compare the first samples in the vectors. + for (size_t k = 0; k < reference.size(); ++k) { + if (fabs(output[k] - reference[k]) > element_error_bound) { + equal = false; + break; + } + } + } + + if (equal) { + return ::testing::AssertionSuccess(); + } + + // Lambda function that produces a formatted string with the data in the + // vector. + auto print_vector_in_c_format = [](rtc::ArrayView v, + size_t num_values_to_print) { + std::string s = "{ "; + for (size_t k = 0; k < std::min(num_values_to_print, v.size()); ++k) { + s += std::to_string(v[k]) + "f"; + s += (k < (num_values_to_print - 1)) ? ", " : ""; + } + return s + " }"; + }; + + // If the vectors are deemed not to be similar, return a report of the + // difference. + return ::testing::AssertionFailure() + << std::endl + << " Actual values : " + << print_vector_in_c_format(output, + std::min(output.size(), reference.size())) + << std::endl + << " Expected values: " + << print_vector_in_c_format(reference, reference.size()) << std::endl; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.h new file mode 100644 index 0000000000..eb5ee081c3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/bitexactness_tools.h @@ -0,0 +1,56 @@ + +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_BITEXACTNESS_TOOLS_H_ +#define MODULES_AUDIO_PROCESSING_TEST_BITEXACTNESS_TOOLS_H_ + +#include + +#include "api/array_view.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +// Returns test vector to use for the render signal in an +// APM bitexactness test. +std::string GetApmRenderTestVectorFileName(int sample_rate_hz); + +// Returns test vector to use for the capture signal in an +// APM bitexactness test. +std::string GetApmCaptureTestVectorFileName(int sample_rate_hz); + +// Extract float samples from a pcm file. +void ReadFloatSamplesFromStereoFile(size_t samples_per_channel, + size_t num_channels, + InputAudioFile* stereo_pcm_file, + rtc::ArrayView data); + +// Verifies a frame against a reference and returns the results as an +// AssertionResult. +::testing::AssertionResult VerifyDeinterleavedArray( + size_t samples_per_channel, + size_t num_channels, + rtc::ArrayView reference, + rtc::ArrayView output, + float element_error_bound); + +// Verifies a vector against a reference and returns the results as an +// AssertionResult. +::testing::AssertionResult VerifyArray(rtc::ArrayView reference, + rtc::ArrayView output, + float element_error_bound); + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_BITEXACTNESS_TOOLS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn new file mode 100644 index 0000000000..c7263301b6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn @@ -0,0 +1,75 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../../webrtc.gni") + +group("conversational_speech") { + testonly = true + deps = [ + ":conversational_speech_generator", + ] +} + +rtc_executable("conversational_speech_generator") { + testonly = true + sources = [ + "generator.cc", + ] + deps = [ + ":lib", + "../../../../rtc_base:rtc_base_approved", + "../../../../test:test_support", + ] +} + +rtc_static_library("lib") { + testonly = true + sources = [ + "config.cc", + "config.h", + "multiend_call.cc", + "multiend_call.h", + "simulator.cc", + "simulator.h", + "timing.cc", + "timing.h", + "wavreader_abstract_factory.h", + "wavreader_factory.cc", + "wavreader_factory.h", + "wavreader_interface.h", + ] + deps = [ + "../../../..:webrtc_common", + "../../../../api:array_view", + "../../../../common_audio", + "../../../../rtc_base:rtc_base_approved", + ] + visibility = [ ":*" ] # Only targets in this file can depend on this. +} + +rtc_source_set("unittest") { + testonly = true + sources = [ + "generator_unittest.cc", + "mock_wavreader.cc", + "mock_wavreader.h", + "mock_wavreader_factory.cc", + "mock_wavreader_factory.h", + ] + deps = [ + ":lib", + "../../../..:webrtc_common", + "../../../../api:array_view", + "../../../../api:optional", + "../../../../common_audio", + "../../../../rtc_base:rtc_base_approved", + "../../../../test:test_support", + "//testing/gmock", + "//testing/gtest", + ] +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/OWNERS new file mode 100644 index 0000000000..0981733ba9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/OWNERS @@ -0,0 +1,6 @@ +alessiob@webrtc.org +henrik.lundin@webrtc.org +peah@webrtc.org + +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/README.md b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/README.md new file mode 100644 index 0000000000..bbb4112fc0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/README.md @@ -0,0 +1,72 @@ +# Conversational Speech generator tool + +Tool to generate multiple-end audio tracks to simulate conversational speech +with two or more participants. + +The input to the tool is a directory containing a number of audio tracks and +a text file indicating how to time the sequence of speech turns (see the Example +section). + +Since the timing of the speaking turns is specified by the user, the generated +tracks may not be suitable for testing scenarios in which there is unpredictable +network delay (e.g., end-to-end RTC assessment). + +Instead, the generated pairs can be used when the delay is constant (obviously +including the case in which there is no delay). +For instance, echo cancellation in the APM module can be evaluated using two-end +audio tracks as input and reverse input. + +By indicating negative and positive time offsets, one can reproduce cross-talk +(aka double-talk) and silence in the conversation. + +### Example + +For each end, there is a set of audio tracks, e.g., a1, a2 and a3 (speaker A) +and b1, b2 (speaker B). +The text file with the timing information may look like this: + +``` +A a1 0 +B b1 0 +A a2 100 +B b2 -200 +A a3 0 +A a4 0 +``` + +The first column indicates the speaker name, the second contains the audio track +file names, and the third the offsets (in milliseconds) used to concatenate the +chunks. + +Assume that all the audio tracks in the example above are 1000 ms long. +The tool will then generate two tracks (A and B) that look like this: + +**Track A** +``` + a1 (1000 ms) + silence (1100 ms) + a2 (1000 ms) + silence (800 ms) + a3 (1000 ms) + a4 (1000 ms) +``` + +**Track B** +``` + silence (1000 ms) + b1 (1000 ms) + silence (900 ms) + b2 (1000 ms) + silence (2000 ms) +``` + +The two tracks can be also visualized as follows (one characheter represents +100 ms, "." is silence and "*" is speech). + +``` +t: 0 1 2 3 4 5 6 (s) +A: **********...........**********........******************** +B: ..........**********.........**********.................... + ^ 200 ms cross-talk + 100 ms silence ^ +``` diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.cc new file mode 100644 index 0000000000..76d3de8108 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/conversational_speech/config.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +const std::string& Config::audiotracks_path() const { + return audiotracks_path_; +} + +const std::string& Config::timing_filepath() const { + return timing_filepath_; +} + +const std::string& Config::output_path() const { + return output_path_; +} + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.h new file mode 100644 index 0000000000..f219a65f60 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/config.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_CONFIG_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_CONFIG_H_ + +#include + +namespace webrtc { +namespace test { +namespace conversational_speech { + +struct Config { + Config(const std::string& audiotracks_path, + const std::string& timing_filepath, + const std::string& output_path) + : audiotracks_path_(audiotracks_path), + timing_filepath_(timing_filepath), + output_path_(output_path) {} + + const std::string& audiotracks_path() const; + const std::string& timing_filepath() const; + const std::string& output_path() const; + + const std::string audiotracks_path_; + const std::string timing_filepath_; + const std::string output_path_; +}; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_CONFIG_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator.cc new file mode 100644 index 0000000000..ee3f81a9a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/test/conversational_speech/config.h" +#include "modules/audio_processing/test/conversational_speech/multiend_call.h" +#include "modules/audio_processing/test/conversational_speech/simulator.h" +#include "modules/audio_processing/test/conversational_speech/timing.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_factory.h" +#include "rtc_base/flags.h" +#include "rtc_base/ptr_util.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { +namespace { + +const char kUsageDescription[] = + "Usage: conversational_speech_generator\n" + " -i \n" + " -t \n" + " -o \n" + "\n\n" + "Command-line tool to generate multiple-end audio tracks to simulate " + "conversational speech with two or more participants.\n"; + +DEFINE_string(i, "", "Directory containing the speech turn wav files"); +DEFINE_string(t, "", "Path to the timing text file"); +DEFINE_string(o, "", "Output wav files destination path"); +DEFINE_bool(help, false, "Prints this message"); + +} // namespace + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || + FLAG_help || argc != 1) { + printf("%s", kUsageDescription); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + RTC_CHECK(DirExists(FLAG_i)); + RTC_CHECK(FileExists(FLAG_t)); + RTC_CHECK(DirExists(FLAG_o)); + + conversational_speech::Config config(FLAG_i, FLAG_t, FLAG_o); + + // Load timing. + std::vector timing = + conversational_speech::LoadTiming(config.timing_filepath()); + + // Parse timing and audio tracks. + auto wavreader_factory = rtc::MakeUnique< + conversational_speech::WavReaderFactory>(); + conversational_speech::MultiEndCall multiend_call( + timing, config.audiotracks_path(), std::move(wavreader_factory)); + + // Generate output audio tracks. + auto generated_audiotrack_pairs = conversational_speech::Simulate( + multiend_call, config.output_path()); + + // Show paths to created audio tracks. + std::cout << "Output files:" << std::endl; + for (const auto& output_paths_entry : *generated_audiotrack_pairs) { + std::cout << " speaker: " << output_paths_entry.first << std::endl; + std::cout << " near end: " << output_paths_entry.second.near_end + << std::endl; + std::cout << " far end: " << output_paths_entry.second.far_end + << std::endl; + } + + return 0; +} + +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator_unittest.cc new file mode 100644 index 0000000000..064e8c82d7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/generator_unittest.cc @@ -0,0 +1,688 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file consists of unit tests for webrtc::test::conversational_speech +// members. Part of them focus on accepting or rejecting different +// conversational speech setups. A setup is defined by a set of audio tracks and +// timing information). +// The docstring at the beginning of each TEST(ConversationalSpeechTest, +// MultiEndCallSetup*) function looks like the drawing below and indicates which +// setup is tested. +// +// Accept: +// A 0****..... +// B .....1**** +// +// The drawing indicates the following: +// - the illustrated setup should be accepted, +// - there are two speakers (namely, A and B), +// - A is the first speaking, B is the second one, +// - each character after the speaker's letter indicates a time unit (e.g., 100 +// ms), +// - "*" indicates speaking, "." listening, +// - numbers indicate the turn index in std::vector. +// +// Note that the same speaker can appear in multiple lines in order to depict +// cases in which there are wrong offsets leading to self cross-talk (which is +// rejected). + +// MSVC++ requires this to be set before any other includes to get M_PI. +#define _USE_MATH_DEFINES + +#include +#include +#include +#include +#include + +#include "api/optional.h" +#include "common_audio/wav_file.h" +#include "modules/audio_processing/test/conversational_speech/config.h" +#include "modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h" +#include "modules/audio_processing/test/conversational_speech/multiend_call.h" +#include "modules/audio_processing/test/conversational_speech/simulator.h" +#include "modules/audio_processing/test/conversational_speech/timing.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_factory.h" +#include "rtc_base/logging.h" +#include "rtc_base/pathutils.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { +namespace { + +using conversational_speech::LoadTiming; +using conversational_speech::SaveTiming; +using conversational_speech::MockWavReaderFactory; +using conversational_speech::MultiEndCall; +using conversational_speech::Turn; +using conversational_speech::WavReaderFactory; + +const char* const audiotracks_path = "/path/to/audiotracks"; +const char* const timing_filepath = "/path/to/timing_file.txt"; +const char* const output_path = "/path/to/output_dir"; + +const std::vector expected_timing = { + {"A", "a1", 0}, + {"B", "b1", 0}, + {"A", "a2", 100}, + {"B", "b2", -200}, + {"A", "a3", 0}, + {"A", "a3", 0}, +}; +const std::size_t kNumberOfTurns = expected_timing.size(); + +// Default arguments for MockWavReaderFactory ctor. +// Fake audio track parameters. +constexpr int kDefaultSampleRate = 48000; +const std::map + kDefaultMockWavReaderFactoryParamsMap = { + {"t300", {kDefaultSampleRate, 1u, 14400u}}, // Mono, 0.3 seconds. + {"t500", {kDefaultSampleRate, 1u, 24000u}}, // Mono, 0.5 seconds. + {"t1000", {kDefaultSampleRate, 1u, 48000u}}, // Mono, 1.0 seconds. + {"sr8000", {8000, 1u, 8000u}}, // 8kHz sample rate, mono, 1 second. + {"sr16000", {16000, 1u, 16000u}}, // 16kHz sample rate, mono, 1 second. + {"sr16000_stereo", {16000, 2u, 16000u}}, // Like sr16000, but stereo. +}; +const MockWavReaderFactory::Params& kDefaultMockWavReaderFactoryParams = + kDefaultMockWavReaderFactoryParamsMap.at("t500"); + +std::unique_ptr CreateMockWavReaderFactory() { + return std::unique_ptr( + new MockWavReaderFactory(kDefaultMockWavReaderFactoryParams, + kDefaultMockWavReaderFactoryParamsMap)); +} + +void CreateSineWavFile(const std::string& filepath, + const MockWavReaderFactory::Params& params, + float frequency = 440.0f) { + // Create samples. + constexpr double two_pi = 2.0 * M_PI; + std::vector samples(params.num_samples); + for (std::size_t i = 0; i < params.num_samples; ++i) { + // TODO(alessiob): the produced tone is not pure, improve. + samples[i] = std::lround(32767.0f * std::sin( + two_pi * i * frequency / params.sample_rate)); + } + + // Write samples. + WavWriter wav_writer(filepath, params.sample_rate, params.num_channels); + wav_writer.WriteSamples(samples.data(), params.num_samples); +} + +// Parameters to generate audio tracks with CreateSineWavFile. +struct SineAudioTrackParams { + MockWavReaderFactory::Params params; + float frequency; +}; + +// Creates a temporary directory in which sine audio tracks are written. +std::string CreateTemporarySineAudioTracks( + const std::map& sine_tracks_params) { + // Create temporary directory. + rtc::Pathname temp_directory(OutputPath()); + temp_directory.AppendFolder("TempConversationalSpeechAudioTracks"); + CreateDir(temp_directory.pathname()); + + // Create sine tracks. + for (const auto& it : sine_tracks_params) { + const rtc::Pathname temp_filepath(temp_directory.pathname(), it.first); + CreateSineWavFile( + temp_filepath.pathname(), it.second.params, it.second.frequency); + } + + return temp_directory.pathname(); +} + +void CheckAudioTrackParams(const WavReaderFactory& wav_reader_factory, + const std::string& filepath, + const MockWavReaderFactory::Params& expeted_params) { + auto wav_reader = wav_reader_factory.Create(filepath); + EXPECT_EQ(expeted_params.sample_rate, wav_reader->SampleRate()); + EXPECT_EQ(expeted_params.num_channels, wav_reader->NumChannels()); + EXPECT_EQ(expeted_params.num_samples, wav_reader->NumSamples()); +} + +void DeleteFolderAndContents(const std::string& dir) { + if (!DirExists(dir)) { return; } + rtc::Optional> dir_content = ReadDirectory(dir); + EXPECT_TRUE(dir_content); + for (const auto& path : *dir_content) { + if (DirExists(path)) { + DeleteFolderAndContents(path); + } else if (FileExists(path)) { + // TODO(alessiob): Wrap with EXPECT_TRUE() once webrtc:7769 bug fixed. + RemoveFile(path); + } else { + FAIL(); + } + } + // TODO(alessiob): Wrap with EXPECT_TRUE() once webrtc:7769 bug fixed. + RemoveDir(dir); +} + +} // namespace + +using testing::_; + +TEST(ConversationalSpeechTest, Settings) { + const conversational_speech::Config config( + audiotracks_path, timing_filepath, output_path); + + // Test getters. + EXPECT_EQ(audiotracks_path, config.audiotracks_path()); + EXPECT_EQ(timing_filepath, config.timing_filepath()); + EXPECT_EQ(output_path, config.output_path()); +} + +TEST(ConversationalSpeechTest, TimingSaveLoad) { + // Save test timing. + const std::string temporary_filepath = TempFilename( + OutputPath(), "TempTimingTestFile"); + SaveTiming(temporary_filepath, expected_timing); + + // Create a std::vector instance by loading from file. + std::vector actual_timing = LoadTiming(temporary_filepath); + std::remove(temporary_filepath.c_str()); + + // Check size. + EXPECT_EQ(expected_timing.size(), actual_timing.size()); + + // Check Turn instances. + for (size_t index = 0; index < expected_timing.size(); ++index) { + EXPECT_EQ(expected_timing[index], actual_timing[index]) + << "turn #" << index << " not matching"; + } +} + +TEST(ConversationalSpeechTest, MultiEndCallCreate) { + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are 5 unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(5); + + // Inject the mock wav reader factory. + conversational_speech::MultiEndCall multiend_call( + expected_timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(2u, multiend_call.speaker_names().size()); + EXPECT_EQ(5u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(6u, multiend_call.speaking_turns().size()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupDifferentSampleRates) { + const std::vector timing = { + {"A", "sr8000", 0}, + {"B", "sr16000", 0}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(testing::_)).Times(2); + + MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupMultipleChannels) { + const std::vector timing = { + {"A", "sr16000_stereo", 0}, + {"B", "sr16000_stereo", 0}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(testing::_)).Times(1); + + MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, + MultiEndCallSetupDifferentSampleRatesAndMultipleNumChannels) { + const std::vector timing = { + {"A", "sr8000", 0}, + {"B", "sr16000_stereo", 0}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(testing::_)).Times(2); + + MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupFirstOffsetNegative) { + const std::vector timing = { + {"A", "t500", -100}, + {"B", "t500", 0}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(1); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupSimple) { + // Accept: + // A 0****..... + // B .....1**** + constexpr std::size_t expected_duration = kDefaultSampleRate; + const std::vector timing = { + {"A", "t500", 0}, + {"B", "t500", 0}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(1); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(2u, multiend_call.speaker_names().size()); + EXPECT_EQ(1u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(2u, multiend_call.speaking_turns().size()); + EXPECT_EQ(expected_duration, multiend_call.total_duration_samples()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupPause) { + // Accept: + // A 0****....... + // B .......1**** + constexpr std::size_t expected_duration = kDefaultSampleRate * 1.2; + const std::vector timing = { + {"A", "t500", 0}, + {"B", "t500", 200}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(1); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(2u, multiend_call.speaker_names().size()); + EXPECT_EQ(1u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(2u, multiend_call.speaking_turns().size()); + EXPECT_EQ(expected_duration, multiend_call.total_duration_samples()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupCrossTalk) { + // Accept: + // A 0****.... + // B ....1**** + constexpr std::size_t expected_duration = kDefaultSampleRate * 0.9; + const std::vector timing = { + {"A", "t500", 0}, + {"B", "t500", -100}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(1); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(2u, multiend_call.speaker_names().size()); + EXPECT_EQ(1u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(2u, multiend_call.speaking_turns().size()); + EXPECT_EQ(expected_duration, multiend_call.total_duration_samples()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupInvalidOrder) { + // Reject: + // A ..0**** + // B .1****. The n-th turn cannot start before the (n-1)-th one. + const std::vector timing = { + {"A", "t500", 200}, + {"B", "t500", -600}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(1); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupCrossTalkThree) { + // Accept: + // A 0****2****... + // B ...1********* + constexpr std::size_t expected_duration = kDefaultSampleRate * 1.3; + const std::vector timing = { + {"A", "t500", 0}, + {"B", "t1000", -200}, + {"A", "t500", -800}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(2); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(2u, multiend_call.speaker_names().size()); + EXPECT_EQ(2u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(3u, multiend_call.speaking_turns().size()); + EXPECT_EQ(expected_duration, multiend_call.total_duration_samples()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupSelfCrossTalkNearInvalid) { + // Reject: + // A 0****...... + // A ...1****... + // B ......2**** + // ^ Turn #1 overlaps with #0 which is from the same speaker. + const std::vector timing = { + {"A", "t500", 0}, + {"A", "t500", -200}, + {"B", "t500", -200}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(1); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupSelfCrossTalkFarInvalid) { + // Reject: + // A 0********* + // B 1**....... + // C ...2**.... + // A ......3**. + // ^ Turn #3 overlaps with #0 which is from the same speaker. + const std::vector timing = { + {"A", "t1000", 0}, + {"B", "t300", -1000}, + {"C", "t300", 0}, + {"A", "t300", 0}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(2); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupCrossTalkMiddleValid) { + // Accept: + // A 0*********.. + // B ..1****..... + // C .......2**** + constexpr std::size_t expected_duration = kDefaultSampleRate * 1.2; + const std::vector timing = { + {"A", "t1000", 0}, + {"B", "t500", -800}, + {"C", "t500", 0}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(2); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(3u, multiend_call.speaker_names().size()); + EXPECT_EQ(2u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(3u, multiend_call.speaking_turns().size()); + EXPECT_EQ(expected_duration, multiend_call.total_duration_samples()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupCrossTalkMiddleInvalid) { + // Reject: + // A 0********* + // B ..1****... + // C ....2****. + // ^ Turn #2 overlaps both with #0 and #1 (cross-talk with 3+ speakers + // not permitted). + const std::vector timing = { + {"A", "t1000", 0}, + {"B", "t500", -800}, + {"C", "t500", -300}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(2); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupCrossTalkMiddleAndPause) { + // Accept: + // A 0*********.. + // B .2****...... + // C .......3**** + constexpr std::size_t expected_duration = kDefaultSampleRate * 1.2; + const std::vector timing = { + {"A", "t1000", 0}, + {"B", "t500", -900}, + {"C", "t500", 100}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(2); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(3u, multiend_call.speaker_names().size()); + EXPECT_EQ(2u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(3u, multiend_call.speaking_turns().size()); + EXPECT_EQ(expected_duration, multiend_call.total_duration_samples()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupCrossTalkFullOverlapValid) { + // Accept: + // A 0**** + // B 1**** + const std::vector timing = { + {"A", "t500", 0}, + {"B", "t500", -500}, + }; + auto mock_wavreader_factory = CreateMockWavReaderFactory(); + + // There is one unique audio track to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(1); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(2u, multiend_call.speaker_names().size()); + EXPECT_EQ(1u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(2u, multiend_call.speaking_turns().size()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupLongSequence) { + // Accept: + // A 0****....3****.5**. + // B .....1****...4**... + // C ......2**.......6**.. + constexpr std::size_t expected_duration = kDefaultSampleRate * 1.9; + const std::vector timing = { + {"A", "t500", 0}, + {"B", "t500", 0}, + {"C", "t300", -400}, + {"A", "t500", 0}, + {"B", "t300", -100}, + {"A", "t300", -100}, + {"C", "t300", -200}, + }; + auto mock_wavreader_factory = std::unique_ptr( + new MockWavReaderFactory(kDefaultMockWavReaderFactoryParams, + kDefaultMockWavReaderFactoryParamsMap)); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(2); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_TRUE(multiend_call.valid()); + + // Test. + EXPECT_EQ(3u, multiend_call.speaker_names().size()); + EXPECT_EQ(2u, multiend_call.audiotrack_readers().size()); + EXPECT_EQ(7u, multiend_call.speaking_turns().size()); + EXPECT_EQ(expected_duration, multiend_call.total_duration_samples()); +} + +TEST(ConversationalSpeechTest, MultiEndCallSetupLongSequenceInvalid) { + // Reject: + // A 0****....3****.6** + // B .....1****...4**.. + // C ......2**.....5**.. + // ^ Turns #4, #5 and #6 overlapping (cross-talk with 3+ + // speakers not permitted). + const std::vector timing = { + {"A", "t500", 0}, + {"B", "t500", 0}, + {"C", "t300", -400}, + {"A", "t500", 0}, + {"B", "t300", -100}, + {"A", "t300", -200}, + {"C", "t300", -200}, + }; + auto mock_wavreader_factory = std::unique_ptr( + new MockWavReaderFactory(kDefaultMockWavReaderFactoryParams, + kDefaultMockWavReaderFactoryParamsMap)); + + // There are two unique audio tracks to read. + EXPECT_CALL(*mock_wavreader_factory, Create(_)).Times(2); + + conversational_speech::MultiEndCall multiend_call( + timing, audiotracks_path, std::move(mock_wavreader_factory)); + EXPECT_FALSE(multiend_call.valid()); +} + +TEST(ConversationalSpeechTest, MultiEndCallWavReaderAdaptorSine) { + // Parameters with which wav files are created. + constexpr int duration_seconds = 5; + const int sample_rates[] = {8000, 11025, 16000, 22050, 32000, 44100, 48000}; + + for (int sample_rate : sample_rates) { + const rtc::Pathname temp_filename( + OutputPath(), "TempSineWavFile_" + std::to_string(sample_rate) + + ".wav"); + + // Write wav file. + const std::size_t num_samples = duration_seconds * sample_rate; + MockWavReaderFactory::Params params = {sample_rate, 1u, num_samples}; + CreateSineWavFile(temp_filename.pathname(), params); + + // Load wav file and check if params match. + WavReaderFactory wav_reader_factory; + MockWavReaderFactory::Params expeted_params = { + sample_rate, 1u, num_samples}; + CheckAudioTrackParams( + wav_reader_factory, temp_filename.pathname(), expeted_params); + + // Clean up. + remove(temp_filename.pathname().c_str()); + } +} + +TEST(ConversationalSpeechTest, DISABLED_MultiEndCallSimulator) { + // Simulated call (one character corresponding to 500 ms): + // A 0*********...........2*********..... + // B ...........1*********.....3********* + const std::vector expected_timing = { + {"A", "t5000_440.wav", 0}, + {"B", "t5000_880.wav", 500}, + {"A", "t5000_440.wav", 0}, + {"B", "t5000_880.wav", -2500}, + }; + const std::size_t expected_duration_seconds = 18; + + // Create temporary audio track files. + const int sample_rate = 16000; + const std::map sine_tracks_params = { + {"t5000_440.wav", {{sample_rate, 1u, sample_rate * 5}, 440.0}}, + {"t5000_880.wav", {{sample_rate, 1u, sample_rate * 5}, 880.0}}, + }; + const std::string audiotracks_path = CreateTemporarySineAudioTracks( + sine_tracks_params); + + // Set up the multi-end call. + auto wavreader_factory = std::unique_ptr( + new WavReaderFactory()); + MultiEndCall multiend_call( + expected_timing, audiotracks_path, std::move(wavreader_factory)); + + // Simulate the call. + rtc::Pathname output_path(audiotracks_path); + output_path.AppendFolder("output"); + CreateDir(output_path.pathname()); + RTC_LOG(LS_VERBOSE) << "simulator output path: " << output_path.pathname(); + auto generated_audiotrak_pairs = conversational_speech::Simulate( + multiend_call, output_path.pathname()); + EXPECT_EQ(2u, generated_audiotrak_pairs->size()); + + // Check the output. + WavReaderFactory wav_reader_factory; + const MockWavReaderFactory::Params expeted_params = { + sample_rate, 1u, sample_rate * expected_duration_seconds}; + for (const auto& it : *generated_audiotrak_pairs) { + RTC_LOG(LS_VERBOSE) << "checking far/near-end for <" << it.first << ">"; + CheckAudioTrackParams( + wav_reader_factory, it.second.near_end, expeted_params); + CheckAudioTrackParams( + wav_reader_factory, it.second.far_end, expeted_params); + } + + // Clean. + EXPECT_NO_FATAL_FAILURE(DeleteFolderAndContents(audiotracks_path)); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.cc new file mode 100644 index 0000000000..103483d83e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.cc @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/conversational_speech/mock_wavreader.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +using testing::Return; + +MockWavReader::MockWavReader( + int sample_rate, size_t num_channels, size_t num_samples) + : sample_rate_(sample_rate), num_channels_(num_channels), + num_samples_(num_samples) { + ON_CALL(*this, SampleRate()).WillByDefault(Return(sample_rate_)); + ON_CALL(*this, NumChannels()).WillByDefault(Return(num_channels_)); + ON_CALL(*this, NumSamples()).WillByDefault(Return(num_samples_)); +} + +MockWavReader::~MockWavReader() = default; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h new file mode 100644 index 0000000000..55d00b31ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MOCK_WAVREADER_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MOCK_WAVREADER_H_ + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_interface.h" +#include "test/gmock.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { +namespace conversational_speech { + +class MockWavReader : public WavReaderInterface { + public: + MockWavReader(int sample_rate, size_t num_channels, size_t num_samples); + ~MockWavReader(); + + // TODO(alessiob): use ON_CALL to return random samples if needed. + MOCK_METHOD1(ReadFloatSamples, size_t(rtc::ArrayView)); + MOCK_METHOD1(ReadInt16Samples, size_t(rtc::ArrayView)); + + MOCK_CONST_METHOD0(SampleRate, int()); + MOCK_CONST_METHOD0(NumChannels, size_t()); + MOCK_CONST_METHOD0(NumSamples, size_t()); + + private: + const int sample_rate_; + const size_t num_channels_; + const size_t num_samples_; +}; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MOCK_WAVREADER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.cc new file mode 100644 index 0000000000..76f5c80718 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.cc @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h" + +#include "modules/audio_processing/test/conversational_speech/mock_wavreader.h" +#include "rtc_base/logging.h" +#include "rtc_base/pathutils.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +using testing::_; +using testing::Invoke; + +MockWavReaderFactory::MockWavReaderFactory( + const Params& default_params, + const std::map& params) + : default_params_(default_params), + audiotrack_names_params_(params) { + ON_CALL(*this, Create(_)).WillByDefault(Invoke( + this, &MockWavReaderFactory::CreateMock)); +} + +MockWavReaderFactory::MockWavReaderFactory(const Params& default_params) + : MockWavReaderFactory(default_params, + std::map{}) {} + +MockWavReaderFactory::~MockWavReaderFactory() = default; + +std::unique_ptr MockWavReaderFactory::CreateMock( + const std::string& filepath) { + // Search the parameters corresponding to filepath. + const rtc::Pathname audiotrack_file_path(filepath); + const auto it = audiotrack_names_params_.find( + audiotrack_file_path.filename()); + + // If not found, use default parameters. + if (it == audiotrack_names_params_.end()) { + RTC_LOG(LS_VERBOSE) << "using default parameters for " << filepath; + return std::unique_ptr( + new MockWavReader(default_params_.sample_rate, + default_params_.num_channels, + default_params_.num_samples)); + } + + // Found, use the audiotrack-specific parameters. + RTC_LOG(LS_VERBOSE) << "using ad-hoc parameters for " << filepath; + RTC_LOG(LS_VERBOSE) << "sample_rate " << it->second.sample_rate; + RTC_LOG(LS_VERBOSE) << "num_channels " << it->second.num_channels; + RTC_LOG(LS_VERBOSE) << "num_samples " << it->second.num_samples; + return std::unique_ptr( + new MockWavReader(it->second.sample_rate, + it->second.num_channels, + it->second.num_samples)); +} + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h new file mode 100644 index 0000000000..4c055cf3ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MOCK_WAVREADER_FACTORY_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MOCK_WAVREADER_FACTORY_H_ + +#include +#include +#include + +#include "modules/audio_processing/test/conversational_speech/wavreader_abstract_factory.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_interface.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +class MockWavReaderFactory : public WavReaderAbstractFactory { + public: + struct Params{ + int sample_rate; + size_t num_channels; + size_t num_samples; + }; + + MockWavReaderFactory(const Params& default_params, + const std::map& params); + explicit MockWavReaderFactory(const Params& default_params); + ~MockWavReaderFactory(); + + MOCK_CONST_METHOD1(Create, std::unique_ptr( + const std::string&)); + + private: + // Creates a MockWavReader instance using the parameters in + // audiotrack_names_params_ if the entry corresponding to filepath exists, + // otherwise creates a MockWavReader instance using the default parameters. + std::unique_ptr CreateMock(const std::string& filepath); + + const Params& default_params_; + std::map audiotrack_names_params_; +}; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MOCK_WAVREADER_FACTORY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.cc new file mode 100644 index 0000000000..76cf774afc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.cc @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/conversational_speech/multiend_call.h" + +#include +#include + +#include "rtc_base/logging.h" +#include "rtc_base/pathutils.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +MultiEndCall::MultiEndCall( + rtc::ArrayView timing, const std::string& audiotracks_path, + std::unique_ptr wavreader_abstract_factory) + : timing_(timing), audiotracks_path_(audiotracks_path), + wavreader_abstract_factory_(std::move(wavreader_abstract_factory)), + valid_(false) { + FindSpeakerNames(); + if (CreateAudioTrackReaders()) + valid_ = CheckTiming(); +} + +MultiEndCall::~MultiEndCall() = default; + +void MultiEndCall::FindSpeakerNames() { + RTC_DCHECK(speaker_names_.empty()); + for (const Turn& turn : timing_) { + speaker_names_.emplace(turn.speaker_name); + } +} + +bool MultiEndCall::CreateAudioTrackReaders() { + RTC_DCHECK(audiotrack_readers_.empty()); + sample_rate_hz_ = 0; // Sample rate will be set when reading the first track. + for (const Turn& turn : timing_) { + auto it = audiotrack_readers_.find(turn.audiotrack_file_name); + if (it != audiotrack_readers_.end()) + continue; + + // Instance Pathname to retrieve the full path to the audiotrack file. + const rtc::Pathname audiotrack_file_path( + audiotracks_path_, turn.audiotrack_file_name); + + // Map the audiotrack file name to a new instance of WavReaderInterface. + std::unique_ptr wavreader = + wavreader_abstract_factory_->Create(audiotrack_file_path.pathname()); + + if (sample_rate_hz_ == 0) { + sample_rate_hz_ = wavreader->SampleRate(); + } else if (sample_rate_hz_ != wavreader->SampleRate()) { + RTC_LOG(LS_ERROR) + << "All the audio tracks should have the same sample rate."; + return false; + } + + if (wavreader->NumChannels() != 1) { + RTC_LOG(LS_ERROR) << "Only mono audio tracks supported."; + return false; + } + + audiotrack_readers_.emplace( + turn.audiotrack_file_name, std::move(wavreader)); + } + + return true; +} + +bool MultiEndCall::CheckTiming() { + struct Interval { + size_t begin; + size_t end; + }; + size_t number_of_turns = timing_.size(); + auto millisecond_to_samples = [](int ms, int sr) -> int { + // Truncation may happen if the sampling rate is not an integer multiple + // of 1000 (e.g., 44100). + return ms * sr / 1000; + }; + auto in_interval = [](size_t value, const Interval& interval) { + return interval.begin <= value && value < interval.end; + }; + total_duration_samples_ = 0; + speaking_turns_.clear(); + + // Begin and end timestamps for the last two turns (unit: number of samples). + Interval second_last_turn = {0, 0}; + Interval last_turn = {0, 0}; + + // Initialize map to store speaking turn indices of each speaker (used to + // detect self cross-talk). + std::map> speaking_turn_indices; + for (const std::string& speaker_name : speaker_names_) { + speaking_turn_indices.emplace( + std::piecewise_construct, + std::forward_as_tuple(speaker_name), + std::forward_as_tuple()); + } + + // Parse turns. + for (size_t turn_index = 0; turn_index < number_of_turns; ++turn_index) { + const Turn& turn = timing_[turn_index]; + auto it = audiotrack_readers_.find(turn.audiotrack_file_name); + RTC_CHECK(it != audiotrack_readers_.end()) + << "Audio track reader not created"; + + // Begin and end timestamps for the current turn. + int offset_samples = millisecond_to_samples( + turn.offset, it->second->SampleRate()); + std::size_t begin_timestamp = last_turn.end + offset_samples; + std::size_t end_timestamp = begin_timestamp + it->second->NumSamples(); + RTC_LOG(LS_INFO) << "turn #" << turn_index << " " << begin_timestamp << "-" + << end_timestamp << " ms"; + + // The order is invalid if the offset is negative and its absolute value is + // larger then the duration of the previous turn. + if (offset_samples < 0 && -offset_samples > static_cast( + last_turn.end - last_turn.begin)) { + RTC_LOG(LS_ERROR) << "invalid order"; + return false; + } + + // Cross-talk with 3 or more speakers occurs when the beginning of the + // current interval falls in the last two turns. + if (turn_index > 1 && in_interval(begin_timestamp, last_turn) + && in_interval(begin_timestamp, second_last_turn)) { + RTC_LOG(LS_ERROR) << "cross-talk with 3+ speakers"; + return false; + } + + // Append turn. + speaking_turns_.emplace_back( + turn.speaker_name, turn.audiotrack_file_name, + begin_timestamp, end_timestamp); + + // Save speaking turn index for self cross-talk detection. + RTC_DCHECK_EQ(speaking_turns_.size(), turn_index + 1); + speaking_turn_indices[turn.speaker_name].push_back(turn_index); + + // Update total duration of the consversational speech. + if (total_duration_samples_ < end_timestamp) + total_duration_samples_ = end_timestamp; + + // Update and continue with next turn. + second_last_turn = last_turn; + last_turn.begin = begin_timestamp; + last_turn.end = end_timestamp; + } + + // Detect self cross-talk. + for (const std::string& speaker_name : speaker_names_) { + RTC_LOG(LS_INFO) << "checking self cross-talk for <" << speaker_name << ">"; + + // Copy all turns for this speaker to new vector. + std::vector speaking_turns_for_name; + std::copy_if(speaking_turns_.begin(), speaking_turns_.end(), + std::back_inserter(speaking_turns_for_name), + [&speaker_name](const SpeakingTurn& st){ + return st.speaker_name == speaker_name; }); + + // Check for overlap between adjacent elements. + // This is a sufficient condition for self cross-talk since the intervals + // are sorted by begin timestamp. + auto overlap = std::adjacent_find( + speaking_turns_for_name.begin(), speaking_turns_for_name.end(), + [](const SpeakingTurn& a, const SpeakingTurn& b) { + return a.end > b.begin; }); + + if (overlap != speaking_turns_for_name.end()) { + RTC_LOG(LS_ERROR) << "Self cross-talk detected"; + return false; + } + } + + return true; +} + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.h new file mode 100644 index 0000000000..e4cee7ac96 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/multiend_call.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MULTIEND_CALL_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MULTIEND_CALL_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/test/conversational_speech/timing.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_abstract_factory.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_interface.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +class MultiEndCall { + public: + struct SpeakingTurn { + // Constructor required in order to use std::vector::emplace_back(). + SpeakingTurn(std::string new_speaker_name, + std::string new_audiotrack_file_name, + size_t new_begin, size_t new_end) + : speaker_name(std::move(new_speaker_name)), + audiotrack_file_name(std::move(new_audiotrack_file_name)), + begin(new_begin), end(new_end) {} + std::string speaker_name; + std::string audiotrack_file_name; + size_t begin; + size_t end; + }; + + MultiEndCall( + rtc::ArrayView timing, const std::string& audiotracks_path, + std::unique_ptr wavreader_abstract_factory); + ~MultiEndCall(); + + const std::set& speaker_names() const { return speaker_names_; } + const std::map>& + audiotrack_readers() const { return audiotrack_readers_; } + bool valid() const { return valid_; } + int sample_rate() const { return sample_rate_hz_; } + size_t total_duration_samples() const { return total_duration_samples_; } + const std::vector& speaking_turns() const { + return speaking_turns_; } + + private: + // Finds unique speaker names. + void FindSpeakerNames(); + + // Creates one WavReader instance for each unique audiotrack. It returns false + // if the audio tracks do not have the same sample rate or if they are not + // mono. + bool CreateAudioTrackReaders(); + + // Validates the speaking turns timing information. Accepts cross-talk, but + // only up to 2 speakers. Rejects unordered turns and self cross-talk. + bool CheckTiming(); + + rtc::ArrayView timing_; + const std::string& audiotracks_path_; + std::unique_ptr wavreader_abstract_factory_; + std::set speaker_names_; + std::map> + audiotrack_readers_; + bool valid_; + int sample_rate_hz_; + size_t total_duration_samples_; + std::vector speaking_turns_; + + RTC_DISALLOW_COPY_AND_ASSIGN(MultiEndCall); +}; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_MULTIEND_CALL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.cc new file mode 100644 index 0000000000..84a9ef55c8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.cc @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/conversational_speech/simulator.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "common_audio/wav_file.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_interface.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/logging.h" +#include "rtc_base/pathutils.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { +namespace test { +namespace { + +using conversational_speech::MultiEndCall; +using conversational_speech::SpeakerOutputFilePaths; +using conversational_speech::WavReaderInterface; + +// Combines output path and speaker names to define the output file paths for +// the near-end and far=end audio tracks. +std::unique_ptr> + InitSpeakerOutputFilePaths(const std::set& speaker_names, + const std::string& output_path) { + // Create map. + auto speaker_output_file_paths_map = rtc::MakeUnique< + std::map>(); + + // Add near-end and far-end output paths into the map. + for (const auto& speaker_name : speaker_names) { + const rtc::Pathname near_end_path( + output_path, "s_" + speaker_name + "-near_end.wav"); + RTC_LOG(LS_VERBOSE) << "The near-end audio track will be created in " + << near_end_path.pathname() << "."; + + const rtc::Pathname far_end_path( + output_path, "s_" + speaker_name + "-far_end.wav"); + RTC_LOG(LS_VERBOSE) << "The far-end audio track will be created in " + << far_end_path.pathname() << "."; + + // Add to map. + speaker_output_file_paths_map->emplace( + std::piecewise_construct, + std::forward_as_tuple(speaker_name), + std::forward_as_tuple(near_end_path.pathname(), + far_end_path.pathname())); + } + + return speaker_output_file_paths_map; +} + +// Class that provides one WavWriter for the near-end and one for the far-end +// output track of a speaker. +class SpeakerWavWriters { + public: + SpeakerWavWriters( + const SpeakerOutputFilePaths& output_file_paths, int sample_rate) + : near_end_wav_writer_(output_file_paths.near_end, sample_rate, 1u), + far_end_wav_writer_(output_file_paths.far_end, sample_rate, 1u) {} + WavWriter* near_end_wav_writer() { + return &near_end_wav_writer_; + } + WavWriter* far_end_wav_writer() { + return &far_end_wav_writer_; + } + private: + WavWriter near_end_wav_writer_; + WavWriter far_end_wav_writer_; +}; + +// Initializes one WavWriter instance for each speaker and both the near-end and +// far-end output tracks. +std::unique_ptr> + InitSpeakersWavWriters(const std::map& + speaker_output_file_paths, int sample_rate) { + // Create map. + auto speaker_wav_writers_map = rtc::MakeUnique< + std::map>(); + + // Add SpeakerWavWriters instance into the map. + for (auto it = speaker_output_file_paths.begin(); + it != speaker_output_file_paths.end(); ++it) { + speaker_wav_writers_map->emplace( + std::piecewise_construct, + std::forward_as_tuple(it->first), + std::forward_as_tuple(it->second, sample_rate)); + } + + return speaker_wav_writers_map; +} + +// Reads all the samples for each audio track. +std::unique_ptr>> PreloadAudioTracks( + const std::map>& + audiotrack_readers) { + // Create map. + auto audiotracks_map = rtc::MakeUnique< + std::map>>(); + + // Add audio track vectors. + for (auto it = audiotrack_readers.begin(); it != audiotrack_readers.end(); + ++it) { + // Add map entry. + audiotracks_map->emplace( + std::piecewise_construct, + std::forward_as_tuple(it->first), + std::forward_as_tuple(it->second->NumSamples())); + + // Read samples. + it->second->ReadInt16Samples(audiotracks_map->at(it->first)); + } + + return audiotracks_map; +} + +// Writes all the values in |source_samples| via |wav_writer|. If the number of +// previously written samples in |wav_writer| is less than |interval_begin|, it +// adds zeros as left padding. The padding corresponds to intervals during which +// a speaker is not active. +void PadLeftWriteChunk(rtc::ArrayView source_samples, + size_t interval_begin, WavWriter* wav_writer) { + // Add left padding. + RTC_CHECK(wav_writer); + RTC_CHECK_GE(interval_begin, wav_writer->num_samples()); + size_t padding_size = interval_begin - wav_writer->num_samples(); + if (padding_size != 0) { + const std::vector padding(padding_size, 0); + wav_writer->WriteSamples(padding.data(), padding_size); + } + + // Write source samples. + wav_writer->WriteSamples(source_samples.data(), source_samples.size()); +} + +// Appends zeros via |wav_writer|. The number of zeros is always non-negative +// and equal to the difference between the previously written samples and +// |pad_samples|. +void PadRightWrite(WavWriter* wav_writer, size_t pad_samples) { + RTC_CHECK(wav_writer); + RTC_CHECK_GE(pad_samples, wav_writer->num_samples()); + size_t padding_size = pad_samples - wav_writer->num_samples(); + if (padding_size != 0) { + const std::vector padding(padding_size, 0); + wav_writer->WriteSamples(padding.data(), padding_size); + } +} + +} // namespace + +namespace conversational_speech { + +std::unique_ptr> Simulate( + const MultiEndCall& multiend_call, const std::string& output_path) { + // Set output file paths and initialize wav writers. + const auto& speaker_names = multiend_call.speaker_names(); + auto speaker_output_file_paths = InitSpeakerOutputFilePaths( + speaker_names, output_path); + auto speakers_wav_writers = InitSpeakersWavWriters( + *speaker_output_file_paths, multiend_call.sample_rate()); + + // Preload all the input audio tracks. + const auto& audiotrack_readers = multiend_call.audiotrack_readers(); + auto audiotracks = PreloadAudioTracks(audiotrack_readers); + + // TODO(alessiob): When speaker_names.size() == 2, near-end and far-end + // across the 2 speakers are symmetric; hence, the code below could be + // replaced by only creating the near-end or the far-end. However, this would + // require to split the unit tests and document the behavior in README.md. + // In practice, it should not be an issue since the files are not expected to + // be signinificant. + + // Write near-end and far-end output tracks. + for (const auto& speaking_turn : multiend_call.speaking_turns()) { + const std::string& active_speaker_name = speaking_turn.speaker_name; + auto source_audiotrack = audiotracks->at( + speaking_turn.audiotrack_file_name); + + // Write active speaker's chunk to active speaker's near-end. + PadLeftWriteChunk(source_audiotrack, speaking_turn.begin, + speakers_wav_writers->at( + active_speaker_name).near_end_wav_writer()); + + // Write active speaker's chunk to other participants' far-ends. + for (const std::string& speaker_name : speaker_names) { + if (speaker_name == active_speaker_name) + continue; + PadLeftWriteChunk(source_audiotrack, speaking_turn.begin, + speakers_wav_writers->at( + speaker_name).far_end_wav_writer()); + } + } + + // Finalize all the output tracks with right padding. + // This is required to make all the output tracks duration equal. + size_t duration_samples = multiend_call.total_duration_samples(); + for (const std::string& speaker_name : speaker_names) { + PadRightWrite(speakers_wav_writers->at(speaker_name).near_end_wav_writer(), + duration_samples); + PadRightWrite(speakers_wav_writers->at(speaker_name).far_end_wav_writer(), + duration_samples); + } + + return speaker_output_file_paths; +} + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.h new file mode 100644 index 0000000000..671c3b9c73 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/simulator.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_SIMULATOR_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_SIMULATOR_H_ + +#include +#include +#include +#include + +#include "modules/audio_processing/test/conversational_speech/multiend_call.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +struct SpeakerOutputFilePaths { + SpeakerOutputFilePaths(const std::string& new_near_end, + const std::string& new_far_end) + : near_end(new_near_end), + far_end(new_far_end) {} + // Paths to the near-end and far-end audio track files. + const std::string near_end; + const std::string far_end; +}; + +// Generates the near-end and far-end audio track pairs for each speaker. +std::unique_ptr> + Simulate(const MultiEndCall& multiend_call, const std::string& output_path); + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_SIMULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.cc new file mode 100644 index 0000000000..773a42ebd7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/conversational_speech/timing.h" + +#include +#include + +#include "rtc_base/stringencode.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +bool Turn::operator==(const Turn &b) const { + return b.speaker_name == speaker_name && + b.audiotrack_file_name == audiotrack_file_name && + b.offset == offset; +} + +std::vector LoadTiming(const std::string& timing_filepath) { + // Line parser. + auto parse_line = [](const std::string& line) { + std::vector fields; + rtc::split(line, ' ', &fields); + RTC_CHECK_EQ(fields.size(), 3); + return Turn(fields[0], fields[1], std::atol(fields[2].c_str())); + }; + + // Init. + std::vector timing; + + // Parse lines. + std::string line; + std::ifstream infile(timing_filepath); + while (std::getline(infile, line)) { + if (line.empty()) + continue; + timing.push_back(parse_line(line)); + } + infile.close(); + + return timing; +} + +void SaveTiming(const std::string& timing_filepath, + rtc::ArrayView timing) { + std::ofstream outfile(timing_filepath); + RTC_CHECK(outfile.is_open()); + for (const Turn& turn : timing) { + outfile << turn.speaker_name << " " << turn.audiotrack_file_name + << " " << turn.offset << std::endl; + } + outfile.close(); +} + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.h new file mode 100644 index 0000000000..dc43361815 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/timing.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_TIMING_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_TIMING_H_ + +#include +#include + +#include "api/array_view.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +struct Turn{ + Turn(std::string new_speaker_name, std::string new_audiotrack_file_name, + int new_offset) + : speaker_name(new_speaker_name), + audiotrack_file_name(new_audiotrack_file_name), + offset(new_offset) {} + bool operator==(const Turn &b) const; + std::string speaker_name; + std::string audiotrack_file_name; + int offset; +}; + +// Loads a list of turns from a file. +std::vector LoadTiming(const std::string& timing_filepath); + +// Writes a list of turns into a file. +void SaveTiming(const std::string& timing_filepath, + rtc::ArrayView timing); + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_TIMING_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_abstract_factory.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_abstract_factory.h new file mode 100644 index 0000000000..83fda0d45c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_abstract_factory.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_ABSTRACT_FACTORY_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_ABSTRACT_FACTORY_H_ + +#include +#include + +#include "modules/audio_processing/test/conversational_speech/wavreader_interface.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +class WavReaderAbstractFactory { + public: + virtual ~WavReaderAbstractFactory() = default; + virtual std::unique_ptr Create( + const std::string& filepath) const = 0; +}; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_ABSTRACT_FACTORY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.cc new file mode 100644 index 0000000000..8342d3fed8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/conversational_speech/wavreader_factory.h" + +#include + +#include "api/array_view.h" +#include "common_audio/wav_file.h" +#include "rtc_base/checks.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { +namespace { + +using conversational_speech::WavReaderInterface; + +class WavReaderAdaptor final : public WavReaderInterface { + public: + explicit WavReaderAdaptor(const std::string& filepath) + : wav_reader_(filepath) {} + ~WavReaderAdaptor() override = default; + + size_t ReadFloatSamples(rtc::ArrayView samples) override { + return wav_reader_.ReadSamples(samples.size(), samples.begin()); + } + + size_t ReadInt16Samples(rtc::ArrayView samples) override { + return wav_reader_.ReadSamples(samples.size(), samples.begin()); + } + + int SampleRate() const override { + return wav_reader_.sample_rate(); + } + + size_t NumChannels() const override { + return wav_reader_.num_channels(); + } + + size_t NumSamples() const override { + return wav_reader_.num_samples(); + } + + private: + WavReader wav_reader_; +}; + +} // namespace + +namespace conversational_speech { + +WavReaderFactory::WavReaderFactory() = default; + +WavReaderFactory::~WavReaderFactory() = default; + +std::unique_ptr WavReaderFactory::Create( + const std::string& filepath) const { + return std::unique_ptr(new WavReaderAdaptor(filepath)); +} + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.h new file mode 100644 index 0000000000..e168d0d71f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_FACTORY_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_FACTORY_H_ + +#include +#include + +#include "modules/audio_processing/test/conversational_speech/wavreader_abstract_factory.h" +#include "modules/audio_processing/test/conversational_speech/wavreader_interface.h" + +namespace webrtc { +namespace test { +namespace conversational_speech { + +class WavReaderFactory : public WavReaderAbstractFactory { + public: + WavReaderFactory(); + ~WavReaderFactory() override; + std::unique_ptr Create(const std::string& filepath) const + override; +}; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_FACTORY_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_interface.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_interface.h new file mode 100644 index 0000000000..3aea273ad2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/conversational_speech/wavreader_interface.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_INTERFACE_H_ +#define MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_INTERFACE_H_ + +#include + +#include "api/array_view.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { +namespace test { +namespace conversational_speech { + +class WavReaderInterface { + public: + virtual ~WavReaderInterface() = default; + + // Returns the number of samples read. + virtual size_t ReadFloatSamples(rtc::ArrayView samples) = 0; + virtual size_t ReadInt16Samples(rtc::ArrayView samples) = 0; + + // Getters. + virtual int SampleRate() const = 0; + virtual size_t NumChannels() const = 0; + virtual size_t NumSamples() const = 0; +}; + +} // namespace conversational_speech +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_CONVERSATIONAL_SPEECH_WAVREADER_INTERFACE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.cc new file mode 100644 index 0000000000..a027d07a9f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.cc @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/debug_dump_replayer.h" + +#include "modules/audio_processing/test/protobuf_utils.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +namespace { + +void MaybeResetBuffer(std::unique_ptr>* buffer, + const StreamConfig& config) { + auto& buffer_ref = *buffer; + if (!buffer_ref.get() || buffer_ref->num_frames() != config.num_frames() || + buffer_ref->num_channels() != config.num_channels()) { + buffer_ref.reset(new ChannelBuffer(config.num_frames(), + config.num_channels())); + } +} + +} // namespace + +DebugDumpReplayer::DebugDumpReplayer() + : input_(nullptr), // will be created upon usage. + reverse_(nullptr), + output_(nullptr), + apm_(nullptr), + debug_file_(nullptr) {} + +DebugDumpReplayer::~DebugDumpReplayer() { + if (debug_file_) + fclose(debug_file_); +} + +bool DebugDumpReplayer::SetDumpFile(const std::string& filename) { + debug_file_ = fopen(filename.c_str(), "rb"); + LoadNextMessage(); + return debug_file_; +} + +// Get next event that has not run. +rtc::Optional DebugDumpReplayer::GetNextEvent() const { + if (!has_next_event_) + return rtc::nullopt; + else + return next_event_; +} + +// Run the next event. Returns the event type. +bool DebugDumpReplayer::RunNextEvent() { + if (!has_next_event_) + return false; + switch (next_event_.type()) { + case audioproc::Event::INIT: + OnInitEvent(next_event_.init()); + break; + case audioproc::Event::STREAM: + OnStreamEvent(next_event_.stream()); + break; + case audioproc::Event::REVERSE_STREAM: + OnReverseStreamEvent(next_event_.reverse_stream()); + break; + case audioproc::Event::CONFIG: + OnConfigEvent(next_event_.config()); + break; + case audioproc::Event::UNKNOWN_EVENT: + // We do not expect to receive UNKNOWN event. + return false; + } + LoadNextMessage(); + return true; +} + +const ChannelBuffer* DebugDumpReplayer::GetOutput() const { + return output_.get(); +} + +StreamConfig DebugDumpReplayer::GetOutputConfig() const { + return output_config_; +} + +// OnInitEvent reset the input/output/reserve channel format. +void DebugDumpReplayer::OnInitEvent(const audioproc::Init& msg) { + RTC_CHECK(msg.has_num_input_channels()); + RTC_CHECK(msg.has_output_sample_rate()); + RTC_CHECK(msg.has_num_output_channels()); + RTC_CHECK(msg.has_reverse_sample_rate()); + RTC_CHECK(msg.has_num_reverse_channels()); + + input_config_ = StreamConfig(msg.sample_rate(), msg.num_input_channels()); + output_config_ = + StreamConfig(msg.output_sample_rate(), msg.num_output_channels()); + reverse_config_ = + StreamConfig(msg.reverse_sample_rate(), msg.num_reverse_channels()); + + MaybeResetBuffer(&input_, input_config_); + MaybeResetBuffer(&output_, output_config_); + MaybeResetBuffer(&reverse_, reverse_config_); +} + +// OnStreamEvent replays an input signal and verifies the output. +void DebugDumpReplayer::OnStreamEvent(const audioproc::Stream& msg) { + // APM should have been created. + RTC_CHECK(apm_.get()); + + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->gain_control()->set_stream_analog_level(msg.level())); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->set_stream_delay_ms(msg.delay())); + + apm_->echo_cancellation()->set_stream_drift_samples(msg.drift()); + if (msg.has_keypress()) { + apm_->set_stream_key_pressed(msg.keypress()); + } else { + apm_->set_stream_key_pressed(true); + } + + RTC_CHECK_EQ(input_config_.num_channels(), + static_cast(msg.input_channel_size())); + RTC_CHECK_EQ(input_config_.num_frames() * sizeof(float), + msg.input_channel(0).size()); + + for (int i = 0; i < msg.input_channel_size(); ++i) { + memcpy(input_->channels()[i], msg.input_channel(i).data(), + msg.input_channel(i).size()); + } + + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->ProcessStream(input_->channels(), input_config_, + output_config_, output_->channels())); +} + +void DebugDumpReplayer::OnReverseStreamEvent( + const audioproc::ReverseStream& msg) { + // APM should have been created. + RTC_CHECK(apm_.get()); + + RTC_CHECK_GT(msg.channel_size(), 0); + RTC_CHECK_EQ(reverse_config_.num_channels(), + static_cast(msg.channel_size())); + RTC_CHECK_EQ(reverse_config_.num_frames() * sizeof(float), + msg.channel(0).size()); + + for (int i = 0; i < msg.channel_size(); ++i) { + memcpy(reverse_->channels()[i], msg.channel(i).data(), + msg.channel(i).size()); + } + + RTC_CHECK_EQ( + AudioProcessing::kNoError, + apm_->ProcessReverseStream(reverse_->channels(), reverse_config_, + reverse_config_, reverse_->channels())); +} + +void DebugDumpReplayer::OnConfigEvent(const audioproc::Config& msg) { + MaybeRecreateApm(msg); + ConfigureApm(msg); +} + +void DebugDumpReplayer::MaybeRecreateApm(const audioproc::Config& msg) { + // These configurations cannot be changed on the fly. + Config config; + RTC_CHECK(msg.has_aec_delay_agnostic_enabled()); + config.Set( + new DelayAgnostic(msg.aec_delay_agnostic_enabled())); + + RTC_CHECK(msg.has_noise_robust_agc_enabled()); + config.Set( + new ExperimentalAgc(msg.noise_robust_agc_enabled())); + + RTC_CHECK(msg.has_transient_suppression_enabled()); + config.Set( + new ExperimentalNs(msg.transient_suppression_enabled())); + + RTC_CHECK(msg.has_aec_extended_filter_enabled()); + config.Set( + new ExtendedFilter(msg.aec_extended_filter_enabled())); + + RTC_CHECK(msg.has_intelligibility_enhancer_enabled()); + config.Set( + new Intelligibility(msg.intelligibility_enhancer_enabled())); + + // We only create APM once, since changes on these fields should not + // happen in current implementation. + if (!apm_.get()) { + apm_.reset(AudioProcessing::Create(config)); + } +} + +void DebugDumpReplayer::ConfigureApm(const audioproc::Config& msg) { + AudioProcessing::Config apm_config; + + // AEC configs. + RTC_CHECK(msg.has_aec_enabled()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->echo_cancellation()->Enable(msg.aec_enabled())); + + RTC_CHECK(msg.has_aec_drift_compensation_enabled()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->echo_cancellation()->enable_drift_compensation( + msg.aec_drift_compensation_enabled())); + + RTC_CHECK(msg.has_aec_suppression_level()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->echo_cancellation()->set_suppression_level( + static_cast( + msg.aec_suppression_level()))); + + // AECM configs. + RTC_CHECK(msg.has_aecm_enabled()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->echo_control_mobile()->Enable(msg.aecm_enabled())); + + RTC_CHECK(msg.has_aecm_comfort_noise_enabled()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->echo_control_mobile()->enable_comfort_noise( + msg.aecm_comfort_noise_enabled())); + + RTC_CHECK(msg.has_aecm_routing_mode()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->echo_control_mobile()->set_routing_mode( + static_cast( + msg.aecm_routing_mode()))); + + // AGC configs. + RTC_CHECK(msg.has_agc_enabled()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->gain_control()->Enable(msg.agc_enabled())); + + RTC_CHECK(msg.has_agc_mode()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->gain_control()->set_mode( + static_cast(msg.agc_mode()))); + + RTC_CHECK(msg.has_agc_limiter_enabled()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->gain_control()->enable_limiter(msg.agc_limiter_enabled())); + + // HPF configs. + RTC_CHECK(msg.has_hpf_enabled()); + apm_config.high_pass_filter.enabled = msg.hpf_enabled(); + + // NS configs. + RTC_CHECK(msg.has_ns_enabled()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->noise_suppression()->Enable(msg.ns_enabled())); + + RTC_CHECK(msg.has_ns_level()); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->noise_suppression()->set_level( + static_cast(msg.ns_level()))); + + apm_->ApplyConfig(apm_config); +} + +void DebugDumpReplayer::LoadNextMessage() { + has_next_event_ = + debug_file_ && ReadMessageFromFile(debug_file_, &next_event_); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.h new file mode 100644 index 0000000000..4cd961dd7f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_replayer.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_DEBUG_DUMP_REPLAYER_H_ +#define MODULES_AUDIO_PROCESSING_TEST_DEBUG_DUMP_REPLAYER_H_ + +#include +#include + +#include "common_audio/channel_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/ignore_wundef.h" + +RTC_PUSH_IGNORING_WUNDEF() +#include "modules/audio_processing/debug.pb.h" +RTC_POP_IGNORING_WUNDEF() + +namespace webrtc { +namespace test { + +class DebugDumpReplayer { + public: + DebugDumpReplayer(); + ~DebugDumpReplayer(); + + // Set dump file + bool SetDumpFile(const std::string& filename); + + // Return next event. + rtc::Optional GetNextEvent() const; + + // Run the next event. Returns true if succeeded. + bool RunNextEvent(); + + const ChannelBuffer* GetOutput() const; + StreamConfig GetOutputConfig() const; + + private: + // Following functions are facilities for replaying debug dumps. + void OnInitEvent(const audioproc::Init& msg); + void OnStreamEvent(const audioproc::Stream& msg); + void OnReverseStreamEvent(const audioproc::ReverseStream& msg); + void OnConfigEvent(const audioproc::Config& msg); + + void MaybeRecreateApm(const audioproc::Config& msg); + void ConfigureApm(const audioproc::Config& msg); + + void LoadNextMessage(); + + // Buffer for APM input/output. + std::unique_ptr> input_; + std::unique_ptr> reverse_; + std::unique_ptr> output_; + + std::unique_ptr apm_; + + FILE* debug_file_; + + StreamConfig input_config_; + StreamConfig reverse_config_; + StreamConfig output_config_; + + bool has_next_event_; + audioproc::Event next_event_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_DEBUG_DUMP_REPLAYER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_test.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_test.cc new file mode 100644 index 0000000000..2c839d3bac --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/debug_dump_test.cc @@ -0,0 +1,620 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include // size_t + +#include +#include +#include + +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "modules/audio_processing/test/debug_dump_replayer.h" +#include "modules/audio_processing/test/test_utils.h" +#include "rtc_base/task_queue.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +namespace { + +void MaybeResetBuffer(std::unique_ptr>* buffer, + const StreamConfig& config) { + auto& buffer_ref = *buffer; + if (!buffer_ref.get() || buffer_ref->num_frames() != config.num_frames() || + buffer_ref->num_channels() != config.num_channels()) { + buffer_ref.reset(new ChannelBuffer(config.num_frames(), + config.num_channels())); + } +} + +class DebugDumpGenerator { + public: + DebugDumpGenerator(const std::string& input_file_name, + int input_rate_hz, + int input_channels, + const std::string& reverse_file_name, + int reverse_rate_hz, + int reverse_channels, + const Config& config, + const std::string& dump_file_name, + bool enable_aec3); + + // Constructor that uses default input files. + explicit DebugDumpGenerator(const Config& config, + const AudioProcessing::Config& apm_config, + bool enable_aec3); + + explicit DebugDumpGenerator(const Config& config, + const AudioProcessing::Config& apm_config); + + ~DebugDumpGenerator(); + + // Changes the sample rate of the input audio to the APM. + void SetInputRate(int rate_hz); + + // Sets if converts stereo input signal to mono by discarding other channels. + void ForceInputMono(bool mono); + + // Changes the sample rate of the reverse audio to the APM. + void SetReverseRate(int rate_hz); + + // Sets if converts stereo reverse signal to mono by discarding other + // channels. + void ForceReverseMono(bool mono); + + // Sets the required sample rate of the APM output. + void SetOutputRate(int rate_hz); + + // Sets the required channels of the APM output. + void SetOutputChannels(int channels); + + std::string dump_file_name() const { return dump_file_name_; } + + void StartRecording(); + void Process(size_t num_blocks); + void StopRecording(); + AudioProcessing* apm() const { return apm_.get(); } + + private: + static void ReadAndDeinterleave(ResampleInputAudioFile* audio, int channels, + const StreamConfig& config, + float* const* buffer); + + // APM input/output settings. + StreamConfig input_config_; + StreamConfig reverse_config_; + StreamConfig output_config_; + + // Input file format. + const std::string input_file_name_; + ResampleInputAudioFile input_audio_; + const int input_file_channels_; + + // Reverse file format. + const std::string reverse_file_name_; + ResampleInputAudioFile reverse_audio_; + const int reverse_file_channels_; + + // Buffer for APM input/output. + std::unique_ptr> input_; + std::unique_ptr> reverse_; + std::unique_ptr> output_; + + rtc::TaskQueue worker_queue_; + std::unique_ptr apm_; + + const std::string dump_file_name_; +}; + +DebugDumpGenerator::DebugDumpGenerator(const std::string& input_file_name, + int input_rate_hz, + int input_channels, + const std::string& reverse_file_name, + int reverse_rate_hz, + int reverse_channels, + const Config& config, + const std::string& dump_file_name, + bool enable_aec3) + : input_config_(input_rate_hz, input_channels), + reverse_config_(reverse_rate_hz, reverse_channels), + output_config_(input_rate_hz, input_channels), + input_audio_(input_file_name, input_rate_hz, input_rate_hz), + input_file_channels_(input_channels), + reverse_audio_(reverse_file_name, reverse_rate_hz, reverse_rate_hz), + reverse_file_channels_(reverse_channels), + input_(new ChannelBuffer(input_config_.num_frames(), + input_config_.num_channels())), + reverse_(new ChannelBuffer(reverse_config_.num_frames(), + reverse_config_.num_channels())), + output_(new ChannelBuffer(output_config_.num_frames(), + output_config_.num_channels())), + worker_queue_("debug_dump_generator_worker_queue"), + apm_(AudioProcessing::Create( + config, + nullptr, + (enable_aec3 ? std::unique_ptr( + new EchoCanceller3Factory()) + : nullptr), + nullptr)), + dump_file_name_(dump_file_name) {} + +DebugDumpGenerator::DebugDumpGenerator( + const Config& config, + const AudioProcessing::Config& apm_config, + bool enable_aec3) + : DebugDumpGenerator(ResourcePath("near32_stereo", "pcm"), + 32000, + 2, + ResourcePath("far32_stereo", "pcm"), + 32000, + 2, + config, + TempFilename(OutputPath(), "debug_aec"), + enable_aec3) { + apm_->ApplyConfig(apm_config); +} + +DebugDumpGenerator::DebugDumpGenerator( + const Config& config, + const AudioProcessing::Config& apm_config) + : DebugDumpGenerator(config, apm_config, false) { + apm_->ApplyConfig(apm_config); +} + +DebugDumpGenerator::~DebugDumpGenerator() { + remove(dump_file_name_.c_str()); +} + +void DebugDumpGenerator::SetInputRate(int rate_hz) { + input_audio_.set_output_rate_hz(rate_hz); + input_config_.set_sample_rate_hz(rate_hz); + MaybeResetBuffer(&input_, input_config_); +} + +void DebugDumpGenerator::ForceInputMono(bool mono) { + const int channels = mono ? 1 : input_file_channels_; + input_config_.set_num_channels(channels); + MaybeResetBuffer(&input_, input_config_); +} + +void DebugDumpGenerator::SetReverseRate(int rate_hz) { + reverse_audio_.set_output_rate_hz(rate_hz); + reverse_config_.set_sample_rate_hz(rate_hz); + MaybeResetBuffer(&reverse_, reverse_config_); +} + +void DebugDumpGenerator::ForceReverseMono(bool mono) { + const int channels = mono ? 1 : reverse_file_channels_; + reverse_config_.set_num_channels(channels); + MaybeResetBuffer(&reverse_, reverse_config_); +} + +void DebugDumpGenerator::SetOutputRate(int rate_hz) { + output_config_.set_sample_rate_hz(rate_hz); + MaybeResetBuffer(&output_, output_config_); +} + +void DebugDumpGenerator::SetOutputChannels(int channels) { + output_config_.set_num_channels(channels); + MaybeResetBuffer(&output_, output_config_); +} + +void DebugDumpGenerator::StartRecording() { + apm_->AttachAecDump( + AecDumpFactory::Create(dump_file_name_.c_str(), -1, &worker_queue_)); +} + +void DebugDumpGenerator::Process(size_t num_blocks) { + for (size_t i = 0; i < num_blocks; ++i) { + ReadAndDeinterleave(&reverse_audio_, reverse_file_channels_, + reverse_config_, reverse_->channels()); + ReadAndDeinterleave(&input_audio_, input_file_channels_, input_config_, + input_->channels()); + RTC_CHECK_EQ(AudioProcessing::kNoError, apm_->set_stream_delay_ms(100)); + apm_->set_stream_key_pressed(i % 10 == 9); + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->ProcessStream(input_->channels(), input_config_, + output_config_, output_->channels())); + + RTC_CHECK_EQ(AudioProcessing::kNoError, + apm_->ProcessReverseStream(reverse_->channels(), + reverse_config_, + reverse_config_, + reverse_->channels())); + } +} + +void DebugDumpGenerator::StopRecording() { + apm_->DetachAecDump(); +} + +void DebugDumpGenerator::ReadAndDeinterleave(ResampleInputAudioFile* audio, + int channels, + const StreamConfig& config, + float* const* buffer) { + const size_t num_frames = config.num_frames(); + const int out_channels = config.num_channels(); + + std::vector signal(channels * num_frames); + + audio->Read(num_frames * channels, &signal[0]); + + // We only allow reducing number of channels by discarding some channels. + RTC_CHECK_LE(out_channels, channels); + for (int channel = 0; channel < out_channels; ++channel) { + for (size_t i = 0; i < num_frames; ++i) { + buffer[channel][i] = S16ToFloat(signal[i * channels + channel]); + } + } +} + +} // namespace + +class DebugDumpTest : public ::testing::Test { + public: + // VerifyDebugDump replays a debug dump using APM and verifies that the result + // is bit-exact-identical to the output channel in the dump. This is only + // guaranteed if the debug dump is started on the first frame. + void VerifyDebugDump(const std::string& in_filename); + + private: + DebugDumpReplayer debug_dump_replayer_; +}; + +void DebugDumpTest::VerifyDebugDump(const std::string& in_filename) { + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(in_filename)); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::STREAM) { + const audioproc::Stream* msg = &event->stream(); + const StreamConfig output_config = debug_dump_replayer_.GetOutputConfig(); + const ChannelBuffer* output = debug_dump_replayer_.GetOutput(); + // Check that output of APM is bit-exact to the output in the dump. + ASSERT_EQ(output_config.num_channels(), + static_cast(msg->output_channel_size())); + ASSERT_EQ(output_config.num_frames() * sizeof(float), + msg->output_channel(0).size()); + for (int i = 0; i < msg->output_channel_size(); ++i) { + ASSERT_EQ(0, memcmp(output->channels()[i], + msg->output_channel(i).data(), + msg->output_channel(i).size())); + } + } + } +} + +TEST_F(DebugDumpTest, SimpleCase) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, ChangeInputFormat) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + + generator.StartRecording(); + generator.Process(100); + generator.SetInputRate(48000); + + generator.ForceInputMono(true); + // Number of output channel should not be larger than that of input. APM will + // fail otherwise. + generator.SetOutputChannels(1); + + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, ChangeReverseFormat) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.SetReverseRate(48000); + generator.ForceReverseMono(true); + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, ChangeOutputFormat) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.SetOutputRate(48000); + generator.SetOutputChannels(1); + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, ToggleAec) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + + EchoCancellation* aec = generator.apm()->echo_cancellation(); + EXPECT_EQ(AudioProcessing::kNoError, aec->Enable(!aec->is_enabled())); + + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, ToggleDelayAgnosticAec) { + Config config; + config.Set(new DelayAgnostic(true)); + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + + EchoCancellation* aec = generator.apm()->echo_cancellation(); + EXPECT_EQ(AudioProcessing::kNoError, aec->Enable(!aec->is_enabled())); + + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, VerifyRefinedAdaptiveFilterExperimentalString) { + Config config; + config.Set(new RefinedAdaptiveFilter(true)); + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + + DebugDumpReplayer debug_dump_replayer_; + + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(generator.dump_file_name())); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::CONFIG) { + const audioproc::Config* msg = &event->config(); + ASSERT_TRUE(msg->has_experiments_description()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "RefinedAdaptiveFilter", + msg->experiments_description().c_str()); + } + } +} + +TEST_F(DebugDumpTest, VerifyCombinedExperimentalStringInclusive) { + Config config; + AudioProcessing::Config apm_config; + config.Set(new RefinedAdaptiveFilter(true)); + // Arbitrarily set clipping gain to 17, which will never be the default. + config.Set(new ExperimentalAgc(true, 0, 17)); + bool enable_aec3 = true; + DebugDumpGenerator generator(config, apm_config, enable_aec3); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + + DebugDumpReplayer debug_dump_replayer_; + + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(generator.dump_file_name())); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::CONFIG) { + const audioproc::Config* msg = &event->config(); + ASSERT_TRUE(msg->has_experiments_description()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "RefinedAdaptiveFilter", + msg->experiments_description().c_str()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "EchoController", + msg->experiments_description().c_str()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "AgcClippingLevelExperiment", + msg->experiments_description().c_str()); + } + } +} + +TEST_F(DebugDumpTest, VerifyCombinedExperimentalStringExclusive) { + Config config; + config.Set(new RefinedAdaptiveFilter(true)); + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + + DebugDumpReplayer debug_dump_replayer_; + + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(generator.dump_file_name())); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::CONFIG) { + const audioproc::Config* msg = &event->config(); + ASSERT_TRUE(msg->has_experiments_description()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "RefinedAdaptiveFilter", + msg->experiments_description().c_str()); + EXPECT_PRED_FORMAT2(testing::IsNotSubstring, "AEC3", + msg->experiments_description().c_str()); + EXPECT_PRED_FORMAT2(testing::IsNotSubstring, "AgcClippingLevelExperiment", + msg->experiments_description().c_str()); + } + } +} + +TEST_F(DebugDumpTest, VerifyAec3ExperimentalString) { + Config config; + AudioProcessing::Config apm_config; + DebugDumpGenerator generator(config, apm_config, true); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + + DebugDumpReplayer debug_dump_replayer_; + + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(generator.dump_file_name())); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::CONFIG) { + const audioproc::Config* msg = &event->config(); + ASSERT_TRUE(msg->has_experiments_description()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "EchoController", + msg->experiments_description().c_str()); + } + } +} + +TEST_F(DebugDumpTest, VerifyLevelControllerExperimentalString) { + Config config; + AudioProcessing::Config apm_config; + apm_config.level_controller.enabled = true; + DebugDumpGenerator generator(config, apm_config); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + + DebugDumpReplayer debug_dump_replayer_; + + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(generator.dump_file_name())); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::CONFIG) { + const audioproc::Config* msg = &event->config(); + ASSERT_TRUE(msg->has_experiments_description()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "LevelController", + msg->experiments_description().c_str()); + } + } +} + +TEST_F(DebugDumpTest, VerifyAgcClippingLevelExperimentalString) { + Config config; + // Arbitrarily set clipping gain to 17, which will never be the default. + config.Set(new ExperimentalAgc(true, 0, 17)); + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + + DebugDumpReplayer debug_dump_replayer_; + + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(generator.dump_file_name())); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::CONFIG) { + const audioproc::Config* msg = &event->config(); + ASSERT_TRUE(msg->has_experiments_description()); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "AgcClippingLevelExperiment", + msg->experiments_description().c_str()); + } + } +} + +TEST_F(DebugDumpTest, VerifyEmptyExperimentalString) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + + DebugDumpReplayer debug_dump_replayer_; + + ASSERT_TRUE(debug_dump_replayer_.SetDumpFile(generator.dump_file_name())); + + while (const rtc::Optional event = + debug_dump_replayer_.GetNextEvent()) { + debug_dump_replayer_.RunNextEvent(); + if (event->type() == audioproc::Event::CONFIG) { + const audioproc::Config* msg = &event->config(); + ASSERT_TRUE(msg->has_experiments_description()); + EXPECT_EQ(0u, msg->experiments_description().size()); + } + } +} + +TEST_F(DebugDumpTest, ToggleAecLevel) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + EchoCancellation* aec = generator.apm()->echo_cancellation(); + EXPECT_EQ(AudioProcessing::kNoError, aec->Enable(true)); + EXPECT_EQ(AudioProcessing::kNoError, + aec->set_suppression_level(EchoCancellation::kLowSuppression)); + generator.StartRecording(); + generator.Process(100); + + EXPECT_EQ(AudioProcessing::kNoError, + aec->set_suppression_level(EchoCancellation::kHighSuppression)); + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +// AGC is not supported on Android or iOS. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_ToggleAgc DISABLED_ToggleAgc +#else +#define MAYBE_ToggleAgc ToggleAgc +#endif +TEST_F(DebugDumpTest, MAYBE_ToggleAgc) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + + GainControl* agc = generator.apm()->gain_control(); + EXPECT_EQ(AudioProcessing::kNoError, agc->Enable(!agc->is_enabled())); + + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, ToggleNs) { + Config config; + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + + NoiseSuppression* ns = generator.apm()->noise_suppression(); + EXPECT_EQ(AudioProcessing::kNoError, ns->Enable(!ns->is_enabled())); + + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +TEST_F(DebugDumpTest, TransientSuppressionOn) { + Config config; + config.Set(new ExperimentalNs(true)); + DebugDumpGenerator generator(config, AudioProcessing::Config()); + generator.StartRecording(); + generator.Process(100); + generator.StopRecording(); + VerifyDebugDump(generator.dump_file_name()); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc new file mode 100644 index 0000000000..eba852d4b5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/echo_canceller_test_tools.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +void RandomizeSampleVector(Random* random_generator, rtc::ArrayView v) { + for (auto& v_k : v) { + v_k = 2 * 32767.f * random_generator->Rand() - 32767.f; + } +} + +template +void DelayBuffer::Delay(rtc::ArrayView x, + rtc::ArrayView x_delayed) { + RTC_DCHECK_EQ(x.size(), x_delayed.size()); + if (buffer_.empty()) { + std::copy(x.begin(), x.end(), x_delayed.begin()); + } else { + for (size_t k = 0; k < x.size(); ++k) { + x_delayed[k] = buffer_[next_insert_index_]; + buffer_[next_insert_index_] = x[k]; + next_insert_index_ = (next_insert_index_ + 1) % buffer_.size(); + } + } +} + +template class DelayBuffer; +template class DelayBuffer; +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.h new file mode 100644 index 0000000000..1cb5b7e762 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_ECHO_CANCELLER_TEST_TOOLS_H_ +#define MODULES_AUDIO_PROCESSING_TEST_ECHO_CANCELLER_TEST_TOOLS_H_ + +#include +#include + +#include "api/array_view.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/random.h" + +namespace webrtc { + +// Randomizes the elements in a vector with values -32767.f:32767.f. +void RandomizeSampleVector(Random* random_generator, rtc::ArrayView v); + +// Class for delaying a signal a fixed number of samples. +template +class DelayBuffer { + public: + explicit DelayBuffer(size_t delay) : buffer_(delay) {} + ~DelayBuffer() = default; + + // Produces a delayed signal copy of x. + void Delay(rtc::ArrayView x, rtc::ArrayView x_delayed); + + private: + std::vector buffer_; + size_t next_insert_index_ = 0; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DelayBuffer); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_ECHO_CANCELLER_TEST_TOOLS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc new file mode 100644 index 0000000000..4d5a4fb34b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/echo_canceller_test_tools.h" + +#include + +#include "api/array_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(EchoCancellerTestTools, FloatDelayBuffer) { + constexpr size_t kDelay = 10; + DelayBuffer delay_buffer(kDelay); + std::vector v(1000, 0.f); + for (size_t k = 0; k < v.size(); ++k) { + v[k] = k; + } + std::vector v_delayed = v; + constexpr size_t kBlockSize = 50; + for (size_t k = 0; k < rtc::CheckedDivExact(v.size(), kBlockSize); ++k) { + delay_buffer.Delay( + rtc::ArrayView(&v[k * kBlockSize], kBlockSize), + rtc::ArrayView(&v_delayed[k * kBlockSize], kBlockSize)); + } + for (size_t k = kDelay; k < v.size(); ++k) { + EXPECT_EQ(v[k - kDelay], v_delayed[k]); + } +} + +TEST(EchoCancellerTestTools, IntDelayBuffer) { + constexpr size_t kDelay = 10; + DelayBuffer delay_buffer(kDelay); + std::vector v(1000, 0); + for (size_t k = 0; k < v.size(); ++k) { + v[k] = k; + } + std::vector v_delayed = v; + const size_t kBlockSize = 50; + for (size_t k = 0; k < rtc::CheckedDivExact(v.size(), kBlockSize); ++k) { + delay_buffer.Delay( + rtc::ArrayView(&v[k * kBlockSize], kBlockSize), + rtc::ArrayView(&v_delayed[k * kBlockSize], kBlockSize)); + } + for (size_t k = kDelay; k < v.size(); ++k) { + EXPECT_EQ(v[k - kDelay], v_delayed[k]); + } +} + +TEST(EchoCancellerTestTools, RandomizeSampleVector) { + Random random_generator(42U); + std::vector v(50, 0.f); + std::vector v_ref = v; + RandomizeSampleVector(&random_generator, v); + EXPECT_NE(v, v_ref); + v_ref = v; + RandomizeSampleVector(&random_generator, v); + EXPECT_NE(v, v_ref); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.cc new file mode 100644 index 0000000000..aee3dcef61 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/fake_recording_device.h" + +#include + +#include "rtc_base/logging.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { +namespace test { + +namespace { + +constexpr int16_t kInt16SampleMin = -32768; +constexpr int16_t kInt16SampleMax = 32767; +constexpr float kFloatSampleMin = -32768.f; +constexpr float kFloatSampleMax = 32767.0f; + +} // namespace + +// Abstract class for the different fake recording devices. +class FakeRecordingDeviceWorker { + public: + explicit FakeRecordingDeviceWorker(const int initial_mic_level) + : mic_level_(initial_mic_level) {} + int mic_level() const { return mic_level_; } + void set_mic_level(const int level) { mic_level_ = level; } + void set_undo_mic_level(const int level) { undo_mic_level_ = level; } + virtual ~FakeRecordingDeviceWorker() = default; + virtual void ModifyBufferInt16(AudioFrame* buffer) = 0; + virtual void ModifyBufferFloat(ChannelBuffer* buffer) = 0; + + protected: + // Mic level to simulate. + int mic_level_; + // Optional mic level to undo. + rtc::Optional undo_mic_level_; +}; + +namespace { + +// Identity fake recording device. The samples are not modified, which is +// equivalent to a constant gain curve at 1.0 - only used for testing. +class FakeRecordingDeviceIdentity final : public FakeRecordingDeviceWorker { + public: + explicit FakeRecordingDeviceIdentity(const int initial_mic_level) + : FakeRecordingDeviceWorker(initial_mic_level) {} + ~FakeRecordingDeviceIdentity() override = default; + void ModifyBufferInt16(AudioFrame* buffer) override {} + void ModifyBufferFloat(ChannelBuffer* buffer) override {} +}; + +// Linear fake recording device. The gain curve is a linear function mapping the +// mic levels range [0, 255] to [0.0, 1.0]. +class FakeRecordingDeviceLinear final : public FakeRecordingDeviceWorker { + public: + explicit FakeRecordingDeviceLinear(const int initial_mic_level) + : FakeRecordingDeviceWorker(initial_mic_level) {} + ~FakeRecordingDeviceLinear() override = default; + void ModifyBufferInt16(AudioFrame* buffer) override { + const size_t number_of_samples = + buffer->samples_per_channel_ * buffer->num_channels_; + int16_t* data = buffer->mutable_data(); + // If an undo level is specified, virtually restore the unmodified + // microphone level; otherwise simulate the mic gain only. + const float divisor = + (undo_mic_level_ && *undo_mic_level_ > 0) ? *undo_mic_level_ : 255.f; + for (size_t i = 0; i < number_of_samples; ++i) { + data[i] = + std::max(kInt16SampleMin, + std::min(kInt16SampleMax, + static_cast(static_cast(data[i]) * + mic_level_ / divisor))); + } + } + void ModifyBufferFloat(ChannelBuffer* buffer) override { + // If an undo level is specified, virtually restore the unmodified + // microphone level; otherwise simulate the mic gain only. + const float divisor = + (undo_mic_level_ && *undo_mic_level_ > 0) ? *undo_mic_level_ : 255.f; + for (size_t c = 0; c < buffer->num_channels(); ++c) { + for (size_t i = 0; i < buffer->num_frames(); ++i) { + buffer->channels()[c][i] = + std::max(kFloatSampleMin, + std::min(kFloatSampleMax, + buffer->channels()[c][i] * mic_level_ / divisor)); + } + } + } +}; + +} // namespace + +FakeRecordingDevice::FakeRecordingDevice(int initial_mic_level, + int device_kind) { + switch (device_kind) { + case 0: + worker_ = rtc::MakeUnique(initial_mic_level); + break; + case 1: + worker_ = rtc::MakeUnique(initial_mic_level); + break; + default: + RTC_NOTREACHED(); + break; + } +} + +FakeRecordingDevice::~FakeRecordingDevice() = default; + +int FakeRecordingDevice::MicLevel() const { + RTC_CHECK(worker_); + return worker_->mic_level(); +} + +void FakeRecordingDevice::SetMicLevel(const int level) { + RTC_CHECK(worker_); + if (level != worker_->mic_level()) + RTC_LOG(LS_INFO) << "Simulate mic level update: " << level; + worker_->set_mic_level(level); +} + +void FakeRecordingDevice::SetUndoMicLevel(const int level) { + RTC_DCHECK(worker_); + // TODO(alessiob): The behavior with undo level equal to zero is not clear yet + // and will be defined in future CLs once more FakeRecordingDeviceWorker + // implementations need to be added. + RTC_CHECK(level > 0) << "Zero undo mic level is unsupported"; + worker_->set_undo_mic_level(level); +} + +void FakeRecordingDevice::SimulateAnalogGain(AudioFrame* buffer) { + RTC_DCHECK(worker_); + worker_->ModifyBufferInt16(buffer); +} + +void FakeRecordingDevice::SimulateAnalogGain(ChannelBuffer* buffer) { + RTC_DCHECK(worker_); + worker_->ModifyBufferFloat(buffer); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.h new file mode 100644 index 0000000000..b1e37a331d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_FAKE_RECORDING_DEVICE_H_ +#define MODULES_AUDIO_PROCESSING_TEST_FAKE_RECORDING_DEVICE_H_ + +#include +#include +#include + +#include "api/array_view.h" +#include "common_audio/channel_buffer.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +class FakeRecordingDeviceWorker; + +// Class for simulating a microphone with analog gain. +// +// The intended modes of operation are the following: +// +// FakeRecordingDevice fake_mic(255, 1); +// +// fake_mic.SetMicLevel(170); +// fake_mic.SimulateAnalogGain(buffer); +// +// When the mic level to undo is known: +// +// fake_mic.SetMicLevel(170); +// fake_mic.SetUndoMicLevel(30); +// fake_mic.SimulateAnalogGain(buffer); +// +// The second option virtually restores the unmodified microphone level. Calling +// SimulateAnalogGain() will first "undo" the gain applied by the real +// microphone (e.g., 30). +class FakeRecordingDevice final { + public: + FakeRecordingDevice(int initial_mic_level, int device_kind); + ~FakeRecordingDevice(); + + int MicLevel() const; + void SetMicLevel(const int level); + void SetUndoMicLevel(const int level); + + // Simulates the analog gain. + // If |real_device_level| is a valid level, the unmodified mic signal is + // virtually restored. To skip the latter step set |real_device_level| to + // an empty value. + void SimulateAnalogGain(AudioFrame* buffer); + + // Simulates the analog gain. + // If |real_device_level| is a valid level, the unmodified mic signal is + // virtually restored. To skip the latter step set |real_device_level| to + // an empty value. + void SimulateAnalogGain(ChannelBuffer* buffer); + + private: + // Fake recording device worker. + std::unique_ptr worker_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_FAKE_RECORDING_DEVICE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device_unittest.cc new file mode 100644 index 0000000000..504459a3c6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/fake_recording_device_unittest.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/test/fake_recording_device.h" +#include "rtc_base/ptr_util.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +constexpr int kInitialMicLevel = 100; + +// TODO(alessiob): Add new fake recording device kind values here as they are +// added in FakeRecordingDevice::FakeRecordingDevice. +const std::vector kFakeRecDeviceKinds = {0, 1}; + +const std::vector> kTestMultiChannelSamples{ + std::vector{-10.f, -1.f, -0.1f, 0.f, 0.1f, 1.f, 10.f}}; + +// Writes samples into ChannelBuffer. +void WritesDataIntoChannelBuffer(const std::vector>& data, + ChannelBuffer* buff) { + EXPECT_EQ(data.size(), buff->num_channels()); + EXPECT_EQ(data[0].size(), buff->num_frames()); + for (size_t c = 0; c < buff->num_channels(); ++c) { + for (size_t f = 0; f < buff->num_frames(); ++f) { + buff->channels()[c][f] = data[c][f]; + } + } +} + +std::unique_ptr> CreateChannelBufferWithData( + const std::vector>& data) { + auto buff = + rtc::MakeUnique>(data[0].size(), data.size()); + WritesDataIntoChannelBuffer(data, buff.get()); + return buff; +} + +// Checks that the samples modified using monotonic level values are also +// monotonic. +void CheckIfMonotoneSamplesModules(const ChannelBuffer* prev, + const ChannelBuffer* curr) { + RTC_DCHECK_EQ(prev->num_channels(), curr->num_channels()); + RTC_DCHECK_EQ(prev->num_frames(), curr->num_frames()); + bool valid = true; + for (size_t i = 0; i < prev->num_channels(); ++i) { + for (size_t j = 0; j < prev->num_frames(); ++j) { + valid = std::fabs(prev->channels()[i][j]) <= + std::fabs(curr->channels()[i][j]); + if (!valid) { + break; + } + } + if (!valid) { + break; + } + } + EXPECT_TRUE(valid); +} + +// Checks that the samples in each pair have the same sign unless the sample in +// |dst| is zero (because of zero gain). +void CheckSameSign(const ChannelBuffer* src, + const ChannelBuffer* dst) { + RTC_DCHECK_EQ(src->num_channels(), dst->num_channels()); + RTC_DCHECK_EQ(src->num_frames(), dst->num_frames()); + const auto fsgn = [](float x) { return ((x < 0) ? -1 : (x > 0) ? 1 : 0); }; + bool valid = true; + for (size_t i = 0; i < src->num_channels(); ++i) { + for (size_t j = 0; j < src->num_frames(); ++j) { + valid = dst->channels()[i][j] == 0.0f || + fsgn(src->channels()[i][j]) == fsgn(dst->channels()[i][j]); + if (!valid) { + break; + } + } + if (!valid) { + break; + } + } + EXPECT_TRUE(valid); +} + +std::string FakeRecordingDeviceKindToString(int fake_rec_device_kind) { + std::ostringstream ss; + ss << "fake recording device: " << fake_rec_device_kind; + return ss.str(); +} + +std::string AnalogLevelToString(int level) { + std::ostringstream ss; + ss << "analog level: " << level; + return ss.str(); +} + +} // namespace + +TEST(FakeRecordingDevice, CheckHelperFunctions) { + constexpr size_t kC = 0; // Channel index. + constexpr size_t kS = 1; // Sample index. + + // Check read. + auto buff = CreateChannelBufferWithData(kTestMultiChannelSamples); + for (size_t c = 0; c < kTestMultiChannelSamples.size(); ++c) { + for (size_t s = 0; s < kTestMultiChannelSamples[0].size(); ++s) { + EXPECT_EQ(kTestMultiChannelSamples[c][s], buff->channels()[c][s]); + } + } + + // Check write. + buff->channels()[kC][kS] = -5.0f; + RTC_DCHECK_NE(buff->channels()[kC][kS], kTestMultiChannelSamples[kC][kS]); + + // Check reset. + WritesDataIntoChannelBuffer(kTestMultiChannelSamples, buff.get()); + EXPECT_EQ(buff->channels()[kC][kS], kTestMultiChannelSamples[kC][kS]); +} + +// Implicitly checks that changes to the mic and undo levels are visible to the +// FakeRecordingDeviceWorker implementation are injected in FakeRecordingDevice. +TEST(FakeRecordingDevice, TestWorkerAbstractClass) { + FakeRecordingDevice fake_recording_device(kInitialMicLevel, 1); + + auto buff1 = CreateChannelBufferWithData(kTestMultiChannelSamples); + fake_recording_device.SetMicLevel(100); + fake_recording_device.SimulateAnalogGain(buff1.get()); + + auto buff2 = CreateChannelBufferWithData(kTestMultiChannelSamples); + fake_recording_device.SetMicLevel(200); + fake_recording_device.SimulateAnalogGain(buff2.get()); + + for (size_t c = 0; c < kTestMultiChannelSamples.size(); ++c) { + for (size_t s = 0; s < kTestMultiChannelSamples[0].size(); ++s) { + EXPECT_LE(std::abs(buff1->channels()[c][s]), + std::abs(buff2->channels()[c][s])); + } + } + + auto buff3 = CreateChannelBufferWithData(kTestMultiChannelSamples); + fake_recording_device.SetMicLevel(200); + fake_recording_device.SetUndoMicLevel(100); + fake_recording_device.SimulateAnalogGain(buff3.get()); + + for (size_t c = 0; c < kTestMultiChannelSamples.size(); ++c) { + for (size_t s = 0; s < kTestMultiChannelSamples[0].size(); ++s) { + EXPECT_LE(std::abs(buff1->channels()[c][s]), + std::abs(buff3->channels()[c][s])); + EXPECT_LE(std::abs(buff2->channels()[c][s]), + std::abs(buff3->channels()[c][s])); + } + } +} + +TEST(FakeRecordingDevice, GainCurveShouldBeMonotone) { + // Create input-output buffers. + auto buff_prev = CreateChannelBufferWithData(kTestMultiChannelSamples); + auto buff_curr = CreateChannelBufferWithData(kTestMultiChannelSamples); + + // Test different mappings. + for (auto fake_rec_device_kind : kFakeRecDeviceKinds) { + SCOPED_TRACE(FakeRecordingDeviceKindToString(fake_rec_device_kind)); + FakeRecordingDevice fake_recording_device(kInitialMicLevel, + fake_rec_device_kind); + // TODO(alessiob): The test below is designed for state-less recording + // devices. If, for instance, a device has memory, the test might need + // to be redesigned (e.g., re-initialize fake recording device). + + // Apply lowest analog level. + WritesDataIntoChannelBuffer(kTestMultiChannelSamples, buff_prev.get()); + fake_recording_device.SetMicLevel(0); + fake_recording_device.SimulateAnalogGain(buff_prev.get()); + + // Increment analog level to check monotonicity. + for (int i = 1; i <= 255; ++i) { + SCOPED_TRACE(AnalogLevelToString(i)); + WritesDataIntoChannelBuffer(kTestMultiChannelSamples, buff_curr.get()); + fake_recording_device.SetMicLevel(i); + fake_recording_device.SimulateAnalogGain(buff_curr.get()); + CheckIfMonotoneSamplesModules(buff_prev.get(), buff_curr.get()); + + // Update prev. + buff_prev.swap(buff_curr); + } + } +} + +TEST(FakeRecordingDevice, GainCurveShouldNotChangeSign) { + // Create view on original samples. + std::unique_ptr> buff_orig = + CreateChannelBufferWithData(kTestMultiChannelSamples); + + // Create output buffer. + auto buff = CreateChannelBufferWithData(kTestMultiChannelSamples); + + // Test different mappings. + for (auto fake_rec_device_kind : kFakeRecDeviceKinds) { + SCOPED_TRACE(FakeRecordingDeviceKindToString(fake_rec_device_kind)); + FakeRecordingDevice fake_recording_device(kInitialMicLevel, + fake_rec_device_kind); + + // TODO(alessiob): The test below is designed for state-less recording + // devices. If, for instance, a device has memory, the test might need + // to be redesigned (e.g., re-initialize fake recording device). + for (int i = 0; i <= 255; ++i) { + SCOPED_TRACE(AnalogLevelToString(i)); + WritesDataIntoChannelBuffer(kTestMultiChannelSamples, buff.get()); + fake_recording_device.SetMicLevel(i); + fake_recording_device.SimulateAnalogGain(buff.get()); + CheckSameSign(buff_orig.get(), buff.get()); + } + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.cc new file mode 100644 index 0000000000..1a82258903 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/performance_timer.h" + +#include + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +PerformanceTimer::PerformanceTimer(int num_frames_to_process) + : clock_(webrtc::Clock::GetRealTimeClock()) { + timestamps_us_.reserve(num_frames_to_process); +} + +PerformanceTimer::~PerformanceTimer() = default; + +void PerformanceTimer::StartTimer() { + start_timestamp_us_ = clock_->TimeInMicroseconds(); +} + +void PerformanceTimer::StopTimer() { + RTC_DCHECK(start_timestamp_us_); + timestamps_us_.push_back(clock_->TimeInMicroseconds() - *start_timestamp_us_); +} + +double PerformanceTimer::GetDurationAverage() const { + return GetDurationAverage(0); +} + +double PerformanceTimer::GetDurationStandardDeviation() const { + return GetDurationStandardDeviation(0); +} + +double PerformanceTimer::GetDurationAverage( + size_t number_of_warmup_samples) const { + RTC_DCHECK_GT(timestamps_us_.size(), number_of_warmup_samples); + const size_t number_of_samples = + timestamps_us_.size() - number_of_warmup_samples; + return static_cast( + std::accumulate(timestamps_us_.begin() + number_of_warmup_samples, + timestamps_us_.end(), static_cast(0))) / + number_of_samples; +} + +double PerformanceTimer::GetDurationStandardDeviation( + size_t number_of_warmup_samples) const { + RTC_DCHECK_GT(timestamps_us_.size(), number_of_warmup_samples); + const size_t number_of_samples = + timestamps_us_.size() - number_of_warmup_samples; + RTC_DCHECK_GT(number_of_samples, 0); + double average_duration = GetDurationAverage(number_of_warmup_samples); + + double variance = std::accumulate( + timestamps_us_.begin() + number_of_warmup_samples, timestamps_us_.end(), + 0.0, [average_duration](const double& a, const int64_t& b) { + return a + (b - average_duration) * (b - average_duration); + }); + + return sqrt(variance / number_of_samples); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.h new file mode 100644 index 0000000000..1c862dc78e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/performance_timer.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_PERFORMANCE_TIMER_H_ +#define MODULES_AUDIO_PROCESSING_TEST_PERFORMANCE_TIMER_H_ + +#include + +#include "api/optional.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { + +class PerformanceTimer { + public: + explicit PerformanceTimer(int num_frames_to_process); + ~PerformanceTimer(); + + void StartTimer(); + void StopTimer(); + + double GetDurationAverage() const; + double GetDurationStandardDeviation() const; + + // These methods are the same as those above, but they ignore the first + // |number_of_warmup_samples| measurements. + double GetDurationAverage(size_t number_of_warmup_samples) const; + double GetDurationStandardDeviation(size_t number_of_warmup_samples) const; + + private: + webrtc::Clock* clock_; + rtc::Optional start_timestamp_us_; + std::vector timestamps_us_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_PERFORMANCE_TIMER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.cc new file mode 100644 index 0000000000..3b623b915c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/protobuf_utils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +size_t ReadMessageBytesFromFile(FILE* file, std::unique_ptr* bytes) { + // The "wire format" for the size is little-endian. Assume we're running on + // a little-endian machine. +#ifndef WEBRTC_ARCH_LITTLE_ENDIAN +#error "Need to convert messsage from little-endian." +#endif + int32_t size = 0; + if (fread(&size, sizeof(size), 1, file) != 1) + return 0; + if (size <= 0) + return 0; + + bytes->reset(new uint8_t[size]); + return fread(bytes->get(), sizeof((*bytes)[0]), size, file); +} + +// Returns true on success, false on error or end-of-file. +bool ReadMessageFromFile(FILE* file, MessageLite* msg) { + std::unique_ptr bytes; + size_t size = ReadMessageBytesFromFile(file, &bytes); + if (!size) + return false; + + msg->Clear(); + return msg->ParseFromArray(bytes.get(), size); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.h new file mode 100644 index 0000000000..dded9b4ca7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/protobuf_utils.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_PROTOBUF_UTILS_H_ +#define MODULES_AUDIO_PROCESSING_TEST_PROTOBUF_UTILS_H_ + +#include + +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/protobuf_utils.h" + +RTC_PUSH_IGNORING_WUNDEF() +#include "modules/audio_processing/debug.pb.h" +RTC_POP_IGNORING_WUNDEF() + +namespace webrtc { + +// Allocates new memory in the unique_ptr to fit the raw message and returns the +// number of bytes read. +size_t ReadMessageBytesFromFile(FILE* file, std::unique_ptr* bytes); + +// Returns true on success, false on error or end-of-file. +bool ReadMessageFromFile(FILE* file, MessageLite* msg); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_PROTOBUF_UTILS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn new file mode 100644 index 0000000000..64e3a30bd3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn @@ -0,0 +1,174 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../../webrtc.gni") + +group("py_quality_assessment") { + testonly = true + deps = [ + ":scripts", + ":unit_tests", + ] +} + +copy("scripts") { + testonly = true + sources = [ + "README.md", + "apm_quality_assessment.py", + "apm_quality_assessment.sh", + "apm_quality_assessment_boxplot.py", + "apm_quality_assessment_export.py", + "apm_quality_assessment_gencfgs.py", + "apm_quality_assessment_optimize.py", + ] + outputs = [ + "$root_build_dir/py_quality_assessment/{{source_file_part}}", + ] + deps = [ + ":apm_configs", + ":lib", + ":output", + "../..:audioproc_f", + "../../../../resources/audio_processing/test/py_quality_assessment:probing_signals", + ] +} + +copy("apm_configs") { + testonly = true + sources = [ + "apm_configs/default.json", + ] + visibility = [ ":*" ] # Only targets in this file can depend on this. + outputs = [ + "$root_build_dir/py_quality_assessment/apm_configs/{{source_file_part}}", + ] +} # apm_configs + +copy("lib") { + testonly = true + sources = [ + "quality_assessment/__init__.py", + "quality_assessment/annotations.py", + "quality_assessment/audioproc_wrapper.py", + "quality_assessment/collect_data.py", + "quality_assessment/data_access.py", + "quality_assessment/echo_path_simulation.py", + "quality_assessment/echo_path_simulation_factory.py", + "quality_assessment/eval_scores.py", + "quality_assessment/eval_scores_factory.py", + "quality_assessment/evaluation.py", + "quality_assessment/exceptions.py", + "quality_assessment/export.py", + "quality_assessment/export_unittest.py", + "quality_assessment/external_vad.py", + "quality_assessment/input_mixer.py", + "quality_assessment/input_signal_creator.py", + "quality_assessment/results.css", + "quality_assessment/results.js", + "quality_assessment/signal_processing.py", + "quality_assessment/simulation.py", + "quality_assessment/test_data_generation.py", + "quality_assessment/test_data_generation_factory.py", + ] + visibility = [ ":*" ] # Only targets in this file can depend on this. + outputs = [ + "$root_build_dir/py_quality_assessment/quality_assessment/{{source_file_part}}", + ] + deps = [ + "../../../../resources/audio_processing/test/py_quality_assessment:noise_tracks", + ] +} + +copy("output") { + testonly = true + sources = [ + "output/README.md", + ] + visibility = [ ":*" ] # Only targets in this file can depend on this. + outputs = [ + "$root_build_dir/py_quality_assessment/output/{{source_file_part}}", + ] +} + +group("unit_tests") { + testonly = true + visibility = [ ":*" ] # Only targets in this file can depend on this. + deps = [ + ":apm_vad", + ":fake_polqa", + ":lib_unit_tests", + ":scripts_unit_tests", + ":vad", + ] +} + +rtc_executable("fake_polqa") { + testonly = true + sources = [ + "quality_assessment/fake_polqa.cc", + ] + visibility = [ ":*" ] # Only targets in this file can depend on this. + output_name = "py_quality_assessment/quality_assessment/fake_polqa" + deps = [ + "../../../..:webrtc_common", + "../../../../rtc_base:rtc_base_approved", + ] +} + +rtc_executable("vad") { + sources = [ + "quality_assessment/vad.cc", + ] + deps = [ + "../../../..:webrtc_common", + "../../../../common_audio", + "../../../../rtc_base:rtc_base_approved", + ] +} + +rtc_executable("apm_vad") { + sources = [ + "quality_assessment/apm_vad.cc", + ] + deps = [ + "../..", + "../../../..:webrtc_common", + "../../../../common_audio", + "../../../../rtc_base:rtc_base_approved", + ] +} + +copy("lib_unit_tests") { + testonly = true + sources = [ + "quality_assessment/annotations_unittest.py", + "quality_assessment/echo_path_simulation_unittest.py", + "quality_assessment/eval_scores_unittest.py", + "quality_assessment/fake_external_vad.py", + "quality_assessment/input_mixer_unittest.py", + "quality_assessment/signal_processing_unittest.py", + "quality_assessment/simulation_unittest.py", + "quality_assessment/test_data_generation_unittest.py", + ] + visibility = [ ":*" ] # Only targets in this file can depend on this. + outputs = [ + "$root_build_dir/py_quality_assessment/quality_assessment/{{source_file_part}}", + ] +} + +copy("scripts_unit_tests") { + testonly = true + sources = [ + "apm_quality_assessment_unittest.py", + ] + visibility = [ ":*" ] # Only targets in this file can depend on this. + outputs = [ + "$root_build_dir/py_quality_assessment/{{source_file_part}}", + ] +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/OWNERS b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/OWNERS new file mode 100644 index 0000000000..4dc254dcee --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/OWNERS @@ -0,0 +1,8 @@ +aleloi@webrtc.org +alessiob@webrtc.org +henrik.lundin@webrtc.org +ivoc@webrtc.org +peah@webrtc.org + +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/README.md b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/README.md new file mode 100644 index 0000000000..6fa0b7004c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/README.md @@ -0,0 +1,125 @@ +# APM Quality Assessment tool + +Python wrapper of APM simulators (e.g., `audioproc_f`) with which quality +assessment can be automatized. The tool allows to simulate different noise +conditions, input signals, APM configurations and it computes different scores. +Once the scores are computed, the results can be easily exported to an HTML page +which allows to listen to the APM input and output signals and also the +reference one used for evaluation. + +## Dependencies + - OS: Linux + - Python 2.7 + - Python libraries: enum34, numpy, scipy, pydub (0.17.0+), pandas (0.20.1+), + pyquery (1.2+), jsmin (2.2+), csscompressor (0.9.4) + - It is recommended that a dedicated Python environment is used + - install `virtualenv` + - `$ sudo apt-get install python-virtualenv` + - setup a new Python environment (e.g., `my_env`) + - `$ cd ~ && virtualenv my_env` + - activate the new Python environment + - `$ source ~/my_env/bin/activate` + - add dependcies via `pip` + - `(my_env)$ pip install enum34 numpy pydub scipy pandas pyquery jsmin \` + `csscompressor` + - PolqaOem64 (see http://www.polqa.info/) + - Tested with POLQA Library v1.180 / P863 v2.400 + - Aachen Impulse Response (AIR) Database + - Download https://www2.iks.rwth-aachen.de/air/air_database_release_1_4.zip + - Input probing signals and noise tracks (you can make your own dataset - *1) + +## Build + - Compile WebRTC + - Go to `out/Default/py_quality_assessment` and check that + `apm_quality_assessment.py` exists + +## Unit tests + - Compile WebRTC + - Go to `out/Default/py_quality_assessment` + - Run `python -m unittest discover -p "*_unittest.py"` + +## First time setup + - Deploy PolqaOem64 and set the `POLQA_PATH` environment variable + - e.g., `$ export POLQA_PATH=/var/opt/PolqaOem64` + - Deploy the AIR Database and set the `AECHEN_IR_DATABASE_PATH` environment + variable + - e.g., `$ export AECHEN_IR_DATABASE_PATH=/var/opt/AIR_1_4` + - Deploy probing signal tracks into + - `out/Default/py_quality_assessment/probing_signals` (*1) + - Deploy noise tracks into + - `out/Default/py_quality_assessment/noise_tracks` (*1, *2) + +(*1) You can use custom files as long as they are mono tracks sampled at 48kHz +encoded in the 16 bit signed format (it is recommended that the tracks are +converted and exported with Audacity). + +## Usage (scores computation) + - Go to `out/Default/py_quality_assessment` + - Check the `apm_quality_assessment.sh` as an example script to parallelize the + experiments + - Adjust the script according to your preferences (e.g., output path) + - Run `apm_quality_assessment.sh` + - The script will end by opening the browser and showing ALL the computed + scores + +## Usage (export reports) +Showing all the results at once can be confusing. You therefore may want to +export separate reports. In this case, you can use the +`apm_quality_assessment_export.py` script as follows: + + - Set `--output_dir, -o` to the same value used in `apm_quality_assessment.sh` + - Use regular expressions to select/filter out scores by + - APM configurations: `--config_names, -c` + - capture signals: `--capture_names, -i` + - render signals: `--render_names, -r` + - echo simulator: `--echo_simulator_names, -e` + - test data generators: `--test_data_generators, -t` + - scores: `--eval_scores, -s` + - Assign a suffix to the report name using `-f ` + +For instance: + +``` +$ ./apm_quality_assessment_export.py \ + -o output/ \ + -c "(^default$)|(.*AE.*)" \ + -t \(white_noise\) \ + -s \(polqa\) \ + -f echo +``` + +## Usage (boxplot) +After generating stats, it can help to visualize how a score depends on a +certain APM simulator parameter. The `apm_quality_assessment_boxplot.py` script +helps with that, producing plots similar to [this +one](https://matplotlib.org/mpl_examples/pylab_examples/boxplot_demo_06.png). + +Suppose some scores come from running the APM simulator `audioproc_f` with +or without the intelligibility enhancer: `--ie=1` or `--ie=0`. Then two boxplots +side by side can be generated with + +``` +$ ./apm_quality_assessment_boxplot.py \ + -o /path/to/output + -v + -n /path/to/dir/with/apm_configs + -z ie +``` + +## Troubleshooting +The input wav file must be: + - sampled at a sample rate that is a multiple of 100 (required by POLQA) + - in the 16 bit format (required by `audioproc_f`) + - encoded in the Microsoft WAV signed 16 bit PCM format (Audacity default + when exporting) + +Depending on the license, the POLQA tool may take “breaks” as a way to limit the +throughput. When this happens, the APM Quality Assessment tool is slowed down. +For more details about this limitation, check Section 10.9.1 in the POLQA manual +v.1.18. + +In case of issues with the POLQA score computation, check +`py_quality_assessment/eval_scores.py` and adapt +`PolqaScore._parse_output_file()`. +The code can be also fixed directly into the build directory (namely, +`out/Default/py_quality_assessment/eval_scores.py`). diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_configs/default.json b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_configs/default.json new file mode 100644 index 0000000000..5c3277bac0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_configs/default.json @@ -0,0 +1 @@ +{"-all_default": null} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py new file mode 100755 index 0000000000..a4cc5f037f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Perform APM module quality assessment on one or more input files using one or + more APM simulator configuration files and one or more test data generators. + +Usage: apm_quality_assessment.py -i audio1.wav [audio2.wav ...] + -c cfg1.json [cfg2.json ...] + -n white [echo ...] + -e audio_level [polqa ...] + -o /path/to/output +""" + +import argparse +import logging +import os +import sys + +import quality_assessment.audioproc_wrapper as audioproc_wrapper +import quality_assessment.echo_path_simulation as echo_path_simulation +import quality_assessment.eval_scores as eval_scores +import quality_assessment.evaluation as evaluation +import quality_assessment.eval_scores_factory as eval_scores_factory +import quality_assessment.external_vad as external_vad +import quality_assessment.test_data_generation as test_data_generation +import quality_assessment.test_data_generation_factory as \ + test_data_generation_factory +import quality_assessment.simulation as simulation + +_ECHO_PATH_SIMULATOR_NAMES = ( + echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES) +_TEST_DATA_GENERATOR_CLASSES = ( + test_data_generation.TestDataGenerator.REGISTERED_CLASSES) +_TEST_DATA_GENERATORS_NAMES = _TEST_DATA_GENERATOR_CLASSES.keys() +_EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES +_EVAL_SCORE_WORKER_NAMES = _EVAL_SCORE_WORKER_CLASSES.keys() + +_DEFAULT_CONFIG_FILE = 'apm_configs/default.json' + +_POLQA_BIN_NAME = 'PolqaOem64' + + +def _InstanceArgumentsParser(): + """Arguments parser factory. + """ + parser = argparse.ArgumentParser(description=( + 'Perform APM module quality assessment on one or more input files using ' + 'one or more APM simulator configuration files and one or more ' + 'test data generators.')) + + parser.add_argument('-c', '--config_files', nargs='+', required=False, + help=('path to the configuration files defining the ' + 'arguments with which the APM simulator tool is ' + 'called'), + default=[_DEFAULT_CONFIG_FILE]) + + parser.add_argument('-i', '--capture_input_files', nargs='+', required=True, + help='path to the capture input wav files (one or more)') + + parser.add_argument('-r', '--render_input_files', nargs='+', required=False, + help=('path to the render input wav files; either ' + 'omitted or one file for each file in ' + '--capture_input_files (files will be paired by ' + 'index)'), default=None) + + parser.add_argument('-p', '--echo_path_simulator', required=False, + help=('custom echo path simulator name; required if ' + '--render_input_files is specified'), + choices=_ECHO_PATH_SIMULATOR_NAMES, + default=echo_path_simulation.NoEchoPathSimulator.NAME) + + parser.add_argument('-t', '--test_data_generators', nargs='+', required=False, + help='custom list of test data generators to use', + choices=_TEST_DATA_GENERATORS_NAMES, + default=_TEST_DATA_GENERATORS_NAMES) + + parser.add_argument('--additive_noise_tracks_path', required=False, + help='path to the wav files for the additive', + default=test_data_generation. \ + AdditiveNoiseTestDataGenerator. \ + DEFAULT_NOISE_TRACKS_PATH) + + parser.add_argument('-e', '--eval_scores', nargs='+', required=False, + help='custom list of evaluation scores to use', + choices=_EVAL_SCORE_WORKER_NAMES, + default=_EVAL_SCORE_WORKER_NAMES) + + parser.add_argument('-o', '--output_dir', required=False, + help=('base path to the output directory in which the ' + 'output wav files and the evaluation outcomes ' + 'are saved'), + default='output') + + parser.add_argument('--polqa_path', required=True, + help='path to the POLQA tool') + + parser.add_argument('--air_db_path', required=True, + help='path to the Aechen IR database') + + parser.add_argument('--apm_sim_path', required=False, + help='path to the APM simulator tool', + default=audioproc_wrapper. \ + AudioProcWrapper. \ + DEFAULT_APM_SIMULATOR_BIN_PATH) + + parser.add_argument('--copy_with_identity_generator', required=False, + help=('If true, the identity test data generator makes a ' + 'copy of the clean speech input file.'), + default=False) + + parser.add_argument('--external_vad_paths', nargs='+', required=False, + help=('Paths to external VAD programs. Each must take' + '\'-i -o \' inputs'), default=[]) + + parser.add_argument('--external_vad_names', nargs='+', required=False, + help=('Keys to the vad paths. Must be different and ' + 'as many as the paths.'), default=[]) + + return parser + + +def _ValidateArguments(args, parser): + if args.capture_input_files and args.render_input_files and ( + len(args.capture_input_files) != len(args.render_input_files)): + parser.error('--render_input_files and --capture_input_files must be lists ' + 'having the same length') + sys.exit(1) + + if args.render_input_files and not args.echo_path_simulator: + parser.error('when --render_input_files is set, --echo_path_simulator is ' + 'also required') + sys.exit(1) + + if len(args.external_vad_names) != len(args.external_vad_paths): + parser.error('If provided, --external_vad_paths and ' + '--external_vad_names must ' + 'have the same number of arguments.') + sys.exit(1) + + +def main(): + # TODO(alessiob): level = logging.INFO once debugged. + logging.basicConfig(level=logging.DEBUG) + parser = _InstanceArgumentsParser() + args = parser.parse_args() + _ValidateArguments(args, parser) + + simulator = simulation.ApmModuleSimulator( + test_data_generator_factory=( + test_data_generation_factory.TestDataGeneratorFactory( + aechen_ir_database_path=args.air_db_path, + noise_tracks_path=args.additive_noise_tracks_path, + copy_with_identity=args.copy_with_identity_generator)), + evaluation_score_factory=eval_scores_factory.EvaluationScoreWorkerFactory( + polqa_tool_bin_path=os.path.join(args.polqa_path, _POLQA_BIN_NAME)), + ap_wrapper=audioproc_wrapper.AudioProcWrapper(args.apm_sim_path), + evaluator=evaluation.ApmModuleEvaluator(), + external_vads=external_vad.ExternalVad.ConstructVadDict( + args.external_vad_paths, args.external_vad_names)) + simulator.Run( + config_filepaths=args.config_files, + capture_input_filepaths=args.capture_input_files, + render_input_filepaths=args.render_input_files, + echo_path_simulator_name=args.echo_path_simulator, + test_data_generator_names=args.test_data_generators, + eval_score_names=args.eval_scores, + output_dir=args.output_dir) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh new file mode 100755 index 0000000000..aa563ee26b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# Path to the POLQA tool. +if [ -z ${POLQA_PATH} ]; then # Check if defined. + # Default location. + export POLQA_PATH='/var/opt/PolqaOem64' +fi +if [ -d "${POLQA_PATH}" ]; then + echo "POLQA found in ${POLQA_PATH}" +else + echo "POLQA not found in ${POLQA_PATH}" + exit 1 +fi + +# Path to the Aechen IR database. +if [ -z ${AECHEN_IR_DATABASE_PATH} ]; then # Check if defined. + # Default location. + export AECHEN_IR_DATABASE_PATH='/var/opt/AIR_1_4' +fi +if [ -d "${AECHEN_IR_DATABASE_PATH}" ]; then + echo "AIR database found in ${AECHEN_IR_DATABASE_PATH}" +else + echo "AIR database not found in ${AECHEN_IR_DATABASE_PATH}" + exit 1 +fi + +# Customize probing signals, test data generators and scores if needed. +CAPTURE_SIGNALS=(probing_signals/*.wav) +TEST_DATA_GENERATORS=( \ + "identity" \ + "white_noise" \ + # "environmental_noise" \ + # "reverberation" \ +) +SCORES=( \ + # "polqa" \ + "audio_level_peak" \ + "audio_level_mean" \ +) +OUTPUT_PATH=output + +# Generate standard APM config files. +chmod +x apm_quality_assessment_gencfgs.py +./apm_quality_assessment_gencfgs.py + +# Customize APM configurations if needed. +APM_CONFIGS=(apm_configs/*.json) + +# Add output path if missing. +if [ ! -d ${OUTPUT_PATH} ]; then + mkdir ${OUTPUT_PATH} +fi + +# Start one process for each "probing signal"-"test data source" pair. +chmod +x apm_quality_assessment.py +for capture_signal_filepath in "${CAPTURE_SIGNALS[@]}" ; do + probing_signal_name="$(basename $capture_signal_filepath)" + probing_signal_name="${probing_signal_name%.*}" + for test_data_gen_name in "${TEST_DATA_GENERATORS[@]}" ; do + LOG_FILE="${OUTPUT_PATH}/apm_qa-${probing_signal_name}-"` + `"${test_data_gen_name}.log" + echo "Starting ${probing_signal_name} ${test_data_gen_name} "` + `"(see ${LOG_FILE})" + ./apm_quality_assessment.py \ + --polqa_path ${POLQA_PATH}\ + --air_db_path ${AECHEN_IR_DATABASE_PATH}\ + -i ${capture_signal_filepath} \ + -o ${OUTPUT_PATH} \ + -t ${test_data_gen_name} \ + -c "${APM_CONFIGS[@]}" \ + -e "${SCORES[@]}" > $LOG_FILE 2>&1 & + done +done + +# Join Python processes running apm_quality_assessment.py. +wait + +# Export results. +chmod +x ./apm_quality_assessment_export.py +./apm_quality_assessment_export.py -o ${OUTPUT_PATH} + +# Show results in the browser. +RESULTS_FILE="$(realpath ${OUTPUT_PATH}/results.html)" +sensible-browser "file://${RESULTS_FILE}" > /dev/null 2>&1 & diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_boxplot.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_boxplot.py new file mode 100644 index 0000000000..ec87970d75 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_boxplot.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Shows boxplots of given score for different values of selected +parameters. Can be used to compare scores by audioproc_f flag. + +Usage: apm_quality_assessment_boxplot.py -o /path/to/output + -v polqa + -n /path/to/dir/with/apm_configs + -z audioproc_f_arg1 [arg2 ...] + +Arguments --config_names, --render_names, --echo_simulator_names, +--test_data_generators, --eval_scores can be used to filter the data +used for plotting. +""" + +import collections +import logging +import matplotlib.pyplot as plt +import os + +import quality_assessment.data_access as data_access +import quality_assessment.collect_data as collect_data + + +def InstanceArgumentsParser(): + """Arguments parser factory. + """ + parser = collect_data.InstanceArgumentsParser() + parser.description = ( + 'Shows boxplot of given score for different values of selected' + 'parameters. Can be used to compare scores by audioproc_f flag') + + parser.add_argument('-v', '--eval_score', required=True, + help=('Score name for constructing boxplots')) + + parser.add_argument('-n', '--config_dir', required=False, + help=('path to the folder with the configuration files'), + default='apm_configs') + + parser.add_argument('-z', '--params_to_plot', required=True, + nargs='+', help=('audioproc_f parameter values' + 'by which to group scores (no leading dash)')) + + return parser + + +def FilterScoresByParams(data_frame, filter_params, score_name, config_dir): + """Filters data on the values of one or more parameters. + + Args: + data_frame: pandas.DataFrame of all used input data. + + filter_params: each config of the input data is assumed to have + exactly one parameter from `filter_params` defined. Every value + of the parameters in `filter_params` is a key in the returned + dict; the associated value is all cells of the data with that + value of the parameter. + + score_name: Name of score which value is boxplotted. Currently cannot do + more than one value. + + config_dir: path to dir with APM configs. + + Returns: dictionary, key is a param value, result is all scores for + that param value (see `filter_params` for explanation). + """ + results = collections.defaultdict(dict) + config_names = data_frame['apm_config'].drop_duplicates().values.tolist() + + for config_name in config_names: + config_json = data_access.AudioProcConfigFile.Load( + os.path.join(config_dir, config_name + '.json')) + data_with_config = data_frame[data_frame.apm_config == config_name] + data_cell_scores = data_with_config[data_with_config.eval_score_name == + score_name] + + # Exactly one of |params_to_plot| must match: + (matching_param, ) = [x for x in filter_params if '-' + x in config_json] + + # Add scores for every track to the result. + for capture_name in data_cell_scores.capture: + result_score = float(data_cell_scores[data_cell_scores.capture == + capture_name].score) + config_dict = results[config_json['-' + matching_param]] + if capture_name not in config_dict: + config_dict[capture_name] = {} + + config_dict[capture_name][matching_param] = result_score + + return results + + +def _FlattenToScoresList(config_param_score_dict): + """Extracts a list of scores from input data structure. + + Args: + config_param_score_dict: of the form {'capture_name': + {'param_name' : score_value,.. } ..} + + Returns: Plain list of all score value present in input data + structure + """ + result = [] + for capture_name in config_param_score_dict: + result += list(config_param_score_dict[capture_name].values()) + return result + + +def main(): + # Init. + # TODO(alessiob): INFO once debugged. + logging.basicConfig(level=logging.DEBUG) + parser = InstanceArgumentsParser() + args = parser.parse_args() + + # Get the scores. + src_path = collect_data.ConstructSrcPath(args) + logging.debug(src_path) + scores_data_frame = collect_data.FindScores(src_path, args) + + # Filter the data by `args.params_to_plot` + scores_filtered = FilterScoresByParams(scores_data_frame, + args.params_to_plot, + args.eval_score, + args.config_dir) + + data_list = sorted(scores_filtered.items()) + data_values = [_FlattenToScoresList(x) for (_, x) in data_list] + data_labels = [x for (x, _) in data_list] + + _, axes = plt.subplots(nrows=1, ncols=1, figsize=(6, 6)) + axes.boxplot(data_values, labels=data_labels) + axes.set_ylabel(args.eval_score) + axes.set_xlabel('/'.join(args.params_to_plot)) + plt.show() + + +if __name__ == "__main__": + main() diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py new file mode 100755 index 0000000000..5c8a52bf16 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Export the scores computed by the apm_quality_assessment.py script into an + HTML file. +""" + +import logging +import os +import sys + +import quality_assessment.collect_data as collect_data +import quality_assessment.export as export + + +def _BuildOutputFilename(filename_suffix): + """Builds the filename for the exported file. + + Args: + filename_suffix: suffix for the output file name. + + Returns: + A string. + """ + if filename_suffix is None: + return 'results.html' + return 'results-{}.html'.format(filename_suffix) + +def main(): + # Init. + logging.basicConfig(level=logging.DEBUG) # TODO(alessio): INFO once debugged. + parser = collect_data.InstanceArgumentsParser() + parser.add_argument('-f', '--filename_suffix', + help=('suffix of the exported file')) + parser.description = ('Exports pre-computed APM module quality assessment ' + 'results into HTML tables') + args = parser.parse_args() + + # Get the scores. + src_path = collect_data.ConstructSrcPath(args) + logging.debug(src_path) + scores_data_frame = collect_data.FindScores(src_path, args) + + # Export. + output_filepath = os.path.join(args.output_dir, _BuildOutputFilename( + args.filename_suffix)) + exporter = export.HtmlExport(output_filepath) + exporter.Export(scores_data_frame) + + logging.info('output file successfully written in %s', output_filepath) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py new file mode 100755 index 0000000000..4017747cc2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Generate .json files with which the APM module can be tested using the + apm_quality_assessment.py script and audioproc_f as APM simulator. +""" + +import logging +import os + +import quality_assessment.data_access as data_access + +OUTPUT_PATH = os.path.abspath('apm_configs') + + +def _GenerateDefaultOverridden(config_override): + """Generates one or more APM overriden configurations. + + For each item in config_override, it overrides the default configuration and + writes a new APM configuration file. + + The default settings are loaded via "-all_default". + Check "src/modules/audio_processing/test/audioproc_float.cc" and search + for "if (FLAG_all_default) {". + + For instance, in 55eb6d621489730084927868fed195d3645a9ec9 the default is this: + settings.use_aec = rtc::Optional(true); + settings.use_aecm = rtc::Optional(false); + settings.use_agc = rtc::Optional(true); + settings.use_bf = rtc::Optional(false); + settings.use_ed = rtc::Optional(false); + settings.use_hpf = rtc::Optional(true); + settings.use_ie = rtc::Optional(false); + settings.use_le = rtc::Optional(true); + settings.use_ns = rtc::Optional(true); + settings.use_ts = rtc::Optional(true); + settings.use_vad = rtc::Optional(true); + + Args: + config_override: dict of APM configuration file names as keys; the values + are dict instances encoding the audioproc_f flags. + """ + for config_filename in config_override: + config = config_override[config_filename] + config['-all_default'] = None + + config_filepath = os.path.join(OUTPUT_PATH, 'default-{}.json'.format( + config_filename)) + logging.debug('config file <%s> | %s', config_filepath, config) + + data_access.AudioProcConfigFile.Save(config_filepath, config) + logging.info('config file created: <%s>', config_filepath) + + +def _GenerateAllDefaultButOne(): + """Disables the flags enabled by default one-by-one. + """ + config_sets = { + 'no_AEC': {'-aec': 0,}, + 'no_AGC': {'-agc': 0,}, + 'no_HP_filter': {'-hpf': 0,}, + 'no_level_estimator': {'-le': 0,}, + 'no_noise_suppressor': {'-ns': 0,}, + 'no_transient_suppressor': {'-ts': 0,}, + 'no_vad': {'-vad': 0,}, + } + _GenerateDefaultOverridden(config_sets) + + +def _GenerateAllDefaultPlusOne(): + """Enables the flags disabled by default one-by-one. + """ + config_sets = { + 'with_AECM': {'-aec': 0, '-aecm': 1,}, # AEC and AECM are exclusive. + 'with_AGC_limiter': {'-agc_limiter': 1,}, + 'with_AEC_delay_agnostic': {'-delay_agnostic': 1,}, + 'with_drift_compensation': {'-drift_compensation': 1,}, + 'with_residual_echo_detector': {'-ed': 1,}, + 'with_AEC_extended_filter': {'-extended_filter': 1,}, + 'with_intelligibility_enhancer': {'-ie': 1,}, + 'with_LC': {'-lc': 1,}, + 'with_refined_adaptive_filter': {'-refined_adaptive_filter': 1,}, + } + _GenerateDefaultOverridden(config_sets) + + +def main(): + logging.basicConfig(level=logging.INFO) + _GenerateAllDefaultPlusOne() + _GenerateAllDefaultButOne() + + +if __name__ == '__main__': + main() diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_optimize.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_optimize.py new file mode 100644 index 0000000000..7946fe2aeb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_optimize.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Finds the APM configuration that maximizes a provided metric by +parsing the output generated apm_quality_assessment.py. +""" + +from __future__ import division + +import collections +import logging +import os + +import quality_assessment.data_access as data_access +import quality_assessment.collect_data as collect_data + +def _InstanceArgumentsParser(): + """Arguments parser factory. Extends the arguments from 'collect_data' + with a few extra for selecting what parameters to optimize for. + """ + parser = collect_data.InstanceArgumentsParser() + parser.description = ( + 'Rudimentary optimization of a function over different parameter' + 'combinations.') + + parser.add_argument('-n', '--config_dir', required=False, + help=('path to the folder with the configuration files'), + default='apm_configs') + + parser.add_argument('-p', '--params', required=True, nargs='+', + help=('parameters to parse from the config files in' + 'config_dir')) + + parser.add_argument('-z', '--params_not_to_optimize', required=False, + nargs='+', default=[], + help=('parameters from `params` not to be optimized for')) + + return parser + + +def _ConfigurationAndScores(data_frame, params, + params_not_to_optimize, config_dir): + """Returns a list of all configurations and scores. + + Args: + data_frame: A pandas data frame with the scores and config name + returned by _FindScores. + params: The parameter names to parse from configs the config + directory + + params_not_to_optimize: The parameter names which shouldn't affect + the optimal parameter + selection. E.g., fixed settings and not + tunable parameters. + + config_dir: Path to folder with config files. + + Returns: + Dictionary of the form + {param_combination: [{params: {param1: value1, ...}, + scores: {score1: value1, ...}}]}. + + The key `param_combination` runs over all parameter combinations + of the parameters in `params` and not in + `params_not_to_optimize`. A corresponding value is a list of all + param combinations for params in `params_not_to_optimize` and + their scores. + """ + results = collections.defaultdict(list) + config_names = data_frame['apm_config'].drop_duplicates().values.tolist() + score_names = data_frame['eval_score_name'].drop_duplicates().values.tolist() + + # Normalize the scores + normalization_constants = {} + for score_name in score_names: + scores = data_frame[data_frame.eval_score_name == score_name].score + normalization_constants[score_name] = max(scores) + + params_to_optimize = [p for p in params if p not in params_not_to_optimize] + param_combination = collections.namedtuple("ParamCombination", + params_to_optimize) + + for config_name in config_names: + config_json = data_access.AudioProcConfigFile.Load( + os.path.join(config_dir, config_name + ".json")) + scores = {} + data_cell = data_frame[data_frame.apm_config == config_name] + for score_name in score_names: + data_cell_scores = data_cell[data_cell.eval_score_name == + score_name].score + scores[score_name] = sum(data_cell_scores) / len(data_cell_scores) + scores[score_name] /= normalization_constants[score_name] + + result = {'scores': scores, 'params': {}} + config_optimize_params = {} + for param in params: + if param in params_to_optimize: + config_optimize_params[param] = config_json['-' + param] + else: + result['params'][param] = config_json['-' + param] + + current_param_combination = param_combination( # pylint: disable=star-args + **config_optimize_params) + results[current_param_combination].append(result) + return results + + +def _FindOptimalParameter(configs_and_scores, score_weighting): + """Finds the config producing the maximal score. + + Args: + configs_and_scores: structure of the form returned by + _ConfigurationAndScores + + score_weighting: a function to weight together all score values of + the form [{params: {param1: value1, ...}, scores: + {score1: value1, ...}}] into a numeric + value + Returns: + the config that has the largest values of |score_weighting| applied + to its scores. + """ + + min_score = float('+inf') + best_params = None + for config in configs_and_scores: + scores_and_params = configs_and_scores[config] + current_score = score_weighting(scores_and_params) + if current_score < min_score: + min_score = current_score + best_params = config + logging.debug("Score: %f", current_score) + logging.debug("Config: %s", str(config)) + return best_params + + +def _ExampleWeighting(scores_and_configs): + """Example argument to `_FindOptimalParameter` + Args: + scores_and_configs: a list of configs and scores, in the form + described in _FindOptimalParameter + Returns: + numeric value, the sum of all scores + """ + res = 0 + for score_config in scores_and_configs: + res += sum(score_config['scores'].values()) + return res + + +def main(): + # Init. + # TODO(alessiob): INFO once debugged. + logging.basicConfig(level=logging.DEBUG) + parser = _InstanceArgumentsParser() + args = parser.parse_args() + + # Get the scores. + src_path = collect_data.ConstructSrcPath(args) + logging.debug('Src path <%s>', src_path) + scores_data_frame = collect_data.FindScores(src_path, args) + all_scores = _ConfigurationAndScores(scores_data_frame, + args.params, + args.params_not_to_optimize, + args.config_dir) + + opt_param = _FindOptimalParameter(all_scores, _ExampleWeighting) + + logging.info('Optimal parameter combination: <%s>', opt_param) + logging.info('It\'s score values: <%s>', all_scores[opt_param]) + +if __name__ == "__main__": + main() diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_unittest.py new file mode 100644 index 0000000000..f5240f8696 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_unittest.py @@ -0,0 +1,33 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the apm_quality_assessment module. +""" + +import os +import sys +import unittest + +SRC = os.path.abspath(os.path.join( + os.path.dirname((__file__)), os.pardir, os.pardir, os.pardir)) +sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) + +import mock + +import apm_quality_assessment + +class TestSimulationScript(unittest.TestCase): + """Unit tests for the apm_quality_assessment module. + """ + + def testMain(self): + # Exit with error code if no arguments are passed. + with self.assertRaises(SystemExit) as cm, mock.patch.object( + sys, 'argv', ['apm_quality_assessment.py']): + apm_quality_assessment.main() + self.assertGreater(cm.exception.code, 0) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/output/README.md b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/output/README.md new file mode 100644 index 0000000000..66e2a1c848 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/output/README.md @@ -0,0 +1 @@ +You can use this folder for the output generated by the apm_quality_assessment scripts. diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/__init__.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/__init__.py new file mode 100644 index 0000000000..b870dfaef3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations.py new file mode 100644 index 0000000000..5c8c8bacbc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations.py @@ -0,0 +1,293 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Extraction of annotations from audio files. +""" + +from __future__ import division +import logging +import os +import shutil +import struct +import subprocess +import sys +import tempfile + +try: + import numpy as np +except ImportError: + logging.critical('Cannot import the third-party Python package numpy') + sys.exit(1) + +from . import external_vad +from . import exceptions +from . import signal_processing + + +class AudioAnnotationsExtractor(object): + """Extracts annotations from audio files. + """ + + # TODO(aleloi): change to enum.IntEnum when py 3.6 is available. + class VadType(object): + ENERGY_THRESHOLD = 1 # TODO(alessiob): Consider switching to P56 standard. + WEBRTC_COMMON_AUDIO = 2 # common_audio/vad/include/vad.h + WEBRTC_APM = 4 # modules/audio_processing/vad/vad.h + + def __init__(self, value): + if (not isinstance(value, int)) or not 0 <= value <= 7: + raise exceptions.InitializationException( + 'Invalid vad type: ' + value) + self._value = value + + def Contains(self, vad_type): + return self._value | vad_type == self._value + + def __str__(self): + vads = [] + if self.Contains(self.ENERGY_THRESHOLD): + vads.append("energy") + if self.Contains(self.WEBRTC_COMMON_AUDIO): + vads.append("common_audio") + if self.Contains(self.WEBRTC_APM): + vads.append("apm") + return "VadType({})".format(", ".join(vads)) + + _OUTPUT_FILENAME_TEMPLATE = '{}annotations.npz' + + # Level estimation params. + _ONE_DB_REDUCTION = np.power(10.0, -1.0 / 20.0) + _LEVEL_FRAME_SIZE_MS = 1.0 + # The time constants in ms indicate the time it takes for the level estimate + # to go down/up by 1 db if the signal is zero. + _LEVEL_ATTACK_MS = 5.0 + _LEVEL_DECAY_MS = 20.0 + + # VAD params. + _VAD_THRESHOLD = 1 + _VAD_WEBRTC_PATH = os.path.join(os.path.dirname( + os.path.abspath(__file__)), os.pardir, os.pardir) + _VAD_WEBRTC_COMMON_AUDIO_PATH = os.path.join(_VAD_WEBRTC_PATH, 'vad') + + _VAD_WEBRTC_APM_PATH = os.path.join( + _VAD_WEBRTC_PATH, 'apm_vad') + + def __init__(self, vad_type, external_vads=None): + self._signal = None + self._level = None + self._level_frame_size = None + self._common_audio_vad = None + self._energy_vad = None + self._apm_vad_probs = None + self._apm_vad_rms = None + self._vad_frame_size = None + self._vad_frame_size_ms = None + self._c_attack = None + self._c_decay = None + + self._vad_type = self.VadType(vad_type) + logging.info('VADs used for annotations: ' + str(self._vad_type)) + + if external_vads is None: + external_vads = {} + self._external_vads = external_vads + + assert len(self._external_vads) == len(external_vads), ( + 'The external VAD names must be unique.') + for vad in external_vads.values(): + if not isinstance(vad, external_vad.ExternalVad): + raise exceptions.InitializationException( + 'Invalid vad type: ' + str(type(vad))) + logging.info('External VAD used for annotation: ' + + str(vad.name)) + + assert os.path.exists(self._VAD_WEBRTC_COMMON_AUDIO_PATH), \ + self._VAD_WEBRTC_COMMON_AUDIO_PATH + assert os.path.exists(self._VAD_WEBRTC_APM_PATH), \ + self._VAD_WEBRTC_APM_PATH + + @classmethod + def GetOutputFileNameTemplate(cls): + return cls._OUTPUT_FILENAME_TEMPLATE + + def GetLevel(self): + return self._level + + def GetLevelFrameSize(self): + return self._level_frame_size + + @classmethod + def GetLevelFrameSizeMs(cls): + return cls._LEVEL_FRAME_SIZE_MS + + def GetVadOutput(self, vad_type): + if vad_type == self.VadType.ENERGY_THRESHOLD: + return self._energy_vad + elif vad_type == self.VadType.WEBRTC_COMMON_AUDIO: + return self._common_audio_vad + elif vad_type == self.VadType.WEBRTC_APM: + return (self._apm_vad_probs, self._apm_vad_rms) + else: + raise exceptions.InitializationException( + 'Invalid vad type: ' + vad_type) + + def GetVadFrameSize(self): + return self._vad_frame_size + + def GetVadFrameSizeMs(self): + return self._vad_frame_size_ms + + def Extract(self, filepath): + # Load signal. + self._signal = signal_processing.SignalProcessingUtils.LoadWav(filepath) + if self._signal.channels != 1: + raise NotImplementedError('Multiple-channel annotations not implemented') + + # Level estimation params. + self._level_frame_size = int(self._signal.frame_rate / 1000 * ( + self._LEVEL_FRAME_SIZE_MS)) + self._c_attack = 0.0 if self._LEVEL_ATTACK_MS == 0 else ( + self._ONE_DB_REDUCTION ** ( + self._LEVEL_FRAME_SIZE_MS / self._LEVEL_ATTACK_MS)) + self._c_decay = 0.0 if self._LEVEL_DECAY_MS == 0 else ( + self._ONE_DB_REDUCTION ** ( + self._LEVEL_FRAME_SIZE_MS / self._LEVEL_DECAY_MS)) + + # Compute level. + self._LevelEstimation() + + # Ideal VAD output, it requires clean speech with high SNR as input. + if self._vad_type.Contains(self.VadType.ENERGY_THRESHOLD): + # Naive VAD based on level thresholding. + vad_threshold = np.percentile(self._level, self._VAD_THRESHOLD) + self._energy_vad = np.uint8(self._level > vad_threshold) + self._vad_frame_size = self._level_frame_size + self._vad_frame_size_ms = self._LEVEL_FRAME_SIZE_MS + if self._vad_type.Contains(self.VadType.WEBRTC_COMMON_AUDIO): + # WebRTC common_audio/ VAD. + self._RunWebRtcCommonAudioVad(filepath, self._signal.frame_rate) + if self._vad_type.Contains(self.VadType.WEBRTC_APM): + # WebRTC modules/audio_processing/ VAD. + self._RunWebRtcApmVad(filepath) + for extvad_name in self._external_vads: + self._external_vads[extvad_name].Run(filepath) + + def Save(self, output_path, annotation_name=""): + ext_kwargs = {'extvad_conf-' + ext_vad: + self._external_vads[ext_vad].GetVadOutput() + for ext_vad in self._external_vads} + # pylint: disable=star-args + np.savez_compressed( + file=os.path.join( + output_path, + self.GetOutputFileNameTemplate().format(annotation_name)), + level=self._level, + level_frame_size=self._level_frame_size, + level_frame_size_ms=self._LEVEL_FRAME_SIZE_MS, + vad_output=self._common_audio_vad, + vad_energy_output=self._energy_vad, + vad_frame_size=self._vad_frame_size, + vad_frame_size_ms=self._vad_frame_size_ms, + vad_probs=self._apm_vad_probs, + vad_rms=self._apm_vad_rms, + **ext_kwargs + ) + + def _LevelEstimation(self): + # Read samples. + samples = signal_processing.SignalProcessingUtils.AudioSegmentToRawData( + self._signal).astype(np.float32) / 32768.0 + num_frames = len(samples) // self._level_frame_size + num_samples = num_frames * self._level_frame_size + + # Envelope. + self._level = np.max(np.reshape(np.abs(samples[:num_samples]), ( + num_frames, self._level_frame_size)), axis=1) + assert len(self._level) == num_frames + + # Envelope smoothing. + smooth = lambda curr, prev, k: (1 - k) * curr + k * prev + self._level[0] = smooth(self._level[0], 0.0, self._c_attack) + for i in range(1, num_frames): + self._level[i] = smooth( + self._level[i], self._level[i - 1], self._c_attack if ( + self._level[i] > self._level[i - 1]) else self._c_decay) + + def _RunWebRtcCommonAudioVad(self, wav_file_path, sample_rate): + self._common_audio_vad = None + self._vad_frame_size = None + + # Create temporary output path. + tmp_path = tempfile.mkdtemp() + output_file_path = os.path.join( + tmp_path, os.path.split(wav_file_path)[1] + '_vad.tmp') + + # Call WebRTC VAD. + try: + subprocess.call([ + self._VAD_WEBRTC_COMMON_AUDIO_PATH, + '-i', wav_file_path, + '-o', output_file_path + ], cwd=self._VAD_WEBRTC_PATH) + + # Read bytes. + with open(output_file_path, 'rb') as f: + raw_data = f.read() + + # Parse side information. + self._vad_frame_size_ms = struct.unpack('B', raw_data[0])[0] + self._vad_frame_size = self._vad_frame_size_ms * sample_rate / 1000 + assert self._vad_frame_size_ms in [10, 20, 30] + extra_bits = struct.unpack('B', raw_data[-1])[0] + assert 0 <= extra_bits <= 8 + + # Init VAD vector. + num_bytes = len(raw_data) + num_frames = 8 * (num_bytes - 2) - extra_bits # 8 frames for each byte. + self._common_audio_vad = np.zeros(num_frames, np.uint8) + + # Read VAD decisions. + for i, byte in enumerate(raw_data[1:-1]): + byte = struct.unpack('B', byte)[0] + for j in range(8 if i < num_bytes - 3 else (8 - extra_bits)): + self._common_audio_vad[i * 8 + j] = int(byte & 1) + byte = byte >> 1 + except Exception as e: + logging.error('Error while running the WebRTC VAD (' + e.message + ')') + finally: + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + + def _RunWebRtcApmVad(self, wav_file_path): + # Create temporary output path. + tmp_path = tempfile.mkdtemp() + output_file_path_probs = os.path.join( + tmp_path, os.path.split(wav_file_path)[1] + '_vad_probs.tmp') + output_file_path_rms = os.path.join( + tmp_path, os.path.split(wav_file_path)[1] + '_vad_rms.tmp') + + # Call WebRTC VAD. + try: + subprocess.call([ + self._VAD_WEBRTC_APM_PATH, + '-i', wav_file_path, + '-o_probs', output_file_path_probs, + '-o_rms', output_file_path_rms + ], cwd=self._VAD_WEBRTC_PATH) + + # Parse annotations. + self._apm_vad_probs = np.fromfile(output_file_path_probs, np.double) + self._apm_vad_rms = np.fromfile(output_file_path_rms, np.double) + assert len(self._apm_vad_rms) == len(self._apm_vad_probs) + + except Exception as e: + logging.error('Error while running the WebRTC APM VAD (' + + e.message + ')') + finally: + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations_unittest.py new file mode 100644 index 0000000000..5a908e7a44 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/annotations_unittest.py @@ -0,0 +1,157 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the annotations module. +""" + +from __future__ import division +import logging +import os +import shutil +import tempfile +import unittest + +import numpy as np + +from . import annotations +from . import external_vad +from . import input_signal_creator +from . import signal_processing + + +class TestAnnotationsExtraction(unittest.TestCase): + """Unit tests for the annotations module. + """ + + _CLEAN_TMP_OUTPUT = True + _DEBUG_PLOT_VAD = False + _VAD_TYPE_CLASS = annotations.AudioAnnotationsExtractor.VadType + _ALL_VAD_TYPES = (_VAD_TYPE_CLASS.ENERGY_THRESHOLD | + _VAD_TYPE_CLASS.WEBRTC_COMMON_AUDIO | + _VAD_TYPE_CLASS.WEBRTC_APM) + + + def setUp(self): + """Create temporary folder.""" + self._tmp_path = tempfile.mkdtemp() + self._wav_file_path = os.path.join(self._tmp_path, 'tone.wav') + pure_tone, _ = input_signal_creator.InputSignalCreator.Create( + 'pure_tone', [440, 1000]) + signal_processing.SignalProcessingUtils.SaveWav( + self._wav_file_path, pure_tone) + self._sample_rate = pure_tone.frame_rate + + def tearDown(self): + """Recursively delete temporary folder.""" + if self._CLEAN_TMP_OUTPUT: + shutil.rmtree(self._tmp_path) + else: + logging.warning(self.id() + ' did not clean the temporary path ' + ( + self._tmp_path)) + + def testFrameSizes(self): + e = annotations.AudioAnnotationsExtractor(self._ALL_VAD_TYPES) + e.Extract(self._wav_file_path) + samples_to_ms = lambda n, sr: 1000 * n // sr + self.assertEqual(samples_to_ms(e.GetLevelFrameSize(), self._sample_rate), + e.GetLevelFrameSizeMs()) + self.assertEqual(samples_to_ms(e.GetVadFrameSize(), self._sample_rate), + e.GetVadFrameSizeMs()) + + def testVoiceActivityDetectors(self): + for vad_type_value in range(0, self._ALL_VAD_TYPES+1): + vad_type = self._VAD_TYPE_CLASS(vad_type_value) + e = annotations.AudioAnnotationsExtractor(vad_type=vad_type_value) + e.Extract(self._wav_file_path) + if vad_type.Contains(self._VAD_TYPE_CLASS.ENERGY_THRESHOLD): + # pylint: disable=unpacking-non-sequence + vad_output = e.GetVadOutput(self._VAD_TYPE_CLASS.ENERGY_THRESHOLD) + self.assertGreater(len(vad_output), 0) + self.assertGreaterEqual(float(np.sum(vad_output)) / len(vad_output), + 0.95) + + if vad_type.Contains(self._VAD_TYPE_CLASS.WEBRTC_COMMON_AUDIO): + # pylint: disable=unpacking-non-sequence + vad_output = e.GetVadOutput(self._VAD_TYPE_CLASS.WEBRTC_COMMON_AUDIO) + self.assertGreater(len(vad_output), 0) + self.assertGreaterEqual(float(np.sum(vad_output)) / len(vad_output), + 0.95) + + if vad_type.Contains(self._VAD_TYPE_CLASS.WEBRTC_APM): + # pylint: disable=unpacking-non-sequence + (vad_probs, vad_rms) = e.GetVadOutput(self._VAD_TYPE_CLASS.WEBRTC_APM) + self.assertGreater(len(vad_probs), 0) + self.assertGreater(len(vad_rms), 0) + self.assertGreaterEqual(float(np.sum(vad_probs)) / len(vad_probs), + 0.5) + self.assertGreaterEqual(float(np.sum(vad_rms)) / len(vad_rms), 20000) + + if self._DEBUG_PLOT_VAD: + frame_times_s = lambda num_frames, frame_size_ms: np.arange( + num_frames).astype(np.float32) * frame_size_ms / 1000.0 + level = e.GetLevel() + t_level = frame_times_s( + num_frames=len(level), + frame_size_ms=e.GetLevelFrameSizeMs()) + t_vad = frame_times_s( + num_frames=len(vad_output), + frame_size_ms=e.GetVadFrameSizeMs()) + import matplotlib.pyplot as plt + plt.figure() + plt.hold(True) + plt.plot(t_level, level) + plt.plot(t_vad, vad_output * np.max(level), '.') + plt.show() + + def testSaveLoad(self): + e = annotations.AudioAnnotationsExtractor(self._ALL_VAD_TYPES) + e.Extract(self._wav_file_path) + e.Save(self._tmp_path, "fake-annotation") + + data = np.load(os.path.join( + self._tmp_path, + e.GetOutputFileNameTemplate().format("fake-annotation"))) + np.testing.assert_array_equal(e.GetLevel(), data['level']) + self.assertEqual(np.float32, data['level'].dtype) + np.testing.assert_array_equal( + e.GetVadOutput(self._VAD_TYPE_CLASS.ENERGY_THRESHOLD), + data['vad_energy_output']) + np.testing.assert_array_equal( + e.GetVadOutput(self._VAD_TYPE_CLASS.WEBRTC_COMMON_AUDIO), + data['vad_output']) + np.testing.assert_array_equal( + e.GetVadOutput(self._VAD_TYPE_CLASS.WEBRTC_APM)[0], data['vad_probs']) + np.testing.assert_array_equal( + e.GetVadOutput(self._VAD_TYPE_CLASS.WEBRTC_APM)[1], data['vad_rms']) + self.assertEqual(np.uint8, data['vad_energy_output'].dtype) + self.assertEqual(np.float64, data['vad_probs'].dtype) + self.assertEqual(np.float64, data['vad_rms'].dtype) + + def testEmptyExternalShouldNotCrash(self): + for vad_type_value in range(0, self._ALL_VAD_TYPES+1): + annotations.AudioAnnotationsExtractor(vad_type_value, {}) + + def testFakeExternalSaveLoad(self): + def FakeExternalFactory(): + return external_vad.ExternalVad( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'fake_external_vad.py'), + 'fake' + ) + for vad_type_value in range(0, self._ALL_VAD_TYPES+1): + e = annotations.AudioAnnotationsExtractor( + vad_type_value, + {'fake': FakeExternalFactory()}) + e.Extract(self._wav_file_path) + e.Save(self._tmp_path, annotation_name="fake-annotation") + data = np.load(os.path.join( + self._tmp_path, + e.GetOutputFileNameTemplate().format("fake-annotation"))) + self.assertEqual(np.float32, data['extvad_conf-fake'].dtype) + np.testing.assert_almost_equal(np.arange(100, dtype=np.float32), + data['extvad_conf-fake']) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_configs/default.json b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_configs/default.json new file mode 100644 index 0000000000..5c3277bac0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_configs/default.json @@ -0,0 +1 @@ +{"-all_default": null} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_vad.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_vad.cc new file mode 100644 index 0000000000..a6184b5f9f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_vad.cc @@ -0,0 +1,94 @@ +// Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +#include +#include +#include + +#include "common_audio/wav_file.h" +#include "modules/audio_processing/vad/voice_activity_detector.h" +#include "rtc_base/flags.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace test { +namespace { + +constexpr uint8_t kAudioFrameLengthMilliseconds = 10; +constexpr int kMaxSampleRate = 48000; +constexpr size_t kMaxFrameLen = + kAudioFrameLengthMilliseconds * kMaxSampleRate / 1000; + +DEFINE_string(i, "", "Input wav file"); +DEFINE_string(o_probs, "", "VAD probabilities output file"); +DEFINE_string(o_rms, "", "VAD output file"); + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true)) + return 1; + + // Open wav input file and check properties. + WavReader wav_reader(FLAG_i); + if (wav_reader.num_channels() != 1) { + RTC_LOG(LS_ERROR) << "Only mono wav files supported"; + return 1; + } + if (wav_reader.sample_rate() > kMaxSampleRate) { + RTC_LOG(LS_ERROR) << "Beyond maximum sample rate (" << kMaxSampleRate + << ")"; + return 1; + } + const size_t audio_frame_len = rtc::CheckedDivExact( + kAudioFrameLengthMilliseconds * wav_reader.sample_rate(), 1000); + if (audio_frame_len > kMaxFrameLen) { + RTC_LOG(LS_ERROR) << "The frame size and/or the sample rate are too large."; + return 1; + } + + // Create output file and write header. + std::ofstream out_probs_file(FLAG_o_probs, std::ofstream::binary); + std::ofstream out_rms_file(FLAG_o_rms, std::ofstream::binary); + + // Run VAD and write decisions. + VoiceActivityDetector vad; + std::array samples; + + while (true) { + // Process frame. + const auto read_samples = + wav_reader.ReadSamples(audio_frame_len, samples.data()); + if (read_samples < audio_frame_len) { + break; + } + vad.ProcessChunk(samples.data(), audio_frame_len, wav_reader.sample_rate()); + // Write output. + auto probs = vad.chunkwise_voice_probabilities(); + auto rms = vad.chunkwise_rms(); + RTC_CHECK_EQ(probs.size(), rms.size()); + RTC_CHECK_EQ(sizeof(double), 8); + + for (const auto& p : probs) { + out_probs_file.write(reinterpret_cast(&p), 8); + } + for (const auto& r : rms) { + out_rms_file.write(reinterpret_cast(&r), 8); + } + } + + out_probs_file.close(); + out_rms_file.close(); + return 0; +} + +} // namespace +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py new file mode 100644 index 0000000000..399e3864dc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py @@ -0,0 +1,96 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Class implementing a wrapper for APM simulators. +""" + +import cProfile +import logging +import os +import subprocess + +from . import data_access +from . import exceptions + + +class AudioProcWrapper(object): + """Wrapper for APM simulators. + """ + + DEFAULT_APM_SIMULATOR_BIN_PATH = os.path.abspath(os.path.join( + os.pardir, 'audioproc_f')) + OUTPUT_FILENAME = 'output.wav' + + def __init__(self, simulator_bin_path): + """Ctor. + + Args: + simulator_bin_path: path to the APM simulator binary. + """ + self._simulator_bin_path = simulator_bin_path + self._config = None + self._output_signal_filepath = None + + # Profiler instance to measure running time. + self._profiler = cProfile.Profile() + + @property + def output_filepath(self): + return self._output_signal_filepath + + def Run(self, config_filepath, capture_input_filepath, output_path, + render_input_filepath=None): + """Runs APM simulator. + + Args: + config_filepath: path to the configuration file specifying the arguments + for the APM simulator. + capture_input_filepath: path to the capture audio track input file (aka + forward or near-end). + output_path: path of the audio track output file. + render_input_filepath: path to the render audio track input file (aka + reverse or far-end). + """ + # Init. + self._output_signal_filepath = os.path.join( + output_path, self.OUTPUT_FILENAME) + profiling_stats_filepath = os.path.join(output_path, 'profiling.stats') + + # Skip if the output has already been generated. + if os.path.exists(self._output_signal_filepath) and os.path.exists( + profiling_stats_filepath): + return + + # Load configuration. + self._config = data_access.AudioProcConfigFile.Load(config_filepath) + + # Set remaining parameters. + if not os.path.exists(capture_input_filepath): + raise exceptions.FileNotFoundError('cannot find capture input file') + self._config['-i'] = capture_input_filepath + self._config['-o'] = self._output_signal_filepath + if render_input_filepath is not None: + if not os.path.exists(render_input_filepath): + raise exceptions.FileNotFoundError('cannot find render input file') + self._config['-ri'] = render_input_filepath + + # Build arguments list. + args = [self._simulator_bin_path] + for param_name in self._config: + args.append(param_name) + if self._config[param_name] is not None: + args.append(str(self._config[param_name])) + logging.debug(' '.join(args)) + + # Run. + self._profiler.enable() + subprocess.call(args) + self._profiler.disable() + + # Save profiling stats. + self._profiler.dump_stats(profiling_stats_filepath) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py new file mode 100644 index 0000000000..fc1f44bcf8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py @@ -0,0 +1,242 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Imports a filtered subset of the scores and configurations computed +by apm_quality_assessment.py into a pandas data frame. +""" + +import argparse +import glob +import logging +import os +import re +import sys + +try: + import pandas as pd +except ImportError: + logging.critical('Cannot import the third-party Python package pandas') + sys.exit(1) + +from . import data_access as data_access +from . import simulation as sim + +# Compiled regular expressions used to extract score descriptors. +RE_CONFIG_NAME = re.compile( + sim.ApmModuleSimulator.GetPrefixApmConfig() + r'(.+)') +RE_CAPTURE_NAME = re.compile( + sim.ApmModuleSimulator.GetPrefixCapture() + r'(.+)') +RE_RENDER_NAME = re.compile( + sim.ApmModuleSimulator.GetPrefixRender() + r'(.+)') +RE_ECHO_SIM_NAME = re.compile( + sim.ApmModuleSimulator.GetPrefixEchoSimulator() + r'(.+)') +RE_TEST_DATA_GEN_NAME = re.compile( + sim.ApmModuleSimulator.GetPrefixTestDataGenerator() + r'(.+)') +RE_TEST_DATA_GEN_PARAMS = re.compile( + sim.ApmModuleSimulator.GetPrefixTestDataGeneratorParameters() + r'(.+)') +RE_SCORE_NAME = re.compile( + sim.ApmModuleSimulator.GetPrefixScore() + r'(.+)(\..+)') + + +def InstanceArgumentsParser(): + """Arguments parser factory. + """ + parser = argparse.ArgumentParser(description=( + 'Override this description in a user script by changing' + ' `parser.description` of the returned parser.')) + + parser.add_argument('-o', '--output_dir', required=True, + help=('the same base path used with the ' + 'apm_quality_assessment tool')) + + parser.add_argument('-c', '--config_names', type=re.compile, + help=('regular expression to filter the APM configuration' + ' names')) + + parser.add_argument('-i', '--capture_names', type=re.compile, + help=('regular expression to filter the capture signal ' + 'names')) + + parser.add_argument('-r', '--render_names', type=re.compile, + help=('regular expression to filter the render signal ' + 'names')) + + parser.add_argument('-e', '--echo_simulator_names', type=re.compile, + help=('regular expression to filter the echo simulator ' + 'names')) + + parser.add_argument('-t', '--test_data_generators', type=re.compile, + help=('regular expression to filter the test data ' + 'generator names')) + + parser.add_argument('-s', '--eval_scores', type=re.compile, + help=('regular expression to filter the evaluation score ' + 'names')) + + return parser + + +def _GetScoreDescriptors(score_filepath): + """Extracts a score descriptor from the given score file path. + + Args: + score_filepath: path to the score file. + + Returns: + A tuple of strings (APM configuration name, capture audio track name, + render audio track name, echo simulator name, test data generator name, + test data generator parameters as string, evaluation score name). + """ + fields = score_filepath.split(os.sep)[-7:] + extract_name = lambda index, reg_expr: ( + reg_expr.match(fields[index]).groups(0)[0]) + return ( + extract_name(0, RE_CONFIG_NAME), + extract_name(1, RE_CAPTURE_NAME), + extract_name(2, RE_RENDER_NAME), + extract_name(3, RE_ECHO_SIM_NAME), + extract_name(4, RE_TEST_DATA_GEN_NAME), + extract_name(5, RE_TEST_DATA_GEN_PARAMS), + extract_name(6, RE_SCORE_NAME), + ) + + +def _ExcludeScore(config_name, capture_name, render_name, echo_simulator_name, + test_data_gen_name, score_name, args): + """Decides whether excluding a score. + + A set of optional regular expressions in args is used to determine if the + score should be excluded (depending on its |*_name| descriptors). + + Args: + config_name: APM configuration name. + capture_name: capture audio track name. + render_name: render audio track name. + echo_simulator_name: echo simulator name. + test_data_gen_name: test data generator name. + score_name: evaluation score name. + args: parsed arguments. + + Returns: + A boolean. + """ + value_regexpr_pairs = [ + (config_name, args.config_names), + (capture_name, args.capture_names), + (render_name, args.render_names), + (echo_simulator_name, args.echo_simulator_names), + (test_data_gen_name, args.test_data_generators), + (score_name, args.eval_scores), + ] + + # Score accepted if each value matches the corresponding regular expression. + for value, regexpr in value_regexpr_pairs: + if regexpr is None: + continue + if not regexpr.match(value): + return True + + return False + + +def FindScores(src_path, args): + """Given a search path, find scores and return a DataFrame object. + + Args: + src_path: Search path pattern. + args: parsed arguments. + + Returns: + A DataFrame object. + """ + # Get scores. + scores = [] + for score_filepath in glob.iglob(src_path): + # Extract score descriptor fields from the path. + (config_name, + capture_name, + render_name, + echo_simulator_name, + test_data_gen_name, + test_data_gen_params, + score_name) = _GetScoreDescriptors(score_filepath) + + # Ignore the score if required. + if _ExcludeScore( + config_name, + capture_name, + render_name, + echo_simulator_name, + test_data_gen_name, + score_name, + args): + logging.info( + 'ignored score: %s %s %s %s %s %s', + config_name, + capture_name, + render_name, + echo_simulator_name, + test_data_gen_name, + score_name) + continue + + # Read metadata and score. + metadata = data_access.Metadata.LoadAudioTestDataPaths( + os.path.split(score_filepath)[0]) + score = data_access.ScoreFile.Load(score_filepath) + + # Add a score with its descriptor fields. + scores.append(( + metadata['clean_capture_input_filepath'], + metadata['echo_free_capture_filepath'], + metadata['echo_filepath'], + metadata['render_filepath'], + metadata['capture_filepath'], + metadata['apm_output_filepath'], + metadata['apm_reference_filepath'], + config_name, + capture_name, + render_name, + echo_simulator_name, + test_data_gen_name, + test_data_gen_params, + score_name, + score, + )) + + return pd.DataFrame( + data=scores, + columns=( + 'clean_capture_input_filepath', + 'echo_free_capture_filepath', + 'echo_filepath', + 'render_filepath', + 'capture_filepath', + 'apm_output_filepath', + 'apm_reference_filepath', + 'apm_config', + 'capture', + 'render', + 'echo_simulator', + 'test_data_gen', + 'test_data_gen_params', + 'eval_score_name', + 'score', + )) + + +def ConstructSrcPath(args): + return os.path.join( + args.output_dir, + sim.ApmModuleSimulator.GetPrefixApmConfig() + '*', + sim.ApmModuleSimulator.GetPrefixCapture() + '*', + sim.ApmModuleSimulator.GetPrefixRender() + '*', + sim.ApmModuleSimulator.GetPrefixEchoSimulator() + '*', + sim.ApmModuleSimulator.GetPrefixTestDataGenerator() + '*', + sim.ApmModuleSimulator.GetPrefixTestDataGeneratorParameters() + '*', + sim.ApmModuleSimulator.GetPrefixScore() + '*') diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py new file mode 100644 index 0000000000..17aa7e2b67 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py @@ -0,0 +1,154 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Data access utility functions and classes. +""" + +import json +import os + + +def MakeDirectory(path): + """Makes a directory recursively without rising exceptions if existing. + + Args: + path: path to the directory to be created. + """ + if os.path.exists(path): + return + os.makedirs(path) + + +class Metadata(object): + """Data access class to save and load metadata. + """ + + def __init__(self): + pass + + _GENERIC_METADATA_SUFFIX = '.mdata' + _AUDIO_TEST_DATA_FILENAME = 'audio_test_data.json' + + @classmethod + def LoadFileMetadata(cls, filepath): + """Loads generic metadata linked to a file. + + Args: + filepath: path to the metadata file to read. + + Returns: + A dict. + """ + with open(filepath + cls._GENERIC_METADATA_SUFFIX) as f: + return json.load(f) + + @classmethod + def SaveFileMetadata(cls, filepath, metadata): + """Saves generic metadata linked to a file. + + Args: + filepath: path to the metadata file to write. + metadata: a dict. + """ + with open(filepath + cls._GENERIC_METADATA_SUFFIX, 'w') as f: + json.dump(metadata, f) + + @classmethod + def LoadAudioTestDataPaths(cls, metadata_path): + """Loads the input and the reference audio track paths. + + Args: + metadata_path: path to the directory containing the metadata file. + + Returns: + Tuple with the paths to the input and output audio tracks. + """ + metadata_filepath = os.path.join( + metadata_path, cls._AUDIO_TEST_DATA_FILENAME) + with open(metadata_filepath) as f: + return json.load(f) + + @classmethod + def SaveAudioTestDataPaths(cls, output_path, **filepaths): + """Saves the input and the reference audio track paths. + + Args: + output_path: path to the directory containing the metadata file. + + Keyword Args: + filepaths: collection of audio track file paths to save. + """ + output_filepath = os.path.join(output_path, cls._AUDIO_TEST_DATA_FILENAME) + with open(output_filepath, 'w') as f: + json.dump(filepaths, f) + + +class AudioProcConfigFile(object): + """Data access to load/save APM simulator argument lists. + + The arguments stored in the config files are used to control the APM flags. + """ + + def __init__(self): + pass + + @classmethod + def Load(cls, filepath): + """Loads a configuration file for an APM simulator. + + Args: + filepath: path to the configuration file. + + Returns: + A dict containing the configuration. + """ + with open(filepath) as f: + return json.load(f) + + @classmethod + def Save(cls, filepath, config): + """Saves a configuration file for an APM simulator. + + Args: + filepath: path to the configuration file. + config: a dict containing the configuration. + """ + with open(filepath, 'w') as f: + json.dump(config, f) + + +class ScoreFile(object): + """Data access class to save and load float scalar scores. + """ + + def __init__(self): + pass + + @classmethod + def Load(cls, filepath): + """Loads a score from file. + + Args: + filepath: path to the score file. + + Returns: + A float encoding the score. + """ + with open(filepath) as f: + return float(f.readline().strip()) + + @classmethod + def Save(cls, filepath, score): + """Saves a score into a file. + + Args: + filepath: path to the score file. + score: float encoding the score. + """ + with open(filepath, 'w') as f: + f.write('{0:f}\n'.format(score)) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation.py new file mode 100644 index 0000000000..a1621966fe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation.py @@ -0,0 +1,136 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Echo path simulation module. +""" + +import hashlib +import os + +from . import signal_processing + + +class EchoPathSimulator(object): + """Abstract class for the echo path simulators. + + In general, an echo path simulator is a function of the render signal and + simulates the propagation of the latter into the microphone (e.g., due to + mechanical or electrical paths). + """ + + NAME = None + REGISTERED_CLASSES = {} + + def __init__(self): + pass + + def Simulate(self, output_path): + """Creates the echo signal and stores it in an audio file (abstract method). + + Args: + output_path: Path in which any output can be saved. + + Returns: + Path to the generated audio track file or None if no echo is present. + """ + raise NotImplementedError() + + @classmethod + def RegisterClass(cls, class_to_register): + """Registers an EchoPathSimulator implementation. + + Decorator to automatically register the classes that extend + EchoPathSimulator. + Example usage: + + @EchoPathSimulator.RegisterClass + class NoEchoPathSimulator(EchoPathSimulator): + pass + """ + cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register + return class_to_register + + +@EchoPathSimulator.RegisterClass +class NoEchoPathSimulator(EchoPathSimulator): + """Simulates absence of echo.""" + + NAME = 'noecho' + + def __init__(self): + EchoPathSimulator.__init__(self) + + def Simulate(self, output_path): + return None + + +@EchoPathSimulator.RegisterClass +class LinearEchoPathSimulator(EchoPathSimulator): + """Simulates linear echo path. + + This class applies a given impulse response to the render input and then it + sums the signal to the capture input signal. + """ + + NAME = 'linear' + + def __init__(self, render_input_filepath, impulse_response): + """ + Args: + render_input_filepath: Render audio track file. + impulse_response: list or numpy vector of float values. + """ + EchoPathSimulator.__init__(self) + self._render_input_filepath = render_input_filepath + self._impulse_response = impulse_response + + def Simulate(self, output_path): + """Simulates linear echo path.""" + # Form the file name with a hash of the impulse response. + impulse_response_hash = hashlib.sha256( + str(self._impulse_response).encode('utf-8', 'ignore')).hexdigest() + echo_filepath = os.path.join(output_path, 'linear_echo_{}.wav'.format( + impulse_response_hash)) + + # If the simulated echo audio track file does not exists, create it. + if not os.path.exists(echo_filepath): + render = signal_processing.SignalProcessingUtils.LoadWav( + self._render_input_filepath) + echo = signal_processing.SignalProcessingUtils.ApplyImpulseResponse( + render, self._impulse_response) + signal_processing.SignalProcessingUtils.SaveWav(echo_filepath, echo) + + return echo_filepath + + +@EchoPathSimulator.RegisterClass +class RecordedEchoPathSimulator(EchoPathSimulator): + """Uses recorded echo. + + This class uses the clean capture input file name to build the file name of + the corresponding recording containing echo (a predefined suffix is used). + Such a file is expected to be already existing. + """ + + NAME = 'recorded' + + _FILE_NAME_SUFFIX = '_echo' + + def __init__(self, render_input_filepath): + EchoPathSimulator.__init__(self) + self._render_input_filepath = render_input_filepath + + def Simulate(self, output_path): + """Uses recorded echo path.""" + path, file_name_ext = os.path.split(self._render_input_filepath) + file_name, file_ext = os.path.splitext(file_name_ext) + echo_filepath = os.path.join(path, '{}{}{}'.format( + file_name, self._FILE_NAME_SUFFIX, file_ext)) + assert os.path.exists(echo_filepath), ( + 'cannot find the echo audio track file {}'.format(echo_filepath)) + return echo_filepath diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_factory.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_factory.py new file mode 100644 index 0000000000..eeffd1d71b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_factory.py @@ -0,0 +1,48 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Echo path simulation factory module. +""" + +import numpy as np + +from . import echo_path_simulation + + +class EchoPathSimulatorFactory(object): + + # TODO(alessiob): Replace 5 ms delay (at 48 kHz sample rate) with a more + # realistic impulse response. + _LINEAR_ECHO_IMPULSE_RESPONSE = np.array([0.0]*(5 * 48) + [0.15]) + + def __init__(self): + pass + + @classmethod + def GetInstance(cls, echo_path_simulator_class, render_input_filepath): + """Creates an EchoPathSimulator instance given a class object. + + Args: + echo_path_simulator_class: EchoPathSimulator class object (not an + instance). + render_input_filepath: Path to the render audio track file. + + Returns: + An EchoPathSimulator instance. + """ + assert render_input_filepath is not None or ( + echo_path_simulator_class == echo_path_simulation.NoEchoPathSimulator) + + if echo_path_simulator_class == echo_path_simulation.NoEchoPathSimulator: + return echo_path_simulation.NoEchoPathSimulator() + elif echo_path_simulator_class == ( + echo_path_simulation.LinearEchoPathSimulator): + return echo_path_simulation.LinearEchoPathSimulator( + render_input_filepath, cls._LINEAR_ECHO_IMPULSE_RESPONSE) + else: + return echo_path_simulator_class(render_input_filepath) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_unittest.py new file mode 100644 index 0000000000..d9ef2c61e4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_unittest.py @@ -0,0 +1,81 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the echo path simulation module. +""" + +import shutil +import os +import tempfile +import unittest + +import pydub + +from . import echo_path_simulation +from . import echo_path_simulation_factory +from . import signal_processing + + +class TestEchoPathSimulators(unittest.TestCase): + """Unit tests for the eval_scores module. + """ + + def setUp(self): + """Creates temporary data.""" + self._tmp_path = tempfile.mkdtemp() + + # Create and save white noise. + silence = pydub.AudioSegment.silent(duration=1000, frame_rate=48000) + white_noise = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + silence) + self._audio_track_num_samples = ( + signal_processing.SignalProcessingUtils.CountSamples(white_noise)) + self._audio_track_filepath = os.path.join(self._tmp_path, 'white_noise.wav') + signal_processing.SignalProcessingUtils.SaveWav( + self._audio_track_filepath, white_noise) + + # Make a copy the white noise audio track file; it will be used by + # echo_path_simulation.RecordedEchoPathSimulator. + shutil.copy(self._audio_track_filepath, os.path.join( + self._tmp_path, 'white_noise_echo.wav')) + + def tearDown(self): + """Recursively deletes temporary folders.""" + shutil.rmtree(self._tmp_path) + + def testRegisteredClasses(self): + # Check that there is at least one registered echo path simulator. + registered_classes = ( + echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES) + self.assertIsInstance(registered_classes, dict) + self.assertGreater(len(registered_classes), 0) + + # Instance factory. + factory = echo_path_simulation_factory.EchoPathSimulatorFactory() + + # Try each registered echo path simulator. + for echo_path_simulator_name in registered_classes: + simulator = factory.GetInstance( + echo_path_simulator_class=registered_classes[ + echo_path_simulator_name], + render_input_filepath=self._audio_track_filepath) + + echo_filepath = simulator.Simulate(self._tmp_path) + if echo_filepath is None: + self.assertEqual(echo_path_simulation.NoEchoPathSimulator.NAME, + echo_path_simulator_name) + # No other tests in this case. + continue + + # Check that the echo audio track file exists and its length is greater or + # equal to that of the render audio track. + self.assertTrue(os.path.exists(echo_filepath)) + echo = signal_processing.SignalProcessingUtils.LoadWav(echo_filepath) + self.assertGreaterEqual( + signal_processing.SignalProcessingUtils.CountSamples(echo), + self._audio_track_num_samples) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py new file mode 100644 index 0000000000..420afd2243 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py @@ -0,0 +1,341 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Evaluation score abstract class and implementations. +""" + +from __future__ import division +import logging +import os +import re +import subprocess +import sys + +try: + import numpy as np +except ImportError: + logging.critical('Cannot import the third-party Python package numpy') + sys.exit(1) + +from . import data_access +from . import exceptions +from . import signal_processing + + +class EvaluationScore(object): + + NAME = None + REGISTERED_CLASSES = {} + + def __init__(self, score_filename_prefix): + self._score_filename_prefix = score_filename_prefix + self._input_signal_metadata = None + self._reference_signal = None + self._reference_signal_filepath = None + self._tested_signal = None + self._tested_signal_filepath = None + self._output_filepath = None + self._score = None + + @classmethod + def RegisterClass(cls, class_to_register): + """Registers an EvaluationScore implementation. + + Decorator to automatically register the classes that extend EvaluationScore. + Example usage: + + @EvaluationScore.RegisterClass + class AudioLevelScore(EvaluationScore): + pass + """ + cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register + return class_to_register + + @property + def output_filepath(self): + return self._output_filepath + + @property + def score(self): + return self._score + + def SetInputSignalMetadata(self, metadata): + """Sets input signal metadata. + + Args: + metadata: dict instance. + """ + self._input_signal_metadata = metadata + + def SetReferenceSignalFilepath(self, filepath): + """Sets the path to the audio track used as reference signal. + + Args: + filepath: path to the reference audio track. + """ + self._reference_signal_filepath = filepath + + def SetTestedSignalFilepath(self, filepath): + """Sets the path to the audio track used as test signal. + + Args: + filepath: path to the test audio track. + """ + self._tested_signal_filepath = filepath + + def Run(self, output_path): + """Extracts the score for the set test data pair. + + Args: + output_path: path to the directory where the output is written. + """ + self._output_filepath = os.path.join( + output_path, self._score_filename_prefix + self.NAME + '.txt') + try: + # If the score has already been computed, load. + self._LoadScore() + logging.debug('score found and loaded') + except IOError: + # Compute the score. + logging.debug('score not found, compute') + self._Run(output_path) + + def _Run(self, output_path): + # Abstract method. + raise NotImplementedError() + + def _LoadReferenceSignal(self): + assert self._reference_signal_filepath is not None + self._reference_signal = signal_processing.SignalProcessingUtils.LoadWav( + self._reference_signal_filepath) + + def _LoadTestedSignal(self): + assert self._tested_signal_filepath is not None + self._tested_signal = signal_processing.SignalProcessingUtils.LoadWav( + self._tested_signal_filepath) + + + def _LoadScore(self): + return data_access.ScoreFile.Load(self._output_filepath) + + def _SaveScore(self): + return data_access.ScoreFile.Save(self._output_filepath, self._score) + + +@EvaluationScore.RegisterClass +class AudioLevelPeakScore(EvaluationScore): + """Peak audio level score. + + Defined as the difference between the peak audio level of the tested and + the reference signals. + + Unit: dB + Ideal: 0 dB + Worst case: +/-inf dB + """ + + NAME = 'audio_level_peak' + + def __init__(self, score_filename_prefix): + EvaluationScore.__init__(self, score_filename_prefix) + + def _Run(self, output_path): + self._LoadReferenceSignal() + self._LoadTestedSignal() + self._score = self._tested_signal.dBFS - self._reference_signal.dBFS + self._SaveScore() + + +@EvaluationScore.RegisterClass +class MeanAudioLevelScore(EvaluationScore): + """Mean audio level score. + + Defined as the difference between the mean audio level of the tested and + the reference signals. + + Unit: dB + Ideal: 0 dB + Worst case: +/-inf dB + """ + + NAME = 'audio_level_mean' + + def __init__(self, score_filename_prefix): + EvaluationScore.__init__(self, score_filename_prefix) + + def _Run(self, output_path): + self._LoadReferenceSignal() + self._LoadTestedSignal() + + dbfs_diffs_sum = 0.0 + seconds = min(len(self._tested_signal), len(self._reference_signal)) // 1000 + for t in range(seconds): + t0 = t * seconds + t1 = t0 + seconds + dbfs_diffs_sum += ( + self._tested_signal[t0:t1].dBFS - self._reference_signal[t0:t1].dBFS) + self._score = dbfs_diffs_sum / float(seconds) + self._SaveScore() + + +@EvaluationScore.RegisterClass +class PolqaScore(EvaluationScore): + """POLQA score. + + See http://www.polqa.info/. + + Unit: MOS + Ideal: 4.5 + Worst case: 1.0 + """ + + NAME = 'polqa' + + def __init__(self, score_filename_prefix, polqa_bin_filepath): + EvaluationScore.__init__(self, score_filename_prefix) + + # POLQA binary file path. + self._polqa_bin_filepath = polqa_bin_filepath + if not os.path.exists(self._polqa_bin_filepath): + logging.error('cannot find POLQA tool binary file') + raise exceptions.FileNotFoundError() + + # Path to the POLQA directory with binary and license files. + self._polqa_tool_path, _ = os.path.split(self._polqa_bin_filepath) + + def _Run(self, output_path): + polqa_out_filepath = os.path.join(output_path, 'polqa.out') + if os.path.exists(polqa_out_filepath): + os.unlink(polqa_out_filepath) + + args = [ + self._polqa_bin_filepath, '-t', '-q', '-Overwrite', + '-Ref', self._reference_signal_filepath, + '-Test', self._tested_signal_filepath, + '-LC', 'NB', + '-Out', polqa_out_filepath, + ] + logging.debug(' '.join(args)) + subprocess.call(args, cwd=self._polqa_tool_path) + + # Parse POLQA tool output and extract the score. + polqa_output = self._ParseOutputFile(polqa_out_filepath) + self._score = float(polqa_output['PolqaScore']) + + self._SaveScore() + + @classmethod + def _ParseOutputFile(cls, polqa_out_filepath): + """ + Parses the POLQA tool output formatted as a table ('-t' option). + + Args: + polqa_out_filepath: path to the POLQA tool output file. + + Returns: + A dict. + """ + data = [] + with open(polqa_out_filepath) as f: + for line in f: + line = line.strip() + if len(line) == 0 or line.startswith('*'): + # Ignore comments. + continue + # Read fields. + data.append(re.split(r'\t+', line)) + + # Two rows expected (header and values). + assert len(data) == 2, 'Cannot parse POLQA output' + number_of_fields = len(data[0]) + assert number_of_fields == len(data[1]) + + # Build and return a dictionary with field names (header) as keys and the + # corresponding field values as values. + return {data[0][index]: data[1][index] for index in range(number_of_fields)} + + +@EvaluationScore.RegisterClass +class TotalHarmonicDistorsionScore(EvaluationScore): + """Total harmonic distorsion plus noise score. + + Total harmonic distorsion plus noise score. + See "https://en.wikipedia.org/wiki/Total_harmonic_distortion#THD.2BN". + + Unit: -. + Ideal: 0. + Worst case: +inf + """ + + NAME = 'thd' + + def __init__(self, score_filename_prefix): + EvaluationScore.__init__(self, score_filename_prefix) + self._input_frequency = None + + def _Run(self, output_path): + # TODO(aleloi): Integrate changes made locally. + self._CheckInputSignal() + + self._LoadTestedSignal() + if self._tested_signal.channels != 1: + raise exceptions.EvaluationScoreException( + 'unsupported number of channels') + samples = signal_processing.SignalProcessingUtils.AudioSegmentToRawData( + self._tested_signal) + + # Init. + num_samples = len(samples) + duration = len(self._tested_signal) / 1000.0 + scaling = 2.0 / num_samples + max_freq = self._tested_signal.frame_rate / 2 + f0_freq = float(self._input_frequency) + t = np.linspace(0, duration, num_samples) + + # Analyze harmonics. + b_terms = [] + n = 1 + while f0_freq * n < max_freq: + x_n = np.sum(samples * np.sin(2.0 * np.pi * n * f0_freq * t)) * scaling + y_n = np.sum(samples * np.cos(2.0 * np.pi * n * f0_freq * t)) * scaling + b_terms.append(np.sqrt(x_n**2 + y_n**2)) + n += 1 + + output_without_fundamental = samples - b_terms[0] * np.sin( + 2.0 * np.pi * f0_freq * t) + distortion_and_noise = np.sqrt(np.sum( + output_without_fundamental**2) * np.pi * scaling) + + # TODO(alessiob): Fix or remove if not needed. + # thd = np.sqrt(np.sum(b_terms[1:]**2)) / b_terms[0] + + # TODO(alessiob): Check the range of |thd_plus_noise| and update the class + # docstring above if accordingly. + thd_plus_noise = distortion_and_noise / b_terms[0] + + self._score = thd_plus_noise + self._SaveScore() + + def _CheckInputSignal(self): + # Check input signal and get properties. + try: + if self._input_signal_metadata['signal'] != 'pure_tone': + raise exceptions.EvaluationScoreException( + 'The THD score requires a pure tone as input signal') + self._input_frequency = self._input_signal_metadata['frequency'] + if self._input_signal_metadata['test_data_gen_name'] != 'identity' or ( + self._input_signal_metadata['test_data_gen_config'] != 'default'): + raise exceptions.EvaluationScoreException( + 'The THD score cannot be used with any test data generator other ' + 'than "identity"') + except TypeError: + raise exceptions.EvaluationScoreException( + 'The THD score requires an input signal with associated metadata') + except KeyError: + raise exceptions.EvaluationScoreException( + 'Invalid input signal metadata to compute the THD score') diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_factory.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_factory.py new file mode 100644 index 0000000000..c2ef317cc9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_factory.py @@ -0,0 +1,51 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""EvaluationScore factory class. +""" + +import logging + +from . import exceptions +from . import eval_scores + + +class EvaluationScoreWorkerFactory(object): + """Factory class used to instantiate evaluation score workers. + + The ctor gets the parametrs that are used to instatiate the evaluation score + workers. + """ + + def __init__(self, polqa_tool_bin_path): + self._score_filename_prefix = None + self._polqa_tool_bin_path = polqa_tool_bin_path + + def SetScoreFilenamePrefix(self, prefix): + self._score_filename_prefix = prefix + + def GetInstance(self, evaluation_score_class): + """Creates an EvaluationScore instance given a class object. + + Args: + evaluation_score_class: EvaluationScore class object (not an instance). + + Returns: + An EvaluationScore instance. + """ + if self._score_filename_prefix is None: + raise exceptions.InitializationException( + 'The score file name prefix for evaluation score workers is not set') + logging.debug( + 'factory producing a %s evaluation score', evaluation_score_class) + + if evaluation_score_class == eval_scores.PolqaScore: + return eval_scores.PolqaScore( + self._score_filename_prefix, self._polqa_tool_bin_path) + else: + return evaluation_score_class(self._score_filename_prefix) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_unittest.py new file mode 100644 index 0000000000..ddb5d0b07a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_unittest.py @@ -0,0 +1,131 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the eval_scores module. +""" + +import os +import shutil +import tempfile +import unittest + +import pydub + +from . import data_access +from . import eval_scores +from . import eval_scores_factory +from . import signal_processing + + +class TestEvalScores(unittest.TestCase): + """Unit tests for the eval_scores module. + """ + + def setUp(self): + """Create temporary output folder and two audio track files.""" + self._output_path = tempfile.mkdtemp() + + # Create fake reference and tested (i.e., APM output) audio track files. + silence = pydub.AudioSegment.silent(duration=1000, frame_rate=48000) + fake_reference_signal = ( + signal_processing.SignalProcessingUtils.GenerateWhiteNoise(silence)) + fake_tested_signal = ( + signal_processing.SignalProcessingUtils.GenerateWhiteNoise(silence)) + + # Save fake audio tracks. + self._fake_reference_signal_filepath = os.path.join( + self._output_path, 'fake_ref.wav') + signal_processing.SignalProcessingUtils.SaveWav( + self._fake_reference_signal_filepath, fake_reference_signal) + self._fake_tested_signal_filepath = os.path.join( + self._output_path, 'fake_test.wav') + signal_processing.SignalProcessingUtils.SaveWav( + self._fake_tested_signal_filepath, fake_tested_signal) + + def tearDown(self): + """Recursively delete temporary folder.""" + shutil.rmtree(self._output_path) + + def testRegisteredClasses(self): + # Evaluation score names to exclude (tested separately). + exceptions = ['thd'] + + # Preliminary check. + self.assertTrue(os.path.exists(self._output_path)) + + # Check that there is at least one registered evaluation score worker. + registered_classes = eval_scores.EvaluationScore.REGISTERED_CLASSES + self.assertIsInstance(registered_classes, dict) + self.assertGreater(len(registered_classes), 0) + + # Instance evaluation score workers factory with fake dependencies. + eval_score_workers_factory = ( + eval_scores_factory.EvaluationScoreWorkerFactory( + polqa_tool_bin_path=os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'fake_polqa'))) + eval_score_workers_factory.SetScoreFilenamePrefix('scores-') + + # Try each registered evaluation score worker. + for eval_score_name in registered_classes: + if eval_score_name in exceptions: + continue + + # Instance evaluation score worker. + eval_score_worker = eval_score_workers_factory.GetInstance( + registered_classes[eval_score_name]) + + # Set fake input metadata and reference and test file paths, then run. + eval_score_worker.SetReferenceSignalFilepath( + self._fake_reference_signal_filepath) + eval_score_worker.SetTestedSignalFilepath( + self._fake_tested_signal_filepath) + eval_score_worker.Run(self._output_path) + + # Check output. + score = data_access.ScoreFile.Load(eval_score_worker.output_filepath) + self.assertTrue(isinstance(score, float)) + + def testTotalHarmonicDistorsionScore(self): + # Init. + pure_tone_freq = 5000.0 + eval_score_worker = eval_scores.TotalHarmonicDistorsionScore('scores-') + eval_score_worker.SetInputSignalMetadata({ + 'signal': 'pure_tone', + 'frequency': pure_tone_freq, + 'test_data_gen_name': 'identity', + 'test_data_gen_config': 'default', + }) + template = pydub.AudioSegment.silent(duration=1000, frame_rate=48000) + + # Create 3 test signals: pure tone, pure tone + white noise, white noise + # only. + pure_tone = signal_processing.SignalProcessingUtils.GeneratePureTone( + template, pure_tone_freq) + white_noise = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + template) + noisy_tone = signal_processing.SignalProcessingUtils.MixSignals( + pure_tone, white_noise) + + # Compute scores for increasingly distorted pure tone signals. + scores = [None, None, None] + for index, tested_signal in enumerate([pure_tone, noisy_tone, white_noise]): + # Save signal. + tmp_filepath = os.path.join(self._output_path, 'tmp_thd.wav') + signal_processing.SignalProcessingUtils.SaveWav( + tmp_filepath, tested_signal) + + # Compute score. + eval_score_worker.SetTestedSignalFilepath(tmp_filepath) + eval_score_worker.Run(self._output_path) + scores[index] = eval_score_worker.score + + # Remove output file to avoid caching. + os.remove(eval_score_worker.output_filepath) + + # Validate scores (lowest score with a pure tone). + self.assertTrue(all([scores[i + 1] > scores[i] for i in range(2)])) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py new file mode 100644 index 0000000000..09ded4cbd5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py @@ -0,0 +1,53 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Evaluator of the APM module. +""" + +import logging + + +class ApmModuleEvaluator(object): + """APM evaluator class. + """ + + def __init__(self): + pass + + @classmethod + def Run(cls, evaluation_score_workers, apm_input_metadata, + apm_output_filepath, reference_input_filepath, output_path): + """Runs the evaluation. + + Iterates over the given evaluation score workers. + + Args: + evaluation_score_workers: list of EvaluationScore instances. + apm_input_metadata: dictionary with metadata of the APM input. + apm_output_filepath: path to the audio track file with the APM output. + reference_input_filepath: path to the reference audio track file. + output_path: output path. + + Returns: + A dict of evaluation score name and score pairs. + """ + # Init. + scores = {} + + for evaluation_score_worker in evaluation_score_workers: + logging.info(' computing <%s> score', evaluation_score_worker.NAME) + evaluation_score_worker.SetInputSignalMetadata(apm_input_metadata) + evaluation_score_worker.SetReferenceSignalFilepath( + reference_input_filepath) + evaluation_score_worker.SetTestedSignalFilepath( + apm_output_filepath) + + evaluation_score_worker.Run(output_path) + scores[evaluation_score_worker.NAME] = evaluation_score_worker.score + + return scores diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/exceptions.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/exceptions.py new file mode 100644 index 0000000000..852e9e8468 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/exceptions.py @@ -0,0 +1,46 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Exception classes. +""" + + +class FileNotFoundError(Exception): + """File not found exception. + """ + pass + + +class SignalProcessingException(Exception): + """Signal processing exception. + """ + pass + + +class InputMixerException(Exception): + """Input mixer exception. + """ + pass + + +class InputSignalCreatorException(Exception): + """Input signal creator exception. + """ + pass + + +class EvaluationScoreException(Exception): + """Evaluation score exception. + """ + pass + + +class InitializationException(Exception): + """Initialization exception. + """ + pass diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py new file mode 100644 index 0000000000..4c50cea21f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py @@ -0,0 +1,402 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import functools +import hashlib +import logging +import os +import re +import sys + +try: + import csscompressor +except ImportError: + logging.critical('Cannot import the third-party Python package csscompressor') + sys.exit(1) + +try: + import jsmin +except ImportError: + logging.critical('Cannot import the third-party Python package jsmin') + sys.exit(1) + + +class HtmlExport(object): + """HTML exporter class for APM quality scores.""" + + _NEW_LINE = '\n' + + # CSS and JS file paths. + _PATH = os.path.dirname(os.path.realpath(__file__)) + _CSS_FILEPATH = os.path.join(_PATH, 'results.css') + _CSS_MINIFIED = True + _JS_FILEPATH = os.path.join(_PATH, 'results.js') + _JS_MINIFIED = True + + def __init__(self, output_filepath): + self._scores_data_frame = None + self._output_filepath = output_filepath + + def Export(self, scores_data_frame): + """Exports scores into an HTML file. + + Args: + scores_data_frame: DataFrame instance. + """ + self._scores_data_frame = scores_data_frame + html = ['', + self._BuildHeader(), + (''), + '', + self._BuildBody(), + '', + ''] + self._Save(self._output_filepath, self._NEW_LINE.join(html)) + + def _BuildHeader(self): + """Builds the section of the HTML file. + + The header contains the page title and either embedded or linked CSS and JS + files. + + Returns: + A string with ... HTML. + """ + html = ['', 'Results'] + + # Add Material Design hosted libs. + html.append('') + html.append('') + html.append('') + html.append('') + + # Embed custom JavaScript and CSS files. + html.append('') + html.append('') + + html.append('') + + return self._NEW_LINE.join(html) + + def _BuildBody(self): + """Builds the content of the section.""" + score_names = self._scores_data_frame['eval_score_name'].drop_duplicates( + ).values.tolist() + + html = [ + ('
'), + '
', + '
', + 'APM QA results ({})'.format( + self._output_filepath), + '
', + ] + + # Tab selectors. + html.append('
') + for tab_index, score_name in enumerate(score_names): + is_active = tab_index == 0 + html.append('' + '{}'.format(tab_index, + ' is-active' if is_active else '', + self._FormatName(score_name))) + html.append('
') + + html.append('
') + html.append('
') + + # Tabs content. + for tab_index, score_name in enumerate(score_names): + html.append('
'.format( + ' is-active' if is_active else '', tab_index)) + html.append('
') + html.append(self._BuildScoreTab(score_name, ('s{}'.format(tab_index),))) + html.append('
') + html.append('
') + + html.append('
') + html.append('
') + + # Add snackbar for notifications. + html.append( + '
' + '
' + '' + '
') + + return self._NEW_LINE.join(html) + + def _BuildScoreTab(self, score_name, anchor_data): + """Builds the content of a tab.""" + # Find unique values. + scores = self._scores_data_frame[ + self._scores_data_frame.eval_score_name == score_name] + apm_configs = sorted(self._FindUniqueTuples(scores, ['apm_config'])) + test_data_gen_configs = sorted(self._FindUniqueTuples( + scores, ['test_data_gen', 'test_data_gen_params'])) + + html = [ + '
', + '
', + '
', + (''), + ] + + # Header. + html.append('') + for test_data_gen_info in test_data_gen_configs: + html.append(''.format( + self._FormatName(test_data_gen_info[0]), test_data_gen_info[1])) + html.append('') + + # Body. + html.append('') + for apm_config in apm_configs: + html.append('') + for test_data_gen_info in test_data_gen_configs: + dialog_id = self._ScoreStatsInspectorDialogId( + score_name, apm_config[0], test_data_gen_info[0], + test_data_gen_info[1]) + html.append( + ''.format( + dialog_id, self._BuildScoreTableCell( + score_name, test_data_gen_info[0], test_data_gen_info[1], + apm_config[0]))) + html.append('') + html.append('') + + html.append('
APM config / Test data generator{} {}
' + self._FormatName(apm_config[0]) + '{}
') + + html.append(self._BuildScoreStatsInspectorDialogs( + score_name, apm_configs, test_data_gen_configs, + anchor_data)) + + return self._NEW_LINE.join(html) + + def _BuildScoreTableCell(self, score_name, test_data_gen, + test_data_gen_params, apm_config): + """Builds the content of a table cell for a score table.""" + scores = self._SliceDataForScoreTableCell( + score_name, apm_config, test_data_gen, test_data_gen_params) + stats = self._ComputeScoreStats(scores) + + html = [] + items_id_prefix = ( + score_name + test_data_gen + test_data_gen_params + apm_config) + if stats['count'] == 1: + # Show the only available score. + item_id = hashlib.md5(items_id_prefix.encode('utf-8')).hexdigest() + html.append('
{1:f}
'.format( + item_id, scores['score'].mean())) + html.append('
{}' + '
'.format(item_id, 'single value')) + else: + # Show stats. + for stat_name in ['min', 'max', 'mean', 'std dev']: + item_id = hashlib.md5( + (items_id_prefix + stat_name).encode('utf-8')).hexdigest() + html.append('
{1:f}
'.format( + item_id, stats[stat_name])) + html.append('
{}' + '
'.format(item_id, stat_name)) + + return self._NEW_LINE.join(html) + + def _BuildScoreStatsInspectorDialogs( + self, score_name, apm_configs, test_data_gen_configs, anchor_data): + """Builds a set of score stats inspector dialogs.""" + html = [] + for apm_config in apm_configs: + for test_data_gen_info in test_data_gen_configs: + dialog_id = self._ScoreStatsInspectorDialogId( + score_name, apm_config[0], + test_data_gen_info[0], test_data_gen_info[1]) + + html.append(''.format(dialog_id)) + + # Content. + html.append('
') + html.append('
APM config preset: {}
' + 'Test data generator: {} ({})
'.format( + self._FormatName(apm_config[0]), + self._FormatName(test_data_gen_info[0]), + test_data_gen_info[1])) + html.append(self._BuildScoreStatsInspectorDialog( + score_name, apm_config[0], test_data_gen_info[0], + test_data_gen_info[1], anchor_data + (dialog_id,))) + html.append('
') + + # Actions. + html.append('
') + html.append('') + html.append('
') + + html.append('') + + return self._NEW_LINE.join(html) + + def _BuildScoreStatsInspectorDialog( + self, score_name, apm_config, test_data_gen, test_data_gen_params, + anchor_data): + """Builds one score stats inspector dialog.""" + scores = self._SliceDataForScoreTableCell( + score_name, apm_config, test_data_gen, test_data_gen_params) + + capture_render_pairs = sorted(self._FindUniqueTuples( + scores, ['capture', 'render'])) + echo_simulators = sorted(self._FindUniqueTuples(scores, ['echo_simulator'])) + + html = [''] + + # Header. + html.append('') + for echo_simulator in echo_simulators: + html.append('') + html.append('') + + # Body. + html.append('') + for row, (capture, render) in enumerate(capture_render_pairs): + html.append(''.format( + capture, render)) + for col, echo_simulator in enumerate(echo_simulators): + score_tuple = self._SliceDataForScoreStatsTableCell( + scores, capture, render, echo_simulator[0]) + cell_class = 'r{}c{}'.format(row, col) + html.append(''.format( + cell_class, self._BuildScoreStatsInspectorTableCell( + score_tuple, anchor_data + (cell_class,)))) + html.append('') + html.append('') + + html.append('
Capture-Render / Echo simulator' + self._FormatName(echo_simulator[0]) +'
{}
{}
{}
') + + # Placeholder for the audio inspector. + html.append('
') + + return self._NEW_LINE.join(html) + + def _BuildScoreStatsInspectorTableCell(self, score_tuple, anchor_data): + """Builds the content of a cell of a score stats inspector.""" + anchor = '&'.join(anchor_data) + html = [('
{}
' + '').format(score_tuple.score, anchor)] + + # Add all the available file paths as hidden data. + for field_name in score_tuple.keys(): + if field_name.endswith('_filepath'): + html.append(''.format( + field_name, score_tuple[field_name])) + + return self._NEW_LINE.join(html) + + def _SliceDataForScoreTableCell( + self, score_name, apm_config, test_data_gen, test_data_gen_params): + """Slices |self._scores_data_frame| to extract the data for a tab.""" + masks = [] + masks.append(self._scores_data_frame.eval_score_name == score_name) + masks.append(self._scores_data_frame.apm_config == apm_config) + masks.append(self._scores_data_frame.test_data_gen == test_data_gen) + masks.append( + self._scores_data_frame.test_data_gen_params == test_data_gen_params) + mask = functools.reduce((lambda i1, i2: i1 & i2), masks) + del masks + return self._scores_data_frame[mask] + + @classmethod + def _SliceDataForScoreStatsTableCell( + cls, scores, capture, render, echo_simulator): + """Slices |scores| to extract the data for a tab.""" + masks = [] + + masks.append(scores.capture == capture) + masks.append(scores.render == render) + masks.append(scores.echo_simulator == echo_simulator) + mask = functools.reduce((lambda i1, i2: i1 & i2), masks) + del masks + + sliced_data = scores[mask] + assert len(sliced_data) == 1, 'single score is expected' + return sliced_data.iloc[0] + + @classmethod + def _FindUniqueTuples(cls, data_frame, fields): + """Slices |data_frame| to a list of fields and finds unique tuples.""" + return data_frame[fields].drop_duplicates().values.tolist() + + @classmethod + def _ComputeScoreStats(cls, data_frame): + """Computes score stats.""" + scores = data_frame['score'] + return { + 'count': scores.count(), + 'min': scores.min(), + 'max': scores.max(), + 'mean': scores.mean(), + 'std dev': scores.std(), + } + + @classmethod + def _ScoreStatsInspectorDialogId(cls, score_name, apm_config, test_data_gen, + test_data_gen_params): + """Assigns a unique name to a dialog.""" + return 'score-stats-dialog-' + hashlib.md5( + 'score-stats-inspector-{}-{}-{}-{}'.format( + score_name, apm_config, test_data_gen, + test_data_gen_params).encode('utf-8')).hexdigest() + + @classmethod + def _Save(cls, output_filepath, html): + """Writes the HTML file. + + Args: + output_filepath: output file path. + html: string with the HTML content. + """ + with open(output_filepath, 'w') as f: + f.write(html) + + @classmethod + def _FormatName(cls, name): + """Formats a name. + + Args: + name: a string. + + Returns: + A copy of name in which underscores and dashes are replaced with a space. + """ + return re.sub(r'[_\-]', ' ', name) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py new file mode 100644 index 0000000000..8b4e20699c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py @@ -0,0 +1,84 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the export module. +""" + +import logging +import os +import shutil +import tempfile +import unittest + +import pyquery as pq + +from . import audioproc_wrapper +from . import collect_data +from . import eval_scores_factory +from . import evaluation +from . import export +from . import simulation +from . import test_data_generation_factory + + +class TestExport(unittest.TestCase): + """Unit tests for the export module. + """ + + _CLEAN_TMP_OUTPUT = True + + def setUp(self): + """Creates temporary data to export.""" + self._tmp_path = tempfile.mkdtemp() + + # Run a fake experiment to produce data to export. + simulator = simulation.ApmModuleSimulator( + test_data_generator_factory=( + test_data_generation_factory.TestDataGeneratorFactory( + aechen_ir_database_path='', + noise_tracks_path='', + copy_with_identity=False)), + evaluation_score_factory=( + eval_scores_factory.EvaluationScoreWorkerFactory( + polqa_tool_bin_path=os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'fake_polqa'))), + ap_wrapper=audioproc_wrapper.AudioProcWrapper( + audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH), + evaluator=evaluation.ApmModuleEvaluator()) + simulator.Run( + config_filepaths=['apm_configs/default.json'], + capture_input_filepaths=[ + os.path.join(self._tmp_path, 'pure_tone-440_1000.wav'), + os.path.join(self._tmp_path, 'pure_tone-880_1000.wav'), + ], + test_data_generator_names=['identity', 'white_noise'], + eval_score_names=['audio_level_peak', 'audio_level_mean'], + output_dir=self._tmp_path) + + # Export results. + p = collect_data.InstanceArgumentsParser() + args = p.parse_args(['--output_dir', self._tmp_path]) + src_path = collect_data.ConstructSrcPath(args) + self._data_to_export = collect_data.FindScores(src_path, args) + + def tearDown(self): + """Recursively deletes temporary folders.""" + if self._CLEAN_TMP_OUTPUT: + shutil.rmtree(self._tmp_path) + else: + logging.warning(self.id() + ' did not clean the temporary path ' + ( + self._tmp_path)) + + def testCreateHtmlReport(self): + fn_out = os.path.join(self._tmp_path, 'results.html') + exporter = export.HtmlExport(fn_out) + exporter.Export(self._data_to_export) + + document = pq.PyQuery(filename=fn_out) + self.assertIsInstance(document, pq.PyQuery) + # TODO(alessiob): Use PyQuery API to check the HTML file. diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/external_vad.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/external_vad.py new file mode 100644 index 0000000000..01418d84fe --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/external_vad.py @@ -0,0 +1,77 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +from __future__ import division + +import logging +import os +import subprocess +import shutil +import sys +import tempfile + +try: + import numpy as np +except ImportError: + logging.critical('Cannot import the third-party Python package numpy') + sys.exit(1) + +from . import signal_processing + +class ExternalVad(object): + + def __init__(self, path_to_binary, name): + """Args: + path_to_binary: path to binary that accepts '-i ', '-o + '. There must be one float value per + 10ms audio + name: a name to identify the external VAD. Used for saving + the output as extvad_output-. + """ + self._path_to_binary = path_to_binary + self.name = name + assert os.path.exists(self._path_to_binary), ( + self._path_to_binary) + self._vad_output = None + + def Run(self, wav_file_path): + _signal = signal_processing.SignalProcessingUtils.LoadWav(wav_file_path) + if _signal.channels != 1: + raise NotImplementedError('Multiple-channel' + ' annotations not implemented') + if _signal.frame_rate != 48000: + raise NotImplementedError('Frame rates ' + 'other than 48000 not implemented') + + tmp_path = tempfile.mkdtemp() + try: + output_file_path = os.path.join( + tmp_path, self.name + '_vad.tmp') + subprocess.call([ + self._path_to_binary, + '-i', wav_file_path, + '-o', output_file_path + ]) + self._vad_output = np.fromfile(output_file_path, np.float32) + except Exception as e: + logging.error('Error while running the ' + self.name + + ' VAD (' + e.message + ')') + finally: + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + + def GetVadOutput(self): + assert self._vad_output is not None + return self._vad_output + + @classmethod + def ConstructVadDict(cls, vad_paths, vad_names): + external_vads = {} + for path, name in zip(vad_paths, vad_names): + external_vads[name] = ExternalVad(path, name) + return external_vads diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_external_vad.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_external_vad.py new file mode 100755 index 0000000000..7c75e8f5c3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_external_vad.py @@ -0,0 +1,24 @@ +#!/usr/bin/python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +import argparse +import numpy as np + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-i', required=True) + parser.add_argument('-o', required=True) + + args = parser.parse_args() + + array = np.arange(100, dtype=np.float32) + array.tofile(open(args.o, 'w')) + + +if __name__ == '__main__': + main() diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_polqa.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_polqa.cc new file mode 100644 index 0000000000..62d8ebb84d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/fake_polqa.cc @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { +namespace { + +const char* const kErrorMessage = "-Out /path/to/output/file is mandatory"; + +// Writes fake output intended to be parsed by +// quality_assessment.eval_scores.PolqaScore. +void WriteOutputFile(const std::string& output_file_path) { + RTC_CHECK_NE(output_file_path, ""); + std::ofstream out(output_file_path); + RTC_CHECK(!out.bad()); + out << "* Fake Polqa output" << std::endl; + out << "FakeField1\tPolqaScore\tFakeField2" << std::endl; + out << "FakeValue1\t3.25\tFakeValue2" << std::endl; + out.close(); +} + +} // namespace + +int main(int argc, char* argv[]) { + // Find "-Out" and use its next argument as output file path. + RTC_CHECK_GE(argc, 3) << kErrorMessage; + const std::string kSoughtFlagName = "-Out"; + for (int i = 1; i < argc - 1; ++i) { + if (kSoughtFlagName.compare(argv[i]) == 0) { + WriteOutputFile(argv[i + 1]); + return 0; + } + } + FATAL() << kErrorMessage; +} + +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer.py new file mode 100644 index 0000000000..b1afe14454 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer.py @@ -0,0 +1,95 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Input mixer module. +""" + +import logging +import os + +from . import exceptions +from . import signal_processing + + +class ApmInputMixer(object): + """Class to mix a set of audio segments down to the APM input.""" + + _HARD_CLIPPING_LOG_MSG = 'hard clipping detected in the mixed signal' + + def __init__(self): + pass + + @classmethod + def HardClippingLogMessage(cls): + """Returns the log message used when hard clipping is detected in the mix. + + This method is mainly intended to be used by the unit tests. + """ + return cls._HARD_CLIPPING_LOG_MSG + + @classmethod + def Mix(cls, output_path, capture_input_filepath, echo_filepath): + """Mixes capture and echo. + + Creates the overall capture input for APM by mixing the "echo-free" capture + signal with the echo signal (e.g., echo simulated via the + echo_path_simulation module). + + The echo signal cannot be shorter than the capture signal and the generated + mix will have the same duration of the capture signal. The latter property + is enforced in order to let the input of APM and the reference signal + created by TestDataGenerator have the same length (required for the + evaluation step). + + Hard-clipping may occur in the mix; a warning is raised when this happens. + + If |echo_filepath| is None, nothing is done and |capture_input_filepath| is + returned. + + Args: + speech: AudioSegment instance. + echo_path: AudioSegment instance or None. + + Returns: + Path to the mix audio track file. + """ + if echo_filepath is None: + return capture_input_filepath + + # Build the mix output file name as a function of the echo file name. + # This ensures that if the internal parameters of the echo path simulator + # change, no erroneous cache hit occurs. + echo_file_name, _ = os.path.splitext(os.path.split(echo_filepath)[1]) + capture_input_file_name, _ = os.path.splitext( + os.path.split(capture_input_filepath)[1]) + mix_filepath = os.path.join(output_path, 'mix_capture_{}_{}.wav'.format( + capture_input_file_name, echo_file_name)) + + # Create the mix if not done yet. + mix = None + if not os.path.exists(mix_filepath): + echo_free_capture = signal_processing.SignalProcessingUtils.LoadWav( + capture_input_filepath) + echo = signal_processing.SignalProcessingUtils.LoadWav(echo_filepath) + + if signal_processing.SignalProcessingUtils.CountSamples(echo) < ( + signal_processing.SignalProcessingUtils.CountSamples( + echo_free_capture)): + raise exceptions.InputMixerException( + 'echo cannot be shorter than capture') + + mix = echo_free_capture.overlay(echo) + signal_processing.SignalProcessingUtils.SaveWav(mix_filepath, mix) + + # Check if hard clipping occurs. + if mix is None: + mix = signal_processing.SignalProcessingUtils.LoadWav(mix_filepath) + if signal_processing.SignalProcessingUtils.DetectHardClipping(mix): + logging.warning(cls._HARD_CLIPPING_LOG_MSG) + + return mix_filepath diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer_unittest.py new file mode 100644 index 0000000000..b212614199 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_mixer_unittest.py @@ -0,0 +1,149 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the input mixer module. +""" + +import logging +import os +import shutil +import sys +import tempfile +import unittest + +SRC = os.path.abspath(os.path.join( + os.path.dirname((__file__)), os.pardir, os.pardir, os.pardir, os.pardir)) +sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) + +import mock + +from . import exceptions +from . import input_mixer +from . import signal_processing + + +class TestApmInputMixer(unittest.TestCase): + """Unit tests for the ApmInputMixer class. + """ + + # Audio track file names created in setUp(). + _FILENAMES = ['capture', 'echo_1', 'echo_2', 'shorter', 'longer'] + + # Target peak power level (dBFS) of each audio track file created in setUp(). + # These values are hand-crafted in order to make saturation happen when + # capture and echo_2 are mixed and the contrary for capture and echo_1. + # None means that the power is not changed. + _MAX_PEAK_POWER_LEVELS = [-10.0, -5.0, 0.0, None, None] + + # Audio track file durations in milliseconds. + _DURATIONS = [1000, 1000, 1000, 800, 1200] + + _SAMPLE_RATE = 48000 + + def setUp(self): + """Creates temporary data.""" + self._tmp_path = tempfile.mkdtemp() + + # Create audio track files. + self._audio_tracks = {} + for filename, peak_power, duration in zip( + self._FILENAMES, self._MAX_PEAK_POWER_LEVELS, self._DURATIONS): + audio_track_filepath = os.path.join(self._tmp_path, '{}.wav'.format( + filename)) + + # Create a pure tone with the target peak power level. + template = signal_processing.SignalProcessingUtils.GenerateSilence( + duration=duration, sample_rate=self._SAMPLE_RATE) + signal = signal_processing.SignalProcessingUtils.GeneratePureTone( + template) + if peak_power is not None: + signal = signal.apply_gain(-signal.max_dBFS + peak_power) + + signal_processing.SignalProcessingUtils.SaveWav( + audio_track_filepath, signal) + self._audio_tracks[filename] = { + 'filepath': audio_track_filepath, + 'num_samples': signal_processing.SignalProcessingUtils.CountSamples( + signal) + } + + def tearDown(self): + """Recursively deletes temporary folders.""" + shutil.rmtree(self._tmp_path) + + def testCheckMixSameDuration(self): + """Checks the duration when mixing capture and echo with same duration.""" + mix_filepath = input_mixer.ApmInputMixer.Mix( + self._tmp_path, + self._audio_tracks['capture']['filepath'], + self._audio_tracks['echo_1']['filepath']) + self.assertTrue(os.path.exists(mix_filepath)) + + mix = signal_processing.SignalProcessingUtils.LoadWav(mix_filepath) + self.assertEqual(self._audio_tracks['capture']['num_samples'], + signal_processing.SignalProcessingUtils.CountSamples(mix)) + + def testRejectShorterEcho(self): + """Rejects echo signals that are shorter than the capture signal.""" + try: + _ = input_mixer.ApmInputMixer.Mix( + self._tmp_path, + self._audio_tracks['capture']['filepath'], + self._audio_tracks['shorter']['filepath']) + self.fail('no exception raised') + except exceptions.InputMixerException: + pass + + def testCheckMixDurationWithLongerEcho(self): + """Checks the duration when mixing an echo longer than the capture.""" + mix_filepath = input_mixer.ApmInputMixer.Mix( + self._tmp_path, + self._audio_tracks['capture']['filepath'], + self._audio_tracks['longer']['filepath']) + self.assertTrue(os.path.exists(mix_filepath)) + + mix = signal_processing.SignalProcessingUtils.LoadWav(mix_filepath) + self.assertEqual(self._audio_tracks['capture']['num_samples'], + signal_processing.SignalProcessingUtils.CountSamples(mix)) + + def testCheckOutputFileNamesConflict(self): + """Checks that different echo files lead to different output file names.""" + mix1_filepath = input_mixer.ApmInputMixer.Mix( + self._tmp_path, + self._audio_tracks['capture']['filepath'], + self._audio_tracks['echo_1']['filepath']) + self.assertTrue(os.path.exists(mix1_filepath)) + + mix2_filepath = input_mixer.ApmInputMixer.Mix( + self._tmp_path, + self._audio_tracks['capture']['filepath'], + self._audio_tracks['echo_2']['filepath']) + self.assertTrue(os.path.exists(mix2_filepath)) + + self.assertNotEqual(mix1_filepath, mix2_filepath) + + def testHardClippingLogExpected(self): + """Checks that hard clipping warning is raised when occurring.""" + logging.warning = mock.MagicMock(name='warning') + _ = input_mixer.ApmInputMixer.Mix( + self._tmp_path, + self._audio_tracks['capture']['filepath'], + self._audio_tracks['echo_2']['filepath']) + logging.warning.assert_called_once_with( + input_mixer.ApmInputMixer.HardClippingLogMessage()) + + def testHardClippingLogNotExpected(self): + """Checks that hard clipping warning is not raised when not occurring.""" + logging.warning = mock.MagicMock(name='warning') + _ = input_mixer.ApmInputMixer.Mix( + self._tmp_path, + self._audio_tracks['capture']['filepath'], + self._audio_tracks['echo_1']['filepath']) + self.assertNotIn( + mock.call(input_mixer.ApmInputMixer.HardClippingLogMessage()), + logging.warning.call_args_list) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_signal_creator.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_signal_creator.py new file mode 100644 index 0000000000..5d97c3b2fc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/input_signal_creator.py @@ -0,0 +1,67 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Input signal creator module. +""" + +from . import exceptions +from . import signal_processing + + +class InputSignalCreator(object): + """Input signal creator class. + """ + + @classmethod + def Create(cls, name, raw_params): + """Creates a input signal and its metadata. + + Args: + name: Input signal creator name. + raw_params: Tuple of parameters to pass to the specific signal creator. + + Returns: + (AudioSegment, dict) tuple. + """ + try: + signal = {} + params = {} + + if name == 'pure_tone': + params['frequency'] = float(raw_params[0]) + params['duration'] = int(raw_params[1]) + signal = cls._CreatePureTone(params['frequency'], params['duration']) + else: + raise exceptions.InputSignalCreatorException( + 'Invalid input signal creator name') + + # Complete metadata. + params['signal'] = name + + return signal, params + except (TypeError, AssertionError) as e: + raise exceptions.InputSignalCreatorException( + 'Invalid signal creator parameters: {}'.format(e)) + + @classmethod + def _CreatePureTone(cls, frequency, duration): + """ + Generates a pure tone at 48000 Hz. + + Args: + frequency: Float in (0-24000] (Hz). + duration: Integer (milliseconds). + + Returns: + AudioSegment instance. + """ + assert 0 < frequency <= 24000 + assert 0 < duration + template = signal_processing.SignalProcessingUtils.GenerateSilence(duration) + return signal_processing.SignalProcessingUtils.GeneratePureTone( + template, frequency) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css new file mode 100644 index 0000000000..2f406bb002 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css @@ -0,0 +1,32 @@ +/* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +td.selected-score { + background-color: #DDD; +} + +td.single-score-cell{ + text-align: center; +} + +.audio-inspector { + text-align: center; +} + +.audio-inspector div{ + margin-bottom: 0; + padding-bottom: 0; + padding-top: 0; +} + +.audio-inspector div div{ + margin-bottom: 0; + padding-bottom: 0; + padding-top: 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js new file mode 100644 index 0000000000..8e47411058 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js @@ -0,0 +1,376 @@ +// Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. + +/** + * Opens the score stats inspector dialog. + * @param {String} dialogId: identifier of the dialog to show. + * @return {DOMElement} The dialog element that has been opened. + */ +function openScoreStatsInspector(dialogId) { + var dialog = document.getElementById(dialogId); + dialog.showModal(); + return dialog; +} + +/** + * Closes the score stats inspector dialog. + */ +function closeScoreStatsInspector() { + var dialog = document.querySelector('dialog[open]'); + if (dialog == null) + return; + dialog.close(); +} + +/** + * Audio inspector class. + * @constructor + */ +function AudioInspector() { + console.debug('Creating an AudioInspector instance.'); + this.audioPlayer_ = new Audio(); + this.metadata_ = {}; + this.currentScore_ = null; + this.audioInspector_ = null; + this.snackbarContainer_ = document.querySelector('#snackbar'); + + // Get base URL without anchors. + this.baseUrl_ = window.location.href; + var index = this.baseUrl_.indexOf('#'); + if (index > 0) + this.baseUrl_ = this.baseUrl_.substr(0, index) + console.info('Base URL set to "' + window.location.href + '".'); + + window.event.stopPropagation(); + this.createTextAreasForCopy_(); + this.createAudioInspector_(); + this.initializeEventHandlers_(); + + // When MDL is ready, parse the anchor (if any) to show the requested + // experiment. + var self = this; + document.querySelectorAll('header a')[0].addEventListener( + 'mdl-componentupgraded', function() { + if (!self.parseWindowAnchor()) { + // If not experiment is requested, open the first section. + console.info('No anchor parsing, opening the first section.'); + document.querySelectorAll('header a > span')[0].click(); + } + }); +} + +/** + * Parse the anchor in the window URL. + * @return {bool} True if the parsing succeeded. + */ +AudioInspector.prototype.parseWindowAnchor = function() { + var index = location.href.indexOf('#'); + if (index == -1) { + console.debug('No # found in the URL.'); + return false; + } + + var anchor = location.href.substr(index - location.href.length + 1); + console.info('Anchor changed: "' + anchor + '".'); + + var parts = anchor.split('&'); + if (parts.length != 3) { + console.info('Ignoring anchor with invalid number of fields.'); + return false; + } + + var openDialog = document.querySelector('dialog[open]'); + try { + // Open the requested dialog if not already open. + if (!openDialog || openDialog.id != parts[1]) { + !openDialog || openDialog.close(); + document.querySelectorAll('header a > span')[ + parseInt(parts[0].substr(1))].click(); + openDialog = openScoreStatsInspector(parts[1]); + } + + // Trigger click on cell. + var cell = openDialog.querySelector('td.' + parts[2]); + cell.focus(); + cell.click(); + + this.showNotification_('Experiment selected.'); + return true; + } catch (e) { + this.showNotification_('Cannot select experiment :('); + console.error('Exception caught while selecting experiment: "' + e + '".'); + } + + return false; +} + +/** + * Set up the inspector for a new score. + * @param {DOMElement} element: Element linked to the selected score. + */ +AudioInspector.prototype.selectedScoreChange = function(element) { + if (this.currentScore_ == element) { return; } + if (this.currentScore_ != null) { + this.currentScore_.classList.remove('selected-score'); + } + this.currentScore_ = element; + this.currentScore_.classList.add('selected-score'); + this.stopAudio(); + + // Read metadata. + var matches = element.querySelectorAll('input[type=hidden]'); + this.metadata_ = {}; + for (var index = 0; index < matches.length; ++index) { + this.metadata_[matches[index].name] = matches[index].value; + } + + // Show the audio inspector interface. + var container = element.parentNode.parentNode.parentNode.parentNode; + var audioInspectorPlaceholder = container.querySelector( + '.audio-inspector-placeholder'); + this.moveInspector_(audioInspectorPlaceholder); +}; + +/** + * Stop playing audio. + */ +AudioInspector.prototype.stopAudio = function() { + console.info('Pausing audio play out.'); + this.audioPlayer_.pause(); +}; + +/** + * Show a text message using the snackbar. + */ +AudioInspector.prototype.showNotification_ = function(text) { + try { + this.snackbarContainer_.MaterialSnackbar.showSnackbar({ + message: text, timeout: 2000}); + } catch (e) { + // Fallback to an alert. + alert(text); + console.warn('Cannot use snackbar: "' + e + '"'); + } +} + +/** + * Move the audio inspector DOM node into the given parent. + * @param {DOMElement} newParentNode: New parent for the inspector. + */ +AudioInspector.prototype.moveInspector_ = function(newParentNode) { + newParentNode.appendChild(this.audioInspector_); +}; + +/** + * Play audio file from url. + * @param {string} metadataFieldName: Metadata field name. + */ +AudioInspector.prototype.playAudio = function(metadataFieldName) { + if (this.metadata_[metadataFieldName] == undefined) { return; } + if (this.metadata_[metadataFieldName] == 'None') { + alert('The selected stream was not used during the experiment.'); + return; + } + this.stopAudio(); + this.audioPlayer_.src = this.metadata_[metadataFieldName]; + console.debug('Audio source URL: "' + this.audioPlayer_.src + '"'); + this.audioPlayer_.play(); + console.info('Playing out audio.'); +}; + +/** + * Create hidden text areas to copy URLs. + * + * For each dialog, one text area is created since it is not possible to select + * text on a text area outside of the active dialog. + */ +AudioInspector.prototype.createTextAreasForCopy_ = function() { + var self = this; + document.querySelectorAll('dialog.mdl-dialog').forEach(function(element) { + var textArea = document.createElement("textarea"); + textArea.classList.add('url-copy'); + textArea.style.position = 'fixed'; + textArea.style.bottom = 0; + textArea.style.left = 0; + textArea.style.width = '2em'; + textArea.style.height = '2em'; + textArea.style.border = 'none'; + textArea.style.outline = 'none'; + textArea.style.boxShadow = 'none'; + textArea.style.background = 'transparent'; + textArea.style.fontSize = '6px'; + element.appendChild(textArea); + }); +} + +/** + * Create audio inspector. + */ +AudioInspector.prototype.createAudioInspector_ = function() { + var buttonIndex = 0; + function getButtonHtml(icon, toolTipText, caption, metadataFieldName) { + var buttonId = 'audioInspectorButton' + buttonIndex++; + html = caption == null ? '' : caption; + html += '' + + return html; + } + + // TODO(alessiob): Add timeline and highlight current track by changing icon + // color. + + this.audioInspector_ = document.createElement('div'); + this.audioInspector_.classList.add('audio-inspector'); + this.audioInspector_.innerHTML = + '
' + + '
' + + '
' + + getButtonHtml('play_arrow', 'Simulated echo', 'Ein', + 'echo_filepath') + + '
' + + '
' + + getButtonHtml('stop', 'Stop playing [S]', null, '__stop__') + + '
' + + '
' + + getButtonHtml('play_arrow', 'Render stream', 'Rin', + 'render_filepath') + + '
' + + '
' + + '
' + + '
' + + '
' + + '
' + + getButtonHtml('play_arrow', 'Capture stream (APM input) [1]', + 'Y\'in', 'capture_filepath') + + '
' + + '
APM
' + + '
' + + getButtonHtml('play_arrow', 'APM output [2]', 'Yout', + 'apm_output_filepath') + + '
' + + '
' + + '
' + + '
' + + '
' + + '
' + + getButtonHtml('play_arrow', 'Echo-free capture stream', + 'Yin', 'echo_free_capture_filepath') + + '
' + + '
' + + getButtonHtml('play_arrow', 'Clean capture stream', + 'Yclean', 'clean_capture_input_filepath') + + '
' + + '
' + + getButtonHtml('play_arrow', 'APM reference [3]', 'Yref', + 'apm_reference_filepath') + + '
' + + '
' + + '
'; + + // Add an invisible node as initial container for the audio inspector. + var parent = document.createElement('div'); + parent.style.display = 'none'; + this.moveInspector_(parent); + document.body.appendChild(parent); +}; + +/** + * Initialize event handlers. + */ +AudioInspector.prototype.initializeEventHandlers_ = function() { + var self = this; + + // Score cells. + document.querySelectorAll('td.single-score-cell').forEach(function(element) { + element.onclick = function() { + self.selectedScoreChange(this); + } + }); + + // Copy anchor URLs icons. + if (document.queryCommandSupported('copy')) { + document.querySelectorAll('td.single-score-cell button').forEach( + function(element) { + element.onclick = function() { + // Find the text area in the dialog. + var textArea = element.closest('dialog').querySelector( + 'textarea.url-copy'); + + // Copy. + textArea.value = self.baseUrl_ + '#' + element.getAttribute( + 'data-anchor'); + textArea.select(); + try { + if (!document.execCommand('copy')) + throw 'Copy returned false'; + self.showNotification_('Experiment URL copied.'); + } catch (e) { + self.showNotification_('Cannot copy experiment URL :('); + console.error(e); + } + } + }); + } else { + self.showNotification_( + 'The copy command is disabled. URL copy is not enabled.'); + } + + // Audio inspector buttons. + this.audioInspector_.querySelectorAll('button').forEach(function(element) { + var target = element.querySelector('input[type=hidden]'); + if (target == null) { return; } + element.onclick = function() { + if (target.value == '__stop__') { + self.stopAudio(); + } else { + self.playAudio(target.value); + } + }; + }); + + // Dialog close handlers. + var dialogs = document.querySelectorAll('dialog').forEach(function(element) { + element.onclose = function() { + self.stopAudio(); + } + }); + + // Keyboard shortcuts. + window.onkeyup = function(e) { + var key = e.keyCode ? e.keyCode : e.which; + switch (key) { + case 49: // 1. + self.playAudio('capture_filepath'); + break; + case 50: // 2. + self.playAudio('apm_output_filepath'); + break; + case 51: // 3. + self.playAudio('apm_reference_filepath'); + break; + case 83: // S. + case 115: // s. + self.stopAudio(); + break; + } + }; + + // Hash change. + window.onhashchange = function(e) { + self.parseWindowAnchor(); + } +}; diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing.py new file mode 100644 index 0000000000..9e0198da85 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing.py @@ -0,0 +1,356 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Signal processing utility module. +""" + +import array +import enum +import logging +import os +import sys + +try: + import numpy as np +except ImportError: + logging.critical('Cannot import the third-party Python package numpy') + sys.exit(1) + +try: + import pydub + import pydub.generators +except ImportError: + logging.critical('Cannot import the third-party Python package pydub') + sys.exit(1) + +try: + import scipy.signal + import scipy.fftpack +except ImportError: + logging.critical('Cannot import the third-party Python package scipy') + sys.exit(1) + +from . import exceptions + + +class SignalProcessingUtils(object): + """Collection of signal processing utilities. + """ + + @enum.unique + class MixPadding(enum.Enum): + NO_PADDING = 0 + ZERO_PADDING = 1 + LOOP = 2 + + def __init__(self): + pass + + @classmethod + def LoadWav(cls, filepath, channels=1): + """Loads wav file. + + Args: + filepath: path to the wav audio track file to load. + channels: number of channels (downmixing to mono by default). + + Returns: + AudioSegment instance. + """ + if not os.path.exists(filepath): + logging.error('cannot find the <%s> audio track file', filepath) + raise exceptions.FileNotFoundError() + return pydub.AudioSegment.from_file( + filepath, format='wav', channels=channels) + + @classmethod + def SaveWav(cls, output_filepath, signal): + """Saves wav file. + + Args: + output_filepath: path to the wav audio track file to save. + signal: AudioSegment instance. + """ + return signal.export(output_filepath, format='wav') + + @classmethod + def CountSamples(cls, signal): + """Number of samples per channel. + + Args: + signal: AudioSegment instance. + + Returns: + An integer. + """ + number_of_samples = len(signal.get_array_of_samples()) + assert signal.channels > 0 + assert number_of_samples % signal.channels == 0 + return number_of_samples / signal.channels + + @classmethod + def GenerateSilence(cls, duration=1000, sample_rate=48000): + """Generates silence. + + This method can also be used to create a template AudioSegment instance. + A template can then be used with other Generate*() methods accepting an + AudioSegment instance as argument. + + Args: + duration: duration in ms. + sample_rate: sample rate. + + Returns: + AudioSegment instance. + """ + return pydub.AudioSegment.silent(duration, sample_rate) + + @classmethod + def GeneratePureTone(cls, template, frequency=440.0): + """Generates a pure tone. + + The pure tone is generated with the same duration and in the same format of + the given template signal. + + Args: + template: AudioSegment instance. + frequency: Frequency of the pure tone in Hz. + + Return: + AudioSegment instance. + """ + if frequency > template.frame_rate >> 1: + raise exceptions.SignalProcessingException('Invalid frequency') + + generator = pydub.generators.Sine( + sample_rate=template.frame_rate, + bit_depth=template.sample_width * 8, + freq=frequency) + + return generator.to_audio_segment( + duration=len(template), + volume=0.0) + + @classmethod + def GenerateWhiteNoise(cls, template): + """Generates white noise. + + The white noise is generated with the same duration and in the same format + of the given template signal. + + Args: + template: AudioSegment instance. + + Return: + AudioSegment instance. + """ + generator = pydub.generators.WhiteNoise( + sample_rate=template.frame_rate, + bit_depth=template.sample_width * 8) + return generator.to_audio_segment( + duration=len(template), + volume=0.0) + + @classmethod + def AudioSegmentToRawData(cls, signal): + samples = signal.get_array_of_samples() + if samples.typecode != 'h': + raise exceptions.SignalProcessingException('Unsupported samples type') + return np.array(signal.get_array_of_samples(), np.int16) + + @classmethod + def Fft(cls, signal, normalize=True): + if signal.channels != 1: + raise NotImplementedError('multiple-channel FFT not implemented') + x = cls.AudioSegmentToRawData(signal).astype(np.float32) + if normalize: + x /= max(abs(np.max(x)), 1.0) + y = scipy.fftpack.fft(x) + return y[:len(y) / 2] + + @classmethod + def DetectHardClipping(cls, signal, threshold=2): + """Detects hard clipping. + + Hard clipping is simply detected by counting samples that touch either the + lower or upper bound too many times in a row (according to |threshold|). + The presence of a single sequence of samples meeting such property is enough + to label the signal as hard clipped. + + Args: + signal: AudioSegment instance. + threshold: minimum number of samples at full-scale in a row. + + Returns: + True if hard clipping is detect, False otherwise. + """ + if signal.channels != 1: + raise NotImplementedError('multiple-channel clipping not implemented') + if signal.sample_width != 2: # Note that signal.sample_width is in bytes. + raise exceptions.SignalProcessingException( + 'hard-clipping detection only supported for 16 bit samples') + samples = cls.AudioSegmentToRawData(signal) + + # Detect adjacent clipped samples. + samples_type_info = np.iinfo(samples.dtype) + mask_min = samples == samples_type_info.min + mask_max = samples == samples_type_info.max + + def HasLongSequence(vector, min_legth=threshold): + """Returns True if there are one or more long sequences of True flags.""" + seq_length = 0 + for b in vector: + seq_length = seq_length + 1 if b else 0 + if seq_length >= min_legth: + return True + return False + + return HasLongSequence(mask_min) or HasLongSequence(mask_max) + + @classmethod + def ApplyImpulseResponse(cls, signal, impulse_response): + """Applies an impulse response to a signal. + + Args: + signal: AudioSegment instance. + impulse_response: list or numpy vector of float values. + + Returns: + AudioSegment instance. + """ + # Get samples. + assert signal.channels == 1, ( + 'multiple-channel recordings not supported') + samples = signal.get_array_of_samples() + + # Convolve. + logging.info('applying %d order impulse response to a signal lasting %d ms', + len(impulse_response), len(signal)) + convolved_samples = scipy.signal.fftconvolve( + in1=samples, + in2=impulse_response, + mode='full').astype(np.int16) + logging.info('convolution computed') + + # Cast. + convolved_samples = array.array(signal.array_type, convolved_samples) + + # Verify. + logging.debug('signal length: %d samples', len(samples)) + logging.debug('convolved signal length: %d samples', len(convolved_samples)) + assert len(convolved_samples) > len(samples) + + # Generate convolved signal AudioSegment instance. + convolved_signal = pydub.AudioSegment( + data=convolved_samples, + metadata={ + 'sample_width': signal.sample_width, + 'frame_rate': signal.frame_rate, + 'frame_width': signal.frame_width, + 'channels': signal.channels, + }) + assert len(convolved_signal) > len(signal) + + return convolved_signal + + @classmethod + def Normalize(cls, signal): + """Normalizes a signal. + + Args: + signal: AudioSegment instance. + + Returns: + An AudioSegment instance. + """ + return signal.apply_gain(-signal.max_dBFS) + + @classmethod + def Copy(cls, signal): + """Makes a copy os a signal. + + Args: + signal: AudioSegment instance. + + Returns: + An AudioSegment instance. + """ + return pydub.AudioSegment( + data=signal.get_array_of_samples(), + metadata={ + 'sample_width': signal.sample_width, + 'frame_rate': signal.frame_rate, + 'frame_width': signal.frame_width, + 'channels': signal.channels, + }) + + @classmethod + def MixSignals(cls, signal, noise, target_snr=0.0, + pad_noise=MixPadding.NO_PADDING): + """Mixes |signal| and |noise| with a target SNR. + + Mix |signal| and |noise| with a desired SNR by scaling |noise|. + If the target SNR is +/- infinite, a copy of signal/noise is returned. + If |signal| is shorter than |noise|, the length of the mix equals that of + |signal|. Otherwise, the mix length depends on whether padding is applied. + When padding is not applied, that is |pad_noise| is set to NO_PADDING + (default), the mix length equals that of |noise| - i.e., |signal| is + truncated. Otherwise, |noise| is extended and the resulting mix has the same + length of |signal|. + + Args: + signal: AudioSegment instance (signal). + noise: AudioSegment instance (noise). + target_snr: float, numpy.Inf or -numpy.Inf (dB). + pad_noise: SignalProcessingUtils.MixPadding, default: NO_PADDING. + + Returns: + An AudioSegment instance. + """ + # Handle infinite target SNR. + if target_snr == -np.Inf: + # Return a copy of noise. + logging.warning('SNR = -Inf, returning noise') + return cls.Copy(noise) + elif target_snr == np.Inf: + # Return a copy of signal. + logging.warning('SNR = +Inf, returning signal') + return cls.Copy(signal) + + # Check signal and noise power. + signal_power = float(signal.dBFS) + noise_power = float(noise.dBFS) + if signal_power == -np.Inf: + logging.error('signal has -Inf power, cannot mix') + raise exceptions.SignalProcessingException( + 'cannot mix a signal with -Inf power') + if noise_power == -np.Inf: + logging.error('noise has -Inf power, cannot mix') + raise exceptions.SignalProcessingException( + 'cannot mix a signal with -Inf power') + + # Mix. + gain_db = signal_power - noise_power - target_snr + signal_duration = len(signal) + noise_duration = len(noise) + if signal_duration <= noise_duration: + # Ignore |pad_noise|, |noise| is truncated if longer that |signal|, the + # mix will have the same length of |signal|. + return signal.overlay(noise.apply_gain(gain_db)) + elif pad_noise == cls.MixPadding.NO_PADDING: + # |signal| is longer than |noise|, but no padding is applied to |noise|. + # Truncate |signal|. + return noise.overlay(signal, gain_during_overlay=gain_db) + elif pad_noise == cls.MixPadding.ZERO_PADDING: + # TODO(alessiob): Check that this works as expected. + return signal.overlay(noise.apply_gain(gain_db)) + elif pad_noise == cls.MixPadding.LOOP: + # |signal| is longer than |noise|, extend |noise| by looping. + return signal.overlay(noise.apply_gain(gain_db), loop=True) + else: + raise exceptions.SignalProcessingException('invalid padding type') diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing_unittest.py new file mode 100644 index 0000000000..30ada41fb9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/signal_processing_unittest.py @@ -0,0 +1,186 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the signal_processing module. +""" + +import unittest + +import numpy as np +import pydub + +from . import exceptions +from . import signal_processing + + +class TestSignalProcessing(unittest.TestCase): + """Unit tests for the signal_processing module. + """ + + def testMixSignals(self): + # Generate a template signal with which white noise can be generated. + silence = pydub.AudioSegment.silent(duration=1000, frame_rate=48000) + + # Generate two distinct AudioSegment instances with 1 second of white noise. + signal = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + silence) + noise = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + silence) + + # Extract samples. + signal_samples = signal.get_array_of_samples() + noise_samples = noise.get_array_of_samples() + + # Test target SNR -Inf (noise expected). + mix_neg_inf = signal_processing.SignalProcessingUtils.MixSignals( + signal, noise, -np.Inf) + self.assertTrue(len(noise), len(mix_neg_inf)) # Check duration. + mix_neg_inf_samples = mix_neg_inf.get_array_of_samples() + self.assertTrue( # Check samples. + all([x == y for x, y in zip(noise_samples, mix_neg_inf_samples)])) + + # Test target SNR 0.0 (different data expected). + mix_0 = signal_processing.SignalProcessingUtils.MixSignals( + signal, noise, 0.0) + self.assertTrue(len(signal), len(mix_0)) # Check duration. + self.assertTrue(len(noise), len(mix_0)) + mix_0_samples = mix_0.get_array_of_samples() + self.assertTrue( + any([x != y for x, y in zip(signal_samples, mix_0_samples)])) + self.assertTrue( + any([x != y for x, y in zip(noise_samples, mix_0_samples)])) + + # Test target SNR +Inf (signal expected). + mix_pos_inf = signal_processing.SignalProcessingUtils.MixSignals( + signal, noise, np.Inf) + self.assertTrue(len(signal), len(mix_pos_inf)) # Check duration. + mix_pos_inf_samples = mix_pos_inf.get_array_of_samples() + self.assertTrue( # Check samples. + all([x == y for x, y in zip(signal_samples, mix_pos_inf_samples)])) + + def testMixSignalsMinInfPower(self): + silence = pydub.AudioSegment.silent(duration=1000, frame_rate=48000) + signal = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + silence) + + with self.assertRaises(exceptions.SignalProcessingException): + _ = signal_processing.SignalProcessingUtils.MixSignals( + signal, silence, 0.0) + + with self.assertRaises(exceptions.SignalProcessingException): + _ = signal_processing.SignalProcessingUtils.MixSignals( + silence, signal, 0.0) + + def testMixSignalNoiseDifferentLengths(self): + # Test signals. + shorter = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + pydub.AudioSegment.silent(duration=1000, frame_rate=8000)) + longer = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + pydub.AudioSegment.silent(duration=2000, frame_rate=8000)) + + # When the signal is shorter than the noise, the mix length always equals + # that of the signal regardless of whether padding is applied. + # No noise padding, length of signal less than that of noise. + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=shorter, + noise=longer, + pad_noise=signal_processing.SignalProcessingUtils.MixPadding.NO_PADDING) + self.assertEqual(len(shorter), len(mix)) + # With noise padding, length of signal less than that of noise. + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=shorter, + noise=longer, + pad_noise=signal_processing.SignalProcessingUtils.MixPadding.ZERO_PADDING) + self.assertEqual(len(shorter), len(mix)) + + # When the signal is longer than the noise, the mix length depends on + # whether padding is applied. + # No noise padding, length of signal greater than that of noise. + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=longer, + noise=shorter, + pad_noise=signal_processing.SignalProcessingUtils.MixPadding.NO_PADDING) + self.assertEqual(len(shorter), len(mix)) + # With noise padding, length of signal greater than that of noise. + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=longer, + noise=shorter, + pad_noise=signal_processing.SignalProcessingUtils.MixPadding.ZERO_PADDING) + self.assertEqual(len(longer), len(mix)) + + def testMixSignalNoisePaddingTypes(self): + # Test signals. + shorter = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + pydub.AudioSegment.silent(duration=1000, frame_rate=8000)) + longer = signal_processing.SignalProcessingUtils.GeneratePureTone( + pydub.AudioSegment.silent(duration=2000, frame_rate=8000), 440.0) + + # Zero padding: expect pure tone only in 1-2s. + mix_zero_pad = signal_processing.SignalProcessingUtils.MixSignals( + signal=longer, + noise=shorter, + target_snr=-6, + pad_noise=signal_processing.SignalProcessingUtils.MixPadding.ZERO_PADDING) + + # Loop: expect pure tone plus noise in 1-2s. + mix_loop = signal_processing.SignalProcessingUtils.MixSignals( + signal=longer, + noise=shorter, + target_snr=-6, + pad_noise=signal_processing.SignalProcessingUtils.MixPadding.LOOP) + + def Energy(signal): + samples = signal_processing.SignalProcessingUtils.AudioSegmentToRawData( + signal).astype(np.float32) + return np.sum(samples * samples) + + e_mix_zero_pad = Energy(mix_zero_pad[-1000:]) + e_mix_loop = Energy(mix_loop[-1000:]) + self.assertLess(0, e_mix_zero_pad) + self.assertLess(e_mix_zero_pad, e_mix_loop) + + def testMixSignalSnr(self): + # Test signals. + tone_low = signal_processing.SignalProcessingUtils.GeneratePureTone( + pydub.AudioSegment.silent(duration=64, frame_rate=8000), 250.0) + tone_high = signal_processing.SignalProcessingUtils.GeneratePureTone( + pydub.AudioSegment.silent(duration=64, frame_rate=8000), 3000.0) + + def ToneAmplitudes(mix): + """Returns the amplitude of the coefficients #16 and #192, which + correspond to the tones at 250 and 3k Hz respectively.""" + mix_fft = np.absolute(signal_processing.SignalProcessingUtils.Fft(mix)) + return mix_fft[16], mix_fft[192] + + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=tone_low, + noise=tone_high, + target_snr=-6) + ampl_low, ampl_high = ToneAmplitudes(mix) + self.assertLess(ampl_low, ampl_high) + + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=tone_high, + noise=tone_low, + target_snr=-6) + ampl_low, ampl_high = ToneAmplitudes(mix) + self.assertLess(ampl_high, ampl_low) + + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=tone_low, + noise=tone_high, + target_snr=6) + ampl_low, ampl_high = ToneAmplitudes(mix) + self.assertLess(ampl_high, ampl_low) + + mix = signal_processing.SignalProcessingUtils.MixSignals( + signal=tone_high, + noise=tone_low, + target_snr=6) + ampl_low, ampl_high = ToneAmplitudes(mix) + self.assertLess(ampl_low, ampl_high) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py new file mode 100644 index 0000000000..e313bf34b4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py @@ -0,0 +1,422 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""APM module simulator. +""" + +import logging +import os + +from . import annotations +from . import data_access +from . import echo_path_simulation +from . import echo_path_simulation_factory +from . import eval_scores +from . import exceptions +from . import input_mixer +from . import input_signal_creator +from . import signal_processing +from . import test_data_generation + + +class ApmModuleSimulator(object): + """Audio processing module (APM) simulator class. + """ + + _TEST_DATA_GENERATOR_CLASSES = ( + test_data_generation.TestDataGenerator.REGISTERED_CLASSES) + _EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES + + _PREFIX_APM_CONFIG = 'apmcfg-' + _PREFIX_CAPTURE = 'capture-' + _PREFIX_RENDER = 'render-' + _PREFIX_ECHO_SIMULATOR = 'echosim-' + _PREFIX_TEST_DATA_GEN = 'datagen-' + _PREFIX_TEST_DATA_GEN_PARAMS = 'datagen_params-' + _PREFIX_SCORE = 'score-' + + def __init__(self, test_data_generator_factory, evaluation_score_factory, + ap_wrapper, evaluator, external_vads=None): + if external_vads is None: + external_vads = {} + self._test_data_generator_factory = test_data_generator_factory + self._evaluation_score_factory = evaluation_score_factory + self._audioproc_wrapper = ap_wrapper + self._evaluator = evaluator + self._annotator = annotations.AudioAnnotationsExtractor( + annotations.AudioAnnotationsExtractor.VadType.ENERGY_THRESHOLD | + annotations.AudioAnnotationsExtractor.VadType.WEBRTC_COMMON_AUDIO | + annotations.AudioAnnotationsExtractor.VadType.WEBRTC_APM, + external_vads + ) + + # Init. + self._test_data_generator_factory.SetOutputDirectoryPrefix( + self._PREFIX_TEST_DATA_GEN_PARAMS) + self._evaluation_score_factory.SetScoreFilenamePrefix( + self._PREFIX_SCORE) + + # Properties for each run. + self._base_output_path = None + self._output_cache_path = None + self._test_data_generators = None + self._evaluation_score_workers = None + self._config_filepaths = None + self._capture_input_filepaths = None + self._render_input_filepaths = None + self._echo_path_simulator_class = None + + @classmethod + def GetPrefixApmConfig(cls): + return cls._PREFIX_APM_CONFIG + + @classmethod + def GetPrefixCapture(cls): + return cls._PREFIX_CAPTURE + + @classmethod + def GetPrefixRender(cls): + return cls._PREFIX_RENDER + + @classmethod + def GetPrefixEchoSimulator(cls): + return cls._PREFIX_ECHO_SIMULATOR + + @classmethod + def GetPrefixTestDataGenerator(cls): + return cls._PREFIX_TEST_DATA_GEN + + @classmethod + def GetPrefixTestDataGeneratorParameters(cls): + return cls._PREFIX_TEST_DATA_GEN_PARAMS + + @classmethod + def GetPrefixScore(cls): + return cls._PREFIX_SCORE + + def Run(self, config_filepaths, capture_input_filepaths, + test_data_generator_names, eval_score_names, output_dir, + render_input_filepaths=None, echo_path_simulator_name=( + echo_path_simulation.NoEchoPathSimulator.NAME)): + """Runs the APM simulation. + + Initializes paths and required instances, then runs all the simulations. + The render input can be optionally added. If added, the number of capture + input audio tracks and the number of render input audio tracks have to be + equal. The two lists are used to form pairs of capture and render input. + + Args: + config_filepaths: set of APM configuration files to test. + capture_input_filepaths: set of capture input audio track files to test. + test_data_generator_names: set of test data generator names to test. + eval_score_names: set of evaluation score names to test. + output_dir: base path to the output directory for wav files and outcomes. + render_input_filepaths: set of render input audio track files to test. + echo_path_simulator_name: name of the echo path simulator to use when + render input is provided. + """ + assert render_input_filepaths is None or ( + len(capture_input_filepaths) == len(render_input_filepaths)), ( + 'render input set size not matching input set size') + assert render_input_filepaths is None or echo_path_simulator_name in ( + echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES), ( + 'invalid echo path simulator') + self._base_output_path = os.path.abspath(output_dir) + + # Output path used to cache the data shared across simulations. + self._output_cache_path = os.path.join(self._base_output_path, '_cache') + + # Instance test data generators. + self._test_data_generators = [self._test_data_generator_factory.GetInstance( + test_data_generators_class=( + self._TEST_DATA_GENERATOR_CLASSES[name])) for name in ( + test_data_generator_names)] + + # Instance evaluation score workers. + self._evaluation_score_workers = [ + self._evaluation_score_factory.GetInstance( + evaluation_score_class=self._EVAL_SCORE_WORKER_CLASSES[name]) for ( + name) in eval_score_names] + + # Set APM configuration file paths. + self._config_filepaths = self._CreatePathsCollection(config_filepaths) + + # Set probing signal file paths. + if render_input_filepaths is None: + # Capture input only. + self._capture_input_filepaths = self._CreatePathsCollection( + capture_input_filepaths) + self._render_input_filepaths = None + else: + # Set both capture and render input signals. + self._SetTestInputSignalFilePaths( + capture_input_filepaths, render_input_filepaths) + + # Set the echo path simulator class. + self._echo_path_simulator_class = ( + echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES[ + echo_path_simulator_name]) + + self._SimulateAll() + + def _SimulateAll(self): + """Runs all the simulations. + + Iterates over the combinations of APM configurations, probing signals, and + test data generators. This method is mainly responsible for the creation of + the cache and output directories required in order to call _Simulate(). + """ + without_render_input = self._render_input_filepaths is None + + # Try different APM config files. + for config_name in self._config_filepaths: + config_filepath = self._config_filepaths[config_name] + + # Try different capture-render pairs. + for capture_input_name in self._capture_input_filepaths: + # Output path for the capture signal annotations. + capture_annotations_cache_path = os.path.join( + self._output_cache_path, + self._PREFIX_CAPTURE + capture_input_name) + data_access.MakeDirectory(capture_annotations_cache_path) + + # Capture. + capture_input_filepath = self._capture_input_filepaths[ + capture_input_name] + if not os.path.exists(capture_input_filepath): + # If the input signal file does not exist, try to create using the + # available input signal creators. + self._CreateInputSignal(capture_input_filepath) + assert os.path.exists(capture_input_filepath) + self._ExtractCaptureAnnotations( + capture_input_filepath, capture_annotations_cache_path) + + # Render and simulated echo path (optional). + render_input_filepath = None if without_render_input else ( + self._render_input_filepaths[capture_input_name]) + render_input_name = '(none)' if without_render_input else ( + self._ExtractFileName(render_input_filepath)) + echo_path_simulator = ( + echo_path_simulation_factory.EchoPathSimulatorFactory.GetInstance( + self._echo_path_simulator_class, render_input_filepath)) + + # Try different test data generators. + for test_data_generators in self._test_data_generators: + logging.info('APM config preset: <%s>, capture: <%s>, render: <%s>,' + 'test data generator: <%s>, echo simulator: <%s>', + config_name, capture_input_name, render_input_name, + test_data_generators.NAME, echo_path_simulator.NAME) + + # Output path for the generated test data. + test_data_cache_path = os.path.join( + capture_annotations_cache_path, + self._PREFIX_TEST_DATA_GEN + test_data_generators.NAME) + data_access.MakeDirectory(test_data_cache_path) + logging.debug('test data cache path: <%s>', test_data_cache_path) + + # Output path for the echo simulator and APM input mixer output. + echo_test_data_cache_path = os.path.join( + test_data_cache_path, 'echosim-{}'.format( + echo_path_simulator.NAME)) + data_access.MakeDirectory(echo_test_data_cache_path) + logging.debug('echo test data cache path: <%s>', + echo_test_data_cache_path) + + # Full output path. + output_path = os.path.join( + self._base_output_path, + self._PREFIX_APM_CONFIG + config_name, + self._PREFIX_CAPTURE + capture_input_name, + self._PREFIX_RENDER + render_input_name, + self._PREFIX_ECHO_SIMULATOR + echo_path_simulator.NAME, + self._PREFIX_TEST_DATA_GEN + test_data_generators.NAME) + data_access.MakeDirectory(output_path) + logging.debug('output path: <%s>', output_path) + + self._Simulate(test_data_generators, capture_input_filepath, + render_input_filepath, test_data_cache_path, + echo_test_data_cache_path, output_path, + config_filepath, echo_path_simulator) + + @staticmethod + def _CreateInputSignal(input_signal_filepath): + """Creates a missing input signal file. + + The file name is parsed to extract input signal creator and params. If a + creator is matched and the parameters are valid, a new signal is generated + and written in |input_signal_filepath|. + + Args: + input_signal_filepath: Path to the input signal audio file to write. + + Raises: + InputSignalCreatorException + """ + filename = os.path.splitext(os.path.split(input_signal_filepath)[-1])[0] + filename_parts = filename.split('-') + + if len(filename_parts) < 2: + raise exceptions.InputSignalCreatorException( + 'Cannot parse input signal file name') + + signal, metadata = input_signal_creator.InputSignalCreator.Create( + filename_parts[0], filename_parts[1].split('_')) + + signal_processing.SignalProcessingUtils.SaveWav( + input_signal_filepath, signal) + data_access.Metadata.SaveFileMetadata(input_signal_filepath, metadata) + + def _ExtractCaptureAnnotations(self, input_filepath, output_path, + annotation_name=""): + self._annotator.Extract(input_filepath) + self._annotator.Save(output_path, annotation_name) + + def _Simulate(self, test_data_generators, clean_capture_input_filepath, + render_input_filepath, test_data_cache_path, + echo_test_data_cache_path, output_path, config_filepath, + echo_path_simulator): + """Runs a single set of simulation. + + Simulates a given combination of APM configuration, probing signal, and + test data generator. It iterates over the test data generator + internal configurations. + + Args: + test_data_generators: TestDataGenerator instance. + clean_capture_input_filepath: capture input audio track file to be + processed by a test data generator and + not affected by echo. + render_input_filepath: render input audio track file to test. + test_data_cache_path: path for the generated test audio track files. + echo_test_data_cache_path: path for the echo simulator. + output_path: base output path for the test data generator. + config_filepath: APM configuration file to test. + echo_path_simulator: EchoPathSimulator instance. + """ + # Generate pairs of noisy input and reference signal files. + test_data_generators.Generate( + input_signal_filepath=clean_capture_input_filepath, + test_data_cache_path=test_data_cache_path, + base_output_path=output_path) + + # Extract metadata linked to the clean input file (if any). + apm_input_metadata = None + try: + apm_input_metadata = data_access.Metadata.LoadFileMetadata( + clean_capture_input_filepath) + except IOError as e: + apm_input_metadata = {} + apm_input_metadata['test_data_gen_name'] = test_data_generators.NAME + apm_input_metadata['test_data_gen_config'] = None + + # For each test data pair, simulate a call and evaluate. + for config_name in test_data_generators.config_names: + logging.info(' - test data generator config: <%s>', config_name) + apm_input_metadata['test_data_gen_config'] = config_name + + # Paths to the test data generator output. + # Note that the reference signal does not depend on the render input + # which is optional. + noisy_capture_input_filepath = ( + test_data_generators.noisy_signal_filepaths[config_name]) + reference_signal_filepath = ( + test_data_generators.reference_signal_filepaths[config_name]) + + # Output path for the evaluation (e.g., APM output file). + evaluation_output_path = test_data_generators.apm_output_paths[ + config_name] + + # Paths to the APM input signals. + echo_path_filepath = echo_path_simulator.Simulate( + echo_test_data_cache_path) + apm_input_filepath = input_mixer.ApmInputMixer.Mix( + echo_test_data_cache_path, noisy_capture_input_filepath, + echo_path_filepath) + + # Extract annotations for the APM input mix. + apm_input_basepath, apm_input_filename = os.path.split( + apm_input_filepath) + self._ExtractCaptureAnnotations( + apm_input_filepath, apm_input_basepath, + os.path.splitext(apm_input_filename)[0] + '-') + + # Simulate a call using APM. + self._audioproc_wrapper.Run( + config_filepath=config_filepath, + capture_input_filepath=apm_input_filepath, + render_input_filepath=render_input_filepath, + output_path=evaluation_output_path) + + try: + # Evaluate. + self._evaluator.Run( + evaluation_score_workers=self._evaluation_score_workers, + apm_input_metadata=apm_input_metadata, + apm_output_filepath=self._audioproc_wrapper.output_filepath, + reference_input_filepath=reference_signal_filepath, + output_path=evaluation_output_path) + + # Save simulation metadata. + data_access.Metadata.SaveAudioTestDataPaths( + output_path=evaluation_output_path, + clean_capture_input_filepath=clean_capture_input_filepath, + echo_free_capture_filepath=noisy_capture_input_filepath, + echo_filepath=echo_path_filepath, + render_filepath=render_input_filepath, + capture_filepath=apm_input_filepath, + apm_output_filepath=self._audioproc_wrapper.output_filepath, + apm_reference_filepath=reference_signal_filepath) + except exceptions.EvaluationScoreException as e: + logging.warning('the evaluation failed: %s', e.message) + continue + + def _SetTestInputSignalFilePaths(self, capture_input_filepaths, + render_input_filepaths): + """Sets input and render input file paths collections. + + Pairs the input and render input files by storing the file paths into two + collections. The key is the file name of the input file. + + Args: + capture_input_filepaths: list of file paths. + render_input_filepaths: list of file paths. + """ + self._capture_input_filepaths = {} + self._render_input_filepaths = {} + assert len(capture_input_filepaths) == len(render_input_filepaths) + for capture_input_filepath, render_input_filepath in zip( + capture_input_filepaths, render_input_filepaths): + name = self._ExtractFileName(capture_input_filepath) + self._capture_input_filepaths[name] = os.path.abspath( + capture_input_filepath) + self._render_input_filepaths[name] = os.path.abspath( + render_input_filepath) + + @classmethod + def _CreatePathsCollection(cls, filepaths): + """Creates a collection of file paths. + + Given a list of file paths, makes a collection with one item for each file + path. The value is absolute path, the key is the file name without + extenstion. + + Args: + filepaths: list of file paths. + + Returns: + A dict. + """ + filepaths_collection = {} + for filepath in filepaths: + name = cls._ExtractFileName(filepath) + filepaths_collection[name] = os.path.abspath(filepath) + return filepaths_collection + + @classmethod + def _ExtractFileName(cls, filepath): + return os.path.splitext(os.path.split(filepath)[-1])[0] diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py new file mode 100644 index 0000000000..c7ebcbc87a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py @@ -0,0 +1,203 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the simulation module. +""" + +import logging +import os +import shutil +import sys +import tempfile +import unittest + +SRC = os.path.abspath(os.path.join( + os.path.dirname((__file__)), os.pardir, os.pardir, os.pardir, os.pardir)) +sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) + +import mock +import pydub + +from . import audioproc_wrapper +from . import eval_scores_factory +from . import evaluation +from . import external_vad +from . import signal_processing +from . import simulation +from . import test_data_generation_factory + + +class TestApmModuleSimulator(unittest.TestCase): + """Unit tests for the ApmModuleSimulator class. + """ + + def setUp(self): + """Create temporary folders and fake audio track.""" + self._output_path = tempfile.mkdtemp() + self._tmp_path = tempfile.mkdtemp() + + silence = pydub.AudioSegment.silent(duration=1000, frame_rate=48000) + fake_signal = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + silence) + self._fake_audio_track_path = os.path.join(self._output_path, 'fake.wav') + signal_processing.SignalProcessingUtils.SaveWav( + self._fake_audio_track_path, fake_signal) + + def tearDown(self): + """Recursively delete temporary folders.""" + shutil.rmtree(self._output_path) + shutil.rmtree(self._tmp_path) + + def testSimulation(self): + # Instance dependencies to mock and inject. + ap_wrapper = audioproc_wrapper.AudioProcWrapper( + audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH) + evaluator = evaluation.ApmModuleEvaluator() + ap_wrapper.Run = mock.MagicMock(name='Run') + evaluator.Run = mock.MagicMock(name='Run') + + # Instance non-mocked dependencies. + test_data_generator_factory = ( + test_data_generation_factory.TestDataGeneratorFactory( + aechen_ir_database_path='', + noise_tracks_path='', + copy_with_identity=False)) + evaluation_score_factory = eval_scores_factory.EvaluationScoreWorkerFactory( + polqa_tool_bin_path=os.path.join( + os.path.dirname(__file__), 'fake_polqa')) + + # Instance simulator. + simulator = simulation.ApmModuleSimulator( + test_data_generator_factory=test_data_generator_factory, + evaluation_score_factory=evaluation_score_factory, + ap_wrapper=ap_wrapper, + evaluator=evaluator, + external_vads={'fake': external_vad.ExternalVad(os.path.join( + os.path.dirname(__file__), 'fake_external_vad.py'), 'fake')} + ) + + # What to simulate. + config_files = ['apm_configs/default.json'] + input_files = [self._fake_audio_track_path] + test_data_generators = ['identity', 'white_noise'] + eval_scores = ['audio_level_mean', 'polqa'] + + # Run all simulations. + simulator.Run( + config_filepaths=config_files, + capture_input_filepaths=input_files, + test_data_generator_names=test_data_generators, + eval_score_names=eval_scores, + output_dir=self._output_path) + + # Check. + # TODO(alessiob): Once the TestDataGenerator classes can be configured by + # the client code (e.g., number of SNR pairs for the white noise test data + # generator), the exact number of calls to ap_wrapper.Run and evaluator.Run + # is known; use that with assertEqual. + min_number_of_simulations = len(config_files) * len(input_files) * len( + test_data_generators) + self.assertGreaterEqual(len(ap_wrapper.Run.call_args_list), + min_number_of_simulations) + self.assertGreaterEqual(len(evaluator.Run.call_args_list), + min_number_of_simulations) + + def testInputSignalCreation(self): + # Instance simulator. + simulator = simulation.ApmModuleSimulator( + test_data_generator_factory=( + test_data_generation_factory.TestDataGeneratorFactory( + aechen_ir_database_path='', + noise_tracks_path='', + copy_with_identity=False)), + evaluation_score_factory=( + eval_scores_factory.EvaluationScoreWorkerFactory( + polqa_tool_bin_path=os.path.join( + os.path.dirname(__file__), 'fake_polqa'))), + ap_wrapper=audioproc_wrapper.AudioProcWrapper( + audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH), + evaluator=evaluation.ApmModuleEvaluator()) + + # Inexistent input files to be silently created. + input_files = [ + os.path.join(self._tmp_path, 'pure_tone-440_1000.wav'), + os.path.join(self._tmp_path, 'pure_tone-1000_500.wav'), + ] + self.assertFalse(any([os.path.exists(input_file) for input_file in ( + input_files)])) + + # The input files are created during the simulation. + simulator.Run( + config_filepaths=['apm_configs/default.json'], + capture_input_filepaths=input_files, + test_data_generator_names=['identity'], + eval_score_names=['audio_level_peak'], + output_dir=self._output_path) + self.assertTrue(all([os.path.exists(input_file) for input_file in ( + input_files)])) + + def testPureToneGenerationWithTotalHarmonicDistorsion(self): + logging.warning = mock.MagicMock(name='warning') + + # Instance simulator. + simulator = simulation.ApmModuleSimulator( + test_data_generator_factory=( + test_data_generation_factory.TestDataGeneratorFactory( + aechen_ir_database_path='', + noise_tracks_path='', + copy_with_identity=False)), + evaluation_score_factory=( + eval_scores_factory.EvaluationScoreWorkerFactory( + polqa_tool_bin_path=os.path.join( + os.path.dirname(__file__), 'fake_polqa'))), + ap_wrapper=audioproc_wrapper.AudioProcWrapper( + audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH), + evaluator=evaluation.ApmModuleEvaluator()) + + # What to simulate. + config_files = ['apm_configs/default.json'] + input_files = [os.path.join(self._tmp_path, 'pure_tone-440_1000.wav')] + eval_scores = ['thd'] + + # Should work. + simulator.Run( + config_filepaths=config_files, + capture_input_filepaths=input_files, + test_data_generator_names=['identity'], + eval_score_names=eval_scores, + output_dir=self._output_path) + self.assertFalse(logging.warning.called) + + # Warning expected. + simulator.Run( + config_filepaths=config_files, + capture_input_filepaths=input_files, + test_data_generator_names=['white_noise'], # Not allowed with THD. + eval_score_names=eval_scores, + output_dir=self._output_path) + logging.warning.assert_called_with('the evaluation failed: %s', ( + 'The THD score cannot be used with any test data generator other than ' + '"identity"')) + + # # Init. + # generator = test_data_generation.IdentityTestDataGenerator('tmp') + # input_signal_filepath = os.path.join( + # self._test_data_cache_path, 'pure_tone-440_1000.wav') + + # # Check that the input signal is generated. + # self.assertFalse(os.path.exists(input_signal_filepath)) + # generator.Generate( + # input_signal_filepath=input_signal_filepath, + # test_data_cache_path=self._test_data_cache_path, + # base_output_path=self._base_output_path) + # self.assertTrue(os.path.exists(input_signal_filepath)) + + # # Check input signal properties. + # input_signal = signal_processing.SignalProcessingUtils.LoadWav( + # input_signal_filepath) + # self.assertEqual(1000, len(input_signal)) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation.py new file mode 100644 index 0000000000..dac4328588 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation.py @@ -0,0 +1,511 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Test data generators producing signals pairs intended to be used to +test the APM module. Each pair consists of a noisy input and a reference signal. +The former is used as APM input and it is generated by adding noise to a +clean audio track. The reference is the expected APM output. + +Throughout this file, the following naming convention is used: + - input signal: the clean signal (e.g., speech), + - noise signal: the noise to be summed up to the input signal (e.g., white + noise, Gaussian noise), + - noisy signal: input + noise. +The noise signal may or may not be a function of the clean signal. For +instance, white noise is independently generated, whereas reverberation is +obtained by convolving the input signal with an impulse response. +""" + +import logging +import os +import shutil +import sys + +try: + import scipy.io +except ImportError: + logging.critical('Cannot import the third-party Python package scipy') + sys.exit(1) + +from . import data_access +from . import exceptions +from . import signal_processing + + +class TestDataGenerator(object): + """Abstract class responsible for the generation of noisy signals. + + Given a clean signal, it generates two streams named noisy signal and + reference. The former is the clean signal deteriorated by the noise source, + the latter goes through the same deterioration process, but more "gently". + Noisy signal and reference are produced so that the reference is the signal + expected at the output of the APM module when the latter is fed with the noisy + signal. + + An test data generator generates one or more pairs. + """ + + NAME = None + REGISTERED_CLASSES = {} + + def __init__(self, output_directory_prefix): + self._output_directory_prefix = output_directory_prefix + # Init dictionaries with one entry for each test data generator + # configuration (e.g., different SNRs). + # Noisy audio track files (stored separately in a cache folder). + self._noisy_signal_filepaths = None + # Path to be used for the APM simulation output files. + self._apm_output_paths = None + # Reference audio track files (stored separately in a cache folder). + self._reference_signal_filepaths = None + self.Clear() + + @classmethod + def RegisterClass(cls, class_to_register): + """Registers a TestDataGenerator implementation. + + Decorator to automatically register the classes that extend + TestDataGenerator. + Example usage: + + @TestDataGenerator.RegisterClass + class IdentityGenerator(TestDataGenerator): + pass + """ + cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register + return class_to_register + + @property + def config_names(self): + return self._noisy_signal_filepaths.keys() + + @property + def noisy_signal_filepaths(self): + return self._noisy_signal_filepaths + + @property + def apm_output_paths(self): + return self._apm_output_paths + + @property + def reference_signal_filepaths(self): + return self._reference_signal_filepaths + + def Generate( + self, input_signal_filepath, test_data_cache_path, base_output_path): + """Generates a set of noisy input and reference audiotrack file pairs. + + This method initializes an empty set of pairs and calls the _Generate() + method implemented in a concrete class. + + Args: + input_signal_filepath: path to the clean input audio track file. + test_data_cache_path: path to the cache of the generated audio track + files. + base_output_path: base path where output is written. + """ + self.Clear() + self._Generate( + input_signal_filepath, test_data_cache_path, base_output_path) + + def Clear(self): + """Clears the generated output path dictionaries. + """ + self._noisy_signal_filepaths = {} + self._apm_output_paths = {} + self._reference_signal_filepaths = {} + + def _Generate( + self, input_signal_filepath, test_data_cache_path, base_output_path): + """Abstract method to be implemented in each concrete class. + """ + raise NotImplementedError() + + def _AddNoiseSnrPairs(self, base_output_path, noisy_mix_filepaths, + snr_value_pairs): + """Adds noisy-reference signal pairs. + + Args: + base_output_path: noisy tracks base output path. + noisy_mix_filepaths: nested dictionary of noisy signal paths organized + by noisy track name and SNR level. + snr_value_pairs: list of SNR pairs. + """ + for noise_track_name in noisy_mix_filepaths: + for snr_noisy, snr_refence in snr_value_pairs: + config_name = '{0}_{1:d}_{2:d}_SNR'.format( + noise_track_name, snr_noisy, snr_refence) + output_path = self._MakeDir(base_output_path, config_name) + self._AddNoiseReferenceFilesPair( + config_name=config_name, + noisy_signal_filepath=noisy_mix_filepaths[ + noise_track_name][snr_noisy], + reference_signal_filepath=noisy_mix_filepaths[ + noise_track_name][snr_refence], + output_path=output_path) + + def _AddNoiseReferenceFilesPair(self, config_name, noisy_signal_filepath, + reference_signal_filepath, output_path): + """Adds one noisy-reference signal pair. + + Args: + config_name: name of the APM configuration. + noisy_signal_filepath: path to noisy audio track file. + reference_signal_filepath: path to reference audio track file. + output_path: APM output path. + """ + assert config_name not in self._noisy_signal_filepaths + self._noisy_signal_filepaths[config_name] = os.path.abspath( + noisy_signal_filepath) + self._apm_output_paths[config_name] = os.path.abspath(output_path) + self._reference_signal_filepaths[config_name] = os.path.abspath( + reference_signal_filepath) + + def _MakeDir(self, base_output_path, test_data_generator_config_name): + output_path = os.path.join( + base_output_path, + self._output_directory_prefix + test_data_generator_config_name) + data_access.MakeDirectory(output_path) + return output_path + + +@TestDataGenerator.RegisterClass +class IdentityTestDataGenerator(TestDataGenerator): + """Generator that adds no noise. + + Both the noisy and the reference signals are the input signal. + """ + + NAME = 'identity' + + def __init__(self, output_directory_prefix, copy_with_identity): + TestDataGenerator.__init__(self, output_directory_prefix) + self._copy_with_identity = copy_with_identity + + @property + def copy_with_identity(self): + return self._copy_with_identity + + def _Generate( + self, input_signal_filepath, test_data_cache_path, base_output_path): + config_name = 'default' + output_path = self._MakeDir(base_output_path, config_name) + + if self._copy_with_identity: + input_signal_filepath_new = os.path.join( + test_data_cache_path, os.path.split(input_signal_filepath)[1]) + logging.info('copying ' + input_signal_filepath + ' to ' + ( + input_signal_filepath_new)) + shutil.copy(input_signal_filepath, input_signal_filepath_new) + input_signal_filepath = input_signal_filepath_new + + self._AddNoiseReferenceFilesPair( + config_name=config_name, + noisy_signal_filepath=input_signal_filepath, + reference_signal_filepath=input_signal_filepath, + output_path=output_path) + + +@TestDataGenerator.RegisterClass +class WhiteNoiseTestDataGenerator(TestDataGenerator): + """Generator that adds white noise. + """ + + NAME = 'white_noise' + + # Each pair indicates the clean vs. noisy and reference vs. noisy SNRs. + # The reference (second value of each pair) always has a lower amount of noise + # - i.e., the SNR is 10 dB higher. + _SNR_VALUE_PAIRS = [ + [20, 30], # Smallest noise. + [10, 20], + [5, 15], + [0, 10], # Largest noise. + ] + + _NOISY_SIGNAL_FILENAME_TEMPLATE = 'noise_{0:d}_SNR.wav' + + def __init__(self, output_directory_prefix): + TestDataGenerator.__init__(self, output_directory_prefix) + + def _Generate( + self, input_signal_filepath, test_data_cache_path, base_output_path): + # Load the input signal. + input_signal = signal_processing.SignalProcessingUtils.LoadWav( + input_signal_filepath) + + # Create the noise track. + noise_signal = signal_processing.SignalProcessingUtils.GenerateWhiteNoise( + input_signal) + + # Create the noisy mixes (once for each unique SNR value). + noisy_mix_filepaths = {} + snr_values = set([snr for pair in self._SNR_VALUE_PAIRS for snr in pair]) + for snr in snr_values: + noisy_signal_filepath = os.path.join( + test_data_cache_path, + self._NOISY_SIGNAL_FILENAME_TEMPLATE.format(snr)) + + # Create and save if not done. + if not os.path.exists(noisy_signal_filepath): + # Create noisy signal. + noisy_signal = signal_processing.SignalProcessingUtils.MixSignals( + input_signal, noise_signal, snr) + + # Save. + signal_processing.SignalProcessingUtils.SaveWav( + noisy_signal_filepath, noisy_signal) + + # Add file to the collection of mixes. + noisy_mix_filepaths[snr] = noisy_signal_filepath + + # Add all the noisy-reference signal pairs. + for snr_noisy, snr_refence in self._SNR_VALUE_PAIRS: + config_name = '{0:d}_{1:d}_SNR'.format(snr_noisy, snr_refence) + output_path = self._MakeDir(base_output_path, config_name) + self._AddNoiseReferenceFilesPair( + config_name=config_name, + noisy_signal_filepath=noisy_mix_filepaths[snr_noisy], + reference_signal_filepath=noisy_mix_filepaths[snr_refence], + output_path=output_path) + + +# TODO(alessiob): remove comment when class implemented. +# @TestDataGenerator.RegisterClass +class NarrowBandNoiseTestDataGenerator(TestDataGenerator): + """Generator that adds narrow-band noise. + """ + + NAME = 'narrow_band_noise' + + def __init__(self, output_directory_prefix): + TestDataGenerator.__init__(self, output_directory_prefix) + + def _Generate( + self, input_signal_filepath, test_data_cache_path, base_output_path): + # TODO(alessiob): implement. + pass + + +@TestDataGenerator.RegisterClass +class AdditiveNoiseTestDataGenerator(TestDataGenerator): + """Generator that adds noise loops. + + This generator uses all the wav files in a given path (default: noise_tracks/) + and mixes them to the clean speech with different target SNRs (hard-coded). + """ + + NAME = 'additive_noise' + _NOISY_SIGNAL_FILENAME_TEMPLATE = '{0}_{1:d}_SNR.wav' + + DEFAULT_NOISE_TRACKS_PATH = os.path.join( + os.path.dirname(__file__), os.pardir, 'noise_tracks') + + # TODO(alessiob): Make the list of SNR pairs customizable. + # Each pair indicates the clean vs. noisy and reference vs. noisy SNRs. + # The reference (second value of each pair) always has a lower amount of noise + # - i.e., the SNR is 10 dB higher. + _SNR_VALUE_PAIRS = [ + [20, 30], # Smallest noise. + [10, 20], + [5, 15], + [0, 10], # Largest noise. + ] + + def __init__(self, output_directory_prefix, noise_tracks_path): + TestDataGenerator.__init__(self, output_directory_prefix) + self._noise_tracks_path = noise_tracks_path + self._noise_tracks_file_names = [n for n in os.listdir( + self._noise_tracks_path) if n.lower().endswith('.wav')] + if len(self._noise_tracks_file_names) == 0: + raise exceptions.InitializationException( + 'No wav files found in the noise tracks path %s' % ( + self._noise_tracks_path)) + + def _Generate( + self, input_signal_filepath, test_data_cache_path, base_output_path): + """Generates test data pairs using environmental noise. + + For each noise track and pair of SNR values, the following two audio tracks + are created: the noisy signal and the reference signal. The former is + obtained by mixing the (clean) input signal to the corresponding noise + track enforcing the target SNR. + """ + # Init. + snr_values = set([snr for pair in self._SNR_VALUE_PAIRS for snr in pair]) + + # Load the input signal. + input_signal = signal_processing.SignalProcessingUtils.LoadWav( + input_signal_filepath) + + noisy_mix_filepaths = {} + for noise_track_filename in self._noise_tracks_file_names: + # Load the noise track. + noise_track_name, _ = os.path.splitext(noise_track_filename) + noise_track_filepath = os.path.join( + self._noise_tracks_path, noise_track_filename) + if not os.path.exists(noise_track_filepath): + logging.error('cannot find the <%s> noise track', noise_track_filename) + raise exceptions.FileNotFoundError() + + noise_signal = signal_processing.SignalProcessingUtils.LoadWav( + noise_track_filepath) + + # Create the noisy mixes (once for each unique SNR value). + noisy_mix_filepaths[noise_track_name] = {} + for snr in snr_values: + noisy_signal_filepath = os.path.join( + test_data_cache_path, + self._NOISY_SIGNAL_FILENAME_TEMPLATE.format(noise_track_name, snr)) + + # Create and save if not done. + if not os.path.exists(noisy_signal_filepath): + # Create noisy signal. + noisy_signal = signal_processing.SignalProcessingUtils.MixSignals( + input_signal, noise_signal, snr, + pad_noise=signal_processing.SignalProcessingUtils.MixPadding.LOOP) + + # Save. + signal_processing.SignalProcessingUtils.SaveWav( + noisy_signal_filepath, noisy_signal) + + # Add file to the collection of mixes. + noisy_mix_filepaths[noise_track_name][snr] = noisy_signal_filepath + + # Add all the noise-SNR pairs. + self._AddNoiseSnrPairs( + base_output_path, noisy_mix_filepaths, self._SNR_VALUE_PAIRS) + + +@TestDataGenerator.RegisterClass +class ReverberationTestDataGenerator(TestDataGenerator): + """Generator that adds reverberation noise. + + TODO(alessiob): Make this class more generic since the impulse response can be + anything (not just reverberation); call it e.g., + ConvolutionalNoiseTestDataGenerator. + """ + + NAME = 'reverberation' + + _IMPULSE_RESPONSES = { + 'lecture': 'air_binaural_lecture_0_0_1.mat', # Long echo. + 'booth': 'air_binaural_booth_0_0_1.mat', # Short echo. + } + _MAX_IMPULSE_RESPONSE_LENGTH = None + + # Each pair indicates the clean vs. noisy and reference vs. noisy SNRs. + # The reference (second value of each pair) always has a lower amount of noise + # - i.e., the SNR is 5 dB higher. + _SNR_VALUE_PAIRS = [ + [3, 8], # Smallest noise. + [-3, 2], # Largest noise. + ] + + _NOISE_TRACK_FILENAME_TEMPLATE = '{0}.wav' + _NOISY_SIGNAL_FILENAME_TEMPLATE = '{0}_{1:d}_SNR.wav' + + def __init__(self, output_directory_prefix, aechen_ir_database_path): + TestDataGenerator.__init__(self, output_directory_prefix) + self._aechen_ir_database_path = aechen_ir_database_path + + def _Generate( + self, input_signal_filepath, test_data_cache_path, base_output_path): + """Generates test data pairs using reverberation noise. + + For each impulse response, one noise track is created. For each impulse + response and pair of SNR values, the following 2 audio tracks are + created: the noisy signal and the reference signal. The former is + obtained by mixing the (clean) input signal to the corresponding noise + track enforcing the target SNR. + """ + # Init. + snr_values = set([snr for pair in self._SNR_VALUE_PAIRS for snr in pair]) + + # Load the input signal. + input_signal = signal_processing.SignalProcessingUtils.LoadWav( + input_signal_filepath) + + noisy_mix_filepaths = {} + for impulse_response_name in self._IMPULSE_RESPONSES: + noise_track_filename = self._NOISE_TRACK_FILENAME_TEMPLATE.format( + impulse_response_name) + noise_track_filepath = os.path.join( + test_data_cache_path, noise_track_filename) + noise_signal = None + try: + # Load noise track. + noise_signal = signal_processing.SignalProcessingUtils.LoadWav( + noise_track_filepath) + except exceptions.FileNotFoundError: + # Generate noise track by applying the impulse response. + impulse_response_filepath = os.path.join( + self._aechen_ir_database_path, + self._IMPULSE_RESPONSES[impulse_response_name]) + noise_signal = self._GenerateNoiseTrack( + noise_track_filepath, input_signal, impulse_response_filepath) + assert noise_signal is not None + + # Create the noisy mixes (once for each unique SNR value). + noisy_mix_filepaths[impulse_response_name] = {} + for snr in snr_values: + noisy_signal_filepath = os.path.join( + test_data_cache_path, + self._NOISY_SIGNAL_FILENAME_TEMPLATE.format( + impulse_response_name, snr)) + + # Create and save if not done. + if not os.path.exists(noisy_signal_filepath): + # Create noisy signal. + noisy_signal = signal_processing.SignalProcessingUtils.MixSignals( + input_signal, noise_signal, snr) + + # Save. + signal_processing.SignalProcessingUtils.SaveWav( + noisy_signal_filepath, noisy_signal) + + # Add file to the collection of mixes. + noisy_mix_filepaths[impulse_response_name][snr] = noisy_signal_filepath + + # Add all the noise-SNR pairs. + self._AddNoiseSnrPairs(base_output_path, noisy_mix_filepaths, + self._SNR_VALUE_PAIRS) + + def _GenerateNoiseTrack(self, noise_track_filepath, input_signal, + impulse_response_filepath): + """Generates noise track. + + Generate a signal by convolving input_signal with the impulse response in + impulse_response_filepath; then save to noise_track_filepath. + + Args: + noise_track_filepath: output file path for the noise track. + input_signal: (clean) input signal samples. + impulse_response_filepath: impulse response file path. + + Returns: + AudioSegment instance. + """ + # Load impulse response. + data = scipy.io.loadmat(impulse_response_filepath) + impulse_response = data['h_air'].flatten() + if self._MAX_IMPULSE_RESPONSE_LENGTH is not None: + logging.info('truncating impulse response from %d to %d samples', + len(impulse_response), self._MAX_IMPULSE_RESPONSE_LENGTH) + impulse_response = impulse_response[:self._MAX_IMPULSE_RESPONSE_LENGTH] + + # Apply impulse response. + processed_signal = ( + signal_processing.SignalProcessingUtils.ApplyImpulseResponse( + input_signal, impulse_response)) + + # Save. + signal_processing.SignalProcessingUtils.SaveWav( + noise_track_filepath, processed_signal) + + return processed_signal diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_factory.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_factory.py new file mode 100644 index 0000000000..c80d150228 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_factory.py @@ -0,0 +1,71 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""TestDataGenerator factory class. +""" + +import logging + +from . import exceptions +from . import test_data_generation + + +class TestDataGeneratorFactory(object): + """Factory class used to create test data generators. + + Usage: Create a factory passing parameters to the ctor with which the + generators will be produced. + """ + + def __init__(self, aechen_ir_database_path, noise_tracks_path, + copy_with_identity): + """Ctor. + + Args: + aechen_ir_database_path: Path to the Aechen Impulse Response database. + noise_tracks_path: Path to the noise tracks to add. + copy_with_identity: Flag indicating whether the identity generator has to + make copies of the clean speech input files. + """ + self._output_directory_prefix = None + self._aechen_ir_database_path = aechen_ir_database_path + self._noise_tracks_path = noise_tracks_path + self._copy_with_identity = copy_with_identity + + def SetOutputDirectoryPrefix(self, prefix): + self._output_directory_prefix = prefix + + def GetInstance(self, test_data_generators_class): + """Creates an TestDataGenerator instance given a class object. + + Args: + test_data_generators_class: TestDataGenerator class object (not an + instance). + + Returns: + TestDataGenerator instance. + """ + if self._output_directory_prefix is None: + raise exceptions.InitializationException( + 'The output directory prefix for test data generators is not set') + logging.debug('factory producing %s', test_data_generators_class) + + if test_data_generators_class == ( + test_data_generation.IdentityTestDataGenerator): + return test_data_generation.IdentityTestDataGenerator( + self._output_directory_prefix, self._copy_with_identity) + elif test_data_generators_class == ( + test_data_generation.ReverberationTestDataGenerator): + return test_data_generation.ReverberationTestDataGenerator( + self._output_directory_prefix, self._aechen_ir_database_path) + elif test_data_generators_class == ( + test_data_generation.AdditiveNoiseTestDataGenerator): + return test_data_generation.AdditiveNoiseTestDataGenerator( + self._output_directory_prefix, self._noise_tracks_path) + else: + return test_data_generators_class(self._output_directory_prefix) diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_unittest.py b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_unittest.py new file mode 100644 index 0000000000..b0d003dbe8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_unittest.py @@ -0,0 +1,206 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the test_data_generation module. +""" + +import os +import shutil +import tempfile +import unittest + +import numpy as np +import scipy.io + +from . import test_data_generation +from . import test_data_generation_factory +from . import signal_processing + + +class TestTestDataGenerators(unittest.TestCase): + """Unit tests for the test_data_generation module. + """ + + def setUp(self): + """Create temporary folders.""" + self._base_output_path = tempfile.mkdtemp() + self._test_data_cache_path = tempfile.mkdtemp() + self._fake_air_db_path = tempfile.mkdtemp() + + # Fake AIR DB impulse responses. + # TODO(alessiob): ReverberationTestDataGenerator will change to allow custom + # impulse responses. When changed, the coupling below between + # impulse_response_mat_file_names and + # ReverberationTestDataGenerator._IMPULSE_RESPONSES can be removed. + impulse_response_mat_file_names = [ + 'air_binaural_lecture_0_0_1.mat', + 'air_binaural_booth_0_0_1.mat', + ] + for impulse_response_mat_file_name in impulse_response_mat_file_names: + data = {'h_air': np.random.rand(1, 1000).astype(' +#include +#include + +#include "common_audio/vad/include/vad.h" +#include "common_audio/wav_file.h" +#include "rtc_base/flags.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace test { +namespace { + +// The allowed values are 10, 20 or 30 ms. +constexpr uint8_t kAudioFrameLengthMilliseconds = 30; +constexpr int kMaxSampleRate = 48000; +constexpr size_t kMaxFrameLen = + kAudioFrameLengthMilliseconds * kMaxSampleRate / 1000; + +constexpr uint8_t kBitmaskBuffSize = 8; + +DEFINE_string(i, "", "Input wav file"); +DEFINE_string(o, "", "VAD output file"); + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true)) + return 1; + + // Open wav input file and check properties. + WavReader wav_reader(FLAG_i); + if (wav_reader.num_channels() != 1) { + RTC_LOG(LS_ERROR) << "Only mono wav files supported"; + return 1; + } + if (wav_reader.sample_rate() > kMaxSampleRate) { + RTC_LOG(LS_ERROR) << "Beyond maximum sample rate (" << kMaxSampleRate + << ")"; + return 1; + } + const size_t audio_frame_length = rtc::CheckedDivExact( + kAudioFrameLengthMilliseconds * wav_reader.sample_rate(), 1000); + if (audio_frame_length > kMaxFrameLen) { + RTC_LOG(LS_ERROR) << "The frame size and/or the sample rate are too large."; + return 1; + } + + // Create output file and write header. + std::ofstream out_file(FLAG_o, std::ofstream::binary); + const char audio_frame_length_ms = kAudioFrameLengthMilliseconds; + out_file.write(&audio_frame_length_ms, 1); // Header. + + // Run VAD and write decisions. + std::unique_ptr vad = CreateVad(Vad::Aggressiveness::kVadNormal); + std::array samples; + char buff = 0; // Buffer to write one bit per frame. + uint8_t next = 0; // Points to the next bit to write in |buff|. + while (true) { + // Process frame. + const auto read_samples = + wav_reader.ReadSamples(audio_frame_length, samples.data()); + if (read_samples < audio_frame_length) + break; + const auto is_speech = vad->VoiceActivity( + samples.data(), audio_frame_length, wav_reader.sample_rate()); + + // Write output. + buff = is_speech ? buff | (1 << next) : buff & ~(1 << next); + if (++next == kBitmaskBuffSize) { + out_file.write(&buff, 1); // Flush. + buff = 0; // Reset. + next = 0; + } + } + + // Finalize. + char extra_bits = 0; + if (next > 0) { + extra_bits = kBitmaskBuffSize - next; + out_file.write(&buff, 1); // Flush. + } + out_file.write(&extra_bits, 1); + out_file.close(); + + return 0; +} + +} // namespace +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.cc new file mode 100644 index 0000000000..90c6d5ea72 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/simulator_buffers.h" + +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +SimulatorBuffers::SimulatorBuffers(int render_input_sample_rate_hz, + int capture_input_sample_rate_hz, + int render_output_sample_rate_hz, + int capture_output_sample_rate_hz, + size_t num_render_input_channels, + size_t num_capture_input_channels, + size_t num_render_output_channels, + size_t num_capture_output_channels) { + Random rand_gen(42); + CreateConfigAndBuffer(render_input_sample_rate_hz, num_render_input_channels, + &rand_gen, &render_input_buffer, &render_input_config, + &render_input, &render_input_samples); + + CreateConfigAndBuffer(render_output_sample_rate_hz, + num_render_output_channels, &rand_gen, + &render_output_buffer, &render_output_config, + &render_output, &render_output_samples); + + CreateConfigAndBuffer(capture_input_sample_rate_hz, + num_capture_input_channels, &rand_gen, + &capture_input_buffer, &capture_input_config, + &capture_input, &capture_input_samples); + + CreateConfigAndBuffer(capture_output_sample_rate_hz, + num_capture_output_channels, &rand_gen, + &capture_output_buffer, &capture_output_config, + &capture_output, &capture_output_samples); + + UpdateInputBuffers(); +} + +SimulatorBuffers::~SimulatorBuffers() = default; + +void SimulatorBuffers::CreateConfigAndBuffer( + int sample_rate_hz, + size_t num_channels, + Random* rand_gen, + std::unique_ptr* buffer, + StreamConfig* config, + std::vector* buffer_data, + std::vector* buffer_data_samples) { + int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + *config = StreamConfig(sample_rate_hz, num_channels, false); + buffer->reset(new AudioBuffer(config->num_frames(), config->num_channels(), + config->num_frames(), config->num_channels(), + config->num_frames())); + + buffer_data_samples->resize(samples_per_channel * num_channels); + for (auto& v : *buffer_data_samples) { + v = rand_gen->Rand(); + } + + buffer_data->resize(num_channels); + for (size_t ch = 0; ch < num_channels; ++ch) { + (*buffer_data)[ch] = &(*buffer_data_samples)[ch * samples_per_channel]; + } +} + +void SimulatorBuffers::UpdateInputBuffers() { + test::CopyVectorToAudioBuffer(capture_input_config, capture_input_samples, + capture_input_buffer.get()); + test::CopyVectorToAudioBuffer(render_input_config, render_input_samples, + render_input_buffer.get()); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.h new file mode 100644 index 0000000000..36dcf301a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/simulator_buffers.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_SIMULATOR_BUFFERS_H_ +#define MODULES_AUDIO_PROCESSING_TEST_SIMULATOR_BUFFERS_H_ + +#include +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/random.h" + +namespace webrtc { +namespace test { + +struct SimulatorBuffers { + SimulatorBuffers(int render_input_sample_rate_hz, + int capture_input_sample_rate_hz, + int render_output_sample_rate_hz, + int capture_output_sample_rate_hz, + size_t num_render_input_channels, + size_t num_capture_input_channels, + size_t num_render_output_channels, + size_t num_capture_output_channels); + ~SimulatorBuffers(); + + void CreateConfigAndBuffer(int sample_rate_hz, + size_t num_channels, + Random* rand_gen, + std::unique_ptr* buffer, + StreamConfig* config, + std::vector* buffer_data, + std::vector* buffer_data_samples); + + void UpdateInputBuffers(); + + std::unique_ptr render_input_buffer; + std::unique_ptr capture_input_buffer; + std::unique_ptr render_output_buffer; + std::unique_ptr capture_output_buffer; + StreamConfig render_input_config; + StreamConfig capture_input_config; + StreamConfig render_output_config; + StreamConfig capture_output_config; + std::vector render_input; + std::vector render_input_samples; + std::vector capture_input; + std::vector capture_input_samples; + std::vector render_output; + std::vector render_output_samples; + std::vector capture_output; + std::vector capture_output_samples; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_SIMULATOR_BUFFERS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.cc new file mode 100644 index 0000000000..23948c6dde --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/test/test_utils.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +RawFile::RawFile(const std::string& filename) + : file_handle_(fopen(filename.c_str(), "wb")) {} + +RawFile::~RawFile() { + fclose(file_handle_); +} + +void RawFile::WriteSamples(const int16_t* samples, size_t num_samples) { +#ifndef WEBRTC_ARCH_LITTLE_ENDIAN + //convert to big-endian + int16_t* s = (int16_t*)samples; + for(size_t idx = 0; idx < num_samples; idx++) { + s[idx] = (samples[idx]<<8) | (samples[idx]>>8); + } +#endif + fwrite(samples, sizeof(*samples), num_samples, file_handle_); +} + +void RawFile::WriteSamples(const float* samples, size_t num_samples) { + fwrite(samples, sizeof(*samples), num_samples, file_handle_); +} + +ChannelBufferWavReader::ChannelBufferWavReader(std::unique_ptr file) + : file_(std::move(file)) {} + +ChannelBufferWavReader::~ChannelBufferWavReader() = default; + +bool ChannelBufferWavReader::Read(ChannelBuffer* buffer) { + RTC_CHECK_EQ(file_->num_channels(), buffer->num_channels()); + interleaved_.resize(buffer->size()); + if (file_->ReadSamples(interleaved_.size(), &interleaved_[0]) != + interleaved_.size()) { + return false; + } + + FloatS16ToFloat(&interleaved_[0], interleaved_.size(), &interleaved_[0]); + Deinterleave(&interleaved_[0], buffer->num_frames(), buffer->num_channels(), + buffer->channels()); + return true; +} + +ChannelBufferWavWriter::ChannelBufferWavWriter(std::unique_ptr file) + : file_(std::move(file)) {} + +ChannelBufferWavWriter::~ChannelBufferWavWriter() = default; + +void ChannelBufferWavWriter::Write(const ChannelBuffer& buffer) { + RTC_CHECK_EQ(file_->num_channels(), buffer.num_channels()); + interleaved_.resize(buffer.size()); + Interleave(buffer.channels(), buffer.num_frames(), buffer.num_channels(), + &interleaved_[0]); + FloatToFloatS16(&interleaved_[0], interleaved_.size(), &interleaved_[0]); + file_->WriteSamples(&interleaved_[0], interleaved_.size()); +} + +void WriteIntData(const int16_t* data, + size_t length, + WavWriter* wav_file, + RawFile* raw_file) { + if (wav_file) { + wav_file->WriteSamples(data, length); + } + if (raw_file) { + raw_file->WriteSamples(data, length); + } +} + +void WriteFloatData(const float* const* data, + size_t samples_per_channel, + size_t num_channels, + WavWriter* wav_file, + RawFile* raw_file) { + size_t length = num_channels * samples_per_channel; + std::unique_ptr buffer(new float[length]); + Interleave(data, samples_per_channel, num_channels, buffer.get()); + if (raw_file) { + raw_file->WriteSamples(buffer.get(), length); + } + // TODO(aluebs): Use ScaleToInt16Range() from audio_util + for (size_t i = 0; i < length; ++i) { + buffer[i] = buffer[i] > 0 ? + buffer[i] * std::numeric_limits::max() : + -buffer[i] * std::numeric_limits::min(); + } + if (wav_file) { + wav_file->WriteSamples(buffer.get(), length); + } +} + +FILE* OpenFile(const std::string& filename, const char* mode) { + FILE* file = fopen(filename.c_str(), mode); + if (!file) { + printf("Unable to open file %s\n", filename.c_str()); + exit(1); + } + return file; +} + +size_t SamplesFromRate(int rate) { + return static_cast(AudioProcessing::kChunkSizeMs * rate / 1000); +} + +void SetFrameSampleRate(AudioFrame* frame, + int sample_rate_hz) { + frame->sample_rate_hz_ = sample_rate_hz; + frame->samples_per_channel_ = AudioProcessing::kChunkSizeMs * + sample_rate_hz / 1000; +} + +AudioProcessing::ChannelLayout LayoutFromChannels(size_t num_channels) { + switch (num_channels) { + case 1: + return AudioProcessing::kMono; + case 2: + return AudioProcessing::kStereo; + default: + RTC_CHECK(false); + return AudioProcessing::kMono; + } +} + +std::vector ParseArrayGeometry(const std::string& mic_positions) { + const std::vector values = ParseList(mic_positions); + const size_t num_mics = + rtc::CheckedDivExact(values.size(), static_cast(3)); + RTC_CHECK_GT(num_mics, 0) << "mic_positions is not large enough."; + + std::vector result; + result.reserve(num_mics); + for (size_t i = 0; i < values.size(); i += 3) { + result.push_back(Point(values[i + 0], values[i + 1], values[i + 2])); + } + + return result; +} + +std::vector ParseArrayGeometry(const std::string& mic_positions, + size_t num_mics) { + std::vector result = ParseArrayGeometry(mic_positions); + RTC_CHECK_EQ(result.size(), num_mics) + << "Could not parse mic_positions or incorrect number of points."; + return result; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.h new file mode 100644 index 0000000000..57dc7b3797 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/test_utils.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_TEST_UTILS_H_ +#define MODULES_AUDIO_PROCESSING_TEST_TEST_UTILS_H_ + +#include +#include +#include +#include +#include +#include + +#include "common_audio/channel_buffer.h" +#include "common_audio/wav_file.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +static const AudioProcessing::Error kNoErr = AudioProcessing::kNoError; +#define EXPECT_NOERR(expr) EXPECT_EQ(kNoErr, (expr)) + +class RawFile final { + public: + explicit RawFile(const std::string& filename); + ~RawFile(); + + void WriteSamples(const int16_t* samples, size_t num_samples); + void WriteSamples(const float* samples, size_t num_samples); + + private: + FILE* file_handle_; + + RTC_DISALLOW_COPY_AND_ASSIGN(RawFile); +}; + +// Reads ChannelBuffers from a provided WavReader. +class ChannelBufferWavReader final { + public: + explicit ChannelBufferWavReader(std::unique_ptr file); + ~ChannelBufferWavReader(); + + // Reads data from the file according to the |buffer| format. Returns false if + // a full buffer can't be read from the file. + bool Read(ChannelBuffer* buffer); + + private: + std::unique_ptr file_; + std::vector interleaved_; + + RTC_DISALLOW_COPY_AND_ASSIGN(ChannelBufferWavReader); +}; + +// Writes ChannelBuffers to a provided WavWriter. +class ChannelBufferWavWriter final { + public: + explicit ChannelBufferWavWriter(std::unique_ptr file); + ~ChannelBufferWavWriter(); + + void Write(const ChannelBuffer& buffer); + + private: + std::unique_ptr file_; + std::vector interleaved_; + + RTC_DISALLOW_COPY_AND_ASSIGN(ChannelBufferWavWriter); +}; + +void WriteIntData(const int16_t* data, + size_t length, + WavWriter* wav_file, + RawFile* raw_file); + +void WriteFloatData(const float* const* data, + size_t samples_per_channel, + size_t num_channels, + WavWriter* wav_file, + RawFile* raw_file); + +// Exits on failure; do not use in unit tests. +FILE* OpenFile(const std::string& filename, const char* mode); + +size_t SamplesFromRate(int rate); + +void SetFrameSampleRate(AudioFrame* frame, + int sample_rate_hz); + +template +void SetContainerFormat(int sample_rate_hz, + size_t num_channels, + AudioFrame* frame, + std::unique_ptr >* cb) { + SetFrameSampleRate(frame, sample_rate_hz); + frame->num_channels_ = num_channels; + cb->reset(new ChannelBuffer(frame->samples_per_channel_, num_channels)); +} + +AudioProcessing::ChannelLayout LayoutFromChannels(size_t num_channels); + +template +float ComputeSNR(const T* ref, const T* test, size_t length, float* variance) { + float mse = 0; + float mean = 0; + *variance = 0; + for (size_t i = 0; i < length; ++i) { + T error = ref[i] - test[i]; + mse += error * error; + *variance += ref[i] * ref[i]; + mean += ref[i]; + } + mse /= length; + *variance /= length; + mean /= length; + *variance -= mean * mean; + + float snr = 100; // We assign 100 dB to the zero-error case. + if (mse > 0) + snr = 10 * log10(*variance / mse); + return snr; +} + +// Returns a vector parsed from whitespace delimited values in to_parse, +// or an empty vector if the string could not be parsed. +template +std::vector ParseList(const std::string& to_parse) { + std::vector values; + + std::istringstream str(to_parse); + std::copy( + std::istream_iterator(str), + std::istream_iterator(), + std::back_inserter(values)); + + return values; +} + +// Parses the array geometry from the command line. +// +// If a vector with size != num_mics is returned, an error has occurred and an +// appropriate error message has been printed to stdout. +std::vector ParseArrayGeometry(const std::string& mic_positions, + size_t num_mics); + +// Same as above, but without the num_mics check for when it isn't available. +std::vector ParseArrayGeometry(const std::string& mic_positions); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_TEST_UTILS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/unittest.proto b/third_party/libwebrtc/webrtc/modules/audio_processing/test/unittest.proto new file mode 100644 index 0000000000..3c93bfd713 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/unittest.proto @@ -0,0 +1,58 @@ +syntax = "proto2"; +option optimize_for = LITE_RUNTIME; +package webrtc.audioproc; + +message Test { + optional int32 num_reverse_channels = 1; + optional int32 num_input_channels = 2; + optional int32 num_output_channels = 3; + optional int32 sample_rate = 4; + + message Frame { + } + + repeated Frame frame = 5; + + optional int32 analog_level_average = 6; + optional int32 max_output_average = 7; + + optional int32 has_echo_count = 8; + optional int32 has_voice_count = 9; + optional int32 is_saturated_count = 10; + + message Statistic { + optional int32 instant = 1; + optional int32 average = 2; + optional int32 maximum = 3; + optional int32 minimum = 4; + } + + message EchoMetrics { + optional Statistic residual_echo_return_loss = 1; + optional Statistic echo_return_loss = 2; + optional Statistic echo_return_loss_enhancement = 3; + optional Statistic a_nlp = 4; + optional float divergent_filter_fraction = 5; + } + + repeated EchoMetrics echo_metrics = 11; + + message DelayMetrics { + optional int32 median = 1; + optional int32 std = 2; + optional float fraction_poor_delays = 3; + } + + repeated DelayMetrics delay_metrics = 12; + + repeated int32 rms_level = 13; + + optional float ns_speech_probability_average = 14; + + optional bool use_aec_extended_filter = 15; +} + +message OutputData { + repeated Test test = 1; +} + diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/unpack.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/unpack.cc new file mode 100644 index 0000000000..8e27706560 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/unpack.cc @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Commandline tool to unpack audioproc debug files. +// +// The debug files are dumped as protobuf blobs. For analysis, it's necessary +// to unpack the file into its component parts: audio and other data. + +#include + +#include + +#include "modules/audio_processing/test/protobuf_utils.h" +#include "modules/audio_processing/test/test_utils.h" +#include "rtc_base/flags.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/ignore_wundef.h" +#include "typedefs.h" // NOLINT(build/include) + +RTC_PUSH_IGNORING_WUNDEF() +#include "modules/audio_processing/debug.pb.h" +RTC_POP_IGNORING_WUNDEF() + +// TODO(andrew): unpack more of the data. +DEFINE_string(input_file, "input", "The name of the input stream file."); +DEFINE_string(output_file, "ref_out", + "The name of the reference output stream file."); +DEFINE_string(reverse_file, "reverse", + "The name of the reverse input stream file."); +DEFINE_string(delay_file, "delay.int32", "The name of the delay file."); +DEFINE_string(drift_file, "drift.int32", "The name of the drift file."); +DEFINE_string(level_file, "level.int32", "The name of the level file."); +DEFINE_string(keypress_file, "keypress.bool", "The name of the keypress file."); +DEFINE_string(settings_file, "settings.txt", "The name of the settings file."); +DEFINE_bool(full, false, + "Unpack the full set of files (normally not needed)."); +DEFINE_bool(raw, false, "Write raw data instead of a WAV file."); +DEFINE_bool(text, + false, + "Write non-audio files as text files instead of binary files."); +DEFINE_bool(help, false, "Print this message."); + +#define PRINT_CONFIG(field_name) \ + if (msg.has_##field_name()) { \ + fprintf(settings_file, " " #field_name ": %d\n", msg.field_name()); \ + } + +namespace webrtc { + +using audioproc::Event; +using audioproc::ReverseStream; +using audioproc::Stream; +using audioproc::Init; + +void WriteData(const void* data, size_t size, FILE* file, + const std::string& filename) { + if (fwrite(data, size, 1, file) != 1) { + printf("Error when writing to %s\n", filename.c_str()); + exit(1); + } +} + +int do_main(int argc, char* argv[]) { + std::string program_name = argv[0]; + std::string usage = "Commandline tool to unpack audioproc debug files.\n" + "Example usage:\n" + program_name + " debug_dump.pb\n"; + + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || + FLAG_help || argc < 2) { + printf("%s", usage.c_str()); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + + FILE* debug_file = OpenFile(argv[1], "rb"); + + Event event_msg; + int frame_count = 0; + size_t reverse_samples_per_channel = 0; + size_t input_samples_per_channel = 0; + size_t output_samples_per_channel = 0; + size_t num_reverse_channels = 0; + size_t num_input_channels = 0; + size_t num_output_channels = 0; + std::unique_ptr reverse_wav_file; + std::unique_ptr input_wav_file; + std::unique_ptr output_wav_file; + std::unique_ptr reverse_raw_file; + std::unique_ptr input_raw_file; + std::unique_ptr output_raw_file; + + FILE* settings_file = OpenFile(FLAG_settings_file, "wb"); + + while (ReadMessageFromFile(debug_file, &event_msg)) { + if (event_msg.type() == Event::REVERSE_STREAM) { + if (!event_msg.has_reverse_stream()) { + printf("Corrupt input file: ReverseStream missing.\n"); + return 1; + } + + const ReverseStream msg = event_msg.reverse_stream(); + if (msg.has_data()) { + if (FLAG_raw && !reverse_raw_file) { + reverse_raw_file.reset(new RawFile(std::string(FLAG_reverse_file) + + ".pcm")); + } + // TODO(aluebs): Replace "num_reverse_channels * + // reverse_samples_per_channel" with "msg.data().size() / + // sizeof(int16_t)" and so on when this fix in audio_processing has made + // it into stable: https://webrtc-codereview.appspot.com/15299004/ + WriteIntData(reinterpret_cast(msg.data().data()), + num_reverse_channels * reverse_samples_per_channel, + reverse_wav_file.get(), + reverse_raw_file.get()); + } else if (msg.channel_size() > 0) { + if (FLAG_raw && !reverse_raw_file) { + reverse_raw_file.reset(new RawFile(std::string(FLAG_reverse_file) + + ".float")); + } + std::unique_ptr data( + new const float* [num_reverse_channels]); + for (size_t i = 0; i < num_reverse_channels; ++i) { + data[i] = reinterpret_cast(msg.channel(i).data()); + } + WriteFloatData(data.get(), + reverse_samples_per_channel, + num_reverse_channels, + reverse_wav_file.get(), + reverse_raw_file.get()); + } + } else if (event_msg.type() == Event::STREAM) { + frame_count++; + if (!event_msg.has_stream()) { + printf("Corrupt input file: Stream missing.\n"); + return 1; + } + + const Stream msg = event_msg.stream(); + if (msg.has_input_data()) { + if (FLAG_raw && !input_raw_file) { + input_raw_file.reset(new RawFile(std::string(FLAG_input_file) + + ".pcm")); + } + WriteIntData(reinterpret_cast(msg.input_data().data()), + num_input_channels * input_samples_per_channel, + input_wav_file.get(), + input_raw_file.get()); + } else if (msg.input_channel_size() > 0) { + if (FLAG_raw && !input_raw_file) { + input_raw_file.reset(new RawFile(std::string(FLAG_input_file) + + ".float")); + } + std::unique_ptr data( + new const float* [num_input_channels]); + for (size_t i = 0; i < num_input_channels; ++i) { + data[i] = reinterpret_cast(msg.input_channel(i).data()); + } + WriteFloatData(data.get(), + input_samples_per_channel, + num_input_channels, + input_wav_file.get(), + input_raw_file.get()); + } + + if (msg.has_output_data()) { + if (FLAG_raw && !output_raw_file) { + output_raw_file.reset(new RawFile(std::string(FLAG_output_file) + + ".pcm")); + } + WriteIntData(reinterpret_cast(msg.output_data().data()), + num_output_channels * output_samples_per_channel, + output_wav_file.get(), + output_raw_file.get()); + } else if (msg.output_channel_size() > 0) { + if (FLAG_raw && !output_raw_file) { + output_raw_file.reset(new RawFile(std::string(FLAG_output_file) + + ".float")); + } + std::unique_ptr data( + new const float* [num_output_channels]); + for (size_t i = 0; i < num_output_channels; ++i) { + data[i] = + reinterpret_cast(msg.output_channel(i).data()); + } + WriteFloatData(data.get(), + output_samples_per_channel, + num_output_channels, + output_wav_file.get(), + output_raw_file.get()); + } + + if (FLAG_full) { + if (msg.has_delay()) { + static FILE* delay_file = OpenFile(FLAG_delay_file, "wb"); + int32_t delay = msg.delay(); + if (FLAG_text) { + fprintf(delay_file, "%d\n", delay); + } else { + WriteData(&delay, sizeof(delay), delay_file, FLAG_delay_file); + } + } + + if (msg.has_drift()) { + static FILE* drift_file = OpenFile(FLAG_drift_file, "wb"); + int32_t drift = msg.drift(); + if (FLAG_text) { + fprintf(drift_file, "%d\n", drift); + } else { + WriteData(&drift, sizeof(drift), drift_file, FLAG_drift_file); + } + } + + if (msg.has_level()) { + static FILE* level_file = OpenFile(FLAG_level_file, "wb"); + int32_t level = msg.level(); + if (FLAG_text) { + fprintf(level_file, "%d\n", level); + } else { + WriteData(&level, sizeof(level), level_file, FLAG_level_file); + } + } + + if (msg.has_keypress()) { + static FILE* keypress_file = OpenFile(FLAG_keypress_file, "wb"); + bool keypress = msg.keypress(); + if (FLAG_text) { + fprintf(keypress_file, "%d\n", keypress); + } else { + WriteData(&keypress, sizeof(keypress), keypress_file, + FLAG_keypress_file); + } + } + } + } else if (event_msg.type() == Event::CONFIG) { + if (!event_msg.has_config()) { + printf("Corrupt input file: Config missing.\n"); + return 1; + } + const audioproc::Config msg = event_msg.config(); + + fprintf(settings_file, "APM re-config at frame: %d\n", frame_count); + + PRINT_CONFIG(aec_enabled); + PRINT_CONFIG(aec_delay_agnostic_enabled); + PRINT_CONFIG(aec_drift_compensation_enabled); + PRINT_CONFIG(aec_extended_filter_enabled); + PRINT_CONFIG(aec_suppression_level); + PRINT_CONFIG(aecm_enabled); + PRINT_CONFIG(aecm_comfort_noise_enabled); + PRINT_CONFIG(aecm_routing_mode); + PRINT_CONFIG(agc_enabled); + PRINT_CONFIG(agc_mode); + PRINT_CONFIG(agc_limiter_enabled); + PRINT_CONFIG(noise_robust_agc_enabled); + PRINT_CONFIG(hpf_enabled); + PRINT_CONFIG(ns_enabled); + PRINT_CONFIG(ns_level); + PRINT_CONFIG(transient_suppression_enabled); + PRINT_CONFIG(intelligibility_enhancer_enabled); + if (msg.has_experiments_description()) { + fprintf(settings_file, " experiments_description: %s\n", + msg.experiments_description().c_str()); + } + } else if (event_msg.type() == Event::INIT) { + if (!event_msg.has_init()) { + printf("Corrupt input file: Init missing.\n"); + return 1; + } + + const Init msg = event_msg.init(); + // These should print out zeros if they're missing. + fprintf(settings_file, "Init at frame: %d\n", frame_count); + int input_sample_rate = msg.sample_rate(); + fprintf(settings_file, " Input sample rate: %d\n", input_sample_rate); + int output_sample_rate = msg.output_sample_rate(); + fprintf(settings_file, " Output sample rate: %d\n", output_sample_rate); + int reverse_sample_rate = msg.reverse_sample_rate(); + fprintf(settings_file, + " Reverse sample rate: %d\n", + reverse_sample_rate); + num_input_channels = msg.num_input_channels(); + fprintf(settings_file, " Input channels: %" PRIuS "\n", + num_input_channels); + num_output_channels = msg.num_output_channels(); + fprintf(settings_file, " Output channels: %" PRIuS "\n", + num_output_channels); + num_reverse_channels = msg.num_reverse_channels(); + fprintf(settings_file, " Reverse channels: %" PRIuS "\n", + num_reverse_channels); + + fprintf(settings_file, "\n"); + + if (reverse_sample_rate == 0) { + reverse_sample_rate = input_sample_rate; + } + if (output_sample_rate == 0) { + output_sample_rate = input_sample_rate; + } + + reverse_samples_per_channel = + static_cast(reverse_sample_rate / 100); + input_samples_per_channel = + static_cast(input_sample_rate / 100); + output_samples_per_channel = + static_cast(output_sample_rate / 100); + + if (!FLAG_raw) { + // The WAV files need to be reset every time, because they cant change + // their sample rate or number of channels. + std::stringstream reverse_name; + reverse_name << FLAG_reverse_file << frame_count << ".wav"; + reverse_wav_file.reset(new WavWriter(reverse_name.str(), + reverse_sample_rate, + num_reverse_channels)); + std::stringstream input_name; + input_name << FLAG_input_file << frame_count << ".wav"; + input_wav_file.reset(new WavWriter(input_name.str(), + input_sample_rate, + num_input_channels)); + std::stringstream output_name; + output_name << FLAG_output_file << frame_count << ".wav"; + output_wav_file.reset(new WavWriter(output_name.str(), + output_sample_rate, + num_output_channels)); + } + } + } + + return 0; +} + +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::do_main(argc, argv); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.cc new file mode 100644 index 0000000000..f53d1e5d8d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.cc @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/test/wav_based_simulator.h" + +#include +#include + +#include "modules/audio_processing/test/test_utils.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +std::vector +WavBasedSimulator::GetCustomEventChain(const std::string& filename) { + std::vector call_chain; + FILE* stream = OpenFile(filename.c_str(), "r"); + + RTC_CHECK(stream) << "Could not open the custom call order file, reverting " + "to using the default call order"; + + char c; + size_t num_read = fread(&c, sizeof(char), 1, stream); + while (num_read > 0) { + switch (c) { + case 'r': + call_chain.push_back(SimulationEventType::kProcessReverseStream); + break; + case 'c': + call_chain.push_back(SimulationEventType::kProcessStream); + break; + case '\n': + break; + default: + FATAL() << "Incorrect custom call order file, reverting to using the " + "default call order"; + fclose(stream); + return WavBasedSimulator::GetDefaultEventChain(); + } + + num_read = fread(&c, sizeof(char), 1, stream); + } + + fclose(stream); + return call_chain; +} + +WavBasedSimulator::WavBasedSimulator(const SimulationSettings& settings) + : AudioProcessingSimulator(settings) {} + +WavBasedSimulator::~WavBasedSimulator() = default; + +std::vector +WavBasedSimulator::GetDefaultEventChain() { + std::vector call_chain(2); + call_chain[0] = SimulationEventType::kProcessStream; + call_chain[1] = SimulationEventType::kProcessReverseStream; + return call_chain; +} + +void WavBasedSimulator::PrepareProcessStreamCall() { + if (settings_.fixed_interface) { + CopyToAudioFrame(*in_buf_, &fwd_frame_); + } + ap_->set_stream_key_pressed(settings_.use_ts && (*settings_.use_ts)); + + RTC_CHECK_EQ(AudioProcessing::kNoError, + ap_->set_stream_delay_ms( + settings_.stream_delay ? *settings_.stream_delay : 0)); + + ap_->echo_cancellation()->set_stream_drift_samples( + settings_.stream_drift_samples ? *settings_.stream_drift_samples : 0); +} + +void WavBasedSimulator::PrepareReverseProcessStreamCall() { + if (settings_.fixed_interface) { + CopyToAudioFrame(*reverse_in_buf_, &rev_frame_); + } +} + +void WavBasedSimulator::Process() { + if (settings_.custom_call_order_filename) { + call_chain_ = WavBasedSimulator::GetCustomEventChain( + *settings_.custom_call_order_filename); + } else { + call_chain_ = WavBasedSimulator::GetDefaultEventChain(); + } + CreateAudioProcessor(); + + Initialize(); + + bool samples_left_to_process = true; + int call_chain_index = 0; + int num_forward_chunks_processed = 0; + while (samples_left_to_process) { + switch (call_chain_[call_chain_index]) { + case SimulationEventType::kProcessStream: + samples_left_to_process = HandleProcessStreamCall(); + ++num_forward_chunks_processed; + break; + case SimulationEventType::kProcessReverseStream: + if (settings_.reverse_input_filename) { + samples_left_to_process = HandleProcessReverseStreamCall(); + } + break; + default: + RTC_CHECK(false); + } + + call_chain_index = (call_chain_index + 1) % call_chain_.size(); + } + + DestroyAudioProcessor(); +} + +bool WavBasedSimulator::HandleProcessStreamCall() { + bool samples_left_to_process = buffer_reader_->Read(in_buf_.get()); + if (samples_left_to_process) { + PrepareProcessStreamCall(); + ProcessStream(settings_.fixed_interface); + } + return samples_left_to_process; +} + +bool WavBasedSimulator::HandleProcessReverseStreamCall() { + bool samples_left_to_process = + reverse_buffer_reader_->Read(reverse_in_buf_.get()); + if (samples_left_to_process) { + PrepareReverseProcessStreamCall(); + ProcessReverseStream(settings_.fixed_interface); + } + return samples_left_to_process; +} + +void WavBasedSimulator::Initialize() { + std::unique_ptr in_file( + new WavReader(settings_.input_filename->c_str())); + int input_sample_rate_hz = in_file->sample_rate(); + int input_num_channels = in_file->num_channels(); + buffer_reader_.reset(new ChannelBufferWavReader(std::move(in_file))); + + int output_sample_rate_hz = settings_.output_sample_rate_hz + ? *settings_.output_sample_rate_hz + : input_sample_rate_hz; + int output_num_channels = settings_.output_num_channels + ? *settings_.output_num_channels + : input_num_channels; + + int reverse_sample_rate_hz = 48000; + int reverse_num_channels = 1; + int reverse_output_sample_rate_hz = 48000; + int reverse_output_num_channels = 1; + if (settings_.reverse_input_filename) { + std::unique_ptr reverse_in_file( + new WavReader(settings_.reverse_input_filename->c_str())); + reverse_sample_rate_hz = reverse_in_file->sample_rate(); + reverse_num_channels = reverse_in_file->num_channels(); + reverse_buffer_reader_.reset( + new ChannelBufferWavReader(std::move(reverse_in_file))); + + reverse_output_sample_rate_hz = + settings_.reverse_output_sample_rate_hz + ? *settings_.reverse_output_sample_rate_hz + : reverse_sample_rate_hz; + reverse_output_num_channels = settings_.reverse_output_num_channels + ? *settings_.reverse_output_num_channels + : reverse_num_channels; + } + + SetupBuffersConfigsOutputs( + input_sample_rate_hz, output_sample_rate_hz, reverse_sample_rate_hz, + reverse_output_sample_rate_hz, input_num_channels, output_num_channels, + reverse_num_channels, reverse_output_num_channels); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.h new file mode 100644 index 0000000000..febcffb62c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/test/wav_based_simulator.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TEST_WAV_BASED_SIMULATOR_H_ +#define MODULES_AUDIO_PROCESSING_TEST_WAV_BASED_SIMULATOR_H_ + +#include + +#include "modules/audio_processing/test/audio_processing_simulator.h" + +#include "rtc_base/constructormagic.h" + +namespace webrtc { +namespace test { + +// Used to perform an audio processing simulation from wav files. +class WavBasedSimulator final : public AudioProcessingSimulator { + public: + explicit WavBasedSimulator(const SimulationSettings& settings); + ~WavBasedSimulator() override; + + // Processes the WAV input. + void Process() override; + + private: + enum SimulationEventType { + kProcessStream, + kProcessReverseStream, + }; + + void Initialize(); + bool HandleProcessStreamCall(); + bool HandleProcessReverseStreamCall(); + void PrepareProcessStreamCall(); + void PrepareReverseProcessStreamCall(); + static std::vector GetDefaultEventChain(); + static std::vector GetCustomEventChain( + const std::string& filename); + + std::vector call_chain_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WavBasedSimulator); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TEST_WAV_BASED_SIMULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.cc new file mode 100644 index 0000000000..f5a319bc54 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.cc @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// An implementation of a 3-band FIR filter-bank with DCT modulation, similar to +// the proposed in "Multirate Signal Processing for Communication Systems" by +// Fredric J Harris. +// +// The idea is to take a heterodyne system and change the order of the +// components to get something which is efficient to implement digitally. +// +// It is possible to separate the filter using the noble identity as follows: +// +// H(z) = H0(z^3) + z^-1 * H1(z^3) + z^-2 * H2(z^3) +// +// This is used in the analysis stage to first downsample serial to parallel +// and then filter each branch with one of these polyphase decompositions of the +// lowpass prototype. Because each filter is only a modulation of the prototype, +// it is enough to multiply each coefficient by the respective cosine value to +// shift it to the desired band. But because the cosine period is 12 samples, +// it requires separating the prototype even further using the noble identity. +// After filtering and modulating for each band, the output of all filters is +// accumulated to get the downsampled bands. +// +// A similar logic can be applied to the synthesis stage. + +// MSVC++ requires this to be set before any other includes to get M_PI. +#define _USE_MATH_DEFINES + +#include "modules/audio_processing/three_band_filter_bank.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +const size_t kNumBands = 3; +const size_t kSparsity = 4; + +// Factors to take into account when choosing |kNumCoeffs|: +// 1. Higher |kNumCoeffs|, means faster transition, which ensures less +// aliasing. This is especially important when there is non-linear +// processing between the splitting and merging. +// 2. The delay that this filter bank introduces is +// |kNumBands| * |kSparsity| * |kNumCoeffs| / 2, so it increases linearly +// with |kNumCoeffs|. +// 3. The computation complexity also increases linearly with |kNumCoeffs|. +const size_t kNumCoeffs = 4; + +// The Matlab code to generate these |kLowpassCoeffs| is: +// +// N = kNumBands * kSparsity * kNumCoeffs - 1; +// h = fir1(N, 1 / (2 * kNumBands), kaiser(N + 1, 3.5)); +// reshape(h, kNumBands * kSparsity, kNumCoeffs); +// +// Because the total bandwidth of the lower and higher band is double the middle +// one (because of the spectrum parity), the low-pass prototype is half the +// bandwidth of 1 / (2 * |kNumBands|) and is then shifted with cosine modulation +// to the right places. +// A Kaiser window is used because of its flexibility and the alpha is set to +// 3.5, since that sets a stop band attenuation of 40dB ensuring a fast +// transition. +const float kLowpassCoeffs[kNumBands * kSparsity][kNumCoeffs] = + {{-0.00047749f, -0.00496888f, +0.16547118f, +0.00425496f}, + {-0.00173287f, -0.01585778f, +0.14989004f, +0.00994113f}, + {-0.00304815f, -0.02536082f, +0.12154542f, +0.01157993f}, + {-0.00383509f, -0.02982767f, +0.08543175f, +0.00983212f}, + {-0.00346946f, -0.02587886f, +0.04760441f, +0.00607594f}, + {-0.00154717f, -0.01136076f, +0.01387458f, +0.00186353f}, + {+0.00186353f, +0.01387458f, -0.01136076f, -0.00154717f}, + {+0.00607594f, +0.04760441f, -0.02587886f, -0.00346946f}, + {+0.00983212f, +0.08543175f, -0.02982767f, -0.00383509f}, + {+0.01157993f, +0.12154542f, -0.02536082f, -0.00304815f}, + {+0.00994113f, +0.14989004f, -0.01585778f, -0.00173287f}, + {+0.00425496f, +0.16547118f, -0.00496888f, -0.00047749f}}; + +// Downsamples |in| into |out|, taking one every |kNumbands| starting from +// |offset|. |split_length| is the |out| length. |in| has to be at least +// |kNumBands| * |split_length| long. +void Downsample(const float* in, + size_t split_length, + size_t offset, + float* out) { + for (size_t i = 0; i < split_length; ++i) { + out[i] = in[kNumBands * i + offset]; + } +} + +// Upsamples |in| into |out|, scaling by |kNumBands| and accumulating it every +// |kNumBands| starting from |offset|. |split_length| is the |in| length. |out| +// has to be at least |kNumBands| * |split_length| long. +void Upsample(const float* in, size_t split_length, size_t offset, float* out) { + for (size_t i = 0; i < split_length; ++i) { + out[kNumBands * i + offset] += kNumBands * in[i]; + } +} + +} // namespace + +// Because the low-pass filter prototype has half bandwidth it is possible to +// use a DCT to shift it in both directions at the same time, to the center +// frequencies [1 / 12, 3 / 12, 5 / 12]. +ThreeBandFilterBank::ThreeBandFilterBank(size_t length) + : in_buffer_(rtc::CheckedDivExact(length, kNumBands)), + out_buffer_(in_buffer_.size()) { + for (size_t i = 0; i < kSparsity; ++i) { + for (size_t j = 0; j < kNumBands; ++j) { + analysis_filters_.push_back( + std::unique_ptr(new SparseFIRFilter( + kLowpassCoeffs[i * kNumBands + j], kNumCoeffs, kSparsity, i))); + synthesis_filters_.push_back( + std::unique_ptr(new SparseFIRFilter( + kLowpassCoeffs[i * kNumBands + j], kNumCoeffs, kSparsity, i))); + } + } + dct_modulation_.resize(kNumBands * kSparsity); + for (size_t i = 0; i < dct_modulation_.size(); ++i) { + dct_modulation_[i].resize(kNumBands); + for (size_t j = 0; j < kNumBands; ++j) { + dct_modulation_[i][j] = + 2.f * cos(2.f * M_PI * i * (2.f * j + 1.f) / dct_modulation_.size()); + } + } +} + +ThreeBandFilterBank::~ThreeBandFilterBank() = default; + +// The analysis can be separated in these steps: +// 1. Serial to parallel downsampling by a factor of |kNumBands|. +// 2. Filtering of |kSparsity| different delayed signals with polyphase +// decomposition of the low-pass prototype filter and upsampled by a factor +// of |kSparsity|. +// 3. Modulating with cosines and accumulating to get the desired band. +void ThreeBandFilterBank::Analysis(const float* in, + size_t length, + float* const* out) { + RTC_CHECK_EQ(in_buffer_.size(), rtc::CheckedDivExact(length, kNumBands)); + for (size_t i = 0; i < kNumBands; ++i) { + memset(out[i], 0, in_buffer_.size() * sizeof(*out[i])); + } + for (size_t i = 0; i < kNumBands; ++i) { + Downsample(in, in_buffer_.size(), kNumBands - i - 1, &in_buffer_[0]); + for (size_t j = 0; j < kSparsity; ++j) { + const size_t offset = i + j * kNumBands; + analysis_filters_[offset]->Filter(&in_buffer_[0], + in_buffer_.size(), + &out_buffer_[0]); + DownModulate(&out_buffer_[0], out_buffer_.size(), offset, out); + } + } +} + +// The synthesis can be separated in these steps: +// 1. Modulating with cosines. +// 2. Filtering each one with a polyphase decomposition of the low-pass +// prototype filter upsampled by a factor of |kSparsity| and accumulating +// |kSparsity| signals with different delays. +// 3. Parallel to serial upsampling by a factor of |kNumBands|. +void ThreeBandFilterBank::Synthesis(const float* const* in, + size_t split_length, + float* out) { + RTC_CHECK_EQ(in_buffer_.size(), split_length); + memset(out, 0, kNumBands * in_buffer_.size() * sizeof(*out)); + for (size_t i = 0; i < kNumBands; ++i) { + for (size_t j = 0; j < kSparsity; ++j) { + const size_t offset = i + j * kNumBands; + UpModulate(in, in_buffer_.size(), offset, &in_buffer_[0]); + synthesis_filters_[offset]->Filter(&in_buffer_[0], + in_buffer_.size(), + &out_buffer_[0]); + Upsample(&out_buffer_[0], out_buffer_.size(), i, out); + } + } +} + + +// Modulates |in| by |dct_modulation_| and accumulates it in each of the +// |kNumBands| bands of |out|. |offset| is the index in the period of the +// cosines used for modulation. |split_length| is the length of |in| and each +// band of |out|. +void ThreeBandFilterBank::DownModulate(const float* in, + size_t split_length, + size_t offset, + float* const* out) { + for (size_t i = 0; i < kNumBands; ++i) { + for (size_t j = 0; j < split_length; ++j) { + out[i][j] += dct_modulation_[offset][i] * in[j]; + } + } +} + +// Modulates each of the |kNumBands| bands of |in| by |dct_modulation_| and +// accumulates them in |out|. |out| is cleared before starting to accumulate. +// |offset| is the index in the period of the cosines used for modulation. +// |split_length| is the length of each band of |in| and |out|. +void ThreeBandFilterBank::UpModulate(const float* const* in, + size_t split_length, + size_t offset, + float* out) { + memset(out, 0, split_length * sizeof(*out)); + for (size_t i = 0; i < kNumBands; ++i) { + for (size_t j = 0; j < split_length; ++j) { + out[j] += dct_modulation_[offset][i] * in[i][j]; + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.h b/third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.h new file mode 100644 index 0000000000..ccbf2ddf97 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/three_band_filter_bank.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_THREE_BAND_FILTER_BANK_H_ +#define MODULES_AUDIO_PROCESSING_THREE_BAND_FILTER_BANK_H_ + +#include +#include +#include + +#include "common_audio/sparse_fir_filter.h" + +namespace webrtc { + +// An implementation of a 3-band FIR filter-bank with DCT modulation, similar to +// the proposed in "Multirate Signal Processing for Communication Systems" by +// Fredric J Harris. +// The low-pass filter prototype has these characteristics: +// * Pass-band ripple = 0.3dB +// * Pass-band frequency = 0.147 (7kHz at 48kHz) +// * Stop-band attenuation = 40dB +// * Stop-band frequency = 0.192 (9.2kHz at 48kHz) +// * Delay = 24 samples (500us at 48kHz) +// * Linear phase +// This filter bank does not satisfy perfect reconstruction. The SNR after +// analysis and synthesis (with no processing in between) is approximately 9.5dB +// depending on the input signal after compensating for the delay. +class ThreeBandFilterBank final { + public: + explicit ThreeBandFilterBank(size_t length); + ~ThreeBandFilterBank(); + + // Splits |in| into 3 downsampled frequency bands in |out|. + // |length| is the |in| length. Each of the 3 bands of |out| has to have a + // length of |length| / 3. + void Analysis(const float* in, size_t length, float* const* out); + + // Merges the 3 downsampled frequency bands in |in| into |out|. + // |split_length| is the length of each band of |in|. |out| has to have at + // least a length of 3 * |split_length|. + void Synthesis(const float* const* in, size_t split_length, float* out); + + private: + void DownModulate(const float* in, + size_t split_length, + size_t offset, + float* const* out); + void UpModulate(const float* const* in, + size_t split_length, + size_t offset, + float* out); + + std::vector in_buffer_; + std::vector out_buffer_; + std::vector> analysis_filters_; + std::vector> synthesis_filters_; + std::vector> dct_modulation_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_THREE_BAND_FILTER_BANK_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/click_annotate.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/click_annotate.cc new file mode 100644 index 0000000000..a8b4a307e1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/click_annotate.cc @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include +#include + +#include "modules/audio_processing/transient/transient_detector.h" +#include "modules/audio_processing/transient/file_utils.h" +#include "system_wrappers/include/file_wrapper.h" + +using webrtc::FileWrapper; +using webrtc::TransientDetector; + +// Application to generate a RTP timing file. +// Opens the PCM file and divides the signal in frames. +// Creates a send times array, one for each step. +// Each block that contains a transient, has an infinite send time. +// The resultant array is written to a DAT file +// Returns -1 on error or |lost_packets| otherwise. +int main(int argc, char* argv[]) { + if (argc != 5) { + printf("\n%s - Application to generate a RTP timing file.\n\n", argv[0]); + printf("%s PCMfile DATfile chunkSize sampleRate\n\n", argv[0]); + printf("Opens the PCMfile with sampleRate in Hertz.\n"); + printf("Creates a send times array, one for each chunkSize "); + printf("milliseconds step.\n"); + printf("Each block that contains a transient, has an infinite send time. "); + printf("The resultant array is written to a DATfile.\n\n"); + return 0; + } + + std::unique_ptr pcm_file(FileWrapper::Create()); + pcm_file->OpenFile(argv[1], true); + if (!pcm_file->is_open()) { + printf("\nThe %s could not be opened.\n\n", argv[1]); + return -1; + } + + std::unique_ptr dat_file(FileWrapper::Create()); + dat_file->OpenFile(argv[2], false); + if (!dat_file->is_open()) { + printf("\nThe %s could not be opened.\n\n", argv[2]); + return -1; + } + + int chunk_size_ms = atoi(argv[3]); + if (chunk_size_ms <= 0) { + printf("\nThe chunkSize must be a positive integer\n\n"); + return -1; + } + + int sample_rate_hz = atoi(argv[4]); + if (sample_rate_hz <= 0) { + printf("\nThe sampleRate must be a positive integer\n\n"); + return -1; + } + + TransientDetector detector(sample_rate_hz); + int lost_packets = 0; + size_t audio_buffer_length = chunk_size_ms * sample_rate_hz / 1000; + std::unique_ptr audio_buffer(new float[audio_buffer_length]); + std::vector send_times; + + // Read first buffer from the PCM test file. + size_t file_samples_read = ReadInt16FromFileToFloatBuffer( + pcm_file.get(), + audio_buffer_length, + audio_buffer.get()); + for (int time = 0; file_samples_read > 0; time += chunk_size_ms) { + // Pad the rest of the buffer with zeros. + for (size_t i = file_samples_read; i < audio_buffer_length; ++i) { + audio_buffer[i] = 0.0; + } + float value = + detector.Detect(audio_buffer.get(), audio_buffer_length, NULL, 0); + if (value < 0.5f) { + value = time; + } else { + value = FLT_MAX; + ++lost_packets; + } + send_times.push_back(value); + + // Read next buffer from the PCM test file. + file_samples_read = ReadInt16FromFileToFloatBuffer(pcm_file.get(), + audio_buffer_length, + audio_buffer.get()); + } + + size_t floats_written = WriteFloatBufferToFile(dat_file.get(), + send_times.size(), + &send_times[0]); + + if (floats_written == 0) { + printf("\nThe send times could not be written to DAT file\n\n"); + return -1; + } + + pcm_file->CloseFile(); + dat_file->CloseFile(); + + return lost_packets; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/common.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/common.h new file mode 100644 index 0000000000..69546fc57b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/common.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_COMMON_H_ +namespace webrtc { +namespace ts { + +static const float kPi = 3.14159265358979323846f; +static const int kChunkSizeMs = 10; +enum { + kSampleRate8kHz = 8000, + kSampleRate16kHz = 16000, + kSampleRate32kHz = 32000, + kSampleRate48kHz = 48000 +}; + +} // namespace ts +} // namespace webrtc +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/daubechies_8_wavelet_coeffs.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/daubechies_8_wavelet_coeffs.h new file mode 100644 index 0000000000..4de24e0fcb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/daubechies_8_wavelet_coeffs.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This header file defines the coefficients of the FIR based approximation of +// the Meyer Wavelet +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_DAUBECHIES_8_WAVELET_COEFFS_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_DAUBECHIES_8_WAVELET_COEFFS_H_ + +// Decomposition coefficients Daubechies 8. + +namespace webrtc { + +const int kDaubechies8CoefficientsLength = 16; + +const float kDaubechies8HighPassCoefficients[kDaubechies8CoefficientsLength] + = { + -5.44158422430816093862e-02f, + 3.12871590914465924627e-01f, + -6.75630736298012846142e-01f, + 5.85354683654869090148e-01f, + 1.58291052560238926228e-02f, + -2.84015542962428091389e-01f, + -4.72484573997972536787e-04f, + 1.28747426620186011803e-01f, + 1.73693010020221083600e-02f, + -4.40882539310647192377e-02f, + -1.39810279170155156436e-02f, + 8.74609404701565465445e-03f, + 4.87035299301066034600e-03f, + -3.91740372995977108837e-04f, + -6.75449405998556772109e-04f, + -1.17476784002281916305e-04f +}; + +const float kDaubechies8LowPassCoefficients[kDaubechies8CoefficientsLength] = { + -1.17476784002281916305e-04f, + 6.75449405998556772109e-04f, + -3.91740372995977108837e-04f, + -4.87035299301066034600e-03f, + 8.74609404701565465445e-03f, + 1.39810279170155156436e-02f, + -4.40882539310647192377e-02f, + -1.73693010020221083600e-02f, + 1.28747426620186011803e-01f, + 4.72484573997972536787e-04f, + -2.84015542962428091389e-01f, + -1.58291052560238926228e-02f, + 5.85354683654869090148e-01f, + 6.75630736298012846142e-01f, + 3.12871590914465924627e-01f, + 5.44158422430816093862e-02f +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_DAUBECHIES_8_WAVELET_COEFFS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator.h new file mode 100644 index 0000000000..104f95d3af --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_DYADIC_DECIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_DYADIC_DECIMATOR_H_ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +// Provides a set of static methods to perform dyadic decimations. + +namespace webrtc { + +// Returns the proper length of the output buffer that you should use for the +// given |in_length| and decimation |odd_sequence|. +// Return -1 on error. +inline size_t GetOutLengthToDyadicDecimate(size_t in_length, + bool odd_sequence) { + size_t out_length = in_length / 2; + + if (in_length % 2 == 1 && !odd_sequence) { + ++out_length; + } + + return out_length; +} + +// Performs a dyadic decimation: removes every odd/even member of a sequence +// halving its overall length. +// Arguments: +// in: array of |in_length|. +// odd_sequence: If false, the odd members will be removed (1, 3, 5, ...); +// if true, the even members will be removed (0, 2, 4, ...). +// out: array of |out_length|. |out_length| must be large enough to +// hold the decimated output. The necessary length can be provided by +// GetOutLengthToDyadicDecimate(). +// Must be previously allocated. +// Returns the number of output samples, -1 on error. +template +static size_t DyadicDecimate(const T* in, + size_t in_length, + bool odd_sequence, + T* out, + size_t out_length) { + size_t half_length = GetOutLengthToDyadicDecimate(in_length, odd_sequence); + + if (!in || !out || in_length <= 0 || out_length < half_length) { + return 0; + } + + size_t output_samples = 0; + size_t index_adjustment = odd_sequence ? 1 : 0; + for (output_samples = 0; output_samples < half_length; ++output_samples) { + out[output_samples] = in[output_samples * 2 + index_adjustment]; + } + + return output_samples; +} + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_DYADIC_DECIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator_unittest.cc new file mode 100644 index 0000000000..c407f47ffb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/dyadic_decimator_unittest.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/dyadic_decimator.h" + +#include "test/gtest.h" + +namespace webrtc { + +static const size_t kEvenBufferLength = 6; +static const size_t kOddBufferLength = 5; +static const size_t kOutBufferLength = 3; + +int16_t const test_buffer_even_len[] = {0, 1, 2, 3, 4, 5}; +int16_t const test_buffer_odd_len[] = {0, 1, 2, 3, 4}; +int16_t test_buffer_out[kOutBufferLength]; + +TEST(DyadicDecimatorTest, GetOutLengthToDyadicDecimate) { + EXPECT_EQ(3u, GetOutLengthToDyadicDecimate(6, false)); + EXPECT_EQ(3u, GetOutLengthToDyadicDecimate(6, true)); + EXPECT_EQ(3u, GetOutLengthToDyadicDecimate(5, false)); + EXPECT_EQ(2u, GetOutLengthToDyadicDecimate(5, true)); +} + + +TEST(DyadicDecimatorTest, DyadicDecimateErrorValues) { + size_t out_samples = 0; + + out_samples = DyadicDecimate(static_cast(NULL), + kEvenBufferLength, + false, // Even sequence. + test_buffer_out, + kOutBufferLength); + EXPECT_EQ(0u, out_samples); + + out_samples = DyadicDecimate(test_buffer_even_len, + kEvenBufferLength, + false, // Even sequence. + static_cast(NULL), + kOutBufferLength); + EXPECT_EQ(0u, out_samples); + + // Less than required |out_length|. + out_samples = DyadicDecimate(test_buffer_even_len, + kEvenBufferLength, + false, // Even sequence. + test_buffer_out, + 2); + EXPECT_EQ(0u, out_samples); +} + +TEST(DyadicDecimatorTest, DyadicDecimateEvenLengthEvenSequence) { + size_t expected_out_samples = + GetOutLengthToDyadicDecimate(kEvenBufferLength, false); + + size_t out_samples = DyadicDecimate(test_buffer_even_len, + kEvenBufferLength, + false, // Even sequence. + test_buffer_out, + kOutBufferLength); + + EXPECT_EQ(expected_out_samples, out_samples); + + EXPECT_EQ(0, test_buffer_out[0]); + EXPECT_EQ(2, test_buffer_out[1]); + EXPECT_EQ(4, test_buffer_out[2]); +} + +TEST(DyadicDecimatorTest, DyadicDecimateEvenLengthOddSequence) { + size_t expected_out_samples = + GetOutLengthToDyadicDecimate(kEvenBufferLength, true); + + size_t out_samples = DyadicDecimate(test_buffer_even_len, + kEvenBufferLength, + true, // Odd sequence. + test_buffer_out, + kOutBufferLength); + + EXPECT_EQ(expected_out_samples, out_samples); + + EXPECT_EQ(1, test_buffer_out[0]); + EXPECT_EQ(3, test_buffer_out[1]); + EXPECT_EQ(5, test_buffer_out[2]); +} + +TEST(DyadicDecimatorTest, DyadicDecimateOddLengthEvenSequence) { + size_t expected_out_samples = + GetOutLengthToDyadicDecimate(kOddBufferLength, false); + + size_t out_samples = DyadicDecimate(test_buffer_odd_len, + kOddBufferLength, + false, // Even sequence. + test_buffer_out, + kOutBufferLength); + + EXPECT_EQ(expected_out_samples, out_samples); + + EXPECT_EQ(0, test_buffer_out[0]); + EXPECT_EQ(2, test_buffer_out[1]); + EXPECT_EQ(4, test_buffer_out[2]); +} + +TEST(DyadicDecimatorTest, DyadicDecimateOddLengthOddSequence) { + size_t expected_out_samples = + GetOutLengthToDyadicDecimate(kOddBufferLength, true); + + size_t out_samples = DyadicDecimate(test_buffer_odd_len, + kOddBufferLength, + true, // Odd sequence. + test_buffer_out, + kOutBufferLength); + + EXPECT_EQ(expected_out_samples, out_samples); + + EXPECT_EQ(1, test_buffer_out[0]); + EXPECT_EQ(3, test_buffer_out[1]); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.cc new file mode 100644 index 0000000000..7bf2e084f8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.cc @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/file_utils.h" + +#include + +#include "system_wrappers/include/file_wrapper.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +int ConvertByteArrayToFloat(const uint8_t bytes[4], float* out) { + if (!bytes || !out) { + return -1; + } + + uint32_t binary_value = 0; + for (int i = 3; i >= 0; --i) { + binary_value <<= 8; + binary_value += bytes[i]; + } + + *out = bit_cast(binary_value); + + return 0; +} + +int ConvertByteArrayToDouble(const uint8_t bytes[8], double* out) { + if (!bytes || !out) { + return -1; + } + + uint64_t binary_value = 0; + for (int i = 7; i >= 0; --i) { + binary_value <<= 8; + binary_value += bytes[i]; + } + + *out = bit_cast(binary_value); + + return 0; +} + +int ConvertFloatToByteArray(float value, uint8_t out_bytes[4]) { + if (!out_bytes) { + return -1; + } + + uint32_t binary_value = bit_cast(value); + for (size_t i = 0; i < 4; ++i) { + out_bytes[i] = binary_value; + binary_value >>= 8; + } + + return 0; +} + +int ConvertDoubleToByteArray(double value, uint8_t out_bytes[8]) { + if (!out_bytes) { + return -1; + } + + uint64_t binary_value = bit_cast(value); + for (size_t i = 0; i < 8; ++i) { + out_bytes[i] = binary_value; + binary_value >>= 8; + } + + return 0; +} + +size_t ReadInt16BufferFromFile(FileWrapper* file, + size_t length, + int16_t* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr byte_array(new uint8_t[2]); + + size_t int16s_read = 0; + + while (int16s_read < length) { + size_t bytes_read = file->Read(byte_array.get(), 2); + if (bytes_read < 2) { + break; + } + int16_t value = byte_array[1]; + value <<= 8; + value += byte_array[0]; + buffer[int16s_read] = value; + ++int16s_read; + } + + return int16s_read; +} + +size_t ReadInt16FromFileToFloatBuffer(FileWrapper* file, + size_t length, + float* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr buffer16(new int16_t[length]); + + size_t int16s_read = ReadInt16BufferFromFile(file, length, buffer16.get()); + + for (size_t i = 0; i < int16s_read; ++i) { + buffer[i] = buffer16[i]; + } + + return int16s_read; +} + +size_t ReadInt16FromFileToDoubleBuffer(FileWrapper* file, + size_t length, + double* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr buffer16(new int16_t[length]); + + size_t int16s_read = ReadInt16BufferFromFile(file, length, buffer16.get()); + + for (size_t i = 0; i < int16s_read; ++i) { + buffer[i] = buffer16[i]; + } + + return int16s_read; +} + +size_t ReadFloatBufferFromFile(FileWrapper* file, + size_t length, + float* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr byte_array(new uint8_t[4]); + + size_t floats_read = 0; + + while (floats_read < length) { + size_t bytes_read = file->Read(byte_array.get(), 4); + if (bytes_read < 4) { + break; + } + ConvertByteArrayToFloat(byte_array.get(), &buffer[floats_read]); + ++floats_read; + } + + return floats_read; +} + +size_t ReadDoubleBufferFromFile(FileWrapper* file, + size_t length, + double* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr byte_array(new uint8_t[8]); + + size_t doubles_read = 0; + + while (doubles_read < length) { + size_t bytes_read = file->Read(byte_array.get(), 8); + if (bytes_read < 8) { + break; + } + ConvertByteArrayToDouble(byte_array.get(), &buffer[doubles_read]); + ++doubles_read; + } + + return doubles_read; +} + +size_t WriteInt16BufferToFile(FileWrapper* file, + size_t length, + const int16_t* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr byte_array(new uint8_t[2]); + + size_t int16s_written = 0; + + for (int16s_written = 0; int16s_written < length; ++int16s_written) { + // Get byte representation. + byte_array[0] = buffer[int16s_written] & 0xFF; + byte_array[1] = (buffer[int16s_written] >> 8) & 0xFF; + + file->Write(byte_array.get(), 2); + } + + file->Flush(); + + return int16s_written; +} + +size_t WriteFloatBufferToFile(FileWrapper* file, + size_t length, + const float* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr byte_array(new uint8_t[4]); + + size_t floats_written = 0; + + for (floats_written = 0; floats_written < length; ++floats_written) { + // Get byte representation. + ConvertFloatToByteArray(buffer[floats_written], byte_array.get()); + + file->Write(byte_array.get(), 4); + } + + file->Flush(); + + return floats_written; +} + +size_t WriteDoubleBufferToFile(FileWrapper* file, + size_t length, + const double* buffer) { + if (!file || !file->is_open() || !buffer || length <= 0) { + return 0; + } + + std::unique_ptr byte_array(new uint8_t[8]); + + size_t doubles_written = 0; + + for (doubles_written = 0; doubles_written < length; ++doubles_written) { + // Get byte representation. + ConvertDoubleToByteArray(buffer[doubles_written], byte_array.get()); + + file->Write(byte_array.get(), 8); + } + + file->Flush(); + + return doubles_written; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.h new file mode 100644 index 0000000000..3f05c1dd52 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_FILE_UTILS_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_FILE_UTILS_H_ + +#include + +#include "system_wrappers/include/file_wrapper.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// This is a copy of the cast included in the Chromium codebase here: +// http://cs.chromium.org/src/third_party/cld/base/casts.h +template +inline Dest bit_cast(const Source& source) { + // A compile error here means your Dest and Source have different sizes. + static_assert(sizeof(Dest) == sizeof(Source), + "Dest and Source have different sizes"); + + Dest dest; + memcpy(&dest, &source, sizeof(dest)); + return dest; +} + +// Converts the byte array with binary float representation to float. +// Bytes must be in little-endian order. +// Returns 0 if correct, -1 on error. +int ConvertByteArrayToFloat(const uint8_t bytes[4], float* out); + +// Converts the byte array with binary double representation to double. +// Bytes must be in little-endian order. +// Returns 0 if correct, -1 on error. +int ConvertByteArrayToDouble(const uint8_t bytes[8], double* out); + +// Converts a float to a byte array with binary float representation. +// Bytes will be in little-endian order. +// Returns 0 if correct, -1 on error. +int ConvertFloatToByteArray(float value, uint8_t out_bytes[4]); + +// Converts a double to a byte array with binary double representation. +// Bytes will be in little-endian order. +// Returns 0 if correct, -1 on error. +int ConvertDoubleToByteArray(double value, uint8_t out_bytes[8]); + +// Reads |length| 16-bit integers from |file| to |buffer|. +// |file| must be previously opened. +// Returns the number of 16-bit integers read or -1 on error. +size_t ReadInt16BufferFromFile(FileWrapper* file, + size_t length, + int16_t* buffer); + +// Reads |length| 16-bit integers from |file| and stores those values +// (converting them) in |buffer|. +// |file| must be previously opened. +// Returns the number of 16-bit integers read or -1 on error. +size_t ReadInt16FromFileToFloatBuffer(FileWrapper* file, + size_t length, + float* buffer); + +// Reads |length| 16-bit integers from |file| and stores those values +// (converting them) in |buffer|. +// |file| must be previously opened. +// Returns the number of 16-bit integers read or -1 on error. +size_t ReadInt16FromFileToDoubleBuffer(FileWrapper* file, + size_t length, + double* buffer); + +// Reads |length| floats in binary representation (4 bytes) from |file| to +// |buffer|. +// |file| must be previously opened. +// Returns the number of floats read or -1 on error. +size_t ReadFloatBufferFromFile(FileWrapper* file, size_t length, float* buffer); + +// Reads |length| doubles in binary representation (8 bytes) from |file| to +// |buffer|. +// |file| must be previously opened. +// Returns the number of doubles read or -1 on error. +size_t ReadDoubleBufferFromFile(FileWrapper* file, + size_t length, + double* buffer); + +// Writes |length| 16-bit integers from |buffer| in binary representation (2 +// bytes) to |file|. It flushes |file|, so after this call there are no +// writings pending. +// |file| must be previously opened. +// Returns the number of doubles written or -1 on error. +size_t WriteInt16BufferToFile(FileWrapper* file, + size_t length, + const int16_t* buffer); + +// Writes |length| floats from |buffer| in binary representation (4 bytes) to +// |file|. It flushes |file|, so after this call there are no writtings pending. +// |file| must be previously opened. +// Returns the number of doubles written or -1 on error. +size_t WriteFloatBufferToFile(FileWrapper* file, + size_t length, + const float* buffer); + +// Writes |length| doubles from |buffer| in binary representation (8 bytes) to +// |file|. It flushes |file|, so after this call there are no writings pending. +// |file| must be previously opened. +// Returns the number of doubles written or -1 on error. +size_t WriteDoubleBufferToFile(FileWrapper* file, + size_t length, + const double* buffer); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_FILE_UTILS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils_unittest.cc new file mode 100644 index 0000000000..c5e03990b5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/file_utils_unittest.cc @@ -0,0 +1,539 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/file_utils.h" + +#include +#include +#include +#include + +#include "system_wrappers/include/file_wrapper.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +static const uint8_t kPiBytesf[4] = {0xDB, 0x0F, 0x49, 0x40}; +static const uint8_t kEBytesf[4] = {0x54, 0xF8, 0x2D, 0x40}; +static const uint8_t kAvogadroBytesf[4] = {0x2F, 0x0C, 0xFF, 0x66}; + +static const uint8_t kPiBytes[8] = + {0x18, 0x2D, 0x44, 0x54, 0xFB, 0x21, 0x09, 0x40}; +static const uint8_t kEBytes[8] = + {0x69, 0x57, 0x14, 0x8B, 0x0A, 0xBF, 0x05, 0x40}; +static const uint8_t kAvogadroBytes[8] = + {0xF4, 0xBC, 0xA8, 0xDF, 0x85, 0xE1, 0xDF, 0x44}; + +static const double kPi = 3.14159265358979323846; +static const double kE = 2.71828182845904523536; +static const double kAvogadro = 602214100000000000000000.0; + +class TransientFileUtilsTest: public ::testing::Test { + protected: + TransientFileUtilsTest() + : kTestFileName( + test::ResourcePath("audio_processing/transient/double-utils", + "dat")), + kTestFileNamef( + test::ResourcePath("audio_processing/transient/float-utils", + "dat")) {} + + ~TransientFileUtilsTest() override { + CleanupTempFiles(); + } + + std::string CreateTempFilename(const std::string& dir, + const std::string& prefix) { + std::string filename = test::TempFilename(dir, prefix); + temp_filenames_.push_back(filename); + return filename; + } + + void CleanupTempFiles() { + for (const std::string& filename : temp_filenames_) { + remove(filename.c_str()); + } + temp_filenames_.clear(); + } + + // This file (used in some tests) contains binary data. The data correspond to + // the double representation of the constants: Pi, E, and the Avogadro's + // Number; + // appended in that order. + const std::string kTestFileName; + + // This file (used in some tests) contains binary data. The data correspond to + // the float representation of the constants: Pi, E, and the Avogadro's + // Number; + // appended in that order. + const std::string kTestFileNamef; + + // List of temporary filenames created by CreateTempFilename. + std::vector temp_filenames_; +}; + +#if defined(WEBRTC_IOS) +#define MAYBE_ConvertByteArrayToFloat DISABLED_ConvertByteArrayToFloat +#else +#define MAYBE_ConvertByteArrayToFloat ConvertByteArrayToFloat +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ConvertByteArrayToFloat) { + float value = 0.0; + + EXPECT_EQ(0, ConvertByteArrayToFloat(kPiBytesf, &value)); + EXPECT_FLOAT_EQ(kPi, value); + + EXPECT_EQ(0, ConvertByteArrayToFloat(kEBytesf, &value)); + EXPECT_FLOAT_EQ(kE, value); + + EXPECT_EQ(0, ConvertByteArrayToFloat(kAvogadroBytesf, &value)); + EXPECT_FLOAT_EQ(kAvogadro, value); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ConvertByteArrayToDouble DISABLED_ConvertByteArrayToDouble +#else +#define MAYBE_ConvertByteArrayToDouble ConvertByteArrayToDouble +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ConvertByteArrayToDouble) { + double value = 0.0; + + EXPECT_EQ(0, ConvertByteArrayToDouble(kPiBytes, &value)); + EXPECT_DOUBLE_EQ(kPi, value); + + EXPECT_EQ(0, ConvertByteArrayToDouble(kEBytes, &value)); + EXPECT_DOUBLE_EQ(kE, value); + + EXPECT_EQ(0, ConvertByteArrayToDouble(kAvogadroBytes, &value)); + EXPECT_DOUBLE_EQ(kAvogadro, value); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ConvertFloatToByteArray DISABLED_ConvertFloatToByteArray +#else +#define MAYBE_ConvertFloatToByteArray ConvertFloatToByteArray +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ConvertFloatToByteArray) { + std::unique_ptr bytes(new uint8_t[4]); + + EXPECT_EQ(0, ConvertFloatToByteArray(kPi, bytes.get())); + EXPECT_EQ(0, memcmp(bytes.get(), kPiBytesf, 4)); + + EXPECT_EQ(0, ConvertFloatToByteArray(kE, bytes.get())); + EXPECT_EQ(0, memcmp(bytes.get(), kEBytesf, 4)); + + EXPECT_EQ(0, ConvertFloatToByteArray(kAvogadro, bytes.get())); + EXPECT_EQ(0, memcmp(bytes.get(), kAvogadroBytesf, 4)); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ConvertDoubleToByteArray DISABLED_ConvertDoubleToByteArray +#else +#define MAYBE_ConvertDoubleToByteArray ConvertDoubleToByteArray +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ConvertDoubleToByteArray) { + std::unique_ptr bytes(new uint8_t[8]); + + EXPECT_EQ(0, ConvertDoubleToByteArray(kPi, bytes.get())); + EXPECT_EQ(0, memcmp(bytes.get(), kPiBytes, 8)); + + EXPECT_EQ(0, ConvertDoubleToByteArray(kE, bytes.get())); + EXPECT_EQ(0, memcmp(bytes.get(), kEBytes, 8)); + + EXPECT_EQ(0, ConvertDoubleToByteArray(kAvogadro, bytes.get())); + EXPECT_EQ(0, memcmp(bytes.get(), kAvogadroBytes, 8)); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ReadInt16BufferFromFile DISABLED_ReadInt16BufferFromFile +#else +#define MAYBE_ReadInt16BufferFromFile ReadInt16BufferFromFile +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ReadInt16BufferFromFile) { + std::string test_filename = kTestFileName; + + std::unique_ptr file(FileWrapper::Create()); + + file->OpenFile(test_filename.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kTestFileName.c_str(); + + const size_t kBufferLength = 12; + std::unique_ptr buffer(new int16_t[kBufferLength]); + + EXPECT_EQ(kBufferLength, ReadInt16BufferFromFile(file.get(), + kBufferLength, + buffer.get())); + EXPECT_EQ(22377, buffer[4]); + EXPECT_EQ(16389, buffer[7]); + EXPECT_EQ(17631, buffer[kBufferLength - 1]); + + file->Rewind(); + + // The next test is for checking the case where there are not as much data as + // needed in the file, but reads to the end, and it returns the number of + // int16s read. + const size_t kBufferLenghtLargerThanFile = kBufferLength * 2; + buffer.reset(new int16_t[kBufferLenghtLargerThanFile]); + EXPECT_EQ(kBufferLength, ReadInt16BufferFromFile(file.get(), + kBufferLenghtLargerThanFile, + buffer.get())); + EXPECT_EQ(11544, buffer[0]); + EXPECT_EQ(22377, buffer[4]); + EXPECT_EQ(16389, buffer[7]); + EXPECT_EQ(17631, buffer[kBufferLength - 1]); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ReadInt16FromFileToFloatBuffer \ + DISABLED_ReadInt16FromFileToFloatBuffer +#else +#define MAYBE_ReadInt16FromFileToFloatBuffer ReadInt16FromFileToFloatBuffer +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ReadInt16FromFileToFloatBuffer) { + std::string test_filename = kTestFileName; + + std::unique_ptr file(FileWrapper::Create()); + + file->OpenFile(test_filename.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kTestFileName.c_str(); + + const size_t kBufferLength = 12; + std::unique_ptr buffer(new float[kBufferLength]); + + EXPECT_EQ(kBufferLength, ReadInt16FromFileToFloatBuffer(file.get(), + kBufferLength, + buffer.get())); + + EXPECT_DOUBLE_EQ(11544, buffer[0]); + EXPECT_DOUBLE_EQ(22377, buffer[4]); + EXPECT_DOUBLE_EQ(16389, buffer[7]); + EXPECT_DOUBLE_EQ(17631, buffer[kBufferLength - 1]); + + file->Rewind(); + + // The next test is for checking the case where there are not as much data as + // needed in the file, but reads to the end, and it returns the number of + // int16s read. + const size_t kBufferLenghtLargerThanFile = kBufferLength * 2; + buffer.reset(new float[kBufferLenghtLargerThanFile]); + EXPECT_EQ(kBufferLength, + ReadInt16FromFileToFloatBuffer(file.get(), + kBufferLenghtLargerThanFile, + buffer.get())); + EXPECT_DOUBLE_EQ(11544, buffer[0]); + EXPECT_DOUBLE_EQ(22377, buffer[4]); + EXPECT_DOUBLE_EQ(16389, buffer[7]); + EXPECT_DOUBLE_EQ(17631, buffer[kBufferLength - 1]); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ReadInt16FromFileToDoubleBuffer \ + DISABLED_ReadInt16FromFileToDoubleBuffer +#else +#define MAYBE_ReadInt16FromFileToDoubleBuffer ReadInt16FromFileToDoubleBuffer +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ReadInt16FromFileToDoubleBuffer) { + std::string test_filename = kTestFileName; + + std::unique_ptr file(FileWrapper::Create()); + + file->OpenFile(test_filename.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kTestFileName.c_str(); + + const size_t kBufferLength = 12; + std::unique_ptr buffer(new double[kBufferLength]); + + EXPECT_EQ(kBufferLength, ReadInt16FromFileToDoubleBuffer(file.get(), + kBufferLength, + buffer.get())); + EXPECT_DOUBLE_EQ(11544, buffer[0]); + EXPECT_DOUBLE_EQ(22377, buffer[4]); + EXPECT_DOUBLE_EQ(16389, buffer[7]); + EXPECT_DOUBLE_EQ(17631, buffer[kBufferLength - 1]); + + file->Rewind(); + + // The next test is for checking the case where there are not as much data as + // needed in the file, but reads to the end, and it returns the number of + // int16s read. + const size_t kBufferLenghtLargerThanFile = kBufferLength * 2; + buffer.reset(new double[kBufferLenghtLargerThanFile]); + EXPECT_EQ(kBufferLength, + ReadInt16FromFileToDoubleBuffer(file.get(), + kBufferLenghtLargerThanFile, + buffer.get())); + EXPECT_DOUBLE_EQ(11544, buffer[0]); + EXPECT_DOUBLE_EQ(22377, buffer[4]); + EXPECT_DOUBLE_EQ(16389, buffer[7]); + EXPECT_DOUBLE_EQ(17631, buffer[kBufferLength - 1]); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ReadFloatBufferFromFile DISABLED_ReadFloatBufferFromFile +#else +#define MAYBE_ReadFloatBufferFromFile ReadFloatBufferFromFile +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ReadFloatBufferFromFile) { + std::string test_filename = kTestFileNamef; + + std::unique_ptr file(FileWrapper::Create()); + + file->OpenFile(test_filename.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kTestFileNamef.c_str(); + + const size_t kBufferLength = 3; + std::unique_ptr buffer(new float[kBufferLength]); + + EXPECT_EQ(kBufferLength, ReadFloatBufferFromFile(file.get(), + kBufferLength, + buffer.get())); + EXPECT_FLOAT_EQ(kPi, buffer[0]); + EXPECT_FLOAT_EQ(kE, buffer[1]); + EXPECT_FLOAT_EQ(kAvogadro, buffer[2]); + + file->Rewind(); + + // The next test is for checking the case where there are not as much data as + // needed in the file, but reads to the end, and it returns the number of + // doubles read. + const size_t kBufferLenghtLargerThanFile = kBufferLength * 2; + buffer.reset(new float[kBufferLenghtLargerThanFile]); + EXPECT_EQ(kBufferLength, ReadFloatBufferFromFile(file.get(), + kBufferLenghtLargerThanFile, + buffer.get())); + EXPECT_FLOAT_EQ(kPi, buffer[0]); + EXPECT_FLOAT_EQ(kE, buffer[1]); + EXPECT_FLOAT_EQ(kAvogadro, buffer[2]); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ReadDoubleBufferFromFile DISABLED_ReadDoubleBufferFromFile +#else +#define MAYBE_ReadDoubleBufferFromFile ReadDoubleBufferFromFile +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ReadDoubleBufferFromFile) { + std::string test_filename = kTestFileName; + + std::unique_ptr file(FileWrapper::Create()); + + file->OpenFile(test_filename.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kTestFileName.c_str(); + + const size_t kBufferLength = 3; + std::unique_ptr buffer(new double[kBufferLength]); + + EXPECT_EQ(kBufferLength, ReadDoubleBufferFromFile(file.get(), + kBufferLength, + buffer.get())); + EXPECT_DOUBLE_EQ(kPi, buffer[0]); + EXPECT_DOUBLE_EQ(kE, buffer[1]); + EXPECT_DOUBLE_EQ(kAvogadro, buffer[2]); + + file->Rewind(); + + // The next test is for checking the case where there are not as much data as + // needed in the file, but reads to the end, and it returns the number of + // doubles read. + const size_t kBufferLenghtLargerThanFile = kBufferLength * 2; + buffer.reset(new double[kBufferLenghtLargerThanFile]); + EXPECT_EQ(kBufferLength, ReadDoubleBufferFromFile(file.get(), + kBufferLenghtLargerThanFile, + buffer.get())); + EXPECT_DOUBLE_EQ(kPi, buffer[0]); + EXPECT_DOUBLE_EQ(kE, buffer[1]); + EXPECT_DOUBLE_EQ(kAvogadro, buffer[2]); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_WriteInt16BufferToFile DISABLED_WriteInt16BufferToFile +#else +#define MAYBE_WriteInt16BufferToFile WriteInt16BufferToFile +#endif +TEST_F(TransientFileUtilsTest, MAYBE_WriteInt16BufferToFile) { + std::unique_ptr file(FileWrapper::Create()); + + std::string kOutFileName = CreateTempFilename(test::OutputPath(), + "utils_test"); + + file->OpenFile(kOutFileName.c_str(), false); // Write mode. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kOutFileName.c_str(); + + const size_t kBufferLength = 3; + std::unique_ptr written_buffer(new int16_t[kBufferLength]); + std::unique_ptr read_buffer(new int16_t[kBufferLength]); + + written_buffer[0] = 1; + written_buffer[1] = 2; + written_buffer[2] = 3; + + EXPECT_EQ(kBufferLength, WriteInt16BufferToFile(file.get(), + kBufferLength, + written_buffer.get())); + + file->CloseFile(); + + file->OpenFile(kOutFileName.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kOutFileName.c_str(); + + EXPECT_EQ(kBufferLength, ReadInt16BufferFromFile(file.get(), + kBufferLength, + read_buffer.get())); + EXPECT_EQ(0, memcmp(written_buffer.get(), + read_buffer.get(), + kBufferLength * sizeof(written_buffer[0]))); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_WriteFloatBufferToFile DISABLED_WriteFloatBufferToFile +#else +#define MAYBE_WriteFloatBufferToFile WriteFloatBufferToFile +#endif +TEST_F(TransientFileUtilsTest, MAYBE_WriteFloatBufferToFile) { + std::unique_ptr file(FileWrapper::Create()); + + std::string kOutFileName = CreateTempFilename(test::OutputPath(), + "utils_test"); + + file->OpenFile(kOutFileName.c_str(), false); // Write mode. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kOutFileName.c_str(); + + const size_t kBufferLength = 3; + std::unique_ptr written_buffer(new float[kBufferLength]); + std::unique_ptr read_buffer(new float[kBufferLength]); + + written_buffer[0] = static_cast(kPi); + written_buffer[1] = static_cast(kE); + written_buffer[2] = static_cast(kAvogadro); + + EXPECT_EQ(kBufferLength, WriteFloatBufferToFile(file.get(), + kBufferLength, + written_buffer.get())); + + file->CloseFile(); + + file->OpenFile(kOutFileName.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kOutFileName.c_str(); + + EXPECT_EQ(kBufferLength, ReadFloatBufferFromFile(file.get(), + kBufferLength, + read_buffer.get())); + EXPECT_EQ(0, memcmp(written_buffer.get(), + read_buffer.get(), + kBufferLength * sizeof(written_buffer[0]))); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_WriteDoubleBufferToFile DISABLED_WriteDoubleBufferToFile +#else +#define MAYBE_WriteDoubleBufferToFile WriteDoubleBufferToFile +#endif +TEST_F(TransientFileUtilsTest, MAYBE_WriteDoubleBufferToFile) { + std::unique_ptr file(FileWrapper::Create()); + + std::string kOutFileName = CreateTempFilename(test::OutputPath(), + "utils_test"); + + file->OpenFile(kOutFileName.c_str(), false); // Write mode. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kOutFileName.c_str(); + + const size_t kBufferLength = 3; + std::unique_ptr written_buffer(new double[kBufferLength]); + std::unique_ptr read_buffer(new double[kBufferLength]); + + written_buffer[0] = kPi; + written_buffer[1] = kE; + written_buffer[2] = kAvogadro; + + EXPECT_EQ(kBufferLength, WriteDoubleBufferToFile(file.get(), + kBufferLength, + written_buffer.get())); + + file->CloseFile(); + + file->OpenFile(kOutFileName.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kOutFileName.c_str(); + + EXPECT_EQ(kBufferLength, ReadDoubleBufferFromFile(file.get(), + kBufferLength, + read_buffer.get())); + EXPECT_EQ(0, memcmp(written_buffer.get(), + read_buffer.get(), + kBufferLength * sizeof(written_buffer[0]))); +} + +#if defined(WEBRTC_IOS) +#define MAYBE_ExpectedErrorReturnValues DISABLED_ExpectedErrorReturnValues +#else +#define MAYBE_ExpectedErrorReturnValues ExpectedErrorReturnValues +#endif +TEST_F(TransientFileUtilsTest, MAYBE_ExpectedErrorReturnValues) { + std::string test_filename = kTestFileName; + + double value; + std::unique_ptr int16_buffer(new int16_t[1]); + std::unique_ptr double_buffer(new double[1]); + std::unique_ptr file(FileWrapper::Create()); + + EXPECT_EQ(-1, ConvertByteArrayToDouble(NULL, &value)); + EXPECT_EQ(-1, ConvertByteArrayToDouble(kPiBytes, NULL)); + + EXPECT_EQ(-1, ConvertDoubleToByteArray(kPi, NULL)); + + // Tests with file not opened. + EXPECT_EQ(0u, ReadInt16BufferFromFile(file.get(), 1, int16_buffer.get())); + EXPECT_EQ(0u, ReadInt16FromFileToDoubleBuffer(file.get(), + 1, + double_buffer.get())); + EXPECT_EQ(0u, ReadDoubleBufferFromFile(file.get(), 1, double_buffer.get())); + EXPECT_EQ(0u, WriteInt16BufferToFile(file.get(), 1, int16_buffer.get())); + EXPECT_EQ(0u, WriteDoubleBufferToFile(file.get(), 1, double_buffer.get())); + + file->OpenFile(test_filename.c_str(), true); // Read only. + ASSERT_TRUE(file->is_open()) << "File could not be opened:\n" + << kTestFileName.c_str(); + + EXPECT_EQ(0u, ReadInt16BufferFromFile(NULL, 1, int16_buffer.get())); + EXPECT_EQ(0u, ReadInt16BufferFromFile(file.get(), 1, NULL)); + EXPECT_EQ(0u, ReadInt16BufferFromFile(file.get(), 0, int16_buffer.get())); + + EXPECT_EQ(0u, ReadInt16FromFileToDoubleBuffer(NULL, 1, double_buffer.get())); + EXPECT_EQ(0u, ReadInt16FromFileToDoubleBuffer(file.get(), 1, NULL)); + EXPECT_EQ(0u, ReadInt16FromFileToDoubleBuffer(file.get(), + 0, + double_buffer.get())); + + EXPECT_EQ(0u, ReadDoubleBufferFromFile(NULL, 1, double_buffer.get())); + EXPECT_EQ(0u, ReadDoubleBufferFromFile(file.get(), 1, NULL)); + EXPECT_EQ(0u, ReadDoubleBufferFromFile(file.get(), 0, double_buffer.get())); + + EXPECT_EQ(0u, WriteInt16BufferToFile(NULL, 1, int16_buffer.get())); + EXPECT_EQ(0u, WriteInt16BufferToFile(file.get(), 1, NULL)); + EXPECT_EQ(0u, WriteInt16BufferToFile(file.get(), 0, int16_buffer.get())); + + EXPECT_EQ(0u, WriteDoubleBufferToFile(NULL, 1, double_buffer.get())); + EXPECT_EQ(0u, WriteDoubleBufferToFile(file.get(), 1, NULL)); + EXPECT_EQ(0u, WriteDoubleBufferToFile(file.get(), 0, double_buffer.get())); +} + +} // namespace webrtc + diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.cc new file mode 100644 index 0000000000..46b16b82bc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/moving_moments.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +MovingMoments::MovingMoments(size_t length) + : length_(length), + queue_(), + sum_(0.0), + sum_of_squares_(0.0) { + RTC_DCHECK_GT(length, 0); + for (size_t i = 0; i < length; ++i) { + queue_.push(0.0); + } +} + +MovingMoments::~MovingMoments() {} + +void MovingMoments::CalculateMoments(const float* in, size_t in_length, + float* first, float* second) { + RTC_DCHECK(in); + RTC_DCHECK_GT(in_length, 0); + RTC_DCHECK(first); + RTC_DCHECK(second); + + for (size_t i = 0; i < in_length; ++i) { + const float old_value = queue_.front(); + queue_.pop(); + queue_.push(in[i]); + + sum_ += in[i] - old_value; + sum_of_squares_ += in[i] * in[i] - old_value * old_value; + first[i] = sum_ / length_; + second[i] = sum_of_squares_ / length_; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.h new file mode 100644 index 0000000000..f1b3e38cc0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_MOVING_MOMENTS_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_MOVING_MOMENTS_H_ + +#include + +#include + +namespace webrtc { + +// Calculates the first and second moments for each value of a buffer taking +// into account a given number of previous values. +// It preserves its state, so it can be multiple-called. +// TODO(chadan): Implement a function that takes a buffer of first moments and a +// buffer of second moments; and calculates the variances. When needed. +// TODO(chadan): Add functionality to update with a buffer but only output are +// the last values of the moments. When needed. +class MovingMoments { + public: + // Creates a Moving Moments object, that uses the last |length| values + // (including the new value introduced in every new calculation). + explicit MovingMoments(size_t length); + ~MovingMoments(); + + // Calculates the new values using |in|. Results will be in the out buffers. + // |first| and |second| must be allocated with at least |in_length|. + void CalculateMoments(const float* in, size_t in_length, + float* first, float* second); + + private: + size_t length_; + // A queue holding the |length_| latest input values. + std::queue queue_; + // Sum of the values of the queue. + float sum_; + // Sum of the squares of the values of the queue. + float sum_of_squares_; +}; + +} // namespace webrtc + + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_MOVING_MOMENTS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments_unittest.cc new file mode 100644 index 0000000000..057bc454e0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/moving_moments_unittest.cc @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/moving_moments.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { + +static const float kTolerance = 0.0001f; + +class MovingMomentsTest : public ::testing::Test { + protected: + static const size_t kMovingMomentsBufferLength = 5; + static const size_t kMaxOutputLength = 20; // Valid for this tests only. + + virtual void SetUp(); + // Calls CalculateMoments and verifies that it produces the expected + // outputs. + void CalculateMomentsAndVerify(const float* input, size_t input_length, + const float* expected_mean, + const float* expected_mean_squares); + + std::unique_ptr moving_moments_; + float output_mean_[kMaxOutputLength]; + float output_mean_squares_[kMaxOutputLength]; +}; + +const size_t MovingMomentsTest::kMaxOutputLength; + +void MovingMomentsTest::SetUp() { + moving_moments_.reset(new MovingMoments(kMovingMomentsBufferLength)); +} + +void MovingMomentsTest::CalculateMomentsAndVerify( + const float* input, size_t input_length, + const float* expected_mean, + const float* expected_mean_squares) { + ASSERT_LE(input_length, kMaxOutputLength); + + moving_moments_->CalculateMoments(input, + input_length, + output_mean_, + output_mean_squares_); + + for (size_t i = 1; i < input_length; ++i) { + EXPECT_NEAR(expected_mean[i], output_mean_[i], kTolerance); + EXPECT_NEAR(expected_mean_squares[i], output_mean_squares_[i], kTolerance); + } +} + +TEST_F(MovingMomentsTest, CorrectMomentsOfAnAllZerosBuffer) { + const float kInput[] = {0.f, 0.f, 0.f, 0.f, 0.f}; + const size_t kInputLength = sizeof(kInput) / sizeof(kInput[0]); + + const float expected_mean[kInputLength] = {0.f, 0.f, 0.f, 0.f, 0.f}; + const float expected_mean_squares[kInputLength] = {0.f, 0.f, 0.f, 0.f, 0.f}; + + CalculateMomentsAndVerify(kInput, kInputLength, expected_mean, + expected_mean_squares); +} + +TEST_F(MovingMomentsTest, CorrectMomentsOfAConstantBuffer) { + const float kInput[] = {5.f, 5.f, 5.f, 5.f, 5.f, 5.f, 5.f, 5.f, 5.f, 5.f}; + const size_t kInputLength = sizeof(kInput) / sizeof(kInput[0]); + + const float expected_mean[kInputLength] = + {1.f, 2.f, 3.f, 4.f, 5.f, 5.f, 5.f, 5.f, 5.f, 5.f}; + const float expected_mean_squares[kInputLength] = + {5.f, 10.f, 15.f, 20.f, 25.f, 25.f, 25.f, 25.f, 25.f, 25.f}; + + CalculateMomentsAndVerify(kInput, kInputLength, expected_mean, + expected_mean_squares); +} + +TEST_F(MovingMomentsTest, CorrectMomentsOfAnIncreasingBuffer) { + const float kInput[] = {1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f}; + const size_t kInputLength = sizeof(kInput) / sizeof(kInput[0]); + + const float expected_mean[kInputLength] = + {0.2f, 0.6f, 1.2f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f}; + const float expected_mean_squares[kInputLength] = + {0.2f, 1.f, 2.8f, 6.f, 11.f, 18.f, 27.f, 38.f, 51.f}; + + CalculateMomentsAndVerify(kInput, kInputLength, expected_mean, + expected_mean_squares); +} + +TEST_F(MovingMomentsTest, CorrectMomentsOfADecreasingBuffer) { + const float kInput[] = + {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f, -7.f, -8.f, -9.f}; + const size_t kInputLength = sizeof(kInput) / sizeof(kInput[0]); + + const float expected_mean[kInputLength] = + {-0.2f, -0.6f, -1.2f, -2.f, -3.f, -4.f, -5.f, -6.f, -7.f}; + const float expected_mean_squares[kInputLength] = + {0.2f, 1.f, 2.8f, 6.f, 11.f, 18.f, 27.f, 38.f, 51.f}; + + CalculateMomentsAndVerify(kInput, kInputLength, expected_mean, + expected_mean_squares); +} + +TEST_F(MovingMomentsTest, CorrectMomentsOfAZeroMeanSequence) { + const size_t kMovingMomentsBufferLength = 4; + moving_moments_.reset(new MovingMoments(kMovingMomentsBufferLength)); + const float kInput[] = + {1.f, -1.f, 1.f, -1.f, 1.f, -1.f, 1.f, -1.f, 1.f, -1.f}; + const size_t kInputLength = sizeof(kInput) / sizeof(kInput[0]); + + const float expected_mean[kInputLength] = + {0.25f, 0.f, 0.25f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + const float expected_mean_squares[kInputLength] = + {0.25f, 0.5f, 0.75f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f}; + + CalculateMomentsAndVerify(kInput, kInputLength, expected_mean, + expected_mean_squares); +} + +TEST_F(MovingMomentsTest, CorrectMomentsOfAnArbitraryBuffer) { + const float kInput[] = + {0.2f, 0.3f, 0.5f, 0.7f, 0.11f, 0.13f, 0.17f, 0.19f, 0.23f}; + const size_t kInputLength = sizeof(kInput) / sizeof(kInput[0]); + + const float expected_mean[kInputLength] = + {0.04f, 0.1f, 0.2f, 0.34f, 0.362f, 0.348f, 0.322f, 0.26f, 0.166f}; + const float expected_mean_squares[kInputLength] = + {0.008f, 0.026f, 0.076f, 0.174f, 0.1764f, 0.1718f, 0.1596f, 0.1168f, + 0.0294f}; + + CalculateMomentsAndVerify(kInput, kInputLength, expected_mean, + expected_mean_squares); +} + +TEST_F(MovingMomentsTest, MutipleCalculateMomentsCalls) { + const float kInputFirstCall[] = + {0.2f, 0.3f, 0.5f, 0.7f, 0.11f, 0.13f, 0.17f, 0.19f, 0.23f}; + const size_t kInputFirstCallLength = sizeof(kInputFirstCall) / + sizeof(kInputFirstCall[0]); + const float kInputSecondCall[] = {0.29f, 0.31f}; + const size_t kInputSecondCallLength = sizeof(kInputSecondCall) / + sizeof(kInputSecondCall[0]); + const float kInputThirdCall[] = {0.37f, 0.41f, 0.43f, 0.47f}; + const size_t kInputThirdCallLength = sizeof(kInputThirdCall) / + sizeof(kInputThirdCall[0]); + + const float expected_mean_first_call[kInputFirstCallLength] = + {0.04f, 0.1f, 0.2f, 0.34f, 0.362f, 0.348f, 0.322f, 0.26f, 0.166f}; + const float expected_mean_squares_first_call[kInputFirstCallLength] = + {0.008f, 0.026f, 0.076f, 0.174f, 0.1764f, 0.1718f, 0.1596f, 0.1168f, + 0.0294f}; + + const float expected_mean_second_call[kInputSecondCallLength] = + {0.202f, 0.238f}; + const float expected_mean_squares_second_call[kInputSecondCallLength] = + {0.0438f, 0.0596f}; + + const float expected_mean_third_call[kInputThirdCallLength] = + {0.278f, 0.322f, 0.362f, 0.398f}; + const float expected_mean_squares_third_call[kInputThirdCallLength] = + {0.0812f, 0.1076f, 0.134f, 0.1614f}; + + CalculateMomentsAndVerify(kInputFirstCall, kInputFirstCallLength, + expected_mean_first_call, expected_mean_squares_first_call); + + CalculateMomentsAndVerify(kInputSecondCall, kInputSecondCallLength, + expected_mean_second_call, expected_mean_squares_second_call); + + CalculateMomentsAndVerify(kInputThirdCall, kInputThirdCallLength, + expected_mean_third_call, expected_mean_squares_third_call); +} + +TEST_F(MovingMomentsTest, + VerifySampleBasedVsBlockBasedCalculation) { + const float kInput[] = + {0.2f, 0.3f, 0.5f, 0.7f, 0.11f, 0.13f, 0.17f, 0.19f, 0.23f}; + const size_t kInputLength = sizeof(kInput) / sizeof(kInput[0]); + + float output_mean_block_based[kInputLength]; + float output_mean_squares_block_based[kInputLength]; + + float output_mean_sample_based; + float output_mean_squares_sample_based; + + moving_moments_->CalculateMoments( + kInput, kInputLength, output_mean_block_based, + output_mean_squares_block_based); + moving_moments_.reset(new MovingMoments(kMovingMomentsBufferLength)); + for (size_t i = 0; i < kInputLength; ++i) { + moving_moments_->CalculateMoments( + &kInput[i], 1, &output_mean_sample_based, + &output_mean_squares_sample_based); + EXPECT_FLOAT_EQ(output_mean_block_based[i], output_mean_sample_based); + EXPECT_FLOAT_EQ(output_mean_squares_block_based[i], + output_mean_squares_sample_based); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/plotDetection.m b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/plotDetection.m new file mode 100644 index 0000000000..8e12ab920b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/plotDetection.m @@ -0,0 +1,22 @@ +% +% Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function [] = plotDetection(PCMfile, DATfile, fs, chunkSize) +%[] = plotDetection(PCMfile, DATfile, fs, chunkSize) +% +%Plots the signal alongside the detection values. +% +%PCMfile: The file of the input signal in PCM format. +%DATfile: The file containing the detection values in binary float format. +%fs: The sample rate of the signal in Hertz. +%chunkSize: The chunk size used to compute the detection values in seconds. +[x, tx] = readPCM(PCMfile, fs); +[d, td] = readDetection(DATfile, fs, chunkSize); +plot(tx, x, td, d); diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readDetection.m b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readDetection.m new file mode 100644 index 0000000000..832bf31ec8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readDetection.m @@ -0,0 +1,26 @@ +% +% Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function [d, t] = readDetection(file, fs, chunkSize) +%[d, t] = readDetection(file, fs, chunkSize) +% +%Reads a detection signal from a DAT file. +% +%d: The detection signal. +%t: The respective time vector. +% +%file: The DAT file where the detection signal is stored in float format. +%fs: The signal sample rate in Hertz. +%chunkSize: The chunk size used for the detection in seconds. +fid = fopen(file); +d = fread(fid, inf, 'float'); +fclose(fid); +t = 0:(1 / fs):(length(d) * chunkSize - 1 / fs); +d = d(floor(t / chunkSize) + 1); diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readPCM.m b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readPCM.m new file mode 100644 index 0000000000..cd3cef8a3c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/test/readPCM.m @@ -0,0 +1,26 @@ +% +% Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function [x, t] = readPCM(file, fs) +%[x, t] = readPCM(file, fs) +% +%Reads a signal from a PCM file. +% +%x: The read signal after normalization. +%t: The respective time vector. +% +%file: The PCM file where the signal is stored in int16 format. +%fs: The signal sample rate in Hertz. +fid = fopen(file); +x = fread(fid, inf, 'int16'); +fclose(fid); +x = x - mean(x); +x = x / max(abs(x)); +t = 0:(1 / fs):((length(x) - 1) / fs); diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.cc new file mode 100644 index 0000000000..1bb6f9f10a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.cc @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/transient_detector.h" + +#include +#include +#include + +#include + +#include "modules/audio_processing/transient/common.h" +#include "modules/audio_processing/transient/daubechies_8_wavelet_coeffs.h" +#include "modules/audio_processing/transient/moving_moments.h" +#include "modules/audio_processing/transient/wpd_tree.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +static const int kTransientLengthMs = 30; +static const int kChunksAtStartupLeftToDelete = + kTransientLengthMs / ts::kChunkSizeMs; +static const float kDetectThreshold = 16.f; + +TransientDetector::TransientDetector(int sample_rate_hz) + : samples_per_chunk_(sample_rate_hz * ts::kChunkSizeMs / 1000), + last_first_moment_(), + last_second_moment_(), + chunks_at_startup_left_to_delete_(kChunksAtStartupLeftToDelete), + reference_energy_(1.f), + using_reference_(false) { + RTC_DCHECK(sample_rate_hz == ts::kSampleRate8kHz || + sample_rate_hz == ts::kSampleRate16kHz || + sample_rate_hz == ts::kSampleRate32kHz || + sample_rate_hz == ts::kSampleRate48kHz); + int samples_per_transient = sample_rate_hz * kTransientLengthMs / 1000; + // Adjustment to avoid data loss while downsampling, making + // |samples_per_chunk_| and |samples_per_transient| always divisible by + // |kLeaves|. + samples_per_chunk_ -= samples_per_chunk_ % kLeaves; + samples_per_transient -= samples_per_transient % kLeaves; + + tree_leaves_data_length_ = samples_per_chunk_ / kLeaves; + wpd_tree_.reset(new WPDTree(samples_per_chunk_, + kDaubechies8HighPassCoefficients, + kDaubechies8LowPassCoefficients, + kDaubechies8CoefficientsLength, + kLevels)); + for (size_t i = 0; i < kLeaves; ++i) { + moving_moments_[i].reset( + new MovingMoments(samples_per_transient / kLeaves)); + } + + first_moments_.reset(new float[tree_leaves_data_length_]); + second_moments_.reset(new float[tree_leaves_data_length_]); + + for (int i = 0; i < kChunksAtStartupLeftToDelete; ++i) { + previous_results_.push_back(0.f); + } +} + +TransientDetector::~TransientDetector() {} + +float TransientDetector::Detect(const float* data, + size_t data_length, + const float* reference_data, + size_t reference_length) { + RTC_DCHECK(data); + RTC_DCHECK_EQ(samples_per_chunk_, data_length); + + // TODO(aluebs): Check if these errors can logically happen and if not assert + // on them. + if (wpd_tree_->Update(data, samples_per_chunk_) != 0) { + return -1.f; + } + + float result = 0.f; + + for (size_t i = 0; i < kLeaves; ++i) { + WPDNode* leaf = wpd_tree_->NodeAt(kLevels, i); + + moving_moments_[i]->CalculateMoments(leaf->data(), + tree_leaves_data_length_, + first_moments_.get(), + second_moments_.get()); + + // Add value delayed (Use the last moments from the last call to Detect). + float unbiased_data = leaf->data()[0] - last_first_moment_[i]; + result += + unbiased_data * unbiased_data / (last_second_moment_[i] + FLT_MIN); + + // Add new values. + for (size_t j = 1; j < tree_leaves_data_length_; ++j) { + unbiased_data = leaf->data()[j] - first_moments_[j - 1]; + result += + unbiased_data * unbiased_data / (second_moments_[j - 1] + FLT_MIN); + } + + last_first_moment_[i] = first_moments_[tree_leaves_data_length_ - 1]; + last_second_moment_[i] = second_moments_[tree_leaves_data_length_ - 1]; + } + + result /= tree_leaves_data_length_; + + result *= ReferenceDetectionValue(reference_data, reference_length); + + if (chunks_at_startup_left_to_delete_ > 0) { + chunks_at_startup_left_to_delete_--; + result = 0.f; + } + + if (result >= kDetectThreshold) { + result = 1.f; + } else { + // Get proportional value. + // Proportion achieved with a squared raised cosine function with domain + // [0, kDetectThreshold) and image [0, 1), it's always increasing. + const float horizontal_scaling = ts::kPi / kDetectThreshold; + const float kHorizontalShift = ts::kPi; + const float kVerticalScaling = 0.5f; + const float kVerticalShift = 1.f; + + result = (cos(result * horizontal_scaling + kHorizontalShift) + + kVerticalShift) * kVerticalScaling; + result *= result; + } + + previous_results_.pop_front(); + previous_results_.push_back(result); + + // In the current implementation we return the max of the current result and + // the previous results, so the high results have a width equals to + // |transient_length|. + return *std::max_element(previous_results_.begin(), previous_results_.end()); +} + +// Looks for the highest slope and compares it with the previous ones. +// An exponential transformation takes this to the [0, 1] range. This value is +// multiplied by the detection result to avoid false positives. +float TransientDetector::ReferenceDetectionValue(const float* data, + size_t length) { + if (data == NULL) { + using_reference_ = false; + return 1.f; + } + static const float kEnergyRatioThreshold = 0.2f; + static const float kReferenceNonLinearity = 20.f; + static const float kMemory = 0.99f; + float reference_energy = 0.f; + for (size_t i = 1; i < length; ++i) { + reference_energy += data[i] * data[i]; + } + if (reference_energy == 0.f) { + using_reference_ = false; + return 1.f; + } + RTC_DCHECK_NE(0, reference_energy_); + float result = 1.f / (1.f + exp(kReferenceNonLinearity * + (kEnergyRatioThreshold - + reference_energy / reference_energy_))); + reference_energy_ = + kMemory * reference_energy_ + (1.f - kMemory) * reference_energy; + + using_reference_ = true; + + return result; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.h new file mode 100644 index 0000000000..3267b3a6cd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_DETECTOR_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_DETECTOR_H_ + +#include +#include + +#include "modules/audio_processing/transient/moving_moments.h" +#include "modules/audio_processing/transient/wpd_tree.h" + +namespace webrtc { + +// This is an implementation of the transient detector described in "Causal +// Wavelet based transient detector". +// Calculates the log-likelihood of a transient to happen on a signal at any +// given time based on the previous samples; it uses a WPD tree to analyze the +// signal. It preserves its state, so it can be multiple-called. +class TransientDetector { + public: + // TODO(chadan): The only supported wavelet is Daubechies 8 using a WPD tree + // of 3 levels. Make an overloaded constructor to allow different wavelets and + // depths of the tree. When needed. + + // Creates a wavelet based transient detector. + TransientDetector(int sample_rate_hz); + + ~TransientDetector(); + + // Calculates the log-likelihood of the existence of a transient in |data|. + // |data_length| has to be equal to |samples_per_chunk_|. + // Returns a value between 0 and 1, as a non linear representation of this + // likelihood. + // Returns a negative value on error. + float Detect(const float* data, + size_t data_length, + const float* reference_data, + size_t reference_length); + + bool using_reference() { return using_reference_; } + + private: + float ReferenceDetectionValue(const float* data, size_t length); + + static const size_t kLevels = 3; + static const size_t kLeaves = 1 << kLevels; + + size_t samples_per_chunk_; + + std::unique_ptr wpd_tree_; + size_t tree_leaves_data_length_; + + // A MovingMoments object is needed for each leaf in the WPD tree. + std::unique_ptr moving_moments_[kLeaves]; + + std::unique_ptr first_moments_; + std::unique_ptr second_moments_; + + // Stores the last calculated moments from the previous detection. + float last_first_moment_[kLeaves]; + float last_second_moment_[kLeaves]; + + // We keep track of the previous results from the previous chunks, so it can + // be used to effectively give results according to the |transient_length|. + std::deque previous_results_; + + // Number of chunks that are going to return only zeros at the beginning of + // the detection. It helps to avoid infs and nans due to the lack of + // information. + int chunks_at_startup_left_to_delete_; + + float reference_energy_; + + bool using_reference_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_DETECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector_unittest.cc new file mode 100644 index 0000000000..96af179c42 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_detector_unittest.cc @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/transient_detector.h" + +#include +#include +#include + +#include "modules/audio_processing/transient/common.h" +#include "modules/audio_processing/transient/file_utils.h" +#include "system_wrappers/include/file_wrapper.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +static const int kSampleRatesHz[] = {ts::kSampleRate8kHz, + ts::kSampleRate16kHz, + ts::kSampleRate32kHz, + ts::kSampleRate48kHz}; +static const size_t kNumberOfSampleRates = + sizeof(kSampleRatesHz) / sizeof(*kSampleRatesHz); + +// This test is for the correctness of the transient detector. +// Checks the results comparing them with the ones stored in the detect files in +// the directory: resources/audio_processing/transient/ +// The files contain all the results in double precision (Little endian). +// The audio files used with different sample rates are stored in the same +// directory. +#if defined(WEBRTC_IOS) +TEST(TransientDetectorTest, DISABLED_CorrectnessBasedOnFiles) { +#else +TEST(TransientDetectorTest, CorrectnessBasedOnFiles) { +#endif + for (size_t i = 0; i < kNumberOfSampleRates; ++i) { + int sample_rate_hz = kSampleRatesHz[i]; + + // Prepare detect file. + std::stringstream detect_file_name; + detect_file_name << "audio_processing/transient/detect" + << (sample_rate_hz / 1000) << "kHz"; + + std::unique_ptr detect_file(FileWrapper::Create()); + + detect_file->OpenFile( + test::ResourcePath(detect_file_name.str(), "dat").c_str(), + true); // Read only. + + bool file_opened = detect_file->is_open(); + ASSERT_TRUE(file_opened) << "File could not be opened.\n" + << detect_file_name.str().c_str(); + + // Prepare audio file. + std::stringstream audio_file_name; + audio_file_name << "audio_processing/transient/audio" + << (sample_rate_hz / 1000) << "kHz"; + + std::unique_ptr audio_file(FileWrapper::Create()); + + audio_file->OpenFile( + test::ResourcePath(audio_file_name.str(), "pcm").c_str(), + true); // Read only. + + // Create detector. + TransientDetector detector(sample_rate_hz); + + const size_t buffer_length = sample_rate_hz * ts::kChunkSizeMs / 1000; + std::unique_ptr buffer(new float[buffer_length]); + + const float kTolerance = 0.02f; + + size_t frames_read = 0; + + while (ReadInt16FromFileToFloatBuffer(audio_file.get(), + buffer_length, + buffer.get()) == buffer_length) { + ++frames_read; + + float detector_value = + detector.Detect(buffer.get(), buffer_length, NULL, 0); + double file_value; + ASSERT_EQ(1u, ReadDoubleBufferFromFile(detect_file.get(), 1, &file_value)) + << "Detect test file is malformed.\n"; + + // Compare results with data from the matlab test file. + EXPECT_NEAR(file_value, detector_value, kTolerance) << "Frame: " + << frames_read; + } + + detect_file->CloseFile(); + audio_file->CloseFile(); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppression_test.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppression_test.cc new file mode 100644 index 0000000000..3442ee0af6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppression_test.cc @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/transient_suppressor.h" + +#include +#include +#include + +#include +#include + +#include "common_audio/include/audio_util.h" +#include "modules/audio_processing/agc/agc.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/flags.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" +#include "typedefs.h" // NOLINT(build/include) + +DEFINE_string(in_file_name, "", "PCM file that contains the signal."); +DEFINE_string(detection_file_name, + "", + "PCM file that contains the detection signal."); +DEFINE_string(reference_file_name, + "", + "PCM file that contains the reference signal."); + +DEFINE_int(chunk_size_ms, + 10, + "Time between each chunk of samples in milliseconds."); + +DEFINE_int(sample_rate_hz, + 16000, + "Sampling frequency of the signal in Hertz."); +DEFINE_int(detection_rate_hz, + 0, + "Sampling frequency of the detection signal in Hertz."); + +DEFINE_int(num_channels, 1, "Number of channels."); + +DEFINE_bool(help, false, "Print this message."); + +namespace webrtc { + +const char kUsage[] = + "\nDetects and suppresses transients from file.\n\n" + "This application loads the signal from the in_file_name with a specific\n" + "num_channels and sample_rate_hz, the detection signal from the\n" + "detection_file_name with a specific detection_rate_hz, and the reference\n" + "signal from the reference_file_name with sample_rate_hz, divides them\n" + "into chunk_size_ms blocks, computes its voice value and depending on the\n" + "voice_threshold does the respective restoration. You can always get the\n" + "all-voiced or all-unvoiced cases by setting the voice_threshold to 0 or\n" + "1 respectively.\n\n"; + +// Read next buffers from the test files (signed 16-bit host-endian PCM +// format). audio_buffer has int16 samples, detection_buffer has float samples +// with range [-32768,32767], and reference_buffer has float samples with range +// [-1,1]. Return true iff all the buffers were filled completely. +bool ReadBuffers(FILE* in_file, + size_t audio_buffer_size, + int num_channels, + int16_t* audio_buffer, + FILE* detection_file, + size_t detection_buffer_size, + float* detection_buffer, + FILE* reference_file, + float* reference_buffer) { + std::unique_ptr tmpbuf; + int16_t* read_ptr = audio_buffer; + if (num_channels > 1) { + tmpbuf.reset(new int16_t[num_channels * audio_buffer_size]); + read_ptr = tmpbuf.get(); + } + if (fread(read_ptr, + sizeof(*read_ptr), + num_channels * audio_buffer_size, + in_file) != num_channels * audio_buffer_size) { + return false; + } + // De-interleave. + if (num_channels > 1) { + for (int i = 0; i < num_channels; ++i) { + for (size_t j = 0; j < audio_buffer_size; ++j) { + audio_buffer[i * audio_buffer_size + j] = + read_ptr[i + j * num_channels]; + } + } + } + if (detection_file) { + std::unique_ptr ibuf(new int16_t[detection_buffer_size]); + if (fread(ibuf.get(), sizeof(ibuf[0]), detection_buffer_size, + detection_file) != detection_buffer_size) + return false; + for (size_t i = 0; i < detection_buffer_size; ++i) + detection_buffer[i] = ibuf[i]; + } + if (reference_file) { + std::unique_ptr ibuf(new int16_t[audio_buffer_size]); + if (fread(ibuf.get(), sizeof(ibuf[0]), audio_buffer_size, reference_file) + != audio_buffer_size) + return false; + S16ToFloat(ibuf.get(), audio_buffer_size, reference_buffer); + } + return true; +} + +// Write a number of samples to an open signed 16-bit host-endian PCM file. +static void WritePCM(FILE* f, + size_t num_samples, + int num_channels, + const float* buffer) { + std::unique_ptr ibuf(new int16_t[num_channels * num_samples]); + // Interleave. + for (int i = 0; i < num_channels; ++i) { + for (size_t j = 0; j < num_samples; ++j) { + ibuf[i + j * num_channels] = FloatS16ToS16(buffer[i * num_samples + j]); + } + } + fwrite(ibuf.get(), sizeof(ibuf[0]), num_channels * num_samples, f); +} + +// This application tests the transient suppression by providing a processed +// PCM file, which has to be listened to in order to evaluate the +// performance. +// It gets an audio file, and its voice gain information, and the suppressor +// process it giving the output file "suppressed_keystrokes.pcm". +void void_main() { + // TODO(aluebs): Remove all FileWrappers. + // Prepare the input file. + FILE* in_file = fopen(FLAG_in_file_name, "rb"); + ASSERT_TRUE(in_file != NULL); + + // Prepare the detection file. + FILE* detection_file = NULL; + if (strlen(FLAG_detection_file_name) > 0) { + detection_file = fopen(FLAG_detection_file_name, "rb"); + } + + // Prepare the reference file. + FILE* reference_file = NULL; + if (strlen(FLAG_reference_file_name) > 0) { + reference_file = fopen(FLAG_reference_file_name, "rb"); + } + + // Prepare the output file. + std::string out_file_name = test::OutputPath() + "suppressed_keystrokes.pcm"; + FILE* out_file = fopen(out_file_name.c_str(), "wb"); + ASSERT_TRUE(out_file != NULL); + + int detection_rate_hz = FLAG_detection_rate_hz; + if (detection_rate_hz == 0) { + detection_rate_hz = FLAG_sample_rate_hz; + } + + Agc agc; + + TransientSuppressor suppressor; + suppressor.Initialize( + FLAG_sample_rate_hz, detection_rate_hz, FLAG_num_channels); + + const size_t audio_buffer_size = + FLAG_chunk_size_ms * FLAG_sample_rate_hz / 1000; + const size_t detection_buffer_size = + FLAG_chunk_size_ms * detection_rate_hz / 1000; + + // int16 and float variants of the same data. + std::unique_ptr audio_buffer_i( + new int16_t[FLAG_num_channels * audio_buffer_size]); + std::unique_ptr audio_buffer_f( + new float[FLAG_num_channels * audio_buffer_size]); + + std::unique_ptr detection_buffer, reference_buffer; + + if (detection_file) + detection_buffer.reset(new float[detection_buffer_size]); + if (reference_file) + reference_buffer.reset(new float[audio_buffer_size]); + + while (ReadBuffers(in_file, + audio_buffer_size, + FLAG_num_channels, + audio_buffer_i.get(), + detection_file, + detection_buffer_size, + detection_buffer.get(), + reference_file, + reference_buffer.get())) { + ASSERT_EQ(0, + agc.Process(audio_buffer_i.get(), + static_cast(audio_buffer_size), + FLAG_sample_rate_hz)) + << "The AGC could not process the frame"; + + for (size_t i = 0; i < FLAG_num_channels * audio_buffer_size; ++i) { + audio_buffer_f[i] = audio_buffer_i[i]; + } + + ASSERT_EQ(0, + suppressor.Suppress(audio_buffer_f.get(), + audio_buffer_size, + FLAG_num_channels, + detection_buffer.get(), + detection_buffer_size, + reference_buffer.get(), + audio_buffer_size, + agc.voice_probability(), + true)) + << "The transient suppressor could not suppress the frame"; + + // Write result to out file. + WritePCM( + out_file, audio_buffer_size, FLAG_num_channels, audio_buffer_f.get()); + } + + fclose(in_file); + if (detection_file) { + fclose(detection_file); + } + if (reference_file) { + fclose(reference_file); + } + fclose(out_file); +} + +} // namespace webrtc + +int main(int argc, char* argv[]) { + if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || + FLAG_help || argc != 1) { + printf("%s", webrtc::kUsage); + if (FLAG_help) { + rtc::FlagList::Print(nullptr, false); + return 0; + } + return 1; + } + RTC_CHECK_GT(FLAG_chunk_size_ms, 0); + RTC_CHECK_GT(FLAG_sample_rate_hz, 0); + RTC_CHECK_GT(FLAG_num_channels, 0); + + webrtc::void_main(); + return 0; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.cc new file mode 100644 index 0000000000..9bbd7d9500 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.cc @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/transient_suppressor.h" + +#include +#include +#include +#include +#include +#include + +#include "common_audio/fft4g.h" +#include "common_audio/include/audio_util.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_processing/ns/windows_private.h" +#include "modules/audio_processing/transient/common.h" +#include "modules/audio_processing/transient/transient_detector.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +static const float kMeanIIRCoefficient = 0.5f; +static const float kVoiceThreshold = 0.02f; + +// TODO(aluebs): Check if these values work also for 48kHz. +static const size_t kMinVoiceBin = 3; +static const size_t kMaxVoiceBin = 60; + +namespace { + +float ComplexMagnitude(float a, float b) { + return std::abs(a) + std::abs(b); +} + +} // namespace + +TransientSuppressor::TransientSuppressor() + : data_length_(0), + detection_length_(0), + analysis_length_(0), + buffer_delay_(0), + complex_analysis_length_(0), + num_channels_(0), + window_(NULL), + detector_smoothed_(0.f), + keypress_counter_(0), + chunks_since_keypress_(0), + detection_enabled_(false), + suppression_enabled_(false), + use_hard_restoration_(false), + chunks_since_voice_change_(0), + seed_(182), + using_reference_(false) { +} + +TransientSuppressor::~TransientSuppressor() {} + +int TransientSuppressor::Initialize(int sample_rate_hz, + int detection_rate_hz, + int num_channels) { + switch (sample_rate_hz) { + case ts::kSampleRate8kHz: + analysis_length_ = 128u; + window_ = kBlocks80w128; + break; + case ts::kSampleRate16kHz: + analysis_length_ = 256u; + window_ = kBlocks160w256; + break; + case ts::kSampleRate32kHz: + analysis_length_ = 512u; + window_ = kBlocks320w512; + break; + case ts::kSampleRate48kHz: + analysis_length_ = 1024u; + window_ = kBlocks480w1024; + break; + default: + return -1; + } + if (detection_rate_hz != ts::kSampleRate8kHz && + detection_rate_hz != ts::kSampleRate16kHz && + detection_rate_hz != ts::kSampleRate32kHz && + detection_rate_hz != ts::kSampleRate48kHz) { + return -1; + } + if (num_channels <= 0) { + return -1; + } + + detector_.reset(new TransientDetector(detection_rate_hz)); + data_length_ = sample_rate_hz * ts::kChunkSizeMs / 1000; + if (data_length_ > analysis_length_) { + RTC_NOTREACHED(); + return -1; + } + buffer_delay_ = analysis_length_ - data_length_; + + complex_analysis_length_ = analysis_length_ / 2 + 1; + RTC_DCHECK_GE(complex_analysis_length_, kMaxVoiceBin); + num_channels_ = num_channels; + in_buffer_.reset(new float[analysis_length_ * num_channels_]); + memset(in_buffer_.get(), + 0, + analysis_length_ * num_channels_ * sizeof(in_buffer_[0])); + detection_length_ = detection_rate_hz * ts::kChunkSizeMs / 1000; + detection_buffer_.reset(new float[detection_length_]); + memset(detection_buffer_.get(), + 0, + detection_length_ * sizeof(detection_buffer_[0])); + out_buffer_.reset(new float[analysis_length_ * num_channels_]); + memset(out_buffer_.get(), + 0, + analysis_length_ * num_channels_ * sizeof(out_buffer_[0])); + // ip[0] must be zero to trigger initialization using rdft(). + size_t ip_length = 2 + sqrtf(analysis_length_); + ip_.reset(new size_t[ip_length]()); + memset(ip_.get(), 0, ip_length * sizeof(ip_[0])); + wfft_.reset(new float[complex_analysis_length_ - 1]); + memset(wfft_.get(), 0, (complex_analysis_length_ - 1) * sizeof(wfft_[0])); + spectral_mean_.reset(new float[complex_analysis_length_ * num_channels_]); + memset(spectral_mean_.get(), + 0, + complex_analysis_length_ * num_channels_ * sizeof(spectral_mean_[0])); + fft_buffer_.reset(new float[analysis_length_ + 2]); + memset(fft_buffer_.get(), 0, (analysis_length_ + 2) * sizeof(fft_buffer_[0])); + magnitudes_.reset(new float[complex_analysis_length_]); + memset(magnitudes_.get(), + 0, + complex_analysis_length_ * sizeof(magnitudes_[0])); + mean_factor_.reset(new float[complex_analysis_length_]); + + static const float kFactorHeight = 10.f; + static const float kLowSlope = 1.f; + static const float kHighSlope = 0.3f; + for (size_t i = 0; i < complex_analysis_length_; ++i) { + mean_factor_[i] = + kFactorHeight / + (1.f + exp(kLowSlope * static_cast(i - kMinVoiceBin))) + + kFactorHeight / + (1.f + exp(kHighSlope * static_cast(kMaxVoiceBin - i))); + } + detector_smoothed_ = 0.f; + keypress_counter_ = 0; + chunks_since_keypress_ = 0; + detection_enabled_ = false; + suppression_enabled_ = false; + use_hard_restoration_ = false; + chunks_since_voice_change_ = 0; + seed_ = 182; + using_reference_ = false; + return 0; +} + +int TransientSuppressor::Suppress(float* data, + size_t data_length, + int num_channels, + const float* detection_data, + size_t detection_length, + const float* reference_data, + size_t reference_length, + float voice_probability, + bool key_pressed) { + if (!data || data_length != data_length_ || num_channels != num_channels_ || + detection_length != detection_length_ || voice_probability < 0 || + voice_probability > 1) { + return -1; + } + + UpdateKeypress(key_pressed); + UpdateBuffers(data); + + int result = 0; + if (detection_enabled_) { + UpdateRestoration(voice_probability); + + if (!detection_data) { + // Use the input data of the first channel if special detection data is + // not supplied. + detection_data = &in_buffer_[buffer_delay_]; + } + + float detector_result = detector_->Detect( + detection_data, detection_length, reference_data, reference_length); + if (detector_result < 0) { + return -1; + } + + using_reference_ = detector_->using_reference(); + + // |detector_smoothed_| follows the |detector_result| when this last one is + // increasing, but has an exponential decaying tail to be able to suppress + // the ringing of keyclicks. + float smooth_factor = using_reference_ ? 0.6 : 0.1; + detector_smoothed_ = detector_result >= detector_smoothed_ + ? detector_result + : smooth_factor * detector_smoothed_ + + (1 - smooth_factor) * detector_result; + + for (int i = 0; i < num_channels_; ++i) { + Suppress(&in_buffer_[i * analysis_length_], + &spectral_mean_[i * complex_analysis_length_], + &out_buffer_[i * analysis_length_]); + } + } + + // If the suppression isn't enabled, we use the in buffer to delay the signal + // appropriately. This also gives time for the out buffer to be refreshed with + // new data between detection and suppression getting enabled. + for (int i = 0; i < num_channels_; ++i) { + memcpy(&data[i * data_length_], + suppression_enabled_ ? &out_buffer_[i * analysis_length_] + : &in_buffer_[i * analysis_length_], + data_length_ * sizeof(*data)); + } + return result; +} + +// This should only be called when detection is enabled. UpdateBuffers() must +// have been called. At return, |out_buffer_| will be filled with the +// processed output. +void TransientSuppressor::Suppress(float* in_ptr, + float* spectral_mean, + float* out_ptr) { + // Go to frequency domain. + for (size_t i = 0; i < analysis_length_; ++i) { + // TODO(aluebs): Rename windows + fft_buffer_[i] = in_ptr[i] * window_[i]; + } + + WebRtc_rdft(analysis_length_, 1, fft_buffer_.get(), ip_.get(), wfft_.get()); + + // Since WebRtc_rdft puts R[n/2] in fft_buffer_[1], we move it to the end + // for convenience. + fft_buffer_[analysis_length_] = fft_buffer_[1]; + fft_buffer_[analysis_length_ + 1] = 0.f; + fft_buffer_[1] = 0.f; + + for (size_t i = 0; i < complex_analysis_length_; ++i) { + magnitudes_[i] = ComplexMagnitude(fft_buffer_[i * 2], + fft_buffer_[i * 2 + 1]); + } + // Restore audio if necessary. + if (suppression_enabled_) { + if (use_hard_restoration_) { + HardRestoration(spectral_mean); + } else { + SoftRestoration(spectral_mean); + } + } + + // Update the spectral mean. + for (size_t i = 0; i < complex_analysis_length_; ++i) { + spectral_mean[i] = (1 - kMeanIIRCoefficient) * spectral_mean[i] + + kMeanIIRCoefficient * magnitudes_[i]; + } + + // Back to time domain. + // Put R[n/2] back in fft_buffer_[1]. + fft_buffer_[1] = fft_buffer_[analysis_length_]; + + WebRtc_rdft(analysis_length_, + -1, + fft_buffer_.get(), + ip_.get(), + wfft_.get()); + const float fft_scaling = 2.f / analysis_length_; + + for (size_t i = 0; i < analysis_length_; ++i) { + out_ptr[i] += fft_buffer_[i] * window_[i] * fft_scaling; + } +} + +void TransientSuppressor::UpdateKeypress(bool key_pressed) { + const int kKeypressPenalty = 1000 / ts::kChunkSizeMs; + const int kIsTypingThreshold = 1000 / ts::kChunkSizeMs; + const int kChunksUntilNotTyping = 4000 / ts::kChunkSizeMs; // 4 seconds. + + if (key_pressed) { + keypress_counter_ += kKeypressPenalty; + chunks_since_keypress_ = 0; + detection_enabled_ = true; + } + keypress_counter_ = std::max(0, keypress_counter_ - 1); + + if (keypress_counter_ > kIsTypingThreshold) { + if (!suppression_enabled_) { + RTC_LOG(LS_INFO) << "[ts] Transient suppression is now enabled."; + } + suppression_enabled_ = true; + keypress_counter_ = 0; + } + + if (detection_enabled_ && + ++chunks_since_keypress_ > kChunksUntilNotTyping) { + if (suppression_enabled_) { + RTC_LOG(LS_INFO) << "[ts] Transient suppression is now disabled."; + } + detection_enabled_ = false; + suppression_enabled_ = false; + keypress_counter_ = 0; + } +} + +void TransientSuppressor::UpdateRestoration(float voice_probability) { + const int kHardRestorationOffsetDelay = 3; + const int kHardRestorationOnsetDelay = 80; + + bool not_voiced = voice_probability < kVoiceThreshold; + + if (not_voiced == use_hard_restoration_) { + chunks_since_voice_change_ = 0; + } else { + ++chunks_since_voice_change_; + + if ((use_hard_restoration_ && + chunks_since_voice_change_ > kHardRestorationOffsetDelay) || + (!use_hard_restoration_ && + chunks_since_voice_change_ > kHardRestorationOnsetDelay)) { + use_hard_restoration_ = not_voiced; + chunks_since_voice_change_ = 0; + } + } +} + +// Shift buffers to make way for new data. Must be called after +// |detection_enabled_| is updated by UpdateKeypress(). +void TransientSuppressor::UpdateBuffers(float* data) { + // TODO(aluebs): Change to ring buffer. + memmove(in_buffer_.get(), + &in_buffer_[data_length_], + (buffer_delay_ + (num_channels_ - 1) * analysis_length_) * + sizeof(in_buffer_[0])); + // Copy new chunk to buffer. + for (int i = 0; i < num_channels_; ++i) { + memcpy(&in_buffer_[buffer_delay_ + i * analysis_length_], + &data[i * data_length_], + data_length_ * sizeof(*data)); + } + if (detection_enabled_) { + // Shift previous chunk in out buffer. + memmove(out_buffer_.get(), + &out_buffer_[data_length_], + (buffer_delay_ + (num_channels_ - 1) * analysis_length_) * + sizeof(out_buffer_[0])); + // Initialize new chunk in out buffer. + for (int i = 0; i < num_channels_; ++i) { + memset(&out_buffer_[buffer_delay_ + i * analysis_length_], + 0, + data_length_ * sizeof(out_buffer_[0])); + } + } +} + +// Restores the unvoiced signal if a click is present. +// Attenuates by a certain factor every peak in the |fft_buffer_| that exceeds +// the spectral mean. The attenuation depends on |detector_smoothed_|. +// If a restoration takes place, the |magnitudes_| are updated to the new value. +void TransientSuppressor::HardRestoration(float* spectral_mean) { + const float detector_result = + 1.f - pow(1.f - detector_smoothed_, using_reference_ ? 200.f : 50.f); + // To restore, we get the peaks in the spectrum. If higher than the previous + // spectral mean we adjust them. + for (size_t i = 0; i < complex_analysis_length_; ++i) { + if (magnitudes_[i] > spectral_mean[i] && magnitudes_[i] > 0) { + // RandU() generates values on [0, int16::max()] + const float phase = 2 * ts::kPi * WebRtcSpl_RandU(&seed_) / + std::numeric_limits::max(); + const float scaled_mean = detector_result * spectral_mean[i]; + + fft_buffer_[i * 2] = (1 - detector_result) * fft_buffer_[i * 2] + + scaled_mean * cosf(phase); + fft_buffer_[i * 2 + 1] = (1 - detector_result) * fft_buffer_[i * 2 + 1] + + scaled_mean * sinf(phase); + magnitudes_[i] = magnitudes_[i] - + detector_result * (magnitudes_[i] - spectral_mean[i]); + } + } +} + +// Restores the voiced signal if a click is present. +// Attenuates by a certain factor every peak in the |fft_buffer_| that exceeds +// the spectral mean and that is lower than some function of the current block +// frequency mean. The attenuation depends on |detector_smoothed_|. +// If a restoration takes place, the |magnitudes_| are updated to the new value. +void TransientSuppressor::SoftRestoration(float* spectral_mean) { + // Get the spectral magnitude mean of the current block. + float block_frequency_mean = 0; + for (size_t i = kMinVoiceBin; i < kMaxVoiceBin; ++i) { + block_frequency_mean += magnitudes_[i]; + } + block_frequency_mean /= (kMaxVoiceBin - kMinVoiceBin); + + // To restore, we get the peaks in the spectrum. If higher than the + // previous spectral mean and lower than a factor of the block mean + // we adjust them. The factor is a double sigmoid that has a minimum in the + // voice frequency range (300Hz - 3kHz). + for (size_t i = 0; i < complex_analysis_length_; ++i) { + if (magnitudes_[i] > spectral_mean[i] && magnitudes_[i] > 0 && + (using_reference_ || + magnitudes_[i] < block_frequency_mean * mean_factor_[i])) { + const float new_magnitude = + magnitudes_[i] - + detector_smoothed_ * (magnitudes_[i] - spectral_mean[i]); + const float magnitude_ratio = new_magnitude / magnitudes_[i]; + + fft_buffer_[i * 2] *= magnitude_ratio; + fft_buffer_[i * 2 + 1] *= magnitude_ratio; + magnitudes_[i] = new_magnitude; + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.h new file mode 100644 index 0000000000..27b096c795 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_SUPPRESSOR_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_SUPPRESSOR_H_ + +#include +#include +#include + +#include "rtc_base/gtest_prod_util.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class TransientDetector; + +// Detects transients in an audio stream and suppress them using a simple +// restoration algorithm that attenuates unexpected spikes in the spectrum. +class TransientSuppressor { + public: + TransientSuppressor(); + ~TransientSuppressor(); + + int Initialize(int sample_rate_hz, int detector_rate_hz, int num_channels); + + // Processes a |data| chunk, and returns it with keystrokes suppressed from + // it. The float format is assumed to be int16 ranged. If there are more than + // one channel, the chunks are concatenated one after the other in |data|. + // |data_length| must be equal to |data_length_|. + // |num_channels| must be equal to |num_channels_|. + // A sub-band, ideally the higher, can be used as |detection_data|. If it is + // NULL, |data| is used for the detection too. The |detection_data| is always + // assumed mono. + // If a reference signal (e.g. keyboard microphone) is available, it can be + // passed in as |reference_data|. It is assumed mono and must have the same + // length as |data|. NULL is accepted if unavailable. + // This suppressor performs better if voice information is available. + // |voice_probability| is the probability of voice being present in this chunk + // of audio. If voice information is not available, |voice_probability| must + // always be set to 1. + // |key_pressed| determines if a key was pressed on this audio chunk. + // Returns 0 on success and -1 otherwise. + int Suppress(float* data, + size_t data_length, + int num_channels, + const float* detection_data, + size_t detection_length, + const float* reference_data, + size_t reference_length, + float voice_probability, + bool key_pressed); + + private: + FRIEND_TEST_ALL_PREFIXES(TransientSuppressorTest, + TypingDetectionLogicWorksAsExpectedForMono); + void Suppress(float* in_ptr, float* spectral_mean, float* out_ptr); + + void UpdateKeypress(bool key_pressed); + void UpdateRestoration(float voice_probability); + + void UpdateBuffers(float* data); + + void HardRestoration(float* spectral_mean); + void SoftRestoration(float* spectral_mean); + + std::unique_ptr detector_; + + size_t data_length_; + size_t detection_length_; + size_t analysis_length_; + size_t buffer_delay_; + size_t complex_analysis_length_; + int num_channels_; + // Input buffer where the original samples are stored. + std::unique_ptr in_buffer_; + std::unique_ptr detection_buffer_; + // Output buffer where the restored samples are stored. + std::unique_ptr out_buffer_; + + // Arrays for fft. + std::unique_ptr ip_; + std::unique_ptr wfft_; + + std::unique_ptr spectral_mean_; + + // Stores the data for the fft. + std::unique_ptr fft_buffer_; + + std::unique_ptr magnitudes_; + + const float* window_; + + std::unique_ptr mean_factor_; + + float detector_smoothed_; + + int keypress_counter_; + int chunks_since_keypress_; + bool detection_enabled_; + bool suppression_enabled_; + + bool use_hard_restoration_; + int chunks_since_voice_change_; + + uint32_t seed_; + + bool using_reference_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_SUPPRESSOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor_unittest.cc new file mode 100644 index 0000000000..32d9858c64 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/transient_suppressor_unittest.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/transient_suppressor.h" + +#include "modules/audio_processing/transient/common.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(TransientSuppressorTest, TypingDetectionLogicWorksAsExpectedForMono) { + static const int kNumChannels = 1; + + TransientSuppressor ts; + ts.Initialize(ts::kSampleRate16kHz, ts::kSampleRate16kHz, kNumChannels); + + // Each key-press enables detection. + EXPECT_FALSE(ts.detection_enabled_); + ts.UpdateKeypress(true); + EXPECT_TRUE(ts.detection_enabled_); + + // It takes four seconds without any key-press to disable the detection + for (int time_ms = 0; time_ms < 3990; time_ms += ts::kChunkSizeMs) { + ts.UpdateKeypress(false); + EXPECT_TRUE(ts.detection_enabled_); + } + ts.UpdateKeypress(false); + EXPECT_FALSE(ts.detection_enabled_); + + // Key-presses that are more than a second apart from each other don't enable + // suppression. + for (int i = 0; i < 100; ++i) { + EXPECT_FALSE(ts.suppression_enabled_); + ts.UpdateKeypress(true); + EXPECT_TRUE(ts.detection_enabled_); + EXPECT_FALSE(ts.suppression_enabled_); + for (int time_ms = 0; time_ms < 990; time_ms += ts::kChunkSizeMs) { + ts.UpdateKeypress(false); + EXPECT_TRUE(ts.detection_enabled_); + EXPECT_FALSE(ts.suppression_enabled_); + } + ts.UpdateKeypress(false); + } + + // Two consecutive key-presses is enough to enable the suppression. + ts.UpdateKeypress(true); + EXPECT_FALSE(ts.suppression_enabled_); + ts.UpdateKeypress(true); + EXPECT_TRUE(ts.suppression_enabled_); + + // Key-presses that are less than a second apart from each other don't disable + // detection nor suppression. + for (int i = 0; i < 100; ++i) { + for (int time_ms = 0; time_ms < 1000; time_ms += ts::kChunkSizeMs) { + ts.UpdateKeypress(false); + EXPECT_TRUE(ts.detection_enabled_); + EXPECT_TRUE(ts.suppression_enabled_); + } + ts.UpdateKeypress(true); + EXPECT_TRUE(ts.detection_enabled_); + EXPECT_TRUE(ts.suppression_enabled_); + } + + // It takes four seconds without any key-press to disable the detection and + // suppression. + for (int time_ms = 0; time_ms < 3990; time_ms += ts::kChunkSizeMs) { + ts.UpdateKeypress(false); + EXPECT_TRUE(ts.detection_enabled_); + EXPECT_TRUE(ts.suppression_enabled_); + } + for (int time_ms = 0; time_ms < 1000; time_ms += ts::kChunkSizeMs) { + ts.UpdateKeypress(false); + EXPECT_FALSE(ts.detection_enabled_); + EXPECT_FALSE(ts.suppression_enabled_); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.cc new file mode 100644 index 0000000000..20d6a9013e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/wpd_node.h" + +#include +#include + +#include "common_audio/fir_filter.h" +#include "common_audio/fir_filter_factory.h" +#include "modules/audio_processing/transient/dyadic_decimator.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +WPDNode::WPDNode(size_t length, + const float* coefficients, + size_t coefficients_length) + : // The data buffer has parent data length to be able to contain and filter + // it. + data_(new float[2 * length + 1]), + length_(length), + filter_(CreateFirFilter(coefficients, + coefficients_length, + 2 * length + 1)) { + RTC_DCHECK_GT(length, 0); + RTC_DCHECK(coefficients); + RTC_DCHECK_GT(coefficients_length, 0); + memset(data_.get(), 0.f, (2 * length + 1) * sizeof(data_[0])); +} + +WPDNode::~WPDNode() {} + +int WPDNode::Update(const float* parent_data, size_t parent_data_length) { + if (!parent_data || (parent_data_length / 2) != length_) { + return -1; + } + + // Filter data. + filter_->Filter(parent_data, parent_data_length, data_.get()); + + // Decimate data. + const bool kOddSequence = true; + size_t output_samples = DyadicDecimate( + data_.get(), parent_data_length, kOddSequence, data_.get(), length_); + if (output_samples != length_) { + return -1; + } + + // Get abs to all values. + for (size_t i = 0; i < length_; ++i) { + data_[i] = fabs(data_[i]); + } + + return 0; +} + +int WPDNode::set_data(const float* new_data, size_t length) { + if (!new_data || length != length_) { + return -1; + } + memcpy(data_.get(), new_data, length * sizeof(data_[0])); + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.h new file mode 100644 index 0000000000..117a6338a0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_NODE_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_NODE_H_ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class FIRFilter; + +// A single node of a Wavelet Packet Decomposition (WPD) tree. +class WPDNode { + public: + // Creates a WPDNode. The data vector will contain zeros. The filter will have + // the coefficients provided. + WPDNode(size_t length, const float* coefficients, size_t coefficients_length); + ~WPDNode(); + + // Updates the node data. |parent_data| / 2 must be equals to |length_|. + // Returns 0 if correct, and -1 otherwise. + int Update(const float* parent_data, size_t parent_data_length); + + const float* data() const { return data_.get(); } + // Returns 0 if correct, and -1 otherwise. + int set_data(const float* new_data, size_t length); + size_t length() const { return length_; } + + private: + std::unique_ptr data_; + size_t length_; + std::unique_ptr filter_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_NODE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node_unittest.cc new file mode 100644 index 0000000000..1929361055 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_node_unittest.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/wpd_node.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { + +static const size_t kDataLength = 5; +static const float kTolerance = 0.0001f; + +static const size_t kParentDataLength = kDataLength * 2; +static const float kParentData[kParentDataLength] = + {1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f}; + +static const float kCoefficients[] = {0.2f, -0.3f, 0.5f, -0.7f, 0.11f}; +static const size_t kCoefficientsLength = sizeof(kCoefficients) / + sizeof(kCoefficients[0]); + +TEST(WPDNodeTest, Accessors) { + WPDNode node(kDataLength, kCoefficients, kCoefficientsLength); + EXPECT_EQ(0, node.set_data(kParentData, kDataLength)); + EXPECT_EQ(0, memcmp(node.data(), + kParentData, + kDataLength * sizeof(node.data()[0]))); +} + +TEST(WPDNodeTest, UpdateThatOnlyDecimates) { + const float kIndentyCoefficient = 1.f; + WPDNode node(kDataLength, &kIndentyCoefficient, 1); + EXPECT_EQ(0, node.Update(kParentData, kParentDataLength)); + for (size_t i = 0; i < kDataLength; ++i) { + EXPECT_FLOAT_EQ(kParentData[i * 2 + 1], node.data()[i]); + } +} + +TEST(WPDNodeTest, UpdateWithArbitraryDataAndArbitraryFilter) { + WPDNode node(kDataLength, kCoefficients, kCoefficientsLength); + EXPECT_EQ(0, node.Update(kParentData, kParentDataLength)); + EXPECT_NEAR(0.1f, node.data()[0], kTolerance); + EXPECT_NEAR(0.2f, node.data()[1], kTolerance); + EXPECT_NEAR(0.18f, node.data()[2], kTolerance); + EXPECT_NEAR(0.56f, node.data()[3], kTolerance); + EXPECT_NEAR(0.94f, node.data()[4], kTolerance); +} + +TEST(WPDNodeTest, ExpectedErrorReturnValue) { + WPDNode node(kDataLength, kCoefficients, kCoefficientsLength); + EXPECT_EQ(-1, node.Update(kParentData, kParentDataLength - 1)); + EXPECT_EQ(-1, node.Update(NULL, kParentDataLength)); + EXPECT_EQ(-1, node.set_data(kParentData, kDataLength - 1)); + EXPECT_EQ(-1, node.set_data(NULL, kDataLength)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.cc new file mode 100644 index 0000000000..a01b816daf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/wpd_tree.h" + +#include +#include + +#include "modules/audio_processing/transient/dyadic_decimator.h" +#include "modules/audio_processing/transient/wpd_node.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +WPDTree::WPDTree(size_t data_length, const float* high_pass_coefficients, + const float* low_pass_coefficients, size_t coefficients_length, + int levels) + : data_length_(data_length), + levels_(levels), + num_nodes_((1 << (levels + 1)) - 1) { + RTC_DCHECK_GT(data_length, (static_cast(1) << levels)); + RTC_DCHECK(high_pass_coefficients); + RTC_DCHECK(low_pass_coefficients); + RTC_DCHECK_GT(levels, 0); + // Size is 1 more, so we can use the array as 1-based. nodes_[0] is never + // allocated. + nodes_.reset(new std::unique_ptr[num_nodes_ + 1]); + + // Create the first node + const float kRootCoefficient = 1.f; // Identity Coefficient. + nodes_[1].reset(new WPDNode(data_length, &kRootCoefficient, 1)); + // Variables used to create the rest of the nodes. + size_t index = 1; + size_t index_left_child = 0; + size_t index_right_child = 0; + + int num_nodes_at_curr_level = 0; + + // Branching each node in each level to create its children. The last level is + // not branched (all the nodes of that level are leaves). + for (int current_level = 0; current_level < levels; ++current_level) { + num_nodes_at_curr_level = 1 << current_level; + for (int i = 0; i < num_nodes_at_curr_level; ++i) { + index = (1 << current_level) + i; + // Obtain the index of the current node children. + index_left_child = index * 2; + index_right_child = index_left_child + 1; + nodes_[index_left_child].reset(new WPDNode(nodes_[index]->length() / 2, + low_pass_coefficients, + coefficients_length)); + nodes_[index_right_child].reset(new WPDNode(nodes_[index]->length() / 2, + high_pass_coefficients, + coefficients_length)); + } + } +} + +WPDTree::~WPDTree() {} + +WPDNode* WPDTree::NodeAt(int level, int index) { + if (level < 0 || level > levels_ || index < 0 || index >= 1 << level) { + return NULL; + } + + return nodes_[(1 << level) + index].get(); +} + +int WPDTree::Update(const float* data, size_t data_length) { + if (!data || data_length != data_length_) { + return -1; + } + + // Update the root node. + int update_result = nodes_[1]->set_data(data, data_length); + if (update_result != 0) { + return -1; + } + + // Variables used to update the rest of the nodes. + size_t index = 1; + size_t index_left_child = 0; + size_t index_right_child = 0; + + int num_nodes_at_curr_level = 0; + + for (int current_level = 0; current_level < levels_; ++current_level) { + num_nodes_at_curr_level = 1 << current_level; + for (int i = 0; i < num_nodes_at_curr_level; ++i) { + index = (1 << current_level) + i; + // Obtain the index of the current node children. + index_left_child = index * 2; + index_right_child = index_left_child + 1; + + update_result = nodes_[index_left_child]->Update( + nodes_[index]->data(), nodes_[index]->length()); + if (update_result != 0) { + return -1; + } + + update_result = nodes_[index_right_child]->Update( + nodes_[index]->data(), nodes_[index]->length()); + if (update_result != 0) { + return -1; + } + } + } + + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.h b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.h new file mode 100644 index 0000000000..53fc06b294 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_TREE_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_TREE_H_ + +#include + +#include "modules/audio_processing/transient/wpd_node.h" + +namespace webrtc { + +// Tree of a Wavelet Packet Decomposition (WPD). +// +// The root node contains all the data provided; for each node in the tree, the +// left child contains the approximation coefficients extracted from the node, +// and the right child contains the detail coefficients. +// It preserves its state, so it can be multiple-called. +// +// The number of nodes in the tree will be 2 ^ levels - 1. +// +// Implementation details: Since the tree always will be a complete binary tree, +// it is implemented using a single linear array instead of managing the +// relationships in each node. For convience is better to use a array that +// starts in 1 (instead of 0). Taking that into account, the following formulas +// apply: +// Root node index: 1. +// Node(Level, Index in that level): 2 ^ Level + (Index in that level). +// Left Child: Current node index * 2. +// Right Child: Current node index * 2 + 1. +// Parent: Current Node Index / 2 (Integer division). +class WPDTree { + public: + // Creates a WPD tree using the data length and coefficients provided. + WPDTree(size_t data_length, + const float* high_pass_coefficients, + const float* low_pass_coefficients, + size_t coefficients_length, + int levels); + ~WPDTree(); + + // Returns the number of nodes at any given level. + static int NumberOfNodesAtLevel(int level) { + return 1 << level; + } + + // Returns a pointer to the node at the given level and index(of that level). + // Level goes from 0 to levels(). + // Index goes from 0 to the number of NumberOfNodesAtLevel(level) - 1. + // + // You can use the following formulas to get any node within the tree: + // Notation: (Level, Index of node in that level). + // Root node: (0/0). + // Left Child: (Current node level + 1, Current node index * 2). + // Right Child: (Current node level + 1, Current node index * 2 + 1). + // Parent: (Current node level - 1, Current node index / 2) (Integer division) + // + // If level or index are out of bounds the function will return NULL. + WPDNode* NodeAt(int level, int index); + + // Updates all the nodes of the tree with the new data. |data_length| must be + // teh same that was used for the creation of the tree. + // Returns 0 if correct, and -1 otherwise. + int Update(const float* data, size_t data_length); + + // Returns the total number of levels below the root. Root is cosidered level + // 0. + int levels() const { return levels_; } + + // Returns the total number of nodes. + int num_nodes() const { return num_nodes_; } + + // Returns the total number of leaves. + int num_leaves() const { return 1 << levels_; } + + private: + size_t data_length_; + int levels_; + int num_nodes_; + std::unique_ptr[]> nodes_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_TREE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree_unittest.cc new file mode 100644 index 0000000000..a90af7766b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/transient/wpd_tree_unittest.cc @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/wpd_tree.h" + +#include +#include +#include + +#include "modules/audio_processing/transient/daubechies_8_wavelet_coeffs.h" +#include "modules/audio_processing/transient/file_utils.h" +#include "system_wrappers/include/file_wrapper.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +TEST(WPDTreeTest, Construction) { + const size_t kTestBufferSize = 100; + const int kLevels = 5; + const int kExpectedNumberOfNodes = (1 << (kLevels + 1)) - 1; + + float test_buffer[kTestBufferSize]; + memset(test_buffer, 0.f, kTestBufferSize * sizeof(*test_buffer)); + float test_coefficients[] = {1.f, 2.f, 3.f, 4.f, 5.f}; + const size_t kTestCoefficientsLength = sizeof(test_coefficients) / + sizeof(test_coefficients[0]); + WPDTree tree(kTestBufferSize, + test_coefficients, + test_coefficients, + kTestCoefficientsLength, + kLevels); + ASSERT_EQ(kExpectedNumberOfNodes, tree.num_nodes()); + // Checks for NodeAt(level, index). + int nodes_at_level = 0; + for (int level = 0; level <= kLevels; ++level) { + nodes_at_level = 1 << level; + for (int i = 0; i < nodes_at_level; ++i) { + ASSERT_TRUE(NULL != tree.NodeAt(level, i)); + } + // Out of bounds. + EXPECT_EQ(NULL, tree.NodeAt(level, -1)); + EXPECT_EQ(NULL, tree.NodeAt(level, -12)); + EXPECT_EQ(NULL, tree.NodeAt(level, nodes_at_level)); + EXPECT_EQ(NULL, tree.NodeAt(level, nodes_at_level + 5)); + } + // Out of bounds. + EXPECT_EQ(NULL, tree.NodeAt(-1, 0)); + EXPECT_EQ(NULL, tree.NodeAt(-12, 0)); + EXPECT_EQ(NULL, tree.NodeAt(kLevels + 1, 0)); + EXPECT_EQ(NULL, tree.NodeAt(kLevels + 5, 0)); + // Checks for Update(). + EXPECT_EQ(0, tree.Update(test_buffer, kTestBufferSize)); + EXPECT_EQ(-1, tree.Update(NULL, kTestBufferSize)); + EXPECT_EQ(-1, tree.Update(test_buffer, kTestBufferSize - 1)); +} + +// This test is for the correctness of the tree. +// Checks the results from the Matlab equivalent, it is done comparing the +// results that are stored in the output files from Matlab. +// It also writes the results in its own set of files in the out directory. +// Matlab and output files contain all the results in double precision (Little +// endian) appended. +#if defined(WEBRTC_IOS) +TEST(WPDTreeTest, DISABLED_CorrectnessBasedOnMatlabFiles) { +#else +TEST(WPDTreeTest, CorrectnessBasedOnMatlabFiles) { +#endif + // 10 ms at 16000 Hz. + const size_t kTestBufferSize = 160; + const int kLevels = 3; + const int kLeaves = 1 << kLevels; + const size_t kLeavesSamples = kTestBufferSize >> kLevels; + // Create tree with Discrete Meyer Wavelet Coefficients. + WPDTree tree(kTestBufferSize, + kDaubechies8HighPassCoefficients, + kDaubechies8LowPassCoefficients, + kDaubechies8CoefficientsLength, + kLevels); + // Allocate and open all matlab and out files. + std::unique_ptr matlab_files_data[kLeaves]; + std::unique_ptr out_files_data[kLeaves]; + + for (int i = 0; i < kLeaves; ++i) { + // Matlab files. + matlab_files_data[i].reset(FileWrapper::Create()); + + std::ostringstream matlab_stream; + matlab_stream << "audio_processing/transient/wpd" << i; + std::string matlab_string = test::ResourcePath(matlab_stream.str(), "dat"); + matlab_files_data[i]->OpenFile(matlab_string.c_str(), true); // Read only. + + bool file_opened = matlab_files_data[i]->is_open(); + ASSERT_TRUE(file_opened) << "File could not be opened.\n" << matlab_string; + + // Out files. + out_files_data[i].reset(FileWrapper::Create()); + + std::ostringstream out_stream; + out_stream << test::OutputPath() << "wpd_" << i << ".out"; + std::string out_string = out_stream.str(); + + out_files_data[i]->OpenFile(out_string.c_str(), false); // Write mode. + + file_opened = out_files_data[i]->is_open(); + ASSERT_TRUE(file_opened) << "File could not be opened.\n" << out_string; + } + + // Prepare the test file. + std::string test_file_name = test::ResourcePath( + "audio_processing/transient/ajm-macbook-1-spke16m", "pcm"); + + std::unique_ptr test_file(FileWrapper::Create()); + + test_file->OpenFile(test_file_name.c_str(), true); // Read only. + + bool file_opened = test_file->is_open(); + ASSERT_TRUE(file_opened) << "File could not be opened.\n" << test_file_name; + + float test_buffer[kTestBufferSize]; + + // Only the first frames of the audio file are tested. The matlab files also + // only contains information about the first frames. + const size_t kMaxFramesToTest = 100; + const float kTolerance = 0.03f; + + size_t frames_read = 0; + + // Read first buffer from the PCM test file. + size_t file_samples_read = ReadInt16FromFileToFloatBuffer(test_file.get(), + kTestBufferSize, + test_buffer); + while (file_samples_read > 0 && frames_read < kMaxFramesToTest) { + ++frames_read; + + if (file_samples_read < kTestBufferSize) { + // Pad the rest of the buffer with zeros. + for (size_t i = file_samples_read; i < kTestBufferSize; ++i) { + test_buffer[i] = 0.0; + } + } + tree.Update(test_buffer, kTestBufferSize); + double matlab_buffer[kTestBufferSize]; + + // Compare results with data from the matlab test files. + for (int i = 0; i < kLeaves; ++i) { + // Compare data values + size_t matlab_samples_read = + ReadDoubleBufferFromFile(matlab_files_data[i].get(), + kLeavesSamples, + matlab_buffer); + + ASSERT_EQ(kLeavesSamples, matlab_samples_read) + << "Matlab test files are malformed.\n" + << "File: 3_" << i; + // Get output data from the corresponding node + const float* node_data = tree.NodeAt(kLevels, i)->data(); + // Compare with matlab files. + for (size_t j = 0; j < kLeavesSamples; ++j) { + EXPECT_NEAR(matlab_buffer[j], node_data[j], kTolerance) + << "\nLeaf: " << i << "\nSample: " << j + << "\nFrame: " << frames_read - 1; + } + + // Write results to out files. + WriteFloatBufferToFile(out_files_data[i].get(), + kLeavesSamples, + node_data); + } + + // Read next buffer from the PCM test file. + file_samples_read = ReadInt16FromFileToFloatBuffer(test_file.get(), + kTestBufferSize, + test_buffer); + } + + // Close all matlab and out files. + for (int i = 0; i < kLeaves; ++i) { + matlab_files_data[i]->CloseFile(); + out_files_data[i]->CloseFile(); + } + + test_file->CloseFile(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.cc new file mode 100644 index 0000000000..6e18124aad --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/typing_detection.h" + +namespace webrtc { + +TypingDetection::TypingDetection() + : time_active_(0), + time_since_last_typing_(0), + penalty_counter_(0), + counter_since_last_detection_update_(0), + detection_to_report_(false), + new_detection_to_report_(false), + time_window_(10), + cost_per_typing_(100), + reporting_threshold_(300), + penalty_decay_(1), + type_event_delay_(2), + report_detection_update_period_(1) { +} + +TypingDetection::~TypingDetection() {} + +bool TypingDetection::Process(bool key_pressed, bool vad_activity) { + if (vad_activity) + time_active_++; + else + time_active_ = 0; + + // Keep track if time since last typing event + if (key_pressed) + time_since_last_typing_ = 0; + else + ++time_since_last_typing_; + + if (time_since_last_typing_ < type_event_delay_ && + vad_activity && + time_active_ < time_window_) { + penalty_counter_ += cost_per_typing_; + if (penalty_counter_ > reporting_threshold_) + new_detection_to_report_ = true; + } + + if (penalty_counter_ > 0) + penalty_counter_ -= penalty_decay_; + + if (++counter_since_last_detection_update_ == + report_detection_update_period_) { + detection_to_report_ = new_detection_to_report_; + new_detection_to_report_ = false; + counter_since_last_detection_update_ = 0; + } + + return detection_to_report_; +} + +int TypingDetection::TimeSinceLastDetectionInSeconds() { + // Round to whole seconds. + return (time_since_last_typing_ + 50) / 100; +} + +void TypingDetection::SetParameters(int time_window, + int cost_per_typing, + int reporting_threshold, + int penalty_decay, + int type_event_delay, + int report_detection_update_period) { + if (time_window) time_window_ = time_window; + + if (cost_per_typing) cost_per_typing_ = cost_per_typing; + + if (reporting_threshold) reporting_threshold_ = reporting_threshold; + + if (penalty_decay) penalty_decay_ = penalty_decay; + + if (type_event_delay) type_event_delay_ = type_event_delay; + + if (report_detection_update_period) + report_detection_update_period_ = report_detection_update_period; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.h b/third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.h new file mode 100644 index 0000000000..fe74a5956e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/typing_detection.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_ +#define MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_ + +#include "modules/include/module_common_types.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class TypingDetection { + public: + TypingDetection(); + virtual ~TypingDetection(); + + // Run the detection algortihm. Shall be called every 10 ms. Returns true if + // typing is detected, or false if not, based on the update period as set with + // SetParameters(). See |report_detection_update_period_| description below. + bool Process(bool key_pressed, bool vad_activity); + + // Gets the time in seconds since the last detection. + int TimeSinceLastDetectionInSeconds(); + + // Sets the algorithm parameters. A parameter value of 0 leaves it unchanged. + // See the correspondning member variables below for descriptions. + void SetParameters(int time_window, + int cost_per_typing, + int reporting_threshold, + int penalty_decay, + int type_event_delay, + int report_detection_update_period); + + private: + int time_active_; + int time_since_last_typing_; + int penalty_counter_; + + // Counter since last time the detection status reported by Process() was + // updated. See also |report_detection_update_period_|. + int counter_since_last_detection_update_; + + // The detection status to report. Updated every + // |report_detection_update_period_| call to Process(). + bool detection_to_report_; + + // What |detection_to_report_| should be set to next time it is updated. + bool new_detection_to_report_; + + // Settable threshold values. + + // Number of 10 ms slots accepted to count as a hit. + int time_window_; + + // Penalty added for a typing + activity coincide. + int cost_per_typing_; + + // Threshold for |penalty_counter_|. + int reporting_threshold_; + + // How much we reduce |penalty_counter_| every 10 ms. + int penalty_decay_; + + // How old typing events we allow. + int type_event_delay_; + + // Settable update period. + + // Number of 10 ms slots between each update of the detection status returned + // by Process(). This inertia added to the algorithm is usually desirable and + // provided so that consumers of the class don't have to implement that + // themselves if they don't wish. + // If set to 1, each call to Process() will return the detection status for + // that 10 ms slot. + // If set to N (where N > 1), the detection status returned from Process() + // will remain the same until Process() has been called N times. Then, if none + // of the last N calls to Process() has detected typing for each respective + // 10 ms slot, Process() will return false. If at least one of the last N + // calls has detected typing, Process() will return true. And that returned + // status will then remain the same until the next N calls have been done. + int report_detection_update_period_; +}; + +} // namespace webrtc + +#endif // #ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.cc new file mode 100644 index 0000000000..3d766929c6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.cc @@ -0,0 +1,53 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/utility/block_mean_calculator.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +BlockMeanCalculator::BlockMeanCalculator(size_t block_length) + : block_length_(block_length), + count_(0), + sum_(0.0), + mean_(0.0) { + RTC_DCHECK(block_length_ != 0); +} + +void BlockMeanCalculator::Reset() { + Clear(); + mean_ = 0.0; +} + +void BlockMeanCalculator::AddValue(float value) { + sum_ += value; + ++count_; + if (count_ == block_length_) { + mean_ = sum_ / block_length_; + Clear(); + } +} + +bool BlockMeanCalculator::EndOfBlock() const { + return count_ == 0; +} + +float BlockMeanCalculator::GetLatestMean() const { + return mean_; +} + +// Flush all samples added. +void BlockMeanCalculator::Clear() { + count_ = 0; + sum_ = 0.0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.h new file mode 100644 index 0000000000..cfa7cfbeba --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator.h @@ -0,0 +1,52 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_UTILITY_BLOCK_MEAN_CALCULATOR_H_ +#define MODULES_AUDIO_PROCESSING_UTILITY_BLOCK_MEAN_CALCULATOR_H_ + +#include + +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// BlockMeanCalculator calculates the mean of a block of values. Values are +// added one after another, and the mean is updated at the end of every block. +class BlockMeanCalculator { + public: + explicit BlockMeanCalculator(size_t block_length); + + // Reset. + void Reset(); + + // Add one value to the sequence. + void AddValue(float value); + + // Return whether the latest added value was at the end of a block. + bool EndOfBlock() const; + + // Return the latest mean. + float GetLatestMean() const; + + private: + // Clear all values added. + void Clear(); + + const size_t block_length_; + size_t count_; + float sum_; + float mean_; + + RTC_DISALLOW_COPY_AND_ASSIGN(BlockMeanCalculator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_UTILITY_BLOCK_MEAN_CALCULATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator_unittest.cc new file mode 100644 index 0000000000..1f4ebf1b67 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/block_mean_calculator_unittest.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/utility/block_mean_calculator.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(MeanCalculatorTest, Correctness) { + const size_t kBlockLength = 10; + BlockMeanCalculator mean_calculator(kBlockLength); + size_t i = 0; + float reference = 0.0; + + for (; i < kBlockLength - 1; ++i) { + mean_calculator.AddValue(static_cast(i)); + EXPECT_FALSE(mean_calculator.EndOfBlock()); + } + mean_calculator.AddValue(static_cast(i++)); + EXPECT_TRUE(mean_calculator.EndOfBlock()); + + for (; i < 3 * kBlockLength; ++i) { + const bool end_of_block = i % kBlockLength == 0; + if (end_of_block) { + // Sum of (i - kBlockLength) ... (i - 1) + reference = i - 0.5 * (1 + kBlockLength); + } + EXPECT_EQ(mean_calculator.EndOfBlock(), end_of_block); + EXPECT_EQ(reference, mean_calculator.GetLatestMean()); + mean_calculator.AddValue(static_cast(i)); + } +} + +TEST(MeanCalculatorTest, Reset) { + const size_t kBlockLength = 10; + BlockMeanCalculator mean_calculator(kBlockLength); + for (size_t i = 0; i < kBlockLength - 1; ++i) { + mean_calculator.AddValue(static_cast(i)); + } + mean_calculator.Reset(); + size_t i = 0; + for (; i < kBlockLength - 1; ++i) { + mean_calculator.AddValue(static_cast(i)); + EXPECT_FALSE(mean_calculator.EndOfBlock()); + } + mean_calculator.AddValue(static_cast(i)); + EXPECT_TRUE(mean_calculator.EndOfBlock()); + EXPECT_EQ(mean_calculator.GetLatestMean(), 0.5 * (kBlockLength - 1)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.cc new file mode 100644 index 0000000000..871b541651 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.cc @@ -0,0 +1,703 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/utility/delay_estimator.h" + +#include +#include +#include + +#include "rtc_base/checks.h" + +// Number of right shifts for scaling is linearly depending on number of bits in +// the far-end binary spectrum. +static const int kShiftsAtZero = 13; // Right shifts at zero binary spectrum. +static const int kShiftsLinearSlope = 3; + +static const int32_t kProbabilityOffset = 1024; // 2 in Q9. +static const int32_t kProbabilityLowerLimit = 8704; // 17 in Q9. +static const int32_t kProbabilityMinSpread = 2816; // 5.5 in Q9. + +// Robust validation settings +static const float kHistogramMax = 3000.f; +static const float kLastHistogramMax = 250.f; +static const float kMinHistogramThreshold = 1.5f; +static const int kMinRequiredHits = 10; +static const int kMaxHitsWhenPossiblyNonCausal = 10; +static const int kMaxHitsWhenPossiblyCausal = 1000; +static const float kQ14Scaling = 1.f / (1 << 14); // Scaling by 2^14 to get Q0. +static const float kFractionSlope = 0.05f; +static const float kMinFractionWhenPossiblyCausal = 0.5f; +static const float kMinFractionWhenPossiblyNonCausal = 0.25f; + +// Counts and returns number of bits of a 32-bit word. +static int BitCount(uint32_t u32) { + uint32_t tmp = u32 - ((u32 >> 1) & 033333333333) - + ((u32 >> 2) & 011111111111); + tmp = ((tmp + (tmp >> 3)) & 030707070707); + tmp = (tmp + (tmp >> 6)); + tmp = (tmp + (tmp >> 12) + (tmp >> 24)) & 077; + + return ((int) tmp); +} + +// Compares the |binary_vector| with all rows of the |binary_matrix| and counts +// per row the number of times they have the same value. +// +// Inputs: +// - binary_vector : binary "vector" stored in a long +// - binary_matrix : binary "matrix" stored as a vector of long +// - matrix_size : size of binary "matrix" +// +// Output: +// - bit_counts : "Vector" stored as a long, containing for each +// row the number of times the matrix row and the +// input vector have the same value +// +static void BitCountComparison(uint32_t binary_vector, + const uint32_t* binary_matrix, + int matrix_size, + int32_t* bit_counts) { + int n = 0; + + // Compare |binary_vector| with all rows of the |binary_matrix| + for (; n < matrix_size; n++) { + bit_counts[n] = (int32_t) BitCount(binary_vector ^ binary_matrix[n]); + } +} + +// Collects necessary statistics for the HistogramBasedValidation(). This +// function has to be called prior to calling HistogramBasedValidation(). The +// statistics updated and used by the HistogramBasedValidation() are: +// 1. the number of |candidate_hits|, which states for how long we have had the +// same |candidate_delay| +// 2. the |histogram| of candidate delays over time. This histogram is +// weighted with respect to a reliability measure and time-varying to cope +// with possible delay shifts. +// For further description see commented code. +// +// Inputs: +// - candidate_delay : The delay to validate. +// - valley_depth_q14 : The cost function has a valley/minimum at the +// |candidate_delay| location. |valley_depth_q14| is the +// cost function difference between the minimum and +// maximum locations. The value is in the Q14 domain. +// - valley_level_q14 : Is the cost function value at the minimum, in Q14. +static void UpdateRobustValidationStatistics(BinaryDelayEstimator* self, + int candidate_delay, + int32_t valley_depth_q14, + int32_t valley_level_q14) { + const float valley_depth = valley_depth_q14 * kQ14Scaling; + float decrease_in_last_set = valley_depth; + const int max_hits_for_slow_change = (candidate_delay < self->last_delay) ? + kMaxHitsWhenPossiblyNonCausal : kMaxHitsWhenPossiblyCausal; + int i = 0; + + RTC_DCHECK_EQ(self->history_size, self->farend->history_size); + // Reset |candidate_hits| if we have a new candidate. + if (candidate_delay != self->last_candidate_delay) { + self->candidate_hits = 0; + self->last_candidate_delay = candidate_delay; + } + self->candidate_hits++; + + // The |histogram| is updated differently across the bins. + // 1. The |candidate_delay| histogram bin is increased with the + // |valley_depth|, which is a simple measure of how reliable the + // |candidate_delay| is. The histogram is not increased above + // |kHistogramMax|. + self->histogram[candidate_delay] += valley_depth; + if (self->histogram[candidate_delay] > kHistogramMax) { + self->histogram[candidate_delay] = kHistogramMax; + } + // 2. The histogram bins in the neighborhood of |candidate_delay| are + // unaffected. The neighborhood is defined as x + {-2, -1, 0, 1}. + // 3. The histogram bins in the neighborhood of |last_delay| are decreased + // with |decrease_in_last_set|. This value equals the difference between + // the cost function values at the locations |candidate_delay| and + // |last_delay| until we reach |max_hits_for_slow_change| consecutive hits + // at the |candidate_delay|. If we exceed this amount of hits the + // |candidate_delay| is a "potential" candidate and we start decreasing + // these histogram bins more rapidly with |valley_depth|. + if (self->candidate_hits < max_hits_for_slow_change) { + decrease_in_last_set = (self->mean_bit_counts[self->compare_delay] - + valley_level_q14) * kQ14Scaling; + } + // 4. All other bins are decreased with |valley_depth|. + // TODO(bjornv): Investigate how to make this loop more efficient. Split up + // the loop? Remove parts that doesn't add too much. + for (i = 0; i < self->history_size; ++i) { + int is_in_last_set = (i >= self->last_delay - 2) && + (i <= self->last_delay + 1) && (i != candidate_delay); + int is_in_candidate_set = (i >= candidate_delay - 2) && + (i <= candidate_delay + 1); + self->histogram[i] -= decrease_in_last_set * is_in_last_set + + valley_depth * (!is_in_last_set && !is_in_candidate_set); + // 5. No histogram bin can go below 0. + if (self->histogram[i] < 0) { + self->histogram[i] = 0; + } + } +} + +// Validates the |candidate_delay|, estimated in WebRtc_ProcessBinarySpectrum(), +// based on a mix of counting concurring hits with a modified histogram +// of recent delay estimates. In brief a candidate is valid (returns 1) if it +// is the most likely according to the histogram. There are a couple of +// exceptions that are worth mentioning: +// 1. If the |candidate_delay| < |last_delay| it can be that we are in a +// non-causal state, breaking a possible echo control algorithm. Hence, we +// open up for a quicker change by allowing the change even if the +// |candidate_delay| is not the most likely one according to the histogram. +// 2. There's a minimum number of hits (kMinRequiredHits) and the histogram +// value has to reached a minimum (kMinHistogramThreshold) to be valid. +// 3. The action is also depending on the filter length used for echo control. +// If the delay difference is larger than what the filter can capture, we +// also move quicker towards a change. +// For further description see commented code. +// +// Input: +// - candidate_delay : The delay to validate. +// +// Return value: +// - is_histogram_valid : 1 - The |candidate_delay| is valid. +// 0 - Otherwise. +static int HistogramBasedValidation(const BinaryDelayEstimator* self, + int candidate_delay) { + float fraction = 1.f; + float histogram_threshold = self->histogram[self->compare_delay]; + const int delay_difference = candidate_delay - self->last_delay; + int is_histogram_valid = 0; + + // The histogram based validation of |candidate_delay| is done by comparing + // the |histogram| at bin |candidate_delay| with a |histogram_threshold|. + // This |histogram_threshold| equals a |fraction| of the |histogram| at bin + // |last_delay|. The |fraction| is a piecewise linear function of the + // |delay_difference| between the |candidate_delay| and the |last_delay| + // allowing for a quicker move if + // i) a potential echo control filter can not handle these large differences. + // ii) keeping |last_delay| instead of updating to |candidate_delay| could + // force an echo control into a non-causal state. + // We further require the histogram to have reached a minimum value of + // |kMinHistogramThreshold|. In addition, we also require the number of + // |candidate_hits| to be more than |kMinRequiredHits| to remove spurious + // values. + + // Calculate a comparison histogram value (|histogram_threshold|) that is + // depending on the distance between the |candidate_delay| and |last_delay|. + // TODO(bjornv): How much can we gain by turning the fraction calculation + // into tables? + if (delay_difference > self->allowed_offset) { + fraction = 1.f - kFractionSlope * (delay_difference - self->allowed_offset); + fraction = (fraction > kMinFractionWhenPossiblyCausal ? fraction : + kMinFractionWhenPossiblyCausal); + } else if (delay_difference < 0) { + fraction = kMinFractionWhenPossiblyNonCausal - + kFractionSlope * delay_difference; + fraction = (fraction > 1.f ? 1.f : fraction); + } + histogram_threshold *= fraction; + histogram_threshold = (histogram_threshold > kMinHistogramThreshold ? + histogram_threshold : kMinHistogramThreshold); + + is_histogram_valid = + (self->histogram[candidate_delay] >= histogram_threshold) && + (self->candidate_hits > kMinRequiredHits); + + return is_histogram_valid; +} + +// Performs a robust validation of the |candidate_delay| estimated in +// WebRtc_ProcessBinarySpectrum(). The algorithm takes the +// |is_instantaneous_valid| and the |is_histogram_valid| and combines them +// into a robust validation. The HistogramBasedValidation() has to be called +// prior to this call. +// For further description on how the combination is done, see commented code. +// +// Inputs: +// - candidate_delay : The delay to validate. +// - is_instantaneous_valid : The instantaneous validation performed in +// WebRtc_ProcessBinarySpectrum(). +// - is_histogram_valid : The histogram based validation. +// +// Return value: +// - is_robust : 1 - The candidate_delay is valid according to a +// combination of the two inputs. +// : 0 - Otherwise. +static int RobustValidation(const BinaryDelayEstimator* self, + int candidate_delay, + int is_instantaneous_valid, + int is_histogram_valid) { + int is_robust = 0; + + // The final robust validation is based on the two algorithms; 1) the + // |is_instantaneous_valid| and 2) the histogram based with result stored in + // |is_histogram_valid|. + // i) Before we actually have a valid estimate (|last_delay| == -2), we say + // a candidate is valid if either algorithm states so + // (|is_instantaneous_valid| OR |is_histogram_valid|). + is_robust = (self->last_delay < 0) && + (is_instantaneous_valid || is_histogram_valid); + // ii) Otherwise, we need both algorithms to be certain + // (|is_instantaneous_valid| AND |is_histogram_valid|) + is_robust |= is_instantaneous_valid && is_histogram_valid; + // iii) With one exception, i.e., the histogram based algorithm can overrule + // the instantaneous one if |is_histogram_valid| = 1 and the histogram + // is significantly strong. + is_robust |= is_histogram_valid && + (self->histogram[candidate_delay] > self->last_delay_histogram); + + return is_robust; +} + +void WebRtc_FreeBinaryDelayEstimatorFarend(BinaryDelayEstimatorFarend* self) { + + if (self == NULL) { + return; + } + + free(self->binary_far_history); + self->binary_far_history = NULL; + + free(self->far_bit_counts); + self->far_bit_counts = NULL; + + free(self); +} + +BinaryDelayEstimatorFarend* WebRtc_CreateBinaryDelayEstimatorFarend( + int history_size) { + BinaryDelayEstimatorFarend* self = NULL; + + if (history_size > 1) { + // Sanity conditions fulfilled. + self = static_cast( + malloc(sizeof(BinaryDelayEstimatorFarend))); + } + if (self == NULL) { + return NULL; + } + + self->history_size = 0; + self->binary_far_history = NULL; + self->far_bit_counts = NULL; + if (WebRtc_AllocateFarendBufferMemory(self, history_size) == 0) { + WebRtc_FreeBinaryDelayEstimatorFarend(self); + self = NULL; + } + return self; +} + +int WebRtc_AllocateFarendBufferMemory(BinaryDelayEstimatorFarend* self, + int history_size) { + RTC_DCHECK(self); + // (Re-)Allocate memory for history buffers. + self->binary_far_history = static_cast( + realloc(self->binary_far_history, + history_size * sizeof(*self->binary_far_history))); + self->far_bit_counts = static_cast( + realloc(self->far_bit_counts, + history_size * sizeof(*self->far_bit_counts))); + if ((self->binary_far_history == NULL) || (self->far_bit_counts == NULL)) { + history_size = 0; + } + // Fill with zeros if we have expanded the buffers. + if (history_size > self->history_size) { + int size_diff = history_size - self->history_size; + memset(&self->binary_far_history[self->history_size], + 0, + sizeof(*self->binary_far_history) * size_diff); + memset(&self->far_bit_counts[self->history_size], + 0, + sizeof(*self->far_bit_counts) * size_diff); + } + self->history_size = history_size; + + return self->history_size; +} + +void WebRtc_InitBinaryDelayEstimatorFarend(BinaryDelayEstimatorFarend* self) { + RTC_DCHECK(self); + memset(self->binary_far_history, 0, sizeof(uint32_t) * self->history_size); + memset(self->far_bit_counts, 0, sizeof(int) * self->history_size); +} + +void WebRtc_SoftResetBinaryDelayEstimatorFarend( + BinaryDelayEstimatorFarend* self, int delay_shift) { + int abs_shift = abs(delay_shift); + int shift_size = 0; + int dest_index = 0; + int src_index = 0; + int padding_index = 0; + + RTC_DCHECK(self); + shift_size = self->history_size - abs_shift; + RTC_DCHECK_GT(shift_size, 0); + if (delay_shift == 0) { + return; + } else if (delay_shift > 0) { + dest_index = abs_shift; + } else if (delay_shift < 0) { + src_index = abs_shift; + padding_index = shift_size; + } + + // Shift and zero pad buffers. + memmove(&self->binary_far_history[dest_index], + &self->binary_far_history[src_index], + sizeof(*self->binary_far_history) * shift_size); + memset(&self->binary_far_history[padding_index], 0, + sizeof(*self->binary_far_history) * abs_shift); + memmove(&self->far_bit_counts[dest_index], + &self->far_bit_counts[src_index], + sizeof(*self->far_bit_counts) * shift_size); + memset(&self->far_bit_counts[padding_index], 0, + sizeof(*self->far_bit_counts) * abs_shift); +} + +void WebRtc_AddBinaryFarSpectrum(BinaryDelayEstimatorFarend* handle, + uint32_t binary_far_spectrum) { + RTC_DCHECK(handle); + // Shift binary spectrum history and insert current |binary_far_spectrum|. + memmove(&(handle->binary_far_history[1]), &(handle->binary_far_history[0]), + (handle->history_size - 1) * sizeof(uint32_t)); + handle->binary_far_history[0] = binary_far_spectrum; + + // Shift history of far-end binary spectrum bit counts and insert bit count + // of current |binary_far_spectrum|. + memmove(&(handle->far_bit_counts[1]), &(handle->far_bit_counts[0]), + (handle->history_size - 1) * sizeof(int)); + handle->far_bit_counts[0] = BitCount(binary_far_spectrum); +} + +void WebRtc_FreeBinaryDelayEstimator(BinaryDelayEstimator* self) { + + if (self == NULL) { + return; + } + + free(self->mean_bit_counts); + self->mean_bit_counts = NULL; + + free(self->bit_counts); + self->bit_counts = NULL; + + free(self->binary_near_history); + self->binary_near_history = NULL; + + free(self->histogram); + self->histogram = NULL; + + // BinaryDelayEstimator does not have ownership of |farend|, hence we do not + // free the memory here. That should be handled separately by the user. + self->farend = NULL; + + free(self); +} + +BinaryDelayEstimator* WebRtc_CreateBinaryDelayEstimator( + BinaryDelayEstimatorFarend* farend, int max_lookahead) { + BinaryDelayEstimator* self = NULL; + + if ((farend != NULL) && (max_lookahead >= 0)) { + // Sanity conditions fulfilled. + self = static_cast( + malloc(sizeof(BinaryDelayEstimator))); + } + if (self == NULL) { + return NULL; + } + + self->farend = farend; + self->near_history_size = max_lookahead + 1; + self->history_size = 0; + self->robust_validation_enabled = 0; // Disabled by default. + self->allowed_offset = 0; + + self->lookahead = max_lookahead; + + // Allocate memory for spectrum and history buffers. + self->mean_bit_counts = NULL; + self->bit_counts = NULL; + self->histogram = NULL; + self->binary_near_history = static_cast( + malloc((max_lookahead + 1) * sizeof(*self->binary_near_history))); + if (self->binary_near_history == NULL || + WebRtc_AllocateHistoryBufferMemory(self, farend->history_size) == 0) { + WebRtc_FreeBinaryDelayEstimator(self); + self = NULL; + } + + return self; +} + +int WebRtc_AllocateHistoryBufferMemory(BinaryDelayEstimator* self, + int history_size) { + BinaryDelayEstimatorFarend* far = self->farend; + // (Re-)Allocate memory for spectrum and history buffers. + if (history_size != far->history_size) { + // Only update far-end buffers if we need. + history_size = WebRtc_AllocateFarendBufferMemory(far, history_size); + } + // The extra array element in |mean_bit_counts| and |histogram| is a dummy + // element only used while |last_delay| == -2, i.e., before we have a valid + // estimate. + self->mean_bit_counts = static_cast( + realloc(self->mean_bit_counts, + (history_size + 1) * sizeof(*self->mean_bit_counts))); + self->bit_counts = static_cast( + realloc(self->bit_counts, history_size * sizeof(*self->bit_counts))); + self->histogram = static_cast( + realloc(self->histogram, (history_size + 1) * sizeof(*self->histogram))); + + if ((self->mean_bit_counts == NULL) || + (self->bit_counts == NULL) || + (self->histogram == NULL)) { + history_size = 0; + } + // Fill with zeros if we have expanded the buffers. + if (history_size > self->history_size) { + int size_diff = history_size - self->history_size; + memset(&self->mean_bit_counts[self->history_size], + 0, + sizeof(*self->mean_bit_counts) * size_diff); + memset(&self->bit_counts[self->history_size], + 0, + sizeof(*self->bit_counts) * size_diff); + memset(&self->histogram[self->history_size], + 0, + sizeof(*self->histogram) * size_diff); + } + self->history_size = history_size; + + return self->history_size; +} + +void WebRtc_InitBinaryDelayEstimator(BinaryDelayEstimator* self) { + int i = 0; + RTC_DCHECK(self); + + memset(self->bit_counts, 0, sizeof(int32_t) * self->history_size); + memset(self->binary_near_history, + 0, + sizeof(uint32_t) * self->near_history_size); + for (i = 0; i <= self->history_size; ++i) { + self->mean_bit_counts[i] = (20 << 9); // 20 in Q9. + self->histogram[i] = 0.f; + } + self->minimum_probability = kMaxBitCountsQ9; // 32 in Q9. + self->last_delay_probability = (int) kMaxBitCountsQ9; // 32 in Q9. + + // Default return value if we're unable to estimate. -1 is used for errors. + self->last_delay = -2; + + self->last_candidate_delay = -2; + self->compare_delay = self->history_size; + self->candidate_hits = 0; + self->last_delay_histogram = 0.f; +} + +int WebRtc_SoftResetBinaryDelayEstimator(BinaryDelayEstimator* self, + int delay_shift) { + int lookahead = 0; + RTC_DCHECK(self); + lookahead = self->lookahead; + self->lookahead -= delay_shift; + if (self->lookahead < 0) { + self->lookahead = 0; + } + if (self->lookahead > self->near_history_size - 1) { + self->lookahead = self->near_history_size - 1; + } + return lookahead - self->lookahead; +} + +int WebRtc_ProcessBinarySpectrum(BinaryDelayEstimator* self, + uint32_t binary_near_spectrum) { + int i = 0; + int candidate_delay = -1; + int valid_candidate = 0; + + int32_t value_best_candidate = kMaxBitCountsQ9; + int32_t value_worst_candidate = 0; + int32_t valley_depth = 0; + + RTC_DCHECK(self); + if (self->farend->history_size != self->history_size) { + // Non matching history sizes. + return -1; + } + if (self->near_history_size > 1) { + // If we apply lookahead, shift near-end binary spectrum history. Insert + // current |binary_near_spectrum| and pull out the delayed one. + memmove(&(self->binary_near_history[1]), &(self->binary_near_history[0]), + (self->near_history_size - 1) * sizeof(uint32_t)); + self->binary_near_history[0] = binary_near_spectrum; + binary_near_spectrum = self->binary_near_history[self->lookahead]; + } + + // Compare with delayed spectra and store the |bit_counts| for each delay. + BitCountComparison(binary_near_spectrum, self->farend->binary_far_history, + self->history_size, self->bit_counts); + + // Update |mean_bit_counts|, which is the smoothed version of |bit_counts|. + for (i = 0; i < self->history_size; i++) { + // |bit_counts| is constrained to [0, 32], meaning we can smooth with a + // factor up to 2^26. We use Q9. + int32_t bit_count = (self->bit_counts[i] << 9); // Q9. + + // Update |mean_bit_counts| only when far-end signal has something to + // contribute. If |far_bit_counts| is zero the far-end signal is weak and + // we likely have a poor echo condition, hence don't update. + if (self->farend->far_bit_counts[i] > 0) { + // Make number of right shifts piecewise linear w.r.t. |far_bit_counts|. + int shifts = kShiftsAtZero; + shifts -= (kShiftsLinearSlope * self->farend->far_bit_counts[i]) >> 4; + WebRtc_MeanEstimatorFix(bit_count, shifts, &(self->mean_bit_counts[i])); + } + } + + // Find |candidate_delay|, |value_best_candidate| and |value_worst_candidate| + // of |mean_bit_counts|. + for (i = 0; i < self->history_size; i++) { + if (self->mean_bit_counts[i] < value_best_candidate) { + value_best_candidate = self->mean_bit_counts[i]; + candidate_delay = i; + } + if (self->mean_bit_counts[i] > value_worst_candidate) { + value_worst_candidate = self->mean_bit_counts[i]; + } + } + valley_depth = value_worst_candidate - value_best_candidate; + + // The |value_best_candidate| is a good indicator on the probability of + // |candidate_delay| being an accurate delay (a small |value_best_candidate| + // means a good binary match). In the following sections we make a decision + // whether to update |last_delay| or not. + // 1) If the difference bit counts between the best and the worst delay + // candidates is too small we consider the situation to be unreliable and + // don't update |last_delay|. + // 2) If the situation is reliable we update |last_delay| if the value of the + // best candidate delay has a value less than + // i) an adaptive threshold |minimum_probability|, or + // ii) this corresponding value |last_delay_probability|, but updated at + // this time instant. + + // Update |minimum_probability|. + if ((self->minimum_probability > kProbabilityLowerLimit) && + (valley_depth > kProbabilityMinSpread)) { + // The "hard" threshold can't be lower than 17 (in Q9). + // The valley in the curve also has to be distinct, i.e., the + // difference between |value_worst_candidate| and |value_best_candidate| has + // to be large enough. + int32_t threshold = value_best_candidate + kProbabilityOffset; + if (threshold < kProbabilityLowerLimit) { + threshold = kProbabilityLowerLimit; + } + if (self->minimum_probability > threshold) { + self->minimum_probability = threshold; + } + } + // Update |last_delay_probability|. + // We use a Markov type model, i.e., a slowly increasing level over time. + self->last_delay_probability++; + // Validate |candidate_delay|. We have a reliable instantaneous delay + // estimate if + // 1) The valley is distinct enough (|valley_depth| > |kProbabilityOffset|) + // and + // 2) The depth of the valley is deep enough + // (|value_best_candidate| < |minimum_probability|) + // and deeper than the best estimate so far + // (|value_best_candidate| < |last_delay_probability|) + valid_candidate = ((valley_depth > kProbabilityOffset) && + ((value_best_candidate < self->minimum_probability) || + (value_best_candidate < self->last_delay_probability))); + + // Check for nonstationary farend signal. + const bool non_stationary_farend = + std::any_of(self->farend->far_bit_counts, + self->farend->far_bit_counts + self->history_size, + [](int a) { return a > 0; }); + + if (non_stationary_farend) { + // Only update the validation statistics when the farend is nonstationary + // as the underlying estimates are otherwise frozen. + UpdateRobustValidationStatistics(self, candidate_delay, valley_depth, + value_best_candidate); + } + + if (self->robust_validation_enabled) { + int is_histogram_valid = HistogramBasedValidation(self, candidate_delay); + valid_candidate = RobustValidation(self, candidate_delay, valid_candidate, + is_histogram_valid); + + } + + // Only update the delay estimate when the farend is nonstationary and when + // a valid delay candidate is available. + if (non_stationary_farend && valid_candidate) { + if (candidate_delay != self->last_delay) { + self->last_delay_histogram = + (self->histogram[candidate_delay] > kLastHistogramMax ? + kLastHistogramMax : self->histogram[candidate_delay]); + // Adjust the histogram if we made a change to |last_delay|, though it was + // not the most likely one according to the histogram. + if (self->histogram[candidate_delay] < + self->histogram[self->compare_delay]) { + self->histogram[self->compare_delay] = self->histogram[candidate_delay]; + } + } + self->last_delay = candidate_delay; + if (value_best_candidate < self->last_delay_probability) { + self->last_delay_probability = value_best_candidate; + } + self->compare_delay = self->last_delay; + } + + return self->last_delay; +} + +int WebRtc_binary_last_delay(BinaryDelayEstimator* self) { + RTC_DCHECK(self); + return self->last_delay; +} + +float WebRtc_binary_last_delay_quality(BinaryDelayEstimator* self) { + float quality = 0; + RTC_DCHECK(self); + + if (self->robust_validation_enabled) { + // Simply a linear function of the histogram height at delay estimate. + quality = self->histogram[self->compare_delay] / kHistogramMax; + } else { + // Note that |last_delay_probability| states how deep the minimum of the + // cost function is, so it is rather an error probability. + quality = (float) (kMaxBitCountsQ9 - self->last_delay_probability) / + kMaxBitCountsQ9; + if (quality < 0) { + quality = 0; + } + } + return quality; +} + +void WebRtc_MeanEstimatorFix(int32_t new_value, + int factor, + int32_t* mean_value) { + int32_t diff = new_value - *mean_value; + + // mean_new = mean_value + ((new_value - mean_value) >> factor); + if (diff < 0) { + diff = -((-diff) >> factor); + } else { + diff = (diff >> factor); + } + *mean_value += diff; +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.h b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.h new file mode 100644 index 0000000000..cce6113a53 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator.h @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Performs delay estimation on binary converted spectra. +// The return value is 0 - OK and -1 - Error, unless otherwise stated. + +#ifndef MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_H_ + +#include "typedefs.h" // NOLINT(build/include) + +static const int32_t kMaxBitCountsQ9 = (32 << 9); // 32 matching bits in Q9. + +typedef struct { + // Pointer to bit counts. + int* far_bit_counts; + // Binary history variables. + uint32_t* binary_far_history; + int history_size; +} BinaryDelayEstimatorFarend; + +typedef struct { + // Pointer to bit counts. + int32_t* mean_bit_counts; + // Array only used locally in ProcessBinarySpectrum() but whose size is + // determined at run-time. + int32_t* bit_counts; + + // Binary history variables. + uint32_t* binary_near_history; + int near_history_size; + int history_size; + + // Delay estimation variables. + int32_t minimum_probability; + int last_delay_probability; + + // Delay memory. + int last_delay; + + // Robust validation + int robust_validation_enabled; + int allowed_offset; + int last_candidate_delay; + int compare_delay; + int candidate_hits; + float* histogram; + float last_delay_histogram; + + // For dynamically changing the lookahead when using SoftReset...(). + int lookahead; + + // Far-end binary spectrum history buffer etc. + BinaryDelayEstimatorFarend* farend; +} BinaryDelayEstimator; + +// Releases the memory allocated by +// WebRtc_CreateBinaryDelayEstimatorFarend(...). +// Input: +// - self : Pointer to the binary delay estimation far-end +// instance which is the return value of +// WebRtc_CreateBinaryDelayEstimatorFarend(). +// +void WebRtc_FreeBinaryDelayEstimatorFarend(BinaryDelayEstimatorFarend* self); + +// Allocates the memory needed by the far-end part of the binary delay +// estimation. The memory needs to be initialized separately through +// WebRtc_InitBinaryDelayEstimatorFarend(...). +// +// Inputs: +// - history_size : Size of the far-end binary spectrum history. +// +// Return value: +// - BinaryDelayEstimatorFarend* +// : Created |handle|. If the memory can't be allocated +// or if any of the input parameters are invalid NULL +// is returned. +// +BinaryDelayEstimatorFarend* WebRtc_CreateBinaryDelayEstimatorFarend( + int history_size); + +// Re-allocates the buffers. +// +// Inputs: +// - self : Pointer to the binary estimation far-end instance +// which is the return value of +// WebRtc_CreateBinaryDelayEstimatorFarend(). +// - history_size : Size of the far-end binary spectrum history. +// +// Return value: +// - history_size : The history size allocated. +int WebRtc_AllocateFarendBufferMemory(BinaryDelayEstimatorFarend* self, + int history_size); + +// Initializes the delay estimation far-end instance created with +// WebRtc_CreateBinaryDelayEstimatorFarend(...). +// +// Input: +// - self : Pointer to the delay estimation far-end instance. +// +// Output: +// - self : Initialized far-end instance. +// +void WebRtc_InitBinaryDelayEstimatorFarend(BinaryDelayEstimatorFarend* self); + +// Soft resets the delay estimation far-end instance created with +// WebRtc_CreateBinaryDelayEstimatorFarend(...). +// +// Input: +// - delay_shift : The amount of blocks to shift history buffers. +// +void WebRtc_SoftResetBinaryDelayEstimatorFarend( + BinaryDelayEstimatorFarend* self, int delay_shift); + +// Adds the binary far-end spectrum to the internal far-end history buffer. This +// spectrum is used as reference when calculating the delay using +// WebRtc_ProcessBinarySpectrum(). +// +// Inputs: +// - self : Pointer to the delay estimation far-end +// instance. +// - binary_far_spectrum : Far-end binary spectrum. +// +// Output: +// - self : Updated far-end instance. +// +void WebRtc_AddBinaryFarSpectrum(BinaryDelayEstimatorFarend* self, + uint32_t binary_far_spectrum); + +// Releases the memory allocated by WebRtc_CreateBinaryDelayEstimator(...). +// +// Note that BinaryDelayEstimator utilizes BinaryDelayEstimatorFarend, but does +// not take ownership of it, hence the BinaryDelayEstimator has to be torn down +// before the far-end. +// +// Input: +// - self : Pointer to the binary delay estimation instance +// which is the return value of +// WebRtc_CreateBinaryDelayEstimator(). +// +void WebRtc_FreeBinaryDelayEstimator(BinaryDelayEstimator* self); + +// Allocates the memory needed by the binary delay estimation. The memory needs +// to be initialized separately through WebRtc_InitBinaryDelayEstimator(...). +// +// See WebRtc_CreateDelayEstimator(..) in delay_estimator_wrapper.c for detailed +// description. +BinaryDelayEstimator* WebRtc_CreateBinaryDelayEstimator( + BinaryDelayEstimatorFarend* farend, int max_lookahead); + +// Re-allocates |history_size| dependent buffers. The far-end buffers will be +// updated at the same time if needed. +// +// Input: +// - self : Pointer to the binary estimation instance which is +// the return value of +// WebRtc_CreateBinaryDelayEstimator(). +// - history_size : Size of the history buffers. +// +// Return value: +// - history_size : The history size allocated. +int WebRtc_AllocateHistoryBufferMemory(BinaryDelayEstimator* self, + int history_size); + +// Initializes the delay estimation instance created with +// WebRtc_CreateBinaryDelayEstimator(...). +// +// Input: +// - self : Pointer to the delay estimation instance. +// +// Output: +// - self : Initialized instance. +// +void WebRtc_InitBinaryDelayEstimator(BinaryDelayEstimator* self); + +// Soft resets the delay estimation instance created with +// WebRtc_CreateBinaryDelayEstimator(...). +// +// Input: +// - delay_shift : The amount of blocks to shift history buffers. +// +// Return value: +// - actual_shifts : The actual number of shifts performed. +// +int WebRtc_SoftResetBinaryDelayEstimator(BinaryDelayEstimator* self, + int delay_shift); + +// Estimates and returns the delay between the binary far-end and binary near- +// end spectra. It is assumed the binary far-end spectrum has been added using +// WebRtc_AddBinaryFarSpectrum() prior to this call. The value will be offset by +// the lookahead (i.e. the lookahead should be subtracted from the returned +// value). +// +// Inputs: +// - self : Pointer to the delay estimation instance. +// - binary_near_spectrum : Near-end binary spectrum of the current block. +// +// Output: +// - self : Updated instance. +// +// Return value: +// - delay : >= 0 - Calculated delay value. +// -2 - Insufficient data for estimation. +// +int WebRtc_ProcessBinarySpectrum(BinaryDelayEstimator* self, + uint32_t binary_near_spectrum); + +// Returns the last calculated delay updated by the function +// WebRtc_ProcessBinarySpectrum(...). +// +// Input: +// - self : Pointer to the delay estimation instance. +// +// Return value: +// - delay : >= 0 - Last calculated delay value +// -2 - Insufficient data for estimation. +// +int WebRtc_binary_last_delay(BinaryDelayEstimator* self); + +// Returns the estimation quality of the last calculated delay updated by the +// function WebRtc_ProcessBinarySpectrum(...). The estimation quality is a value +// in the interval [0, 1]. The higher the value, the better the quality. +// +// Return value: +// - delay_quality : >= 0 - Estimation quality of last calculated +// delay value. +float WebRtc_binary_last_delay_quality(BinaryDelayEstimator* self); + +// Updates the |mean_value| recursively with a step size of 2^-|factor|. This +// function is used internally in the Binary Delay Estimator as well as the +// Fixed point wrapper. +// +// Inputs: +// - new_value : The new value the mean should be updated with. +// - factor : The step size, in number of right shifts. +// +// Input/Output: +// - mean_value : Pointer to the mean value. +// +void WebRtc_MeanEstimatorFix(int32_t new_value, + int factor, + int32_t* mean_value); + +#endif // MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_internal.h b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_internal.h new file mode 100644 index 0000000000..46eea3ec18 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_internal.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Header file including the delay estimator handle used for testing. + +#ifndef MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_INTERNAL_H_ +#define MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_INTERNAL_H_ + +#include "modules/audio_processing/utility/delay_estimator.h" +#include "typedefs.h" // NOLINT(build/include) + +typedef union { + float float_; + int32_t int32_; +} SpectrumType; + +typedef struct { + // Pointers to mean values of spectrum. + SpectrumType* mean_far_spectrum; + // |mean_far_spectrum| initialization indicator. + int far_spectrum_initialized; + + int spectrum_size; + + // Far-end part of binary spectrum based delay estimation. + BinaryDelayEstimatorFarend* binary_farend; +} DelayEstimatorFarend; + +typedef struct { + // Pointers to mean values of spectrum. + SpectrumType* mean_near_spectrum; + // |mean_near_spectrum| initialization indicator. + int near_spectrum_initialized; + + int spectrum_size; + + // Binary spectrum based delay estimator + BinaryDelayEstimator* binary_handle; +} DelayEstimator; + +#endif // MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_INTERNAL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_unittest.cc new file mode 100644 index 0000000000..36700e5706 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_unittest.cc @@ -0,0 +1,618 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/utility/delay_estimator.h" +#include "modules/audio_processing/utility/delay_estimator_internal.h" +#include "modules/audio_processing/utility/delay_estimator_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace { + +enum { kSpectrumSize = 65 }; +// Delay history sizes. +enum { kMaxDelay = 100 }; +enum { kLookahead = 10 }; +enum { kHistorySize = kMaxDelay + kLookahead }; +// Length of binary spectrum sequence. +enum { kSequenceLength = 400 }; + +const int kDifferentHistorySize = 3; +const int kDifferentLookahead = 1; + +const int kEnable[] = { 0, 1 }; +const size_t kSizeEnable = sizeof(kEnable) / sizeof(*kEnable); + +class DelayEstimatorTest : public ::testing::Test { + protected: + DelayEstimatorTest(); + virtual void SetUp(); + virtual void TearDown(); + + void Init(); + void InitBinary(); + void VerifyDelay(BinaryDelayEstimator* binary_handle, int offset, int delay); + void RunBinarySpectra(BinaryDelayEstimator* binary1, + BinaryDelayEstimator* binary2, + int near_offset, int lookahead_offset, int far_offset); + void RunBinarySpectraTest(int near_offset, int lookahead_offset, + int ref_robust_validation, int robust_validation); + + void* handle_; + DelayEstimator* self_; + void* farend_handle_; + DelayEstimatorFarend* farend_self_; + BinaryDelayEstimator* binary_; + BinaryDelayEstimatorFarend* binary_farend_; + int spectrum_size_; + // Dummy input spectra. + float far_f_[kSpectrumSize]; + float near_f_[kSpectrumSize]; + uint16_t far_u16_[kSpectrumSize]; + uint16_t near_u16_[kSpectrumSize]; + uint32_t binary_spectrum_[kSequenceLength + kHistorySize]; +}; + +DelayEstimatorTest::DelayEstimatorTest() + : handle_(NULL), + self_(NULL), + farend_handle_(NULL), + farend_self_(NULL), + binary_(NULL), + binary_farend_(NULL), + spectrum_size_(kSpectrumSize) { + // Dummy input data are set with more or less arbitrary non-zero values. + memset(far_f_, 1, sizeof(far_f_)); + memset(near_f_, 2, sizeof(near_f_)); + memset(far_u16_, 1, sizeof(far_u16_)); + memset(near_u16_, 2, sizeof(near_u16_)); + // Construct a sequence of binary spectra used to verify delay estimate. The + // |kSequenceLength| has to be long enough for the delay estimation to leave + // the initialized state. + binary_spectrum_[0] = 1; + for (int i = 1; i < (kSequenceLength + kHistorySize); i++) { + binary_spectrum_[i] = 3 * binary_spectrum_[i - 1]; + } +} + +void DelayEstimatorTest::SetUp() { + farend_handle_ = WebRtc_CreateDelayEstimatorFarend(kSpectrumSize, + kHistorySize); + ASSERT_TRUE(farend_handle_ != NULL); + farend_self_ = reinterpret_cast(farend_handle_); + handle_ = WebRtc_CreateDelayEstimator(farend_handle_, kLookahead); + ASSERT_TRUE(handle_ != NULL); + self_ = reinterpret_cast(handle_); + binary_farend_ = WebRtc_CreateBinaryDelayEstimatorFarend(kHistorySize); + ASSERT_TRUE(binary_farend_ != NULL); + binary_ = WebRtc_CreateBinaryDelayEstimator(binary_farend_, kLookahead); + ASSERT_TRUE(binary_ != NULL); +} + +void DelayEstimatorTest::TearDown() { + WebRtc_FreeDelayEstimator(handle_); + handle_ = NULL; + self_ = NULL; + WebRtc_FreeDelayEstimatorFarend(farend_handle_); + farend_handle_ = NULL; + farend_self_ = NULL; + WebRtc_FreeBinaryDelayEstimator(binary_); + binary_ = NULL; + WebRtc_FreeBinaryDelayEstimatorFarend(binary_farend_); + binary_farend_ = NULL; +} + +void DelayEstimatorTest::Init() { + // Initialize Delay Estimator + EXPECT_EQ(0, WebRtc_InitDelayEstimatorFarend(farend_handle_)); + EXPECT_EQ(0, WebRtc_InitDelayEstimator(handle_)); + // Verify initialization. + EXPECT_EQ(0, farend_self_->far_spectrum_initialized); + EXPECT_EQ(0, self_->near_spectrum_initialized); + EXPECT_EQ(-2, WebRtc_last_delay(handle_)); // Delay in initial state. + EXPECT_FLOAT_EQ(0, WebRtc_last_delay_quality(handle_)); // Zero quality. +} + +void DelayEstimatorTest::InitBinary() { + // Initialize Binary Delay Estimator (far-end part). + WebRtc_InitBinaryDelayEstimatorFarend(binary_farend_); + // Initialize Binary Delay Estimator + WebRtc_InitBinaryDelayEstimator(binary_); + // Verify initialization. This does not guarantee a complete check, since + // |last_delay| may be equal to -2 before initialization if done on the fly. + EXPECT_EQ(-2, binary_->last_delay); +} + +void DelayEstimatorTest::VerifyDelay(BinaryDelayEstimator* binary_handle, + int offset, int delay) { + // Verify that we WebRtc_binary_last_delay() returns correct delay. + EXPECT_EQ(delay, WebRtc_binary_last_delay(binary_handle)); + + if (delay != -2) { + // Verify correct delay estimate. In the non-causal case the true delay + // is equivalent with the |offset|. + EXPECT_EQ(offset, delay); + } +} + +void DelayEstimatorTest::RunBinarySpectra(BinaryDelayEstimator* binary1, + BinaryDelayEstimator* binary2, + int near_offset, + int lookahead_offset, + int far_offset) { + int different_validations = binary1->robust_validation_enabled ^ + binary2->robust_validation_enabled; + WebRtc_InitBinaryDelayEstimatorFarend(binary_farend_); + WebRtc_InitBinaryDelayEstimator(binary1); + WebRtc_InitBinaryDelayEstimator(binary2); + // Verify initialization. This does not guarantee a complete check, since + // |last_delay| may be equal to -2 before initialization if done on the fly. + EXPECT_EQ(-2, binary1->last_delay); + EXPECT_EQ(-2, binary2->last_delay); + for (int i = kLookahead; i < (kSequenceLength + kLookahead); i++) { + WebRtc_AddBinaryFarSpectrum(binary_farend_, + binary_spectrum_[i + far_offset]); + int delay_1 = WebRtc_ProcessBinarySpectrum(binary1, binary_spectrum_[i]); + int delay_2 = + WebRtc_ProcessBinarySpectrum(binary2, + binary_spectrum_[i - near_offset]); + + VerifyDelay(binary1, far_offset + kLookahead, delay_1); + VerifyDelay(binary2, + far_offset + kLookahead + lookahead_offset + near_offset, + delay_2); + // Expect the two delay estimates to be offset by |lookahead_offset| + + // |near_offset| when we have left the initial state. + if ((delay_1 != -2) && (delay_2 != -2)) { + EXPECT_EQ(delay_1, delay_2 - lookahead_offset - near_offset); + } + // For the case of identical signals |delay_1| and |delay_2| should match + // all the time, unless one of them has robust validation turned on. In + // that case the robust validation leaves the initial state faster. + if ((near_offset == 0) && (lookahead_offset == 0)) { + if (!different_validations) { + EXPECT_EQ(delay_1, delay_2); + } else { + if (binary1->robust_validation_enabled) { + EXPECT_GE(delay_1, delay_2); + } else { + EXPECT_GE(delay_2, delay_1); + } + } + } + } + // Verify that we have left the initialized state. + EXPECT_NE(-2, WebRtc_binary_last_delay(binary1)); + EXPECT_LT(0, WebRtc_binary_last_delay_quality(binary1)); + EXPECT_NE(-2, WebRtc_binary_last_delay(binary2)); + EXPECT_LT(0, WebRtc_binary_last_delay_quality(binary2)); +} + +void DelayEstimatorTest::RunBinarySpectraTest(int near_offset, + int lookahead_offset, + int ref_robust_validation, + int robust_validation) { + BinaryDelayEstimator* binary2 = + WebRtc_CreateBinaryDelayEstimator(binary_farend_, + kLookahead + lookahead_offset); + // Verify the delay for both causal and non-causal systems. For causal systems + // the delay is equivalent with a positive |offset| of the far-end sequence. + // For non-causal systems the delay is equivalent with a negative |offset| of + // the far-end sequence. + binary_->robust_validation_enabled = ref_robust_validation; + binary2->robust_validation_enabled = robust_validation; + for (int offset = -kLookahead; + offset < kMaxDelay - lookahead_offset - near_offset; + offset++) { + RunBinarySpectra(binary_, binary2, near_offset, lookahead_offset, offset); + } + WebRtc_FreeBinaryDelayEstimator(binary2); + binary2 = NULL; + binary_->robust_validation_enabled = 0; // Reset reference. +} + +TEST_F(DelayEstimatorTest, CorrectErrorReturnsOfWrapper) { + // In this test we verify correct error returns on invalid API calls. + + // WebRtc_CreateDelayEstimatorFarend() and WebRtc_CreateDelayEstimator() + // should return a NULL pointer on invalid input values. + // Make sure we have a non-NULL value at start, so we can detect NULL after + // create failure. + void* handle = farend_handle_; + handle = WebRtc_CreateDelayEstimatorFarend(33, kHistorySize); + EXPECT_TRUE(handle == NULL); + handle = WebRtc_CreateDelayEstimatorFarend(kSpectrumSize, 1); + EXPECT_TRUE(handle == NULL); + + handle = handle_; + handle = WebRtc_CreateDelayEstimator(NULL, kLookahead); + EXPECT_TRUE(handle == NULL); + handle = WebRtc_CreateDelayEstimator(farend_handle_, -1); + EXPECT_TRUE(handle == NULL); + + // WebRtc_InitDelayEstimatorFarend() and WebRtc_InitDelayEstimator() should + // return -1 if we have a NULL pointer as |handle|. + EXPECT_EQ(-1, WebRtc_InitDelayEstimatorFarend(NULL)); + EXPECT_EQ(-1, WebRtc_InitDelayEstimator(NULL)); + + // WebRtc_AddFarSpectrumFloat() should return -1 if we have: + // 1) NULL pointer as |handle|. + // 2) NULL pointer as far-end spectrum. + // 3) Incorrect spectrum size. + EXPECT_EQ(-1, WebRtc_AddFarSpectrumFloat(NULL, far_f_, spectrum_size_)); + // Use |farend_handle_| which is properly created at SetUp(). + EXPECT_EQ(-1, WebRtc_AddFarSpectrumFloat(farend_handle_, NULL, + spectrum_size_)); + EXPECT_EQ(-1, WebRtc_AddFarSpectrumFloat(farend_handle_, far_f_, + spectrum_size_ + 1)); + + // WebRtc_AddFarSpectrumFix() should return -1 if we have: + // 1) NULL pointer as |handle|. + // 2) NULL pointer as far-end spectrum. + // 3) Incorrect spectrum size. + // 4) Too high precision in far-end spectrum (Q-domain > 15). + EXPECT_EQ(-1, WebRtc_AddFarSpectrumFix(NULL, far_u16_, spectrum_size_, 0)); + EXPECT_EQ(-1, WebRtc_AddFarSpectrumFix(farend_handle_, NULL, spectrum_size_, + 0)); + EXPECT_EQ(-1, WebRtc_AddFarSpectrumFix(farend_handle_, far_u16_, + spectrum_size_ + 1, 0)); + EXPECT_EQ(-1, WebRtc_AddFarSpectrumFix(farend_handle_, far_u16_, + spectrum_size_, 16)); + + // WebRtc_set_history_size() should return -1 if: + // 1) |handle| is a NULL. + // 2) |history_size| <= 1. + EXPECT_EQ(-1, WebRtc_set_history_size(NULL, 1)); + EXPECT_EQ(-1, WebRtc_set_history_size(handle_, 1)); + // WebRtc_history_size() should return -1 if: + // 1) NULL pointer input. + EXPECT_EQ(-1, WebRtc_history_size(NULL)); + // 2) there is a mismatch between history size. + void* tmp_handle = WebRtc_CreateDelayEstimator(farend_handle_, kHistorySize); + EXPECT_EQ(0, WebRtc_InitDelayEstimator(tmp_handle)); + EXPECT_EQ(kDifferentHistorySize, + WebRtc_set_history_size(tmp_handle, kDifferentHistorySize)); + EXPECT_EQ(kDifferentHistorySize, WebRtc_history_size(tmp_handle)); + EXPECT_EQ(kHistorySize, WebRtc_set_history_size(handle_, kHistorySize)); + EXPECT_EQ(-1, WebRtc_history_size(tmp_handle)); + + // WebRtc_set_lookahead() should return -1 if we try a value outside the + /// buffer. + EXPECT_EQ(-1, WebRtc_set_lookahead(handle_, kLookahead + 1)); + EXPECT_EQ(-1, WebRtc_set_lookahead(handle_, -1)); + + // WebRtc_set_allowed_offset() should return -1 if we have: + // 1) NULL pointer as |handle|. + // 2) |allowed_offset| < 0. + EXPECT_EQ(-1, WebRtc_set_allowed_offset(NULL, 0)); + EXPECT_EQ(-1, WebRtc_set_allowed_offset(handle_, -1)); + + EXPECT_EQ(-1, WebRtc_get_allowed_offset(NULL)); + + // WebRtc_enable_robust_validation() should return -1 if we have: + // 1) NULL pointer as |handle|. + // 2) Incorrect |enable| value (not 0 or 1). + EXPECT_EQ(-1, WebRtc_enable_robust_validation(NULL, kEnable[0])); + EXPECT_EQ(-1, WebRtc_enable_robust_validation(handle_, -1)); + EXPECT_EQ(-1, WebRtc_enable_robust_validation(handle_, 2)); + + // WebRtc_is_robust_validation_enabled() should return -1 if we have NULL + // pointer as |handle|. + EXPECT_EQ(-1, WebRtc_is_robust_validation_enabled(NULL)); + + // WebRtc_DelayEstimatorProcessFloat() should return -1 if we have: + // 1) NULL pointer as |handle|. + // 2) NULL pointer as near-end spectrum. + // 3) Incorrect spectrum size. + // 4) Non matching history sizes if multiple delay estimators using the same + // far-end reference. + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFloat(NULL, near_f_, + spectrum_size_)); + // Use |handle_| which is properly created at SetUp(). + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFloat(handle_, NULL, + spectrum_size_)); + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFloat(handle_, near_f_, + spectrum_size_ + 1)); + // |tmp_handle| is already in a non-matching state. + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFloat(tmp_handle, + near_f_, + spectrum_size_)); + + // WebRtc_DelayEstimatorProcessFix() should return -1 if we have: + // 1) NULL pointer as |handle|. + // 2) NULL pointer as near-end spectrum. + // 3) Incorrect spectrum size. + // 4) Too high precision in near-end spectrum (Q-domain > 15). + // 5) Non matching history sizes if multiple delay estimators using the same + // far-end reference. + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFix(NULL, near_u16_, spectrum_size_, + 0)); + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFix(handle_, NULL, spectrum_size_, + 0)); + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFix(handle_, near_u16_, + spectrum_size_ + 1, 0)); + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFix(handle_, near_u16_, + spectrum_size_, 16)); + // |tmp_handle| is already in a non-matching state. + EXPECT_EQ(-1, WebRtc_DelayEstimatorProcessFix(tmp_handle, + near_u16_, + spectrum_size_, + 0)); + WebRtc_FreeDelayEstimator(tmp_handle); + + // WebRtc_last_delay() should return -1 if we have a NULL pointer as |handle|. + EXPECT_EQ(-1, WebRtc_last_delay(NULL)); + + // Free any local memory if needed. + WebRtc_FreeDelayEstimator(handle); +} + +TEST_F(DelayEstimatorTest, VerifyAllowedOffset) { + // Is set to zero by default. + EXPECT_EQ(0, WebRtc_get_allowed_offset(handle_)); + for (int i = 1; i >= 0; i--) { + EXPECT_EQ(0, WebRtc_set_allowed_offset(handle_, i)); + EXPECT_EQ(i, WebRtc_get_allowed_offset(handle_)); + Init(); + // Unaffected over a reset. + EXPECT_EQ(i, WebRtc_get_allowed_offset(handle_)); + } +} + +TEST_F(DelayEstimatorTest, VerifyEnableRobustValidation) { + // Disabled by default. + EXPECT_EQ(0, WebRtc_is_robust_validation_enabled(handle_)); + for (size_t i = 0; i < kSizeEnable; ++i) { + EXPECT_EQ(0, WebRtc_enable_robust_validation(handle_, kEnable[i])); + EXPECT_EQ(kEnable[i], WebRtc_is_robust_validation_enabled(handle_)); + Init(); + // Unaffected over a reset. + EXPECT_EQ(kEnable[i], WebRtc_is_robust_validation_enabled(handle_)); + } +} + +TEST_F(DelayEstimatorTest, InitializedSpectrumAfterProcess) { + // In this test we verify that the mean spectra are initialized after first + // time we call WebRtc_AddFarSpectrum() and Process() respectively. The test + // also verifies the state is not left for zero spectra. + const float kZerosFloat[kSpectrumSize] = { 0.0 }; + const uint16_t kZerosU16[kSpectrumSize] = { 0 }; + + // For floating point operations, process one frame and verify initialization + // flag. + Init(); + EXPECT_EQ(0, WebRtc_AddFarSpectrumFloat(farend_handle_, kZerosFloat, + spectrum_size_)); + EXPECT_EQ(0, farend_self_->far_spectrum_initialized); + EXPECT_EQ(0, WebRtc_AddFarSpectrumFloat(farend_handle_, far_f_, + spectrum_size_)); + EXPECT_EQ(1, farend_self_->far_spectrum_initialized); + EXPECT_EQ(-2, WebRtc_DelayEstimatorProcessFloat(handle_, kZerosFloat, + spectrum_size_)); + EXPECT_EQ(0, self_->near_spectrum_initialized); + EXPECT_EQ(-2, WebRtc_DelayEstimatorProcessFloat(handle_, near_f_, + spectrum_size_)); + EXPECT_EQ(1, self_->near_spectrum_initialized); + + // For fixed point operations, process one frame and verify initialization + // flag. + Init(); + EXPECT_EQ(0, WebRtc_AddFarSpectrumFix(farend_handle_, kZerosU16, + spectrum_size_, 0)); + EXPECT_EQ(0, farend_self_->far_spectrum_initialized); + EXPECT_EQ(0, WebRtc_AddFarSpectrumFix(farend_handle_, far_u16_, + spectrum_size_, 0)); + EXPECT_EQ(1, farend_self_->far_spectrum_initialized); + EXPECT_EQ(-2, WebRtc_DelayEstimatorProcessFix(handle_, kZerosU16, + spectrum_size_, 0)); + EXPECT_EQ(0, self_->near_spectrum_initialized); + EXPECT_EQ(-2, WebRtc_DelayEstimatorProcessFix(handle_, near_u16_, + spectrum_size_, 0)); + EXPECT_EQ(1, self_->near_spectrum_initialized); +} + +TEST_F(DelayEstimatorTest, CorrectLastDelay) { + // In this test we verify that we get the correct last delay upon valid call. + // We simply process the same data until we leave the initialized state + // (|last_delay| = -2). Then we compare the Process() output with the + // last_delay() call. + + // TODO(bjornv): Update quality values for robust validation. + int last_delay = 0; + // Floating point operations. + Init(); + for (int i = 0; i < 200; i++) { + EXPECT_EQ(0, WebRtc_AddFarSpectrumFloat(farend_handle_, far_f_, + spectrum_size_)); + last_delay = WebRtc_DelayEstimatorProcessFloat(handle_, near_f_, + spectrum_size_); + if (last_delay != -2) { + EXPECT_EQ(last_delay, WebRtc_last_delay(handle_)); + if (!WebRtc_is_robust_validation_enabled(handle_)) { + EXPECT_FLOAT_EQ(7203.f / kMaxBitCountsQ9, + WebRtc_last_delay_quality(handle_)); + } + break; + } + } + // Verify that we have left the initialized state. + EXPECT_NE(-2, WebRtc_last_delay(handle_)); + EXPECT_LT(0, WebRtc_last_delay_quality(handle_)); + + // Fixed point operations. + Init(); + for (int i = 0; i < 200; i++) { + EXPECT_EQ(0, WebRtc_AddFarSpectrumFix(farend_handle_, far_u16_, + spectrum_size_, 0)); + last_delay = WebRtc_DelayEstimatorProcessFix(handle_, near_u16_, + spectrum_size_, 0); + if (last_delay != -2) { + EXPECT_EQ(last_delay, WebRtc_last_delay(handle_)); + if (!WebRtc_is_robust_validation_enabled(handle_)) { + EXPECT_FLOAT_EQ(7203.f / kMaxBitCountsQ9, + WebRtc_last_delay_quality(handle_)); + } + break; + } + } + // Verify that we have left the initialized state. + EXPECT_NE(-2, WebRtc_last_delay(handle_)); + EXPECT_LT(0, WebRtc_last_delay_quality(handle_)); +} + +TEST_F(DelayEstimatorTest, CorrectErrorReturnsOfBinaryEstimatorFarend) { + // In this test we verify correct output on invalid API calls to the Binary + // Delay Estimator (far-end part). + + BinaryDelayEstimatorFarend* binary = binary_farend_; + // WebRtc_CreateBinaryDelayEstimatorFarend() should return -1 if the input + // history size is less than 2. This is to make sure the buffer shifting + // applies properly. + // Make sure we have a non-NULL value at start, so we can detect NULL after + // create failure. + binary = WebRtc_CreateBinaryDelayEstimatorFarend(1); + EXPECT_TRUE(binary == NULL); +} + +TEST_F(DelayEstimatorTest, CorrectErrorReturnsOfBinaryEstimator) { + // In this test we verify correct output on invalid API calls to the Binary + // Delay Estimator. + + BinaryDelayEstimator* binary_handle = binary_; + // WebRtc_CreateBinaryDelayEstimator() should return -1 if we have a NULL + // pointer as |binary_farend| or invalid input values. Upon failure, the + // |binary_handle| should be NULL. + // Make sure we have a non-NULL value at start, so we can detect NULL after + // create failure. + binary_handle = WebRtc_CreateBinaryDelayEstimator(NULL, kLookahead); + EXPECT_TRUE(binary_handle == NULL); + binary_handle = WebRtc_CreateBinaryDelayEstimator(binary_farend_, -1); + EXPECT_TRUE(binary_handle == NULL); +} + +TEST_F(DelayEstimatorTest, MeanEstimatorFix) { + // In this test we verify that we update the mean value in correct direction + // only. With "direction" we mean increase or decrease. + + int32_t mean_value = 4000; + int32_t mean_value_before = mean_value; + int32_t new_mean_value = mean_value * 2; + + // Increasing |mean_value|. + WebRtc_MeanEstimatorFix(new_mean_value, 10, &mean_value); + EXPECT_LT(mean_value_before, mean_value); + EXPECT_GT(new_mean_value, mean_value); + + // Decreasing |mean_value|. + new_mean_value = mean_value / 2; + mean_value_before = mean_value; + WebRtc_MeanEstimatorFix(new_mean_value, 10, &mean_value); + EXPECT_GT(mean_value_before, mean_value); + EXPECT_LT(new_mean_value, mean_value); +} + +TEST_F(DelayEstimatorTest, ExactDelayEstimateMultipleNearSameSpectrum) { + // In this test we verify that we get the correct delay estimates if we shift + // the signal accordingly. We create two Binary Delay Estimators and feed them + // with the same signals, so they should output the same results. + // We verify both causal and non-causal delays. + // For these noise free signals, the robust validation should not have an + // impact, hence we turn robust validation on/off for both reference and + // delayed near end. + + for (size_t i = 0; i < kSizeEnable; ++i) { + for (size_t j = 0; j < kSizeEnable; ++j) { + RunBinarySpectraTest(0, 0, kEnable[i], kEnable[j]); + } + } +} + +TEST_F(DelayEstimatorTest, ExactDelayEstimateMultipleNearDifferentSpectrum) { + // In this test we use the same setup as above, but we now feed the two Binary + // Delay Estimators with different signals, so they should output different + // results. + // For these noise free signals, the robust validation should not have an + // impact, hence we turn robust validation on/off for both reference and + // delayed near end. + + const int kNearOffset = 1; + for (size_t i = 0; i < kSizeEnable; ++i) { + for (size_t j = 0; j < kSizeEnable; ++j) { + RunBinarySpectraTest(kNearOffset, 0, kEnable[i], kEnable[j]); + } + } +} + +TEST_F(DelayEstimatorTest, ExactDelayEstimateMultipleNearDifferentLookahead) { + // In this test we use the same setup as above, feeding the two Binary + // Delay Estimators with the same signals. The difference is that we create + // them with different lookahead. + // For these noise free signals, the robust validation should not have an + // impact, hence we turn robust validation on/off for both reference and + // delayed near end. + + const int kLookaheadOffset = 1; + for (size_t i = 0; i < kSizeEnable; ++i) { + for (size_t j = 0; j < kSizeEnable; ++j) { + RunBinarySpectraTest(0, kLookaheadOffset, kEnable[i], kEnable[j]); + } + } +} + +TEST_F(DelayEstimatorTest, AllowedOffsetNoImpactWhenRobustValidationDisabled) { + // The same setup as in ExactDelayEstimateMultipleNearSameSpectrum with the + // difference that |allowed_offset| is set for the reference binary delay + // estimator. + + binary_->allowed_offset = 10; + RunBinarySpectraTest(0, 0, 0, 0); + binary_->allowed_offset = 0; // Reset reference. +} + +TEST_F(DelayEstimatorTest, VerifyLookaheadAtCreate) { + void* farend_handle = WebRtc_CreateDelayEstimatorFarend(kSpectrumSize, + kMaxDelay); + ASSERT_TRUE(farend_handle != NULL); + void* handle = WebRtc_CreateDelayEstimator(farend_handle, kLookahead); + ASSERT_TRUE(handle != NULL); + EXPECT_EQ(kLookahead, WebRtc_lookahead(handle)); + WebRtc_FreeDelayEstimator(handle); + WebRtc_FreeDelayEstimatorFarend(farend_handle); +} + +TEST_F(DelayEstimatorTest, VerifyLookaheadIsSetAndKeptAfterInit) { + EXPECT_EQ(kLookahead, WebRtc_lookahead(handle_)); + EXPECT_EQ(kDifferentLookahead, + WebRtc_set_lookahead(handle_, kDifferentLookahead)); + EXPECT_EQ(kDifferentLookahead, WebRtc_lookahead(handle_)); + EXPECT_EQ(0, WebRtc_InitDelayEstimatorFarend(farend_handle_)); + EXPECT_EQ(kDifferentLookahead, WebRtc_lookahead(handle_)); + EXPECT_EQ(0, WebRtc_InitDelayEstimator(handle_)); + EXPECT_EQ(kDifferentLookahead, WebRtc_lookahead(handle_)); +} + +TEST_F(DelayEstimatorTest, VerifyHistorySizeAtCreate) { + EXPECT_EQ(kHistorySize, WebRtc_history_size(handle_)); +} + +TEST_F(DelayEstimatorTest, VerifyHistorySizeIsSetAndKeptAfterInit) { + EXPECT_EQ(kHistorySize, WebRtc_history_size(handle_)); + EXPECT_EQ(kDifferentHistorySize, + WebRtc_set_history_size(handle_, kDifferentHistorySize)); + EXPECT_EQ(kDifferentHistorySize, WebRtc_history_size(handle_)); + EXPECT_EQ(0, WebRtc_InitDelayEstimator(handle_)); + EXPECT_EQ(kDifferentHistorySize, WebRtc_history_size(handle_)); + EXPECT_EQ(0, WebRtc_InitDelayEstimatorFarend(farend_handle_)); + EXPECT_EQ(kDifferentHistorySize, WebRtc_history_size(handle_)); +} + +// TODO(bjornv): Add tests for SoftReset...(...). + +} // namespace diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc new file mode 100644 index 0000000000..f907c80a35 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc @@ -0,0 +1,486 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/utility/delay_estimator_wrapper.h" + +#include +#include + +#include "modules/audio_processing/utility/delay_estimator.h" +#include "modules/audio_processing/utility/delay_estimator_internal.h" +#include "rtc_base/checks.h" + +// Only bit |kBandFirst| through bit |kBandLast| are processed and +// |kBandFirst| - |kBandLast| must be < 32. +enum { kBandFirst = 12 }; +enum { kBandLast = 43 }; + +static __inline uint32_t SetBit(uint32_t in, int pos) { + uint32_t mask = (1 << pos); + uint32_t out = (in | mask); + + return out; +} + +// Calculates the mean recursively. Same version as WebRtc_MeanEstimatorFix(), +// but for float. +// +// Inputs: +// - new_value : New additional value. +// - scale : Scale for smoothing (should be less than 1.0). +// +// Input/Output: +// - mean_value : Pointer to the mean value for updating. +// +static void MeanEstimatorFloat(float new_value, + float scale, + float* mean_value) { + RTC_DCHECK_LT(scale, 1.0f); + *mean_value += (new_value - *mean_value) * scale; +} + +// Computes the binary spectrum by comparing the input |spectrum| with a +// |threshold_spectrum|. Float and fixed point versions. +// +// Inputs: +// - spectrum : Spectrum of which the binary spectrum should be +// calculated. +// - threshold_spectrum : Threshold spectrum with which the input +// spectrum is compared. +// Return: +// - out : Binary spectrum. +// +static uint32_t BinarySpectrumFix(const uint16_t* spectrum, + SpectrumType* threshold_spectrum, + int q_domain, + int* threshold_initialized) { + int i = kBandFirst; + uint32_t out = 0; + + RTC_DCHECK_LT(q_domain, 16); + + if (!(*threshold_initialized)) { + // Set the |threshold_spectrum| to half the input |spectrum| as starting + // value. This speeds up the convergence. + for (i = kBandFirst; i <= kBandLast; i++) { + if (spectrum[i] > 0) { + // Convert input spectrum from Q(|q_domain|) to Q15. + int32_t spectrum_q15 = ((int32_t) spectrum[i]) << (15 - q_domain); + threshold_spectrum[i].int32_ = (spectrum_q15 >> 1); + *threshold_initialized = 1; + } + } + } + for (i = kBandFirst; i <= kBandLast; i++) { + // Convert input spectrum from Q(|q_domain|) to Q15. + int32_t spectrum_q15 = ((int32_t) spectrum[i]) << (15 - q_domain); + // Update the |threshold_spectrum|. + WebRtc_MeanEstimatorFix(spectrum_q15, 6, &(threshold_spectrum[i].int32_)); + // Convert |spectrum| at current frequency bin to a binary value. + if (spectrum_q15 > threshold_spectrum[i].int32_) { + out = SetBit(out, i - kBandFirst); + } + } + + return out; +} + +static uint32_t BinarySpectrumFloat(const float* spectrum, + SpectrumType* threshold_spectrum, + int* threshold_initialized) { + int i = kBandFirst; + uint32_t out = 0; + const float kScale = 1 / 64.0; + + if (!(*threshold_initialized)) { + // Set the |threshold_spectrum| to half the input |spectrum| as starting + // value. This speeds up the convergence. + for (i = kBandFirst; i <= kBandLast; i++) { + if (spectrum[i] > 0.0f) { + threshold_spectrum[i].float_ = (spectrum[i] / 2); + *threshold_initialized = 1; + } + } + } + + for (i = kBandFirst; i <= kBandLast; i++) { + // Update the |threshold_spectrum|. + MeanEstimatorFloat(spectrum[i], kScale, &(threshold_spectrum[i].float_)); + // Convert |spectrum| at current frequency bin to a binary value. + if (spectrum[i] > threshold_spectrum[i].float_) { + out = SetBit(out, i - kBandFirst); + } + } + + return out; +} + +void WebRtc_FreeDelayEstimatorFarend(void* handle) { + DelayEstimatorFarend* self = (DelayEstimatorFarend*) handle; + + if (handle == NULL) { + return; + } + + free(self->mean_far_spectrum); + self->mean_far_spectrum = NULL; + + WebRtc_FreeBinaryDelayEstimatorFarend(self->binary_farend); + self->binary_farend = NULL; + + free(self); +} + +void* WebRtc_CreateDelayEstimatorFarend(int spectrum_size, int history_size) { + DelayEstimatorFarend* self = NULL; + + // Check if the sub band used in the delay estimation is small enough to fit + // the binary spectra in a uint32_t. + static_assert(kBandLast - kBandFirst < 32, ""); + + if (spectrum_size >= kBandLast) { + self = static_cast( + malloc(sizeof(DelayEstimatorFarend))); + } + + if (self != NULL) { + int memory_fail = 0; + + // Allocate memory for the binary far-end spectrum handling. + self->binary_farend = WebRtc_CreateBinaryDelayEstimatorFarend(history_size); + memory_fail |= (self->binary_farend == NULL); + + // Allocate memory for spectrum buffers. + self->mean_far_spectrum = + static_cast(malloc(spectrum_size * sizeof(SpectrumType))); + memory_fail |= (self->mean_far_spectrum == NULL); + + self->spectrum_size = spectrum_size; + + if (memory_fail) { + WebRtc_FreeDelayEstimatorFarend(self); + self = NULL; + } + } + + return self; +} + +int WebRtc_InitDelayEstimatorFarend(void* handle) { + DelayEstimatorFarend* self = (DelayEstimatorFarend*) handle; + + if (self == NULL) { + return -1; + } + + // Initialize far-end part of binary delay estimator. + WebRtc_InitBinaryDelayEstimatorFarend(self->binary_farend); + + // Set averaged far and near end spectra to zero. + memset(self->mean_far_spectrum, 0, + sizeof(SpectrumType) * self->spectrum_size); + // Reset initialization indicators. + self->far_spectrum_initialized = 0; + + return 0; +} + +void WebRtc_SoftResetDelayEstimatorFarend(void* handle, int delay_shift) { + DelayEstimatorFarend* self = (DelayEstimatorFarend*) handle; + RTC_DCHECK(self); + WebRtc_SoftResetBinaryDelayEstimatorFarend(self->binary_farend, delay_shift); +} + +int WebRtc_AddFarSpectrumFix(void* handle, + const uint16_t* far_spectrum, + int spectrum_size, + int far_q) { + DelayEstimatorFarend* self = (DelayEstimatorFarend*) handle; + uint32_t binary_spectrum = 0; + + if (self == NULL) { + return -1; + } + if (far_spectrum == NULL) { + // Empty far end spectrum. + return -1; + } + if (spectrum_size != self->spectrum_size) { + // Data sizes don't match. + return -1; + } + if (far_q > 15) { + // If |far_q| is larger than 15 we cannot guarantee no wrap around. + return -1; + } + + // Get binary spectrum. + binary_spectrum = BinarySpectrumFix(far_spectrum, self->mean_far_spectrum, + far_q, &(self->far_spectrum_initialized)); + WebRtc_AddBinaryFarSpectrum(self->binary_farend, binary_spectrum); + + return 0; +} + +int WebRtc_AddFarSpectrumFloat(void* handle, + const float* far_spectrum, + int spectrum_size) { + DelayEstimatorFarend* self = (DelayEstimatorFarend*) handle; + uint32_t binary_spectrum = 0; + + if (self == NULL) { + return -1; + } + if (far_spectrum == NULL) { + // Empty far end spectrum. + return -1; + } + if (spectrum_size != self->spectrum_size) { + // Data sizes don't match. + return -1; + } + + // Get binary spectrum. + binary_spectrum = BinarySpectrumFloat(far_spectrum, self->mean_far_spectrum, + &(self->far_spectrum_initialized)); + WebRtc_AddBinaryFarSpectrum(self->binary_farend, binary_spectrum); + + return 0; +} + +void WebRtc_FreeDelayEstimator(void* handle) { + DelayEstimator* self = (DelayEstimator*) handle; + + if (handle == NULL) { + return; + } + + free(self->mean_near_spectrum); + self->mean_near_spectrum = NULL; + + WebRtc_FreeBinaryDelayEstimator(self->binary_handle); + self->binary_handle = NULL; + + free(self); +} + +void* WebRtc_CreateDelayEstimator(void* farend_handle, int max_lookahead) { + DelayEstimator* self = NULL; + DelayEstimatorFarend* farend = (DelayEstimatorFarend*) farend_handle; + + if (farend_handle != NULL) { + self = static_cast(malloc(sizeof(DelayEstimator))); + } + + if (self != NULL) { + int memory_fail = 0; + + // Allocate memory for the farend spectrum handling. + self->binary_handle = + WebRtc_CreateBinaryDelayEstimator(farend->binary_farend, max_lookahead); + memory_fail |= (self->binary_handle == NULL); + + // Allocate memory for spectrum buffers. + self->mean_near_spectrum = static_cast( + malloc(farend->spectrum_size * sizeof(SpectrumType))); + memory_fail |= (self->mean_near_spectrum == NULL); + + self->spectrum_size = farend->spectrum_size; + + if (memory_fail) { + WebRtc_FreeDelayEstimator(self); + self = NULL; + } + } + + return self; +} + +int WebRtc_InitDelayEstimator(void* handle) { + DelayEstimator* self = (DelayEstimator*) handle; + + if (self == NULL) { + return -1; + } + + // Initialize binary delay estimator. + WebRtc_InitBinaryDelayEstimator(self->binary_handle); + + // Set averaged far and near end spectra to zero. + memset(self->mean_near_spectrum, 0, + sizeof(SpectrumType) * self->spectrum_size); + // Reset initialization indicators. + self->near_spectrum_initialized = 0; + + return 0; +} + +int WebRtc_SoftResetDelayEstimator(void* handle, int delay_shift) { + DelayEstimator* self = (DelayEstimator*) handle; + RTC_DCHECK(self); + return WebRtc_SoftResetBinaryDelayEstimator(self->binary_handle, delay_shift); +} + +int WebRtc_set_history_size(void* handle, int history_size) { + DelayEstimator* self = static_cast(handle); + + if ((self == NULL) || (history_size <= 1)) { + return -1; + } + return WebRtc_AllocateHistoryBufferMemory(self->binary_handle, history_size); +} + +int WebRtc_history_size(const void* handle) { + const DelayEstimator* self = static_cast(handle); + + if (self == NULL) { + return -1; + } + if (self->binary_handle->farend->history_size != + self->binary_handle->history_size) { + // Non matching history sizes. + return -1; + } + return self->binary_handle->history_size; +} + +int WebRtc_set_lookahead(void* handle, int lookahead) { + DelayEstimator* self = (DelayEstimator*) handle; + RTC_DCHECK(self); + RTC_DCHECK(self->binary_handle); + if ((lookahead > self->binary_handle->near_history_size - 1) || + (lookahead < 0)) { + return -1; + } + self->binary_handle->lookahead = lookahead; + return self->binary_handle->lookahead; +} + +int WebRtc_lookahead(void* handle) { + DelayEstimator* self = (DelayEstimator*) handle; + RTC_DCHECK(self); + RTC_DCHECK(self->binary_handle); + return self->binary_handle->lookahead; +} + +int WebRtc_set_allowed_offset(void* handle, int allowed_offset) { + DelayEstimator* self = (DelayEstimator*) handle; + + if ((self == NULL) || (allowed_offset < 0)) { + return -1; + } + self->binary_handle->allowed_offset = allowed_offset; + return 0; +} + +int WebRtc_get_allowed_offset(const void* handle) { + const DelayEstimator* self = (const DelayEstimator*) handle; + + if (self == NULL) { + return -1; + } + return self->binary_handle->allowed_offset; +} + +int WebRtc_enable_robust_validation(void* handle, int enable) { + DelayEstimator* self = (DelayEstimator*) handle; + + if (self == NULL) { + return -1; + } + if ((enable < 0) || (enable > 1)) { + return -1; + } + RTC_DCHECK(self->binary_handle); + self->binary_handle->robust_validation_enabled = enable; + return 0; +} + +int WebRtc_is_robust_validation_enabled(const void* handle) { + const DelayEstimator* self = (const DelayEstimator*) handle; + + if (self == NULL) { + return -1; + } + return self->binary_handle->robust_validation_enabled; +} + +int WebRtc_DelayEstimatorProcessFix(void* handle, + const uint16_t* near_spectrum, + int spectrum_size, + int near_q) { + DelayEstimator* self = (DelayEstimator*) handle; + uint32_t binary_spectrum = 0; + + if (self == NULL) { + return -1; + } + if (near_spectrum == NULL) { + // Empty near end spectrum. + return -1; + } + if (spectrum_size != self->spectrum_size) { + // Data sizes don't match. + return -1; + } + if (near_q > 15) { + // If |near_q| is larger than 15 we cannot guarantee no wrap around. + return -1; + } + + // Get binary spectra. + binary_spectrum = BinarySpectrumFix(near_spectrum, + self->mean_near_spectrum, + near_q, + &(self->near_spectrum_initialized)); + + return WebRtc_ProcessBinarySpectrum(self->binary_handle, binary_spectrum); +} + +int WebRtc_DelayEstimatorProcessFloat(void* handle, + const float* near_spectrum, + int spectrum_size) { + DelayEstimator* self = (DelayEstimator*) handle; + uint32_t binary_spectrum = 0; + + if (self == NULL) { + return -1; + } + if (near_spectrum == NULL) { + // Empty near end spectrum. + return -1; + } + if (spectrum_size != self->spectrum_size) { + // Data sizes don't match. + return -1; + } + + // Get binary spectrum. + binary_spectrum = BinarySpectrumFloat(near_spectrum, self->mean_near_spectrum, + &(self->near_spectrum_initialized)); + + return WebRtc_ProcessBinarySpectrum(self->binary_handle, binary_spectrum); +} + +int WebRtc_last_delay(void* handle) { + DelayEstimator* self = (DelayEstimator*) handle; + + if (self == NULL) { + return -1; + } + + return WebRtc_binary_last_delay(self->binary_handle); +} + +float WebRtc_last_delay_quality(void* handle) { + DelayEstimator* self = (DelayEstimator*) handle; + RTC_DCHECK(self); + return WebRtc_binary_last_delay_quality(self->binary_handle); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h new file mode 100644 index 0000000000..6b6e51f82c --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Performs delay estimation on block by block basis. +// The return value is 0 - OK and -1 - Error, unless otherwise stated. + +#ifndef MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_WRAPPER_H_ +#define MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_WRAPPER_H_ + +#include "typedefs.h" // NOLINT(build/include) + +// Releases the memory allocated by WebRtc_CreateDelayEstimatorFarend(...) +void WebRtc_FreeDelayEstimatorFarend(void* handle); + +// Allocates the memory needed by the far-end part of the delay estimation. The +// memory needs to be initialized separately through +// WebRtc_InitDelayEstimatorFarend(...). +// +// Inputs: +// - spectrum_size : Size of the spectrum used both in far-end and +// near-end. Used to allocate memory for spectrum +// specific buffers. +// - history_size : The far-end history buffer size. A change in buffer +// size can be forced with WebRtc_set_history_size(). +// Note that the maximum delay which can be estimated is +// determined together with WebRtc_set_lookahead(). +// +// Return value: +// - void* : Created |handle|. If the memory can't be allocated or +// if any of the input parameters are invalid NULL is +// returned. +void* WebRtc_CreateDelayEstimatorFarend(int spectrum_size, int history_size); + +// Initializes the far-end part of the delay estimation instance returned by +// WebRtc_CreateDelayEstimatorFarend(...) +int WebRtc_InitDelayEstimatorFarend(void* handle); + +// Soft resets the far-end part of the delay estimation instance returned by +// WebRtc_CreateDelayEstimatorFarend(...). +// Input: +// - delay_shift : The amount of blocks to shift history buffers. +void WebRtc_SoftResetDelayEstimatorFarend(void* handle, int delay_shift); + +// Adds the far-end spectrum to the far-end history buffer. This spectrum is +// used as reference when calculating the delay using +// WebRtc_ProcessSpectrum(). +// +// Inputs: +// - far_spectrum : Far-end spectrum. +// - spectrum_size : The size of the data arrays (same for both far- and +// near-end). +// - far_q : The Q-domain of the far-end data. +// +// Output: +// - handle : Updated far-end instance. +// +int WebRtc_AddFarSpectrumFix(void* handle, + const uint16_t* far_spectrum, + int spectrum_size, + int far_q); + +// See WebRtc_AddFarSpectrumFix() for description. +int WebRtc_AddFarSpectrumFloat(void* handle, + const float* far_spectrum, + int spectrum_size); + +// Releases the memory allocated by WebRtc_CreateDelayEstimator(...) +void WebRtc_FreeDelayEstimator(void* handle); + +// Allocates the memory needed by the delay estimation. The memory needs to be +// initialized separately through WebRtc_InitDelayEstimator(...). +// +// Inputs: +// - farend_handle : Pointer to the far-end part of the delay estimation +// instance created prior to this call using +// WebRtc_CreateDelayEstimatorFarend(). +// +// Note that WebRtc_CreateDelayEstimator does not take +// ownership of |farend_handle|, which has to be torn +// down properly after this instance. +// +// - max_lookahead : Maximum amount of non-causal lookahead allowed. The +// actual amount of lookahead used can be controlled by +// WebRtc_set_lookahead(...). The default |lookahead| is +// set to |max_lookahead| at create time. Use +// WebRtc_set_lookahead(...) before start if a different +// value is desired. +// +// Using lookahead can detect cases in which a near-end +// signal occurs before the corresponding far-end signal. +// It will delay the estimate for the current block by an +// equal amount, and the returned values will be offset +// by it. +// +// A value of zero is the typical no-lookahead case. +// This also represents the minimum delay which can be +// estimated. +// +// Note that the effective range of delay estimates is +// [-|lookahead|,... ,|history_size|-|lookahead|) +// where |history_size| is set through +// WebRtc_set_history_size(). +// +// Return value: +// - void* : Created |handle|. If the memory can't be allocated or +// if any of the input parameters are invalid NULL is +// returned. +void* WebRtc_CreateDelayEstimator(void* farend_handle, int max_lookahead); + +// Initializes the delay estimation instance returned by +// WebRtc_CreateDelayEstimator(...) +int WebRtc_InitDelayEstimator(void* handle); + +// Soft resets the delay estimation instance returned by +// WebRtc_CreateDelayEstimator(...) +// Input: +// - delay_shift : The amount of blocks to shift history buffers. +// +// Return value: +// - actual_shifts : The actual number of shifts performed. +int WebRtc_SoftResetDelayEstimator(void* handle, int delay_shift); + +// Sets the effective |history_size| used. Valid values from 2. We simply need +// at least two delays to compare to perform an estimate. If |history_size| is +// changed, buffers are reallocated filling in with zeros if necessary. +// Note that changing the |history_size| affects both buffers in far-end and +// near-end. Hence it is important to change all DelayEstimators that use the +// same reference far-end, to the same |history_size| value. +// Inputs: +// - handle : Pointer to the delay estimation instance. +// - history_size : Effective history size to be used. +// Return value: +// - new_history_size : The new history size used. If the memory was not able +// to be allocated 0 is returned. +int WebRtc_set_history_size(void* handle, int history_size); + +// Returns the history_size currently used. +// Input: +// - handle : Pointer to the delay estimation instance. +int WebRtc_history_size(const void* handle); + +// Sets the amount of |lookahead| to use. Valid values are [0, max_lookahead] +// where |max_lookahead| was set at create time through +// WebRtc_CreateDelayEstimator(...). +// +// Input: +// - handle : Pointer to the delay estimation instance. +// - lookahead : The amount of lookahead to be used. +// +// Return value: +// - new_lookahead : The actual amount of lookahead set, unless |handle| is +// a NULL pointer or |lookahead| is invalid, for which an +// error is returned. +int WebRtc_set_lookahead(void* handle, int lookahead); + +// Returns the amount of lookahead we currently use. +// Input: +// - handle : Pointer to the delay estimation instance. +int WebRtc_lookahead(void* handle); + +// Sets the |allowed_offset| used in the robust validation scheme. If the +// delay estimator is used in an echo control component, this parameter is +// related to the filter length. In principle |allowed_offset| should be set to +// the echo control filter length minus the expected echo duration, i.e., the +// delay offset the echo control can handle without quality regression. The +// default value, used if not set manually, is zero. Note that |allowed_offset| +// has to be non-negative. +// Inputs: +// - handle : Pointer to the delay estimation instance. +// - allowed_offset : The amount of delay offset, measured in partitions, +// the echo control filter can handle. +int WebRtc_set_allowed_offset(void* handle, int allowed_offset); + +// Returns the |allowed_offset| in number of partitions. +int WebRtc_get_allowed_offset(const void* handle); + +// Enables/Disables a robust validation functionality in the delay estimation. +// This is by default set to disabled at create time. The state is preserved +// over a reset. +// Inputs: +// - handle : Pointer to the delay estimation instance. +// - enable : Enable (1) or disable (0) this feature. +int WebRtc_enable_robust_validation(void* handle, int enable); + +// Returns 1 if robust validation is enabled and 0 if disabled. +int WebRtc_is_robust_validation_enabled(const void* handle); + +// Estimates and returns the delay between the far-end and near-end blocks. The +// value will be offset by the lookahead (i.e. the lookahead should be +// subtracted from the returned value). +// Inputs: +// - handle : Pointer to the delay estimation instance. +// - near_spectrum : Pointer to the near-end spectrum data of the current +// block. +// - spectrum_size : The size of the data arrays (same for both far- and +// near-end). +// - near_q : The Q-domain of the near-end data. +// +// Output: +// - handle : Updated instance. +// +// Return value: +// - delay : >= 0 - Calculated delay value. +// -1 - Error. +// -2 - Insufficient data for estimation. +int WebRtc_DelayEstimatorProcessFix(void* handle, + const uint16_t* near_spectrum, + int spectrum_size, + int near_q); + +// See WebRtc_DelayEstimatorProcessFix() for description. +int WebRtc_DelayEstimatorProcessFloat(void* handle, + const float* near_spectrum, + int spectrum_size); + +// Returns the last calculated delay updated by the function +// WebRtc_DelayEstimatorProcess(...). +// +// Input: +// - handle : Pointer to the delay estimation instance. +// +// Return value: +// - delay : >= 0 - Last calculated delay value. +// -1 - Error. +// -2 - Insufficient data for estimation. +int WebRtc_last_delay(void* handle); + +// Returns the estimation quality/probability of the last calculated delay +// updated by the function WebRtc_DelayEstimatorProcess(...). The estimation +// quality is a value in the interval [0, 1]. The higher the value, the better +// the quality. +// +// Return value: +// - delay_quality : >= 0 - Estimation quality of last calculated delay. +float WebRtc_last_delay_quality(void* handle); + +#endif // MODULES_AUDIO_PROCESSING_UTILITY_DELAY_ESTIMATOR_WRAPPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.cc new file mode 100644 index 0000000000..59631117a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.cc @@ -0,0 +1,543 @@ +/* + * http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html + * Copyright Takuya OOURA, 1996-2001 + * + * You may use, copy, modify and distribute this code for any purpose (include + * commercial use) and without fee. Please refer to this package when you modify + * this code. + * + * Changes by the WebRTC authors: + * - Trivial type modifications. + * - Minimal code subset to do rdft of length 128. + * - Optimizations because of known length. + * - Removed the global variables by moving the code in to a class in order + * to make it thread safe. + * + * All changes are covered by the WebRTC license and IP grant: + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing//utility/ooura_fft.h" + +#include + +#include "modules/audio_processing/utility/ooura_fft_tables_common.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +namespace { + +#if !(defined(MIPS_FPU_LE) || defined(WEBRTC_HAS_NEON)) +static void cft1st_128_C(float* a) { + const int n = 128; + int j, k1, k2; + float wk1r, wk1i, wk2r, wk2i, wk3r, wk3i; + float x0r, x0i, x1r, x1i, x2r, x2i, x3r, x3i; + + // The processing of the first set of elements was simplified in C to avoid + // some operations (multiplication by zero or one, addition of two elements + // multiplied by the same weight, ...). + x0r = a[0] + a[2]; + x0i = a[1] + a[3]; + x1r = a[0] - a[2]; + x1i = a[1] - a[3]; + x2r = a[4] + a[6]; + x2i = a[5] + a[7]; + x3r = a[4] - a[6]; + x3i = a[5] - a[7]; + a[0] = x0r + x2r; + a[1] = x0i + x2i; + a[4] = x0r - x2r; + a[5] = x0i - x2i; + a[2] = x1r - x3i; + a[3] = x1i + x3r; + a[6] = x1r + x3i; + a[7] = x1i - x3r; + wk1r = rdft_w[2]; + x0r = a[8] + a[10]; + x0i = a[9] + a[11]; + x1r = a[8] - a[10]; + x1i = a[9] - a[11]; + x2r = a[12] + a[14]; + x2i = a[13] + a[15]; + x3r = a[12] - a[14]; + x3i = a[13] - a[15]; + a[8] = x0r + x2r; + a[9] = x0i + x2i; + a[12] = x2i - x0i; + a[13] = x0r - x2r; + x0r = x1r - x3i; + x0i = x1i + x3r; + a[10] = wk1r * (x0r - x0i); + a[11] = wk1r * (x0r + x0i); + x0r = x3i + x1r; + x0i = x3r - x1i; + a[14] = wk1r * (x0i - x0r); + a[15] = wk1r * (x0i + x0r); + k1 = 0; + for (j = 16; j < n; j += 16) { + k1 += 2; + k2 = 2 * k1; + wk2r = rdft_w[k1 + 0]; + wk2i = rdft_w[k1 + 1]; + wk1r = rdft_w[k2 + 0]; + wk1i = rdft_w[k2 + 1]; + wk3r = rdft_wk3ri_first[k1 + 0]; + wk3i = rdft_wk3ri_first[k1 + 1]; + x0r = a[j + 0] + a[j + 2]; + x0i = a[j + 1] + a[j + 3]; + x1r = a[j + 0] - a[j + 2]; + x1i = a[j + 1] - a[j + 3]; + x2r = a[j + 4] + a[j + 6]; + x2i = a[j + 5] + a[j + 7]; + x3r = a[j + 4] - a[j + 6]; + x3i = a[j + 5] - a[j + 7]; + a[j + 0] = x0r + x2r; + a[j + 1] = x0i + x2i; + x0r -= x2r; + x0i -= x2i; + a[j + 4] = wk2r * x0r - wk2i * x0i; + a[j + 5] = wk2r * x0i + wk2i * x0r; + x0r = x1r - x3i; + x0i = x1i + x3r; + a[j + 2] = wk1r * x0r - wk1i * x0i; + a[j + 3] = wk1r * x0i + wk1i * x0r; + x0r = x1r + x3i; + x0i = x1i - x3r; + a[j + 6] = wk3r * x0r - wk3i * x0i; + a[j + 7] = wk3r * x0i + wk3i * x0r; + wk1r = rdft_w[k2 + 2]; + wk1i = rdft_w[k2 + 3]; + wk3r = rdft_wk3ri_second[k1 + 0]; + wk3i = rdft_wk3ri_second[k1 + 1]; + x0r = a[j + 8] + a[j + 10]; + x0i = a[j + 9] + a[j + 11]; + x1r = a[j + 8] - a[j + 10]; + x1i = a[j + 9] - a[j + 11]; + x2r = a[j + 12] + a[j + 14]; + x2i = a[j + 13] + a[j + 15]; + x3r = a[j + 12] - a[j + 14]; + x3i = a[j + 13] - a[j + 15]; + a[j + 8] = x0r + x2r; + a[j + 9] = x0i + x2i; + x0r -= x2r; + x0i -= x2i; + a[j + 12] = -wk2i * x0r - wk2r * x0i; + a[j + 13] = -wk2i * x0i + wk2r * x0r; + x0r = x1r - x3i; + x0i = x1i + x3r; + a[j + 10] = wk1r * x0r - wk1i * x0i; + a[j + 11] = wk1r * x0i + wk1i * x0r; + x0r = x1r + x3i; + x0i = x1i - x3r; + a[j + 14] = wk3r * x0r - wk3i * x0i; + a[j + 15] = wk3r * x0i + wk3i * x0r; + } +} + +static void cftmdl_128_C(float* a) { + const int l = 8; + const int n = 128; + const int m = 32; + int j0, j1, j2, j3, k, k1, k2, m2; + float wk1r, wk1i, wk2r, wk2i, wk3r, wk3i; + float x0r, x0i, x1r, x1i, x2r, x2i, x3r, x3i; + + for (j0 = 0; j0 < l; j0 += 2) { + j1 = j0 + 8; + j2 = j0 + 16; + j3 = j0 + 24; + x0r = a[j0 + 0] + a[j1 + 0]; + x0i = a[j0 + 1] + a[j1 + 1]; + x1r = a[j0 + 0] - a[j1 + 0]; + x1i = a[j0 + 1] - a[j1 + 1]; + x2r = a[j2 + 0] + a[j3 + 0]; + x2i = a[j2 + 1] + a[j3 + 1]; + x3r = a[j2 + 0] - a[j3 + 0]; + x3i = a[j2 + 1] - a[j3 + 1]; + a[j0 + 0] = x0r + x2r; + a[j0 + 1] = x0i + x2i; + a[j2 + 0] = x0r - x2r; + a[j2 + 1] = x0i - x2i; + a[j1 + 0] = x1r - x3i; + a[j1 + 1] = x1i + x3r; + a[j3 + 0] = x1r + x3i; + a[j3 + 1] = x1i - x3r; + } + wk1r = rdft_w[2]; + for (j0 = m; j0 < l + m; j0 += 2) { + j1 = j0 + 8; + j2 = j0 + 16; + j3 = j0 + 24; + x0r = a[j0 + 0] + a[j1 + 0]; + x0i = a[j0 + 1] + a[j1 + 1]; + x1r = a[j0 + 0] - a[j1 + 0]; + x1i = a[j0 + 1] - a[j1 + 1]; + x2r = a[j2 + 0] + a[j3 + 0]; + x2i = a[j2 + 1] + a[j3 + 1]; + x3r = a[j2 + 0] - a[j3 + 0]; + x3i = a[j2 + 1] - a[j3 + 1]; + a[j0 + 0] = x0r + x2r; + a[j0 + 1] = x0i + x2i; + a[j2 + 0] = x2i - x0i; + a[j2 + 1] = x0r - x2r; + x0r = x1r - x3i; + x0i = x1i + x3r; + a[j1 + 0] = wk1r * (x0r - x0i); + a[j1 + 1] = wk1r * (x0r + x0i); + x0r = x3i + x1r; + x0i = x3r - x1i; + a[j3 + 0] = wk1r * (x0i - x0r); + a[j3 + 1] = wk1r * (x0i + x0r); + } + k1 = 0; + m2 = 2 * m; + for (k = m2; k < n; k += m2) { + k1 += 2; + k2 = 2 * k1; + wk2r = rdft_w[k1 + 0]; + wk2i = rdft_w[k1 + 1]; + wk1r = rdft_w[k2 + 0]; + wk1i = rdft_w[k2 + 1]; + wk3r = rdft_wk3ri_first[k1 + 0]; + wk3i = rdft_wk3ri_first[k1 + 1]; + for (j0 = k; j0 < l + k; j0 += 2) { + j1 = j0 + 8; + j2 = j0 + 16; + j3 = j0 + 24; + x0r = a[j0 + 0] + a[j1 + 0]; + x0i = a[j0 + 1] + a[j1 + 1]; + x1r = a[j0 + 0] - a[j1 + 0]; + x1i = a[j0 + 1] - a[j1 + 1]; + x2r = a[j2 + 0] + a[j3 + 0]; + x2i = a[j2 + 1] + a[j3 + 1]; + x3r = a[j2 + 0] - a[j3 + 0]; + x3i = a[j2 + 1] - a[j3 + 1]; + a[j0 + 0] = x0r + x2r; + a[j0 + 1] = x0i + x2i; + x0r -= x2r; + x0i -= x2i; + a[j2 + 0] = wk2r * x0r - wk2i * x0i; + a[j2 + 1] = wk2r * x0i + wk2i * x0r; + x0r = x1r - x3i; + x0i = x1i + x3r; + a[j1 + 0] = wk1r * x0r - wk1i * x0i; + a[j1 + 1] = wk1r * x0i + wk1i * x0r; + x0r = x1r + x3i; + x0i = x1i - x3r; + a[j3 + 0] = wk3r * x0r - wk3i * x0i; + a[j3 + 1] = wk3r * x0i + wk3i * x0r; + } + wk1r = rdft_w[k2 + 2]; + wk1i = rdft_w[k2 + 3]; + wk3r = rdft_wk3ri_second[k1 + 0]; + wk3i = rdft_wk3ri_second[k1 + 1]; + for (j0 = k + m; j0 < l + (k + m); j0 += 2) { + j1 = j0 + 8; + j2 = j0 + 16; + j3 = j0 + 24; + x0r = a[j0 + 0] + a[j1 + 0]; + x0i = a[j0 + 1] + a[j1 + 1]; + x1r = a[j0 + 0] - a[j1 + 0]; + x1i = a[j0 + 1] - a[j1 + 1]; + x2r = a[j2 + 0] + a[j3 + 0]; + x2i = a[j2 + 1] + a[j3 + 1]; + x3r = a[j2 + 0] - a[j3 + 0]; + x3i = a[j2 + 1] - a[j3 + 1]; + a[j0 + 0] = x0r + x2r; + a[j0 + 1] = x0i + x2i; + x0r -= x2r; + x0i -= x2i; + a[j2 + 0] = -wk2i * x0r - wk2r * x0i; + a[j2 + 1] = -wk2i * x0i + wk2r * x0r; + x0r = x1r - x3i; + x0i = x1i + x3r; + a[j1 + 0] = wk1r * x0r - wk1i * x0i; + a[j1 + 1] = wk1r * x0i + wk1i * x0r; + x0r = x1r + x3i; + x0i = x1i - x3r; + a[j3 + 0] = wk3r * x0r - wk3i * x0i; + a[j3 + 1] = wk3r * x0i + wk3i * x0r; + } + } +} + +static void rftfsub_128_C(float* a) { + const float* c = rdft_w + 32; + int j1, j2, k1, k2; + float wkr, wki, xr, xi, yr, yi; + + for (j1 = 1, j2 = 2; j2 < 64; j1 += 1, j2 += 2) { + k2 = 128 - j2; + k1 = 32 - j1; + wkr = 0.5f - c[k1]; + wki = c[j1]; + xr = a[j2 + 0] - a[k2 + 0]; + xi = a[j2 + 1] + a[k2 + 1]; + yr = wkr * xr - wki * xi; + yi = wkr * xi + wki * xr; + a[j2 + 0] -= yr; + a[j2 + 1] -= yi; + a[k2 + 0] += yr; + a[k2 + 1] -= yi; + } +} + +static void rftbsub_128_C(float* a) { + const float* c = rdft_w + 32; + int j1, j2, k1, k2; + float wkr, wki, xr, xi, yr, yi; + + a[1] = -a[1]; + for (j1 = 1, j2 = 2; j2 < 64; j1 += 1, j2 += 2) { + k2 = 128 - j2; + k1 = 32 - j1; + wkr = 0.5f - c[k1]; + wki = c[j1]; + xr = a[j2 + 0] - a[k2 + 0]; + xi = a[j2 + 1] + a[k2 + 1]; + yr = wkr * xr + wki * xi; + yi = wkr * xi - wki * xr; + a[j2 + 0] = a[j2 + 0] - yr; + a[j2 + 1] = yi - a[j2 + 1]; + a[k2 + 0] = yr + a[k2 + 0]; + a[k2 + 1] = yi - a[k2 + 1]; + } + a[65] = -a[65]; +} +#endif + + +} // namespace + +OouraFft::OouraFft() { +#if defined(WEBRTC_ARCH_X86_FAMILY) + use_sse2_ = (WebRtc_GetCPUInfo(kSSE2) != 0); +#else + use_sse2_ = false; +#endif +} + +OouraFft::~OouraFft() = default; + +void OouraFft::Fft(float* a) const { + float xi; + bitrv2_128(a); + cftfsub_128(a); + rftfsub_128(a); + xi = a[0] - a[1]; + a[0] += a[1]; + a[1] = xi; +} +void OouraFft::InverseFft(float* a) const { + a[1] = 0.5f * (a[0] - a[1]); + a[0] -= a[1]; + rftbsub_128(a); + bitrv2_128(a); + cftbsub_128(a); +} + +void OouraFft::cft1st_128(float* a) const { +#if defined(MIPS_FPU_LE) + cft1st_128_mips(a); +#elif defined(WEBRTC_HAS_NEON) + cft1st_128_neon(a); +#elif defined(WEBRTC_ARCH_X86_FAMILY) + if (use_sse2_) { + cft1st_128_SSE2(a); + } else { + cft1st_128_C(a); + } +#else + cft1st_128_C(a); +#endif +} +void OouraFft::cftmdl_128(float* a) const { +#if defined(MIPS_FPU_LE) + cftmdl_128_mips(a); +#elif defined(WEBRTC_HAS_NEON) + cftmdl_128_neon(a); +#elif defined(WEBRTC_ARCH_X86_FAMILY) + if (use_sse2_) { + cftmdl_128_SSE2(a); + } else { + cftmdl_128_C(a); + } +#else + cftmdl_128_C(a); +#endif +} +void OouraFft::rftfsub_128(float* a) const { +#if defined(MIPS_FPU_LE) + rftfsub_128_mips(a); +#elif defined(WEBRTC_HAS_NEON) + rftfsub_128_neon(a); +#elif defined(WEBRTC_ARCH_X86_FAMILY) + if (use_sse2_) { + rftfsub_128_SSE2(a); + } else { + rftfsub_128_C(a); + } +#else + rftfsub_128_C(a); +#endif +} + +void OouraFft::rftbsub_128(float* a) const { +#if defined(MIPS_FPU_LE) + rftbsub_128_mips(a); +#elif defined(WEBRTC_HAS_NEON) + rftbsub_128_neon(a); +#elif defined(WEBRTC_ARCH_X86_FAMILY) + if (use_sse2_) { + rftbsub_128_SSE2(a); + } else { + rftbsub_128_C(a); + } +#else + rftbsub_128_C(a); +#endif +} + +void OouraFft::cftbsub_128(float* a) const { + int j, j1, j2, j3, l; + float x0r, x0i, x1r, x1i, x2r, x2i, x3r, x3i; + + cft1st_128(a); + cftmdl_128(a); + l = 32; + + for (j = 0; j < l; j += 2) { + j1 = j + l; + j2 = j1 + l; + j3 = j2 + l; + x0r = a[j] + a[j1]; + x0i = -a[j + 1] - a[j1 + 1]; + x1r = a[j] - a[j1]; + x1i = -a[j + 1] + a[j1 + 1]; + x2r = a[j2] + a[j3]; + x2i = a[j2 + 1] + a[j3 + 1]; + x3r = a[j2] - a[j3]; + x3i = a[j2 + 1] - a[j3 + 1]; + a[j] = x0r + x2r; + a[j + 1] = x0i - x2i; + a[j2] = x0r - x2r; + a[j2 + 1] = x0i + x2i; + a[j1] = x1r - x3i; + a[j1 + 1] = x1i - x3r; + a[j3] = x1r + x3i; + a[j3 + 1] = x1i + x3r; + } +} + +void OouraFft::cftfsub_128(float* a) const { + int j, j1, j2, j3, l; + float x0r, x0i, x1r, x1i, x2r, x2i, x3r, x3i; + + cft1st_128(a); + cftmdl_128(a); + l = 32; + for (j = 0; j < l; j += 2) { + j1 = j + l; + j2 = j1 + l; + j3 = j2 + l; + x0r = a[j] + a[j1]; + x0i = a[j + 1] + a[j1 + 1]; + x1r = a[j] - a[j1]; + x1i = a[j + 1] - a[j1 + 1]; + x2r = a[j2] + a[j3]; + x2i = a[j2 + 1] + a[j3 + 1]; + x3r = a[j2] - a[j3]; + x3i = a[j2 + 1] - a[j3 + 1]; + a[j] = x0r + x2r; + a[j + 1] = x0i + x2i; + a[j2] = x0r - x2r; + a[j2 + 1] = x0i - x2i; + a[j1] = x1r - x3i; + a[j1 + 1] = x1i + x3r; + a[j3] = x1r + x3i; + a[j3 + 1] = x1i - x3r; + } +} + +void OouraFft::bitrv2_128(float* a) const { + /* + Following things have been attempted but are no faster: + (a) Storing the swap indexes in a LUT (index calculations are done + for 'free' while waiting on memory/L1). + (b) Consolidate the load/store of two consecutive floats by a 64 bit + integer (execution is memory/L1 bound). + (c) Do a mix of floats and 64 bit integer to maximize register + utilization (execution is memory/L1 bound). + (d) Replacing ip[i] by ((k<<31)>>25) + ((k >> 1)<<5). + (e) Hard-coding of the offsets to completely eliminates index + calculations. + */ + + unsigned int j, j1, k, k1; + float xr, xi, yr, yi; + + const int ip[4] = {0, 64, 32, 96}; + for (k = 0; k < 4; k++) { + for (j = 0; j < k; j++) { + j1 = 2 * j + ip[k]; + k1 = 2 * k + ip[j]; + xr = a[j1 + 0]; + xi = a[j1 + 1]; + yr = a[k1 + 0]; + yi = a[k1 + 1]; + a[j1 + 0] = yr; + a[j1 + 1] = yi; + a[k1 + 0] = xr; + a[k1 + 1] = xi; + j1 += 8; + k1 += 16; + xr = a[j1 + 0]; + xi = a[j1 + 1]; + yr = a[k1 + 0]; + yi = a[k1 + 1]; + a[j1 + 0] = yr; + a[j1 + 1] = yi; + a[k1 + 0] = xr; + a[k1 + 1] = xi; + j1 += 8; + k1 -= 8; + xr = a[j1 + 0]; + xi = a[j1 + 1]; + yr = a[k1 + 0]; + yi = a[k1 + 1]; + a[j1 + 0] = yr; + a[j1 + 1] = yi; + a[k1 + 0] = xr; + a[k1 + 1] = xi; + j1 += 8; + k1 += 16; + xr = a[j1 + 0]; + xi = a[j1 + 1]; + yr = a[k1 + 0]; + yi = a[k1 + 1]; + a[j1 + 0] = yr; + a[j1 + 1] = yi; + a[k1 + 0] = xr; + a[k1 + 1] = xi; + } + j1 = 2 * k + 8 + ip[k]; + k1 = j1 + 8; + xr = a[j1 + 0]; + xi = a[j1 + 1]; + yr = a[k1 + 0]; + yi = a[k1 + 1]; + a[j1 + 0] = yr; + a[j1 + 1] = yi; + a[k1 + 0] = xr; + a[k1 + 1] = xi; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.h b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.h new file mode 100644 index 0000000000..96d57dc908 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_H_ +#define MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_H_ + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#if defined(WEBRTC_ARCH_X86_FAMILY) +void cft1st_128_SSE2(float* a); +void cftmdl_128_SSE2(float* a); +void rftfsub_128_SSE2(float* a); +void rftbsub_128_SSE2(float* a); +#endif + +#if defined(MIPS_FPU_LE) +void cft1st_128_mips(float* a); +void cftmdl_128_mips(float* a); +void rftfsub_128_mips(float* a); +void rftbsub_128_mips(float* a); +#endif + +#if defined(WEBRTC_HAS_NEON) +void cft1st_128_neon(float* a); +void cftmdl_128_neon(float* a); +void rftfsub_128_neon(float* a); +void rftbsub_128_neon(float* a); +#endif + +class OouraFft { + public: + OouraFft(); + ~OouraFft(); + void Fft(float* a) const; + void InverseFft(float* a) const; + + private: + void cft1st_128(float* a) const; + void cftmdl_128(float* a) const; + void rftfsub_128(float* a) const; + void rftbsub_128(float* a) const; + + void cftfsub_128(float* a) const; + void cftbsub_128(float* a) const; + void bitrv2_128(float* a) const; + bool use_sse2_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_mips.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_mips.cc new file mode 100644 index 0000000000..569e1d7e82 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_mips.cc @@ -0,0 +1,1185 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/utility/ooura_fft.h" + +#include "modules/audio_processing/utility/ooura_fft_tables_common.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +#if defined(MIPS_FPU_LE) +void bitrv2_128_mips(float* a) { + // n is 128 + float xr, xi, yr, yi; + + xr = a[8]; + xi = a[9]; + yr = a[16]; + yi = a[17]; + a[8] = yr; + a[9] = yi; + a[16] = xr; + a[17] = xi; + + xr = a[64]; + xi = a[65]; + yr = a[2]; + yi = a[3]; + a[64] = yr; + a[65] = yi; + a[2] = xr; + a[3] = xi; + + xr = a[72]; + xi = a[73]; + yr = a[18]; + yi = a[19]; + a[72] = yr; + a[73] = yi; + a[18] = xr; + a[19] = xi; + + xr = a[80]; + xi = a[81]; + yr = a[10]; + yi = a[11]; + a[80] = yr; + a[81] = yi; + a[10] = xr; + a[11] = xi; + + xr = a[88]; + xi = a[89]; + yr = a[26]; + yi = a[27]; + a[88] = yr; + a[89] = yi; + a[26] = xr; + a[27] = xi; + + xr = a[74]; + xi = a[75]; + yr = a[82]; + yi = a[83]; + a[74] = yr; + a[75] = yi; + a[82] = xr; + a[83] = xi; + + xr = a[32]; + xi = a[33]; + yr = a[4]; + yi = a[5]; + a[32] = yr; + a[33] = yi; + a[4] = xr; + a[5] = xi; + + xr = a[40]; + xi = a[41]; + yr = a[20]; + yi = a[21]; + a[40] = yr; + a[41] = yi; + a[20] = xr; + a[21] = xi; + + xr = a[48]; + xi = a[49]; + yr = a[12]; + yi = a[13]; + a[48] = yr; + a[49] = yi; + a[12] = xr; + a[13] = xi; + + xr = a[56]; + xi = a[57]; + yr = a[28]; + yi = a[29]; + a[56] = yr; + a[57] = yi; + a[28] = xr; + a[29] = xi; + + xr = a[34]; + xi = a[35]; + yr = a[68]; + yi = a[69]; + a[34] = yr; + a[35] = yi; + a[68] = xr; + a[69] = xi; + + xr = a[42]; + xi = a[43]; + yr = a[84]; + yi = a[85]; + a[42] = yr; + a[43] = yi; + a[84] = xr; + a[85] = xi; + + xr = a[50]; + xi = a[51]; + yr = a[76]; + yi = a[77]; + a[50] = yr; + a[51] = yi; + a[76] = xr; + a[77] = xi; + + xr = a[58]; + xi = a[59]; + yr = a[92]; + yi = a[93]; + a[58] = yr; + a[59] = yi; + a[92] = xr; + a[93] = xi; + + xr = a[44]; + xi = a[45]; + yr = a[52]; + yi = a[53]; + a[44] = yr; + a[45] = yi; + a[52] = xr; + a[53] = xi; + + xr = a[96]; + xi = a[97]; + yr = a[6]; + yi = a[7]; + a[96] = yr; + a[97] = yi; + a[6] = xr; + a[7] = xi; + + xr = a[104]; + xi = a[105]; + yr = a[22]; + yi = a[23]; + a[104] = yr; + a[105] = yi; + a[22] = xr; + a[23] = xi; + + xr = a[112]; + xi = a[113]; + yr = a[14]; + yi = a[15]; + a[112] = yr; + a[113] = yi; + a[14] = xr; + a[15] = xi; + + xr = a[120]; + xi = a[121]; + yr = a[30]; + yi = a[31]; + a[120] = yr; + a[121] = yi; + a[30] = xr; + a[31] = xi; + + xr = a[98]; + xi = a[99]; + yr = a[70]; + yi = a[71]; + a[98] = yr; + a[99] = yi; + a[70] = xr; + a[71] = xi; + + xr = a[106]; + xi = a[107]; + yr = a[86]; + yi = a[87]; + a[106] = yr; + a[107] = yi; + a[86] = xr; + a[87] = xi; + + xr = a[114]; + xi = a[115]; + yr = a[78]; + yi = a[79]; + a[114] = yr; + a[115] = yi; + a[78] = xr; + a[79] = xi; + + xr = a[122]; + xi = a[123]; + yr = a[94]; + yi = a[95]; + a[122] = yr; + a[123] = yi; + a[94] = xr; + a[95] = xi; + + xr = a[100]; + xi = a[101]; + yr = a[38]; + yi = a[39]; + a[100] = yr; + a[101] = yi; + a[38] = xr; + a[39] = xi; + + xr = a[108]; + xi = a[109]; + yr = a[54]; + yi = a[55]; + a[108] = yr; + a[109] = yi; + a[54] = xr; + a[55] = xi; + + xr = a[116]; + xi = a[117]; + yr = a[46]; + yi = a[47]; + a[116] = yr; + a[117] = yi; + a[46] = xr; + a[47] = xi; + + xr = a[124]; + xi = a[125]; + yr = a[62]; + yi = a[63]; + a[124] = yr; + a[125] = yi; + a[62] = xr; + a[63] = xi; + + xr = a[110]; + xi = a[111]; + yr = a[118]; + yi = a[119]; + a[110] = yr; + a[111] = yi; + a[118] = xr; + a[119] = xi; +} + +void cft1st_128_mips(float* a) { + float f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14; + int a_ptr, p1_rdft, p2_rdft, count; + const float* first = rdft_wk3ri_first; + const float* second = rdft_wk3ri_second; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + // first 8 + "lwc1 %[f0], 0(%[a]) \n\t" + "lwc1 %[f1], 4(%[a]) \n\t" + "lwc1 %[f2], 8(%[a]) \n\t" + "lwc1 %[f3], 12(%[a]) \n\t" + "lwc1 %[f4], 16(%[a]) \n\t" + "lwc1 %[f5], 20(%[a]) \n\t" + "lwc1 %[f6], 24(%[a]) \n\t" + "lwc1 %[f7], 28(%[a]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "add.s %[f7], %[f8], %[f2] \n\t" + "sub.s %[f8], %[f8], %[f2] \n\t" + "sub.s %[f2], %[f1], %[f4] \n\t" + "add.s %[f1], %[f1], %[f4] \n\t" + "add.s %[f4], %[f6], %[f3] \n\t" + "sub.s %[f6], %[f6], %[f3] \n\t" + "sub.s %[f3], %[f0], %[f5] \n\t" + "add.s %[f0], %[f0], %[f5] \n\t" + "swc1 %[f7], 0(%[a]) \n\t" + "swc1 %[f8], 16(%[a]) \n\t" + "swc1 %[f2], 28(%[a]) \n\t" + "swc1 %[f1], 12(%[a]) \n\t" + "swc1 %[f4], 4(%[a]) \n\t" + "swc1 %[f6], 20(%[a]) \n\t" + "swc1 %[f3], 8(%[a]) \n\t" + "swc1 %[f0], 24(%[a]) \n\t" + // second 8 + "lwc1 %[f0], 32(%[a]) \n\t" + "lwc1 %[f1], 36(%[a]) \n\t" + "lwc1 %[f2], 40(%[a]) \n\t" + "lwc1 %[f3], 44(%[a]) \n\t" + "lwc1 %[f4], 48(%[a]) \n\t" + "lwc1 %[f5], 52(%[a]) \n\t" + "lwc1 %[f6], 56(%[a]) \n\t" + "lwc1 %[f7], 60(%[a]) \n\t" + "add.s %[f8], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "add.s %[f7], %[f4], %[f1] \n\t" + "sub.s %[f4], %[f4], %[f1] \n\t" + "add.s %[f1], %[f3], %[f8] \n\t" + "sub.s %[f3], %[f3], %[f8] \n\t" + "sub.s %[f8], %[f0], %[f5] \n\t" + "add.s %[f0], %[f0], %[f5] \n\t" + "add.s %[f5], %[f6], %[f2] \n\t" + "sub.s %[f6], %[f2], %[f6] \n\t" + "lwc1 %[f9], 8(%[rdft_w]) \n\t" + "sub.s %[f2], %[f8], %[f7] \n\t" + "add.s %[f8], %[f8], %[f7] \n\t" + "sub.s %[f7], %[f4], %[f0] \n\t" + "add.s %[f4], %[f4], %[f0] \n\t" + // prepare for loop + "addiu %[a_ptr], %[a], 64 \n\t" + "addiu %[p1_rdft], %[rdft_w], 8 \n\t" + "addiu %[p2_rdft], %[rdft_w], 16 \n\t" + "addiu %[count], $zero, 7 \n\t" + // finish second 8 + "mul.s %[f2], %[f9], %[f2] \n\t" + "mul.s %[f8], %[f9], %[f8] \n\t" + "mul.s %[f7], %[f9], %[f7] \n\t" + "mul.s %[f4], %[f9], %[f4] \n\t" + "swc1 %[f1], 32(%[a]) \n\t" + "swc1 %[f3], 52(%[a]) \n\t" + "swc1 %[f5], 36(%[a]) \n\t" + "swc1 %[f6], 48(%[a]) \n\t" + "swc1 %[f2], 40(%[a]) \n\t" + "swc1 %[f8], 44(%[a]) \n\t" + "swc1 %[f7], 56(%[a]) \n\t" + "swc1 %[f4], 60(%[a]) \n\t" + // loop + "1: \n\t" + "lwc1 %[f0], 0(%[a_ptr]) \n\t" + "lwc1 %[f1], 4(%[a_ptr]) \n\t" + "lwc1 %[f2], 8(%[a_ptr]) \n\t" + "lwc1 %[f3], 12(%[a_ptr]) \n\t" + "lwc1 %[f4], 16(%[a_ptr]) \n\t" + "lwc1 %[f5], 20(%[a_ptr]) \n\t" + "lwc1 %[f6], 24(%[a_ptr]) \n\t" + "lwc1 %[f7], 28(%[a_ptr]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "lwc1 %[f10], 4(%[p1_rdft]) \n\t" + "lwc1 %[f11], 0(%[p2_rdft]) \n\t" + "lwc1 %[f12], 4(%[p2_rdft]) \n\t" + "lwc1 %[f13], 8(%[first]) \n\t" + "lwc1 %[f14], 12(%[first]) \n\t" + "add.s %[f7], %[f8], %[f2] \n\t" + "sub.s %[f8], %[f8], %[f2] \n\t" + "add.s %[f2], %[f6], %[f3] \n\t" + "sub.s %[f6], %[f6], %[f3] \n\t" + "add.s %[f3], %[f0], %[f5] \n\t" + "sub.s %[f0], %[f0], %[f5] \n\t" + "add.s %[f5], %[f1], %[f4] \n\t" + "sub.s %[f1], %[f1], %[f4] \n\t" + "swc1 %[f7], 0(%[a_ptr]) \n\t" + "swc1 %[f2], 4(%[a_ptr]) \n\t" + "mul.s %[f4], %[f9], %[f8] \n\t" +#if defined(MIPS32_R2_LE) + "mul.s %[f8], %[f10], %[f8] \n\t" + "mul.s %[f7], %[f11], %[f0] \n\t" + "mul.s %[f0], %[f12], %[f0] \n\t" + "mul.s %[f2], %[f13], %[f3] \n\t" + "mul.s %[f3], %[f14], %[f3] \n\t" + "nmsub.s %[f4], %[f4], %[f10], %[f6] \n\t" + "madd.s %[f8], %[f8], %[f9], %[f6] \n\t" + "nmsub.s %[f7], %[f7], %[f12], %[f5] \n\t" + "madd.s %[f0], %[f0], %[f11], %[f5] \n\t" + "nmsub.s %[f2], %[f2], %[f14], %[f1] \n\t" + "madd.s %[f3], %[f3], %[f13], %[f1] \n\t" +#else + "mul.s %[f7], %[f10], %[f6] \n\t" + "mul.s %[f6], %[f9], %[f6] \n\t" + "mul.s %[f8], %[f10], %[f8] \n\t" + "mul.s %[f2], %[f11], %[f0] \n\t" + "mul.s %[f11], %[f11], %[f5] \n\t" + "mul.s %[f5], %[f12], %[f5] \n\t" + "mul.s %[f0], %[f12], %[f0] \n\t" + "mul.s %[f12], %[f13], %[f3] \n\t" + "mul.s %[f13], %[f13], %[f1] \n\t" + "mul.s %[f1], %[f14], %[f1] \n\t" + "mul.s %[f3], %[f14], %[f3] \n\t" + "sub.s %[f4], %[f4], %[f7] \n\t" + "add.s %[f8], %[f6], %[f8] \n\t" + "sub.s %[f7], %[f2], %[f5] \n\t" + "add.s %[f0], %[f11], %[f0] \n\t" + "sub.s %[f2], %[f12], %[f1] \n\t" + "add.s %[f3], %[f13], %[f3] \n\t" +#endif + "swc1 %[f4], 16(%[a_ptr]) \n\t" + "swc1 %[f8], 20(%[a_ptr]) \n\t" + "swc1 %[f7], 8(%[a_ptr]) \n\t" + "swc1 %[f0], 12(%[a_ptr]) \n\t" + "swc1 %[f2], 24(%[a_ptr]) \n\t" + "swc1 %[f3], 28(%[a_ptr]) \n\t" + "lwc1 %[f0], 32(%[a_ptr]) \n\t" + "lwc1 %[f1], 36(%[a_ptr]) \n\t" + "lwc1 %[f2], 40(%[a_ptr]) \n\t" + "lwc1 %[f3], 44(%[a_ptr]) \n\t" + "lwc1 %[f4], 48(%[a_ptr]) \n\t" + "lwc1 %[f5], 52(%[a_ptr]) \n\t" + "lwc1 %[f6], 56(%[a_ptr]) \n\t" + "lwc1 %[f7], 60(%[a_ptr]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "lwc1 %[f11], 8(%[p2_rdft]) \n\t" + "lwc1 %[f12], 12(%[p2_rdft]) \n\t" + "lwc1 %[f13], 8(%[second]) \n\t" + "lwc1 %[f14], 12(%[second]) \n\t" + "add.s %[f7], %[f8], %[f2] \n\t" + "sub.s %[f8], %[f2], %[f8] \n\t" + "add.s %[f2], %[f6], %[f3] \n\t" + "sub.s %[f6], %[f3], %[f6] \n\t" + "add.s %[f3], %[f0], %[f5] \n\t" + "sub.s %[f0], %[f0], %[f5] \n\t" + "add.s %[f5], %[f1], %[f4] \n\t" + "sub.s %[f1], %[f1], %[f4] \n\t" + "swc1 %[f7], 32(%[a_ptr]) \n\t" + "swc1 %[f2], 36(%[a_ptr]) \n\t" + "mul.s %[f4], %[f10], %[f8] \n\t" +#if defined(MIPS32_R2_LE) + "mul.s %[f10], %[f10], %[f6] \n\t" + "mul.s %[f7], %[f11], %[f0] \n\t" + "mul.s %[f11], %[f11], %[f5] \n\t" + "mul.s %[f2], %[f13], %[f3] \n\t" + "mul.s %[f13], %[f13], %[f1] \n\t" + "madd.s %[f4], %[f4], %[f9], %[f6] \n\t" + "nmsub.s %[f10], %[f10], %[f9], %[f8] \n\t" + "nmsub.s %[f7], %[f7], %[f12], %[f5] \n\t" + "madd.s %[f11], %[f11], %[f12], %[f0] \n\t" + "nmsub.s %[f2], %[f2], %[f14], %[f1] \n\t" + "madd.s %[f13], %[f13], %[f14], %[f3] \n\t" +#else + "mul.s %[f2], %[f9], %[f6] \n\t" + "mul.s %[f10], %[f10], %[f6] \n\t" + "mul.s %[f9], %[f9], %[f8] \n\t" + "mul.s %[f7], %[f11], %[f0] \n\t" + "mul.s %[f8], %[f12], %[f5] \n\t" + "mul.s %[f11], %[f11], %[f5] \n\t" + "mul.s %[f12], %[f12], %[f0] \n\t" + "mul.s %[f5], %[f13], %[f3] \n\t" + "mul.s %[f0], %[f14], %[f1] \n\t" + "mul.s %[f13], %[f13], %[f1] \n\t" + "mul.s %[f14], %[f14], %[f3] \n\t" + "add.s %[f4], %[f4], %[f2] \n\t" + "sub.s %[f10], %[f10], %[f9] \n\t" + "sub.s %[f7], %[f7], %[f8] \n\t" + "add.s %[f11], %[f11], %[f12] \n\t" + "sub.s %[f2], %[f5], %[f0] \n\t" + "add.s %[f13], %[f13], %[f14] \n\t" +#endif + "swc1 %[f4], 48(%[a_ptr]) \n\t" + "swc1 %[f10], 52(%[a_ptr]) \n\t" + "swc1 %[f7], 40(%[a_ptr]) \n\t" + "swc1 %[f11], 44(%[a_ptr]) \n\t" + "swc1 %[f2], 56(%[a_ptr]) \n\t" + "swc1 %[f13], 60(%[a_ptr]) \n\t" + "addiu %[count], %[count], -1 \n\t" + "lwc1 %[f9], 8(%[p1_rdft]) \n\t" + "addiu %[a_ptr], %[a_ptr], 64 \n\t" + "addiu %[p1_rdft], %[p1_rdft], 8 \n\t" + "addiu %[p2_rdft], %[p2_rdft], 16 \n\t" + "addiu %[first], %[first], 8 \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[second], %[second], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), + [f4] "=&f" (f4), [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), + [f8] "=&f" (f8), [f9] "=&f" (f9), [f10] "=&f" (f10), [f11] "=&f" (f11), + [f12] "=&f" (f12), [f13] "=&f" (f13), [f14] "=&f" (f14), + [a_ptr] "=&r" (a_ptr), [p1_rdft] "=&r" (p1_rdft), [first] "+r" (first), + [p2_rdft] "=&r" (p2_rdft), [count] "=&r" (count), [second] "+r" (second) + : [a] "r" (a), [rdft_w] "r" (rdft_w) + : "memory" + ); +} + +void cftmdl_128_mips(float* a) { + float f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14; + int tmp_a, count; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[tmp_a], %[a], 0 \n\t" + "addiu %[count], $zero, 4 \n\t" + "1: \n\t" + "addiu %[count], %[count], -1 \n\t" + "lwc1 %[f0], 0(%[tmp_a]) \n\t" + "lwc1 %[f2], 32(%[tmp_a]) \n\t" + "lwc1 %[f4], 64(%[tmp_a]) \n\t" + "lwc1 %[f6], 96(%[tmp_a]) \n\t" + "lwc1 %[f1], 4(%[tmp_a]) \n\t" + "lwc1 %[f3], 36(%[tmp_a]) \n\t" + "lwc1 %[f5], 68(%[tmp_a]) \n\t" + "lwc1 %[f7], 100(%[tmp_a]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "add.s %[f7], %[f8], %[f2] \n\t" + "sub.s %[f8], %[f8], %[f2] \n\t" + "add.s %[f2], %[f1], %[f4] \n\t" + "sub.s %[f1], %[f1], %[f4] \n\t" + "add.s %[f4], %[f6], %[f3] \n\t" + "sub.s %[f6], %[f6], %[f3] \n\t" + "sub.s %[f3], %[f0], %[f5] \n\t" + "add.s %[f0], %[f0], %[f5] \n\t" + "swc1 %[f7], 0(%[tmp_a]) \n\t" + "swc1 %[f8], 64(%[tmp_a]) \n\t" + "swc1 %[f2], 36(%[tmp_a]) \n\t" + "swc1 %[f1], 100(%[tmp_a]) \n\t" + "swc1 %[f4], 4(%[tmp_a]) \n\t" + "swc1 %[f6], 68(%[tmp_a]) \n\t" + "swc1 %[f3], 32(%[tmp_a]) \n\t" + "swc1 %[f0], 96(%[tmp_a]) \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[tmp_a], %[tmp_a], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), + [f4] "=&f" (f4), [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), + [f8] "=&f" (f8), [tmp_a] "=&r" (tmp_a), [count] "=&r" (count) + : [a] "r" (a) + : "memory" + ); + f9 = rdft_w[2]; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[tmp_a], %[a], 128 \n\t" + "addiu %[count], $zero, 4 \n\t" + "1: \n\t" + "addiu %[count], %[count], -1 \n\t" + "lwc1 %[f0], 0(%[tmp_a]) \n\t" + "lwc1 %[f2], 32(%[tmp_a]) \n\t" + "lwc1 %[f5], 68(%[tmp_a]) \n\t" + "lwc1 %[f7], 100(%[tmp_a]) \n\t" + "lwc1 %[f1], 4(%[tmp_a]) \n\t" + "lwc1 %[f3], 36(%[tmp_a]) \n\t" + "lwc1 %[f4], 64(%[tmp_a]) \n\t" + "lwc1 %[f6], 96(%[tmp_a]) \n\t" + "sub.s %[f8], %[f0], %[f2] \n\t" + "add.s %[f0], %[f0], %[f2] \n\t" + "sub.s %[f2], %[f5], %[f7] \n\t" + "add.s %[f5], %[f5], %[f7] \n\t" + "sub.s %[f7], %[f1], %[f3] \n\t" + "add.s %[f1], %[f1], %[f3] \n\t" + "sub.s %[f3], %[f4], %[f6] \n\t" + "add.s %[f4], %[f4], %[f6] \n\t" + "sub.s %[f6], %[f8], %[f2] \n\t" + "add.s %[f8], %[f8], %[f2] \n\t" + "add.s %[f2], %[f5], %[f1] \n\t" + "sub.s %[f5], %[f5], %[f1] \n\t" + "add.s %[f1], %[f3], %[f7] \n\t" + "sub.s %[f3], %[f3], %[f7] \n\t" + "add.s %[f7], %[f0], %[f4] \n\t" + "sub.s %[f0], %[f0], %[f4] \n\t" + "sub.s %[f4], %[f6], %[f1] \n\t" + "add.s %[f6], %[f6], %[f1] \n\t" + "sub.s %[f1], %[f3], %[f8] \n\t" + "add.s %[f3], %[f3], %[f8] \n\t" + "mul.s %[f4], %[f4], %[f9] \n\t" + "mul.s %[f6], %[f6], %[f9] \n\t" + "mul.s %[f1], %[f1], %[f9] \n\t" + "mul.s %[f3], %[f3], %[f9] \n\t" + "swc1 %[f7], 0(%[tmp_a]) \n\t" + "swc1 %[f2], 4(%[tmp_a]) \n\t" + "swc1 %[f5], 64(%[tmp_a]) \n\t" + "swc1 %[f0], 68(%[tmp_a]) \n\t" + "swc1 %[f4], 32(%[tmp_a]) \n\t" + "swc1 %[f6], 36(%[tmp_a]) \n\t" + "swc1 %[f1], 96(%[tmp_a]) \n\t" + "swc1 %[f3], 100(%[tmp_a]) \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[tmp_a], %[tmp_a], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), + [f4] "=&f" (f4), [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), + [f8] "=&f" (f8), [tmp_a] "=&r" (tmp_a), [count] "=&r" (count) + : [a] "r" (a), [f9] "f" (f9) + : "memory" + ); + f10 = rdft_w[3]; + f11 = rdft_w[4]; + f12 = rdft_w[5]; + f13 = rdft_wk3ri_first[2]; + f14 = rdft_wk3ri_first[3]; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[tmp_a], %[a], 256 \n\t" + "addiu %[count], $zero, 4 \n\t" + "1: \n\t" + "addiu %[count], %[count], -1 \n\t" + "lwc1 %[f0], 0(%[tmp_a]) \n\t" + "lwc1 %[f2], 32(%[tmp_a]) \n\t" + "lwc1 %[f4], 64(%[tmp_a]) \n\t" + "lwc1 %[f6], 96(%[tmp_a]) \n\t" + "lwc1 %[f1], 4(%[tmp_a]) \n\t" + "lwc1 %[f3], 36(%[tmp_a]) \n\t" + "lwc1 %[f5], 68(%[tmp_a]) \n\t" + "lwc1 %[f7], 100(%[tmp_a]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "sub.s %[f7], %[f8], %[f2] \n\t" + "add.s %[f8], %[f8], %[f2] \n\t" + "add.s %[f2], %[f1], %[f4] \n\t" + "sub.s %[f1], %[f1], %[f4] \n\t" + "sub.s %[f4], %[f6], %[f3] \n\t" + "add.s %[f6], %[f6], %[f3] \n\t" + "sub.s %[f3], %[f0], %[f5] \n\t" + "add.s %[f0], %[f0], %[f5] \n\t" + "swc1 %[f8], 0(%[tmp_a]) \n\t" + "swc1 %[f6], 4(%[tmp_a]) \n\t" + "mul.s %[f5], %[f9], %[f7] \n\t" +#if defined(MIPS32_R2_LE) + "mul.s %[f7], %[f10], %[f7] \n\t" + "mul.s %[f8], %[f11], %[f3] \n\t" + "mul.s %[f3], %[f12], %[f3] \n\t" + "mul.s %[f6], %[f13], %[f0] \n\t" + "mul.s %[f0], %[f14], %[f0] \n\t" + "nmsub.s %[f5], %[f5], %[f10], %[f4] \n\t" + "madd.s %[f7], %[f7], %[f9], %[f4] \n\t" + "nmsub.s %[f8], %[f8], %[f12], %[f2] \n\t" + "madd.s %[f3], %[f3], %[f11], %[f2] \n\t" + "nmsub.s %[f6], %[f6], %[f14], %[f1] \n\t" + "madd.s %[f0], %[f0], %[f13], %[f1] \n\t" + "swc1 %[f5], 64(%[tmp_a]) \n\t" + "swc1 %[f7], 68(%[tmp_a]) \n\t" +#else + "mul.s %[f8], %[f10], %[f4] \n\t" + "mul.s %[f4], %[f9], %[f4] \n\t" + "mul.s %[f7], %[f10], %[f7] \n\t" + "mul.s %[f6], %[f11], %[f3] \n\t" + "mul.s %[f3], %[f12], %[f3] \n\t" + "sub.s %[f5], %[f5], %[f8] \n\t" + "mul.s %[f8], %[f12], %[f2] \n\t" + "mul.s %[f2], %[f11], %[f2] \n\t" + "add.s %[f7], %[f4], %[f7] \n\t" + "mul.s %[f4], %[f13], %[f0] \n\t" + "mul.s %[f0], %[f14], %[f0] \n\t" + "sub.s %[f8], %[f6], %[f8] \n\t" + "mul.s %[f6], %[f14], %[f1] \n\t" + "mul.s %[f1], %[f13], %[f1] \n\t" + "add.s %[f3], %[f2], %[f3] \n\t" + "swc1 %[f5], 64(%[tmp_a]) \n\t" + "swc1 %[f7], 68(%[tmp_a]) \n\t" + "sub.s %[f6], %[f4], %[f6] \n\t" + "add.s %[f0], %[f1], %[f0] \n\t" +#endif + "swc1 %[f8], 32(%[tmp_a]) \n\t" + "swc1 %[f3], 36(%[tmp_a]) \n\t" + "swc1 %[f6], 96(%[tmp_a]) \n\t" + "swc1 %[f0], 100(%[tmp_a]) \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[tmp_a], %[tmp_a], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), + [f4] "=&f" (f4), [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), + [f8] "=&f" (f8), [tmp_a] "=&r" (tmp_a), [count] "=&r" (count) + : [a] "r" (a), [f9] "f" (f9), [f10] "f" (f10), [f11] "f" (f11), + [f12] "f" (f12), [f13] "f" (f13), [f14] "f" (f14) + : "memory" + ); + f11 = rdft_w[6]; + f12 = rdft_w[7]; + f13 = rdft_wk3ri_second[2]; + f14 = rdft_wk3ri_second[3]; + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[tmp_a], %[a], 384 \n\t" + "addiu %[count], $zero, 4 \n\t" + "1: \n\t" + "addiu %[count], %[count], -1 \n\t" + "lwc1 %[f0], 0(%[tmp_a]) \n\t" + "lwc1 %[f1], 4(%[tmp_a]) \n\t" + "lwc1 %[f2], 32(%[tmp_a]) \n\t" + "lwc1 %[f3], 36(%[tmp_a]) \n\t" + "lwc1 %[f4], 64(%[tmp_a]) \n\t" + "lwc1 %[f5], 68(%[tmp_a]) \n\t" + "lwc1 %[f6], 96(%[tmp_a]) \n\t" + "lwc1 %[f7], 100(%[tmp_a]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "sub.s %[f7], %[f2], %[f8] \n\t" + "add.s %[f2], %[f2], %[f8] \n\t" + "add.s %[f8], %[f1], %[f4] \n\t" + "sub.s %[f1], %[f1], %[f4] \n\t" + "sub.s %[f4], %[f3], %[f6] \n\t" + "add.s %[f3], %[f3], %[f6] \n\t" + "sub.s %[f6], %[f0], %[f5] \n\t" + "add.s %[f0], %[f0], %[f5] \n\t" + "swc1 %[f2], 0(%[tmp_a]) \n\t" + "swc1 %[f3], 4(%[tmp_a]) \n\t" + "mul.s %[f5], %[f10], %[f7] \n\t" +#if defined(MIPS32_R2_LE) + "mul.s %[f7], %[f9], %[f7] \n\t" + "mul.s %[f2], %[f12], %[f8] \n\t" + "mul.s %[f8], %[f11], %[f8] \n\t" + "mul.s %[f3], %[f14], %[f1] \n\t" + "mul.s %[f1], %[f13], %[f1] \n\t" + "madd.s %[f5], %[f5], %[f9], %[f4] \n\t" + "msub.s %[f7], %[f7], %[f10], %[f4] \n\t" + "msub.s %[f2], %[f2], %[f11], %[f6] \n\t" + "madd.s %[f8], %[f8], %[f12], %[f6] \n\t" + "msub.s %[f3], %[f3], %[f13], %[f0] \n\t" + "madd.s %[f1], %[f1], %[f14], %[f0] \n\t" + "swc1 %[f5], 64(%[tmp_a]) \n\t" + "swc1 %[f7], 68(%[tmp_a]) \n\t" +#else + "mul.s %[f2], %[f9], %[f4] \n\t" + "mul.s %[f4], %[f10], %[f4] \n\t" + "mul.s %[f7], %[f9], %[f7] \n\t" + "mul.s %[f3], %[f11], %[f6] \n\t" + "mul.s %[f6], %[f12], %[f6] \n\t" + "add.s %[f5], %[f5], %[f2] \n\t" + "sub.s %[f7], %[f4], %[f7] \n\t" + "mul.s %[f2], %[f12], %[f8] \n\t" + "mul.s %[f8], %[f11], %[f8] \n\t" + "mul.s %[f4], %[f14], %[f1] \n\t" + "mul.s %[f1], %[f13], %[f1] \n\t" + "sub.s %[f2], %[f3], %[f2] \n\t" + "mul.s %[f3], %[f13], %[f0] \n\t" + "mul.s %[f0], %[f14], %[f0] \n\t" + "add.s %[f8], %[f8], %[f6] \n\t" + "swc1 %[f5], 64(%[tmp_a]) \n\t" + "swc1 %[f7], 68(%[tmp_a]) \n\t" + "sub.s %[f3], %[f3], %[f4] \n\t" + "add.s %[f1], %[f1], %[f0] \n\t" +#endif + "swc1 %[f2], 32(%[tmp_a]) \n\t" + "swc1 %[f8], 36(%[tmp_a]) \n\t" + "swc1 %[f3], 96(%[tmp_a]) \n\t" + "swc1 %[f1], 100(%[tmp_a]) \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[tmp_a], %[tmp_a], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), + [f4] "=&f" (f4), [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), + [f8] "=&f" (f8), [tmp_a] "=&r" (tmp_a), [count] "=&r" (count) + : [a] "r" (a), [f9] "f" (f9), [f10] "f" (f10), [f11] "f" (f11), + [f12] "f" (f12), [f13] "f" (f13), [f14] "f" (f14) + : "memory" + ); +} + +void cftfsub_128_mips(float* a) { + float f0, f1, f2, f3, f4, f5, f6, f7, f8; + int tmp_a, count; + + cft1st_128_mips(a); + cftmdl_128_mips(a); + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[tmp_a], %[a], 0 \n\t" + "addiu %[count], $zero, 16 \n\t" + "1: \n\t" + "addiu %[count], %[count], -1 \n\t" + "lwc1 %[f0], 0(%[tmp_a]) \n\t" + "lwc1 %[f2], 128(%[tmp_a]) \n\t" + "lwc1 %[f4], 256(%[tmp_a]) \n\t" + "lwc1 %[f6], 384(%[tmp_a]) \n\t" + "lwc1 %[f1], 4(%[tmp_a]) \n\t" + "lwc1 %[f3], 132(%[tmp_a]) \n\t" + "lwc1 %[f5], 260(%[tmp_a]) \n\t" + "lwc1 %[f7], 388(%[tmp_a]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f1], %[f3] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "add.s %[f7], %[f8], %[f2] \n\t" + "sub.s %[f8], %[f8], %[f2] \n\t" + "add.s %[f2], %[f1], %[f4] \n\t" + "sub.s %[f1], %[f1], %[f4] \n\t" + "add.s %[f4], %[f6], %[f3] \n\t" + "sub.s %[f6], %[f6], %[f3] \n\t" + "sub.s %[f3], %[f0], %[f5] \n\t" + "add.s %[f0], %[f0], %[f5] \n\t" + "swc1 %[f7], 0(%[tmp_a]) \n\t" + "swc1 %[f8], 256(%[tmp_a]) \n\t" + "swc1 %[f2], 132(%[tmp_a]) \n\t" + "swc1 %[f1], 388(%[tmp_a]) \n\t" + "swc1 %[f4], 4(%[tmp_a]) \n\t" + "swc1 %[f6], 260(%[tmp_a]) \n\t" + "swc1 %[f3], 128(%[tmp_a]) \n\t" + "swc1 %[f0], 384(%[tmp_a]) \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[tmp_a], %[tmp_a], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), + [f4] "=&f" (f4), [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), + [f8] "=&f" (f8), [tmp_a] "=&r" (tmp_a), + [count] "=&r" (count) + : [a] "r" (a) + : "memory" + ); +} + +void cftbsub_128_mips(float* a) { + float f0, f1, f2, f3, f4, f5, f6, f7, f8; + int tmp_a, count; + + cft1st_128_mips(a); + cftmdl_128_mips(a); + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[tmp_a], %[a], 0 \n\t" + "addiu %[count], $zero, 16 \n\t" + "1: \n\t" + "addiu %[count], %[count], -1 \n\t" + "lwc1 %[f0], 0(%[tmp_a]) \n\t" + "lwc1 %[f2], 128(%[tmp_a]) \n\t" + "lwc1 %[f4], 256(%[tmp_a]) \n\t" + "lwc1 %[f6], 384(%[tmp_a]) \n\t" + "lwc1 %[f1], 4(%[tmp_a]) \n\t" + "lwc1 %[f3], 132(%[tmp_a]) \n\t" + "lwc1 %[f5], 260(%[tmp_a]) \n\t" + "lwc1 %[f7], 388(%[tmp_a]) \n\t" + "add.s %[f8], %[f0], %[f2] \n\t" + "sub.s %[f0], %[f0], %[f2] \n\t" + "add.s %[f2], %[f4], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "add.s %[f6], %[f1], %[f3] \n\t" + "sub.s %[f1], %[f3], %[f1] \n\t" + "add.s %[f3], %[f5], %[f7] \n\t" + "sub.s %[f5], %[f5], %[f7] \n\t" + "add.s %[f7], %[f8], %[f2] \n\t" + "sub.s %[f8], %[f8], %[f2] \n\t" + "sub.s %[f2], %[f1], %[f4] \n\t" + "add.s %[f1], %[f1], %[f4] \n\t" + "add.s %[f4], %[f3], %[f6] \n\t" + "sub.s %[f6], %[f3], %[f6] \n\t" + "sub.s %[f3], %[f0], %[f5] \n\t" + "add.s %[f0], %[f0], %[f5] \n\t" + "neg.s %[f4], %[f4] \n\t" + "swc1 %[f7], 0(%[tmp_a]) \n\t" + "swc1 %[f8], 256(%[tmp_a]) \n\t" + "swc1 %[f2], 132(%[tmp_a]) \n\t" + "swc1 %[f1], 388(%[tmp_a]) \n\t" + "swc1 %[f6], 260(%[tmp_a]) \n\t" + "swc1 %[f3], 128(%[tmp_a]) \n\t" + "swc1 %[f0], 384(%[tmp_a]) \n\t" + "swc1 %[f4], 4(%[tmp_a]) \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[tmp_a], %[tmp_a], 8 \n\t" + ".set pop \n\t" + : [f0] "=&f" (f0), [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), + [f4] "=&f" (f4), [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), + [f8] "=&f" (f8), [tmp_a] "=&r" (tmp_a), [count] "=&r" (count) + : [a] "r" (a) + : "memory" + ); +} + +void rftfsub_128_mips(float* a) { + const float* c = rdft_w + 32; + const float f0 = 0.5f; + float* a1 = &a[2]; + float* a2 = &a[126]; + const float* c1 = &c[1]; + const float* c2 = &c[31]; + float f1, f2, f3 ,f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15; + int count; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lwc1 %[f6], 0(%[c2]) \n\t" + "lwc1 %[f1], 0(%[a1]) \n\t" + "lwc1 %[f2], 0(%[a2]) \n\t" + "lwc1 %[f3], 4(%[a1]) \n\t" + "lwc1 %[f4], 4(%[a2]) \n\t" + "lwc1 %[f5], 0(%[c1]) \n\t" + "sub.s %[f6], %[f0], %[f6] \n\t" + "sub.s %[f7], %[f1], %[f2] \n\t" + "add.s %[f8], %[f3], %[f4] \n\t" + "addiu %[count], $zero, 15 \n\t" + "mul.s %[f9], %[f6], %[f7] \n\t" + "mul.s %[f6], %[f6], %[f8] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f8], %[f5], %[f8] \n\t" + "mul.s %[f5], %[f5], %[f7] \n\t" + "sub.s %[f9], %[f9], %[f8] \n\t" + "add.s %[f6], %[f6], %[f5] \n\t" +#else + "nmsub.s %[f9], %[f9], %[f5], %[f8] \n\t" + "madd.s %[f6], %[f6], %[f5], %[f7] \n\t" +#endif + "sub.s %[f1], %[f1], %[f9] \n\t" + "add.s %[f2], %[f2], %[f9] \n\t" + "sub.s %[f3], %[f3], %[f6] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "swc1 %[f1], 0(%[a1]) \n\t" + "swc1 %[f2], 0(%[a2]) \n\t" + "swc1 %[f3], 4(%[a1]) \n\t" + "swc1 %[f4], 4(%[a2]) \n\t" + "addiu %[a1], %[a1], 8 \n\t" + "addiu %[a2], %[a2], -8 \n\t" + "addiu %[c1], %[c1], 4 \n\t" + "addiu %[c2], %[c2], -4 \n\t" + "1: \n\t" + "lwc1 %[f6], 0(%[c2]) \n\t" + "lwc1 %[f1], 0(%[a1]) \n\t" + "lwc1 %[f2], 0(%[a2]) \n\t" + "lwc1 %[f3], 4(%[a1]) \n\t" + "lwc1 %[f4], 4(%[a2]) \n\t" + "lwc1 %[f5], 0(%[c1]) \n\t" + "sub.s %[f6], %[f0], %[f6] \n\t" + "sub.s %[f7], %[f1], %[f2] \n\t" + "add.s %[f8], %[f3], %[f4] \n\t" + "lwc1 %[f10], -4(%[c2]) \n\t" + "lwc1 %[f11], 8(%[a1]) \n\t" + "lwc1 %[f12], -8(%[a2]) \n\t" + "mul.s %[f9], %[f6], %[f7] \n\t" + "mul.s %[f6], %[f6], %[f8] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f8], %[f5], %[f8] \n\t" + "mul.s %[f5], %[f5], %[f7] \n\t" + "lwc1 %[f13], 12(%[a1]) \n\t" + "lwc1 %[f14], -4(%[a2]) \n\t" + "lwc1 %[f15], 4(%[c1]) \n\t" + "sub.s %[f9], %[f9], %[f8] \n\t" + "add.s %[f6], %[f6], %[f5] \n\t" +#else + "lwc1 %[f13], 12(%[a1]) \n\t" + "lwc1 %[f14], -4(%[a2]) \n\t" + "lwc1 %[f15], 4(%[c1]) \n\t" + "nmsub.s %[f9], %[f9], %[f5], %[f8] \n\t" + "madd.s %[f6], %[f6], %[f5], %[f7] \n\t" +#endif + "sub.s %[f10], %[f0], %[f10] \n\t" + "sub.s %[f5], %[f11], %[f12] \n\t" + "add.s %[f7], %[f13], %[f14] \n\t" + "sub.s %[f1], %[f1], %[f9] \n\t" + "add.s %[f2], %[f2], %[f9] \n\t" + "sub.s %[f3], %[f3], %[f6] \n\t" + "mul.s %[f8], %[f10], %[f5] \n\t" + "mul.s %[f10], %[f10], %[f7] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f9], %[f15], %[f7] \n\t" + "mul.s %[f15], %[f15], %[f5] \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "swc1 %[f1], 0(%[a1]) \n\t" + "swc1 %[f2], 0(%[a2]) \n\t" + "sub.s %[f8], %[f8], %[f9] \n\t" + "add.s %[f10], %[f10], %[f15] \n\t" +#else + "swc1 %[f1], 0(%[a1]) \n\t" + "swc1 %[f2], 0(%[a2]) \n\t" + "sub.s %[f4], %[f4], %[f6] \n\t" + "nmsub.s %[f8], %[f8], %[f15], %[f7] \n\t" + "madd.s %[f10], %[f10], %[f15], %[f5] \n\t" +#endif + "swc1 %[f3], 4(%[a1]) \n\t" + "swc1 %[f4], 4(%[a2]) \n\t" + "sub.s %[f11], %[f11], %[f8] \n\t" + "add.s %[f12], %[f12], %[f8] \n\t" + "sub.s %[f13], %[f13], %[f10] \n\t" + "sub.s %[f14], %[f14], %[f10] \n\t" + "addiu %[c2], %[c2], -8 \n\t" + "addiu %[c1], %[c1], 8 \n\t" + "swc1 %[f11], 8(%[a1]) \n\t" + "swc1 %[f12], -8(%[a2]) \n\t" + "swc1 %[f13], 12(%[a1]) \n\t" + "swc1 %[f14], -4(%[a2]) \n\t" + "addiu %[a1], %[a1], 16 \n\t" + "addiu %[count], %[count], -1 \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[a2], %[a2], -16 \n\t" + ".set pop \n\t" + : [a1] "+r" (a1), [a2] "+r" (a2), [c1] "+r" (c1), [c2] "+r" (c2), + [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), [f4] "=&f" (f4), + [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), [f8] "=&f" (f8), + [f9] "=&f" (f9), [f10] "=&f" (f10), [f11] "=&f" (f11), [f12] "=&f" (f12), + [f13] "=&f" (f13), [f14] "=&f" (f14), [f15] "=&f" (f15), + [count] "=&r" (count) + : [f0] "f" (f0) + : "memory" + ); +} + +void rftbsub_128_mips(float* a) { + const float *c = rdft_w + 32; + const float f0 = 0.5f; + float* a1 = &a[2]; + float* a2 = &a[126]; + const float* c1 = &c[1]; + const float* c2 = &c[31]; + float f1, f2, f3 ,f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15; + int count; + + a[1] = -a[1]; + a[65] = -a[65]; + + __asm __volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "lwc1 %[f6], 0(%[c2]) \n\t" + "lwc1 %[f1], 0(%[a1]) \n\t" + "lwc1 %[f2], 0(%[a2]) \n\t" + "lwc1 %[f3], 4(%[a1]) \n\t" + "lwc1 %[f4], 4(%[a2]) \n\t" + "lwc1 %[f5], 0(%[c1]) \n\t" + "sub.s %[f6], %[f0], %[f6] \n\t" + "sub.s %[f7], %[f1], %[f2] \n\t" + "add.s %[f8], %[f3], %[f4] \n\t" + "addiu %[count], $zero, 15 \n\t" + "mul.s %[f9], %[f6], %[f7] \n\t" + "mul.s %[f6], %[f6], %[f8] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f8], %[f5], %[f8] \n\t" + "mul.s %[f5], %[f5], %[f7] \n\t" + "add.s %[f9], %[f9], %[f8] \n\t" + "sub.s %[f6], %[f6], %[f5] \n\t" +#else + "madd.s %[f9], %[f9], %[f5], %[f8] \n\t" + "nmsub.s %[f6], %[f6], %[f5], %[f7] \n\t" +#endif + "sub.s %[f1], %[f1], %[f9] \n\t" + "add.s %[f2], %[f2], %[f9] \n\t" + "sub.s %[f3], %[f6], %[f3] \n\t" + "sub.s %[f4], %[f6], %[f4] \n\t" + "swc1 %[f1], 0(%[a1]) \n\t" + "swc1 %[f2], 0(%[a2]) \n\t" + "swc1 %[f3], 4(%[a1]) \n\t" + "swc1 %[f4], 4(%[a2]) \n\t" + "addiu %[a1], %[a1], 8 \n\t" + "addiu %[a2], %[a2], -8 \n\t" + "addiu %[c1], %[c1], 4 \n\t" + "addiu %[c2], %[c2], -4 \n\t" + "1: \n\t" + "lwc1 %[f6], 0(%[c2]) \n\t" + "lwc1 %[f1], 0(%[a1]) \n\t" + "lwc1 %[f2], 0(%[a2]) \n\t" + "lwc1 %[f3], 4(%[a1]) \n\t" + "lwc1 %[f4], 4(%[a2]) \n\t" + "lwc1 %[f5], 0(%[c1]) \n\t" + "sub.s %[f6], %[f0], %[f6] \n\t" + "sub.s %[f7], %[f1], %[f2] \n\t" + "add.s %[f8], %[f3], %[f4] \n\t" + "lwc1 %[f10], -4(%[c2]) \n\t" + "lwc1 %[f11], 8(%[a1]) \n\t" + "lwc1 %[f12], -8(%[a2]) \n\t" + "mul.s %[f9], %[f6], %[f7] \n\t" + "mul.s %[f6], %[f6], %[f8] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f8], %[f5], %[f8] \n\t" + "mul.s %[f5], %[f5], %[f7] \n\t" + "lwc1 %[f13], 12(%[a1]) \n\t" + "lwc1 %[f14], -4(%[a2]) \n\t" + "lwc1 %[f15], 4(%[c1]) \n\t" + "add.s %[f9], %[f9], %[f8] \n\t" + "sub.s %[f6], %[f6], %[f5] \n\t" +#else + "lwc1 %[f13], 12(%[a1]) \n\t" + "lwc1 %[f14], -4(%[a2]) \n\t" + "lwc1 %[f15], 4(%[c1]) \n\t" + "madd.s %[f9], %[f9], %[f5], %[f8] \n\t" + "nmsub.s %[f6], %[f6], %[f5], %[f7] \n\t" +#endif + "sub.s %[f10], %[f0], %[f10] \n\t" + "sub.s %[f5], %[f11], %[f12] \n\t" + "add.s %[f7], %[f13], %[f14] \n\t" + "sub.s %[f1], %[f1], %[f9] \n\t" + "add.s %[f2], %[f2], %[f9] \n\t" + "sub.s %[f3], %[f6], %[f3] \n\t" + "mul.s %[f8], %[f10], %[f5] \n\t" + "mul.s %[f10], %[f10], %[f7] \n\t" +#if !defined(MIPS32_R2_LE) + "mul.s %[f9], %[f15], %[f7] \n\t" + "mul.s %[f15], %[f15], %[f5] \n\t" + "sub.s %[f4], %[f6], %[f4] \n\t" + "swc1 %[f1], 0(%[a1]) \n\t" + "swc1 %[f2], 0(%[a2]) \n\t" + "add.s %[f8], %[f8], %[f9] \n\t" + "sub.s %[f10], %[f10], %[f15] \n\t" +#else + "swc1 %[f1], 0(%[a1]) \n\t" + "swc1 %[f2], 0(%[a2]) \n\t" + "sub.s %[f4], %[f6], %[f4] \n\t" + "madd.s %[f8], %[f8], %[f15], %[f7] \n\t" + "nmsub.s %[f10], %[f10], %[f15], %[f5] \n\t" +#endif + "swc1 %[f3], 4(%[a1]) \n\t" + "swc1 %[f4], 4(%[a2]) \n\t" + "sub.s %[f11], %[f11], %[f8] \n\t" + "add.s %[f12], %[f12], %[f8] \n\t" + "sub.s %[f13], %[f10], %[f13] \n\t" + "sub.s %[f14], %[f10], %[f14] \n\t" + "addiu %[c2], %[c2], -8 \n\t" + "addiu %[c1], %[c1], 8 \n\t" + "swc1 %[f11], 8(%[a1]) \n\t" + "swc1 %[f12], -8(%[a2]) \n\t" + "swc1 %[f13], 12(%[a1]) \n\t" + "swc1 %[f14], -4(%[a2]) \n\t" + "addiu %[a1], %[a1], 16 \n\t" + "addiu %[count], %[count], -1 \n\t" + "bgtz %[count], 1b \n\t" + " addiu %[a2], %[a2], -16 \n\t" + ".set pop \n\t" + : [a1] "+r" (a1), [a2] "+r" (a2), [c1] "+r" (c1), [c2] "+r" (c2), + [f1] "=&f" (f1), [f2] "=&f" (f2), [f3] "=&f" (f3), [f4] "=&f" (f4), + [f5] "=&f" (f5), [f6] "=&f" (f6), [f7] "=&f" (f7), [f8] "=&f" (f8), + [f9] "=&f" (f9), [f10] "=&f" (f10), [f11] "=&f" (f11), [f12] "=&f" (f12), + [f13] "=&f" (f13), [f14] "=&f" (f14), [f15] "=&f" (f15), + [count] "=&r" (count) + : [f0] "f" (f0) + : "memory" + ); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_neon.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_neon.cc new file mode 100644 index 0000000000..401387a643 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_neon.cc @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * The rdft AEC algorithm, neon version of speed-critical functions. + * + * Based on the sse2 version. + */ + +#include "modules/audio_processing/utility/ooura_fft.h" + +#include + +#include "modules/audio_processing/utility/ooura_fft_tables_common.h" +#include "modules/audio_processing/utility/ooura_fft_tables_neon_sse2.h" + +namespace webrtc { + +#if defined(WEBRTC_HAS_NEON) +void cft1st_128_neon(float* a) { + const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign); + int j, k2; + + for (k2 = 0, j = 0; j < 128; j += 16, k2 += 4) { + float32x4_t a00v = vld1q_f32(&a[j + 0]); + float32x4_t a04v = vld1q_f32(&a[j + 4]); + float32x4_t a08v = vld1q_f32(&a[j + 8]); + float32x4_t a12v = vld1q_f32(&a[j + 12]); + float32x4_t a01v = vcombine_f32(vget_low_f32(a00v), vget_low_f32(a08v)); + float32x4_t a23v = vcombine_f32(vget_high_f32(a00v), vget_high_f32(a08v)); + float32x4_t a45v = vcombine_f32(vget_low_f32(a04v), vget_low_f32(a12v)); + float32x4_t a67v = vcombine_f32(vget_high_f32(a04v), vget_high_f32(a12v)); + const float32x4_t wk1rv = vld1q_f32(&rdft_wk1r[k2]); + const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2]); + const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2]); + const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2]); + const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2]); + const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2]); + float32x4_t x0v = vaddq_f32(a01v, a23v); + const float32x4_t x1v = vsubq_f32(a01v, a23v); + const float32x4_t x2v = vaddq_f32(a45v, a67v); + const float32x4_t x3v = vsubq_f32(a45v, a67v); + const float32x4_t x3w = vrev64q_f32(x3v); + float32x4_t x0w; + a01v = vaddq_f32(x0v, x2v); + x0v = vsubq_f32(x0v, x2v); + x0w = vrev64q_f32(x0v); + a45v = vmulq_f32(wk2rv, x0v); + a45v = vmlaq_f32(a45v, wk2iv, x0w); + x0v = vmlaq_f32(x1v, x3w, vec_swap_sign); + x0w = vrev64q_f32(x0v); + a23v = vmulq_f32(wk1rv, x0v); + a23v = vmlaq_f32(a23v, wk1iv, x0w); + x0v = vmlsq_f32(x1v, x3w, vec_swap_sign); + x0w = vrev64q_f32(x0v); + a67v = vmulq_f32(wk3rv, x0v); + a67v = vmlaq_f32(a67v, wk3iv, x0w); + a00v = vcombine_f32(vget_low_f32(a01v), vget_low_f32(a23v)); + a04v = vcombine_f32(vget_low_f32(a45v), vget_low_f32(a67v)); + a08v = vcombine_f32(vget_high_f32(a01v), vget_high_f32(a23v)); + a12v = vcombine_f32(vget_high_f32(a45v), vget_high_f32(a67v)); + vst1q_f32(&a[j + 0], a00v); + vst1q_f32(&a[j + 4], a04v); + vst1q_f32(&a[j + 8], a08v); + vst1q_f32(&a[j + 12], a12v); + } +} + +void cftmdl_128_neon(float* a) { + int j; + const int l = 8; + const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign); + float32x4_t wk1rv = vld1q_f32(cftmdl_wk1r); + + for (j = 0; j < l; j += 2) { + const float32x2_t a_00 = vld1_f32(&a[j + 0]); + const float32x2_t a_08 = vld1_f32(&a[j + 8]); + const float32x2_t a_32 = vld1_f32(&a[j + 32]); + const float32x2_t a_40 = vld1_f32(&a[j + 40]); + const float32x4_t a_00_32 = vcombine_f32(a_00, a_32); + const float32x4_t a_08_40 = vcombine_f32(a_08, a_40); + const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40); + const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40); + const float32x2_t a_16 = vld1_f32(&a[j + 16]); + const float32x2_t a_24 = vld1_f32(&a[j + 24]); + const float32x2_t a_48 = vld1_f32(&a[j + 48]); + const float32x2_t a_56 = vld1_f32(&a[j + 56]); + const float32x4_t a_16_48 = vcombine_f32(a_16, a_48); + const float32x4_t a_24_56 = vcombine_f32(a_24, a_56); + const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56); + const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56); + const float32x4_t xx0 = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1); + const float32x4_t x1_x3_add = + vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); + const float32x4_t x1_x3_sub = + vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); + const float32x2_t yy0_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 0); + const float32x2_t yy0_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 0); + const float32x4_t yy0_as = vcombine_f32(yy0_a, yy0_s); + const float32x2_t yy1_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 1); + const float32x2_t yy1_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 1); + const float32x4_t yy1_as = vcombine_f32(yy1_a, yy1_s); + const float32x4_t yy0 = vmlaq_f32(yy0_as, vec_swap_sign, yy1_as); + const float32x4_t yy4 = vmulq_f32(wk1rv, yy0); + const float32x4_t xx1_rev = vrev64q_f32(xx1); + const float32x4_t yy4_rev = vrev64q_f32(yy4); + + vst1_f32(&a[j + 0], vget_low_f32(xx0)); + vst1_f32(&a[j + 32], vget_high_f32(xx0)); + vst1_f32(&a[j + 16], vget_low_f32(xx1)); + vst1_f32(&a[j + 48], vget_high_f32(xx1_rev)); + + a[j + 48] = -a[j + 48]; + + vst1_f32(&a[j + 8], vget_low_f32(x1_x3_add)); + vst1_f32(&a[j + 24], vget_low_f32(x1_x3_sub)); + vst1_f32(&a[j + 40], vget_low_f32(yy4)); + vst1_f32(&a[j + 56], vget_high_f32(yy4_rev)); + } + + { + const int k = 64; + const int k1 = 2; + const int k2 = 2 * k1; + const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2 + 0]); + const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2 + 0]); + const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2 + 0]); + const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2 + 0]); + const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2 + 0]); + wk1rv = vld1q_f32(&rdft_wk1r[k2 + 0]); + for (j = k; j < l + k; j += 2) { + const float32x2_t a_00 = vld1_f32(&a[j + 0]); + const float32x2_t a_08 = vld1_f32(&a[j + 8]); + const float32x2_t a_32 = vld1_f32(&a[j + 32]); + const float32x2_t a_40 = vld1_f32(&a[j + 40]); + const float32x4_t a_00_32 = vcombine_f32(a_00, a_32); + const float32x4_t a_08_40 = vcombine_f32(a_08, a_40); + const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40); + const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40); + const float32x2_t a_16 = vld1_f32(&a[j + 16]); + const float32x2_t a_24 = vld1_f32(&a[j + 24]); + const float32x2_t a_48 = vld1_f32(&a[j + 48]); + const float32x2_t a_56 = vld1_f32(&a[j + 56]); + const float32x4_t a_16_48 = vcombine_f32(a_16, a_48); + const float32x4_t a_24_56 = vcombine_f32(a_24, a_56); + const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56); + const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56); + const float32x4_t xx = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1); + const float32x4_t x1_x3_add = + vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); + const float32x4_t x1_x3_sub = + vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); + float32x4_t xx4 = vmulq_f32(wk2rv, xx1); + float32x4_t xx12 = vmulq_f32(wk1rv, x1_x3_add); + float32x4_t xx22 = vmulq_f32(wk3rv, x1_x3_sub); + xx4 = vmlaq_f32(xx4, wk2iv, vrev64q_f32(xx1)); + xx12 = vmlaq_f32(xx12, wk1iv, vrev64q_f32(x1_x3_add)); + xx22 = vmlaq_f32(xx22, wk3iv, vrev64q_f32(x1_x3_sub)); + + vst1_f32(&a[j + 0], vget_low_f32(xx)); + vst1_f32(&a[j + 32], vget_high_f32(xx)); + vst1_f32(&a[j + 16], vget_low_f32(xx4)); + vst1_f32(&a[j + 48], vget_high_f32(xx4)); + vst1_f32(&a[j + 8], vget_low_f32(xx12)); + vst1_f32(&a[j + 40], vget_high_f32(xx12)); + vst1_f32(&a[j + 24], vget_low_f32(xx22)); + vst1_f32(&a[j + 56], vget_high_f32(xx22)); + } + } +} + +__inline static float32x4_t reverse_order_f32x4(float32x4_t in) { + // A B C D -> C D A B + const float32x4_t rev = vcombine_f32(vget_high_f32(in), vget_low_f32(in)); + // C D A B -> D C B A + return vrev64q_f32(rev); +} + +void rftfsub_128_neon(float* a) { + const float* c = rdft_w + 32; + int j1, j2; + const float32x4_t mm_half = vdupq_n_f32(0.5f); + + // Vectorized code (four at once). + // Note: commented number are indexes for the first iteration of the loop. + for (j1 = 1, j2 = 2; j2 + 7 < 64; j1 += 4, j2 += 8) { + // Load 'wk'. + const float32x4_t c_j1 = vld1q_f32(&c[j1]); // 1, 2, 3, 4, + const float32x4_t c_k1 = vld1q_f32(&c[29 - j1]); // 28, 29, 30, 31, + const float32x4_t wkrt = vsubq_f32(mm_half, c_k1); // 28, 29, 30, 31, + const float32x4_t wkr_ = reverse_order_f32x4(wkrt); // 31, 30, 29, 28, + const float32x4_t wki_ = c_j1; // 1, 2, 3, 4, + // Load and shuffle 'a'. + // 2, 4, 6, 8, 3, 5, 7, 9 + float32x4x2_t a_j2_p = vld2q_f32(&a[0 + j2]); + // 120, 122, 124, 126, 121, 123, 125, 127, + const float32x4x2_t k2_0_4 = vld2q_f32(&a[122 - j2]); + // 126, 124, 122, 120 + const float32x4_t a_k2_p0 = reverse_order_f32x4(k2_0_4.val[0]); + // 127, 125, 123, 121 + const float32x4_t a_k2_p1 = reverse_order_f32x4(k2_0_4.val[1]); + // Calculate 'x'. + const float32x4_t xr_ = vsubq_f32(a_j2_p.val[0], a_k2_p0); + // 2-126, 4-124, 6-122, 8-120, + const float32x4_t xi_ = vaddq_f32(a_j2_p.val[1], a_k2_p1); + // 3-127, 5-125, 7-123, 9-121, + // Calculate product into 'y'. + // yr = wkr * xr - wki * xi; + // yi = wkr * xi + wki * xr; + const float32x4_t a_ = vmulq_f32(wkr_, xr_); + const float32x4_t b_ = vmulq_f32(wki_, xi_); + const float32x4_t c_ = vmulq_f32(wkr_, xi_); + const float32x4_t d_ = vmulq_f32(wki_, xr_); + const float32x4_t yr_ = vsubq_f32(a_, b_); // 2-126, 4-124, 6-122, 8-120, + const float32x4_t yi_ = vaddq_f32(c_, d_); // 3-127, 5-125, 7-123, 9-121, + // Update 'a'. + // a[j2 + 0] -= yr; + // a[j2 + 1] -= yi; + // a[k2 + 0] += yr; + // a[k2 + 1] -= yi; + // 126, 124, 122, 120, + const float32x4_t a_k2_p0n = vaddq_f32(a_k2_p0, yr_); + // 127, 125, 123, 121, + const float32x4_t a_k2_p1n = vsubq_f32(a_k2_p1, yi_); + // Shuffle in right order and store. + const float32x4_t a_k2_p0nr = vrev64q_f32(a_k2_p0n); + const float32x4_t a_k2_p1nr = vrev64q_f32(a_k2_p1n); + // 124, 125, 126, 127, 120, 121, 122, 123 + const float32x4x2_t a_k2_n = vzipq_f32(a_k2_p0nr, a_k2_p1nr); + // 2, 4, 6, 8, + a_j2_p.val[0] = vsubq_f32(a_j2_p.val[0], yr_); + // 3, 5, 7, 9, + a_j2_p.val[1] = vsubq_f32(a_j2_p.val[1], yi_); + // 2, 3, 4, 5, 6, 7, 8, 9, + vst2q_f32(&a[0 + j2], a_j2_p); + + vst1q_f32(&a[122 - j2], a_k2_n.val[1]); + vst1q_f32(&a[126 - j2], a_k2_n.val[0]); + } + + // Scalar code for the remaining items. + for (; j2 < 64; j1 += 1, j2 += 2) { + const int k2 = 128 - j2; + const int k1 = 32 - j1; + const float wkr = 0.5f - c[k1]; + const float wki = c[j1]; + const float xr = a[j2 + 0] - a[k2 + 0]; + const float xi = a[j2 + 1] + a[k2 + 1]; + const float yr = wkr * xr - wki * xi; + const float yi = wkr * xi + wki * xr; + a[j2 + 0] -= yr; + a[j2 + 1] -= yi; + a[k2 + 0] += yr; + a[k2 + 1] -= yi; + } +} + +void rftbsub_128_neon(float* a) { + const float* c = rdft_w + 32; + int j1, j2; + const float32x4_t mm_half = vdupq_n_f32(0.5f); + + a[1] = -a[1]; + // Vectorized code (four at once). + // Note: commented number are indexes for the first iteration of the loop. + for (j1 = 1, j2 = 2; j2 + 7 < 64; j1 += 4, j2 += 8) { + // Load 'wk'. + const float32x4_t c_j1 = vld1q_f32(&c[j1]); // 1, 2, 3, 4, + const float32x4_t c_k1 = vld1q_f32(&c[29 - j1]); // 28, 29, 30, 31, + const float32x4_t wkrt = vsubq_f32(mm_half, c_k1); // 28, 29, 30, 31, + const float32x4_t wkr_ = reverse_order_f32x4(wkrt); // 31, 30, 29, 28, + const float32x4_t wki_ = c_j1; // 1, 2, 3, 4, + // Load and shuffle 'a'. + // 2, 4, 6, 8, 3, 5, 7, 9 + float32x4x2_t a_j2_p = vld2q_f32(&a[0 + j2]); + // 120, 122, 124, 126, 121, 123, 125, 127, + const float32x4x2_t k2_0_4 = vld2q_f32(&a[122 - j2]); + // 126, 124, 122, 120 + const float32x4_t a_k2_p0 = reverse_order_f32x4(k2_0_4.val[0]); + // 127, 125, 123, 121 + const float32x4_t a_k2_p1 = reverse_order_f32x4(k2_0_4.val[1]); + // Calculate 'x'. + const float32x4_t xr_ = vsubq_f32(a_j2_p.val[0], a_k2_p0); + // 2-126, 4-124, 6-122, 8-120, + const float32x4_t xi_ = vaddq_f32(a_j2_p.val[1], a_k2_p1); + // 3-127, 5-125, 7-123, 9-121, + // Calculate product into 'y'. + // yr = wkr * xr - wki * xi; + // yi = wkr * xi + wki * xr; + const float32x4_t a_ = vmulq_f32(wkr_, xr_); + const float32x4_t b_ = vmulq_f32(wki_, xi_); + const float32x4_t c_ = vmulq_f32(wkr_, xi_); + const float32x4_t d_ = vmulq_f32(wki_, xr_); + const float32x4_t yr_ = vaddq_f32(a_, b_); // 2-126, 4-124, 6-122, 8-120, + const float32x4_t yi_ = vsubq_f32(c_, d_); // 3-127, 5-125, 7-123, 9-121, + // Update 'a'. + // a[j2 + 0] -= yr; + // a[j2 + 1] -= yi; + // a[k2 + 0] += yr; + // a[k2 + 1] -= yi; + // 126, 124, 122, 120, + const float32x4_t a_k2_p0n = vaddq_f32(a_k2_p0, yr_); + // 127, 125, 123, 121, + const float32x4_t a_k2_p1n = vsubq_f32(yi_, a_k2_p1); + // Shuffle in right order and store. + // 2, 3, 4, 5, 6, 7, 8, 9, + const float32x4_t a_k2_p0nr = vrev64q_f32(a_k2_p0n); + const float32x4_t a_k2_p1nr = vrev64q_f32(a_k2_p1n); + // 124, 125, 126, 127, 120, 121, 122, 123 + const float32x4x2_t a_k2_n = vzipq_f32(a_k2_p0nr, a_k2_p1nr); + // 2, 4, 6, 8, + a_j2_p.val[0] = vsubq_f32(a_j2_p.val[0], yr_); + // 3, 5, 7, 9, + a_j2_p.val[1] = vsubq_f32(yi_, a_j2_p.val[1]); + // 2, 3, 4, 5, 6, 7, 8, 9, + vst2q_f32(&a[0 + j2], a_j2_p); + + vst1q_f32(&a[122 - j2], a_k2_n.val[1]); + vst1q_f32(&a[126 - j2], a_k2_n.val[0]); + } + + // Scalar code for the remaining items. + for (; j2 < 64; j1 += 1, j2 += 2) { + const int k2 = 128 - j2; + const int k1 = 32 - j1; + const float wkr = 0.5f - c[k1]; + const float wki = c[j1]; + const float xr = a[j2 + 0] - a[k2 + 0]; + const float xi = a[j2 + 1] + a[k2 + 1]; + const float yr = wkr * xr + wki * xi; + const float yi = wkr * xi - wki * xr; + a[j2 + 0] = a[j2 + 0] - yr; + a[j2 + 1] = yi - a[j2 + 1]; + a[k2 + 0] = yr + a[k2 + 0]; + a[k2 + 1] = yi - a[k2 + 1]; + } + a[65] = -a[65]; +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_sse2.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_sse2.cc new file mode 100644 index 0000000000..48a05c3bc2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_sse2.cc @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing//utility/ooura_fft.h" + +#include + +#include "modules/audio_processing/utility/ooura_fft_tables_common.h" +#include "modules/audio_processing/utility/ooura_fft_tables_neon_sse2.h" + +namespace webrtc { + +#if defined(WEBRTC_ARCH_X86_FAMILY) + +namespace { +// These intrinsics were unavailable before VS 2008. +// TODO(andrew): move to a common file. +#if defined(_MSC_VER) && _MSC_VER < 1500 +static __inline __m128 _mm_castsi128_ps(__m128i a) { + return *(__m128*)&a; +} +static __inline __m128i _mm_castps_si128(__m128 a) { + return *(__m128i*)&a; +} +#endif + +} // namespace + +void cft1st_128_SSE2(float* a) { + const __m128 mm_swap_sign = _mm_load_ps(k_swap_sign); + int j, k2; + + for (k2 = 0, j = 0; j < 128; j += 16, k2 += 4) { + __m128 a00v = _mm_loadu_ps(&a[j + 0]); + __m128 a04v = _mm_loadu_ps(&a[j + 4]); + __m128 a08v = _mm_loadu_ps(&a[j + 8]); + __m128 a12v = _mm_loadu_ps(&a[j + 12]); + __m128 a01v = _mm_shuffle_ps(a00v, a08v, _MM_SHUFFLE(1, 0, 1, 0)); + __m128 a23v = _mm_shuffle_ps(a00v, a08v, _MM_SHUFFLE(3, 2, 3, 2)); + __m128 a45v = _mm_shuffle_ps(a04v, a12v, _MM_SHUFFLE(1, 0, 1, 0)); + __m128 a67v = _mm_shuffle_ps(a04v, a12v, _MM_SHUFFLE(3, 2, 3, 2)); + + const __m128 wk1rv = _mm_load_ps(&rdft_wk1r[k2]); + const __m128 wk1iv = _mm_load_ps(&rdft_wk1i[k2]); + const __m128 wk2rv = _mm_load_ps(&rdft_wk2r[k2]); + const __m128 wk2iv = _mm_load_ps(&rdft_wk2i[k2]); + const __m128 wk3rv = _mm_load_ps(&rdft_wk3r[k2]); + const __m128 wk3iv = _mm_load_ps(&rdft_wk3i[k2]); + __m128 x0v = _mm_add_ps(a01v, a23v); + const __m128 x1v = _mm_sub_ps(a01v, a23v); + const __m128 x2v = _mm_add_ps(a45v, a67v); + const __m128 x3v = _mm_sub_ps(a45v, a67v); + __m128 x0w; + a01v = _mm_add_ps(x0v, x2v); + x0v = _mm_sub_ps(x0v, x2v); + x0w = _mm_shuffle_ps(x0v, x0v, _MM_SHUFFLE(2, 3, 0, 1)); + { + const __m128 a45_0v = _mm_mul_ps(wk2rv, x0v); + const __m128 a45_1v = _mm_mul_ps(wk2iv, x0w); + a45v = _mm_add_ps(a45_0v, a45_1v); + } + { + __m128 a23_0v, a23_1v; + const __m128 x3w = _mm_shuffle_ps(x3v, x3v, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128 x3s = _mm_mul_ps(mm_swap_sign, x3w); + x0v = _mm_add_ps(x1v, x3s); + x0w = _mm_shuffle_ps(x0v, x0v, _MM_SHUFFLE(2, 3, 0, 1)); + a23_0v = _mm_mul_ps(wk1rv, x0v); + a23_1v = _mm_mul_ps(wk1iv, x0w); + a23v = _mm_add_ps(a23_0v, a23_1v); + + x0v = _mm_sub_ps(x1v, x3s); + x0w = _mm_shuffle_ps(x0v, x0v, _MM_SHUFFLE(2, 3, 0, 1)); + } + { + const __m128 a67_0v = _mm_mul_ps(wk3rv, x0v); + const __m128 a67_1v = _mm_mul_ps(wk3iv, x0w); + a67v = _mm_add_ps(a67_0v, a67_1v); + } + + a00v = _mm_shuffle_ps(a01v, a23v, _MM_SHUFFLE(1, 0, 1, 0)); + a04v = _mm_shuffle_ps(a45v, a67v, _MM_SHUFFLE(1, 0, 1, 0)); + a08v = _mm_shuffle_ps(a01v, a23v, _MM_SHUFFLE(3, 2, 3, 2)); + a12v = _mm_shuffle_ps(a45v, a67v, _MM_SHUFFLE(3, 2, 3, 2)); + _mm_storeu_ps(&a[j + 0], a00v); + _mm_storeu_ps(&a[j + 4], a04v); + _mm_storeu_ps(&a[j + 8], a08v); + _mm_storeu_ps(&a[j + 12], a12v); + } +} + +void cftmdl_128_SSE2(float* a) { + const int l = 8; + const __m128 mm_swap_sign = _mm_load_ps(k_swap_sign); + int j0; + + __m128 wk1rv = _mm_load_ps(cftmdl_wk1r); + for (j0 = 0; j0 < l; j0 += 2) { + const __m128i a_00 = _mm_loadl_epi64((__m128i*)&a[j0 + 0]); + const __m128i a_08 = _mm_loadl_epi64((__m128i*)&a[j0 + 8]); + const __m128i a_32 = _mm_loadl_epi64((__m128i*)&a[j0 + 32]); + const __m128i a_40 = _mm_loadl_epi64((__m128i*)&a[j0 + 40]); + const __m128 a_00_32 = + _mm_shuffle_ps(_mm_castsi128_ps(a_00), _mm_castsi128_ps(a_32), + _MM_SHUFFLE(1, 0, 1, 0)); + const __m128 a_08_40 = + _mm_shuffle_ps(_mm_castsi128_ps(a_08), _mm_castsi128_ps(a_40), + _MM_SHUFFLE(1, 0, 1, 0)); + __m128 x0r0_0i0_0r1_x0i1 = _mm_add_ps(a_00_32, a_08_40); + const __m128 x1r0_1i0_1r1_x1i1 = _mm_sub_ps(a_00_32, a_08_40); + + const __m128i a_16 = _mm_loadl_epi64((__m128i*)&a[j0 + 16]); + const __m128i a_24 = _mm_loadl_epi64((__m128i*)&a[j0 + 24]); + const __m128i a_48 = _mm_loadl_epi64((__m128i*)&a[j0 + 48]); + const __m128i a_56 = _mm_loadl_epi64((__m128i*)&a[j0 + 56]); + const __m128 a_16_48 = + _mm_shuffle_ps(_mm_castsi128_ps(a_16), _mm_castsi128_ps(a_48), + _MM_SHUFFLE(1, 0, 1, 0)); + const __m128 a_24_56 = + _mm_shuffle_ps(_mm_castsi128_ps(a_24), _mm_castsi128_ps(a_56), + _MM_SHUFFLE(1, 0, 1, 0)); + const __m128 x2r0_2i0_2r1_x2i1 = _mm_add_ps(a_16_48, a_24_56); + const __m128 x3r0_3i0_3r1_x3i1 = _mm_sub_ps(a_16_48, a_24_56); + + const __m128 xx0 = _mm_add_ps(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + const __m128 xx1 = _mm_sub_ps(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + + const __m128 x3i0_3r0_3i1_x3r1 = _mm_castsi128_ps(_mm_shuffle_epi32( + _mm_castps_si128(x3r0_3i0_3r1_x3i1), _MM_SHUFFLE(2, 3, 0, 1))); + const __m128 x3_swapped = _mm_mul_ps(mm_swap_sign, x3i0_3r0_3i1_x3r1); + const __m128 x1_x3_add = _mm_add_ps(x1r0_1i0_1r1_x1i1, x3_swapped); + const __m128 x1_x3_sub = _mm_sub_ps(x1r0_1i0_1r1_x1i1, x3_swapped); + + const __m128 yy0 = + _mm_shuffle_ps(x1_x3_add, x1_x3_sub, _MM_SHUFFLE(2, 2, 2, 2)); + const __m128 yy1 = + _mm_shuffle_ps(x1_x3_add, x1_x3_sub, _MM_SHUFFLE(3, 3, 3, 3)); + const __m128 yy2 = _mm_mul_ps(mm_swap_sign, yy1); + const __m128 yy3 = _mm_add_ps(yy0, yy2); + const __m128 yy4 = _mm_mul_ps(wk1rv, yy3); + + _mm_storel_epi64((__m128i*)&a[j0 + 0], _mm_castps_si128(xx0)); + _mm_storel_epi64( + (__m128i*)&a[j0 + 32], + _mm_shuffle_epi32(_mm_castps_si128(xx0), _MM_SHUFFLE(3, 2, 3, 2))); + + _mm_storel_epi64((__m128i*)&a[j0 + 16], _mm_castps_si128(xx1)); + _mm_storel_epi64( + (__m128i*)&a[j0 + 48], + _mm_shuffle_epi32(_mm_castps_si128(xx1), _MM_SHUFFLE(2, 3, 2, 3))); + a[j0 + 48] = -a[j0 + 48]; + + _mm_storel_epi64((__m128i*)&a[j0 + 8], _mm_castps_si128(x1_x3_add)); + _mm_storel_epi64((__m128i*)&a[j0 + 24], _mm_castps_si128(x1_x3_sub)); + + _mm_storel_epi64((__m128i*)&a[j0 + 40], _mm_castps_si128(yy4)); + _mm_storel_epi64( + (__m128i*)&a[j0 + 56], + _mm_shuffle_epi32(_mm_castps_si128(yy4), _MM_SHUFFLE(2, 3, 2, 3))); + } + + { + int k = 64; + int k1 = 2; + int k2 = 2 * k1; + const __m128 wk2rv = _mm_load_ps(&rdft_wk2r[k2 + 0]); + const __m128 wk2iv = _mm_load_ps(&rdft_wk2i[k2 + 0]); + const __m128 wk1iv = _mm_load_ps(&rdft_wk1i[k2 + 0]); + const __m128 wk3rv = _mm_load_ps(&rdft_wk3r[k2 + 0]); + const __m128 wk3iv = _mm_load_ps(&rdft_wk3i[k2 + 0]); + wk1rv = _mm_load_ps(&rdft_wk1r[k2 + 0]); + for (j0 = k; j0 < l + k; j0 += 2) { + const __m128i a_00 = _mm_loadl_epi64((__m128i*)&a[j0 + 0]); + const __m128i a_08 = _mm_loadl_epi64((__m128i*)&a[j0 + 8]); + const __m128i a_32 = _mm_loadl_epi64((__m128i*)&a[j0 + 32]); + const __m128i a_40 = _mm_loadl_epi64((__m128i*)&a[j0 + 40]); + const __m128 a_00_32 = + _mm_shuffle_ps(_mm_castsi128_ps(a_00), _mm_castsi128_ps(a_32), + _MM_SHUFFLE(1, 0, 1, 0)); + const __m128 a_08_40 = + _mm_shuffle_ps(_mm_castsi128_ps(a_08), _mm_castsi128_ps(a_40), + _MM_SHUFFLE(1, 0, 1, 0)); + __m128 x0r0_0i0_0r1_x0i1 = _mm_add_ps(a_00_32, a_08_40); + const __m128 x1r0_1i0_1r1_x1i1 = _mm_sub_ps(a_00_32, a_08_40); + + const __m128i a_16 = _mm_loadl_epi64((__m128i*)&a[j0 + 16]); + const __m128i a_24 = _mm_loadl_epi64((__m128i*)&a[j0 + 24]); + const __m128i a_48 = _mm_loadl_epi64((__m128i*)&a[j0 + 48]); + const __m128i a_56 = _mm_loadl_epi64((__m128i*)&a[j0 + 56]); + const __m128 a_16_48 = + _mm_shuffle_ps(_mm_castsi128_ps(a_16), _mm_castsi128_ps(a_48), + _MM_SHUFFLE(1, 0, 1, 0)); + const __m128 a_24_56 = + _mm_shuffle_ps(_mm_castsi128_ps(a_24), _mm_castsi128_ps(a_56), + _MM_SHUFFLE(1, 0, 1, 0)); + const __m128 x2r0_2i0_2r1_x2i1 = _mm_add_ps(a_16_48, a_24_56); + const __m128 x3r0_3i0_3r1_x3i1 = _mm_sub_ps(a_16_48, a_24_56); + + const __m128 xx = _mm_add_ps(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + const __m128 xx1 = _mm_sub_ps(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); + const __m128 xx2 = _mm_mul_ps(xx1, wk2rv); + const __m128 xx3 = _mm_mul_ps( + wk2iv, _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(xx1), + _MM_SHUFFLE(2, 3, 0, 1)))); + const __m128 xx4 = _mm_add_ps(xx2, xx3); + + const __m128 x3i0_3r0_3i1_x3r1 = _mm_castsi128_ps(_mm_shuffle_epi32( + _mm_castps_si128(x3r0_3i0_3r1_x3i1), _MM_SHUFFLE(2, 3, 0, 1))); + const __m128 x3_swapped = _mm_mul_ps(mm_swap_sign, x3i0_3r0_3i1_x3r1); + const __m128 x1_x3_add = _mm_add_ps(x1r0_1i0_1r1_x1i1, x3_swapped); + const __m128 x1_x3_sub = _mm_sub_ps(x1r0_1i0_1r1_x1i1, x3_swapped); + + const __m128 xx10 = _mm_mul_ps(x1_x3_add, wk1rv); + const __m128 xx11 = _mm_mul_ps( + wk1iv, _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(x1_x3_add), + _MM_SHUFFLE(2, 3, 0, 1)))); + const __m128 xx12 = _mm_add_ps(xx10, xx11); + + const __m128 xx20 = _mm_mul_ps(x1_x3_sub, wk3rv); + const __m128 xx21 = _mm_mul_ps( + wk3iv, _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(x1_x3_sub), + _MM_SHUFFLE(2, 3, 0, 1)))); + const __m128 xx22 = _mm_add_ps(xx20, xx21); + + _mm_storel_epi64((__m128i*)&a[j0 + 0], _mm_castps_si128(xx)); + _mm_storel_epi64( + (__m128i*)&a[j0 + 32], + _mm_shuffle_epi32(_mm_castps_si128(xx), _MM_SHUFFLE(3, 2, 3, 2))); + + _mm_storel_epi64((__m128i*)&a[j0 + 16], _mm_castps_si128(xx4)); + _mm_storel_epi64( + (__m128i*)&a[j0 + 48], + _mm_shuffle_epi32(_mm_castps_si128(xx4), _MM_SHUFFLE(3, 2, 3, 2))); + + _mm_storel_epi64((__m128i*)&a[j0 + 8], _mm_castps_si128(xx12)); + _mm_storel_epi64( + (__m128i*)&a[j0 + 40], + _mm_shuffle_epi32(_mm_castps_si128(xx12), _MM_SHUFFLE(3, 2, 3, 2))); + + _mm_storel_epi64((__m128i*)&a[j0 + 24], _mm_castps_si128(xx22)); + _mm_storel_epi64( + (__m128i*)&a[j0 + 56], + _mm_shuffle_epi32(_mm_castps_si128(xx22), _MM_SHUFFLE(3, 2, 3, 2))); + } + } +} + +void rftfsub_128_SSE2(float* a) { + const float* c = rdft_w + 32; + int j1, j2, k1, k2; + float wkr, wki, xr, xi, yr, yi; + + static const ALIGN16_BEG float ALIGN16_END k_half[4] = {0.5f, 0.5f, 0.5f, + 0.5f}; + const __m128 mm_half = _mm_load_ps(k_half); + + // Vectorized code (four at once). + // Note: commented number are indexes for the first iteration of the loop. + for (j1 = 1, j2 = 2; j2 + 7 < 64; j1 += 4, j2 += 8) { + // Load 'wk'. + const __m128 c_j1 = _mm_loadu_ps(&c[j1]); // 1, 2, 3, 4, + const __m128 c_k1 = _mm_loadu_ps(&c[29 - j1]); // 28, 29, 30, 31, + const __m128 wkrt = _mm_sub_ps(mm_half, c_k1); // 28, 29, 30, 31, + const __m128 wkr_ = + _mm_shuffle_ps(wkrt, wkrt, _MM_SHUFFLE(0, 1, 2, 3)); // 31, 30, 29, 28, + const __m128 wki_ = c_j1; // 1, 2, 3, 4, + // Load and shuffle 'a'. + const __m128 a_j2_0 = _mm_loadu_ps(&a[0 + j2]); // 2, 3, 4, 5, + const __m128 a_j2_4 = _mm_loadu_ps(&a[4 + j2]); // 6, 7, 8, 9, + const __m128 a_k2_0 = _mm_loadu_ps(&a[122 - j2]); // 120, 121, 122, 123, + const __m128 a_k2_4 = _mm_loadu_ps(&a[126 - j2]); // 124, 125, 126, 127, + const __m128 a_j2_p0 = _mm_shuffle_ps( + a_j2_0, a_j2_4, _MM_SHUFFLE(2, 0, 2, 0)); // 2, 4, 6, 8, + const __m128 a_j2_p1 = _mm_shuffle_ps( + a_j2_0, a_j2_4, _MM_SHUFFLE(3, 1, 3, 1)); // 3, 5, 7, 9, + const __m128 a_k2_p0 = _mm_shuffle_ps( + a_k2_4, a_k2_0, _MM_SHUFFLE(0, 2, 0, 2)); // 126, 124, 122, 120, + const __m128 a_k2_p1 = _mm_shuffle_ps( + a_k2_4, a_k2_0, _MM_SHUFFLE(1, 3, 1, 3)); // 127, 125, 123, 121, + // Calculate 'x'. + const __m128 xr_ = _mm_sub_ps(a_j2_p0, a_k2_p0); + // 2-126, 4-124, 6-122, 8-120, + const __m128 xi_ = _mm_add_ps(a_j2_p1, a_k2_p1); + // 3-127, 5-125, 7-123, 9-121, + // Calculate product into 'y'. + // yr = wkr * xr - wki * xi; + // yi = wkr * xi + wki * xr; + const __m128 a_ = _mm_mul_ps(wkr_, xr_); + const __m128 b_ = _mm_mul_ps(wki_, xi_); + const __m128 c_ = _mm_mul_ps(wkr_, xi_); + const __m128 d_ = _mm_mul_ps(wki_, xr_); + const __m128 yr_ = _mm_sub_ps(a_, b_); // 2-126, 4-124, 6-122, 8-120, + const __m128 yi_ = _mm_add_ps(c_, d_); // 3-127, 5-125, 7-123, 9-121, + // Update 'a'. + // a[j2 + 0] -= yr; + // a[j2 + 1] -= yi; + // a[k2 + 0] += yr; + // a[k2 + 1] -= yi; + const __m128 a_j2_p0n = _mm_sub_ps(a_j2_p0, yr_); // 2, 4, 6, 8, + const __m128 a_j2_p1n = _mm_sub_ps(a_j2_p1, yi_); // 3, 5, 7, 9, + const __m128 a_k2_p0n = _mm_add_ps(a_k2_p0, yr_); // 126, 124, 122, 120, + const __m128 a_k2_p1n = _mm_sub_ps(a_k2_p1, yi_); // 127, 125, 123, 121, + // Shuffle in right order and store. + const __m128 a_j2_0n = _mm_unpacklo_ps(a_j2_p0n, a_j2_p1n); + // 2, 3, 4, 5, + const __m128 a_j2_4n = _mm_unpackhi_ps(a_j2_p0n, a_j2_p1n); + // 6, 7, 8, 9, + const __m128 a_k2_0nt = _mm_unpackhi_ps(a_k2_p0n, a_k2_p1n); + // 122, 123, 120, 121, + const __m128 a_k2_4nt = _mm_unpacklo_ps(a_k2_p0n, a_k2_p1n); + // 126, 127, 124, 125, + const __m128 a_k2_0n = _mm_shuffle_ps( + a_k2_0nt, a_k2_0nt, _MM_SHUFFLE(1, 0, 3, 2)); // 120, 121, 122, 123, + const __m128 a_k2_4n = _mm_shuffle_ps( + a_k2_4nt, a_k2_4nt, _MM_SHUFFLE(1, 0, 3, 2)); // 124, 125, 126, 127, + _mm_storeu_ps(&a[0 + j2], a_j2_0n); + _mm_storeu_ps(&a[4 + j2], a_j2_4n); + _mm_storeu_ps(&a[122 - j2], a_k2_0n); + _mm_storeu_ps(&a[126 - j2], a_k2_4n); + } + // Scalar code for the remaining items. + for (; j2 < 64; j1 += 1, j2 += 2) { + k2 = 128 - j2; + k1 = 32 - j1; + wkr = 0.5f - c[k1]; + wki = c[j1]; + xr = a[j2 + 0] - a[k2 + 0]; + xi = a[j2 + 1] + a[k2 + 1]; + yr = wkr * xr - wki * xi; + yi = wkr * xi + wki * xr; + a[j2 + 0] -= yr; + a[j2 + 1] -= yi; + a[k2 + 0] += yr; + a[k2 + 1] -= yi; + } +} + +void rftbsub_128_SSE2(float* a) { + const float* c = rdft_w + 32; + int j1, j2, k1, k2; + float wkr, wki, xr, xi, yr, yi; + + static const ALIGN16_BEG float ALIGN16_END k_half[4] = {0.5f, 0.5f, 0.5f, + 0.5f}; + const __m128 mm_half = _mm_load_ps(k_half); + + a[1] = -a[1]; + // Vectorized code (four at once). + // Note: commented number are indexes for the first iteration of the loop. + for (j1 = 1, j2 = 2; j2 + 7 < 64; j1 += 4, j2 += 8) { + // Load 'wk'. + const __m128 c_j1 = _mm_loadu_ps(&c[j1]); // 1, 2, 3, 4, + const __m128 c_k1 = _mm_loadu_ps(&c[29 - j1]); // 28, 29, 30, 31, + const __m128 wkrt = _mm_sub_ps(mm_half, c_k1); // 28, 29, 30, 31, + const __m128 wkr_ = + _mm_shuffle_ps(wkrt, wkrt, _MM_SHUFFLE(0, 1, 2, 3)); // 31, 30, 29, 28, + const __m128 wki_ = c_j1; // 1, 2, 3, 4, + // Load and shuffle 'a'. + const __m128 a_j2_0 = _mm_loadu_ps(&a[0 + j2]); // 2, 3, 4, 5, + const __m128 a_j2_4 = _mm_loadu_ps(&a[4 + j2]); // 6, 7, 8, 9, + const __m128 a_k2_0 = _mm_loadu_ps(&a[122 - j2]); // 120, 121, 122, 123, + const __m128 a_k2_4 = _mm_loadu_ps(&a[126 - j2]); // 124, 125, 126, 127, + const __m128 a_j2_p0 = _mm_shuffle_ps( + a_j2_0, a_j2_4, _MM_SHUFFLE(2, 0, 2, 0)); // 2, 4, 6, 8, + const __m128 a_j2_p1 = _mm_shuffle_ps( + a_j2_0, a_j2_4, _MM_SHUFFLE(3, 1, 3, 1)); // 3, 5, 7, 9, + const __m128 a_k2_p0 = _mm_shuffle_ps( + a_k2_4, a_k2_0, _MM_SHUFFLE(0, 2, 0, 2)); // 126, 124, 122, 120, + const __m128 a_k2_p1 = _mm_shuffle_ps( + a_k2_4, a_k2_0, _MM_SHUFFLE(1, 3, 1, 3)); // 127, 125, 123, 121, + // Calculate 'x'. + const __m128 xr_ = _mm_sub_ps(a_j2_p0, a_k2_p0); + // 2-126, 4-124, 6-122, 8-120, + const __m128 xi_ = _mm_add_ps(a_j2_p1, a_k2_p1); + // 3-127, 5-125, 7-123, 9-121, + // Calculate product into 'y'. + // yr = wkr * xr + wki * xi; + // yi = wkr * xi - wki * xr; + const __m128 a_ = _mm_mul_ps(wkr_, xr_); + const __m128 b_ = _mm_mul_ps(wki_, xi_); + const __m128 c_ = _mm_mul_ps(wkr_, xi_); + const __m128 d_ = _mm_mul_ps(wki_, xr_); + const __m128 yr_ = _mm_add_ps(a_, b_); // 2-126, 4-124, 6-122, 8-120, + const __m128 yi_ = _mm_sub_ps(c_, d_); // 3-127, 5-125, 7-123, 9-121, + // Update 'a'. + // a[j2 + 0] = a[j2 + 0] - yr; + // a[j2 + 1] = yi - a[j2 + 1]; + // a[k2 + 0] = yr + a[k2 + 0]; + // a[k2 + 1] = yi - a[k2 + 1]; + const __m128 a_j2_p0n = _mm_sub_ps(a_j2_p0, yr_); // 2, 4, 6, 8, + const __m128 a_j2_p1n = _mm_sub_ps(yi_, a_j2_p1); // 3, 5, 7, 9, + const __m128 a_k2_p0n = _mm_add_ps(a_k2_p0, yr_); // 126, 124, 122, 120, + const __m128 a_k2_p1n = _mm_sub_ps(yi_, a_k2_p1); // 127, 125, 123, 121, + // Shuffle in right order and store. + const __m128 a_j2_0n = _mm_unpacklo_ps(a_j2_p0n, a_j2_p1n); + // 2, 3, 4, 5, + const __m128 a_j2_4n = _mm_unpackhi_ps(a_j2_p0n, a_j2_p1n); + // 6, 7, 8, 9, + const __m128 a_k2_0nt = _mm_unpackhi_ps(a_k2_p0n, a_k2_p1n); + // 122, 123, 120, 121, + const __m128 a_k2_4nt = _mm_unpacklo_ps(a_k2_p0n, a_k2_p1n); + // 126, 127, 124, 125, + const __m128 a_k2_0n = _mm_shuffle_ps( + a_k2_0nt, a_k2_0nt, _MM_SHUFFLE(1, 0, 3, 2)); // 120, 121, 122, 123, + const __m128 a_k2_4n = _mm_shuffle_ps( + a_k2_4nt, a_k2_4nt, _MM_SHUFFLE(1, 0, 3, 2)); // 124, 125, 126, 127, + _mm_storeu_ps(&a[0 + j2], a_j2_0n); + _mm_storeu_ps(&a[4 + j2], a_j2_4n); + _mm_storeu_ps(&a[122 - j2], a_k2_0n); + _mm_storeu_ps(&a[126 - j2], a_k2_4n); + } + // Scalar code for the remaining items. + for (; j2 < 64; j1 += 1, j2 += 2) { + k2 = 128 - j2; + k1 = 32 - j1; + wkr = 0.5f - c[k1]; + wki = c[j1]; + xr = a[j2 + 0] - a[k2 + 0]; + xi = a[j2 + 1] + a[k2 + 1]; + yr = wkr * xr + wki * xi; + yi = wkr * xi - wki * xr; + a[j2 + 0] = a[j2 + 0] - yr; + a[j2 + 1] = yi - a[j2 + 1]; + a[k2 + 0] = yr + a[k2 + 0]; + a[k2 + 1] = yi - a[k2 + 1]; + } + a[65] = -a[65]; +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_common.h b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_common.h new file mode 100644 index 0000000000..47d076ea2a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_common.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_TABLES_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_TABLES_COMMON_H_ + +#include "modules/audio_processing/utility/ooura_fft.h" + +namespace webrtc { + +// This tables used to be computed at run-time. For example, refer to: +// https://code.google.com/p/webrtc/source/browse/trunk/webrtc/modules/audio_processing/utility/apm_rdft.c?r=6564 +// to see the initialization code. +// Constants shared by all paths (C, SSE2, NEON). +const float rdft_w[64] = { + 1.0000000000f, 0.0000000000f, 0.7071067691f, 0.7071067691f, 0.9238795638f, + 0.3826834559f, 0.3826834559f, 0.9238795638f, 0.9807852507f, 0.1950903237f, + 0.5555702448f, 0.8314695954f, 0.8314695954f, 0.5555702448f, 0.1950903237f, + 0.9807852507f, 0.9951847196f, 0.0980171412f, 0.6343933344f, 0.7730104327f, + 0.8819212914f, 0.4713967443f, 0.2902846634f, 0.9569403529f, 0.9569403529f, + 0.2902846634f, 0.4713967443f, 0.8819212914f, 0.7730104327f, 0.6343933344f, + 0.0980171412f, 0.9951847196f, 0.7071067691f, 0.4993977249f, 0.4975923598f, + 0.4945882559f, 0.4903926253f, 0.4850156307f, 0.4784701765f, 0.4707720280f, + 0.4619397819f, 0.4519946277f, 0.4409606457f, 0.4288643003f, 0.4157347977f, + 0.4016037583f, 0.3865052164f, 0.3704755902f, 0.3535533845f, 0.3357794881f, + 0.3171966672f, 0.2978496552f, 0.2777851224f, 0.2570513785f, 0.2356983721f, + 0.2137775421f, 0.1913417280f, 0.1684449315f, 0.1451423317f, 0.1214900985f, + 0.0975451618f, 0.0733652338f, 0.0490085706f, 0.0245338380f, +}; + +// Constants used by the C and MIPS paths. +const float rdft_wk3ri_first[16] = { + 1.000000000f, 0.000000000f, 0.382683456f, 0.923879564f, + 0.831469536f, 0.555570245f, -0.195090353f, 0.980785251f, + 0.956940353f, 0.290284693f, 0.098017156f, 0.995184720f, + 0.634393334f, 0.773010492f, -0.471396863f, 0.881921172f, +}; +const float rdft_wk3ri_second[16] = { + -0.707106769f, 0.707106769f, -0.923879564f, -0.382683456f, + -0.980785251f, 0.195090353f, -0.555570245f, -0.831469536f, + -0.881921172f, 0.471396863f, -0.773010492f, -0.634393334f, + -0.995184720f, -0.098017156f, -0.290284693f, -0.956940353f, +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_TABLES_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_neon_sse2.h b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_neon_sse2.h new file mode 100644 index 0000000000..1c44ae7197 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/utility/ooura_fft_tables_neon_sse2.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_TABLES_NEON_SSE2_H_ +#define MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_TABLES_NEON_SSE2_H_ + +#include "modules/audio_processing/utility/ooura_fft.h" + +#ifdef _MSC_VER /* visual c++ */ +#define ALIGN16_BEG __declspec(align(16)) +#define ALIGN16_END +#else /* gcc or icc */ +#define ALIGN16_BEG +#define ALIGN16_END __attribute__((aligned(16))) +#endif + +namespace webrtc { + +// These tables used to be computed at run-time. For example, refer to: +// https://code.google.com/p/webrtc/source/browse/trunk/webrtc/modules/audio_processing/utility/apm_rdft.c?r=6564 +// to see the initialization code. +#if defined(WEBRTC_ARCH_X86_FAMILY) || defined(WEBRTC_HAS_NEON) +// Constants used by SSE2 and NEON but initialized in the C path. +const ALIGN16_BEG float ALIGN16_END k_swap_sign[4] = {-1.f, 1.f, -1.f, 1.f}; + +ALIGN16_BEG const float ALIGN16_END rdft_wk1r[32] = { + 1.000000000f, 1.000000000f, 0.707106769f, 0.707106769f, 0.923879564f, + 0.923879564f, 0.382683456f, 0.382683456f, 0.980785251f, 0.980785251f, + 0.555570245f, 0.555570245f, 0.831469595f, 0.831469595f, 0.195090324f, + 0.195090324f, 0.995184720f, 0.995184720f, 0.634393334f, 0.634393334f, + 0.881921291f, 0.881921291f, 0.290284663f, 0.290284663f, 0.956940353f, + 0.956940353f, 0.471396744f, 0.471396744f, 0.773010433f, 0.773010433f, + 0.098017141f, 0.098017141f, +}; +ALIGN16_BEG const float ALIGN16_END rdft_wk2r[32] = { + 1.000000000f, 1.000000000f, -0.000000000f, -0.000000000f, 0.707106769f, + 0.707106769f, -0.707106769f, -0.707106769f, 0.923879564f, 0.923879564f, + -0.382683456f, -0.382683456f, 0.382683456f, 0.382683456f, -0.923879564f, + -0.923879564f, 0.980785251f, 0.980785251f, -0.195090324f, -0.195090324f, + 0.555570245f, 0.555570245f, -0.831469595f, -0.831469595f, 0.831469595f, + 0.831469595f, -0.555570245f, -0.555570245f, 0.195090324f, 0.195090324f, + -0.980785251f, -0.980785251f, +}; +ALIGN16_BEG const float ALIGN16_END rdft_wk3r[32] = { + 1.000000000f, 1.000000000f, -0.707106769f, -0.707106769f, 0.382683456f, + 0.382683456f, -0.923879564f, -0.923879564f, 0.831469536f, 0.831469536f, + -0.980785251f, -0.980785251f, -0.195090353f, -0.195090353f, -0.555570245f, + -0.555570245f, 0.956940353f, 0.956940353f, -0.881921172f, -0.881921172f, + 0.098017156f, 0.098017156f, -0.773010492f, -0.773010492f, 0.634393334f, + 0.634393334f, -0.995184720f, -0.995184720f, -0.471396863f, -0.471396863f, + -0.290284693f, -0.290284693f, +}; +ALIGN16_BEG const float ALIGN16_END rdft_wk1i[32] = { + -0.000000000f, 0.000000000f, -0.707106769f, 0.707106769f, -0.382683456f, + 0.382683456f, -0.923879564f, 0.923879564f, -0.195090324f, 0.195090324f, + -0.831469595f, 0.831469595f, -0.555570245f, 0.555570245f, -0.980785251f, + 0.980785251f, -0.098017141f, 0.098017141f, -0.773010433f, 0.773010433f, + -0.471396744f, 0.471396744f, -0.956940353f, 0.956940353f, -0.290284663f, + 0.290284663f, -0.881921291f, 0.881921291f, -0.634393334f, 0.634393334f, + -0.995184720f, 0.995184720f, +}; +ALIGN16_BEG const float ALIGN16_END rdft_wk2i[32] = { + -0.000000000f, 0.000000000f, -1.000000000f, 1.000000000f, -0.707106769f, + 0.707106769f, -0.707106769f, 0.707106769f, -0.382683456f, 0.382683456f, + -0.923879564f, 0.923879564f, -0.923879564f, 0.923879564f, -0.382683456f, + 0.382683456f, -0.195090324f, 0.195090324f, -0.980785251f, 0.980785251f, + -0.831469595f, 0.831469595f, -0.555570245f, 0.555570245f, -0.555570245f, + 0.555570245f, -0.831469595f, 0.831469595f, -0.980785251f, 0.980785251f, + -0.195090324f, 0.195090324f, +}; +ALIGN16_BEG const float ALIGN16_END rdft_wk3i[32] = { + -0.000000000f, 0.000000000f, -0.707106769f, 0.707106769f, -0.923879564f, + 0.923879564f, 0.382683456f, -0.382683456f, -0.555570245f, 0.555570245f, + -0.195090353f, 0.195090353f, -0.980785251f, 0.980785251f, 0.831469536f, + -0.831469536f, -0.290284693f, 0.290284693f, -0.471396863f, 0.471396863f, + -0.995184720f, 0.995184720f, 0.634393334f, -0.634393334f, -0.773010492f, + 0.773010492f, 0.098017156f, -0.098017156f, -0.881921172f, 0.881921172f, + 0.956940353f, -0.956940353f, +}; +ALIGN16_BEG const float ALIGN16_END cftmdl_wk1r[4] = { + 0.707106769f, 0.707106769f, 0.707106769f, -0.707106769f, +}; +#endif + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_UTILITY_OOURA_FFT_TABLES_NEON_SSE2_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/common.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/common.h new file mode 100644 index 0000000000..b5a5fb385b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/common.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_VAD_COMMON_H_ + +#include + +static const int kSampleRateHz = 16000; +static const size_t kLength10Ms = kSampleRateHz / 100; +static const size_t kMaxNumFrames = 4; + +struct AudioFeatures { + double log_pitch_gain[kMaxNumFrames]; + double pitch_lag_hz[kMaxNumFrames]; + double spectral_peak[kMaxNumFrames]; + double rms[kMaxNumFrames]; + size_t num_frames; + bool silence; +}; + +#endif // MODULES_AUDIO_PROCESSING_VAD_COMMON_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.cc new file mode 100644 index 0000000000..266ca44cc9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.cc @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/gmm.h" + +#include +#include + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +static const int kMaxDimension = 10; + +static void RemoveMean(const double* in, + const double* mean_vec, + int dimension, + double* out) { + for (int n = 0; n < dimension; ++n) + out[n] = in[n] - mean_vec[n]; +} + +static double ComputeExponent(const double* in, + const double* covar_inv, + int dimension) { + double q = 0; + for (int i = 0; i < dimension; ++i) { + double v = 0; + for (int j = 0; j < dimension; j++) + v += (*covar_inv++) * in[j]; + q += v * in[i]; + } + q *= -0.5; + return q; +} + +double EvaluateGmm(const double* x, const GmmParameters& gmm_parameters) { + if (gmm_parameters.dimension > kMaxDimension) { + return -1; // This is invalid pdf so the caller can check this. + } + double f = 0; + double v[kMaxDimension]; + const double* mean_vec = gmm_parameters.mean; + const double* covar_inv = gmm_parameters.covar_inverse; + + for (int n = 0; n < gmm_parameters.num_mixtures; n++) { + RemoveMean(x, mean_vec, gmm_parameters.dimension, v); + double q = ComputeExponent(v, covar_inv, gmm_parameters.dimension) + + gmm_parameters.weight[n]; + f += exp(q); + mean_vec += gmm_parameters.dimension; + covar_inv += gmm_parameters.dimension * gmm_parameters.dimension; + } + return f; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.h new file mode 100644 index 0000000000..93eb675c46 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_GMM_H_ +#define MODULES_AUDIO_PROCESSING_VAD_GMM_H_ + +namespace webrtc { + +// A structure that specifies a GMM. +// A GMM is formulated as +// f(x) = w[0] * mixture[0] + w[1] * mixture[1] + ... + +// w[num_mixtures - 1] * mixture[num_mixtures - 1]; +// Where a 'mixture' is a Gaussian density. + +struct GmmParameters { + // weight[n] = log(w[n]) - |dimension|/2 * log(2*pi) - 1/2 * log(det(cov[n])); + // where cov[n] is the covariance matrix of mixture n; + const double* weight; + // pointer to the first element of a |num_mixtures|x|dimension| matrix + // where kth row is the mean of the kth mixture. + const double* mean; + // pointer to the first element of a |num_mixtures|x|dimension|x|dimension| + // 3D-matrix, where the kth 2D-matrix is the inverse of the covariance + // matrix of the kth mixture. + const double* covar_inverse; + // Dimensionality of the mixtures. + int dimension; + // number of the mixtures. + int num_mixtures; +}; + +// Evaluate the given GMM, according to |gmm_parameters|, at the given point +// |x|. If the dimensionality of the given GMM is larger that the maximum +// acceptable dimension by the following function -1 is returned. +double EvaluateGmm(const double* x, const GmmParameters& gmm_parameters); + +} // namespace webrtc +#endif // MODULES_AUDIO_PROCESSING_VAD_GMM_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm_unittest.cc new file mode 100644 index 0000000000..dfc8855a14 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/gmm_unittest.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/gmm.h" + +#include + +#include "modules/audio_processing/vad/noise_gmm_tables.h" +#include "modules/audio_processing/vad/voice_gmm_tables.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(GmmTest, EvaluateGmm) { + GmmParameters noise_gmm; + GmmParameters voice_gmm; + + // Setup noise GMM. + noise_gmm.dimension = kNoiseGmmDim; + noise_gmm.num_mixtures = kNoiseGmmNumMixtures; + noise_gmm.weight = kNoiseGmmWeights; + noise_gmm.mean = &kNoiseGmmMean[0][0]; + noise_gmm.covar_inverse = &kNoiseGmmCovarInverse[0][0][0]; + + // Setup voice GMM. + voice_gmm.dimension = kVoiceGmmDim; + voice_gmm.num_mixtures = kVoiceGmmNumMixtures; + voice_gmm.weight = kVoiceGmmWeights; + voice_gmm.mean = &kVoiceGmmMean[0][0]; + voice_gmm.covar_inverse = &kVoiceGmmCovarInverse[0][0][0]; + + // Test vectors. These are the mean of the GMM means. + const double kXVoice[kVoiceGmmDim] = { + -1.35893162459863, 602.862491970368, 178.022069191324}; + const double kXNoise[kNoiseGmmDim] = { + -2.33443722724409, 2827.97828765184, 141.114178166812}; + + // Expected pdf values. These values are computed in MATLAB using EvalGmm.m + const double kPdfNoise = 1.88904409403101e-07; + const double kPdfVoice = 1.30453996982266e-06; + + // Relative error should be smaller that the following value. + const double kAcceptedRelativeErr = 1e-10; + + // Test Voice. + double pdf = EvaluateGmm(kXVoice, voice_gmm); + EXPECT_GT(pdf, 0); + double relative_error = fabs(pdf - kPdfVoice) / kPdfVoice; + EXPECT_LE(relative_error, kAcceptedRelativeErr); + + // Test Noise. + pdf = EvaluateGmm(kXNoise, noise_gmm); + EXPECT_GT(pdf, 0); + relative_error = fabs(pdf - kPdfNoise) / kPdfNoise; + EXPECT_LE(relative_error, kAcceptedRelativeErr); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/noise_gmm_tables.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/noise_gmm_tables.h new file mode 100644 index 0000000000..c07dadebd5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/noise_gmm_tables.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// GMM tables for inactive segments. Generated by MakeGmmTables.m. + +#ifndef MODULES_AUDIO_PROCESSING_VAD_NOISE_GMM_TABLES_H_ +#define MODULES_AUDIO_PROCESSING_VAD_NOISE_GMM_TABLES_H_ + +static const int kNoiseGmmNumMixtures = 12; +static const int kNoiseGmmDim = 3; + +static const double + kNoiseGmmCovarInverse[kNoiseGmmNumMixtures][kNoiseGmmDim][kNoiseGmmDim] = { + {{7.36219567592941e+00, 4.83060785179861e-03, 1.23335151497610e-02}, + {4.83060785179861e-03, 1.65289507047817e-04, -2.41490588169997e-04}, + {1.23335151497610e-02, -2.41490588169997e-04, 6.59472060689382e-03}}, + {{8.70265239309140e+00, -5.30636201431086e-04, 5.44014966585347e-03}, + {-5.30636201431086e-04, 3.11095453521008e-04, -1.86287206836035e-04}, + {5.44014966585347e-03, -1.86287206836035e-04, 6.29493388790744e-04}}, + {{4.53467851955055e+00, -3.92977536695197e-03, -2.46521420693317e-03}, + {-3.92977536695197e-03, 4.94650752632750e-05, -1.08587438501826e-05}, + {-2.46521420693317e-03, -1.08587438501826e-05, 9.28793975422261e-05}}, + {{9.26817997114275e-01, -4.03976069276753e-04, -3.56441427392165e-03}, + {-4.03976069276753e-04, 2.51976251631430e-06, 1.46914206734572e-07}, + {-3.56441427392165e-03, 1.46914206734572e-07, 8.19914567685373e-05}}, + {{7.61715986787441e+00, -1.54889041216888e-04, 2.41756280071656e-02}, + {-1.54889041216888e-04, 3.50282550461672e-07, -6.27251196972490e-06}, + {2.41756280071656e-02, -6.27251196972490e-06, 1.45061847649872e-02}}, + {{8.31193642663158e+00, -3.84070508164323e-04, -3.09750630821876e-02}, + {-3.84070508164323e-04, 3.80433432277336e-07, -1.14321142836636e-06}, + {-3.09750630821876e-02, -1.14321142836636e-06, 8.35091486289997e-04}}, + {{9.67283151270894e-01, 5.82465812445039e-05, -3.18350798617053e-03}, + {5.82465812445039e-05, 2.23762672000318e-07, -7.74196587408623e-07}, + {-3.18350798617053e-03, -7.74196587408623e-07, 3.85120938338325e-04}}, + {{8.28066236985388e+00, 5.87634508319763e-05, 6.99303090891743e-03}, + {5.87634508319763e-05, 2.93746018618058e-07, 3.40843332882272e-07}, + {6.99303090891743e-03, 3.40843332882272e-07, 1.99379171190344e-04}}, + {{6.07488998675646e+00, -1.11494526618473e-02, 5.10013111123381e-03}, + {-1.11494526618473e-02, 6.99238879921751e-04, 5.36718550370870e-05}, + {5.10013111123381e-03, 5.36718550370870e-05, 5.26909853276753e-04}}, + {{6.90492021419175e+00, 4.20639355257863e-04, -2.38612752336481e-03}, + {4.20639355257863e-04, 3.31246767338153e-06, -2.42052288150859e-08}, + {-2.38612752336481e-03, -2.42052288150859e-08, 4.46608368363412e-04}}, + {{1.31069150869715e+01, -1.73718583865670e-04, -1.97591814508578e-02}, + {-1.73718583865670e-04, 2.80451716300124e-07, 9.96570755379865e-07}, + {-1.97591814508578e-02, 9.96570755379865e-07, 2.41361900868847e-03}}, + {{4.69566344239814e+00, -2.61077567563690e-04, 5.26359000761433e-03}, + {-2.61077567563690e-04, 1.82420859823767e-06, -7.83645887541601e-07}, + {5.26359000761433e-03, -7.83645887541601e-07, 1.33586288288802e-02}}}; + +static const double kNoiseGmmMean[kNoiseGmmNumMixtures][kNoiseGmmDim] = { + {-2.01386094766163e+00, 1.69702162045397e+02, 7.41715804872181e+01}, + {-1.94684591777290e+00, 1.42398396732668e+02, 1.64186321157831e+02}, + {-2.29319297562437e+00, 3.86415425589868e+02, 2.13452215267125e+02}, + {-3.25487177070268e+00, 1.08668712553616e+03, 2.33119949467419e+02}, + {-2.13159632447467e+00, 4.83821702557717e+03, 6.86786166673740e+01}, + {-2.26171410780526e+00, 4.79420193982422e+03, 1.53222513286450e+02}, + {-3.32166740703185e+00, 4.35161135834358e+03, 1.33206448431316e+02}, + {-2.19290322814343e+00, 3.98325506609408e+03, 2.13249167359934e+02}, + {-2.02898459255404e+00, 7.37039893155007e+03, 1.12518527491926e+02}, + {-2.26150236399500e+00, 1.54896745196145e+03, 1.49717357868579e+02}, + {-2.00417668301790e+00, 3.82434760310304e+03, 1.07438913004312e+02}, + {-2.30193040814533e+00, 1.43953696546439e+03, 7.04085275122649e+01}}; + +static const double kNoiseGmmWeights[kNoiseGmmNumMixtures] = { + -1.09422832086193e+01, + -1.10847897513425e+01, + -1.36767587732187e+01, + -1.79789356118641e+01, + -1.42830169160894e+01, + -1.56500228061379e+01, + -1.83124990950113e+01, + -1.69979436177477e+01, + -1.12329424387828e+01, + -1.41311785780639e+01, + -1.47171861448585e+01, + -1.35963362781839e+01}; +#endif // MODULES_AUDIO_PROCESSING_VAD_NOISE_GMM_TABLES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.cc new file mode 100644 index 0000000000..bca2552c35 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pitch_based_vad.h" + +#include +#include + +#include "modules/audio_processing/vad/vad_circular_buffer.h" +#include "modules/audio_processing/vad/common.h" +#include "modules/audio_processing/vad/noise_gmm_tables.h" +#include "modules/audio_processing/vad/voice_gmm_tables.h" +#include "modules/include/module_common_types.h" + +namespace webrtc { + +static_assert(kNoiseGmmDim == kVoiceGmmDim, + "noise and voice gmm dimension not equal"); + +// These values should match MATLAB counterparts for unit-tests to pass. +static const int kPosteriorHistorySize = 500; // 5 sec of 10 ms frames. +static const double kInitialPriorProbability = 0.3; +static const int kTransientWidthThreshold = 7; +static const double kLowProbabilityThreshold = 0.2; + +static double LimitProbability(double p) { + const double kLimHigh = 0.99; + const double kLimLow = 0.01; + + if (p > kLimHigh) + p = kLimHigh; + else if (p < kLimLow) + p = kLimLow; + return p; +} + +PitchBasedVad::PitchBasedVad() + : p_prior_(kInitialPriorProbability), + circular_buffer_(VadCircularBuffer::Create(kPosteriorHistorySize)) { + // Setup noise GMM. + noise_gmm_.dimension = kNoiseGmmDim; + noise_gmm_.num_mixtures = kNoiseGmmNumMixtures; + noise_gmm_.weight = kNoiseGmmWeights; + noise_gmm_.mean = &kNoiseGmmMean[0][0]; + noise_gmm_.covar_inverse = &kNoiseGmmCovarInverse[0][0][0]; + + // Setup voice GMM. + voice_gmm_.dimension = kVoiceGmmDim; + voice_gmm_.num_mixtures = kVoiceGmmNumMixtures; + voice_gmm_.weight = kVoiceGmmWeights; + voice_gmm_.mean = &kVoiceGmmMean[0][0]; + voice_gmm_.covar_inverse = &kVoiceGmmCovarInverse[0][0][0]; +} + +PitchBasedVad::~PitchBasedVad() { +} + +int PitchBasedVad::VoicingProbability(const AudioFeatures& features, + double* p_combined) { + double p; + double gmm_features[3]; + double pdf_features_given_voice; + double pdf_features_given_noise; + // These limits are the same in matlab implementation 'VoicingProbGMM().' + const double kLimLowLogPitchGain = -2.0; + const double kLimHighLogPitchGain = -0.9; + const double kLimLowSpectralPeak = 200; + const double kLimHighSpectralPeak = 2000; + const double kEps = 1e-12; + for (size_t n = 0; n < features.num_frames; n++) { + gmm_features[0] = features.log_pitch_gain[n]; + gmm_features[1] = features.spectral_peak[n]; + gmm_features[2] = features.pitch_lag_hz[n]; + + pdf_features_given_voice = EvaluateGmm(gmm_features, voice_gmm_); + pdf_features_given_noise = EvaluateGmm(gmm_features, noise_gmm_); + + if (features.spectral_peak[n] < kLimLowSpectralPeak || + features.spectral_peak[n] > kLimHighSpectralPeak || + features.log_pitch_gain[n] < kLimLowLogPitchGain) { + pdf_features_given_voice = kEps * pdf_features_given_noise; + } else if (features.log_pitch_gain[n] > kLimHighLogPitchGain) { + pdf_features_given_noise = kEps * pdf_features_given_voice; + } + + p = p_prior_ * pdf_features_given_voice / + (pdf_features_given_voice * p_prior_ + + pdf_features_given_noise * (1 - p_prior_)); + + p = LimitProbability(p); + + // Combine pitch-based probability with standalone probability, before + // updating prior probabilities. + double prod_active = p * p_combined[n]; + double prod_inactive = (1 - p) * (1 - p_combined[n]); + p_combined[n] = prod_active / (prod_active + prod_inactive); + + if (UpdatePrior(p_combined[n]) < 0) + return -1; + // Limit prior probability. With a zero prior probability the posterior + // probability is always zero. + p_prior_ = LimitProbability(p_prior_); + } + return 0; +} + +int PitchBasedVad::UpdatePrior(double p) { + circular_buffer_->Insert(p); + if (circular_buffer_->RemoveTransient(kTransientWidthThreshold, + kLowProbabilityThreshold) < 0) + return -1; + p_prior_ = circular_buffer_->Mean(); + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.h new file mode 100644 index 0000000000..584dcc73ce --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_PITCH_BASED_VAD_H_ +#define MODULES_AUDIO_PROCESSING_VAD_PITCH_BASED_VAD_H_ + +#include + +#include "modules/audio_processing/vad/common.h" +#include "modules/audio_processing/vad/gmm.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioFrame; +class VadCircularBuffer; + +// Computes the probability of the input audio frame to be active given +// the corresponding pitch-gain and lag of the frame. +class PitchBasedVad { + public: + PitchBasedVad(); + ~PitchBasedVad(); + + // Compute pitch-based voicing probability, given the features. + // features: a structure containing features required for computing voicing + // probabilities. + // + // p_combined: an array which contains the combined activity probabilities + // computed prior to the call of this function. The method, + // then, computes the voicing probabilities and combine them + // with the given values. The result are returned in |p|. + int VoicingProbability(const AudioFeatures& features, double* p_combined); + + private: + int UpdatePrior(double p); + + // TODO(turajs): maybe defining this at a higher level (maybe enum) so that + // all the code recognize it as "no-error." + static const int kNoError = 0; + + GmmParameters noise_gmm_; + GmmParameters voice_gmm_; + + double p_prior_; + + std::unique_ptr circular_buffer_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_PROCESSING_VAD_PITCH_BASED_VAD_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad_unittest.cc new file mode 100644 index 0000000000..fb6daa523e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_based_vad_unittest.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pitch_based_vad.h" + +#include +#include + +#include + +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +TEST(PitchBasedVadTest, VoicingProbabilityTest) { + std::string spectral_peak_file_name = + test::ResourcePath("audio_processing/agc/agc_spectral_peak", "dat"); + FILE* spectral_peak_file = fopen(spectral_peak_file_name.c_str(), "rb"); + ASSERT_TRUE(spectral_peak_file != NULL); + + std::string pitch_gain_file_name = + test::ResourcePath("audio_processing/agc/agc_pitch_gain", "dat"); + FILE* pitch_gain_file = fopen(pitch_gain_file_name.c_str(), "rb"); + ASSERT_TRUE(pitch_gain_file != NULL); + + std::string pitch_lag_file_name = + test::ResourcePath("audio_processing/agc/agc_pitch_lag", "dat"); + FILE* pitch_lag_file = fopen(pitch_lag_file_name.c_str(), "rb"); + ASSERT_TRUE(pitch_lag_file != NULL); + + std::string voicing_prob_file_name = + test::ResourcePath("audio_processing/agc/agc_voicing_prob", "dat"); + FILE* voicing_prob_file = fopen(voicing_prob_file_name.c_str(), "rb"); + ASSERT_TRUE(voicing_prob_file != NULL); + + PitchBasedVad vad_; + + double reference_activity_probability; + + AudioFeatures audio_features; + memset(&audio_features, 0, sizeof(audio_features)); + audio_features.num_frames = 1; + while (fread(audio_features.spectral_peak, + sizeof(audio_features.spectral_peak[0]), 1, + spectral_peak_file) == 1u) { + double p; + ASSERT_EQ(1u, fread(audio_features.log_pitch_gain, + sizeof(audio_features.log_pitch_gain[0]), 1, + pitch_gain_file)); + ASSERT_EQ(1u, + fread(audio_features.pitch_lag_hz, + sizeof(audio_features.pitch_lag_hz[0]), 1, pitch_lag_file)); + ASSERT_EQ(1u, fread(&reference_activity_probability, + sizeof(reference_activity_probability), 1, + voicing_prob_file)); + + p = 0.5; // Initialize to the neutral value for combining probabilities. + EXPECT_EQ(0, vad_.VoicingProbability(audio_features, &p)); + EXPECT_NEAR(p, reference_activity_probability, 0.01); + } + + fclose(spectral_peak_file); + fclose(pitch_gain_file); + fclose(pitch_lag_file); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.cc new file mode 100644 index 0000000000..7e6bd3e616 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.cc @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pitch_internal.h" + +#include + +// A 4-to-3 linear interpolation. +// The interpolation constants are derived as following: +// Input pitch parameters are updated every 7.5 ms. Within a 30-ms interval +// we are interested in pitch parameters of 0-5 ms, 10-15ms and 20-25ms. This is +// like interpolating 4-to-6 and keep the odd samples. +// The reason behind this is that LPC coefficients are computed for the first +// half of each 10ms interval. +static void PitchInterpolation(double old_val, const double* in, double* out) { + out[0] = 1. / 6. * old_val + 5. / 6. * in[0]; + out[1] = 5. / 6. * in[1] + 1. / 6. * in[2]; + out[2] = 0.5 * in[2] + 0.5 * in[3]; +} + +void GetSubframesPitchParameters(int sampling_rate_hz, + double* gains, + double* lags, + int num_in_frames, + int num_out_frames, + double* log_old_gain, + double* old_lag, + double* log_pitch_gain, + double* pitch_lag_hz) { + // Gain interpolation is in log-domain, also returned in log-domain. + for (int n = 0; n < num_in_frames; n++) + gains[n] = log(gains[n] + 1e-12); + + // Interpolate lags and gains. + PitchInterpolation(*log_old_gain, gains, log_pitch_gain); + *log_old_gain = gains[num_in_frames - 1]; + PitchInterpolation(*old_lag, lags, pitch_lag_hz); + *old_lag = lags[num_in_frames - 1]; + + // Convert pitch-lags to Hertz. + for (int n = 0; n < num_out_frames; n++) { + pitch_lag_hz[n] = (sampling_rate_hz) / (pitch_lag_hz[n]); + } +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.h new file mode 100644 index 0000000000..67e0522328 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_PITCH_INTERNAL_H_ +#define MODULES_AUDIO_PROCESSING_VAD_PITCH_INTERNAL_H_ + +// TODO(turajs): Write a description of this function. Also be consistent with +// usage of |sampling_rate_hz| vs |kSamplingFreqHz|. +void GetSubframesPitchParameters(int sampling_rate_hz, + double* gains, + double* lags, + int num_in_frames, + int num_out_frames, + double* log_old_gain, + double* old_lag, + double* log_pitch_gain, + double* pitch_lag_hz); + +#endif // MODULES_AUDIO_PROCESSING_VAD_PITCH_INTERNAL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal_unittest.cc new file mode 100644 index 0000000000..c1fde10008 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pitch_internal_unittest.cc @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pitch_internal.h" + +#include + +#include "test/gtest.h" + +TEST(PitchInternalTest, test) { + const int kSamplingRateHz = 8000; + const int kNumInputParameters = 4; + const int kNumOutputParameters = 3; + // Inputs + double log_old_gain = log(0.5); + double gains[] = {0.6, 0.2, 0.5, 0.4}; + + double old_lag = 70; + double lags[] = {90, 111, 122, 50}; + + // Expected outputs + double expected_log_pitch_gain[] = { + -0.541212549898316, -1.45672279045507, -0.80471895621705}; + double expected_log_old_gain = log(gains[kNumInputParameters - 1]); + + double expected_pitch_lag_hz[] = { + 92.3076923076923, 70.9010339734121, 93.0232558139535}; + double expected_old_lag = lags[kNumInputParameters - 1]; + + double log_pitch_gain[kNumOutputParameters]; + double pitch_lag_hz[kNumInputParameters]; + + GetSubframesPitchParameters(kSamplingRateHz, gains, lags, kNumInputParameters, + kNumOutputParameters, &log_old_gain, &old_lag, + log_pitch_gain, pitch_lag_hz); + + for (int n = 0; n < 3; n++) { + EXPECT_NEAR(pitch_lag_hz[n], expected_pitch_lag_hz[n], 1e-6); + EXPECT_NEAR(log_pitch_gain[n], expected_log_pitch_gain[n], 1e-8); + } + EXPECT_NEAR(old_lag, expected_old_lag, 1e-6); + EXPECT_NEAR(log_old_gain, expected_log_old_gain, 1e-8); +} diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.cc new file mode 100644 index 0000000000..fa56a3c15e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.cc @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pole_zero_filter.h" + +#include +#include +#include + +namespace webrtc { + +PoleZeroFilter* PoleZeroFilter::Create(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator) { + if (order_numerator > kMaxFilterOrder || + order_denominator > kMaxFilterOrder || denominator_coefficients[0] == 0 || + numerator_coefficients == NULL || denominator_coefficients == NULL) + return NULL; + return new PoleZeroFilter(numerator_coefficients, order_numerator, + denominator_coefficients, order_denominator); +} + +PoleZeroFilter::PoleZeroFilter(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator) + : past_input_(), + past_output_(), + numerator_coefficients_(), + denominator_coefficients_(), + order_numerator_(order_numerator), + order_denominator_(order_denominator), + highest_order_(std::max(order_denominator, order_numerator)) { + memcpy(numerator_coefficients_, numerator_coefficients, + sizeof(numerator_coefficients_[0]) * (order_numerator_ + 1)); + memcpy(denominator_coefficients_, denominator_coefficients, + sizeof(denominator_coefficients_[0]) * (order_denominator_ + 1)); + + if (denominator_coefficients_[0] != 1) { + for (size_t n = 0; n <= order_numerator_; n++) + numerator_coefficients_[n] /= denominator_coefficients_[0]; + for (size_t n = 0; n <= order_denominator_; n++) + denominator_coefficients_[n] /= denominator_coefficients_[0]; + } +} + +template +static float FilterArPast(const T* past, size_t order, + const float* coefficients) { + float sum = 0.0f; + size_t past_index = order - 1; + for (size_t k = 1; k <= order; k++, past_index--) + sum += coefficients[k] * past[past_index]; + return sum; +} + +int PoleZeroFilter::Filter(const int16_t* in, + size_t num_input_samples, + float* output) { + if (in == NULL || output == NULL) + return -1; + // This is the typical case, just a memcpy. + const size_t k = std::min(num_input_samples, highest_order_); + size_t n; + for (n = 0; n < k; n++) { + output[n] = in[n] * numerator_coefficients_[0]; + output[n] += FilterArPast(&past_input_[n], order_numerator_, + numerator_coefficients_); + output[n] -= FilterArPast(&past_output_[n], order_denominator_, + denominator_coefficients_); + + past_input_[n + order_numerator_] = in[n]; + past_output_[n + order_denominator_] = output[n]; + } + if (highest_order_ < num_input_samples) { + for (size_t m = 0; n < num_input_samples; n++, m++) { + output[n] = in[n] * numerator_coefficients_[0]; + output[n] += + FilterArPast(&in[m], order_numerator_, numerator_coefficients_); + output[n] -= FilterArPast(&output[m], order_denominator_, + denominator_coefficients_); + } + // Record into the past signal. + memcpy(past_input_, &in[num_input_samples - order_numerator_], + sizeof(in[0]) * order_numerator_); + memcpy(past_output_, &output[num_input_samples - order_denominator_], + sizeof(output[0]) * order_denominator_); + } else { + // Odd case that the length of the input is shorter that filter order. + memmove(past_input_, &past_input_[num_input_samples], + order_numerator_ * sizeof(past_input_[0])); + memmove(past_output_, &past_output_[num_input_samples], + order_denominator_ * sizeof(past_output_[0])); + } + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.h new file mode 100644 index 0000000000..283deecde3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_POLE_ZERO_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_VAD_POLE_ZERO_FILTER_H_ + +#include + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class PoleZeroFilter { + public: + ~PoleZeroFilter() {} + + static PoleZeroFilter* Create(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator); + + int Filter(const int16_t* in, size_t num_input_samples, float* output); + + private: + PoleZeroFilter(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator); + + static const int kMaxFilterOrder = 24; + + int16_t past_input_[kMaxFilterOrder * 2]; + float past_output_[kMaxFilterOrder * 2]; + + float numerator_coefficients_[kMaxFilterOrder + 1]; + float denominator_coefficients_[kMaxFilterOrder + 1]; + + size_t order_numerator_; + size_t order_denominator_; + size_t highest_order_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_POLE_ZERO_FILTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter_unittest.cc new file mode 100644 index 0000000000..aef07a6e44 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/pole_zero_filter_unittest.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pole_zero_filter.h" + +#include +#include + +#include + +#include "modules/audio_processing/vad/vad_audio_proc_internal.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +static const int kInputSamples = 50; + +static const int16_t kInput[kInputSamples] = { + -2136, -7116, 10715, 2464, 3164, 8139, 11393, 24013, -32117, -5544, + -27740, 10181, 14190, -24055, -15912, 17393, 6359, -9950, -13894, 32432, + -23944, 3437, -8381, 19768, 3087, -19795, -5920, 13310, 1407, 3876, + 4059, 3524, -23130, 19121, -27900, -24840, 4089, 21422, -3625, 3015, + -11236, 28856, 13424, 6571, -19761, -6361, 15821, -9469, 29727, 32229}; + +static const float kReferenceOutput[kInputSamples] = { + -2082.230472f, -6878.572941f, 10697.090871f, 2358.373952f, + 2973.936512f, 7738.580650f, 10690.803213f, 22687.091576f, + -32676.684717f, -5879.621684f, -27359.297432f, 10368.735888f, + 13994.584604f, -23676.126249f, -15078.250390f, 17818.253338f, + 6577.743123f, -9498.369315f, -13073.651079f, 32460.026588f, + -23391.849347f, 3953.805667f, -7667.761363f, 19995.153447f, + 3185.575477f, -19207.365160f, -5143.103201f, 13756.317237f, + 1779.654794f, 4142.269755f, 4209.475034f, 3572.991789f, + -22509.089546f, 19307.878964f, -27060.439759f, -23319.042810f, + 5547.685267f, 22312.718676f, -2707.309027f, 3852.358490f, + -10135.510093f, 29241.509970f, 13394.397233f, 6340.721417f, + -19510.207905f, -5908.442086f, 15882.301634f, -9211.335255f, + 29253.056735f, 30874.443046f}; + +class PoleZeroFilterTest : public ::testing::Test { + protected: + PoleZeroFilterTest() + : my_filter_(PoleZeroFilter::Create(kCoeffNumerator, + kFilterOrder, + kCoeffDenominator, + kFilterOrder)) {} + + ~PoleZeroFilterTest() {} + + void FilterSubframes(int num_subframes); + + private: + void TestClean(); + std::unique_ptr my_filter_; +}; + +void PoleZeroFilterTest::FilterSubframes(int num_subframes) { + float output[kInputSamples]; + const int num_subframe_samples = kInputSamples / num_subframes; + EXPECT_EQ(num_subframe_samples * num_subframes, kInputSamples); + + for (int n = 0; n < num_subframes; n++) { + my_filter_->Filter(&kInput[n * num_subframe_samples], num_subframe_samples, + &output[n * num_subframe_samples]); + } + for (int n = 0; n < kInputSamples; n++) { + EXPECT_NEAR(output[n], kReferenceOutput[n], 1); + } +} + +TEST_F(PoleZeroFilterTest, OneSubframe) { + FilterSubframes(1); +} + +TEST_F(PoleZeroFilterTest, TwoSubframes) { + FilterSubframes(2); +} + +TEST_F(PoleZeroFilterTest, FiveSubframes) { + FilterSubframes(5); +} + +TEST_F(PoleZeroFilterTest, TenSubframes) { + FilterSubframes(10); +} + +TEST_F(PoleZeroFilterTest, TwentyFiveSubframes) { + FilterSubframes(25); +} + +TEST_F(PoleZeroFilterTest, FiftySubframes) { + FilterSubframes(50); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.cc new file mode 100644 index 0000000000..004cefebb8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/standalone_vad.h" + +#include "audio/utility/audio_frame_operations.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +static const int kDefaultStandaloneVadMode = 3; + +StandaloneVad::StandaloneVad(VadInst* vad) + : vad_(vad), buffer_(), index_(0), mode_(kDefaultStandaloneVadMode) { +} + +StandaloneVad::~StandaloneVad() { + WebRtcVad_Free(vad_); +} + +StandaloneVad* StandaloneVad::Create() { + VadInst* vad = WebRtcVad_Create(); + if (!vad) + return nullptr; + + int err = WebRtcVad_Init(vad); + err |= WebRtcVad_set_mode(vad, kDefaultStandaloneVadMode); + if (err != 0) { + WebRtcVad_Free(vad); + return nullptr; + } + return new StandaloneVad(vad); +} + +int StandaloneVad::AddAudio(const int16_t* data, size_t length) { + if (length != kLength10Ms) + return -1; + + if (index_ + length > kLength10Ms * kMaxNum10msFrames) + // Reset the buffer if it's full. + // TODO(ajm): Instead, consider just processing every 10 ms frame. Then we + // can forgo the buffering. + index_ = 0; + + memcpy(&buffer_[index_], data, sizeof(int16_t) * length); + index_ += length; + return 0; +} + +int StandaloneVad::GetActivity(double* p, size_t length_p) { + if (index_ == 0) + return -1; + + const size_t num_frames = index_ / kLength10Ms; + if (num_frames > length_p) + return -1; + RTC_DCHECK_EQ(0, WebRtcVad_ValidRateAndFrameLength(kSampleRateHz, index_)); + + int activity = WebRtcVad_Process(vad_, kSampleRateHz, buffer_, index_); + if (activity < 0) + return -1; + else if (activity == 0) + p[0] = 0.01; // Arbitrary but small and non-zero. + else + p[0] = 0.5; // 0.5 is neutral values when combinned by other probabilities. + for (size_t n = 1; n < num_frames; n++) + p[n] = p[0]; + // Reset the buffer to start from the beginning. + index_ = 0; + return activity; +} + +int StandaloneVad::set_mode(int mode) { + if (mode < 0 || mode > 3) + return -1; + if (WebRtcVad_set_mode(vad_, mode) != 0) + return -1; + + mode_ = mode; + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.h new file mode 100644 index 0000000000..b85de0a75a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_STANDALONE_VAD_H_ +#define MODULES_AUDIO_PROCESSING_AGC_STANDALONE_VAD_H_ + +#include "modules/audio_processing/vad/common.h" +#include "common_audio/vad/include/webrtc_vad.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioFrame; + +class StandaloneVad { + public: + static StandaloneVad* Create(); + ~StandaloneVad(); + + // Outputs + // p: a buffer where probabilities are written to. + // length_p: number of elements of |p|. + // + // return value: + // -1: if no audio is stored or VAD returns error. + // 0: in success. + // In case of error the content of |activity| is unchanged. + // + // Note that due to a high false-positive (VAD decision is active while the + // processed audio is just background noise) rate, stand-alone VAD is used as + // a one-sided indicator. The activity probability is 0.5 if the frame is + // classified as active, and the probability is 0.01 if the audio is + // classified as passive. In this way, when probabilities are combined, the + // effect of the stand-alone VAD is neutral if the input is classified as + // active. + int GetActivity(double* p, size_t length_p); + + // Expecting 10 ms of 16 kHz audio to be pushed in. + int AddAudio(const int16_t* data, size_t length); + + // Set aggressiveness of VAD, 0 is the least aggressive and 3 is the most + // aggressive mode. Returns -1 if the input is less than 0 or larger than 3, + // otherwise 0 is returned. + int set_mode(int mode); + // Get the agressiveness of the current VAD. + int mode() const { return mode_; } + + private: + explicit StandaloneVad(VadInst* vad); + + static const size_t kMaxNum10msFrames = 3; + + // TODO(turajs): Is there a way to use scoped-pointer here? + VadInst* vad_; + int16_t buffer_[kMaxNum10msFrames * kLength10Ms]; + size_t index_; + int mode_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC_STANDALONE_VAD_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad_unittest.cc new file mode 100644 index 0000000000..28d1349396 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/standalone_vad_unittest.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/standalone_vad.h" + +#include + +#include + +#include "modules/include/module_common_types.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +TEST(StandaloneVadTest, Api) { + std::unique_ptr vad(StandaloneVad::Create()); + int16_t data[kLength10Ms] = {0}; + + // Valid frame length (for 32 kHz rate), but not what the VAD is expecting. + EXPECT_EQ(-1, vad->AddAudio(data, 320)); + + const size_t kMaxNumFrames = 3; + double p[kMaxNumFrames]; + for (size_t n = 0; n < kMaxNumFrames; n++) + EXPECT_EQ(0, vad->AddAudio(data, kLength10Ms)); + + // Pretend |p| is shorter that it should be. + EXPECT_EQ(-1, vad->GetActivity(p, kMaxNumFrames - 1)); + + EXPECT_EQ(0, vad->GetActivity(p, kMaxNumFrames)); + + // Ask for activity when buffer is empty. + EXPECT_EQ(-1, vad->GetActivity(p, kMaxNumFrames)); + + // Should reset and result in one buffer. + for (size_t n = 0; n < kMaxNumFrames + 1; n++) + EXPECT_EQ(0, vad->AddAudio(data, kLength10Ms)); + EXPECT_EQ(0, vad->GetActivity(p, 1)); + + // Wrong modes + EXPECT_EQ(-1, vad->set_mode(-1)); + EXPECT_EQ(-1, vad->set_mode(4)); + + // Valid mode. + const int kMode = 2; + EXPECT_EQ(0, vad->set_mode(kMode)); + EXPECT_EQ(kMode, vad->mode()); +} + +#if defined(WEBRTC_IOS) +TEST(StandaloneVadTest, DISABLED_ActivityDetection) { +#else +TEST(StandaloneVadTest, ActivityDetection) { +#endif + std::unique_ptr vad(StandaloneVad::Create()); + const size_t kDataLength = kLength10Ms; + int16_t data[kDataLength] = {0}; + + FILE* pcm_file = + fopen(test::ResourcePath("audio_processing/agc/agc_audio", "pcm").c_str(), + "rb"); + ASSERT_TRUE(pcm_file != NULL); + + FILE* reference_file = fopen( + test::ResourcePath("audio_processing/agc/agc_vad", "dat").c_str(), "rb"); + ASSERT_TRUE(reference_file != NULL); + + // Reference activities are prepared with 0 aggressiveness. + ASSERT_EQ(0, vad->set_mode(0)); + + // Stand-alone VAD can operate on 1, 2 or 3 frames of length 10 ms. The + // reference file is created for 30 ms frame. + const int kNumVadFramesToProcess = 3; + int num_frames = 0; + while (fread(data, sizeof(int16_t), kDataLength, pcm_file) == kDataLength) { + vad->AddAudio(data, kDataLength); + num_frames++; + if (num_frames == kNumVadFramesToProcess) { + num_frames = 0; + int referece_activity; + double p[kNumVadFramesToProcess]; + EXPECT_EQ(1u, fread(&referece_activity, sizeof(referece_activity), 1, + reference_file)); + int activity = vad->GetActivity(p, kNumVadFramesToProcess); + EXPECT_EQ(referece_activity, activity); + if (activity != 0) { + // When active, probabilities are set to 0.5. + for (int n = 0; n < kNumVadFramesToProcess; n++) + EXPECT_EQ(0.5, p[n]); + } else { + // When inactive, probabilities are set to 0.01. + for (int n = 0; n < kNumVadFramesToProcess; n++) + EXPECT_EQ(0.01, p[n]); + } + } + } + fclose(reference_file); + fclose(pcm_file); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.cc new file mode 100644 index 0000000000..b1841d0b21 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.cc @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/vad_audio_proc.h" + +#include +#include + +#include "common_audio/fft4g.h" +#include "modules/audio_processing/vad/pitch_internal.h" +#include "modules/audio_processing/vad/pole_zero_filter.h" +#include "modules/audio_processing/vad/vad_audio_proc_internal.h" +#include "rtc_base/checks.h" +extern "C" { +#include "modules/audio_coding/codecs/isac/main/source/codec.h" +#include "modules/audio_coding/codecs/isac/main/source/lpc_analysis.h" +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" +#include "modules/audio_coding/codecs/isac/main/source/structs.h" +} +#include "modules/include/module_common_types.h" + +namespace webrtc { + +// The following structures are declared anonymous in iSAC's structs.h. To +// forward declare them, we use this derived class trick. +struct VadAudioProc::PitchAnalysisStruct : public ::PitchAnalysisStruct {}; +struct VadAudioProc::PreFiltBankstr : public ::PreFiltBankstr {}; + +static constexpr float kFrequencyResolution = + kSampleRateHz / static_cast(VadAudioProc::kDftSize); +static constexpr int kSilenceRms = 5; + +// TODO(turajs): Make a Create or Init for VadAudioProc. +VadAudioProc::VadAudioProc() + : audio_buffer_(), + num_buffer_samples_(kNumPastSignalSamples), + log_old_gain_(-2), + old_lag_(50), // Arbitrary but valid as pitch-lag (in samples). + pitch_analysis_handle_(new PitchAnalysisStruct), + pre_filter_handle_(new PreFiltBankstr), + high_pass_filter_(PoleZeroFilter::Create(kCoeffNumerator, + kFilterOrder, + kCoeffDenominator, + kFilterOrder)) { + static_assert(kNumPastSignalSamples + kNumSubframeSamples == + sizeof(kLpcAnalWin) / sizeof(kLpcAnalWin[0]), + "lpc analysis window incorrect size"); + static_assert(kLpcOrder + 1 == sizeof(kCorrWeight) / sizeof(kCorrWeight[0]), + "correlation weight incorrect size"); + + // TODO(turajs): Are we doing too much in the constructor? + float data[kDftSize]; + // Make FFT to initialize. + ip_[0] = 0; + WebRtc_rdft(kDftSize, 1, data, ip_, w_fft_); + // TODO(turajs): Need to initialize high-pass filter. + + // Initialize iSAC components. + WebRtcIsac_InitPreFilterbank(pre_filter_handle_.get()); + WebRtcIsac_InitPitchAnalysis(pitch_analysis_handle_.get()); +} + +VadAudioProc::~VadAudioProc() { +} + +void VadAudioProc::ResetBuffer() { + memcpy(audio_buffer_, &audio_buffer_[kNumSamplesToProcess], + sizeof(audio_buffer_[0]) * kNumPastSignalSamples); + num_buffer_samples_ = kNumPastSignalSamples; +} + +int VadAudioProc::ExtractFeatures(const int16_t* frame, + size_t length, + AudioFeatures* features) { + features->num_frames = 0; + if (length != kNumSubframeSamples) { + return -1; + } + + // High-pass filter to remove the DC component and very low frequency content. + // We have experienced that this high-pass filtering improves voice/non-voiced + // classification. + if (high_pass_filter_->Filter(frame, kNumSubframeSamples, + &audio_buffer_[num_buffer_samples_]) != 0) { + return -1; + } + + num_buffer_samples_ += kNumSubframeSamples; + if (num_buffer_samples_ < kBufferLength) { + return 0; + } + RTC_DCHECK_EQ(num_buffer_samples_, kBufferLength); + features->num_frames = kNum10msSubframes; + features->silence = false; + + Rms(features->rms, kMaxNumFrames); + for (size_t i = 0; i < kNum10msSubframes; ++i) { + if (features->rms[i] < kSilenceRms) { + // PitchAnalysis can cause NaNs in the pitch gain if it's fed silence. + // Bail out here instead. + features->silence = true; + ResetBuffer(); + return 0; + } + } + + PitchAnalysis(features->log_pitch_gain, features->pitch_lag_hz, + kMaxNumFrames); + FindFirstSpectralPeaks(features->spectral_peak, kMaxNumFrames); + ResetBuffer(); + return 0; +} + +// Computes |kLpcOrder + 1| correlation coefficients. +void VadAudioProc::SubframeCorrelation(double* corr, + size_t length_corr, + size_t subframe_index) { + RTC_DCHECK_GE(length_corr, kLpcOrder + 1); + double windowed_audio[kNumSubframeSamples + kNumPastSignalSamples]; + size_t buffer_index = subframe_index * kNumSubframeSamples; + + for (size_t n = 0; n < kNumSubframeSamples + kNumPastSignalSamples; n++) + windowed_audio[n] = audio_buffer_[buffer_index++] * kLpcAnalWin[n]; + + WebRtcIsac_AutoCorr(corr, windowed_audio, + kNumSubframeSamples + kNumPastSignalSamples, kLpcOrder); +} + +// Compute |kNum10msSubframes| sets of LPC coefficients, one per 10 ms input. +// The analysis window is 15 ms long and it is centered on the first half of +// each 10ms sub-frame. This is equivalent to computing LPC coefficients for the +// first half of each 10 ms subframe. +void VadAudioProc::GetLpcPolynomials(double* lpc, size_t length_lpc) { + RTC_DCHECK_GE(length_lpc, kNum10msSubframes * (kLpcOrder + 1)); + double corr[kLpcOrder + 1]; + double reflec_coeff[kLpcOrder]; + for (size_t i = 0, offset_lpc = 0; i < kNum10msSubframes; + i++, offset_lpc += kLpcOrder + 1) { + SubframeCorrelation(corr, kLpcOrder + 1, i); + corr[0] *= 1.0001; + // This makes Lev-Durb a bit more stable. + for (size_t k = 0; k < kLpcOrder + 1; k++) { + corr[k] *= kCorrWeight[k]; + } + WebRtcIsac_LevDurb(&lpc[offset_lpc], reflec_coeff, corr, kLpcOrder); + } +} + +// Fit a second order curve to these 3 points and find the location of the +// extremum. The points are inverted before curve fitting. +static float QuadraticInterpolation(float prev_val, + float curr_val, + float next_val) { + // Doing the interpolation in |1 / A(z)|^2. + float fractional_index = 0; + next_val = 1.0f / next_val; + prev_val = 1.0f / prev_val; + curr_val = 1.0f / curr_val; + + fractional_index = + -(next_val - prev_val) * 0.5f / (next_val + prev_val - 2.f * curr_val); + RTC_DCHECK_LT(fabs(fractional_index), 1); + return fractional_index; +} + +// 1 / A(z), where A(z) is defined by |lpc| is a model of the spectral envelope +// of the input signal. The local maximum of the spectral envelope corresponds +// with the local minimum of A(z). It saves complexity, as we save one +// inversion. Furthermore, we find the first local maximum of magnitude squared, +// to save on one square root. +void VadAudioProc::FindFirstSpectralPeaks(double* f_peak, + size_t length_f_peak) { + RTC_DCHECK_GE(length_f_peak, kNum10msSubframes); + double lpc[kNum10msSubframes * (kLpcOrder + 1)]; + // For all sub-frames. + GetLpcPolynomials(lpc, kNum10msSubframes * (kLpcOrder + 1)); + + const size_t kNumDftCoefficients = kDftSize / 2 + 1; + float data[kDftSize]; + + for (size_t i = 0; i < kNum10msSubframes; i++) { + // Convert to float with zero pad. + memset(data, 0, sizeof(data)); + for (size_t n = 0; n < kLpcOrder + 1; n++) { + data[n] = static_cast(lpc[i * (kLpcOrder + 1) + n]); + } + // Transform to frequency domain. + WebRtc_rdft(kDftSize, 1, data, ip_, w_fft_); + + size_t index_peak = 0; + float prev_magn_sqr = data[0] * data[0]; + float curr_magn_sqr = data[2] * data[2] + data[3] * data[3]; + float next_magn_sqr; + bool found_peak = false; + for (size_t n = 2; n < kNumDftCoefficients - 1; n++) { + next_magn_sqr = + data[2 * n] * data[2 * n] + data[2 * n + 1] * data[2 * n + 1]; + if (curr_magn_sqr < prev_magn_sqr && curr_magn_sqr < next_magn_sqr) { + found_peak = true; + index_peak = n - 1; + break; + } + prev_magn_sqr = curr_magn_sqr; + curr_magn_sqr = next_magn_sqr; + } + float fractional_index = 0; + if (!found_peak) { + // Checking if |kNumDftCoefficients - 1| is the local minimum. + next_magn_sqr = data[1] * data[1]; + if (curr_magn_sqr < prev_magn_sqr && curr_magn_sqr < next_magn_sqr) { + index_peak = kNumDftCoefficients - 1; + } + } else { + // A peak is found, do a simple quadratic interpolation to get a more + // accurate estimate of the peak location. + fractional_index = + QuadraticInterpolation(prev_magn_sqr, curr_magn_sqr, next_magn_sqr); + } + f_peak[i] = (index_peak + fractional_index) * kFrequencyResolution; + } +} + +// Using iSAC functions to estimate pitch gains & lags. +void VadAudioProc::PitchAnalysis(double* log_pitch_gains, + double* pitch_lags_hz, + size_t length) { + // TODO(turajs): This can be "imported" from iSAC & and the next two + // constants. + RTC_DCHECK_GE(length, kNum10msSubframes); + const int kNumPitchSubframes = 4; + double gains[kNumPitchSubframes]; + double lags[kNumPitchSubframes]; + + const int kNumSubbandFrameSamples = 240; + const int kNumLookaheadSamples = 24; + + float lower[kNumSubbandFrameSamples]; + float upper[kNumSubbandFrameSamples]; + double lower_lookahead[kNumSubbandFrameSamples]; + double upper_lookahead[kNumSubbandFrameSamples]; + double lower_lookahead_pre_filter[kNumSubbandFrameSamples + + kNumLookaheadSamples]; + + // Split signal to lower and upper bands + WebRtcIsac_SplitAndFilterFloat(&audio_buffer_[kNumPastSignalSamples], lower, + upper, lower_lookahead, upper_lookahead, + pre_filter_handle_.get()); + WebRtcIsac_PitchAnalysis(lower_lookahead, lower_lookahead_pre_filter, + pitch_analysis_handle_.get(), lags, gains); + + // Lags are computed on lower-band signal with sampling rate half of the + // input signal. + GetSubframesPitchParameters( + kSampleRateHz / 2, gains, lags, kNumPitchSubframes, kNum10msSubframes, + &log_old_gain_, &old_lag_, log_pitch_gains, pitch_lags_hz); +} + +void VadAudioProc::Rms(double* rms, size_t length_rms) { + RTC_DCHECK_GE(length_rms, kNum10msSubframes); + size_t offset = kNumPastSignalSamples; + for (size_t i = 0; i < kNum10msSubframes; i++) { + rms[i] = 0; + for (size_t n = 0; n < kNumSubframeSamples; n++, offset++) + rms[i] += audio_buffer_[offset] * audio_buffer_[offset]; + rms[i] = sqrt(rms[i] / kNumSubframeSamples); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.h new file mode 100644 index 0000000000..b1441a08c0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_H_ + +#include + +#include "modules/audio_processing/vad/common.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +class AudioFrame; +class PoleZeroFilter; + +class VadAudioProc { + public: + // Forward declare iSAC structs. + struct PitchAnalysisStruct; + struct PreFiltBankstr; + + VadAudioProc(); + ~VadAudioProc(); + + int ExtractFeatures(const int16_t* audio_frame, + size_t length, + AudioFeatures* audio_features); + + static const size_t kDftSize = 512; + + private: + void PitchAnalysis(double* pitch_gains, double* pitch_lags_hz, size_t length); + void SubframeCorrelation(double* corr, + size_t length_corr, + size_t subframe_index); + void GetLpcPolynomials(double* lpc, size_t length_lpc); + void FindFirstSpectralPeaks(double* f_peak, size_t length_f_peak); + void Rms(double* rms, size_t length_rms); + void ResetBuffer(); + + // To compute spectral peak we perform LPC analysis to get spectral envelope. + // For every 30 ms we compute 3 spectral peak there for 3 LPC analysis. + // LPC is computed over 15 ms of windowed audio. For every 10 ms sub-frame + // we need 5 ms of past signal to create the input of LPC analysis. + enum : size_t { + kNumPastSignalSamples = static_cast(kSampleRateHz / 200) + }; + + // TODO(turajs): maybe defining this at a higher level (maybe enum) so that + // all the code recognize it as "no-error." + enum : int { kNoError = 0 }; + + enum : size_t { kNum10msSubframes = 3 }; + enum : size_t { + kNumSubframeSamples = static_cast(kSampleRateHz / 100) + }; + enum : size_t { + // Samples in 30 ms @ given sampling rate. + kNumSamplesToProcess = kNum10msSubframes * kNumSubframeSamples + }; + enum : size_t { + kBufferLength = kNumPastSignalSamples + kNumSamplesToProcess + }; + enum : size_t { kIpLength = kDftSize >> 1 }; + enum : size_t { kWLength = kDftSize >> 1 }; + enum : size_t { kLpcOrder = 16 }; + + size_t ip_[kIpLength]; + float w_fft_[kWLength]; + + // A buffer of 5 ms (past audio) + 30 ms (one iSAC frame ). + float audio_buffer_[kBufferLength]; + size_t num_buffer_samples_; + + double log_old_gain_; + double old_lag_; + + std::unique_ptr pitch_analysis_handle_; + std::unique_ptr pre_filter_handle_; + std::unique_ptr high_pass_filter_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h new file mode 100644 index 0000000000..ab1e63651f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_INTERNAL_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_INTERNAL_H_ + +namespace webrtc { + +// These values should match MATLAB counterparts for unit-tests to pass. +static const double kCorrWeight[] = {1.000000, + 0.985000, + 0.970225, + 0.955672, + 0.941337, + 0.927217, + 0.913308, + 0.899609, + 0.886115, + 0.872823, + 0.859730, + 0.846834, + 0.834132, + 0.821620, + 0.809296, + 0.797156, + 0.785199}; + +static const double kLpcAnalWin[] = { + 0.00000000, 0.01314436, 0.02628645, 0.03942400, 0.05255473, 0.06567639, + 0.07878670, 0.09188339, 0.10496421, 0.11802689, 0.13106918, 0.14408883, + 0.15708358, 0.17005118, 0.18298941, 0.19589602, 0.20876878, 0.22160547, + 0.23440387, 0.24716177, 0.25987696, 0.27254725, 0.28517045, 0.29774438, + 0.31026687, 0.32273574, 0.33514885, 0.34750406, 0.35979922, 0.37203222, + 0.38420093, 0.39630327, 0.40833713, 0.42030043, 0.43219112, 0.44400713, + 0.45574642, 0.46740697, 0.47898676, 0.49048379, 0.50189608, 0.51322164, + 0.52445853, 0.53560481, 0.54665854, 0.55761782, 0.56848075, 0.57924546, + 0.58991008, 0.60047278, 0.61093173, 0.62128512, 0.63153117, 0.64166810, + 0.65169416, 0.66160761, 0.67140676, 0.68108990, 0.69065536, 0.70010148, + 0.70942664, 0.71862923, 0.72770765, 0.73666033, 0.74548573, 0.75418233, + 0.76274862, 0.77118312, 0.77948437, 0.78765094, 0.79568142, 0.80357442, + 0.81132858, 0.81894256, 0.82641504, 0.83374472, 0.84093036, 0.84797069, + 0.85486451, 0.86161063, 0.86820787, 0.87465511, 0.88095122, 0.88709512, + 0.89308574, 0.89892206, 0.90460306, 0.91012776, 0.91549520, 0.92070447, + 0.92575465, 0.93064488, 0.93537432, 0.93994213, 0.94434755, 0.94858979, + 0.95266814, 0.95658189, 0.96033035, 0.96391289, 0.96732888, 0.97057773, + 0.97365889, 0.97657181, 0.97931600, 0.98189099, 0.98429632, 0.98653158, + 0.98859639, 0.99049038, 0.99221324, 0.99376466, 0.99514438, 0.99635215, + 0.99738778, 0.99825107, 0.99894188, 0.99946010, 0.99980562, 0.99997840, + 0.99997840, 0.99980562, 0.99946010, 0.99894188, 0.99825107, 0.99738778, + 0.99635215, 0.99514438, 0.99376466, 0.99221324, 0.99049038, 0.98859639, + 0.98653158, 0.98429632, 0.98189099, 0.97931600, 0.97657181, 0.97365889, + 0.97057773, 0.96732888, 0.96391289, 0.96033035, 0.95658189, 0.95266814, + 0.94858979, 0.94434755, 0.93994213, 0.93537432, 0.93064488, 0.92575465, + 0.92070447, 0.91549520, 0.91012776, 0.90460306, 0.89892206, 0.89308574, + 0.88709512, 0.88095122, 0.87465511, 0.86820787, 0.86161063, 0.85486451, + 0.84797069, 0.84093036, 0.83374472, 0.82641504, 0.81894256, 0.81132858, + 0.80357442, 0.79568142, 0.78765094, 0.77948437, 0.77118312, 0.76274862, + 0.75418233, 0.74548573, 0.73666033, 0.72770765, 0.71862923, 0.70942664, + 0.70010148, 0.69065536, 0.68108990, 0.67140676, 0.66160761, 0.65169416, + 0.64166810, 0.63153117, 0.62128512, 0.61093173, 0.60047278, 0.58991008, + 0.57924546, 0.56848075, 0.55761782, 0.54665854, 0.53560481, 0.52445853, + 0.51322164, 0.50189608, 0.49048379, 0.47898676, 0.46740697, 0.45574642, + 0.44400713, 0.43219112, 0.42030043, 0.40833713, 0.39630327, 0.38420093, + 0.37203222, 0.35979922, 0.34750406, 0.33514885, 0.32273574, 0.31026687, + 0.29774438, 0.28517045, 0.27254725, 0.25987696, 0.24716177, 0.23440387, + 0.22160547, 0.20876878, 0.19589602, 0.18298941, 0.17005118, 0.15708358, + 0.14408883, 0.13106918, 0.11802689, 0.10496421, 0.09188339, 0.07878670, + 0.06567639, 0.05255473, 0.03942400, 0.02628645, 0.01314436, 0.00000000}; + +static const size_t kFilterOrder = 2; +static const float kCoeffNumerator[kFilterOrder + 1] = {0.974827f, + -1.949650f, + 0.974827f}; +static const float kCoeffDenominator[kFilterOrder + 1] = {1.0f, + -1.971999f, + 0.972457f}; + +static_assert(kFilterOrder + 1 == + sizeof(kCoeffNumerator) / sizeof(kCoeffNumerator[0]), + "numerator coefficients incorrect size"); +static_assert(kFilterOrder + 1 == + sizeof(kCoeffDenominator) / sizeof(kCoeffDenominator[0]), + "denominator coefficients incorrect size"); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROCESSING_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_unittest.cc new file mode 100644 index 0000000000..c520257f8e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_audio_proc_unittest.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// We don't test the value of pitch gain and lags as they are created by iSAC +// routines. However, interpolation of pitch-gain and lags is in a separate +// class and has its own unit-test. + +#include "modules/audio_processing/vad/vad_audio_proc.h" + +#include +#include + +#include + +#include "modules/audio_processing/vad/common.h" +#include "modules/include/module_common_types.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { + +TEST(AudioProcessingTest, DISABLED_ComputingFirstSpectralPeak) { + VadAudioProc audioproc; + + std::string peak_file_name = + test::ResourcePath("audio_processing/agc/agc_spectral_peak", "dat"); + FILE* peak_file = fopen(peak_file_name.c_str(), "rb"); + ASSERT_TRUE(peak_file != NULL); + + std::string pcm_file_name = + test::ResourcePath("audio_processing/agc/agc_audio", "pcm"); + FILE* pcm_file = fopen(pcm_file_name.c_str(), "rb"); + ASSERT_TRUE(pcm_file != NULL); + + // Read 10 ms audio in each iteration. + const size_t kDataLength = kLength10Ms; + int16_t data[kDataLength] = {0}; + AudioFeatures features; + double sp[kMaxNumFrames]; + while (fread(data, sizeof(int16_t), kDataLength, pcm_file) == kDataLength) { + audioproc.ExtractFeatures(data, kDataLength, &features); + if (features.num_frames > 0) { + ASSERT_LT(features.num_frames, kMaxNumFrames); + // Read reference values. + const size_t num_frames = features.num_frames; + ASSERT_EQ(num_frames, fread(sp, sizeof(sp[0]), num_frames, peak_file)); + for (size_t n = 0; n < features.num_frames; n++) + EXPECT_NEAR(features.spectral_peak[n], sp[n], 3); + } + } + + fclose(peak_file); + fclose(pcm_file); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.cc new file mode 100644 index 0000000000..3c4d5ad0e0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.cc @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/vad_circular_buffer.h" + +#include + +namespace webrtc { + +VadCircularBuffer::VadCircularBuffer(int buffer_size) + : buffer_(new double[buffer_size]), + is_full_(false), + index_(0), + buffer_size_(buffer_size), + sum_(0) { +} + +VadCircularBuffer::~VadCircularBuffer() { +} + +void VadCircularBuffer::Reset() { + is_full_ = false; + index_ = 0; + sum_ = 0; +} + +VadCircularBuffer* VadCircularBuffer::Create(int buffer_size) { + if (buffer_size <= 0) + return NULL; + return new VadCircularBuffer(buffer_size); +} + +double VadCircularBuffer::Oldest() const { + if (!is_full_) + return buffer_[0]; + else + return buffer_[index_]; +} + +double VadCircularBuffer::Mean() { + double m; + if (is_full_) { + m = sum_ / buffer_size_; + } else { + if (index_ > 0) + m = sum_ / index_; + else + m = 0; + } + return m; +} + +void VadCircularBuffer::Insert(double value) { + if (is_full_) { + sum_ -= buffer_[index_]; + } + sum_ += value; + buffer_[index_] = value; + index_++; + if (index_ >= buffer_size_) { + is_full_ = true; + index_ = 0; + } +} +int VadCircularBuffer::BufferLevel() { + if (is_full_) + return buffer_size_; + return index_; +} + +int VadCircularBuffer::Get(int index, double* value) const { + int err = ConvertToLinearIndex(&index); + if (err < 0) + return -1; + *value = buffer_[index]; + return 0; +} + +int VadCircularBuffer::Set(int index, double value) { + int err = ConvertToLinearIndex(&index); + if (err < 0) + return -1; + + sum_ -= buffer_[index]; + buffer_[index] = value; + sum_ += value; + return 0; +} + +int VadCircularBuffer::ConvertToLinearIndex(int* index) const { + if (*index < 0 || *index >= buffer_size_) + return -1; + + if (!is_full_ && *index >= index_) + return -1; + + *index = index_ - 1 - *index; + if (*index < 0) + *index += buffer_size_; + return 0; +} + +int VadCircularBuffer::RemoveTransient(int width_threshold, + double val_threshold) { + if (!is_full_ && index_ < width_threshold + 2) + return 0; + + int index_1 = 0; + int index_2 = width_threshold + 1; + double v = 0; + if (Get(index_1, &v) < 0) + return -1; + if (v < val_threshold) { + Set(index_1, 0); + int index; + for (index = index_2; index > index_1; index--) { + if (Get(index, &v) < 0) + return -1; + if (v < val_threshold) + break; + } + for (; index > index_1; index--) { + if (Set(index, 0.0) < 0) + return -1; + } + } + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.h new file mode 100644 index 0000000000..46b03d49bf --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_CIRCULAR_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VAD_CIRCULAR_BUFFER_H_ + +#include + +namespace webrtc { + +// A circular buffer tailored to the need of this project. It stores last +// K samples of the input, and keeps track of the mean of the last samples. +// +// It is used in class "PitchBasedActivity" to keep track of posterior +// probabilities in the past few seconds. The posterior probabilities are used +// to recursively update prior probabilities. +class VadCircularBuffer { + public: + static VadCircularBuffer* Create(int buffer_size); + ~VadCircularBuffer(); + + // If buffer is wrapped around. + bool is_full() const { return is_full_; } + // Get the oldest entry in the buffer. + double Oldest() const; + // Insert new value into the buffer. + void Insert(double value); + // Reset buffer, forget the past, start fresh. + void Reset(); + + // The mean value of the elements in the buffer. The return value is zero if + // buffer is empty, i.e. no value is inserted. + double Mean(); + // Remove transients. If the values exceed |val_threshold| for a period + // shorter then or equal to |width_threshold|, then that period is considered + // transient and set to zero. + int RemoveTransient(int width_threshold, double val_threshold); + + private: + explicit VadCircularBuffer(int buffer_size); + // Get previous values. |index = 0| corresponds to the most recent + // insertion. |index = 1| is the one before the most recent insertion, and + // so on. + int Get(int index, double* value) const; + // Set a given position to |value|. |index| is interpreted as above. + int Set(int index, double value); + // Return the number of valid elements in the buffer. + int BufferLevel(); + + // Convert an index with the interpretation as get() method to the + // corresponding linear index. + int ConvertToLinearIndex(int* index) const; + + std::unique_ptr buffer_; + bool is_full_; + int index_; + int buffer_size_; + double sum_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_PROCESSING_VAD_VAD_CIRCULAR_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer_unittest.cc new file mode 100644 index 0000000000..efbd70d9d9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/vad_circular_buffer_unittest.cc @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/vad_circular_buffer.h" + +#include + +#include + +#include "test/gtest.h" + +namespace webrtc { + +static const int kWidthThreshold = 7; +static const double kValThreshold = 1.0; +static const int kLongBuffSize = 100; +static const int kShortBuffSize = 10; + +static void InsertSequentially(int k, VadCircularBuffer* circular_buffer) { + double mean_val; + for (int n = 1; n <= k; n++) { + EXPECT_TRUE(!circular_buffer->is_full()); + circular_buffer->Insert(n); + mean_val = circular_buffer->Mean(); + EXPECT_EQ((n + 1.0) / 2., mean_val); + } +} + +static void Insert(double value, + int num_insertion, + VadCircularBuffer* circular_buffer) { + for (int n = 0; n < num_insertion; n++) + circular_buffer->Insert(value); +} + +static void InsertZeros(int num_zeros, VadCircularBuffer* circular_buffer) { + Insert(0.0, num_zeros, circular_buffer); +} + +TEST(VadCircularBufferTest, GeneralTest) { + std::unique_ptr circular_buffer( + VadCircularBuffer::Create(kShortBuffSize)); + double mean_val; + + // Mean should return zero if nothing is inserted. + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(0.0, mean_val); + InsertSequentially(kShortBuffSize, circular_buffer.get()); + + // Should be full. + EXPECT_TRUE(circular_buffer->is_full()); + // Correct update after being full. + for (int n = 1; n < kShortBuffSize; n++) { + circular_buffer->Insert(n); + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ((kShortBuffSize + 1.) / 2., mean_val); + EXPECT_TRUE(circular_buffer->is_full()); + } + + // Check reset. This should be like starting fresh. + circular_buffer->Reset(); + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(0, mean_val); + InsertSequentially(kShortBuffSize, circular_buffer.get()); + EXPECT_TRUE(circular_buffer->is_full()); +} + +TEST(VadCircularBufferTest, TransientsRemoval) { + std::unique_ptr circular_buffer( + VadCircularBuffer::Create(kLongBuffSize)); + // Let the first transient be in wrap-around. + InsertZeros(kLongBuffSize - kWidthThreshold / 2, circular_buffer.get()); + + double push_val = kValThreshold; + double mean_val; + for (int k = kWidthThreshold; k >= 1; k--) { + Insert(push_val, k, circular_buffer.get()); + circular_buffer->Insert(0); + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(k * push_val / kLongBuffSize, mean_val); + circular_buffer->RemoveTransient(kWidthThreshold, kValThreshold); + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(0, mean_val); + } +} + +TEST(VadCircularBufferTest, TransientDetection) { + std::unique_ptr circular_buffer( + VadCircularBuffer::Create(kLongBuffSize)); + // Let the first transient be in wrap-around. + int num_insertion = kLongBuffSize - kWidthThreshold / 2; + InsertZeros(num_insertion, circular_buffer.get()); + + double push_val = 2; + // This is longer than a transient and shouldn't be removed. + int num_non_zero_elements = kWidthThreshold + 1; + Insert(push_val, num_non_zero_elements, circular_buffer.get()); + + double mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(num_non_zero_elements * push_val / kLongBuffSize, mean_val); + circular_buffer->Insert(0); + EXPECT_EQ(0, + circular_buffer->RemoveTransient(kWidthThreshold, kValThreshold)); + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(num_non_zero_elements * push_val / kLongBuffSize, mean_val); + + // A transient right after a non-transient, should be removed and mean is + // not changed. + num_insertion = 3; + Insert(push_val, num_insertion, circular_buffer.get()); + circular_buffer->Insert(0); + EXPECT_EQ(0, + circular_buffer->RemoveTransient(kWidthThreshold, kValThreshold)); + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(num_non_zero_elements * push_val / kLongBuffSize, mean_val); + + // Last input is larger than threshold, although the sequence is short but + // it shouldn't be considered transient. + Insert(push_val, num_insertion, circular_buffer.get()); + num_non_zero_elements += num_insertion; + EXPECT_EQ(0, + circular_buffer->RemoveTransient(kWidthThreshold, kValThreshold)); + mean_val = circular_buffer->Mean(); + EXPECT_DOUBLE_EQ(num_non_zero_elements * push_val / kLongBuffSize, mean_val); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.cc new file mode 100644 index 0000000000..66a704f6be --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/voice_activity_detector.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +const size_t kNumChannels = 1; + +const double kDefaultVoiceValue = 1.0; +const double kNeutralProbability = 0.5; +const double kLowProbability = 0.01; + +} // namespace + +VoiceActivityDetector::VoiceActivityDetector() + : last_voice_probability_(kDefaultVoiceValue), + standalone_vad_(StandaloneVad::Create()) { +} + +VoiceActivityDetector::~VoiceActivityDetector() = default; + +// Because ISAC has a different chunk length, it updates +// |chunkwise_voice_probabilities_| and |chunkwise_rms_| when there is new data. +// Otherwise it clears them. +void VoiceActivityDetector::ProcessChunk(const int16_t* audio, + size_t length, + int sample_rate_hz) { + RTC_DCHECK_EQ(length, sample_rate_hz / 100); + // Resample to the required rate. + const int16_t* resampled_ptr = audio; + if (sample_rate_hz != kSampleRateHz) { + RTC_CHECK_EQ( + resampler_.ResetIfNeeded(sample_rate_hz, kSampleRateHz, kNumChannels), + 0); + resampler_.Push(audio, length, resampled_, kLength10Ms, length); + resampled_ptr = resampled_; + } + RTC_DCHECK_EQ(length, kLength10Ms); + + // Each chunk needs to be passed into |standalone_vad_|, because internally it + // buffers the audio and processes it all at once when GetActivity() is + // called. + RTC_CHECK_EQ(standalone_vad_->AddAudio(resampled_ptr, length), 0); + + audio_processing_.ExtractFeatures(resampled_ptr, length, &features_); + + chunkwise_voice_probabilities_.resize(features_.num_frames); + chunkwise_rms_.resize(features_.num_frames); + std::copy(features_.rms, features_.rms + chunkwise_rms_.size(), + chunkwise_rms_.begin()); + if (features_.num_frames > 0) { + if (features_.silence) { + // The other features are invalid, so set the voice probabilities to an + // arbitrary low value. + std::fill(chunkwise_voice_probabilities_.begin(), + chunkwise_voice_probabilities_.end(), kLowProbability); + } else { + std::fill(chunkwise_voice_probabilities_.begin(), + chunkwise_voice_probabilities_.end(), kNeutralProbability); + RTC_CHECK_GE( + standalone_vad_->GetActivity(&chunkwise_voice_probabilities_[0], + chunkwise_voice_probabilities_.size()), + 0); + RTC_CHECK_GE(pitch_based_vad_.VoicingProbability( + features_, &chunkwise_voice_probabilities_[0]), + 0); + } + last_voice_probability_ = chunkwise_voice_probabilities_.back(); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.h new file mode 100644 index 0000000000..0079cb2184 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VOICE_ACTIVITY_DETECTOR_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VOICE_ACTIVITY_DETECTOR_H_ + +#include +#include + +#include "common_audio/resampler/include/resampler.h" +#include "modules/audio_processing/vad/vad_audio_proc.h" +#include "modules/audio_processing/vad/common.h" +#include "modules/audio_processing/vad/pitch_based_vad.h" +#include "modules/audio_processing/vad/standalone_vad.h" + +namespace webrtc { + +// A Voice Activity Detector (VAD) that combines the voice probability from the +// StandaloneVad and PitchBasedVad to get a more robust estimation. +class VoiceActivityDetector { + public: + VoiceActivityDetector(); + ~VoiceActivityDetector(); + + // Processes each audio chunk and estimates the voice probability. + void ProcessChunk(const int16_t* audio, size_t length, int sample_rate_hz); + + // Returns a vector of voice probabilities for each chunk. It can be empty for + // some chunks, but it catches up afterwards returning multiple values at + // once. + const std::vector& chunkwise_voice_probabilities() const { + return chunkwise_voice_probabilities_; + } + + // Returns a vector of RMS values for each chunk. It has the same length as + // chunkwise_voice_probabilities(). + const std::vector& chunkwise_rms() const { return chunkwise_rms_; } + + // Returns the last voice probability, regardless of the internal + // implementation, although it has a few chunks of delay. + float last_voice_probability() const { return last_voice_probability_; } + + private: + // TODO(aluebs): Change these to float. + std::vector chunkwise_voice_probabilities_; + std::vector chunkwise_rms_; + + float last_voice_probability_; + + Resampler resampler_; + VadAudioProc audio_processing_; + + std::unique_ptr standalone_vad_; + PitchBasedVad pitch_based_vad_; + + int16_t resampled_[kLength10Ms]; + AudioFeatures features_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_VOICE_ACTIVITY_DETECTOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector_unittest.cc new file mode 100644 index 0000000000..62eda2bd3d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_activity_detector_unittest.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/voice_activity_detector.h" + +#include +#include + +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace { + +const int kStartTimeSec = 16; +const float kMeanSpeechProbability = 0.3f; +const float kMaxNoiseProbability = 0.1f; +const size_t kNumChunks = 300u; +const size_t kNumChunksPerIsacBlock = 3; + +void GenerateNoise(std::vector* data) { + for (size_t i = 0; i < data->size(); ++i) { + // std::rand returns between 0 and RAND_MAX, but this will work because it + // wraps into some random place. + (*data)[i] = std::rand(); + } +} + +} // namespace + +TEST(VoiceActivityDetectorTest, ConstructorSetsDefaultValues) { + const float kDefaultVoiceValue = 1.f; + + VoiceActivityDetector vad; + + std::vector p = vad.chunkwise_voice_probabilities(); + std::vector rms = vad.chunkwise_rms(); + + EXPECT_EQ(p.size(), 0u); + EXPECT_EQ(rms.size(), 0u); + + EXPECT_FLOAT_EQ(vad.last_voice_probability(), kDefaultVoiceValue); +} + +TEST(VoiceActivityDetectorTest, Speech16kHzHasHighVoiceProbabilities) { + const int kSampleRateHz = 16000; + const int kLength10Ms = kSampleRateHz / 100; + + VoiceActivityDetector vad; + + std::vector data(kLength10Ms); + float mean_probability = 0.f; + + FILE* pcm_file = + fopen(test::ResourcePath("audio_processing/transient/audio16kHz", "pcm") + .c_str(), + "rb"); + ASSERT_TRUE(pcm_file != nullptr); + // The silences in the file are skipped to get a more robust voice probability + // for speech. + ASSERT_EQ(fseek(pcm_file, kStartTimeSec * kSampleRateHz * sizeof(data[0]), + SEEK_SET), + 0); + + size_t num_chunks = 0; + while (fread(&data[0], sizeof(data[0]), data.size(), pcm_file) == + data.size()) { + vad.ProcessChunk(&data[0], data.size(), kSampleRateHz); + + mean_probability += vad.last_voice_probability(); + + ++num_chunks; + } + + mean_probability /= num_chunks; + + EXPECT_GT(mean_probability, kMeanSpeechProbability); +} + +TEST(VoiceActivityDetectorTest, Speech32kHzHasHighVoiceProbabilities) { + const int kSampleRateHz = 32000; + const int kLength10Ms = kSampleRateHz / 100; + + VoiceActivityDetector vad; + + std::vector data(kLength10Ms); + float mean_probability = 0.f; + + FILE* pcm_file = + fopen(test::ResourcePath("audio_processing/transient/audio32kHz", "pcm") + .c_str(), + "rb"); + ASSERT_TRUE(pcm_file != nullptr); + // The silences in the file are skipped to get a more robust voice probability + // for speech. + ASSERT_EQ(fseek(pcm_file, kStartTimeSec * kSampleRateHz * sizeof(data[0]), + SEEK_SET), + 0); + + size_t num_chunks = 0; + while (fread(&data[0], sizeof(data[0]), data.size(), pcm_file) == + data.size()) { + vad.ProcessChunk(&data[0], data.size(), kSampleRateHz); + + mean_probability += vad.last_voice_probability(); + + ++num_chunks; + } + + mean_probability /= num_chunks; + + EXPECT_GT(mean_probability, kMeanSpeechProbability); +} + +TEST(VoiceActivityDetectorTest, Noise16kHzHasLowVoiceProbabilities) { + VoiceActivityDetector vad; + + std::vector data(kLength10Ms); + float max_probability = 0.f; + + std::srand(42); + + for (size_t i = 0; i < kNumChunks; ++i) { + GenerateNoise(&data); + + vad.ProcessChunk(&data[0], data.size(), kSampleRateHz); + + // Before the |vad has enough data to process an ISAC block it will return + // the default value, 1.f, which would ruin the |max_probability| value. + if (i > kNumChunksPerIsacBlock) { + max_probability = std::max(max_probability, vad.last_voice_probability()); + } + } + + EXPECT_LT(max_probability, kMaxNoiseProbability); +} + +TEST(VoiceActivityDetectorTest, Noise32kHzHasLowVoiceProbabilities) { + VoiceActivityDetector vad; + + std::vector data(2 * kLength10Ms); + float max_probability = 0.f; + + std::srand(42); + + for (size_t i = 0; i < kNumChunks; ++i) { + GenerateNoise(&data); + + vad.ProcessChunk(&data[0], data.size(), 2 * kSampleRateHz); + + // Before the |vad has enough data to process an ISAC block it will return + // the default value, 1.f, which would ruin the |max_probability| value. + if (i > kNumChunksPerIsacBlock) { + max_probability = std::max(max_probability, vad.last_voice_probability()); + } + } + + EXPECT_LT(max_probability, kMaxNoiseProbability); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_gmm_tables.h b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_gmm_tables.h new file mode 100644 index 0000000000..29cc7d6cb2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/vad/voice_gmm_tables.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// GMM tables for active segments. Generated by MakeGmmTables.m. + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VOICE_GMM_TABLES_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VOICE_GMM_TABLES_H_ + +static const int kVoiceGmmNumMixtures = 12; +static const int kVoiceGmmDim = 3; + +static const double + kVoiceGmmCovarInverse[kVoiceGmmNumMixtures][kVoiceGmmDim][kVoiceGmmDim] = { + {{1.83673825579513e+00, -8.09791637570095e-04, 4.60106414365986e-03}, + {-8.09791637570095e-04, 8.89351738394608e-04, -9.80188953277734e-04}, + {4.60106414365986e-03, -9.80188953277734e-04, 1.38706060206582e-03}}, + {{6.76228912850703e+01, -1.98893120119660e-02, -3.53548357253551e-03}, + {-1.98893120119660e-02, 3.96216858500530e-05, -4.08492938394097e-05}, + {-3.53548357253551e-03, -4.08492938394097e-05, 9.31864352856416e-04}}, + {{9.98612435944558e+00, -5.27880954316893e-03, -6.30342541619017e-03}, + {-5.27880954316893e-03, 4.54359480225226e-05, 6.30804591626044e-05}, + {-6.30342541619017e-03, 6.30804591626044e-05, 5.36466441382942e-04}}, + {{3.39917474216349e+01, -1.56213579433191e-03, -4.01459014990225e-02}, + {-1.56213579433191e-03, 6.40415424897724e-05, 6.20076342427833e-05}, + {-4.01459014990225e-02, 6.20076342427833e-05, 3.51199070103063e-03}}, + {{1.34545062271428e+01, -7.94513610147144e-03, -5.34401019341728e-02}, + {-7.94513610147144e-03, 1.16511820098649e-04, 4.66063702069293e-05}, + {-5.34401019341728e-02, 4.66063702069293e-05, 2.72354323774163e-03}}, + {{1.08557844314806e+02, -1.54885805673668e-02, -1.88029692674851e-02}, + {-1.54885805673668e-02, 1.16404042786406e-04, 6.45579292702802e-06}, + {-1.88029692674851e-02, 6.45579292702802e-06, 4.32330478391416e-04}}, + {{8.22940066541450e+01, -1.15903110231303e-02, -4.92166764865343e-02}, + {-1.15903110231303e-02, 7.42510742165261e-05, 3.73007314191290e-06}, + {-4.92166764865343e-02, 3.73007314191290e-06, 3.64005221593244e-03}}, + {{2.31133605685660e+00, -7.83261568950254e-04, 7.45744012346313e-04}, + {-7.83261568950254e-04, 1.29460648214142e-05, -2.22774455093730e-06}, + {7.45744012346313e-04, -2.22774455093730e-06, 1.05117294093010e-04}}, + {{3.78767849189611e+02, 1.57759761011568e-03, -2.08551217988774e-02}, + {1.57759761011568e-03, 4.76066236886865e-05, -2.33977412299324e-05}, + {-2.08551217988774e-02, -2.33977412299324e-05, 5.24261005371196e-04}}, + {{6.98580096506135e-01, -5.13850255217378e-04, -4.01124551717056e-04}, + {-5.13850255217378e-04, 1.40501021984840e-06, -2.09496928716569e-06}, + {-4.01124551717056e-04, -2.09496928716569e-06, 2.82879357740037e-04}}, + {{2.62770945162399e+00, -2.31825753241430e-03, -5.30447217466318e-03}, + {-2.31825753241430e-03, 4.59108572227649e-05, 7.67631886355405e-05}, + {-5.30447217466318e-03, 7.67631886355405e-05, 2.28521601674098e-03}}, + {{1.89940391362152e+02, -4.23280856852379e-03, -2.70608873541399e-02}, + {-4.23280856852379e-03, 6.77547582742563e-05, 2.69154203800467e-05}, + {-2.70608873541399e-02, 2.69154203800467e-05, 3.88574543373470e-03}}}; + +static const double kVoiceGmmMean[kVoiceGmmNumMixtures][kVoiceGmmDim] = { + {-2.15020241646536e+00, 4.97079062999877e+02, 4.77078119504505e+02}, + {-8.92097680029190e-01, 5.92064964199921e+02, 1.81045145941059e+02}, + {-1.29435784144398e+00, 4.98450293410611e+02, 1.71991263804064e+02}, + {-1.03925228397884e+00, 4.99511274321571e+02, 1.05838336539105e+02}, + {-1.29229047206129e+00, 4.15026762566707e+02, 1.12861119017125e+02}, + {-7.88748114599810e-01, 4.48739336688113e+02, 1.89784216956337e+02}, + {-8.77777402332642e-01, 4.86620285054533e+02, 1.13477708016491e+02}, + {-2.06465957063057e+00, 6.33385049870607e+02, 2.32758546796149e+02}, + {-6.98893789231685e-01, 5.93622051503385e+02, 1.92536982473203e+02}, + {-2.55901217508894e+00, 1.55914919756205e+03, 1.39769980835570e+02}, + {-1.92070024165837e+00, 4.87983940444185e+02, 1.02745468128289e+02}, + {-7.29187507662854e-01, 5.22717685022855e+02, 1.16377942283991e+02}}; + +static const double kVoiceGmmWeights[kVoiceGmmNumMixtures] = { + -1.39789694361035e+01, + -1.19527720202104e+01, + -1.32396317929055e+01, + -1.09436815209238e+01, + -1.13440027478149e+01, + -1.12200721834504e+01, + -1.02537324043693e+01, + -1.60789861938302e+01, + -1.03394494048344e+01, + -1.83207938586818e+01, + -1.31186044948288e+01, + -9.52479998673554e+00}; +#endif // MODULES_AUDIO_PROCESSING_VAD_VOICE_GMM_TABLES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.cc new file mode 100644 index 0000000000..5ee0c7b9c2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.cc @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/voice_detection_impl.h" + +#include "common_audio/vad/include/webrtc_vad.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { +class VoiceDetectionImpl::Vad { + public: + Vad() { + state_ = WebRtcVad_Create(); + RTC_CHECK(state_); + int error = WebRtcVad_Init(state_); + RTC_DCHECK_EQ(0, error); + } + ~Vad() { + WebRtcVad_Free(state_); + } + VadInst* state() { return state_; } + private: + VadInst* state_ = nullptr; + RTC_DISALLOW_COPY_AND_ASSIGN(Vad); +}; + +VoiceDetectionImpl::VoiceDetectionImpl(rtc::CriticalSection* crit) + : crit_(crit) { + RTC_DCHECK(crit); +} + +VoiceDetectionImpl::~VoiceDetectionImpl() {} + +void VoiceDetectionImpl::Initialize(int sample_rate_hz) { + rtc::CritScope cs(crit_); + sample_rate_hz_ = sample_rate_hz; + std::unique_ptr new_vad; + if (enabled_) { + new_vad.reset(new Vad()); + } + vad_.swap(new_vad); + using_external_vad_ = false; + frame_size_samples_ = + static_cast(frame_size_ms_ * sample_rate_hz_) / 1000; + set_likelihood(likelihood_); +} + +void VoiceDetectionImpl::ProcessCaptureAudio(AudioBuffer* audio) { + rtc::CritScope cs(crit_); + if (!enabled_) { + return; + } + if (using_external_vad_) { + using_external_vad_ = false; + return; + } + + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + // TODO(ajm): concatenate data in frame buffer here. + int vad_ret = WebRtcVad_Process(vad_->state(), sample_rate_hz_, + audio->mixed_low_pass_data(), + frame_size_samples_); + if (vad_ret == 0) { + stream_has_voice_ = false; + audio->set_activity(AudioFrame::kVadPassive); + } else if (vad_ret == 1) { + stream_has_voice_ = true; + audio->set_activity(AudioFrame::kVadActive); + } else { + RTC_NOTREACHED(); + } +} + +int VoiceDetectionImpl::Enable(bool enable) { + rtc::CritScope cs(crit_); + if (enabled_ != enable) { + enabled_ = enable; + Initialize(sample_rate_hz_); + } + return AudioProcessing::kNoError; +} + +bool VoiceDetectionImpl::is_enabled() const { + rtc::CritScope cs(crit_); + return enabled_; +} + +int VoiceDetectionImpl::set_stream_has_voice(bool has_voice) { + rtc::CritScope cs(crit_); + using_external_vad_ = true; + stream_has_voice_ = has_voice; + return AudioProcessing::kNoError; +} + +bool VoiceDetectionImpl::stream_has_voice() const { + rtc::CritScope cs(crit_); + // TODO(ajm): enable this assertion? + //RTC_DCHECK(using_external_vad_ || is_component_enabled()); + return stream_has_voice_; +} + +int VoiceDetectionImpl::set_likelihood(VoiceDetection::Likelihood likelihood) { + rtc::CritScope cs(crit_); + likelihood_ = likelihood; + if (enabled_) { + int mode = 2; + switch (likelihood) { + case VoiceDetection::kVeryLowLikelihood: + mode = 3; + break; + case VoiceDetection::kLowLikelihood: + mode = 2; + break; + case VoiceDetection::kModerateLikelihood: + mode = 1; + break; + case VoiceDetection::kHighLikelihood: + mode = 0; + break; + default: + RTC_NOTREACHED(); + break; + } + int error = WebRtcVad_set_mode(vad_->state(), mode); + RTC_DCHECK_EQ(0, error); + } + return AudioProcessing::kNoError; +} + +VoiceDetection::Likelihood VoiceDetectionImpl::likelihood() const { + rtc::CritScope cs(crit_); + return likelihood_; +} + +int VoiceDetectionImpl::set_frame_size_ms(int size) { + rtc::CritScope cs(crit_); + RTC_DCHECK_EQ(10, size); // TODO(ajm): remove when supported. + frame_size_ms_ = size; + Initialize(sample_rate_hz_); + return AudioProcessing::kNoError; +} + +int VoiceDetectionImpl::frame_size_ms() const { + rtc::CritScope cs(crit_); + return frame_size_ms_; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.h b/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.h new file mode 100644 index 0000000000..4b724bdd25 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_impl.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VOICE_DETECTION_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_VOICE_DETECTION_IMPL_H_ + +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +class AudioBuffer; + +class VoiceDetectionImpl : public VoiceDetection { + public: + explicit VoiceDetectionImpl(rtc::CriticalSection* crit); + ~VoiceDetectionImpl() override; + + // TODO(peah): Fold into ctor, once public API is removed. + void Initialize(int sample_rate_hz); + void ProcessCaptureAudio(AudioBuffer* audio); + + // VoiceDetection implementation. + int Enable(bool enable) override; + bool is_enabled() const override; + int set_stream_has_voice(bool has_voice) override; + bool stream_has_voice() const override; + int set_likelihood(Likelihood likelihood) override; + Likelihood likelihood() const override; + int set_frame_size_ms(int size) override; + int frame_size_ms() const override; + + private: + class Vad; + rtc::CriticalSection* const crit_; + bool enabled_ RTC_GUARDED_BY(crit_) = false; + bool stream_has_voice_ RTC_GUARDED_BY(crit_) = false; + bool using_external_vad_ RTC_GUARDED_BY(crit_) = false; + Likelihood likelihood_ RTC_GUARDED_BY(crit_) = kLowLikelihood; + int frame_size_ms_ RTC_GUARDED_BY(crit_) = 10; + size_t frame_size_samples_ RTC_GUARDED_BY(crit_) = 0; + int sample_rate_hz_ RTC_GUARDED_BY(crit_) = 0; + std::unique_ptr vad_ RTC_GUARDED_BY(crit_); + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(VoiceDetectionImpl); +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VOICE_DETECTION_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_unittest.cc b/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_unittest.cc new file mode 100644 index 0000000000..663913b638 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/audio_processing/voice_detection_unittest.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/test/audio_buffer_tools.h" +#include "modules/audio_processing/test/bitexactness_tools.h" +#include "modules/audio_processing/voice_detection_impl.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kNumFramesToProcess = 1000; + +// Process one frame of data and produce the output. +void ProcessOneFrame(int sample_rate_hz, + AudioBuffer* audio_buffer, + VoiceDetectionImpl* voice_detection) { + if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { + audio_buffer->SplitIntoFrequencyBands(); + } + + voice_detection->ProcessCaptureAudio(audio_buffer); +} + +// Processes a specified amount of frames, verifies the results and reports +// any errors. +void RunBitexactnessTest(int sample_rate_hz, + size_t num_channels, + int frame_size_ms_reference, + bool stream_has_voice_reference, + VoiceDetection::Likelihood likelihood_reference) { + rtc::CriticalSection crit_capture; + VoiceDetectionImpl voice_detection(&crit_capture); + voice_detection.Initialize(sample_rate_hz > 16000 ? 16000 : sample_rate_hz); + voice_detection.Enable(true); + + int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100); + const StreamConfig capture_config(sample_rate_hz, num_channels, false); + AudioBuffer capture_buffer( + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames(), capture_config.num_channels(), + capture_config.num_frames()); + test::InputAudioFile capture_file( + test::GetApmCaptureTestVectorFileName(sample_rate_hz)); + std::vector capture_input(samples_per_channel * num_channels); + for (int frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + ReadFloatSamplesFromStereoFile(samples_per_channel, num_channels, + &capture_file, capture_input); + + test::CopyVectorToAudioBuffer(capture_config, capture_input, + &capture_buffer); + + ProcessOneFrame(sample_rate_hz, &capture_buffer, &voice_detection); + } + + int frame_size_ms = voice_detection.frame_size_ms(); + bool stream_has_voice = voice_detection.stream_has_voice(); + VoiceDetection::Likelihood likelihood = voice_detection.likelihood(); + + // Compare the outputs to the references. + EXPECT_EQ(frame_size_ms_reference, frame_size_ms); + EXPECT_EQ(stream_has_voice_reference, stream_has_voice); + EXPECT_EQ(likelihood_reference, likelihood); +} + +const int kFrameSizeMsReference = 10; +const bool kStreamHasVoiceReference = true; +const VoiceDetection::Likelihood kLikelihoodReference = + VoiceDetection::kLowLikelihood; + +} // namespace + +TEST(VoiceDetectionBitExactnessTest, Mono8kHz) { + RunBitexactnessTest(8000, 1, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +TEST(VoiceDetectionBitExactnessTest, Mono16kHz) { + RunBitexactnessTest(16000, 1, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +TEST(VoiceDetectionBitExactnessTest, Mono32kHz) { + RunBitexactnessTest(32000, 1, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +TEST(VoiceDetectionBitExactnessTest, Mono48kHz) { + RunBitexactnessTest(48000, 1, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +TEST(VoiceDetectionBitExactnessTest, Stereo8kHz) { + RunBitexactnessTest(8000, 2, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +TEST(VoiceDetectionBitExactnessTest, Stereo16kHz) { + RunBitexactnessTest(16000, 2, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +TEST(VoiceDetectionBitExactnessTest, Stereo32kHz) { + RunBitexactnessTest(32000, 2, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +TEST(VoiceDetectionBitExactnessTest, Stereo48kHz) { + RunBitexactnessTest(48000, 2, kFrameSizeMsReference, kStreamHasVoiceReference, + kLikelihoodReference); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/BUILD.gn b/third_party/libwebrtc/webrtc/modules/bitrate_controller/BUILD.gn new file mode 100644 index 0000000000..1439f54851 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/BUILD.gn @@ -0,0 +1,68 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_static_library("bitrate_controller") { + # TODO(mbonadei): Remove (bugs.webrtc.org/6828) + # Errors on cyclic dependency with: + # congestion_controller:congestion_controller if enabled. + check_includes = false + + sources = [ + "bitrate_controller_impl.cc", + "bitrate_controller_impl.h", + "include/bitrate_controller.h", + "send_side_bandwidth_estimation.cc", + "send_side_bandwidth_estimation.h", + ] + + if (rtc_enable_bwe_test_logging) { + defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=1" ] + } else { + defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=0" ] + } + + # TODO(jschuh): Bug 1348: fix this warning. + configs += [ "//build/config/compiler:no_size_t_to_int_warning" ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../rtp_rtcp", + ] +} + +if (rtc_include_tests) { + rtc_source_set("bitrate_controller_unittests") { + testonly = true + + sources = [ + "bitrate_controller_unittest.cc", + "send_side_bandwidth_estimation_unittest.cc", + ] + deps = [ + ":bitrate_controller", + "../../logging:rtc_event_log_api", + "../../test:field_trial", + "../../test:test_support", + "../pacing:mock_paced_sender", + "../remote_bitrate_estimator:remote_bitrate_estimator", + "//testing/gmock", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/DEPS b/third_party/libwebrtc/webrtc/modules/bitrate_controller/DEPS new file mode 100644 index 0000000000..01a45e7ba1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/DEPS @@ -0,0 +1,5 @@ +include_rules = [ + "+call", + "+logging/rtc_event_log", + "+system_wrappers", +] diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/OWNERS b/third_party/libwebrtc/webrtc/modules/bitrate_controller/OWNERS new file mode 100644 index 0000000000..81c98ead47 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/OWNERS @@ -0,0 +1,8 @@ +stefan@webrtc.org +terelius@webrtc.org +mflodman@webrtc.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_gn/moz.build b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_gn/moz.build new file mode 100644 index 0000000000..73b43eb236 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_gn/moz.build @@ -0,0 +1,227 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["BWE_TEST_LOGGING_COMPILE_TIME_ENABLE"] = "0" +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.cc", + "/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("bitrate_controller_gn") diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.cc b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.cc new file mode 100644 index 0000000000..5b9b9ad462 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.cc @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "modules/bitrate_controller/bitrate_controller_impl.h" + +#include +#include + +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +class BitrateControllerImpl::RtcpBandwidthObserverImpl + : public RtcpBandwidthObserver { + public: + explicit RtcpBandwidthObserverImpl(BitrateControllerImpl* owner) + : owner_(owner) { + } + ~RtcpBandwidthObserverImpl() override = default; + // Received RTCP REMB or TMMBR. + void OnReceivedEstimatedBitrate(uint32_t bitrate) override { + owner_->OnReceivedEstimatedBitrate(bitrate); + } + // Received RTCP receiver block. + void OnReceivedRtcpReceiverReport(const ReportBlockList& report_blocks, + int64_t rtt, + int64_t now_ms) override { + owner_->OnReceivedRtcpReceiverReport(report_blocks, rtt, now_ms); + } + + private: + BitrateControllerImpl* const owner_; +}; + +BitrateController* BitrateController::CreateBitrateController( + const Clock* clock, + BitrateObserver* observer, + RtcEventLog* event_log) { + return new BitrateControllerImpl(clock, observer, event_log); +} + +BitrateController* BitrateController::CreateBitrateController( + const Clock* clock, + RtcEventLog* event_log) { + return CreateBitrateController(clock, nullptr, event_log); +} + +BitrateControllerImpl::BitrateControllerImpl(const Clock* clock, + BitrateObserver* observer, + RtcEventLog* event_log) + : clock_(clock), + observer_(observer), + last_bitrate_update_ms_(clock_->TimeInMilliseconds()), + event_log_(event_log), + bandwidth_estimation_(event_log), + reserved_bitrate_bps_(0), + last_bitrate_bps_(0), + last_fraction_loss_(0), + last_rtt_ms_(0), + last_reserved_bitrate_bps_(0) { + // This calls the observer_ if set, which means that the observer provided by + // the user must be ready to accept a bitrate update when it constructs the + // controller. We do this to avoid having to keep synchronized initial values + // in both the controller and the allocator. + MaybeTriggerOnNetworkChanged(); +} + +RtcpBandwidthObserver* BitrateControllerImpl::CreateRtcpBandwidthObserver() { + return new RtcpBandwidthObserverImpl(this); +} + +void BitrateControllerImpl::SetStartBitrate(int start_bitrate_bps) { + { + rtc::CritScope cs(&critsect_); + bandwidth_estimation_.SetSendBitrate(start_bitrate_bps); + } + MaybeTriggerOnNetworkChanged(); +} + +void BitrateControllerImpl::SetMinMaxBitrate(int min_bitrate_bps, + int max_bitrate_bps) { + { + rtc::CritScope cs(&critsect_); + bandwidth_estimation_.SetMinMaxBitrate(min_bitrate_bps, max_bitrate_bps); + } + MaybeTriggerOnNetworkChanged(); +} + +void BitrateControllerImpl::SetBitrates(int start_bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) { + { + rtc::CritScope cs(&critsect_); + bandwidth_estimation_.SetBitrates(start_bitrate_bps, + min_bitrate_bps, + max_bitrate_bps); + } + MaybeTriggerOnNetworkChanged(); +} + +void BitrateControllerImpl::ResetBitrates(int bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) { + { + rtc::CritScope cs(&critsect_); + bandwidth_estimation_ = SendSideBandwidthEstimation(event_log_); + bandwidth_estimation_.SetBitrates(bitrate_bps, min_bitrate_bps, + max_bitrate_bps); + } + MaybeTriggerOnNetworkChanged(); +} + +void BitrateControllerImpl::SetReservedBitrate(uint32_t reserved_bitrate_bps) { + { + rtc::CritScope cs(&critsect_); + reserved_bitrate_bps_ = reserved_bitrate_bps; + } + MaybeTriggerOnNetworkChanged(); +} + +// This is called upon reception of REMB or TMMBR. +void BitrateControllerImpl::OnReceivedEstimatedBitrate(uint32_t bitrate) { + { + rtc::CritScope cs(&critsect_); + bandwidth_estimation_.UpdateReceiverEstimate(clock_->TimeInMilliseconds(), + bitrate); + BWE_TEST_LOGGING_PLOT(1, "REMB_kbps", clock_->TimeInMilliseconds(), + bitrate / 1000); + } + MaybeTriggerOnNetworkChanged(); +} + +void BitrateControllerImpl::OnDelayBasedBweResult( + const DelayBasedBwe::Result& result) { + if (!result.updated) + return; + { + rtc::CritScope cs(&critsect_); + if (result.probe) { + bandwidth_estimation_.SetSendBitrate(result.target_bitrate_bps); + } + // Since SetSendBitrate now resets the delay-based estimate, we have to call + // UpdateDelayBasedEstimate after SetSendBitrate. + bandwidth_estimation_.UpdateDelayBasedEstimate(clock_->TimeInMilliseconds(), + result.target_bitrate_bps); + } + MaybeTriggerOnNetworkChanged(); +} + +int64_t BitrateControllerImpl::TimeUntilNextProcess() { + const int64_t kBitrateControllerUpdateIntervalMs = 25; + rtc::CritScope cs(&critsect_); + int64_t time_since_update_ms = + clock_->TimeInMilliseconds() - last_bitrate_update_ms_; + return std::max( + kBitrateControllerUpdateIntervalMs - time_since_update_ms, 0); +} + +void BitrateControllerImpl::Process() { + { + rtc::CritScope cs(&critsect_); + bandwidth_estimation_.UpdateEstimate(clock_->TimeInMilliseconds()); + } + MaybeTriggerOnNetworkChanged(); + last_bitrate_update_ms_ = clock_->TimeInMilliseconds(); +} + +void BitrateControllerImpl::OnReceivedRtcpReceiverReport( + const ReportBlockList& report_blocks, + int64_t rtt, + int64_t now_ms) { + if (report_blocks.empty()) + return; + + { + rtc::CritScope cs(&critsect_); + int fraction_lost_aggregate = 0; + int total_number_of_packets = 0; + + // Compute the a weighted average of the fraction loss from all report + // blocks. + for (const RTCPReportBlock& report_block : report_blocks) { + std::map::iterator seq_num_it = + ssrc_to_last_received_extended_high_seq_num_.find( + report_block.source_ssrc); + + int number_of_packets = 0; + if (seq_num_it != ssrc_to_last_received_extended_high_seq_num_.end()) { + number_of_packets = + report_block.extended_highest_sequence_number - seq_num_it->second; + } + + fraction_lost_aggregate += number_of_packets * report_block.fraction_lost; + total_number_of_packets += number_of_packets; + + // Update last received for this SSRC. + ssrc_to_last_received_extended_high_seq_num_[report_block.source_ssrc] = + report_block.extended_highest_sequence_number; + } + if (total_number_of_packets < 0) { + RTC_LOG(LS_WARNING) + << "Received report block where extended high sequence " + "number goes backwards, ignoring."; + return; + } + if (total_number_of_packets == 0) + fraction_lost_aggregate = 0; + else + fraction_lost_aggregate = + (fraction_lost_aggregate + total_number_of_packets / 2) / + total_number_of_packets; + if (fraction_lost_aggregate > 255) + return; + + RTC_DCHECK_GE(total_number_of_packets, 0); + + bandwidth_estimation_.UpdateReceiverBlock(fraction_lost_aggregate, rtt, + total_number_of_packets, now_ms); + } + MaybeTriggerOnNetworkChanged(); +} + +void BitrateControllerImpl::MaybeTriggerOnNetworkChanged() { + if (!observer_) + return; + + uint32_t bitrate_bps; + uint8_t fraction_loss; + int64_t rtt; + + if (GetNetworkParameters(&bitrate_bps, &fraction_loss, &rtt)) + observer_->OnNetworkChanged(bitrate_bps, fraction_loss, rtt); +} + +bool BitrateControllerImpl::GetNetworkParameters(uint32_t* bitrate, + uint8_t* fraction_loss, + int64_t* rtt) { + rtc::CritScope cs(&critsect_); + int current_bitrate; + bandwidth_estimation_.CurrentEstimate(¤t_bitrate, fraction_loss, rtt); + *bitrate = current_bitrate; + *bitrate -= std::min(*bitrate, reserved_bitrate_bps_); + *bitrate = + std::max(*bitrate, bandwidth_estimation_.GetMinBitrate()); + + bool new_bitrate = false; + if (*bitrate != last_bitrate_bps_ || *fraction_loss != last_fraction_loss_ || + *rtt != last_rtt_ms_ || + last_reserved_bitrate_bps_ != reserved_bitrate_bps_) { + last_bitrate_bps_ = *bitrate; + last_fraction_loss_ = *fraction_loss; + last_rtt_ms_ = *rtt; + last_reserved_bitrate_bps_ = reserved_bitrate_bps_; + new_bitrate = true; + } + + BWE_TEST_LOGGING_PLOT(1, "fraction_loss_%", clock_->TimeInMilliseconds(), + (last_fraction_loss_ * 100) / 256); + BWE_TEST_LOGGING_PLOT(1, "rtt_ms", clock_->TimeInMilliseconds(), + last_rtt_ms_); + BWE_TEST_LOGGING_PLOT(1, "Target_bitrate_kbps", clock_->TimeInMilliseconds(), + last_bitrate_bps_ / 1000); + + return new_bitrate; +} + +bool BitrateControllerImpl::AvailableBandwidth(uint32_t* bandwidth) const { + rtc::CritScope cs(&critsect_); + int bitrate; + uint8_t fraction_loss; + int64_t rtt; + bandwidth_estimation_.CurrentEstimate(&bitrate, &fraction_loss, &rtt); + if (bitrate > 0) { + bitrate = bitrate - std::min(bitrate, reserved_bitrate_bps_); + bitrate = std::max(bitrate, bandwidth_estimation_.GetMinBitrate()); + *bandwidth = bitrate; + return true; + } + return false; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.h b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.h new file mode 100644 index 0000000000..6a678082e9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_impl.h @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + * Usage: this class will register multiple RtcpBitrateObserver's one at each + * RTCP module. It will aggregate the results and run one bandwidth estimation + * and push the result to the encoder via VideoEncoderCallback. + */ + +#ifndef MODULES_BITRATE_CONTROLLER_BITRATE_CONTROLLER_IMPL_H_ +#define MODULES_BITRATE_CONTROLLER_BITRATE_CONTROLLER_IMPL_H_ + +#include "modules/bitrate_controller/include/bitrate_controller.h" + +#include +#include +#include +#include + +#include "modules/bitrate_controller/send_side_bandwidth_estimation.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +class BitrateControllerImpl : public BitrateController { + public: + // TODO(perkj): BitrateObserver has been deprecated and is not used in WebRTC. + // |observer| is left for project that is not yet updated. + BitrateControllerImpl(const Clock* clock, + BitrateObserver* observer, + RtcEventLog* event_log); + virtual ~BitrateControllerImpl() {} + + bool AvailableBandwidth(uint32_t* bandwidth) const override; + + RTC_DEPRECATED RtcpBandwidthObserver* CreateRtcpBandwidthObserver() override; + + // Deprecated + void SetStartBitrate(int start_bitrate_bps) override; + // Deprecated + void SetMinMaxBitrate(int min_bitrate_bps, int max_bitrate_bps) override; + + void SetBitrates(int start_bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) override; + + void ResetBitrates(int bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) override; + + + void SetReservedBitrate(uint32_t reserved_bitrate_bps) override; + + // Returns true if the parameters have changed since the last call. + bool GetNetworkParameters(uint32_t* bitrate, + uint8_t* fraction_loss, + int64_t* rtt) override; + + void OnDelayBasedBweResult(const DelayBasedBwe::Result& result) override; + + int64_t TimeUntilNextProcess() override; + void Process() override; + + private: + class RtcpBandwidthObserverImpl; + + // Called by BitrateObserver's direct from the RTCP module. + // Implements RtcpBandwidthObserver. + void OnReceivedEstimatedBitrate(uint32_t bitrate) override; + + void OnReceivedRtcpReceiverReport(const ReportBlockList& report_blocks, + int64_t rtt, + int64_t now_ms) override; + + // Deprecated + void MaybeTriggerOnNetworkChanged(); + + void OnNetworkChanged(uint32_t bitrate, + uint8_t fraction_loss, // 0 - 255. + int64_t rtt) RTC_EXCLUSIVE_LOCKS_REQUIRED(critsect_); + + // Used by process thread. + const Clock* const clock_; + BitrateObserver* const observer_; + int64_t last_bitrate_update_ms_; + RtcEventLog* const event_log_; + + rtc::CriticalSection critsect_; + std::map ssrc_to_last_received_extended_high_seq_num_ + RTC_GUARDED_BY(critsect_); + SendSideBandwidthEstimation bandwidth_estimation_ RTC_GUARDED_BY(critsect_); + uint32_t reserved_bitrate_bps_ RTC_GUARDED_BY(critsect_); + + uint32_t last_bitrate_bps_ RTC_GUARDED_BY(critsect_); + uint8_t last_fraction_loss_ RTC_GUARDED_BY(critsect_); + int64_t last_rtt_ms_ RTC_GUARDED_BY(critsect_); + uint32_t last_reserved_bitrate_bps_ RTC_GUARDED_BY(critsect_); + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BitrateControllerImpl); +}; +} // namespace webrtc +#endif // MODULES_BITRATE_CONTROLLER_BITRATE_CONTROLLER_IMPL_H_ diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_unittest.cc new file mode 100644 index 0000000000..66ca5b9558 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/bitrate_controller_unittest.cc @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "modules/bitrate_controller/include/bitrate_controller.h" +#include "modules/pacing/mock/mock_paced_sender.h" +#include "modules/remote_bitrate_estimator/include/bwe_defines.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +using ::testing::Exactly; +using ::testing::Return; + +using webrtc::BitrateController; +using webrtc::BitrateObserver; +using webrtc::PacedSender; +using webrtc::RtcpBandwidthObserver; + +uint8_t WeightedLoss(int num_packets1, uint8_t fraction_loss1, + int num_packets2, uint8_t fraction_loss2) { + int weighted_sum = num_packets1 * fraction_loss1 + + num_packets2 * fraction_loss2; + int total_num_packets = num_packets1 + num_packets2; + return (weighted_sum + total_num_packets / 2) / total_num_packets; +} + +webrtc::RTCPReportBlock CreateReportBlock( + uint32_t remote_ssrc, uint32_t source_ssrc, + uint8_t fraction_lost, uint32_t extended_high_sequence_number) { + return webrtc::RTCPReportBlock(remote_ssrc, source_ssrc, fraction_lost, 0, + extended_high_sequence_number, 0, 0, 0); +} + +class TestBitrateObserver: public BitrateObserver { + public: + TestBitrateObserver() + : last_bitrate_(0), + last_fraction_loss_(0), + last_rtt_(0) { + } + + virtual void OnNetworkChanged(uint32_t bitrate, + uint8_t fraction_loss, + int64_t rtt) { + last_bitrate_ = static_cast(bitrate); + last_fraction_loss_ = fraction_loss; + last_rtt_ = rtt; + } + int last_bitrate_; + uint8_t last_fraction_loss_; + int64_t last_rtt_; +}; + +class BitrateControllerTest : public ::testing::Test { + protected: + BitrateControllerTest() : clock_(0) {} + ~BitrateControllerTest() {} + + virtual void SetUp() { + controller_.reset(BitrateController::CreateBitrateController( + &clock_, &bitrate_observer_, &event_log_)); + controller_->SetStartBitrate(kStartBitrateBps); + EXPECT_EQ(kStartBitrateBps, bitrate_observer_.last_bitrate_); + controller_->SetMinMaxBitrate(kMinBitrateBps, kMaxBitrateBps); + EXPECT_EQ(kStartBitrateBps, bitrate_observer_.last_bitrate_); + bandwidth_observer_ = controller_.get(); + } + + virtual void TearDown() { + } + + const int kMinBitrateBps = 100000; + const int kStartBitrateBps = 200000; + const int kMaxBitrateBps = 300000; + + const int kDefaultMinBitrateBps = 10000; + const int kDefaultMaxBitrateBps = 1000000000; + + webrtc::SimulatedClock clock_; + TestBitrateObserver bitrate_observer_; + std::unique_ptr controller_; + RtcpBandwidthObserver* bandwidth_observer_; + testing::NiceMock event_log_; +}; + +TEST_F(BitrateControllerTest, DefaultMinMaxBitrate) { + // Receive successively lower REMBs, verify the reserved bitrate is deducted. + controller_->SetMinMaxBitrate(0, 0); + EXPECT_EQ(kStartBitrateBps, bitrate_observer_.last_bitrate_); + bandwidth_observer_->OnReceivedEstimatedBitrate(kDefaultMinBitrateBps / 2); + EXPECT_EQ(webrtc::congestion_controller::GetMinBitrateBps(), + bitrate_observer_.last_bitrate_); + bandwidth_observer_->OnReceivedEstimatedBitrate(2 * kDefaultMaxBitrateBps); + clock_.AdvanceTimeMilliseconds(1000); + controller_->Process(); + EXPECT_EQ(kDefaultMaxBitrateBps, bitrate_observer_.last_bitrate_); +} + +TEST_F(BitrateControllerTest, OneBitrateObserverOneRtcpObserver) { + // First REMB applies immediately. + int64_t time_ms = 1001; + webrtc::ReportBlockList report_blocks; + report_blocks.push_back(CreateReportBlock(1, 2, 0, 1)); + bandwidth_observer_->OnReceivedEstimatedBitrate(200000); + EXPECT_EQ(200000, bitrate_observer_.last_bitrate_); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(0, bitrate_observer_.last_rtt_); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + report_blocks.clear(); + time_ms += 2000; + + // Receive a high remb, test bitrate inc. + bandwidth_observer_->OnReceivedEstimatedBitrate(400000); + + // Test bitrate increase 8% per second. + report_blocks.push_back(CreateReportBlock(1, 2, 0, 21)); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(217000, bitrate_observer_.last_bitrate_); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + time_ms += 1000; + + report_blocks.clear(); + report_blocks.push_back(CreateReportBlock(1, 2, 0, 41)); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(235360, bitrate_observer_.last_bitrate_); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + time_ms += 1000; + + report_blocks.clear(); + report_blocks.push_back(CreateReportBlock(1, 2, 0, 61)); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(255189, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + report_blocks.clear(); + report_blocks.push_back(CreateReportBlock(1, 2, 0, 81)); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(276604, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + report_blocks.clear(); + report_blocks.push_back(CreateReportBlock(1, 2, 0, 101)); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(299732, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + // Reach max cap. + report_blocks.clear(); + report_blocks.push_back(CreateReportBlock(1, 2, 0, 121)); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(300000, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + report_blocks.clear(); + report_blocks.push_back(CreateReportBlock(1, 2, 0, 141)); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(300000, bitrate_observer_.last_bitrate_); + + // Test that a low delay-based estimate limits the combined estimate. + webrtc::DelayBasedBwe::Result result(false, 280000); + controller_->OnDelayBasedBweResult(result); + EXPECT_EQ(280000, bitrate_observer_.last_bitrate_); + + // Test that a low REMB limits the combined estimate. + bandwidth_observer_->OnReceivedEstimatedBitrate(250000); + EXPECT_EQ(250000, bitrate_observer_.last_bitrate_); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + + bandwidth_observer_->OnReceivedEstimatedBitrate(1000); + EXPECT_EQ(100000, bitrate_observer_.last_bitrate_); +} + +TEST_F(BitrateControllerTest, OneBitrateObserverTwoRtcpObservers) { + const uint32_t kSenderSsrc1 = 1; + const uint32_t kSenderSsrc2 = 2; + const uint32_t kMediaSsrc1 = 3; + const uint32_t kMediaSsrc2 = 4; + + int64_t time_ms = 1; + webrtc::ReportBlockList report_blocks; + report_blocks = {CreateReportBlock(kSenderSsrc1, kMediaSsrc1, 0, 1)}; + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + time_ms += 500; + + RtcpBandwidthObserver* second_bandwidth_observer = controller_.get(); + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 21)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 100, time_ms); + + // Test start bitrate. + EXPECT_EQ(200000, bitrate_observer_.last_bitrate_); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(100, bitrate_observer_.last_rtt_); + time_ms += 500; + + // Test bitrate increase 8% per second. + report_blocks = {CreateReportBlock(kSenderSsrc1, kMediaSsrc1, 0, 21)}; + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + time_ms += 500; + + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 21)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 100, time_ms); + EXPECT_EQ(217000, bitrate_observer_.last_bitrate_); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(100, bitrate_observer_.last_rtt_); + time_ms += 500; + + // Extra report should not change estimate. + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 31)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 100, time_ms); + EXPECT_EQ(217000, bitrate_observer_.last_bitrate_); + time_ms += 500; + + report_blocks = {CreateReportBlock(kSenderSsrc1, kMediaSsrc1, 0, 41)}; + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(235360, bitrate_observer_.last_bitrate_); + + // Second report should not change estimate. + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 41)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 100, time_ms); + EXPECT_EQ(235360, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + // Reports from only one bandwidth observer is ok. + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 61)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 50, time_ms); + EXPECT_EQ(255189, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 81)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 50, time_ms); + EXPECT_EQ(276604, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 121)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 50, time_ms); + EXPECT_EQ(299732, bitrate_observer_.last_bitrate_); + time_ms += 1000; + + // Reach max cap. + report_blocks = {CreateReportBlock(kSenderSsrc2, kMediaSsrc2, 0, 141)}; + second_bandwidth_observer->OnReceivedRtcpReceiverReport( + report_blocks, 50, time_ms); + EXPECT_EQ(300000, bitrate_observer_.last_bitrate_); + + // Test that a low REMB trigger immediately. + // We don't care which bandwidth observer that delivers the REMB. + second_bandwidth_observer->OnReceivedEstimatedBitrate(250000); + EXPECT_EQ(250000, bitrate_observer_.last_bitrate_); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + + // Min cap. + bandwidth_observer_->OnReceivedEstimatedBitrate(1000); + EXPECT_EQ(100000, bitrate_observer_.last_bitrate_); +} + +TEST_F(BitrateControllerTest, OneBitrateObserverMultipleReportBlocks) { + uint32_t sequence_number[2] = {0, 0xFF00}; + const int kStartBitrate = 200000; + const int kMinBitrate = 100000; + const int kMaxBitrate = 300000; + controller_->SetStartBitrate(kStartBitrate); + controller_->SetMinMaxBitrate(kMinBitrate, kMaxBitrate); + + // REMBs during the first 2 seconds apply immediately. + int64_t time_ms = 1001; + webrtc::ReportBlockList report_blocks; + report_blocks.push_back(CreateReportBlock(1, 2, 0, sequence_number[0])); + bandwidth_observer_->OnReceivedEstimatedBitrate(kStartBitrate); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + report_blocks.clear(); + time_ms += 2000; + + // Receive a high REMB, test bitrate increase. + bandwidth_observer_->OnReceivedEstimatedBitrate(400000); + + int last_bitrate = 0; + // Ramp up to max bitrate. + for (int i = 0; i < 7; ++i) { + report_blocks.push_back(CreateReportBlock(1, 2, 0, sequence_number[0])); + report_blocks.push_back(CreateReportBlock(1, 3, 0, sequence_number[1])); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, + time_ms); + EXPECT_GT(bitrate_observer_.last_bitrate_, last_bitrate); + EXPECT_EQ(0, bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + last_bitrate = bitrate_observer_.last_bitrate_; + time_ms += 1000; + sequence_number[0] += 20; + sequence_number[1] += 1; + report_blocks.clear(); + } + + EXPECT_EQ(kMaxBitrate, bitrate_observer_.last_bitrate_); + + // Packet loss on the first stream. Verify that bitrate decreases. + report_blocks.push_back(CreateReportBlock(1, 2, 50, sequence_number[0])); + report_blocks.push_back(CreateReportBlock(1, 3, 0, sequence_number[1])); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_LT(bitrate_observer_.last_bitrate_, last_bitrate); + EXPECT_EQ(WeightedLoss(20, 50, 1, 0), bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + last_bitrate = bitrate_observer_.last_bitrate_; + sequence_number[0] += 20; + sequence_number[1] += 20; + time_ms += 1000; + report_blocks.clear(); + + // Packet loss on the second stream. Verify that bitrate decreases. + report_blocks.push_back(CreateReportBlock(1, 2, 0, sequence_number[0])); + report_blocks.push_back(CreateReportBlock(1, 3, 75, sequence_number[1])); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_LT(bitrate_observer_.last_bitrate_, last_bitrate); + EXPECT_EQ(WeightedLoss(20, 0, 20, 75), bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + last_bitrate = bitrate_observer_.last_bitrate_; + sequence_number[0] += 20; + sequence_number[1] += 1; + time_ms += 1000; + report_blocks.clear(); + + // All packets lost on stream with few packets, no back-off. + report_blocks.push_back(CreateReportBlock(1, 2, 1, sequence_number[0])); + report_blocks.push_back(CreateReportBlock(1, 3, 255, sequence_number[1])); + bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, 50, time_ms); + EXPECT_EQ(bitrate_observer_.last_bitrate_, last_bitrate); + EXPECT_EQ(WeightedLoss(20, 1, 1, 255), bitrate_observer_.last_fraction_loss_); + EXPECT_EQ(50, bitrate_observer_.last_rtt_); + last_bitrate = bitrate_observer_.last_bitrate_; + sequence_number[0] += 20; + sequence_number[1] += 1; + report_blocks.clear(); +} + +TEST_F(BitrateControllerTest, SetReservedBitrate) { + // Receive successively lower REMBs, verify the reserved bitrate is deducted. + controller_->SetReservedBitrate(0); + bandwidth_observer_->OnReceivedEstimatedBitrate(400000); + EXPECT_EQ(200000, bitrate_observer_.last_bitrate_); + controller_->SetReservedBitrate(50000); + bandwidth_observer_->OnReceivedEstimatedBitrate(400000); + EXPECT_EQ(150000, bitrate_observer_.last_bitrate_); + + controller_->SetReservedBitrate(0); + bandwidth_observer_->OnReceivedEstimatedBitrate(250000); + EXPECT_EQ(200000, bitrate_observer_.last_bitrate_); + controller_->SetReservedBitrate(50000); + bandwidth_observer_->OnReceivedEstimatedBitrate(250000); + EXPECT_EQ(150000, bitrate_observer_.last_bitrate_); + + controller_->SetReservedBitrate(0); + bandwidth_observer_->OnReceivedEstimatedBitrate(200000); + EXPECT_EQ(200000, bitrate_observer_.last_bitrate_); + controller_->SetReservedBitrate(30000); + bandwidth_observer_->OnReceivedEstimatedBitrate(200000); + EXPECT_EQ(170000, bitrate_observer_.last_bitrate_); + + controller_->SetReservedBitrate(0); + bandwidth_observer_->OnReceivedEstimatedBitrate(160000); + EXPECT_EQ(160000, bitrate_observer_.last_bitrate_); + controller_->SetReservedBitrate(30000); + bandwidth_observer_->OnReceivedEstimatedBitrate(160000); + EXPECT_EQ(130000, bitrate_observer_.last_bitrate_); + + controller_->SetReservedBitrate(0); + bandwidth_observer_->OnReceivedEstimatedBitrate(120000); + EXPECT_EQ(120000, bitrate_observer_.last_bitrate_); + controller_->SetReservedBitrate(10000); + bandwidth_observer_->OnReceivedEstimatedBitrate(120000); + EXPECT_EQ(110000, bitrate_observer_.last_bitrate_); + + controller_->SetReservedBitrate(0); + bandwidth_observer_->OnReceivedEstimatedBitrate(120000); + EXPECT_EQ(120000, bitrate_observer_.last_bitrate_); + controller_->SetReservedBitrate(50000); + bandwidth_observer_->OnReceivedEstimatedBitrate(120000); + // Limited by min bitrate. + EXPECT_EQ(100000, bitrate_observer_.last_bitrate_); + + controller_->SetReservedBitrate(10000); + bandwidth_observer_->OnReceivedEstimatedBitrate(1); + EXPECT_EQ(100000, bitrate_observer_.last_bitrate_); +} + +TEST_F(BitrateControllerTest, TimeoutsWithoutFeedback) { + { + webrtc::test::ScopedFieldTrials override_field_trials( + "WebRTC-FeedbackTimeout/Enabled/"); + SetUp(); + int expected_bitrate_bps = 300000; + controller_->SetBitrates(300000, kDefaultMinBitrateBps, + kDefaultMaxBitrateBps); + + webrtc::ReportBlockList report_blocks; + report_blocks.push_back(CreateReportBlock(1, 2, 0, 1)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(500); + + report_blocks.push_back(CreateReportBlock(1, 2, 0, 21)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + report_blocks.clear(); + expected_bitrate_bps = expected_bitrate_bps * 1.08 + 1000; + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(1500); + + report_blocks.push_back(CreateReportBlock(1, 2, 0, 41)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + expected_bitrate_bps = expected_bitrate_bps * 1.08 + 1000; + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(4000); + + // 4 seconds since feedback, expect increase. + controller_->Process(); + expected_bitrate_bps = expected_bitrate_bps * 1.08 + 1000; + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(2000); + + // 6 seconds since feedback, expect no increase. + controller_->Process(); + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(9001); + + // More than 15 seconds since feedback, expect decrease. + controller_->Process(); + expected_bitrate_bps *= 0.8; + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(500); + + // Only one timeout every second. + controller_->Process(); + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(501); + + // New timeout allowed. + controller_->Process(); + expected_bitrate_bps *= 0.8; + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + } +} + +TEST_F(BitrateControllerTest, StopIncreaseWithoutPacketReports) { + int expected_bitrate_bps = 300000; + controller_->SetBitrates(300000, kDefaultMinBitrateBps, + kDefaultMaxBitrateBps); + + webrtc::ReportBlockList report_blocks; + report_blocks.push_back(CreateReportBlock(1, 2, 0, 1)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(500); + + report_blocks.push_back(CreateReportBlock(1, 2, 0, 21)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + report_blocks.clear(); + expected_bitrate_bps = expected_bitrate_bps * 1.08 + 1000; + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(1500); + + // 1.2 seconds without packets reported as received, no increase. + report_blocks.push_back(CreateReportBlock(1, 2, 0, 21)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(1000); + + // 5 packets reported as received since last, too few, no increase. + report_blocks.push_back(CreateReportBlock(1, 2, 0, 26)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + report_blocks.clear(); + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(100); + + // 15 packets reported as received since last, enough to increase. + report_blocks.push_back(CreateReportBlock(1, 2, 0, 41)); + bandwidth_observer_->OnReceivedRtcpReceiverReport( + report_blocks, 50, clock_.TimeInMilliseconds()); + expected_bitrate_bps = expected_bitrate_bps * 1.08 + 1000; + EXPECT_EQ(expected_bitrate_bps, bitrate_observer_.last_bitrate_); + clock_.AdvanceTimeMilliseconds(1000); +} diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/include/bitrate_controller.h b/third_party/libwebrtc/webrtc/modules/bitrate_controller/include/bitrate_controller.h new file mode 100644 index 0000000000..5aaf88b95d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/include/bitrate_controller.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + * Usage: this class will register multiple RtcpBitrateObserver's one at each + * RTCP module. It will aggregate the results and run one bandwidth estimation + * and push the result to the encoders via BitrateObserver(s). + */ + +#ifndef MODULES_BITRATE_CONTROLLER_INCLUDE_BITRATE_CONTROLLER_H_ +#define MODULES_BITRATE_CONTROLLER_INCLUDE_BITRATE_CONTROLLER_H_ + +#include + +#include "modules/congestion_controller/delay_based_bwe.h" +#include "modules/include/module.h" +#include "modules/pacing/paced_sender.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { + +class RtcEventLog; + +// Deprecated +// TODO(perkj): Remove BitrateObserver when no implementations use it. +class BitrateObserver { + // Observer class for bitrate changes announced due to change in bandwidth + // estimate or due to bitrate allocation changes. Fraction loss and rtt is + // also part of this callback to allow the obsevrer to optimize its settings + // for different types of network environments. The bitrate does not include + // packet headers and is measured in bits per second. + public: + virtual void OnNetworkChanged(uint32_t bitrate_bps, + uint8_t fraction_loss, // 0 - 255. + int64_t rtt_ms) = 0; + // TODO(gnish): Merge these two into one function. + virtual void OnNetworkChanged(uint32_t bitrate_for_encoder_bps, + uint32_t bitrate_for_pacer_bps, + bool in_probe_rtt, + int64_t target_set_time, + uint64_t congestion_window) {} + virtual void OnBytesAcked(size_t bytes) {} + virtual size_t pacer_queue_size_in_bytes() { return 0; } + virtual ~BitrateObserver() {} +}; + +class BitrateController : public Module, public RtcpBandwidthObserver { + // This class collects feedback from all streams sent to a peer (via + // RTCPBandwidthObservers). It does one aggregated send side bandwidth + // estimation and divide the available bitrate between all its registered + // BitrateObservers. + public: + static const int kDefaultStartBitratebps = 300000; + + // Deprecated: + // TODO(perkj): BitrateObserver has been deprecated and is not used in WebRTC. + // Remove this method once other other projects does not use it. + static BitrateController* CreateBitrateController(const Clock* clock, + BitrateObserver* observer, + RtcEventLog* event_log); + + static BitrateController* CreateBitrateController(const Clock* clock, + RtcEventLog* event_log); + + virtual ~BitrateController() {} + + // Deprecated, use raw pointer to BitrateController instance instead. + // Creates RtcpBandwidthObserver caller responsible to delete. + RTC_DEPRECATED virtual RtcpBandwidthObserver* + CreateRtcpBandwidthObserver() = 0; + + // Deprecated + virtual void SetStartBitrate(int start_bitrate_bps) = 0; + // Deprecated + virtual void SetMinMaxBitrate(int min_bitrate_bps, int max_bitrate_bps) = 0; + virtual void SetBitrates(int start_bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) = 0; + + virtual void ResetBitrates(int bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) = 0; + + virtual void OnDelayBasedBweResult(const DelayBasedBwe::Result& result) = 0; + + // Gets the available payload bandwidth in bits per second. Note that + // this bandwidth excludes packet headers. + virtual bool AvailableBandwidth(uint32_t* bandwidth) const = 0; + + virtual void SetReservedBitrate(uint32_t reserved_bitrate_bps) = 0; + + virtual bool GetNetworkParameters(uint32_t* bitrate, + uint8_t* fraction_loss, + int64_t* rtt) = 0; +}; +} // namespace webrtc +#endif // MODULES_BITRATE_CONTROLLER_INCLUDE_BITRATE_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/include/mock/mock_bitrate_controller.h b/third_party/libwebrtc/webrtc/modules/bitrate_controller/include/mock/mock_bitrate_controller.h new file mode 100644 index 0000000000..4207b7498f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/include/mock/mock_bitrate_controller.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_BITRATE_CONTROLLER_INCLUDE_MOCK_MOCK_BITRATE_CONTROLLER_H_ +#define MODULES_BITRATE_CONTROLLER_INCLUDE_MOCK_MOCK_BITRATE_CONTROLLER_H_ + +#include "modules/bitrate_controller/include/bitrate_controller.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockBitrateObserver : public BitrateObserver { + public: + MOCK_METHOD3(OnNetworkChanged, + void(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt_ms)); +}; + +class MockBitrateController : public BitrateController { + public: + MOCK_METHOD0(CreateRtcpBandwidthObserver, RtcpBandwidthObserver*()); + MOCK_METHOD1(SetStartBitrate, void(int start_bitrate_bps)); + MOCK_METHOD2(SetMinMaxBitrate, + void(int min_bitrate_bps, int max_bitrate_bps)); + MOCK_METHOD3(SetBitrates, + void(int start_bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps)); + MOCK_METHOD3(ResetBitrates, + void(int bitrate_bps, int min_bitrate_bps, int max_bitrate_bps)); + MOCK_METHOD1(UpdateDelayBasedEstimate, void(uint32_t bitrate_bps)); + MOCK_METHOD1(UpdateProbeBitrate, void(uint32_t bitrate_bps)); + MOCK_METHOD1(SetEventLog, void(RtcEventLog* event_log)); + MOCK_CONST_METHOD1(AvailableBandwidth, bool(uint32_t* bandwidth)); + MOCK_METHOD1(SetReservedBitrate, void(uint32_t reserved_bitrate_bps)); + MOCK_METHOD3(GetNetworkParameters, + bool(uint32_t* bitrate, uint8_t* fraction_loss, int64_t* rtt)); + + MOCK_METHOD0(Process, void()); + MOCK_METHOD0(TimeUntilNextProcess, int64_t()); +}; +} // namespace test +} // namespace webrtc + +#endif // MODULES_BITRATE_CONTROLLER_INCLUDE_MOCK_MOCK_BITRATE_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.cc b/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.cc new file mode 100644 index 0000000000..86096a1b73 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.cc @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/bitrate_controller/send_side_bandwidth_estimation.h" + +#include +#include +#include +#include + +#include "logging/rtc_event_log/events/rtc_event_bwe_update_loss_based.h" +#include "logging/rtc_event_log/rtc_event_log.h" +#include "modules/remote_bitrate_estimator/include/bwe_defines.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/ptr_util.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { +const int64_t kBweIncreaseIntervalMs = 1000; +const int64_t kBweDecreaseIntervalMs = 300; +const int64_t kStartPhaseMs = 2000; +const int64_t kBweConverganceTimeMs = 20000; +const int kLimitNumPackets = 20; +const int kDefaultMaxBitrateBps = 1000000000; +const int64_t kLowBitrateLogPeriodMs = 10000; +const int64_t kRtcEventLogPeriodMs = 5000; +// Expecting that RTCP feedback is sent uniformly within [0.5, 1.5]s intervals. +const int64_t kFeedbackIntervalMs = 5000; +const int64_t kFeedbackTimeoutIntervals = 3; +const int64_t kTimeoutIntervalMs = 1000; + +const float kDefaultLowLossThreshold = 0.02f; +const float kDefaultHighLossThreshold = 0.1f; +const int kDefaultBitrateThresholdKbps = 0; + +struct UmaRampUpMetric { + const char* metric_name; + int bitrate_kbps; +}; + +const UmaRampUpMetric kUmaRampupMetrics[] = { + {"WebRTC.BWE.RampUpTimeTo500kbpsInMs", 500}, + {"WebRTC.BWE.RampUpTimeTo1000kbpsInMs", 1000}, + {"WebRTC.BWE.RampUpTimeTo2000kbpsInMs", 2000}}; +const size_t kNumUmaRampupMetrics = + sizeof(kUmaRampupMetrics) / sizeof(kUmaRampupMetrics[0]); + +const char kBweLosExperiment[] = "WebRTC-BweLossExperiment"; + +bool BweLossExperimentIsEnabled() { + std::string experiment_string = + webrtc::field_trial::FindFullName(kBweLosExperiment); + // The experiment is enabled iff the field trial string begins with "Enabled". + return experiment_string.find("Enabled") == 0; +} + +bool ReadBweLossExperimentParameters(float* low_loss_threshold, + float* high_loss_threshold, + uint32_t* bitrate_threshold_kbps) { + RTC_DCHECK(low_loss_threshold); + RTC_DCHECK(high_loss_threshold); + RTC_DCHECK(bitrate_threshold_kbps); + std::string experiment_string = + webrtc::field_trial::FindFullName(kBweLosExperiment); + int parsed_values = + sscanf(experiment_string.c_str(), "Enabled-%f,%f,%u", low_loss_threshold, + high_loss_threshold, bitrate_threshold_kbps); + if (parsed_values == 3) { + RTC_CHECK_GT(*low_loss_threshold, 0.0f) + << "Loss threshold must be greater than 0."; + RTC_CHECK_LE(*low_loss_threshold, 1.0f) + << "Loss threshold must be less than or equal to 1."; + RTC_CHECK_GT(*high_loss_threshold, 0.0f) + << "Loss threshold must be greater than 0."; + RTC_CHECK_LE(*high_loss_threshold, 1.0f) + << "Loss threshold must be less than or equal to 1."; + RTC_CHECK_LE(*low_loss_threshold, *high_loss_threshold) + << "The low loss threshold must be less than or equal to the high loss " + "threshold."; + RTC_CHECK_GE(*bitrate_threshold_kbps, 0) + << "Bitrate threshold can't be negative."; + RTC_CHECK_LT(*bitrate_threshold_kbps, + std::numeric_limits::max() / 1000) + << "Bitrate must be smaller enough to avoid overflows."; + return true; + } + RTC_LOG(LS_WARNING) << "Failed to parse parameters for BweLossExperiment " + "experiment from field trial string. Using default."; + *low_loss_threshold = kDefaultLowLossThreshold; + *high_loss_threshold = kDefaultHighLossThreshold; + *bitrate_threshold_kbps = kDefaultBitrateThresholdKbps; + return false; +} +} // namespace + +SendSideBandwidthEstimation::SendSideBandwidthEstimation(RtcEventLog* event_log) + : lost_packets_since_last_loss_update_Q8_(0), + expected_packets_since_last_loss_update_(0), + current_bitrate_bps_(0), + min_bitrate_configured_(congestion_controller::GetMinBitrateBps()), + max_bitrate_configured_(kDefaultMaxBitrateBps), + last_low_bitrate_log_ms_(-1), + has_decreased_since_last_fraction_loss_(false), + last_feedback_ms_(-1), + last_packet_report_ms_(-1), + last_timeout_ms_(-1), + last_fraction_loss_(0), + last_logged_fraction_loss_(0), + last_round_trip_time_ms_(0), + bwe_incoming_(0), + delay_based_bitrate_bps_(0), + time_last_decrease_ms_(0), + first_report_time_ms_(-1), + initially_lost_packets_(0), + bitrate_at_2_seconds_kbps_(0), + uma_update_state_(kNoUpdate), + rampup_uma_stats_updated_(kNumUmaRampupMetrics, false), + event_log_(event_log), + last_rtc_event_log_ms_(-1), + in_timeout_experiment_( + webrtc::field_trial::IsEnabled("WebRTC-FeedbackTimeout")), + low_loss_threshold_(kDefaultLowLossThreshold), + high_loss_threshold_(kDefaultHighLossThreshold), + bitrate_threshold_bps_(1000 * kDefaultBitrateThresholdKbps) { + RTC_DCHECK(event_log); + if (BweLossExperimentIsEnabled()) { + uint32_t bitrate_threshold_kbps; + if (ReadBweLossExperimentParameters(&low_loss_threshold_, + &high_loss_threshold_, + &bitrate_threshold_kbps)) { + RTC_LOG(LS_INFO) << "Enabled BweLossExperiment with parameters " + << low_loss_threshold_ << ", " << high_loss_threshold_ + << ", " << bitrate_threshold_kbps; + bitrate_threshold_bps_ = bitrate_threshold_kbps * 1000; + } + } +} + +SendSideBandwidthEstimation::~SendSideBandwidthEstimation() {} + +void SendSideBandwidthEstimation::SetBitrates(int send_bitrate, + int min_bitrate, + int max_bitrate) { + SetMinMaxBitrate(min_bitrate, max_bitrate); + if (send_bitrate > 0) + SetSendBitrate(send_bitrate); +} + +void SendSideBandwidthEstimation::SetSendBitrate(int bitrate) { + RTC_DCHECK_GT(bitrate, 0); + delay_based_bitrate_bps_ = 0; // Reset to avoid being capped by the estimate. + CapBitrateToThresholds(Clock::GetRealTimeClock()->TimeInMilliseconds(), + bitrate); + // Clear last sent bitrate history so the new value can be used directly + // and not capped. + min_bitrate_history_.clear(); +} + +void SendSideBandwidthEstimation::SetMinMaxBitrate(int min_bitrate, + int max_bitrate) { + RTC_DCHECK_GE(min_bitrate, 0); + min_bitrate_configured_ = + std::max(min_bitrate, congestion_controller::GetMinBitrateBps()); + if (max_bitrate > 0) { + max_bitrate_configured_ = + std::max(min_bitrate_configured_, max_bitrate); + } else { + max_bitrate_configured_ = kDefaultMaxBitrateBps; + } +} + +int SendSideBandwidthEstimation::GetMinBitrate() const { + return min_bitrate_configured_; +} + +void SendSideBandwidthEstimation::CurrentEstimate(int* bitrate, + uint8_t* loss, + int64_t* rtt) const { + *bitrate = current_bitrate_bps_; + *loss = last_fraction_loss_; + *rtt = last_round_trip_time_ms_; +} + +void SendSideBandwidthEstimation::UpdateReceiverEstimate( + int64_t now_ms, uint32_t bandwidth) { + bwe_incoming_ = bandwidth; + CapBitrateToThresholds(now_ms, current_bitrate_bps_); +} + +void SendSideBandwidthEstimation::UpdateDelayBasedEstimate( + int64_t now_ms, + uint32_t bitrate_bps) { + delay_based_bitrate_bps_ = bitrate_bps; + CapBitrateToThresholds(now_ms, current_bitrate_bps_); +} + +void SendSideBandwidthEstimation::UpdateReceiverBlock(uint8_t fraction_loss, + int64_t rtt, + int number_of_packets, + int64_t now_ms) { + last_feedback_ms_ = now_ms; + if (first_report_time_ms_ == -1) + first_report_time_ms_ = now_ms; + + // Update RTT if we were able to compute an RTT based on this RTCP. + // FlexFEC doesn't send RTCP SR, which means we won't be able to compute RTT. + if (rtt > 0) + last_round_trip_time_ms_ = rtt; + + // Check sequence number diff and weight loss report + if (number_of_packets > 0) { + // Calculate number of lost packets. + const int num_lost_packets_Q8 = fraction_loss * number_of_packets; + // Accumulate reports. + lost_packets_since_last_loss_update_Q8_ += num_lost_packets_Q8; + expected_packets_since_last_loss_update_ += number_of_packets; + + // Don't generate a loss rate until it can be based on enough packets. + if (expected_packets_since_last_loss_update_ < kLimitNumPackets) + return; + + has_decreased_since_last_fraction_loss_ = false; + last_fraction_loss_ = lost_packets_since_last_loss_update_Q8_ / + expected_packets_since_last_loss_update_; + + // Reset accumulators. + lost_packets_since_last_loss_update_Q8_ = 0; + expected_packets_since_last_loss_update_ = 0; + last_packet_report_ms_ = now_ms; + UpdateEstimate(now_ms); + } + UpdateUmaStats(now_ms, rtt, (fraction_loss * number_of_packets) >> 8); +} + +void SendSideBandwidthEstimation::UpdateUmaStats(int64_t now_ms, + int64_t rtt, + int lost_packets) { + int bitrate_kbps = static_cast((current_bitrate_bps_ + 500) / 1000); + for (size_t i = 0; i < kNumUmaRampupMetrics; ++i) { + if (!rampup_uma_stats_updated_[i] && + bitrate_kbps >= kUmaRampupMetrics[i].bitrate_kbps) { + RTC_HISTOGRAMS_COUNTS_100000(i, kUmaRampupMetrics[i].metric_name, + now_ms - first_report_time_ms_); + rampup_uma_stats_updated_[i] = true; + } + } + if (IsInStartPhase(now_ms)) { + initially_lost_packets_ += lost_packets; + } else if (uma_update_state_ == kNoUpdate) { + uma_update_state_ = kFirstDone; + bitrate_at_2_seconds_kbps_ = bitrate_kbps; + RTC_HISTOGRAM_COUNTS("WebRTC.BWE.InitiallyLostPackets", + initially_lost_packets_, 0, 100, 50); + RTC_HISTOGRAM_COUNTS("WebRTC.BWE.InitialRtt", static_cast(rtt), 0, + 2000, 50); + RTC_HISTOGRAM_COUNTS("WebRTC.BWE.InitialBandwidthEstimate", + bitrate_at_2_seconds_kbps_, 0, 2000, 50); + } else if (uma_update_state_ == kFirstDone && + now_ms - first_report_time_ms_ >= kBweConverganceTimeMs) { + uma_update_state_ = kDone; + int bitrate_diff_kbps = + std::max(bitrate_at_2_seconds_kbps_ - bitrate_kbps, 0); + RTC_HISTOGRAM_COUNTS("WebRTC.BWE.InitialVsConvergedDiff", bitrate_diff_kbps, + 0, 2000, 50); + } +} + +void SendSideBandwidthEstimation::UpdateEstimate(int64_t now_ms) { + uint32_t new_bitrate = current_bitrate_bps_; + // We trust the REMB and/or delay-based estimate during the first 2 seconds if + // we haven't had any packet loss reported, to allow startup bitrate probing. + if (last_fraction_loss_ == 0 && IsInStartPhase(now_ms)) { + new_bitrate = std::max(bwe_incoming_, new_bitrate); + new_bitrate = std::max(delay_based_bitrate_bps_, new_bitrate); + + if (new_bitrate != current_bitrate_bps_) { + min_bitrate_history_.clear(); + min_bitrate_history_.push_back( + std::make_pair(now_ms, current_bitrate_bps_)); + CapBitrateToThresholds(now_ms, new_bitrate); + return; + } + } + UpdateMinHistory(now_ms); + if (last_packet_report_ms_ == -1) { + // No feedback received. + CapBitrateToThresholds(now_ms, current_bitrate_bps_); + return; + } + int64_t time_since_packet_report_ms = now_ms - last_packet_report_ms_; + int64_t time_since_feedback_ms = now_ms - last_feedback_ms_; + if (time_since_packet_report_ms < 1.2 * kFeedbackIntervalMs) { + // We only care about loss above a given bitrate threshold. + float loss = last_fraction_loss_ / 256.0f; + // We only make decisions based on loss when the bitrate is above a + // threshold. This is a crude way of handling loss which is uncorrelated + // to congestion. + if (current_bitrate_bps_ < bitrate_threshold_bps_ || + loss <= low_loss_threshold_) { + // Loss < 2%: Increase rate by 8% of the min bitrate in the last + // kBweIncreaseIntervalMs. + // Note that by remembering the bitrate over the last second one can + // rampup up one second faster than if only allowed to start ramping + // at 8% per second rate now. E.g.: + // If sending a constant 100kbps it can rampup immediatly to 108kbps + // whenever a receiver report is received with lower packet loss. + // If instead one would do: current_bitrate_bps_ *= 1.08^(delta time), + // it would take over one second since the lower packet loss to achieve + // 108kbps. + new_bitrate = static_cast( + min_bitrate_history_.front().second * 1.08 + 0.5); + + // Add 1 kbps extra, just to make sure that we do not get stuck + // (gives a little extra increase at low rates, negligible at higher + // rates). + new_bitrate += 1000; + } else if (current_bitrate_bps_ > bitrate_threshold_bps_) { + if (loss <= high_loss_threshold_) { + // Loss between 2% - 10%: Do nothing. + } else { + // Loss > 10%: Limit the rate decreases to once a kBweDecreaseIntervalMs + // + rtt. + if (!has_decreased_since_last_fraction_loss_ && + (now_ms - time_last_decrease_ms_) >= + (kBweDecreaseIntervalMs + last_round_trip_time_ms_)) { + time_last_decrease_ms_ = now_ms; + + // Reduce rate: + // newRate = rate * (1 - 0.5*lossRate); + // where packetLoss = 256*lossRate; + new_bitrate = static_cast( + (current_bitrate_bps_ * + static_cast(512 - last_fraction_loss_)) / + 512.0); + has_decreased_since_last_fraction_loss_ = true; + } + } + } + } else if (time_since_feedback_ms > + kFeedbackTimeoutIntervals * kFeedbackIntervalMs && + (last_timeout_ms_ == -1 || + now_ms - last_timeout_ms_ > kTimeoutIntervalMs)) { + if (in_timeout_experiment_) { + RTC_LOG(LS_WARNING) << "Feedback timed out (" << time_since_feedback_ms + << " ms), reducing bitrate."; + new_bitrate *= 0.8; + // Reset accumulators since we've already acted on missing feedback and + // shouldn't to act again on these old lost packets. + lost_packets_since_last_loss_update_Q8_ = 0; + expected_packets_since_last_loss_update_ = 0; + last_timeout_ms_ = now_ms; + } + } + + CapBitrateToThresholds(now_ms, new_bitrate); +} + +bool SendSideBandwidthEstimation::IsInStartPhase(int64_t now_ms) const { + return first_report_time_ms_ == -1 || + now_ms - first_report_time_ms_ < kStartPhaseMs; +} + +void SendSideBandwidthEstimation::UpdateMinHistory(int64_t now_ms) { + // Remove old data points from history. + // Since history precision is in ms, add one so it is able to increase + // bitrate if it is off by as little as 0.5ms. + while (!min_bitrate_history_.empty() && + now_ms - min_bitrate_history_.front().first + 1 > + kBweIncreaseIntervalMs) { + min_bitrate_history_.pop_front(); + } + + // Typical minimum sliding-window algorithm: Pop values higher than current + // bitrate before pushing it. + while (!min_bitrate_history_.empty() && + current_bitrate_bps_ <= min_bitrate_history_.back().second) { + min_bitrate_history_.pop_back(); + } + + min_bitrate_history_.push_back(std::make_pair(now_ms, current_bitrate_bps_)); +} + +void SendSideBandwidthEstimation::CapBitrateToThresholds(int64_t now_ms, + uint32_t bitrate_bps) { + if (bwe_incoming_ > 0 && bitrate_bps > bwe_incoming_) { + bitrate_bps = bwe_incoming_; + } + if (delay_based_bitrate_bps_ > 0 && bitrate_bps > delay_based_bitrate_bps_) { + bitrate_bps = delay_based_bitrate_bps_; + } + if (bitrate_bps > max_bitrate_configured_) { + bitrate_bps = max_bitrate_configured_; + } + if (bitrate_bps < min_bitrate_configured_) { + if (last_low_bitrate_log_ms_ == -1 || + now_ms - last_low_bitrate_log_ms_ > kLowBitrateLogPeriodMs) { + RTC_LOG(LS_WARNING) << "Estimated available bandwidth " + << bitrate_bps / 1000 + << " kbps is below configured min bitrate " + << min_bitrate_configured_ / 1000 << " kbps."; + last_low_bitrate_log_ms_ = now_ms; + } + bitrate_bps = min_bitrate_configured_; + } + + if (bitrate_bps != current_bitrate_bps_ || + last_fraction_loss_ != last_logged_fraction_loss_ || + now_ms - last_rtc_event_log_ms_ > kRtcEventLogPeriodMs) { + event_log_->Log(rtc::MakeUnique( + bitrate_bps, last_fraction_loss_, + expected_packets_since_last_loss_update_)); + last_logged_fraction_loss_ = last_fraction_loss_; + last_rtc_event_log_ms_ = now_ms; + } + current_bitrate_bps_ = bitrate_bps; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.h b/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.h new file mode 100644 index 0000000000..59d1c3280d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + * FEC and NACK added bitrate is handled outside class + */ + +#ifndef MODULES_BITRATE_CONTROLLER_SEND_SIDE_BANDWIDTH_ESTIMATION_H_ +#define MODULES_BITRATE_CONTROLLER_SEND_SIDE_BANDWIDTH_ESTIMATION_H_ + +#include +#include +#include + +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { + +class RtcEventLog; + +class SendSideBandwidthEstimation { + public: + SendSideBandwidthEstimation() = delete; + explicit SendSideBandwidthEstimation(RtcEventLog* event_log); + virtual ~SendSideBandwidthEstimation(); + + void CurrentEstimate(int* bitrate, uint8_t* loss, int64_t* rtt) const; + + // Call periodically to update estimate. + void UpdateEstimate(int64_t now_ms); + + // Call when we receive a RTCP message with TMMBR or REMB. + void UpdateReceiverEstimate(int64_t now_ms, uint32_t bandwidth); + + // Call when a new delay-based estimate is available. + void UpdateDelayBasedEstimate(int64_t now_ms, uint32_t bitrate_bps); + + // Call when we receive a RTCP message with a ReceiveBlock. + void UpdateReceiverBlock(uint8_t fraction_loss, + int64_t rtt, + int number_of_packets, + int64_t now_ms); + + void SetBitrates(int send_bitrate, + int min_bitrate, + int max_bitrate); + void SetSendBitrate(int bitrate); + void SetMinMaxBitrate(int min_bitrate, int max_bitrate); + int GetMinBitrate() const; + + private: + enum UmaState { kNoUpdate, kFirstDone, kDone }; + + bool IsInStartPhase(int64_t now_ms) const; + + void UpdateUmaStats(int64_t now_ms, int64_t rtt, int lost_packets); + + // Updates history of min bitrates. + // After this method returns min_bitrate_history_.front().second contains the + // min bitrate used during last kBweIncreaseIntervalMs. + void UpdateMinHistory(int64_t now_ms); + + // Cap |bitrate_bps| to [min_bitrate_configured_, max_bitrate_configured_] and + // set |current_bitrate_bps_| to the capped value and updates the event log. + void CapBitrateToThresholds(int64_t now_ms, uint32_t bitrate_bps); + + std::deque > min_bitrate_history_; + + // incoming filters + int lost_packets_since_last_loss_update_Q8_; + int expected_packets_since_last_loss_update_; + + uint32_t current_bitrate_bps_; + uint32_t min_bitrate_configured_; + uint32_t max_bitrate_configured_; + int64_t last_low_bitrate_log_ms_; + + bool has_decreased_since_last_fraction_loss_; + int64_t last_feedback_ms_; + int64_t last_packet_report_ms_; + int64_t last_timeout_ms_; + uint8_t last_fraction_loss_; + uint8_t last_logged_fraction_loss_; + int64_t last_round_trip_time_ms_; + + uint32_t bwe_incoming_; + uint32_t delay_based_bitrate_bps_; + int64_t time_last_decrease_ms_; + int64_t first_report_time_ms_; + int initially_lost_packets_; + int bitrate_at_2_seconds_kbps_; + UmaState uma_update_state_; + std::vector rampup_uma_stats_updated_; + RtcEventLog* event_log_; + int64_t last_rtc_event_log_ms_; + bool in_timeout_experiment_; + float low_loss_threshold_; + float high_loss_threshold_; + uint32_t bitrate_threshold_bps_; +}; +} // namespace webrtc +#endif // MODULES_BITRATE_CONTROLLER_SEND_SIDE_BANDWIDTH_ESTIMATION_H_ diff --git a/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation_unittest.cc b/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation_unittest.cc new file mode 100644 index 0000000000..567d81f610 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/bitrate_controller/send_side_bandwidth_estimation_unittest.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "logging/rtc_event_log/events/rtc_event_bwe_update_loss_based.h" +#include "logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "modules/bitrate_controller/send_side_bandwidth_estimation.h" +#include "test/gtest.h" + +namespace webrtc { + +MATCHER(LossBasedBweUpdateWithBitrateOnly, "") { + if (arg->GetType() != RtcEvent::Type::BweUpdateLossBased) { + return false; + } + auto bwe_event = static_cast(arg); + return bwe_event->bitrate_bps_ > 0 && bwe_event->fraction_loss_ == 0; +} + +MATCHER(LossBasedBweUpdateWithBitrateAndLossFraction, "") { + if (arg->GetType() != RtcEvent::Type::BweUpdateLossBased) { + return false; + } + auto bwe_event = static_cast(arg); + return bwe_event->bitrate_bps_ > 0 && bwe_event->fraction_loss_ > 0; +} + +void TestProbing(bool use_delay_based) { + MockRtcEventLog event_log; + SendSideBandwidthEstimation bwe(&event_log); + bwe.SetMinMaxBitrate(100000, 1500000); + bwe.SetSendBitrate(200000); + + const int kRembBps = 1000000; + const int kSecondRembBps = kRembBps + 500000; + int64_t now_ms = 0; + + bwe.UpdateReceiverBlock(0, 50, 1, now_ms); + + // Initial REMB applies immediately. + if (use_delay_based) { + bwe.UpdateDelayBasedEstimate(now_ms, kRembBps); + } else { + bwe.UpdateReceiverEstimate(now_ms, kRembBps); + } + bwe.UpdateEstimate(now_ms); + int bitrate; + uint8_t fraction_loss; + int64_t rtt; + bwe.CurrentEstimate(&bitrate, &fraction_loss, &rtt); + EXPECT_EQ(kRembBps, bitrate); + + // Second REMB doesn't apply immediately. + now_ms += 2001; + if (use_delay_based) { + bwe.UpdateDelayBasedEstimate(now_ms, kSecondRembBps); + } else { + bwe.UpdateReceiverEstimate(now_ms, kSecondRembBps); + } + bwe.UpdateEstimate(now_ms); + bitrate = 0; + bwe.CurrentEstimate(&bitrate, &fraction_loss, &rtt); + EXPECT_EQ(kRembBps, bitrate); +} + +TEST(SendSideBweTest, InitialRembWithProbing) { + TestProbing(false); +} + +TEST(SendSideBweTest, InitialDelayBasedBweWithProbing) { + TestProbing(true); +} + +TEST(SendSideBweTest, DoesntReapplyBitrateDecreaseWithoutFollowingRemb) { + MockRtcEventLog event_log; + EXPECT_CALL(event_log, LogProxy(LossBasedBweUpdateWithBitrateOnly())) + .Times(1); + EXPECT_CALL(event_log, + LogProxy(LossBasedBweUpdateWithBitrateAndLossFraction())) + .Times(2); + SendSideBandwidthEstimation bwe(&event_log); + static const int kMinBitrateBps = 100000; + static const int kInitialBitrateBps = 1000000; + bwe.SetMinMaxBitrate(kMinBitrateBps, 1500000); + bwe.SetSendBitrate(kInitialBitrateBps); + + static const uint8_t kFractionLoss = 128; + static const int64_t kRttMs = 50; + + int64_t now_ms = 0; + int bitrate_bps; + uint8_t fraction_loss; + int64_t rtt_ms; + bwe.CurrentEstimate(&bitrate_bps, &fraction_loss, &rtt_ms); + EXPECT_EQ(kInitialBitrateBps, bitrate_bps); + EXPECT_EQ(0, fraction_loss); + EXPECT_EQ(0, rtt_ms); + + // Signal heavy loss to go down in bitrate. + bwe.UpdateReceiverBlock(kFractionLoss, kRttMs, 100, now_ms); + // Trigger an update 2 seconds later to not be rate limited. + now_ms += 1000; + bwe.UpdateEstimate(now_ms); + + bwe.CurrentEstimate(&bitrate_bps, &fraction_loss, &rtt_ms); + EXPECT_LT(bitrate_bps, kInitialBitrateBps); + // Verify that the obtained bitrate isn't hitting the min bitrate, or this + // test doesn't make sense. If this ever happens, update the thresholds or + // loss rates so that it doesn't hit min bitrate after one bitrate update. + EXPECT_GT(bitrate_bps, kMinBitrateBps); + EXPECT_EQ(kFractionLoss, fraction_loss); + EXPECT_EQ(kRttMs, rtt_ms); + + // Triggering an update shouldn't apply further downgrade nor upgrade since + // there's no intermediate receiver block received indicating whether this is + // currently good or not. + int last_bitrate_bps = bitrate_bps; + // Trigger an update 2 seconds later to not be rate limited (but it still + // shouldn't update). + now_ms += 1000; + bwe.UpdateEstimate(now_ms); + bwe.CurrentEstimate(&bitrate_bps, &fraction_loss, &rtt_ms); + + EXPECT_EQ(last_bitrate_bps, bitrate_bps); + // The old loss rate should still be applied though. + EXPECT_EQ(kFractionLoss, fraction_loss); + EXPECT_EQ(kRttMs, rtt_ms); +} + +TEST(SendSideBweTest, SettingSendBitrateOverridesDelayBasedEstimate) { + ::testing::NiceMock event_log; + SendSideBandwidthEstimation bwe(&event_log); + static const int kMinBitrateBps = 10000; + static const int kMaxBitrateBps = 10000000; + static const int kInitialBitrateBps = 300000; + static const int kDelayBasedBitrateBps = 350000; + static const int kForcedHighBitrate = 2500000; + + int64_t now_ms = 0; + int bitrate_bps; + uint8_t fraction_loss; + int64_t rtt_ms; + + bwe.SetMinMaxBitrate(kMinBitrateBps, kMaxBitrateBps); + bwe.SetSendBitrate(kInitialBitrateBps); + + bwe.UpdateDelayBasedEstimate(now_ms, kDelayBasedBitrateBps); + bwe.UpdateEstimate(now_ms); + bwe.CurrentEstimate(&bitrate_bps, &fraction_loss, &rtt_ms); + EXPECT_GE(bitrate_bps, kInitialBitrateBps); + EXPECT_LE(bitrate_bps, kDelayBasedBitrateBps); + + bwe.SetSendBitrate(kForcedHighBitrate); + bwe.CurrentEstimate(&bitrate_bps, &fraction_loss, &rtt_ms); + EXPECT_EQ(bitrate_bps, kForcedHighBitrate); +} + + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/BUILD.gn b/third_party/libwebrtc/webrtc/modules/congestion_controller/BUILD.gn new file mode 100644 index 0000000000..6f5232d6eb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/BUILD.gn @@ -0,0 +1,122 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_static_library("congestion_controller") { + sources = [ + "acknowledged_bitrate_estimator.cc", + "acknowledged_bitrate_estimator.h", + "bitrate_estimator.cc", + "bitrate_estimator.h", + "delay_based_bwe.cc", + "delay_based_bwe.h", + "include/receive_side_congestion_controller.h", + "include/send_side_congestion_controller.h", + "median_slope_estimator.cc", + "median_slope_estimator.h", + "probe_bitrate_estimator.cc", + "probe_bitrate_estimator.h", + "probe_controller.cc", + "probe_controller.h", + "receive_side_congestion_controller.cc", + "send_side_congestion_controller.cc", + "transport_feedback_adapter.cc", + "transport_feedback_adapter.h", + "trendline_estimator.cc", + "trendline_estimator.h", + ] + + if (rtc_enable_bwe_test_logging) { + defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=1" ] + } else { + defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=0" ] + } + + # TODO(jschuh): Bug 1348: fix this warning. + configs += [ "//build/config/compiler:no_size_t_to_int_warning" ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + "..:module_api", + "../..:webrtc_common", + "../../api:optional", + "../../logging:rtc_event_log_api", + "../../rtc_base:rtc_base", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_numerics", + "../../system_wrappers", + "../bitrate_controller", + "../pacing", + "../remote_bitrate_estimator", + "../rtp_rtcp", + "../utility", + ] + + if (!build_with_mozilla) { + deps += [ "../../rtc_base:rtc_base" ] + } +} + +if (rtc_include_tests) { + rtc_source_set("congestion_controller_unittests") { + testonly = true + + sources = [ + "acknowledged_bitrate_estimator_unittest.cc", + "congestion_controller_unittests_helper.cc", + "congestion_controller_unittests_helper.h", + "delay_based_bwe_unittest.cc", + "delay_based_bwe_unittest_helper.cc", + "delay_based_bwe_unittest_helper.h", + "median_slope_estimator_unittest.cc", + "probe_bitrate_estimator_unittest.cc", + "probe_controller_unittest.cc", + "receive_side_congestion_controller_unittest.cc", + "send_side_congestion_controller_unittest.cc", + "transport_feedback_adapter_unittest.cc", + "trendline_estimator_unittest.cc", + ] + deps = [ + ":congestion_controller", + ":mock_congestion_controller", + "../../rtc_base:rtc_base", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_base_tests_utils", + "../../system_wrappers:system_wrappers", + "../../test:field_trial", + "../../test:test_support", + "../bitrate_controller:bitrate_controller", + "../pacing:mock_paced_sender", + "../pacing:pacing", + "../remote_bitrate_estimator:remote_bitrate_estimator", + "../rtp_rtcp:rtp_rtcp", + "//testing/gmock", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + + rtc_source_set("mock_congestion_controller") { + testonly = true + sources = [ + "include/mock/mock_congestion_observer.h", + "include/mock/mock_send_side_congestion_controller.h", + ] + deps = [ + ":congestion_controller", + "../../test:test_support", + ] + } +} diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/DEPS b/third_party/libwebrtc/webrtc/modules/congestion_controller/DEPS new file mode 100644 index 0000000000..2ed9952e22 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/DEPS @@ -0,0 +1,5 @@ +include_rules = [ + "+logging/rtc_event_log", + "+system_wrappers", + "+video", +] diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/OWNERS b/third_party/libwebrtc/webrtc/modules/congestion_controller/OWNERS new file mode 100644 index 0000000000..0a94c47784 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/OWNERS @@ -0,0 +1,9 @@ +stefan@webrtc.org +terelius@webrtc.org +philipel@webrtc.org +mflodman@webrtc.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.cc new file mode 100644 index 0000000000..223c4769e6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/acknowledged_bitrate_estimator.h" + +#include + +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { + +namespace { +bool IsInSendTimeHistory(const PacketFeedback& packet) { + return packet.send_time_ms >= 0; +} +} // namespace + +AcknowledgedBitrateEstimator::AcknowledgedBitrateEstimator() + : AcknowledgedBitrateEstimator(rtc::MakeUnique()) {} + +AcknowledgedBitrateEstimator::AcknowledgedBitrateEstimator( + std::unique_ptr bitrate_estimator) + : bitrate_estimator_(std::move(bitrate_estimator)) {} + +void AcknowledgedBitrateEstimator::IncomingPacketFeedbackVector( + const std::vector& packet_feedback_vector) { + RTC_DCHECK(std::is_sorted(packet_feedback_vector.begin(), + packet_feedback_vector.end(), + PacketFeedbackComparator())); + for (const auto& packet : packet_feedback_vector) { + if (IsInSendTimeHistory(packet)) { + MaybeExpectFastRateChange(packet.send_time_ms); + bitrate_estimator_->Update(packet.arrival_time_ms, packet.payload_size); + } + } +} + +rtc::Optional AcknowledgedBitrateEstimator::bitrate_bps() const { + return bitrate_estimator_->bitrate_bps(); +} + +void AcknowledgedBitrateEstimator::SetAlrEndedTimeMs( + int64_t alr_ended_time_ms) { + alr_ended_time_ms_.emplace(alr_ended_time_ms); +} + +void AcknowledgedBitrateEstimator::MaybeExpectFastRateChange( + int64_t packet_send_time_ms) { + if (alr_ended_time_ms_ && packet_send_time_ms > *alr_ended_time_ms_) { + bitrate_estimator_->ExpectFastRateChange(); + alr_ended_time_ms_.reset(); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.h new file mode 100644 index 0000000000..72c1023044 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_ACKNOWLEDGED_BITRATE_ESTIMATOR_H_ +#define MODULES_CONGESTION_CONTROLLER_ACKNOWLEDGED_BITRATE_ESTIMATOR_H_ + +#include +#include + +#include "api/optional.h" +#include "modules/congestion_controller/bitrate_estimator.h" + +namespace webrtc { + +struct PacketFeedback; + +class AcknowledgedBitrateEstimator { + public: + explicit AcknowledgedBitrateEstimator( + std::unique_ptr bitrate_estimator); + + AcknowledgedBitrateEstimator(); + + void IncomingPacketFeedbackVector( + const std::vector& packet_feedback_vector); + rtc::Optional bitrate_bps() const; + void SetAlrEndedTimeMs(int64_t alr_ended_time_ms); + + private: + void MaybeExpectFastRateChange(int64_t packet_arrival_time_ms); + rtc::Optional alr_ended_time_ms_; + std::unique_ptr bitrate_estimator_; +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_ACKNOWLEDGED_BITRATE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator_unittest.cc new file mode 100644 index 0000000000..d7bf736781 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator_unittest.cc @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/acknowledged_bitrate_estimator.h" + +#include + +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "rtc_base/fakeclock.h" +#include "rtc_base/ptr_util.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using testing::_; +using testing::NiceMock; +using testing::InSequence; +using testing::Return; + +namespace webrtc { + +namespace { + +constexpr int64_t kFirstArrivalTimeMs = 10; +constexpr int64_t kFirstSendTimeMs = 10; +constexpr uint16_t kSequenceNumber = 1; +constexpr size_t kPayloadSize = 10; + +class MockBitrateEstimator : public BitrateEstimator { + public: + MOCK_METHOD2(Update, void(int64_t now_ms, int bytes)); + MOCK_CONST_METHOD0(bitrate_bps, rtc::Optional()); + MOCK_METHOD0(ExpectFastRateChange, void()); +}; + +struct AcknowledgedBitrateEstimatorTestStates { + std::unique_ptr acknowledged_bitrate_estimator; + MockBitrateEstimator* mock_bitrate_estimator; +}; + +AcknowledgedBitrateEstimatorTestStates CreateTestStates() { + AcknowledgedBitrateEstimatorTestStates states; + auto mock_bitrate_estimator = rtc::MakeUnique(); + states.mock_bitrate_estimator = mock_bitrate_estimator.get(); + states.acknowledged_bitrate_estimator = + rtc::MakeUnique( + std::move(mock_bitrate_estimator)); + return states; +} + +std::vector CreateFeedbackVector() { + std::vector packet_feedback_vector; + const PacedPacketInfo pacing_info; + packet_feedback_vector.push_back( + PacketFeedback(kFirstArrivalTimeMs, kFirstSendTimeMs, kSequenceNumber, + kPayloadSize, pacing_info)); + packet_feedback_vector.push_back( + PacketFeedback(kFirstArrivalTimeMs + 10, kFirstSendTimeMs + 10, + kSequenceNumber, kPayloadSize + 10, pacing_info)); + return packet_feedback_vector; +} + +} // anonymous namespace + +TEST(TestAcknowledgedBitrateEstimator, DontAddPacketsWhichAreNotInSendHistory) { + auto states = CreateTestStates(); + std::vector packet_feedback_vector; + packet_feedback_vector.push_back( + PacketFeedback(kFirstArrivalTimeMs, kSequenceNumber)); + EXPECT_CALL(*states.mock_bitrate_estimator, Update(_, _)).Times(0); + states.acknowledged_bitrate_estimator->IncomingPacketFeedbackVector( + packet_feedback_vector); +} + +TEST(TestAcknowledgedBitrateEstimator, UpdateBandwith) { + auto states = CreateTestStates(); + auto packet_feedback_vector = CreateFeedbackVector(); + { + InSequence dummy; + EXPECT_CALL( + *states.mock_bitrate_estimator, + Update(packet_feedback_vector[0].arrival_time_ms, + static_cast(packet_feedback_vector[0].payload_size))) + .Times(1); + EXPECT_CALL( + *states.mock_bitrate_estimator, + Update(packet_feedback_vector[1].arrival_time_ms, + static_cast(packet_feedback_vector[1].payload_size))) + .Times(1); + } + states.acknowledged_bitrate_estimator->IncomingPacketFeedbackVector( + packet_feedback_vector); +} + +TEST(TestAcknowledgedBitrateEstimator, ExpectFastRateChangeWhenLeftAlr) { + auto states = CreateTestStates(); + auto packet_feedback_vector = CreateFeedbackVector(); + { + InSequence dummy; + EXPECT_CALL( + *states.mock_bitrate_estimator, + Update(packet_feedback_vector[0].arrival_time_ms, + static_cast(packet_feedback_vector[0].payload_size))) + .Times(1); + EXPECT_CALL(*states.mock_bitrate_estimator, ExpectFastRateChange()) + .Times(1); + EXPECT_CALL( + *states.mock_bitrate_estimator, + Update(packet_feedback_vector[1].arrival_time_ms, + static_cast(packet_feedback_vector[1].payload_size))) + .Times(1); + } + states.acknowledged_bitrate_estimator->SetAlrEndedTimeMs(kFirstArrivalTimeMs + + 1); + states.acknowledged_bitrate_estimator->IncomingPacketFeedbackVector( + packet_feedback_vector); +} + +TEST(TestAcknowledgedBitrateEstimator, ReturnBitrate) { + auto states = CreateTestStates(); + rtc::Optional return_value(42); + EXPECT_CALL(*states.mock_bitrate_estimator, bitrate_bps()) + .Times(1) + .WillOnce(Return(return_value)); + EXPECT_EQ(return_value, states.acknowledged_bitrate_estimator->bitrate_bps()); +} + +} // namespace webrtc*/ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.cc new file mode 100644 index 0000000000..76c132bbe9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/bitrate_estimator.h" + +#include + +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { + +namespace { +constexpr int kInitialRateWindowMs = 500; +constexpr int kRateWindowMs = 150; +} // namespace + +BitrateEstimator::BitrateEstimator() + : sum_(0), + current_win_ms_(0), + prev_time_ms_(-1), + bitrate_estimate_(-1.0f), + bitrate_estimate_var_(50.0f) {} + +BitrateEstimator::~BitrateEstimator() = default; + +void BitrateEstimator::Update(int64_t now_ms, int bytes) { + int rate_window_ms = kRateWindowMs; + // We use a larger window at the beginning to get a more stable sample that + // we can use to initialize the estimate. + if (bitrate_estimate_ < 0.f) + rate_window_ms = kInitialRateWindowMs; + float bitrate_sample = UpdateWindow(now_ms, bytes, rate_window_ms); + if (bitrate_sample < 0.0f) + return; + if (bitrate_estimate_ < 0.0f) { + // This is the very first sample we get. Use it to initialize the estimate. + bitrate_estimate_ = bitrate_sample; + return; + } + // Define the sample uncertainty as a function of how far away it is from the + // current estimate. + float sample_uncertainty = + 10.0f * std::abs(bitrate_estimate_ - bitrate_sample) / bitrate_estimate_; + float sample_var = sample_uncertainty * sample_uncertainty; + // Update a bayesian estimate of the rate, weighting it lower if the sample + // uncertainty is large. + // The bitrate estimate uncertainty is increased with each update to model + // that the bitrate changes over time. + float pred_bitrate_estimate_var = bitrate_estimate_var_ + 5.f; + bitrate_estimate_ = (sample_var * bitrate_estimate_ + + pred_bitrate_estimate_var * bitrate_sample) / + (sample_var + pred_bitrate_estimate_var); + bitrate_estimate_var_ = sample_var * pred_bitrate_estimate_var / + (sample_var + pred_bitrate_estimate_var); + BWE_TEST_LOGGING_PLOT(1, "acknowledged_bitrate", now_ms, + bitrate_estimate_ * 1000); +} + +float BitrateEstimator::UpdateWindow(int64_t now_ms, + int bytes, + int rate_window_ms) { + // Reset if time moves backwards. + if (now_ms < prev_time_ms_) { + prev_time_ms_ = -1; + sum_ = 0; + current_win_ms_ = 0; + } + if (prev_time_ms_ >= 0) { + current_win_ms_ += now_ms - prev_time_ms_; + // Reset if nothing has been received for more than a full window. + if (now_ms - prev_time_ms_ > rate_window_ms) { + sum_ = 0; + current_win_ms_ %= rate_window_ms; + } + } + prev_time_ms_ = now_ms; + float bitrate_sample = -1.0f; + if (current_win_ms_ >= rate_window_ms) { + bitrate_sample = 8.0f * sum_ / static_cast(rate_window_ms); + current_win_ms_ -= rate_window_ms; + sum_ = 0; + } + sum_ += bytes; + return bitrate_sample; +} + +rtc::Optional BitrateEstimator::bitrate_bps() const { + if (bitrate_estimate_ < 0.f) + return rtc::nullopt; + return bitrate_estimate_ * 1000; +} + +void BitrateEstimator::ExpectFastRateChange() { + // By setting the bitrate-estimate variance to a higher value we allow the + // bitrate to change fast for the next few samples. + bitrate_estimate_var_ += 200; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.h new file mode 100644 index 0000000000..2fa1515d19 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_BITRATE_ESTIMATOR_H_ +#define MODULES_CONGESTION_CONTROLLER_BITRATE_ESTIMATOR_H_ + +#include + +#include "api/optional.h" + +namespace webrtc { + +// Computes a bayesian estimate of the throughput given acks containing +// the arrival time and payload size. Samples which are far from the current +// estimate or are based on few packets are given a smaller weight, as they +// are considered to be more likely to have been caused by, e.g., delay spikes +// unrelated to congestion. +class BitrateEstimator { + public: + BitrateEstimator(); + virtual ~BitrateEstimator(); + virtual void Update(int64_t now_ms, int bytes); + + virtual rtc::Optional bitrate_bps() const; + + virtual void ExpectFastRateChange(); + + private: + float UpdateWindow(int64_t now_ms, int bytes, int rate_window_ms); + int sum_; + int64_t current_win_ms_; + int64_t prev_time_ms_; + float bitrate_estimate_; + float bitrate_estimate_var_; +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_BITRATE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_gn/moz.build b/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_gn/moz.build new file mode 100644 index 0000000000..2a98d20d27 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_gn/moz.build @@ -0,0 +1,238 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["BWE_TEST_LOGGING_COMPILE_TIME_ENABLE"] = "0" +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.cc" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/congestion_controller/acknowledged_bitrate_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/bitrate_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller.cc", + "/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION"] = "r12b" + DEFINES["DISABLE_NACL"] = True + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["USE_OPENSSL_CERTS"] = "1" + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["__GNU_SOURCE"] = "1" + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("congestion_controller_gn") diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.cc new file mode 100644 index 0000000000..2ec54171c8 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.cc @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/congestion_controller_unittests_helper.h" + +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { +void ComparePacketFeedbackVectors(const std::vector& truth, + const std::vector& input) { + ASSERT_EQ(truth.size(), input.size()); + size_t len = truth.size(); + // truth contains the input data for the test, and input is what will be + // sent to the bandwidth estimator. truth.arrival_tims_ms is used to + // populate the transport feedback messages. As these times may be changed + // (because of resolution limits in the packets, and because of the time + // base adjustment performed by the TransportFeedbackAdapter at the first + // packet, the truth[x].arrival_time and input[x].arrival_time may not be + // equal. However, the difference must be the same for all x. + int64_t arrival_time_delta = + truth[0].arrival_time_ms - input[0].arrival_time_ms; + for (size_t i = 0; i < len; ++i) { + RTC_CHECK(truth[i].arrival_time_ms != PacketFeedback::kNotReceived); + if (input[i].arrival_time_ms != PacketFeedback::kNotReceived) { + EXPECT_EQ(truth[i].arrival_time_ms, + input[i].arrival_time_ms + arrival_time_delta); + } + EXPECT_EQ(truth[i].send_time_ms, input[i].send_time_ms); + EXPECT_EQ(truth[i].sequence_number, input[i].sequence_number); + EXPECT_EQ(truth[i].payload_size, input[i].payload_size); + EXPECT_EQ(truth[i].pacing_info, input[i].pacing_info); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.h new file mode 100644 index 0000000000..37e3870d3e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/congestion_controller_unittests_helper.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_CONGESTION_CONTROLLER_UNITTESTS_HELPER_H_ +#define MODULES_CONGESTION_CONTROLLER_CONGESTION_CONTROLLER_UNITTESTS_HELPER_H_ + +#include + +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { +void ComparePacketFeedbackVectors(const std::vector& truth, + const std::vector& input); +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_CONGESTION_CONTROLLER_UNITTESTS_HELPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.cc new file mode 100644 index 0000000000..fda0e61553 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.cc @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/delay_based_bwe.h" + +#include +#include +#include + +#include "logging/rtc_event_log/events/rtc_event_bwe_update_delay_based.h" +#include "logging/rtc_event_log/rtc_event_log.h" +#include "modules/pacing/paced_sender.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/logging.h" +#include "rtc_base/ptr_util.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace { +constexpr int kTimestampGroupLengthMs = 5; +constexpr int kAbsSendTimeFraction = 18; +constexpr int kAbsSendTimeInterArrivalUpshift = 8; +constexpr int kInterArrivalShift = + kAbsSendTimeFraction + kAbsSendTimeInterArrivalUpshift; +constexpr double kTimestampToMs = + 1000.0 / static_cast(1 << kInterArrivalShift); +// This ssrc is used to fulfill the current API but will be removed +// after the API has been changed. +constexpr uint32_t kFixedSsrc = 0; + +// Parameters for linear least squares fit of regression line to noisy data. +constexpr size_t kDefaultTrendlineWindowSize = 20; +constexpr double kDefaultTrendlineSmoothingCoeff = 0.9; +constexpr double kDefaultTrendlineThresholdGain = 4.0; + +constexpr int kMaxConsecutiveFailedLookups = 5; + +const char kBweSparseUpdateExperiment[] = "WebRTC-BweSparseUpdateExperiment"; +const char kBweWindowSizeInPacketsExperiment[] = + "WebRTC-BweWindowSizeInPackets"; + +size_t ReadTrendlineFilterWindowSize() { + std::string experiment_string = + webrtc::field_trial::FindFullName(kBweWindowSizeInPacketsExperiment); + size_t window_size; + int parsed_values = + sscanf(experiment_string.c_str(), "Enabled-%zu", &window_size); + if (parsed_values == 1) { + if (window_size > 1) + return window_size; + RTC_LOG(WARNING) << "Window size must be greater than 1."; + } + RTC_LOG(LS_WARNING) << "Failed to parse parameters for BweTrendlineFilter " + "experiment from field trial string. Using default."; + return kDefaultTrendlineWindowSize; +} +} // namespace + +namespace webrtc { + +DelayBasedBwe::Result::Result() + : updated(false), + probe(false), + target_bitrate_bps(0), + recovered_from_overuse(false) {} + +DelayBasedBwe::Result::Result(bool probe, uint32_t target_bitrate_bps) + : updated(true), + probe(probe), + target_bitrate_bps(target_bitrate_bps), + recovered_from_overuse(false) {} + +DelayBasedBwe::Result::~Result() {} + +DelayBasedBwe::DelayBasedBwe(RtcEventLog* event_log, const Clock* clock) + : event_log_(event_log), + clock_(clock), + inter_arrival_(), + trendline_estimator_(), + detector_(), + last_seen_packet_ms_(-1), + uma_recorded_(false), + probe_bitrate_estimator_(event_log), + trendline_window_size_( + webrtc::field_trial::IsEnabled(kBweWindowSizeInPacketsExperiment) + ? ReadTrendlineFilterWindowSize() + : kDefaultTrendlineWindowSize), + trendline_smoothing_coeff_(kDefaultTrendlineSmoothingCoeff), + trendline_threshold_gain_(kDefaultTrendlineThresholdGain), + consecutive_delayed_feedbacks_(0), + prev_bitrate_(0), + prev_state_(BandwidthUsage::kBwNormal), + in_sparse_update_experiment_( + webrtc::field_trial::IsEnabled(kBweSparseUpdateExperiment)) { + RTC_LOG(LS_INFO) + << "Using Trendline filter for delay change estimation with window size " + << trendline_window_size_; +} + +DelayBasedBwe::~DelayBasedBwe() {} + +DelayBasedBwe::Result DelayBasedBwe::IncomingPacketFeedbackVector( + const std::vector& packet_feedback_vector, + rtc::Optional acked_bitrate_bps) { + RTC_DCHECK(std::is_sorted(packet_feedback_vector.begin(), + packet_feedback_vector.end(), + PacketFeedbackComparator())); + RTC_DCHECK_RUNS_SERIALIZED(&network_race_); + + // TOOD(holmer): An empty feedback vector here likely means that + // all acks were too late and that the send time history had + // timed out. We should reduce the rate when this occurs. + if (packet_feedback_vector.empty()) { + RTC_LOG(LS_WARNING) << "Very late feedback received."; + return DelayBasedBwe::Result(); + } + + if (!uma_recorded_) { + RTC_HISTOGRAM_ENUMERATION(kBweTypeHistogram, + BweNames::kSendSideTransportSeqNum, + BweNames::kBweNamesMax); + uma_recorded_ = true; + } + bool overusing = false; + bool delayed_feedback = true; + bool recovered_from_overuse = false; + BandwidthUsage prev_detector_state = detector_.State(); + for (const auto& packet_feedback : packet_feedback_vector) { + if (packet_feedback.send_time_ms < 0) + continue; + delayed_feedback = false; + IncomingPacketFeedback(packet_feedback); + if (!in_sparse_update_experiment_) + overusing |= (detector_.State() == BandwidthUsage::kBwOverusing); + if (prev_detector_state == BandwidthUsage::kBwUnderusing && + detector_.State() == BandwidthUsage::kBwNormal) { + recovered_from_overuse = true; + } + prev_detector_state = detector_.State(); + } + if (in_sparse_update_experiment_) + overusing = (detector_.State() == BandwidthUsage::kBwOverusing); + + if (delayed_feedback) { + ++consecutive_delayed_feedbacks_; + if (consecutive_delayed_feedbacks_ >= kMaxConsecutiveFailedLookups) { + consecutive_delayed_feedbacks_ = 0; + return OnLongFeedbackDelay(packet_feedback_vector.back().arrival_time_ms); + } + } else { + consecutive_delayed_feedbacks_ = 0; + return MaybeUpdateEstimate(overusing, acked_bitrate_bps, + recovered_from_overuse); + } + return Result(); +} + +DelayBasedBwe::Result DelayBasedBwe::OnLongFeedbackDelay( + int64_t arrival_time_ms) { + // Estimate should always be valid since a start bitrate always is set in the + // Call constructor. An alternative would be to return an empty Result here, + // or to estimate the throughput based on the feedback we received. + RTC_DCHECK(rate_control_.ValidEstimate()); + rate_control_.SetEstimate(rate_control_.LatestEstimate() / 2, + arrival_time_ms); + Result result; + result.updated = true; + result.probe = false; + result.target_bitrate_bps = rate_control_.LatestEstimate(); + RTC_LOG(LS_WARNING) << "Long feedback delay detected, reducing BWE to " + << result.target_bitrate_bps; + return result; +} + +void DelayBasedBwe::IncomingPacketFeedback( + const PacketFeedback& packet_feedback) { + int64_t now_ms = clock_->TimeInMilliseconds(); + // Reset if the stream has timed out. + if (last_seen_packet_ms_ == -1 || + now_ms - last_seen_packet_ms_ > kStreamTimeOutMs) { + inter_arrival_.reset( + new InterArrival((kTimestampGroupLengthMs << kInterArrivalShift) / 1000, + kTimestampToMs, true)); + trendline_estimator_.reset(new TrendlineEstimator( + trendline_window_size_, trendline_smoothing_coeff_, + trendline_threshold_gain_)); + } + last_seen_packet_ms_ = now_ms; + + uint32_t send_time_24bits = + static_cast( + ((static_cast(packet_feedback.send_time_ms) + << kAbsSendTimeFraction) + + 500) / + 1000) & + 0x00FFFFFF; + // Shift up send time to use the full 32 bits that inter_arrival works with, + // so wrapping works properly. + uint32_t timestamp = send_time_24bits << kAbsSendTimeInterArrivalUpshift; + + uint32_t ts_delta = 0; + int64_t t_delta = 0; + int size_delta = 0; + if (inter_arrival_->ComputeDeltas(timestamp, packet_feedback.arrival_time_ms, + now_ms, packet_feedback.payload_size, + &ts_delta, &t_delta, &size_delta)) { + double ts_delta_ms = (1000.0 * ts_delta) / (1 << kInterArrivalShift); + trendline_estimator_->Update(t_delta, ts_delta_ms, + packet_feedback.arrival_time_ms); + detector_.Detect(trendline_estimator_->trendline_slope(), ts_delta_ms, + trendline_estimator_->num_of_deltas(), + packet_feedback.arrival_time_ms); + } + if (packet_feedback.pacing_info.probe_cluster_id != + PacedPacketInfo::kNotAProbe) { + probe_bitrate_estimator_.HandleProbeAndEstimateBitrate(packet_feedback); + } +} + +DelayBasedBwe::Result DelayBasedBwe::MaybeUpdateEstimate( + bool overusing, + rtc::Optional acked_bitrate_bps, + bool recovered_from_overuse) { + Result result; + int64_t now_ms = clock_->TimeInMilliseconds(); + + rtc::Optional probe_bitrate_bps = + probe_bitrate_estimator_.FetchAndResetLastEstimatedBitrateBps(); + // Currently overusing the bandwidth. + if (overusing) { + if (acked_bitrate_bps && + rate_control_.TimeToReduceFurther(now_ms, *acked_bitrate_bps)) { + result.updated = UpdateEstimate(now_ms, acked_bitrate_bps, overusing, + &result.target_bitrate_bps); + } else if (!acked_bitrate_bps && rate_control_.ValidEstimate() && + rate_control_.TimeToReduceFurther( + now_ms, rate_control_.LatestEstimate() / 2 - 1)) { + // Overusing before we have a measured acknowledged bitrate. We check + // TimeToReduceFurther (with a fake acknowledged bitrate) to avoid + // reducing too often. + // TODO(tschumim): Improve this and/or the acknowledged bitrate estimator + // so that we (almost) always have a bitrate estimate. + rate_control_.SetEstimate(rate_control_.LatestEstimate() / 2, now_ms); + result.updated = true; + result.probe = false; + result.target_bitrate_bps = rate_control_.LatestEstimate(); + } + } else { + if (probe_bitrate_bps) { + result.probe = true; + result.updated = true; + result.target_bitrate_bps = *probe_bitrate_bps; + rate_control_.SetEstimate(*probe_bitrate_bps, now_ms); + } else { + result.updated = UpdateEstimate(now_ms, acked_bitrate_bps, overusing, + &result.target_bitrate_bps); + result.recovered_from_overuse = recovered_from_overuse; + } + } + if ((result.updated && prev_bitrate_ != result.target_bitrate_bps) || + detector_.State() != prev_state_) { + uint32_t bitrate_bps = + result.updated ? result.target_bitrate_bps : prev_bitrate_; + + BWE_TEST_LOGGING_PLOT(1, "target_bitrate_bps", now_ms, bitrate_bps); + + if (event_log_) { + event_log_->Log(rtc::MakeUnique( + bitrate_bps, detector_.State())); + } + + prev_bitrate_ = bitrate_bps; + prev_state_ = detector_.State(); + } + return result; +} + +bool DelayBasedBwe::UpdateEstimate(int64_t now_ms, + rtc::Optional acked_bitrate_bps, + bool overusing, + uint32_t* target_bitrate_bps) { + // TODO(terelius): RateControlInput::noise_var is deprecated and will be + // removed. In the meantime, we set it to zero. + const RateControlInput input( + overusing ? BandwidthUsage::kBwOverusing : detector_.State(), + acked_bitrate_bps, 0); + *target_bitrate_bps = rate_control_.Update(&input, now_ms); + return rate_control_.ValidEstimate(); +} + +void DelayBasedBwe::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) { + rate_control_.SetRtt(avg_rtt_ms); +} + +bool DelayBasedBwe::LatestEstimate(std::vector* ssrcs, + uint32_t* bitrate_bps) const { + // Currently accessed from both the process thread (see + // ModuleRtpRtcpImpl::Process()) and the configuration thread (see + // Call::GetStats()). Should in the future only be accessed from a single + // thread. + RTC_DCHECK(ssrcs); + RTC_DCHECK(bitrate_bps); + if (!rate_control_.ValidEstimate()) + return false; + + *ssrcs = {kFixedSsrc}; + *bitrate_bps = rate_control_.LatestEstimate(); + return true; +} + +void DelayBasedBwe::SetStartBitrate(int start_bitrate_bps) { + RTC_LOG(LS_WARNING) << "BWE Setting start bitrate to: " << start_bitrate_bps; + rate_control_.SetStartBitrate(start_bitrate_bps); +} + +void DelayBasedBwe::SetMinBitrate(int min_bitrate_bps) { + // Called from both the configuration thread and the network thread. Shouldn't + // be called from the network thread in the future. + rate_control_.SetMinBitrate(min_bitrate_bps); +} + +int64_t DelayBasedBwe::GetExpectedBwePeriodMs() const { + return rate_control_.GetExpectedBandwidthPeriodMs(); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.h new file mode 100644 index 0000000000..3766cd119f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_DELAY_BASED_BWE_H_ +#define MODULES_CONGESTION_CONTROLLER_DELAY_BASED_BWE_H_ + +#include +#include +#include + +#include "modules/congestion_controller/median_slope_estimator.h" +#include "modules/congestion_controller/probe_bitrate_estimator.h" +#include "modules/congestion_controller/trendline_estimator.h" +#include "modules/remote_bitrate_estimator/aimd_rate_control.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/remote_bitrate_estimator/inter_arrival.h" +#include "modules/remote_bitrate_estimator/overuse_detector.h" +#include "modules/remote_bitrate_estimator/overuse_estimator.h" +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/race_checker.h" + +namespace webrtc { + +class RtcEventLog; + +class DelayBasedBwe { + public: + static const int64_t kStreamTimeOutMs = 2000; + + struct Result { + Result(); + Result(bool probe, uint32_t target_bitrate_bps); + ~Result(); + bool updated; + bool probe; + uint32_t target_bitrate_bps; + bool recovered_from_overuse; + }; + + DelayBasedBwe(RtcEventLog* event_log, const Clock* clock); + virtual ~DelayBasedBwe(); + + Result IncomingPacketFeedbackVector( + const std::vector& packet_feedback_vector, + rtc::Optional acked_bitrate_bps); + void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms); + bool LatestEstimate(std::vector* ssrcs, + uint32_t* bitrate_bps) const; + void SetStartBitrate(int start_bitrate_bps); + void SetMinBitrate(int min_bitrate_bps); + int64_t GetExpectedBwePeriodMs() const; + + private: + void IncomingPacketFeedback(const PacketFeedback& packet_feedback); + Result OnLongFeedbackDelay(int64_t arrival_time_ms); + Result MaybeUpdateEstimate(bool overusing, + rtc::Optional acked_bitrate_bps, + bool request_probe); + // Updates the current remote rate estimate and returns true if a valid + // estimate exists. + bool UpdateEstimate(int64_t now_ms, + rtc::Optional acked_bitrate_bps, + bool overusing, + uint32_t* target_bitrate_bps); + + rtc::RaceChecker network_race_; + RtcEventLog* const event_log_; + const Clock* const clock_; + std::unique_ptr inter_arrival_; + std::unique_ptr trendline_estimator_; + OveruseDetector detector_; + int64_t last_seen_packet_ms_; + bool uma_recorded_; + AimdRateControl rate_control_; + ProbeBitrateEstimator probe_bitrate_estimator_; + size_t trendline_window_size_; + double trendline_smoothing_coeff_; + double trendline_threshold_gain_; + int consecutive_delayed_feedbacks_; + uint32_t prev_bitrate_; + BandwidthUsage prev_state_; + bool in_sparse_update_experiment_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DelayBasedBwe); +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_DELAY_BASED_BWE_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest.cc new file mode 100644 index 0000000000..e53667b16f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest.cc @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/delay_based_bwe.h" +#include "modules/congestion_controller/delay_based_bwe_unittest_helper.h" +#include "modules/pacing/paced_sender.h" +#include "rtc_base/constructormagic.h" +#include "system_wrappers/include/clock.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int kNumProbesCluster0 = 5; +constexpr int kNumProbesCluster1 = 8; +const PacedPacketInfo kPacingInfo0(0, kNumProbesCluster0, 2000); +const PacedPacketInfo kPacingInfo1(1, kNumProbesCluster1, 4000); +constexpr float kTargetUtilizationFraction = 0.95f; +} // namespace + +TEST_F(DelayBasedBweTest, NoCrashEmptyFeedback) { + std::vector packet_feedback_vector; + bitrate_estimator_->IncomingPacketFeedbackVector(packet_feedback_vector, + rtc::nullopt); +} + +TEST_F(DelayBasedBweTest, NoCrashOnlyLostFeedback) { + std::vector packet_feedback_vector; + packet_feedback_vector.push_back( + PacketFeedback(-1, -1, 0, 1500, PacedPacketInfo())); + packet_feedback_vector.push_back( + PacketFeedback(-1, -1, 1, 1500, PacedPacketInfo())); + bitrate_estimator_->IncomingPacketFeedbackVector(packet_feedback_vector, + rtc::nullopt); +} + +TEST_F(DelayBasedBweTest, ProbeDetection) { + int64_t now_ms = clock_.TimeInMilliseconds(); + uint16_t seq_num = 0; + + // First burst sent at 8 * 1000 / 10 = 800 kbps. + for (int i = 0; i < kNumProbesCluster0; ++i) { + clock_.AdvanceTimeMilliseconds(10); + now_ms = clock_.TimeInMilliseconds(); + IncomingFeedback(now_ms, now_ms, seq_num++, 1000, kPacingInfo0); + } + EXPECT_TRUE(bitrate_observer_.updated()); + + // Second burst sent at 8 * 1000 / 5 = 1600 kbps. + for (int i = 0; i < kNumProbesCluster1; ++i) { + clock_.AdvanceTimeMilliseconds(5); + now_ms = clock_.TimeInMilliseconds(); + IncomingFeedback(now_ms, now_ms, seq_num++, 1000, kPacingInfo1); + } + + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_GT(bitrate_observer_.latest_bitrate(), 1500000u); +} + +TEST_F(DelayBasedBweTest, ProbeDetectionNonPacedPackets) { + int64_t now_ms = clock_.TimeInMilliseconds(); + uint16_t seq_num = 0; + // First burst sent at 8 * 1000 / 10 = 800 kbps, but with every other packet + // not being paced which could mess things up. + for (int i = 0; i < kNumProbesCluster0; ++i) { + clock_.AdvanceTimeMilliseconds(5); + now_ms = clock_.TimeInMilliseconds(); + IncomingFeedback(now_ms, now_ms, seq_num++, 1000, kPacingInfo0); + // Non-paced packet, arriving 5 ms after. + clock_.AdvanceTimeMilliseconds(5); + IncomingFeedback(now_ms, now_ms, seq_num++, 100, PacedPacketInfo()); + } + + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_GT(bitrate_observer_.latest_bitrate(), 800000u); +} + +TEST_F(DelayBasedBweTest, ProbeDetectionFasterArrival) { + int64_t now_ms = clock_.TimeInMilliseconds(); + uint16_t seq_num = 0; + // First burst sent at 8 * 1000 / 10 = 800 kbps. + // Arriving at 8 * 1000 / 5 = 1600 kbps. + int64_t send_time_ms = 0; + for (int i = 0; i < kNumProbesCluster0; ++i) { + clock_.AdvanceTimeMilliseconds(1); + send_time_ms += 10; + now_ms = clock_.TimeInMilliseconds(); + IncomingFeedback(now_ms, send_time_ms, seq_num++, 1000, kPacingInfo0); + } + + EXPECT_FALSE(bitrate_observer_.updated()); +} + +TEST_F(DelayBasedBweTest, ProbeDetectionSlowerArrival) { + int64_t now_ms = clock_.TimeInMilliseconds(); + uint16_t seq_num = 0; + // First burst sent at 8 * 1000 / 5 = 1600 kbps. + // Arriving at 8 * 1000 / 7 = 1142 kbps. + // Since the receive rate is significantly below the send rate, we expect to + // use 95% of the estimated capacity. + int64_t send_time_ms = 0; + for (int i = 0; i < kNumProbesCluster1; ++i) { + clock_.AdvanceTimeMilliseconds(7); + send_time_ms += 5; + now_ms = clock_.TimeInMilliseconds(); + IncomingFeedback(now_ms, send_time_ms, seq_num++, 1000, kPacingInfo1); + } + + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_NEAR(bitrate_observer_.latest_bitrate(), + kTargetUtilizationFraction * 1140000u, 10000u); +} + +TEST_F(DelayBasedBweTest, ProbeDetectionSlowerArrivalHighBitrate) { + int64_t now_ms = clock_.TimeInMilliseconds(); + uint16_t seq_num = 0; + // Burst sent at 8 * 1000 / 1 = 8000 kbps. + // Arriving at 8 * 1000 / 2 = 4000 kbps. + // Since the receive rate is significantly below the send rate, we expect to + // use 95% of the estimated capacity. + int64_t send_time_ms = 0; + for (int i = 0; i < kNumProbesCluster1; ++i) { + clock_.AdvanceTimeMilliseconds(2); + send_time_ms += 1; + now_ms = clock_.TimeInMilliseconds(); + IncomingFeedback(now_ms, send_time_ms, seq_num++, 1000, kPacingInfo1); + } + + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_NEAR(bitrate_observer_.latest_bitrate(), + kTargetUtilizationFraction * 4000000u, 10000u); +} + +TEST_F(DelayBasedBweTest, GetExpectedBwePeriodMs) { + int64_t default_interval_ms = bitrate_estimator_->GetExpectedBwePeriodMs(); + EXPECT_GT(default_interval_ms, 0); + CapacityDropTestHelper(1, true, 333, 0); + int64_t interval_ms = bitrate_estimator_->GetExpectedBwePeriodMs(); + EXPECT_GT(interval_ms, 0); + EXPECT_NE(interval_ms, default_interval_ms); +} + +TEST_F(DelayBasedBweTest, InitialBehavior) { + InitialBehaviorTestHelper(730000); +} + +TEST_F(DelayBasedBweTest, RateIncreaseReordering) { + RateIncreaseReorderingTestHelper(730000); +} +TEST_F(DelayBasedBweTest, RateIncreaseRtpTimestamps) { + RateIncreaseRtpTimestampsTestHelper(627); +} + +TEST_F(DelayBasedBweTest, CapacityDropOneStream) { + CapacityDropTestHelper(1, false, 300, 0); +} + +TEST_F(DelayBasedBweTest, CapacityDropPosOffsetChange) { + CapacityDropTestHelper(1, false, 867, 30000); +} + +TEST_F(DelayBasedBweTest, CapacityDropNegOffsetChange) { + CapacityDropTestHelper(1, false, 933, -30000); +} + +TEST_F(DelayBasedBweTest, CapacityDropOneStreamWrap) { + CapacityDropTestHelper(1, true, 333, 0); +} + +TEST_F(DelayBasedBweTest, TestTimestampGrouping) { + TestTimestampGroupingTestHelper(); +} + +TEST_F(DelayBasedBweTest, TestShortTimeoutAndWrap) { + // Simulate a client leaving and rejoining the call after 35 seconds. This + // will make abs send time wrap, so if streams aren't timed out properly + // the next 30 seconds of packets will be out of order. + TestWrappingHelper(35); +} + +TEST_F(DelayBasedBweTest, TestLongTimeoutAndWrap) { + // Simulate a client leaving and rejoining the call after some multiple of + // 64 seconds later. This will cause a zero difference in abs send times due + // to the wrap, but a big difference in arrival time, if streams aren't + // properly timed out. + TestWrappingHelper(10 * 64); +} + +TEST_F(DelayBasedBweTest, TestInitialOveruse) { + const uint32_t kStartBitrate = 300e3; + const uint32_t kInitialCapacityBps = 200e3; + const uint32_t kDummySsrc = 0; + // High FPS to ensure that we send a lot of packets in a short time. + const int kFps = 90; + + stream_generator_->AddStream(new test::RtpStream(kFps, kStartBitrate)); + stream_generator_->set_capacity_bps(kInitialCapacityBps); + + // Needed to initialize the AimdRateControl. + bitrate_estimator_->SetStartBitrate(kStartBitrate); + + // Produce 30 frames (in 1/3 second) and give them to the estimator. + uint32_t bitrate_bps = kStartBitrate; + bool seen_overuse = false; + for (int i = 0; i < 30; ++i) { + bool overuse = GenerateAndProcessFrame(kDummySsrc, bitrate_bps); + // The purpose of this test is to ensure that we back down even if we don't + // have any acknowledged bitrate estimate yet. Hence, if the test works + // as expected, we should not have a measured bitrate yet. + EXPECT_FALSE(acknowledged_bitrate_estimator_->bitrate_bps().has_value()); + if (overuse) { + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_NEAR(bitrate_observer_.latest_bitrate(), kStartBitrate / 2, 15000); + bitrate_bps = bitrate_observer_.latest_bitrate(); + seen_overuse = true; + break; + } else if (bitrate_observer_.updated()) { + bitrate_bps = bitrate_observer_.latest_bitrate(); + bitrate_observer_.Reset(); + } + } + EXPECT_TRUE(seen_overuse); + EXPECT_NEAR(bitrate_observer_.latest_bitrate(), kStartBitrate / 2, 15000); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.cc new file mode 100644 index 0000000000..b441e998cd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.cc @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/congestion_controller/delay_based_bwe_unittest_helper.h" + +#include +#include +#include + +#include "modules/congestion_controller/delay_based_bwe.h" +#include "rtc_base/checks.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { + +constexpr size_t kMtu = 1200; +constexpr uint32_t kAcceptedBitrateErrorBps = 50000; + +// Number of packets needed before we have a valid estimate. +constexpr int kNumInitialPackets = 2; + +constexpr int kInitialProbingPackets = 5; + +namespace test { + +void TestBitrateObserver::OnReceiveBitrateChanged( + const std::vector& ssrcs, + uint32_t bitrate) { + latest_bitrate_ = bitrate; + updated_ = true; +} + +RtpStream::RtpStream(int fps, int bitrate_bps) + : fps_(fps), + bitrate_bps_(bitrate_bps), + next_rtp_time_(0), + sequence_number_(0) { + RTC_CHECK_GT(fps_, 0); +} + +// Generates a new frame for this stream. If called too soon after the +// previous frame, no frame will be generated. The frame is split into +// packets. +int64_t RtpStream::GenerateFrame(int64_t time_now_us, + std::vector* packets) { + if (time_now_us < next_rtp_time_) { + return next_rtp_time_; + } + RTC_CHECK(packets != NULL); + size_t bits_per_frame = (bitrate_bps_ + fps_ / 2) / fps_; + size_t n_packets = + std::max((bits_per_frame + 4 * kMtu) / (8 * kMtu), 1u); + size_t payload_size = (bits_per_frame + 4 * n_packets) / (8 * n_packets); + for (size_t i = 0; i < n_packets; ++i) { + PacketFeedback packet(-1, sequence_number_++); + packet.send_time_ms = (time_now_us + kSendSideOffsetUs) / 1000; + packet.payload_size = payload_size; + packets->push_back(packet); + } + next_rtp_time_ = time_now_us + (1000000 + fps_ / 2) / fps_; + return next_rtp_time_; +} + +// The send-side time when the next frame can be generated. +int64_t RtpStream::next_rtp_time() const { + return next_rtp_time_; +} + +void RtpStream::set_bitrate_bps(int bitrate_bps) { + ASSERT_GE(bitrate_bps, 0); + bitrate_bps_ = bitrate_bps; +} + +int RtpStream::bitrate_bps() const { + return bitrate_bps_; +} + +bool RtpStream::Compare(const std::unique_ptr& lhs, + const std::unique_ptr& rhs) { + return lhs->next_rtp_time_ < rhs->next_rtp_time_; +} + +StreamGenerator::StreamGenerator(int capacity, int64_t time_now) + : capacity_(capacity), prev_arrival_time_us_(time_now) {} + +// Add a new stream. +void StreamGenerator::AddStream(RtpStream* stream) { + streams_.push_back(std::unique_ptr(stream)); +} + +// Set the link capacity. +void StreamGenerator::set_capacity_bps(int capacity_bps) { + ASSERT_GT(capacity_bps, 0); + capacity_ = capacity_bps; +} + +// Divides |bitrate_bps| among all streams. The allocated bitrate per stream +// is decided by the current allocation ratios. +void StreamGenerator::SetBitrateBps(int bitrate_bps) { + ASSERT_GE(streams_.size(), 0u); + int total_bitrate_before = 0; + for (const auto& stream : streams_) { + total_bitrate_before += stream->bitrate_bps(); + } + int64_t bitrate_before = 0; + int total_bitrate_after = 0; + for (const auto& stream : streams_) { + bitrate_before += stream->bitrate_bps(); + int64_t bitrate_after = + (bitrate_before * bitrate_bps + total_bitrate_before / 2) / + total_bitrate_before; + stream->set_bitrate_bps(bitrate_after - total_bitrate_after); + total_bitrate_after += stream->bitrate_bps(); + } + ASSERT_EQ(bitrate_before, total_bitrate_before); + EXPECT_EQ(total_bitrate_after, bitrate_bps); +} + +// TODO(holmer): Break out the channel simulation part from this class to make +// it possible to simulate different types of channels. +int64_t StreamGenerator::GenerateFrame(std::vector* packets, + int64_t time_now_us) { + RTC_CHECK(packets != NULL); + RTC_CHECK(packets->empty()); + RTC_CHECK_GT(capacity_, 0); + auto it = + std::min_element(streams_.begin(), streams_.end(), RtpStream::Compare); + (*it)->GenerateFrame(time_now_us, packets); + int i = 0; + for (PacketFeedback& packet : *packets) { + int capacity_bpus = capacity_ / 1000; + int64_t required_network_time_us = + (8 * 1000 * packet.payload_size + capacity_bpus / 2) / capacity_bpus; + prev_arrival_time_us_ = + std::max(time_now_us + required_network_time_us, + prev_arrival_time_us_ + required_network_time_us); + packet.arrival_time_ms = prev_arrival_time_us_ / 1000; + ++i; + } + it = std::min_element(streams_.begin(), streams_.end(), RtpStream::Compare); + return std::max((*it)->next_rtp_time(), time_now_us); +} +} // namespace test + +DelayBasedBweTest::DelayBasedBweTest() + : clock_(100000000), + acknowledged_bitrate_estimator_( + rtc::MakeUnique()), + bitrate_estimator_(new DelayBasedBwe(nullptr, &clock_)), + stream_generator_(new test::StreamGenerator(1e6, // Capacity. + clock_.TimeInMicroseconds())), + arrival_time_offset_ms_(0), + first_update_(true) {} + +DelayBasedBweTest::~DelayBasedBweTest() {} + +void DelayBasedBweTest::AddDefaultStream() { + stream_generator_->AddStream(new test::RtpStream(30, 3e5)); +} + +const uint32_t DelayBasedBweTest::kDefaultSsrc = 0; + +void DelayBasedBweTest::IncomingFeedback(int64_t arrival_time_ms, + int64_t send_time_ms, + uint16_t sequence_number, + size_t payload_size) { + IncomingFeedback(arrival_time_ms, send_time_ms, sequence_number, payload_size, + PacedPacketInfo()); +} + +void DelayBasedBweTest::IncomingFeedback(int64_t arrival_time_ms, + int64_t send_time_ms, + uint16_t sequence_number, + size_t payload_size, + const PacedPacketInfo& pacing_info) { + RTC_CHECK_GE(arrival_time_ms + arrival_time_offset_ms_, 0); + PacketFeedback packet(arrival_time_ms + arrival_time_offset_ms_, send_time_ms, + sequence_number, payload_size, pacing_info); + std::vector packets; + packets.push_back(packet); + acknowledged_bitrate_estimator_->IncomingPacketFeedbackVector(packets); + DelayBasedBwe::Result result = + bitrate_estimator_->IncomingPacketFeedbackVector( + packets, acknowledged_bitrate_estimator_->bitrate_bps()); + const uint32_t kDummySsrc = 0; + if (result.updated) { + bitrate_observer_.OnReceiveBitrateChanged({kDummySsrc}, + result.target_bitrate_bps); + } +} + +// Generates a frame of packets belonging to a stream at a given bitrate and +// with a given ssrc. The stream is pushed through a very simple simulated +// network, and is then given to the receive-side bandwidth estimator. +// Returns true if an over-use was seen, false otherwise. +// The StreamGenerator::updated() should be used to check for any changes in +// target bitrate after the call to this function. +bool DelayBasedBweTest::GenerateAndProcessFrame(uint32_t ssrc, + uint32_t bitrate_bps) { + stream_generator_->SetBitrateBps(bitrate_bps); + std::vector packets; + int64_t next_time_us = + stream_generator_->GenerateFrame(&packets, clock_.TimeInMicroseconds()); + if (packets.empty()) + return false; + + bool overuse = false; + bitrate_observer_.Reset(); + clock_.AdvanceTimeMicroseconds(1000 * packets.back().arrival_time_ms - + clock_.TimeInMicroseconds()); + for (auto& packet : packets) { + RTC_CHECK_GE(packet.arrival_time_ms + arrival_time_offset_ms_, 0); + packet.arrival_time_ms += arrival_time_offset_ms_; + } + + acknowledged_bitrate_estimator_->IncomingPacketFeedbackVector(packets); + DelayBasedBwe::Result result = + bitrate_estimator_->IncomingPacketFeedbackVector( + packets, acknowledged_bitrate_estimator_->bitrate_bps()); + const uint32_t kDummySsrc = 0; + if (result.updated) { + bitrate_observer_.OnReceiveBitrateChanged({kDummySsrc}, + result.target_bitrate_bps); + if (!first_update_ && result.target_bitrate_bps < bitrate_bps) + overuse = true; + first_update_ = false; + } + + clock_.AdvanceTimeMicroseconds(next_time_us - clock_.TimeInMicroseconds()); + return overuse; +} + +// Run the bandwidth estimator with a stream of |number_of_frames| frames, or +// until it reaches |target_bitrate|. +// Can for instance be used to run the estimator for some time to get it +// into a steady state. +uint32_t DelayBasedBweTest::SteadyStateRun(uint32_t ssrc, + int max_number_of_frames, + uint32_t start_bitrate, + uint32_t min_bitrate, + uint32_t max_bitrate, + uint32_t target_bitrate) { + uint32_t bitrate_bps = start_bitrate; + bool bitrate_update_seen = false; + // Produce |number_of_frames| frames and give them to the estimator. + for (int i = 0; i < max_number_of_frames; ++i) { + bool overuse = GenerateAndProcessFrame(ssrc, bitrate_bps); + if (overuse) { + EXPECT_LT(bitrate_observer_.latest_bitrate(), max_bitrate); + EXPECT_GT(bitrate_observer_.latest_bitrate(), min_bitrate); + bitrate_bps = bitrate_observer_.latest_bitrate(); + bitrate_update_seen = true; + } else if (bitrate_observer_.updated()) { + bitrate_bps = bitrate_observer_.latest_bitrate(); + bitrate_observer_.Reset(); + } + if (bitrate_update_seen && bitrate_bps > target_bitrate) { + break; + } + } + EXPECT_TRUE(bitrate_update_seen); + return bitrate_bps; +} + +void DelayBasedBweTest::InitialBehaviorTestHelper( + uint32_t expected_converge_bitrate) { + const int kFramerate = 50; // 50 fps to avoid rounding errors. + const int kFrameIntervalMs = 1000 / kFramerate; + const PacedPacketInfo kPacingInfo(0, 5, 5000); + uint32_t bitrate_bps = 0; + int64_t send_time_ms = 0; + uint16_t sequence_number = 0; + std::vector ssrcs; + EXPECT_FALSE(bitrate_estimator_->LatestEstimate(&ssrcs, &bitrate_bps)); + EXPECT_EQ(0u, ssrcs.size()); + clock_.AdvanceTimeMilliseconds(1000); + EXPECT_FALSE(bitrate_estimator_->LatestEstimate(&ssrcs, &bitrate_bps)); + EXPECT_FALSE(bitrate_observer_.updated()); + bitrate_observer_.Reset(); + clock_.AdvanceTimeMilliseconds(1000); + // Inserting packets for 5 seconds to get a valid estimate. + for (int i = 0; i < 5 * kFramerate + 1 + kNumInitialPackets; ++i) { + // NOTE!!! If the following line is moved under the if case then this test + // wont work on windows realease bots. + PacedPacketInfo pacing_info = + i < kInitialProbingPackets ? kPacingInfo : PacedPacketInfo(); + + if (i == kNumInitialPackets) { + EXPECT_FALSE(bitrate_estimator_->LatestEstimate(&ssrcs, &bitrate_bps)); + EXPECT_EQ(0u, ssrcs.size()); + EXPECT_FALSE(bitrate_observer_.updated()); + bitrate_observer_.Reset(); + } + IncomingFeedback(clock_.TimeInMilliseconds(), send_time_ms, + sequence_number++, kMtu, pacing_info); + clock_.AdvanceTimeMilliseconds(1000 / kFramerate); + send_time_ms += kFrameIntervalMs; + } + EXPECT_TRUE(bitrate_estimator_->LatestEstimate(&ssrcs, &bitrate_bps)); + ASSERT_EQ(1u, ssrcs.size()); + EXPECT_EQ(kDefaultSsrc, ssrcs.front()); + EXPECT_NEAR(expected_converge_bitrate, bitrate_bps, kAcceptedBitrateErrorBps); + EXPECT_TRUE(bitrate_observer_.updated()); + bitrate_observer_.Reset(); + EXPECT_EQ(bitrate_observer_.latest_bitrate(), bitrate_bps); +} + +void DelayBasedBweTest::RateIncreaseReorderingTestHelper( + uint32_t expected_bitrate_bps) { + const int kFramerate = 50; // 50 fps to avoid rounding errors. + const int kFrameIntervalMs = 1000 / kFramerate; + const PacedPacketInfo kPacingInfo(0, 5, 5000); + int64_t send_time_ms = 0; + uint16_t sequence_number = 0; + // Inserting packets for five seconds to get a valid estimate. + for (int i = 0; i < 5 * kFramerate + 1 + kNumInitialPackets; ++i) { + // NOTE!!! If the following line is moved under the if case then this test + // wont work on windows realease bots. + PacedPacketInfo pacing_info = + i < kInitialProbingPackets ? kPacingInfo : PacedPacketInfo(); + + // TODO(sprang): Remove this hack once the single stream estimator is gone, + // as it doesn't do anything in Process(). + if (i == kNumInitialPackets) { + // Process after we have enough frames to get a valid input rate estimate. + + EXPECT_FALSE(bitrate_observer_.updated()); // No valid estimate. + } + IncomingFeedback(clock_.TimeInMilliseconds(), send_time_ms, + sequence_number++, kMtu, pacing_info); + clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + send_time_ms += kFrameIntervalMs; + } + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_NEAR(expected_bitrate_bps, bitrate_observer_.latest_bitrate(), + kAcceptedBitrateErrorBps); + for (int i = 0; i < 10; ++i) { + clock_.AdvanceTimeMilliseconds(2 * kFrameIntervalMs); + send_time_ms += 2 * kFrameIntervalMs; + IncomingFeedback(clock_.TimeInMilliseconds(), send_time_ms, + sequence_number + 2, 1000); + IncomingFeedback(clock_.TimeInMilliseconds(), + send_time_ms - kFrameIntervalMs, sequence_number + 1, + 1000); + sequence_number += 2; + } + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_NEAR(expected_bitrate_bps, bitrate_observer_.latest_bitrate(), + kAcceptedBitrateErrorBps); +} + +// Make sure we initially increase the bitrate as expected. +void DelayBasedBweTest::RateIncreaseRtpTimestampsTestHelper( + int expected_iterations) { + // This threshold corresponds approximately to increasing linearly with + // bitrate(i) = 1.04 * bitrate(i-1) + 1000 + // until bitrate(i) > 500000, with bitrate(1) ~= 30000. + uint32_t bitrate_bps = 30000; + int iterations = 0; + AddDefaultStream(); + // Feed the estimator with a stream of packets and verify that it reaches + // 500 kbps at the expected time. + while (bitrate_bps < 5e5) { + bool overuse = GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); + if (overuse) { + EXPECT_GT(bitrate_observer_.latest_bitrate(), bitrate_bps); + bitrate_bps = bitrate_observer_.latest_bitrate(); + bitrate_observer_.Reset(); + } else if (bitrate_observer_.updated()) { + bitrate_bps = bitrate_observer_.latest_bitrate(); + bitrate_observer_.Reset(); + } + ++iterations; + } + ASSERT_EQ(expected_iterations, iterations); +} + +void DelayBasedBweTest::CapacityDropTestHelper( + int number_of_streams, + bool wrap_time_stamp, + uint32_t expected_bitrate_drop_delta, + int64_t receiver_clock_offset_change_ms) { + const int kFramerate = 30; + const int kStartBitrate = 900e3; + const int kMinExpectedBitrate = 800e3; + const int kMaxExpectedBitrate = 1100e3; + const uint32_t kInitialCapacityBps = 1000e3; + const uint32_t kReducedCapacityBps = 500e3; + + int steady_state_time = 0; + if (number_of_streams <= 1) { + steady_state_time = 10; + AddDefaultStream(); + } else { + steady_state_time = 10 * number_of_streams; + int bitrate_sum = 0; + int kBitrateDenom = number_of_streams * (number_of_streams - 1); + for (int i = 0; i < number_of_streams; i++) { + // First stream gets half available bitrate, while the rest share the + // remaining half i.e.: 1/2 = Sum[n/(N*(N-1))] for n=1..N-1 (rounded up) + int bitrate = kStartBitrate / 2; + if (i > 0) { + bitrate = (kStartBitrate * i + kBitrateDenom / 2) / kBitrateDenom; + } + stream_generator_->AddStream(new test::RtpStream(kFramerate, bitrate)); + bitrate_sum += bitrate; + } + ASSERT_EQ(bitrate_sum, kStartBitrate); + } + + // Run in steady state to make the estimator converge. + stream_generator_->set_capacity_bps(kInitialCapacityBps); + uint32_t bitrate_bps = SteadyStateRun( + kDefaultSsrc, steady_state_time * kFramerate, kStartBitrate, + kMinExpectedBitrate, kMaxExpectedBitrate, kInitialCapacityBps); + EXPECT_NEAR(kInitialCapacityBps, bitrate_bps, 180000u); + bitrate_observer_.Reset(); + + // Add an offset to make sure the BWE can handle it. + arrival_time_offset_ms_ += receiver_clock_offset_change_ms; + + // Reduce the capacity and verify the decrease time. + stream_generator_->set_capacity_bps(kReducedCapacityBps); + int64_t overuse_start_time = clock_.TimeInMilliseconds(); + int64_t bitrate_drop_time = -1; + for (int i = 0; i < 100 * number_of_streams; ++i) { + GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); + if (bitrate_drop_time == -1 && + bitrate_observer_.latest_bitrate() <= kReducedCapacityBps) { + bitrate_drop_time = clock_.TimeInMilliseconds(); + } + if (bitrate_observer_.updated()) + bitrate_bps = bitrate_observer_.latest_bitrate(); + } + + EXPECT_NEAR(expected_bitrate_drop_delta, + bitrate_drop_time - overuse_start_time, 33); +} + +void DelayBasedBweTest::TestTimestampGroupingTestHelper() { + const int kFramerate = 50; // 50 fps to avoid rounding errors. + const int kFrameIntervalMs = 1000 / kFramerate; + int64_t send_time_ms = 0; + uint16_t sequence_number = 0; + // Initial set of frames to increase the bitrate. 6 seconds to have enough + // time for the first estimate to be generated and for Process() to be called. + for (int i = 0; i <= 6 * kFramerate; ++i) { + IncomingFeedback(clock_.TimeInMilliseconds(), send_time_ms, + sequence_number++, 1000); + + clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + send_time_ms += kFrameIntervalMs; + } + EXPECT_TRUE(bitrate_observer_.updated()); + EXPECT_GE(bitrate_observer_.latest_bitrate(), 400000u); + + // Insert batches of frames which were sent very close in time. Also simulate + // capacity over-use to see that we back off correctly. + const int kTimestampGroupLength = 15; + for (int i = 0; i < 100; ++i) { + for (int j = 0; j < kTimestampGroupLength; ++j) { + // Insert |kTimestampGroupLength| frames with just 1 timestamp ticks in + // between. Should be treated as part of the same group by the estimator. + IncomingFeedback(clock_.TimeInMilliseconds(), send_time_ms, + sequence_number++, 100); + clock_.AdvanceTimeMilliseconds(kFrameIntervalMs / kTimestampGroupLength); + send_time_ms += 1; + } + // Increase time until next batch to simulate over-use. + clock_.AdvanceTimeMilliseconds(10); + send_time_ms += kFrameIntervalMs - kTimestampGroupLength; + } + EXPECT_TRUE(bitrate_observer_.updated()); + // Should have reduced the estimate. + EXPECT_LT(bitrate_observer_.latest_bitrate(), 400000u); +} + +void DelayBasedBweTest::TestWrappingHelper(int silence_time_s) { + const int kFramerate = 100; + const int kFrameIntervalMs = 1000 / kFramerate; + int64_t send_time_ms = 0; + uint16_t sequence_number = 0; + + for (size_t i = 0; i < 3000; ++i) { + IncomingFeedback(clock_.TimeInMilliseconds(), send_time_ms, + sequence_number++, 1000); + clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + send_time_ms += kFrameIntervalMs; + } + uint32_t bitrate_before = 0; + std::vector ssrcs; + bitrate_estimator_->LatestEstimate(&ssrcs, &bitrate_before); + + clock_.AdvanceTimeMilliseconds(silence_time_s * 1000); + send_time_ms += silence_time_s * 1000; + + for (size_t i = 0; i < 24; ++i) { + IncomingFeedback(clock_.TimeInMilliseconds(), send_time_ms, + sequence_number++, 1000); + clock_.AdvanceTimeMilliseconds(2 * kFrameIntervalMs); + send_time_ms += kFrameIntervalMs; + } + uint32_t bitrate_after = 0; + bitrate_estimator_->LatestEstimate(&ssrcs, &bitrate_after); + EXPECT_LT(bitrate_after, bitrate_before); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.h new file mode 100644 index 0000000000..23969a92ae --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/delay_based_bwe_unittest_helper.h @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_DELAY_BASED_BWE_UNITTEST_HELPER_H_ +#define MODULES_CONGESTION_CONTROLLER_DELAY_BASED_BWE_UNITTEST_HELPER_H_ + +#include +#include +#include +#include +#include + +#include "modules/congestion_controller/acknowledged_bitrate_estimator.h" +#include "modules/congestion_controller/delay_based_bwe.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "rtc_base/constructormagic.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +class TestBitrateObserver : public RemoteBitrateObserver { + public: + TestBitrateObserver() : updated_(false), latest_bitrate_(0) {} + virtual ~TestBitrateObserver() {} + + void OnReceiveBitrateChanged(const std::vector& ssrcs, + uint32_t bitrate) override; + + void Reset() { updated_ = false; } + + bool updated() const { return updated_; } + + uint32_t latest_bitrate() const { return latest_bitrate_; } + + private: + bool updated_; + uint32_t latest_bitrate_; +}; + +class RtpStream { + public: + enum { kSendSideOffsetUs = 1000000 }; + + RtpStream(int fps, int bitrate_bps); + + // Generates a new frame for this stream. If called too soon after the + // previous frame, no frame will be generated. The frame is split into + // packets. + int64_t GenerateFrame(int64_t time_now_us, + std::vector* packets); + + // The send-side time when the next frame can be generated. + int64_t next_rtp_time() const; + + void set_bitrate_bps(int bitrate_bps); + + int bitrate_bps() const; + + static bool Compare(const std::unique_ptr& lhs, + const std::unique_ptr& rhs); + + private: + int fps_; + int bitrate_bps_; + int64_t next_rtp_time_; + uint16_t sequence_number_; + + RTC_DISALLOW_COPY_AND_ASSIGN(RtpStream); +}; + +class StreamGenerator { + public: + StreamGenerator(int capacity, int64_t time_now); + + // Add a new stream. + void AddStream(RtpStream* stream); + + // Set the link capacity. + void set_capacity_bps(int capacity_bps); + + // Divides |bitrate_bps| among all streams. The allocated bitrate per stream + // is decided by the initial allocation ratios. + void SetBitrateBps(int bitrate_bps); + + // Set the RTP timestamp offset for the stream identified by |ssrc|. + void set_rtp_timestamp_offset(uint32_t ssrc, uint32_t offset); + + // TODO(holmer): Break out the channel simulation part from this class to make + // it possible to simulate different types of channels. + int64_t GenerateFrame(std::vector* packets, + int64_t time_now_us); + + private: + // Capacity of the simulated channel in bits per second. + int capacity_; + // The time when the last packet arrived. + int64_t prev_arrival_time_us_; + // All streams being transmitted on this simulated channel. + std::vector> streams_; + + RTC_DISALLOW_COPY_AND_ASSIGN(StreamGenerator); +}; +} // namespace test + +class DelayBasedBweTest : public ::testing::Test { + public: + DelayBasedBweTest(); + virtual ~DelayBasedBweTest(); + + protected: + void AddDefaultStream(); + + // Helpers to insert a single packet into the delay-based BWE. + void IncomingFeedback(int64_t arrival_time_ms, + int64_t send_time_ms, + uint16_t sequence_number, + size_t payload_size); + void IncomingFeedback(int64_t arrival_time_ms, + int64_t send_time_ms, + uint16_t sequence_number, + size_t payload_size, + const PacedPacketInfo& pacing_info); + + // Generates a frame of packets belonging to a stream at a given bitrate and + // with a given ssrc. The stream is pushed through a very simple simulated + // network, and is then given to the receive-side bandwidth estimator. + // Returns true if an over-use was seen, false otherwise. + // The StreamGenerator::updated() should be used to check for any changes in + // target bitrate after the call to this function. + bool GenerateAndProcessFrame(uint32_t ssrc, uint32_t bitrate_bps); + + // Run the bandwidth estimator with a stream of |number_of_frames| frames, or + // until it reaches |target_bitrate|. + // Can for instance be used to run the estimator for some time to get it + // into a steady state. + uint32_t SteadyStateRun(uint32_t ssrc, + int number_of_frames, + uint32_t start_bitrate, + uint32_t min_bitrate, + uint32_t max_bitrate, + uint32_t target_bitrate); + + void TestTimestampGroupingTestHelper(); + + void TestWrappingHelper(int silence_time_s); + + void InitialBehaviorTestHelper(uint32_t expected_converge_bitrate); + void RateIncreaseReorderingTestHelper(uint32_t expected_bitrate); + void RateIncreaseRtpTimestampsTestHelper(int expected_iterations); + void CapacityDropTestHelper(int number_of_streams, + bool wrap_time_stamp, + uint32_t expected_bitrate_drop_delta, + int64_t receiver_clock_offset_change_ms); + + static const uint32_t kDefaultSsrc; + + SimulatedClock clock_; // Time at the receiver. + test::TestBitrateObserver bitrate_observer_; + std::unique_ptr acknowledged_bitrate_estimator_; + std::unique_ptr bitrate_estimator_; + std::unique_ptr stream_generator_; + int64_t arrival_time_offset_ms_; + bool first_update_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DelayBasedBweTest); +}; +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_DELAY_BASED_BWE_UNITTEST_HELPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_congestion_observer.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_congestion_observer.h new file mode 100644 index 0000000000..c6883d976b --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_congestion_observer.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_INCLUDE_MOCK_MOCK_CONGESTION_OBSERVER_H_ +#define MODULES_CONGESTION_CONTROLLER_INCLUDE_MOCK_MOCK_CONGESTION_OBSERVER_H_ + +#include "modules/congestion_controller/include/send_side_congestion_controller.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockCongestionObserver : public SendSideCongestionController::Observer { + public: + MOCK_METHOD4(OnNetworkChanged, + void(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt_ms, + int64_t probing_interval_ms)); +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_CONGESTION_CONTROLLER_INCLUDE_MOCK_MOCK_CONGESTION_OBSERVER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_send_side_congestion_controller.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_send_side_congestion_controller.h new file mode 100644 index 0000000000..6ec7bb1b6e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/mock/mock_send_side_congestion_controller.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_INCLUDE_MOCK_MOCK_SEND_SIDE_CONGESTION_CONTROLLER_H_ +#define MODULES_CONGESTION_CONTROLLER_INCLUDE_MOCK_MOCK_SEND_SIDE_CONGESTION_CONTROLLER_H_ + +#include "modules/congestion_controller/include/send_side_congestion_controller.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockSendSideCongestionController : public SendSideCongestionController { + public: + MockSendSideCongestionController(const Clock* clock, + RtcEventLog* event_log, + PacedSender* paced_sender) + : SendSideCongestionController(clock, + nullptr /* observer */, + event_log, + paced_sender) {} + + MOCK_METHOD3(SetBweBitrates, + void(int min_bitrate_bps, + int start_bitrate_bps, + int max_bitrate_bps)); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_INCLUDE_MOCK_MOCK_SEND_SIDE_CONGESTION_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/include/receive_side_congestion_controller.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/receive_side_congestion_controller.h new file mode 100644 index 0000000000..a4b593ae88 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/receive_side_congestion_controller.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_INCLUDE_RECEIVE_SIDE_CONGESTION_CONTROLLER_H_ +#define MODULES_CONGESTION_CONTROLLER_INCLUDE_RECEIVE_SIDE_CONGESTION_CONTROLLER_H_ + +#include +#include + +#include "modules/remote_bitrate_estimator/remote_estimator_proxy.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { +class RemoteBitrateEstimator; +class RemoteBitrateObserver; + +// This class represents the congestion control state for receive +// streams. For send side bandwidth estimation, this is simply +// relaying for each received RTP packet back to the sender. While for +// receive side bandwidth estimation, we do the estimation locally and +// send our results back to the sender. +class ReceiveSideCongestionController : public CallStatsObserver, + public Module { + public: + ReceiveSideCongestionController( + const Clock* clock, + PacketRouter* packet_router); + + virtual ~ReceiveSideCongestionController() {} + + virtual void OnReceivedPacket(int64_t arrival_time_ms, + size_t payload_size, + const RTPHeader& header); + + // TODO(nisse): Delete these methods, design a more specific interface. + virtual RemoteBitrateEstimator* GetRemoteBitrateEstimator(bool send_side_bwe); + virtual const RemoteBitrateEstimator* GetRemoteBitrateEstimator( + bool send_side_bwe) const; + + // Implements CallStatsObserver. + void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; + + // This is send bitrate, used to control the rate of feedback messages. + void OnBitrateChanged(int bitrate_bps); + + // Implements Module. + int64_t TimeUntilNextProcess() override; + void Process() override; + + private: + class WrappingBitrateEstimator : public RemoteBitrateEstimator { + public: + WrappingBitrateEstimator(RemoteBitrateObserver* observer, + const Clock* clock); + + virtual ~WrappingBitrateEstimator() {} + + void IncomingPacket(int64_t arrival_time_ms, + size_t payload_size, + const RTPHeader& header) override; + + void Process() override; + + int64_t TimeUntilNextProcess() override; + + void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; + + void RemoveStream(unsigned int ssrc) override; + + bool LatestEstimate(std::vector* ssrcs, + unsigned int* bitrate_bps) const override; + + void SetMinBitrate(int min_bitrate_bps) override; + + private: + void PickEstimatorFromHeader(const RTPHeader& header) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + void PickEstimator() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); + RemoteBitrateObserver* observer_; + const Clock* const clock_; + rtc::CriticalSection crit_sect_; + std::unique_ptr rbe_; + bool using_absolute_send_time_; + uint32_t packets_since_absolute_send_time_; + int min_bitrate_bps_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WrappingBitrateEstimator); + }; + + WrappingBitrateEstimator remote_bitrate_estimator_; + RemoteEstimatorProxy remote_estimator_proxy_; +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_INCLUDE_RECEIVE_SIDE_CONGESTION_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/include/send_side_congestion_controller.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/send_side_congestion_controller.h new file mode 100644 index 0000000000..b33fad0d30 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/include/send_side_congestion_controller.h @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_INCLUDE_SEND_SIDE_CONGESTION_CONTROLLER_H_ +#define MODULES_CONGESTION_CONTROLLER_INCLUDE_SEND_SIDE_CONGESTION_CONTROLLER_H_ + +#include +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/congestion_controller/delay_based_bwe.h" +#include "modules/congestion_controller/transport_feedback_adapter.h" +#include "modules/include/module.h" +#include "modules/include/module_common_types.h" +#include "modules/pacing/paced_sender.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/networkroute.h" +#include "rtc_base/race_checker.h" + +namespace rtc { +struct SentPacket; +} + +namespace webrtc { + +class BitrateController; +class Clock; +class AcknowledgedBitrateEstimator; +class ProbeController; +class RateLimiter; +class RtcEventLog; + +class SendSideCongestionController : public CallStatsObserver, + public Module, + public TransportFeedbackObserver { + public: + // Observer class for bitrate changes announced due to change in bandwidth + // estimate or due to that the send pacer is full. Fraction loss and rtt is + // also part of this callback to allow the observer to optimize its settings + // for different types of network environments. The bitrate does not include + // packet headers and is measured in bits per second. + class Observer { + public: + virtual void OnNetworkChanged(uint32_t bitrate_bps, + uint8_t fraction_loss, // 0 - 255. + int64_t rtt_ms, + int64_t probing_interval_ms) = 0; + + protected: + virtual ~Observer() {} + }; + SendSideCongestionController(const Clock* clock, + Observer* observer, + RtcEventLog* event_log, + PacedSender* pacer); + ~SendSideCongestionController() override; + + void RegisterPacketFeedbackObserver(PacketFeedbackObserver* observer); + void DeRegisterPacketFeedbackObserver(PacketFeedbackObserver* observer); + + // Currently, there can be at most one observer. + // TODO(nisse): The RegisterNetworkObserver method is needed because we first + // construct this object (as part of RtpTransportControllerSend), then pass a + // reference to Call, which then registers itself as the observer. We should + // try to break this circular chain of references, and make the observer a + // construction time constant. + void RegisterNetworkObserver(Observer* observer); + void DeRegisterNetworkObserver(Observer* observer); + + virtual void SetBweBitrates(int min_bitrate_bps, + int start_bitrate_bps, + int max_bitrate_bps); + // Resets the BWE state. Note the first argument is the bitrate_bps. + virtual void OnNetworkRouteChanged(const rtc::NetworkRoute& network_route, + int bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps); + virtual void SignalNetworkState(NetworkState state); + virtual void SetTransportOverhead(size_t transport_overhead_bytes_per_packet); + + // Deprecated: Use GetBandwidthObserver instead. + RTC_DEPRECATED virtual BitrateController* GetBitrateController() const; + + virtual RtcpBandwidthObserver* GetBandwidthObserver() const; + + virtual bool AvailableBandwidth(uint32_t* bandwidth) const; + virtual int64_t GetPacerQueuingDelayMs() const; + virtual int64_t GetFirstPacketTimeMs() const; + + virtual TransportFeedbackObserver* GetTransportFeedbackObserver(); + + RateLimiter* GetRetransmissionRateLimiter(); + void EnablePeriodicAlrProbing(bool enable); + + virtual void OnSentPacket(const rtc::SentPacket& sent_packet); + + // Implements CallStatsObserver. + void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; + + // Implements Module. + int64_t TimeUntilNextProcess() override; + void Process() override; + + // Implements TransportFeedbackObserver. + void AddPacket(uint32_t ssrc, + uint16_t sequence_number, + size_t length, + const PacedPacketInfo& pacing_info) override; + void OnTransportFeedback(const rtcp::TransportFeedback& feedback) override; + std::vector GetTransportFeedbackVector() const override; + + private: + void MaybeTriggerOnNetworkChanged(); + + bool IsSendQueueFull() const; + bool IsNetworkDown() const; + bool HasNetworkParametersToReportChanged(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt); + void LimitOutstandingBytes(size_t num_outstanding_bytes); + const Clock* const clock_; + rtc::CriticalSection observer_lock_; + Observer* observer_ RTC_GUARDED_BY(observer_lock_); + RtcEventLog* const event_log_; + PacedSender* const pacer_; + const std::unique_ptr bitrate_controller_; + std::unique_ptr acknowledged_bitrate_estimator_; + const std::unique_ptr probe_controller_; + const std::unique_ptr retransmission_rate_limiter_; + TransportFeedbackAdapter transport_feedback_adapter_; + rtc::CriticalSection network_state_lock_; + uint32_t last_reported_bitrate_bps_ RTC_GUARDED_BY(network_state_lock_); + uint8_t last_reported_fraction_loss_ RTC_GUARDED_BY(network_state_lock_); + int64_t last_reported_rtt_ RTC_GUARDED_BY(network_state_lock_); + NetworkState network_state_ RTC_GUARDED_BY(network_state_lock_); + bool pause_pacer_ RTC_GUARDED_BY(network_state_lock_); + // Duplicate the pacer paused state to avoid grabbing a lock when + // pausing the pacer. This can be removed when we move this class + // over to the task queue. + bool pacer_paused_; + rtc::CriticalSection bwe_lock_; + int min_bitrate_bps_ RTC_GUARDED_BY(bwe_lock_); + std::unique_ptr delay_based_bwe_ RTC_GUARDED_BY(bwe_lock_); + bool in_cwnd_experiment_; + int64_t accepted_queue_ms_; + bool was_in_alr_; + + rtc::RaceChecker worker_race_; + + bool pacer_pushback_experiment_ = false; + float encoding_rate_ = 1.0; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(SendSideCongestionController); +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_INCLUDE_SEND_SIDE_CONGESTION_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.cc new file mode 100644 index 0000000000..ea76239577 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/median_slope_estimator.h" + +#include +#include + +#include "modules/remote_bitrate_estimator/include/bwe_defines.h" +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +constexpr unsigned int kDeltaCounterMax = 1000; + +MedianSlopeEstimator::MedianSlopeEstimator(size_t window_size, + double threshold_gain) + : window_size_(window_size), + threshold_gain_(threshold_gain), + num_of_deltas_(0), + accumulated_delay_(0), + delay_hist_(), + median_filter_(0.5), + trendline_(0) {} + +MedianSlopeEstimator::~MedianSlopeEstimator() {} + +MedianSlopeEstimator::DelayInfo::DelayInfo(int64_t time, + double delay, + size_t slope_count) + : time(time), delay(delay) { + slopes.reserve(slope_count); +} + +MedianSlopeEstimator::DelayInfo::~DelayInfo() = default; + +void MedianSlopeEstimator::Update(double recv_delta_ms, + double send_delta_ms, + int64_t arrival_time_ms) { + const double delta_ms = recv_delta_ms - send_delta_ms; + ++num_of_deltas_; + if (num_of_deltas_ > kDeltaCounterMax) + num_of_deltas_ = kDeltaCounterMax; + + accumulated_delay_ += delta_ms; + BWE_TEST_LOGGING_PLOT(1, "accumulated_delay_ms", arrival_time_ms, + accumulated_delay_); + + // If the window is full, remove the |window_size_| - 1 slopes that belong to + // the oldest point. + if (delay_hist_.size() == window_size_) { + for (double slope : delay_hist_.front().slopes) { + const bool success = median_filter_.Erase(slope); + RTC_CHECK(success); + } + delay_hist_.pop_front(); + } + // Add |window_size_| - 1 new slopes. + for (auto& old_delay : delay_hist_) { + if (arrival_time_ms - old_delay.time != 0) { + // The C99 standard explicitly states that casts and assignments must + // perform the associated conversions. This means that |slope| will be + // a 64-bit double even if the division is computed using, e.g., 80-bit + // extended precision. I believe this also holds in C++ even though the + // C++11 standard isn't as explicit. Furthermore, there are good reasons + // to believe that compilers couldn't perform optimizations that break + // this assumption even if they wanted to. + double slope = (accumulated_delay_ - old_delay.delay) / + static_cast(arrival_time_ms - old_delay.time); + median_filter_.Insert(slope); + // We want to avoid issues with different rounding mode / precision + // which we might get if we recomputed the slope when we remove it. + old_delay.slopes.push_back(slope); + } + } + delay_hist_.emplace_back(arrival_time_ms, accumulated_delay_, + window_size_ - 1); + // Recompute the median slope. + if (delay_hist_.size() == window_size_) + trendline_ = median_filter_.GetPercentileValue(); + + BWE_TEST_LOGGING_PLOT(1, "trendline_slope", arrival_time_ms, trendline_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.h new file mode 100644 index 0000000000..f837b24a21 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_CONGESTION_CONTROLLER_MEDIAN_SLOPE_ESTIMATOR_H_ +#define MODULES_CONGESTION_CONTROLLER_MEDIAN_SLOPE_ESTIMATOR_H_ + +#include +#include + +#include +#include + +#include "rtc_base/constructormagic.h" +#include "rtc_base/numerics/percentile_filter.h" + +namespace webrtc { + +class MedianSlopeEstimator { + public: + // |window_size| is the number of points required to compute a trend line. + // |threshold_gain| is used to scale the trendline slope for comparison to + // the old threshold. Once the old estimator has been removed (or the + // thresholds been merged into the estimators), we can just set the + // threshold instead of setting a gain. + MedianSlopeEstimator(size_t window_size, double threshold_gain); + ~MedianSlopeEstimator(); + + // Update the estimator with a new sample. The deltas should represent deltas + // between timestamp groups as defined by the InterArrival class. + void Update(double recv_delta_ms, + double send_delta_ms, + int64_t arrival_time_ms); + + // Returns the estimated trend k multiplied by some gain. + // 0 < k < 1 -> the delay increases, queues are filling up + // k == 0 -> the delay does not change + // k < 0 -> the delay decreases, queues are being emptied + double trendline_slope() const { return trendline_ * threshold_gain_; } + + // Returns the number of deltas which the current estimator state is based on. + unsigned int num_of_deltas() const { return num_of_deltas_; } + + private: + struct DelayInfo { + DelayInfo(int64_t time, double delay, size_t slope_count); + ~DelayInfo(); + int64_t time; + double delay; + std::vector slopes; + }; + // Parameters. + const size_t window_size_; + const double threshold_gain_; + // Used by the existing threshold. + unsigned int num_of_deltas_; + // Theil-Sen robust line fitting + double accumulated_delay_; + std::deque delay_hist_; + PercentileFilter median_filter_; + double trendline_; + + RTC_DISALLOW_COPY_AND_ASSIGN(MedianSlopeEstimator); +}; +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_MEDIAN_SLOPE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator_unittest.cc new file mode 100644 index 0000000000..a5b90d8c35 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/median_slope_estimator_unittest.cc @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/median_slope_estimator.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr size_t kWindowSize = 20; +constexpr double kGain = 1; +constexpr int64_t kAvgTimeBetweenPackets = 10; +constexpr size_t kPacketCount = 2 * kWindowSize + 1; + +void TestEstimator(double slope, double jitter_stddev, double tolerance) { + MedianSlopeEstimator estimator(kWindowSize, kGain); + Random random(0x1234567); + int64_t send_times[kPacketCount]; + int64_t recv_times[kPacketCount]; + int64_t send_start_time = random.Rand(1000000); + int64_t recv_start_time = random.Rand(1000000); + for (size_t i = 0; i < kPacketCount; ++i) { + send_times[i] = send_start_time + i * kAvgTimeBetweenPackets; + double latency = i * kAvgTimeBetweenPackets / (1 - slope); + double jitter = random.Gaussian(0, jitter_stddev); + recv_times[i] = recv_start_time + latency + jitter; + } + for (size_t i = 1; i < kPacketCount; ++i) { + double recv_delta = recv_times[i] - recv_times[i - 1]; + double send_delta = send_times[i] - send_times[i - 1]; + estimator.Update(recv_delta, send_delta, recv_times[i]); + if (i < kWindowSize) + EXPECT_NEAR(estimator.trendline_slope(), 0, 0.001); + else + EXPECT_NEAR(estimator.trendline_slope(), slope, tolerance); + } +} +} // namespace + +TEST(MedianSlopeEstimator, PerfectLineSlopeOneHalf) { + TestEstimator(0.5, 0, 0.001); +} + +TEST(MedianSlopeEstimator, PerfectLineSlopeMinusOne) { + TestEstimator(-1, 0, 0.001); +} + +TEST(MedianSlopeEstimator, PerfectLineSlopeZero) { + TestEstimator(0, 0, 0.001); +} + +TEST(MedianSlopeEstimator, JitteryLineSlopeOneHalf) { + TestEstimator(0.5, kAvgTimeBetweenPackets / 3.0, 0.01); +} + +TEST(MedianSlopeEstimator, JitteryLineSlopeMinusOne) { + TestEstimator(-1, kAvgTimeBetweenPackets / 3.0, 0.05); +} + +TEST(MedianSlopeEstimator, JitteryLineSlopeZero) { + TestEstimator(0, kAvgTimeBetweenPackets / 3.0, 0.02); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.cc new file mode 100644 index 0000000000..1a438829b1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.cc @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/probe_bitrate_estimator.h" + +#include + +#include "logging/rtc_event_log/events/rtc_event_probe_result_failure.h" +#include "logging/rtc_event_log/events/rtc_event_probe_result_success.h" +#include "logging/rtc_event_log/rtc_event_log.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/ptr_util.h" + +namespace { +// The minumum number of probes we need to receive feedback about in percent +// in order to have a valid estimate. +constexpr int kMinReceivedProbesPercent = 80; + +// The minumum number of bytes we need to receive feedback about in percent +// in order to have a valid estimate. +constexpr int kMinReceivedBytesPercent = 80; + +// The maximum |receive rate| / |send rate| ratio for a valid estimate. +constexpr float kMaxValidRatio = 2.0f; + +// The minimum |receive rate| / |send rate| ratio assuming that the link is +// not saturated, i.e. we assume that we will receive at least +// kMinRatioForUnsaturatedLink * |send rate| if |send rate| is less than the +// link capacity. +constexpr float kMinRatioForUnsaturatedLink = 0.9f; + +// The target utilization of the link. If we know true link capacity +// we'd like to send at 95% of that rate. +constexpr float kTargetUtilizationFraction = 0.95f; + +// The maximum time period over which the cluster history is retained. +// This is also the maximum time period beyond which a probing burst is not +// expected to last. +constexpr int kMaxClusterHistoryMs = 1000; + +// The maximum time interval between first and the last probe on a cluster +// on the sender side as well as the receive side. +constexpr int kMaxProbeIntervalMs = 1000; +} // namespace + +namespace webrtc { + +ProbeBitrateEstimator::ProbeBitrateEstimator(RtcEventLog* event_log) + : event_log_(event_log) {} + +ProbeBitrateEstimator::~ProbeBitrateEstimator() = default; + +int ProbeBitrateEstimator::HandleProbeAndEstimateBitrate( + const PacketFeedback& packet_feedback) { + int cluster_id = packet_feedback.pacing_info.probe_cluster_id; + RTC_DCHECK_NE(cluster_id, PacedPacketInfo::kNotAProbe); + + EraseOldClusters(packet_feedback.arrival_time_ms - kMaxClusterHistoryMs); + + int payload_size_bits = packet_feedback.payload_size * 8; + AggregatedCluster* cluster = &clusters_[cluster_id]; + + if (packet_feedback.send_time_ms < cluster->first_send_ms) { + cluster->first_send_ms = packet_feedback.send_time_ms; + } + if (packet_feedback.send_time_ms > cluster->last_send_ms) { + cluster->last_send_ms = packet_feedback.send_time_ms; + cluster->size_last_send = payload_size_bits; + } + if (packet_feedback.arrival_time_ms < cluster->first_receive_ms) { + cluster->first_receive_ms = packet_feedback.arrival_time_ms; + cluster->size_first_receive = payload_size_bits; + } + if (packet_feedback.arrival_time_ms > cluster->last_receive_ms) { + cluster->last_receive_ms = packet_feedback.arrival_time_ms; + } + cluster->size_total += payload_size_bits; + cluster->num_probes += 1; + + RTC_DCHECK_GT(packet_feedback.pacing_info.probe_cluster_min_probes, 0); + RTC_DCHECK_GT(packet_feedback.pacing_info.probe_cluster_min_bytes, 0); + + int min_probes = packet_feedback.pacing_info.probe_cluster_min_probes * + kMinReceivedProbesPercent / 100; + int min_bytes = packet_feedback.pacing_info.probe_cluster_min_bytes * + kMinReceivedBytesPercent / 100; + if (cluster->num_probes < min_probes || cluster->size_total < min_bytes * 8) + return -1; + + float send_interval_ms = cluster->last_send_ms - cluster->first_send_ms; + float receive_interval_ms = + cluster->last_receive_ms - cluster->first_receive_ms; + + if (send_interval_ms <= 0 || send_interval_ms > kMaxProbeIntervalMs || + receive_interval_ms <= 0 || receive_interval_ms > kMaxProbeIntervalMs) { + RTC_LOG(LS_INFO) << "Probing unsuccessful, invalid send/receive interval" + << " [cluster id: " << cluster_id + << "] [send interval: " << send_interval_ms << " ms]" + << " [receive interval: " << receive_interval_ms << " ms]"; + if (event_log_) { + event_log_->Log(rtc::MakeUnique( + cluster_id, ProbeFailureReason::kInvalidSendReceiveInterval)); + } + return -1; + } + // Since the |send_interval_ms| does not include the time it takes to actually + // send the last packet the size of the last sent packet should not be + // included when calculating the send bitrate. + RTC_DCHECK_GT(cluster->size_total, cluster->size_last_send); + float send_size = cluster->size_total - cluster->size_last_send; + float send_bps = send_size / send_interval_ms * 1000; + + // Since the |receive_interval_ms| does not include the time it takes to + // actually receive the first packet the size of the first received packet + // should not be included when calculating the receive bitrate. + RTC_DCHECK_GT(cluster->size_total, cluster->size_first_receive); + float receive_size = cluster->size_total - cluster->size_first_receive; + float receive_bps = receive_size / receive_interval_ms * 1000; + + float ratio = receive_bps / send_bps; + if (ratio > kMaxValidRatio) { + RTC_LOG(LS_INFO) << "Probing unsuccessful, receive/send ratio too high" + << " [cluster id: " << cluster_id + << "] [send: " << send_size << " bytes / " + << send_interval_ms << " ms = " << send_bps / 1000 + << " kb/s]" + << " [receive: " << receive_size << " bytes / " + << receive_interval_ms << " ms = " << receive_bps / 1000 + << " kb/s]" + << " [ratio: " << receive_bps / 1000 << " / " + << send_bps / 1000 << " = " << ratio + << " > kMaxValidRatio (" << kMaxValidRatio << ")]"; + if (event_log_) { + event_log_->Log(rtc::MakeUnique( + cluster_id, ProbeFailureReason::kInvalidSendReceiveRatio)); + } + return -1; + } + RTC_LOG(LS_INFO) << "Probing successful" + << " [cluster id: " << cluster_id << "] [send: " << send_size + << " bytes / " << send_interval_ms + << " ms = " << send_bps / 1000 << " kb/s]" + << " [receive: " << receive_size << " bytes / " + << receive_interval_ms << " ms = " << receive_bps / 1000 + << " kb/s]"; + + float res = std::min(send_bps, receive_bps); + // If we're receiving at significantly lower bitrate than we were sending at, + // it suggests that we've found the true capacity of the link. In this case, + // set the target bitrate slightly lower to not immediately overuse. + if (receive_bps < kMinRatioForUnsaturatedLink * send_bps) { + RTC_DCHECK_GT(send_bps, receive_bps); + res = kTargetUtilizationFraction * receive_bps; + } + if (event_log_) { + event_log_->Log( + rtc::MakeUnique(cluster_id, res)); + } + estimated_bitrate_bps_ = res; + return *estimated_bitrate_bps_; +} + +rtc::Optional +ProbeBitrateEstimator::FetchAndResetLastEstimatedBitrateBps() { + rtc::Optional estimated_bitrate_bps = estimated_bitrate_bps_; + estimated_bitrate_bps_.reset(); + return estimated_bitrate_bps; +} + +void ProbeBitrateEstimator::EraseOldClusters(int64_t timestamp_ms) { + for (auto it = clusters_.begin(); it != clusters_.end();) { + if (it->second.last_receive_ms < timestamp_ms) { + it = clusters_.erase(it); + } else { + ++it; + } + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.h new file mode 100644 index 0000000000..bb8a7ed534 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_PROBE_BITRATE_ESTIMATOR_H_ +#define MODULES_CONGESTION_CONTROLLER_PROBE_BITRATE_ESTIMATOR_H_ + +#include +#include + +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { +class RtcEventLog; + +class ProbeBitrateEstimator { + public: + explicit ProbeBitrateEstimator(RtcEventLog* event_log); + ~ProbeBitrateEstimator(); + + // Should be called for every probe packet we receive feedback about. + // Returns the estimated bitrate if the probe completes a valid cluster. + int HandleProbeAndEstimateBitrate(const PacketFeedback& packet_feedback); + + rtc::Optional FetchAndResetLastEstimatedBitrateBps(); + + private: + struct AggregatedCluster { + int num_probes = 0; + int64_t first_send_ms = std::numeric_limits::max(); + int64_t last_send_ms = 0; + int64_t first_receive_ms = std::numeric_limits::max(); + int64_t last_receive_ms = 0; + int size_last_send = 0; + int size_first_receive = 0; + int size_total = 0; + }; + + // Erases old cluster data that was seen before |timestamp_ms|. + void EraseOldClusters(int64_t timestamp_ms); + + std::map clusters_; + RtcEventLog* const event_log_; + rtc::Optional estimated_bitrate_bps_; +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_PROBE_BITRATE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator_unittest.cc new file mode 100644 index 0000000000..6d21de449d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_bitrate_estimator_unittest.cc @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/probe_bitrate_estimator.h" + +#include +#include + +#include "modules/remote_bitrate_estimator/aimd_rate_control.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int kInvalidBitrate = -1; +constexpr int kDefaultMinProbes = 5; +constexpr int kDefaultMinBytes = 5000; +constexpr float kTargetUtilizationFraction = 0.95f; +} // anonymous namespace + +class TestProbeBitrateEstimator : public ::testing::Test { + public: + TestProbeBitrateEstimator() : probe_bitrate_estimator_(nullptr) {} + + // TODO(philipel): Use PacedPacketInfo when ProbeBitrateEstimator is rewritten + // to use that information. + void AddPacketFeedback(int probe_cluster_id, + size_t size_bytes, + int64_t send_time_ms, + int64_t arrival_time_ms, + int min_probes = kDefaultMinProbes, + int min_bytes = kDefaultMinBytes) { + PacedPacketInfo pacing_info(probe_cluster_id, min_probes, min_bytes); + PacketFeedback packet_feedback(arrival_time_ms, send_time_ms, 0, size_bytes, + pacing_info); + measured_bps_ = + probe_bitrate_estimator_.HandleProbeAndEstimateBitrate(packet_feedback); + } + + protected: + int measured_bps_ = kInvalidBitrate; + ProbeBitrateEstimator probe_bitrate_estimator_; +}; + +TEST_F(TestProbeBitrateEstimator, OneCluster) { + AddPacketFeedback(0, 1000, 0, 10); + AddPacketFeedback(0, 1000, 10, 20); + AddPacketFeedback(0, 1000, 20, 30); + AddPacketFeedback(0, 1000, 30, 40); + + EXPECT_NEAR(measured_bps_, 800000, 10); +} + +TEST_F(TestProbeBitrateEstimator, OneClusterTooFewProbes) { + AddPacketFeedback(0, 2000, 0, 10); + AddPacketFeedback(0, 2000, 10, 20); + AddPacketFeedback(0, 2000, 20, 30); + + EXPECT_EQ(kInvalidBitrate, measured_bps_); +} + +TEST_F(TestProbeBitrateEstimator, OneClusterTooFewBytes) { + const int kMinBytes = 6000; + AddPacketFeedback(0, 800, 0, 10, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 800, 10, 20, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 800, 20, 30, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 800, 30, 40, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 800, 40, 50, kDefaultMinProbes, kMinBytes); + + EXPECT_EQ(kInvalidBitrate, measured_bps_); +} + +TEST_F(TestProbeBitrateEstimator, SmallCluster) { + const int kMinBytes = 1000; + AddPacketFeedback(0, 150, 0, 10, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 150, 10, 20, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 150, 20, 30, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 150, 30, 40, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 150, 40, 50, kDefaultMinProbes, kMinBytes); + AddPacketFeedback(0, 150, 50, 60, kDefaultMinProbes, kMinBytes); + EXPECT_NEAR(measured_bps_, 120000, 10); +} + +TEST_F(TestProbeBitrateEstimator, LargeCluster) { + const int kMinProbes = 30; + const int kMinBytes = 312500; + + int64_t send_time = 0; + int64_t receive_time = 5; + for (int i = 0; i < 25; ++i) { + AddPacketFeedback(0, 12500, send_time, receive_time, kMinProbes, kMinBytes); + ++send_time; + ++receive_time; + } + EXPECT_NEAR(measured_bps_, 100000000, 10); +} + +TEST_F(TestProbeBitrateEstimator, FastReceive) { + AddPacketFeedback(0, 1000, 0, 15); + AddPacketFeedback(0, 1000, 10, 30); + AddPacketFeedback(0, 1000, 20, 35); + AddPacketFeedback(0, 1000, 30, 40); + + EXPECT_NEAR(measured_bps_, 800000, 10); +} + +TEST_F(TestProbeBitrateEstimator, TooFastReceive) { + AddPacketFeedback(0, 1000, 0, 19); + AddPacketFeedback(0, 1000, 10, 22); + AddPacketFeedback(0, 1000, 20, 25); + AddPacketFeedback(0, 1000, 40, 27); + + EXPECT_EQ(measured_bps_, kInvalidBitrate); +} + +TEST_F(TestProbeBitrateEstimator, SlowReceive) { + AddPacketFeedback(0, 1000, 0, 10); + AddPacketFeedback(0, 1000, 10, 40); + AddPacketFeedback(0, 1000, 20, 70); + AddPacketFeedback(0, 1000, 30, 85); + // Expected send rate = 800 kbps, expected receive rate = 320 kbps. + + EXPECT_NEAR(measured_bps_, kTargetUtilizationFraction * 320000, 10); +} + +TEST_F(TestProbeBitrateEstimator, BurstReceive) { + AddPacketFeedback(0, 1000, 0, 50); + AddPacketFeedback(0, 1000, 10, 50); + AddPacketFeedback(0, 1000, 20, 50); + AddPacketFeedback(0, 1000, 40, 50); + + EXPECT_EQ(measured_bps_, kInvalidBitrate); +} + +TEST_F(TestProbeBitrateEstimator, MultipleClusters) { + AddPacketFeedback(0, 1000, 0, 10); + AddPacketFeedback(0, 1000, 10, 20); + AddPacketFeedback(0, 1000, 20, 30); + AddPacketFeedback(0, 1000, 40, 60); + // Expected send rate = 600 kbps, expected receive rate = 480 kbps. + EXPECT_NEAR(measured_bps_, kTargetUtilizationFraction * 480000, 10); + + AddPacketFeedback(0, 1000, 50, 60); + // Expected send rate = 640 kbps, expected receive rate = 640 kbps. + EXPECT_NEAR(measured_bps_, 640000, 10); + + AddPacketFeedback(1, 1000, 60, 70); + AddPacketFeedback(1, 1000, 65, 77); + AddPacketFeedback(1, 1000, 70, 84); + AddPacketFeedback(1, 1000, 75, 90); + // Expected send rate = 1600 kbps, expected receive rate = 1200 kbps. + + EXPECT_NEAR(measured_bps_, kTargetUtilizationFraction * 1200000, 10); +} + +TEST_F(TestProbeBitrateEstimator, IgnoreOldClusters) { + AddPacketFeedback(0, 1000, 0, 10); + AddPacketFeedback(0, 1000, 10, 20); + AddPacketFeedback(0, 1000, 20, 30); + + AddPacketFeedback(1, 1000, 60, 70); + AddPacketFeedback(1, 1000, 65, 77); + AddPacketFeedback(1, 1000, 70, 84); + AddPacketFeedback(1, 1000, 75, 90); + // Expected send rate = 1600 kbps, expected receive rate = 1200 kbps. + + EXPECT_NEAR(measured_bps_, kTargetUtilizationFraction * 1200000, 10); + + // Coming in 6s later + AddPacketFeedback(0, 1000, 40 + 6000, 60 + 6000); + + EXPECT_EQ(measured_bps_, kInvalidBitrate); +} + +TEST_F(TestProbeBitrateEstimator, IgnoreSizeLastSendPacket) { + AddPacketFeedback(0, 1000, 0, 10); + AddPacketFeedback(0, 1000, 10, 20); + AddPacketFeedback(0, 1000, 20, 30); + AddPacketFeedback(0, 1000, 30, 40); + AddPacketFeedback(0, 1500, 40, 50); + // Expected send rate = 800 kbps, expected receive rate = 900 kbps. + + EXPECT_NEAR(measured_bps_, 800000, 10); +} + +TEST_F(TestProbeBitrateEstimator, IgnoreSizeFirstReceivePacket) { + AddPacketFeedback(0, 1500, 0, 10); + AddPacketFeedback(0, 1000, 10, 20); + AddPacketFeedback(0, 1000, 20, 30); + AddPacketFeedback(0, 1000, 30, 40); + // Expected send rate = 933 kbps, expected receive rate = 800 kbps. + + EXPECT_NEAR(measured_bps_, kTargetUtilizationFraction * 800000, 10); +} + +TEST_F(TestProbeBitrateEstimator, NoLastEstimatedBitrateBps) { + EXPECT_FALSE(probe_bitrate_estimator_.FetchAndResetLastEstimatedBitrateBps()); +} + +TEST_F(TestProbeBitrateEstimator, FetchLastEstimatedBitrateBps) { + AddPacketFeedback(0, 1000, 0, 10); + AddPacketFeedback(0, 1000, 10, 20); + AddPacketFeedback(0, 1000, 20, 30); + AddPacketFeedback(0, 1000, 30, 40); + + auto estimated_bitrate_bps = + probe_bitrate_estimator_.FetchAndResetLastEstimatedBitrateBps(); + EXPECT_TRUE(estimated_bitrate_bps); + EXPECT_NEAR(*estimated_bitrate_bps, 800000, 10); + EXPECT_FALSE(probe_bitrate_estimator_.FetchAndResetLastEstimatedBitrateBps()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.cc new file mode 100644 index 0000000000..8e6ae7cca1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.cc @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/probe_controller.h" + +#include +#include + +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { +// Maximum waiting time from the time of initiating probing to getting +// the measured results back. +constexpr int64_t kMaxWaitingTimeForProbingResultMs = 1000; + +// Value of |min_bitrate_to_probe_further_bps_| that indicates +// further probing is disabled. +constexpr int kExponentialProbingDisabled = 0; + +// Default probing bitrate limit. Applied only when the application didn't +// specify max bitrate. +constexpr int64_t kDefaultMaxProbingBitrateBps = 5000000; + +// Interval between probes when ALR periodic probing is enabled. +constexpr int64_t kAlrPeriodicProbingIntervalMs = 5000; + +// Minimum probe bitrate percentage to probe further for repeated probes, +// relative to the previous probe. For example, if 1Mbps probe results in +// 80kbps, then we'll probe again at 1.6Mbps. In that case second probe won't be +// sent if we get 600kbps from the first one. +constexpr int kRepeatedProbeMinPercentage = 70; + +// If the bitrate drops to a factor |kBitrateDropThreshold| or lower +// and we recover within |kBitrateDropTimeoutMs|, then we'll send +// a probe at a fraction |kProbeFractionAfterDrop| of the original bitrate. +constexpr double kBitrateDropThreshold = 0.66; +constexpr int kBitrateDropTimeoutMs = 5000; +constexpr double kProbeFractionAfterDrop = 0.85; + +// Timeout for probing after leaving ALR. If the bitrate drops significantly, +// (as determined by the delay based estimator) and we leave ALR, then we will +// send a probe if we recover within |kLeftAlrTimeoutMs| ms. +constexpr int kAlrEndedTimeoutMs = 3000; + +// The expected uncertainty of probe result (as a fraction of the target probe +// This is a limit on how often probing can be done when there is a BW +// drop detected in ALR. +constexpr int64_t kMinTimeBetweenAlrProbesMs = 5000; + +// bitrate). Used to avoid probing if the probe bitrate is close to our current +// estimate. +constexpr double kProbeUncertainty = 0.05; + +// Use probing to recover faster after large bitrate estimate drops. +constexpr char kBweRapidRecoveryExperiment[] = + "WebRTC-BweRapidRecoveryExperiment"; + +} // namespace + +ProbeController::ProbeController(PacedSender* pacer, const Clock* clock) + : pacer_(pacer), clock_(clock), enable_periodic_alr_probing_(false) { + Reset(); + in_rapid_recovery_experiment_ = webrtc::field_trial::FindFullName( + kBweRapidRecoveryExperiment) == "Enabled"; +} + +void ProbeController::SetBitrates(int64_t min_bitrate_bps, + int64_t start_bitrate_bps, + int64_t max_bitrate_bps) { + rtc::CritScope cs(&critsect_); + + if (start_bitrate_bps > 0) { + start_bitrate_bps_ = start_bitrate_bps; + estimated_bitrate_bps_ = start_bitrate_bps; + } else if (start_bitrate_bps_ == 0) { + start_bitrate_bps_ = min_bitrate_bps; + } + + // The reason we use the variable |old_max_bitrate_pbs| is because we + // need to set |max_bitrate_bps_| before we call InitiateProbing. + int64_t old_max_bitrate_bps = max_bitrate_bps_; + max_bitrate_bps_ = max_bitrate_bps; + + switch (state_) { + case State::kInit: + if (network_state_ == kNetworkUp) + InitiateExponentialProbing(); + break; + + case State::kWaitingForProbingResult: + break; + + case State::kProbingComplete: + // If the new max bitrate is higher than the old max bitrate and the + // estimate is lower than the new max bitrate then initiate probing. + if (estimated_bitrate_bps_ != 0 && + old_max_bitrate_bps < max_bitrate_bps_ && + estimated_bitrate_bps_ < max_bitrate_bps_) { + // The assumption is that if we jump more than 20% in the bandwidth + // estimate or if the bandwidth estimate is within 90% of the new + // max bitrate then the probing attempt was successful. + mid_call_probing_succcess_threshold_ = + std::min(estimated_bitrate_bps_ * 1.2, max_bitrate_bps_ * 0.9); + mid_call_probing_waiting_for_result_ = true; + mid_call_probing_bitrate_bps_ = max_bitrate_bps_; + + RTC_HISTOGRAM_COUNTS_10000("WebRTC.BWE.MidCallProbing.Initiated", + max_bitrate_bps_ / 1000); + + InitiateProbing(clock_->TimeInMilliseconds(), {max_bitrate_bps}, false); + } + break; + } +} + +void ProbeController::OnNetworkStateChanged(NetworkState network_state) { + rtc::CritScope cs(&critsect_); + network_state_ = network_state; + if (network_state_ == kNetworkUp && state_ == State::kInit) + InitiateExponentialProbing(); +} + +void ProbeController::InitiateExponentialProbing() { + RTC_DCHECK(network_state_ == kNetworkUp); + RTC_DCHECK(state_ == State::kInit); + RTC_DCHECK_GT(start_bitrate_bps_, 0); + + // When probing at 1.8 Mbps ( 6x 300), this represents a threshold of + // 1.2 Mbps to continue probing. + InitiateProbing(clock_->TimeInMilliseconds(), + {3 * start_bitrate_bps_, 6 * start_bitrate_bps_}, true); +} + +void ProbeController::SetEstimatedBitrate(int64_t bitrate_bps) { + rtc::CritScope cs(&critsect_); + int64_t now_ms = clock_->TimeInMilliseconds(); + + if (mid_call_probing_waiting_for_result_ && + bitrate_bps >= mid_call_probing_succcess_threshold_) { + RTC_HISTOGRAM_COUNTS_10000("WebRTC.BWE.MidCallProbing.Success", + mid_call_probing_bitrate_bps_ / 1000); + RTC_HISTOGRAM_COUNTS_10000("WebRTC.BWE.MidCallProbing.ProbedKbps", + bitrate_bps / 1000); + mid_call_probing_waiting_for_result_ = false; + } + + if (state_ == State::kWaitingForProbingResult) { + // Continue probing if probing results indicate channel has greater + // capacity. + RTC_LOG(LS_INFO) << "Measured bitrate: " << bitrate_bps + << " Minimum to probe further: " + << min_bitrate_to_probe_further_bps_; + + if (min_bitrate_to_probe_further_bps_ != kExponentialProbingDisabled && + bitrate_bps > min_bitrate_to_probe_further_bps_) { + // Double the probing bitrate. + InitiateProbing(now_ms, {2 * bitrate_bps}, true); + } + } + + if (bitrate_bps < kBitrateDropThreshold * estimated_bitrate_bps_) { + time_of_last_large_drop_ms_ = now_ms; + bitrate_before_last_large_drop_bps_ = estimated_bitrate_bps_; + } + + estimated_bitrate_bps_ = bitrate_bps; +} + +void ProbeController::EnablePeriodicAlrProbing(bool enable) { + rtc::CritScope cs(&critsect_); + enable_periodic_alr_probing_ = enable; +} + +void ProbeController::SetAlrEndedTimeMs(int64_t alr_end_time_ms) { + rtc::CritScope cs(&critsect_); + alr_end_time_ms_.emplace(alr_end_time_ms); +} + +void ProbeController::RequestProbe() { + int64_t now_ms = clock_->TimeInMilliseconds(); + rtc::CritScope cs(&critsect_); + // Called once we have returned to normal state after a large drop in + // estimated bandwidth. The current response is to initiate a single probe + // session (if not already probing) at the previous bitrate. + // + // If the probe session fails, the assumption is that this drop was a + // real one from a competing flow or a network change. + bool in_alr = pacer_->GetApplicationLimitedRegionStartTime().has_value(); + bool alr_ended_recently = + (alr_end_time_ms_.has_value() && + now_ms - alr_end_time_ms_.value() < kAlrEndedTimeoutMs); + if (in_alr || alr_ended_recently || in_rapid_recovery_experiment_) { + if (state_ == State::kProbingComplete) { + uint32_t suggested_probe_bps = + kProbeFractionAfterDrop * bitrate_before_last_large_drop_bps_; + uint32_t min_expected_probe_result_bps = + (1 - kProbeUncertainty) * suggested_probe_bps; + int64_t time_since_drop_ms = now_ms - time_of_last_large_drop_ms_; + int64_t time_since_probe_ms = now_ms - last_bwe_drop_probing_time_ms_; + if (min_expected_probe_result_bps > estimated_bitrate_bps_ && + time_since_drop_ms < kBitrateDropTimeoutMs && + time_since_probe_ms > kMinTimeBetweenAlrProbesMs) { + RTC_LOG(LS_INFO) << "Detected big bandwidth drop, start probing."; + // Track how often we probe in response to bandwidth drop in ALR. + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.BWE.BweDropProbingIntervalInS", + (now_ms - last_bwe_drop_probing_time_ms_) / 1000); + InitiateProbing(now_ms, {suggested_probe_bps}, false); + last_bwe_drop_probing_time_ms_ = now_ms; + } + } + } +} + +void ProbeController::Reset() { + rtc::CritScope cs(&critsect_); + network_state_ = kNetworkUp; + state_ = State::kInit; + min_bitrate_to_probe_further_bps_ = kExponentialProbingDisabled; + time_last_probing_initiated_ms_ = 0; + estimated_bitrate_bps_ = 0; + start_bitrate_bps_ = 0; + max_bitrate_bps_ = 0; + int64_t now_ms = clock_->TimeInMilliseconds(); + last_bwe_drop_probing_time_ms_ = now_ms; + alr_end_time_ms_.reset(); + mid_call_probing_waiting_for_result_ = false; + time_of_last_large_drop_ms_ = now_ms; + bitrate_before_last_large_drop_bps_ = 0; +} + +void ProbeController::Process() { + rtc::CritScope cs(&critsect_); + + int64_t now_ms = clock_->TimeInMilliseconds(); + + if (now_ms - time_last_probing_initiated_ms_ > + kMaxWaitingTimeForProbingResultMs) { + mid_call_probing_waiting_for_result_ = false; + + if (state_ == State::kWaitingForProbingResult) { + RTC_LOG(LS_INFO) << "kWaitingForProbingResult: timeout"; + state_ = State::kProbingComplete; + min_bitrate_to_probe_further_bps_ = kExponentialProbingDisabled; + } + } + + if (state_ != State::kProbingComplete || !enable_periodic_alr_probing_) + return; + + // Probe bandwidth periodically when in ALR state. + rtc::Optional alr_start_time = + pacer_->GetApplicationLimitedRegionStartTime(); + if (alr_start_time && estimated_bitrate_bps_ > 0) { + int64_t next_probe_time_ms = + std::max(*alr_start_time, time_last_probing_initiated_ms_) + + kAlrPeriodicProbingIntervalMs; + if (now_ms >= next_probe_time_ms) { + InitiateProbing(now_ms, {estimated_bitrate_bps_ * 2}, true); + } + } +} + +void ProbeController::InitiateProbing( + int64_t now_ms, + std::initializer_list bitrates_to_probe, + bool probe_further) { + for (int64_t bitrate : bitrates_to_probe) { + RTC_DCHECK_GT(bitrate, 0); + int64_t max_probe_bitrate_bps = + max_bitrate_bps_ > 0 ? max_bitrate_bps_ : kDefaultMaxProbingBitrateBps; + if (bitrate > max_probe_bitrate_bps) { + bitrate = max_probe_bitrate_bps; + probe_further = false; + } + pacer_->CreateProbeCluster(rtc::dchecked_cast(bitrate)); + } + time_last_probing_initiated_ms_ = now_ms; + if (probe_further) { + state_ = State::kWaitingForProbingResult; + min_bitrate_to_probe_further_bps_ = + (*(bitrates_to_probe.end() - 1)) * kRepeatedProbeMinPercentage / 100; + } else { + state_ = State::kProbingComplete; + min_bitrate_to_probe_further_bps_ = kExponentialProbingDisabled; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.h new file mode 100644 index 0000000000..cd0e9b1b57 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_PROBE_CONTROLLER_H_ +#define MODULES_CONGESTION_CONTROLLER_PROBE_CONTROLLER_H_ + +#include + +#include "common_types.h" // NOLINT(build/include) +#include "modules/pacing/paced_sender.h" +#include "rtc_base/criticalsection.h" + +namespace webrtc { + +class Clock; + +// This class controls initiation of probing to estimate initial channel +// capacity. There is also support for probing during a session when max +// bitrate is adjusted by an application. +class ProbeController { + public: + ProbeController(PacedSender* pacer, const Clock* clock); + + void SetBitrates(int64_t min_bitrate_bps, + int64_t start_bitrate_bps, + int64_t max_bitrate_bps); + + void OnNetworkStateChanged(NetworkState state); + + void SetEstimatedBitrate(int64_t bitrate_bps); + + void EnablePeriodicAlrProbing(bool enable); + + void SetAlrEndedTimeMs(int64_t alr_end_time); + + void RequestProbe(); + + // Resets the ProbeController to a state equivalent to as if it was just + // created EXCEPT for |enable_periodic_alr_probing_|. + void Reset(); + + void Process(); + + private: + enum class State { + // Initial state where no probing has been triggered yet. + kInit, + // Waiting for probing results to continue further probing. + kWaitingForProbingResult, + // Probing is complete. + kProbingComplete, + }; + + void InitiateExponentialProbing() RTC_EXCLUSIVE_LOCKS_REQUIRED(critsect_); + void InitiateProbing(int64_t now_ms, + std::initializer_list bitrates_to_probe, + bool probe_further) + RTC_EXCLUSIVE_LOCKS_REQUIRED(critsect_); + + rtc::CriticalSection critsect_; + PacedSender* const pacer_; + const Clock* const clock_; + NetworkState network_state_ RTC_GUARDED_BY(critsect_); + State state_ RTC_GUARDED_BY(critsect_); + int64_t min_bitrate_to_probe_further_bps_ RTC_GUARDED_BY(critsect_); + int64_t time_last_probing_initiated_ms_ RTC_GUARDED_BY(critsect_); + int64_t estimated_bitrate_bps_ RTC_GUARDED_BY(critsect_); + int64_t start_bitrate_bps_ RTC_GUARDED_BY(critsect_); + int64_t max_bitrate_bps_ RTC_GUARDED_BY(critsect_); + int64_t last_bwe_drop_probing_time_ms_ RTC_GUARDED_BY(critsect_); + rtc::Optional alr_end_time_ms_ RTC_GUARDED_BY(critsect_); + bool enable_periodic_alr_probing_ RTC_GUARDED_BY(critsect_); + int64_t time_of_last_large_drop_ms_ RTC_GUARDED_BY(critsect_); + int64_t bitrate_before_last_large_drop_bps_ RTC_GUARDED_BY(critsect_); + + bool in_rapid_recovery_experiment_ RTC_GUARDED_BY(critsect_); + // For WebRTC.BWE.MidCallProbing.* metric. + bool mid_call_probing_waiting_for_result_ RTC_GUARDED_BY(&critsect_); + int64_t mid_call_probing_bitrate_bps_ RTC_GUARDED_BY(&critsect_); + int64_t mid_call_probing_succcess_threshold_ RTC_GUARDED_BY(&critsect_); + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(ProbeController); +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_PROBE_CONTROLLER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller_unittest.cc new file mode 100644 index 0000000000..3477c87a26 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/probe_controller_unittest.cc @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include "modules/congestion_controller/probe_controller.h" +#include "modules/pacing/mock/mock_paced_sender.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using testing::_; +using testing::AtLeast; +using testing::NiceMock; +using testing::Return; + +namespace webrtc { +namespace test { + +namespace { + +constexpr int kMinBitrateBps = 100; +constexpr int kStartBitrateBps = 300; +constexpr int kMaxBitrateBps = 10000; + +constexpr int kExponentialProbingTimeoutMs = 5000; + +constexpr int kAlrProbeInterval = 5000; +constexpr int kAlrEndedTimeoutMs = 3000; +constexpr int kBitrateDropTimeoutMs = 5000; + +} // namespace + +class ProbeControllerTest : public ::testing::Test { + protected: + ProbeControllerTest() : clock_(100000000L) { + probe_controller_.reset(new ProbeController(&pacer_, &clock_)); + } + ~ProbeControllerTest() override {} + + SimulatedClock clock_; + NiceMock pacer_; + std::unique_ptr probe_controller_; +}; + +TEST_F(ProbeControllerTest, InitiatesProbingAtStart) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(AtLeast(2)); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); +} + +TEST_F(ProbeControllerTest, ProbeOnlyWhenNetworkIsUp) { + probe_controller_->OnNetworkStateChanged(kNetworkDown); + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(0); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + + testing::Mock::VerifyAndClearExpectations(&pacer_); + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(AtLeast(2)); + probe_controller_->OnNetworkStateChanged(kNetworkUp); +} + +TEST_F(ProbeControllerTest, InitiatesProbingOnMaxBitrateIncrease) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(AtLeast(2)); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + // Long enough to time out exponential probing. + clock_.AdvanceTimeMilliseconds(kExponentialProbingTimeoutMs); + probe_controller_->SetEstimatedBitrate(kStartBitrateBps); + probe_controller_->Process(); + + EXPECT_CALL(pacer_, CreateProbeCluster(kMaxBitrateBps + 100)); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps + 100); +} + +TEST_F(ProbeControllerTest, InitiatesProbingOnMaxBitrateIncreaseAtMaxBitrate) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(AtLeast(2)); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + // Long enough to time out exponential probing. + clock_.AdvanceTimeMilliseconds(kExponentialProbingTimeoutMs); + probe_controller_->SetEstimatedBitrate(kStartBitrateBps); + probe_controller_->Process(); + + probe_controller_->SetEstimatedBitrate(kMaxBitrateBps); + EXPECT_CALL(pacer_, CreateProbeCluster(kMaxBitrateBps + 100)); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps + 100); +} + +TEST_F(ProbeControllerTest, TestExponentialProbing) { + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + + // Repeated probe should only be sent when estimated bitrate climbs above + // 0.7 * 6 * kStartBitrateBps = 1260. + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(0); + probe_controller_->SetEstimatedBitrate(1000); + testing::Mock::VerifyAndClearExpectations(&pacer_); + + EXPECT_CALL(pacer_, CreateProbeCluster(2 * 1800)); + probe_controller_->SetEstimatedBitrate(1800); +} + +TEST_F(ProbeControllerTest, TestExponentialProbingTimeout) { + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + + // Advance far enough to cause a time out in waiting for probing result. + clock_.AdvanceTimeMilliseconds(kExponentialProbingTimeoutMs); + probe_controller_->Process(); + + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(0); + probe_controller_->SetEstimatedBitrate(1800); +} + +TEST_F(ProbeControllerTest, RequestProbeInAlr) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(2); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); + EXPECT_CALL(pacer_, CreateProbeCluster(0.85 * 500)).Times(1); + EXPECT_CALL(pacer_, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(clock_.TimeInMilliseconds())); + clock_.AdvanceTimeMilliseconds(kAlrProbeInterval + 1); + probe_controller_->Process(); + probe_controller_->SetEstimatedBitrate(250); + probe_controller_->RequestProbe(); +} + +TEST_F(ProbeControllerTest, RequestProbeWhenAlrEndedRecently) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(2); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); + EXPECT_CALL(pacer_, CreateProbeCluster(0.85 * 500)).Times(1); + EXPECT_CALL(pacer_, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(rtc::nullopt)); + clock_.AdvanceTimeMilliseconds(kAlrProbeInterval + 1); + probe_controller_->Process(); + probe_controller_->SetEstimatedBitrate(250); + probe_controller_->SetAlrEndedTimeMs(clock_.TimeInMilliseconds()); + clock_.AdvanceTimeMilliseconds(kAlrEndedTimeoutMs - 1); + probe_controller_->RequestProbe(); +} + +TEST_F(ProbeControllerTest, RequestProbeWhenAlrNotEndedRecently) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(2); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(0); + EXPECT_CALL(pacer_, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(rtc::nullopt)); + clock_.AdvanceTimeMilliseconds(kAlrProbeInterval + 1); + probe_controller_->Process(); + probe_controller_->SetEstimatedBitrate(250); + probe_controller_->SetAlrEndedTimeMs(clock_.TimeInMilliseconds()); + clock_.AdvanceTimeMilliseconds(kAlrEndedTimeoutMs + 1); + probe_controller_->RequestProbe(); +} + +TEST_F(ProbeControllerTest, RequestProbeWhenBweDropNotRecent) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(2); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(0); + EXPECT_CALL(pacer_, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(clock_.TimeInMilliseconds())); + clock_.AdvanceTimeMilliseconds(kAlrProbeInterval + 1); + probe_controller_->Process(); + probe_controller_->SetEstimatedBitrate(250); + clock_.AdvanceTimeMilliseconds(kBitrateDropTimeoutMs + 1); + probe_controller_->RequestProbe(); +} + +TEST_F(ProbeControllerTest, PeriodicProbing) { + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(2); + probe_controller_->EnablePeriodicAlrProbing(true); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); + + int64_t start_time = clock_.TimeInMilliseconds(); + + // Expect the controller to send a new probe after 5s has passed. + EXPECT_CALL(pacer_, CreateProbeCluster(1000)).Times(1); + EXPECT_CALL(pacer_, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(start_time)); + clock_.AdvanceTimeMilliseconds(5000); + probe_controller_->Process(); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); + + // The following probe should be sent at 10s into ALR. + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(0); + EXPECT_CALL(pacer_, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(start_time)); + clock_.AdvanceTimeMilliseconds(4000); + probe_controller_->Process(); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); + + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(1); + EXPECT_CALL(pacer_, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(start_time)); + clock_.AdvanceTimeMilliseconds(1000); + probe_controller_->Process(); + probe_controller_->SetEstimatedBitrate(500); + testing::Mock::VerifyAndClearExpectations(&pacer_); +} + +TEST_F(ProbeControllerTest, PeriodicProbingAfterReset) { + testing::StrictMock local_pacer; + probe_controller_.reset(new ProbeController(&local_pacer, &clock_)); + int64_t alr_start_time = clock_.TimeInMilliseconds(); + EXPECT_CALL(local_pacer, GetApplicationLimitedRegionStartTime()) + .WillRepeatedly(Return(alr_start_time)); + + EXPECT_CALL(local_pacer, CreateProbeCluster(_)).Times(2); + probe_controller_->EnablePeriodicAlrProbing(true); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + probe_controller_->Reset(); + + clock_.AdvanceTimeMilliseconds(10000); + probe_controller_->Process(); + + EXPECT_CALL(local_pacer, CreateProbeCluster(_)).Times(2); + probe_controller_->SetBitrates(kMinBitrateBps, kStartBitrateBps, + kMaxBitrateBps); + + // Make sure we use |kStartBitrateBps| as the estimated bitrate + // until SetEstimatedBitrate is called with an updated estimate. + clock_.AdvanceTimeMilliseconds(10000); + EXPECT_CALL(local_pacer, CreateProbeCluster(kStartBitrateBps*2)); + probe_controller_->Process(); +} + +TEST_F(ProbeControllerTest, TestExponentialProbingOverflow) { + const int64_t kMbpsMultiplier = 1000000; + probe_controller_->SetBitrates(kMinBitrateBps, 10 * kMbpsMultiplier, + 100 * kMbpsMultiplier); + + // Verify that probe bitrate is capped at the specified max bitrate + EXPECT_CALL(pacer_, CreateProbeCluster(100 * kMbpsMultiplier)); + probe_controller_->SetEstimatedBitrate(60 * kMbpsMultiplier); + testing::Mock::VerifyAndClearExpectations(&pacer_); + + // Verify that repeated probes aren't sent. + EXPECT_CALL(pacer_, CreateProbeCluster(_)).Times(0); + probe_controller_->SetEstimatedBitrate(100 * kMbpsMultiplier); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller.cc new file mode 100644 index 0000000000..11e66efbc2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller.cc @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/include/receive_side_congestion_controller.h" + +#include "modules/pacing/packet_router.h" +#include "modules/remote_bitrate_estimator/remote_bitrate_estimator_abs_send_time.h" +#include "modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +namespace { +static const uint32_t kTimeOffsetSwitchThreshold = 30; +} // namespace + +ReceiveSideCongestionController::WrappingBitrateEstimator:: + WrappingBitrateEstimator(RemoteBitrateObserver* observer, + const Clock* clock) + : observer_(observer), + clock_(clock), + rbe_(new RemoteBitrateEstimatorSingleStream(observer_, clock_)), + using_absolute_send_time_(false), + packets_since_absolute_send_time_(0), + min_bitrate_bps_(congestion_controller::GetMinBitrateBps()) {} + +void ReceiveSideCongestionController::WrappingBitrateEstimator::IncomingPacket( + int64_t arrival_time_ms, + size_t payload_size, + const RTPHeader& header) { + rtc::CritScope cs(&crit_sect_); + PickEstimatorFromHeader(header); + rbe_->IncomingPacket(arrival_time_ms, payload_size, header); +} + +void ReceiveSideCongestionController::WrappingBitrateEstimator::Process() { + rtc::CritScope cs(&crit_sect_); + rbe_->Process(); +} + +int64_t ReceiveSideCongestionController::WrappingBitrateEstimator:: + TimeUntilNextProcess() { + rtc::CritScope cs(&crit_sect_); + return rbe_->TimeUntilNextProcess(); +} + +void ReceiveSideCongestionController::WrappingBitrateEstimator::OnRttUpdate( + int64_t avg_rtt_ms, + int64_t max_rtt_ms) { + rtc::CritScope cs(&crit_sect_); + rbe_->OnRttUpdate(avg_rtt_ms, max_rtt_ms); +} + +void ReceiveSideCongestionController::WrappingBitrateEstimator::RemoveStream( + unsigned int ssrc) { + rtc::CritScope cs(&crit_sect_); + rbe_->RemoveStream(ssrc); +} + +bool ReceiveSideCongestionController::WrappingBitrateEstimator::LatestEstimate( + std::vector* ssrcs, + unsigned int* bitrate_bps) const { + rtc::CritScope cs(&crit_sect_); + return rbe_->LatestEstimate(ssrcs, bitrate_bps); +} + +void ReceiveSideCongestionController::WrappingBitrateEstimator::SetMinBitrate( + int min_bitrate_bps) { + rtc::CritScope cs(&crit_sect_); + rbe_->SetMinBitrate(min_bitrate_bps); + min_bitrate_bps_ = min_bitrate_bps; +} + +void ReceiveSideCongestionController::WrappingBitrateEstimator:: + PickEstimatorFromHeader(const RTPHeader& header) { + if (header.extension.hasAbsoluteSendTime) { + // If we see AST in header, switch RBE strategy immediately. + if (!using_absolute_send_time_) { + RTC_LOG(LS_INFO) + << "WrappingBitrateEstimator: Switching to absolute send time RBE."; + using_absolute_send_time_ = true; + PickEstimator(); + } + packets_since_absolute_send_time_ = 0; + } else { + // When we don't see AST, wait for a few packets before going back to TOF. + if (using_absolute_send_time_) { + ++packets_since_absolute_send_time_; + if (packets_since_absolute_send_time_ >= kTimeOffsetSwitchThreshold) { + RTC_LOG(LS_INFO) + << "WrappingBitrateEstimator: Switching to transmission " + << "time offset RBE."; + using_absolute_send_time_ = false; + PickEstimator(); + } + } + } +} + +// Instantiate RBE for Time Offset or Absolute Send Time extensions. +void ReceiveSideCongestionController::WrappingBitrateEstimator:: + PickEstimator() { + if (using_absolute_send_time_) { + rbe_.reset(new RemoteBitrateEstimatorAbsSendTime(observer_, clock_)); + } else { + rbe_.reset(new RemoteBitrateEstimatorSingleStream(observer_, clock_)); + } + rbe_->SetMinBitrate(min_bitrate_bps_); +} + +ReceiveSideCongestionController::ReceiveSideCongestionController( + const Clock* clock, + PacketRouter* packet_router) + : remote_bitrate_estimator_(packet_router, clock), + remote_estimator_proxy_(clock, packet_router) {} + +void ReceiveSideCongestionController::OnReceivedPacket( + int64_t arrival_time_ms, + size_t payload_size, + const RTPHeader& header) { + // Send-side BWE. + if (header.extension.hasTransportSequenceNumber) { + remote_estimator_proxy_.IncomingPacket(arrival_time_ms, payload_size, + header); + } else { + // Receive-side BWE. + remote_bitrate_estimator_.IncomingPacket(arrival_time_ms, payload_size, + header); + } +} + +RemoteBitrateEstimator* +ReceiveSideCongestionController::GetRemoteBitrateEstimator(bool send_side_bwe) { + if (send_side_bwe) { + return &remote_estimator_proxy_; + } else { + return &remote_bitrate_estimator_; + } +} + +const RemoteBitrateEstimator* +ReceiveSideCongestionController::GetRemoteBitrateEstimator( + bool send_side_bwe) const { + if (send_side_bwe) { + return &remote_estimator_proxy_; + } else { + return &remote_bitrate_estimator_; + } +} + +void ReceiveSideCongestionController::OnRttUpdate(int64_t avg_rtt_ms, + int64_t max_rtt_ms) { + remote_bitrate_estimator_.OnRttUpdate(avg_rtt_ms, max_rtt_ms); +} + +void ReceiveSideCongestionController::OnBitrateChanged(int bitrate_bps) { + remote_estimator_proxy_.OnBitrateChanged(bitrate_bps); +} + +int64_t ReceiveSideCongestionController::TimeUntilNextProcess() { + return remote_bitrate_estimator_.TimeUntilNextProcess(); +} + +void ReceiveSideCongestionController::Process() { + remote_bitrate_estimator_.Process(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc new file mode 100644 index 0000000000..4b5a66fb0f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/include/receive_side_congestion_controller.h" +#include "modules/pacing/packet_router.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using testing::_; +using testing::AtLeast; +using testing::NiceMock; +using testing::Return; +using testing::SaveArg; +using testing::StrictMock; + +namespace webrtc { + +namespace { + +// Helper to convert some time format to resolution used in absolute send time +// header extension, rounded upwards. |t| is the time to convert, in some +// resolution. |denom| is the value to divide |t| by to get whole seconds, +// e.g. |denom| = 1000 if |t| is in milliseconds. +uint32_t AbsSendTime(int64_t t, int64_t denom) { + return (((t << 18) + (denom >> 1)) / denom) & 0x00fffffful; +} + +class MockPacketRouter : public PacketRouter { + public: + MOCK_METHOD2(OnReceiveBitrateChanged, + void(const std::vector& ssrcs, uint32_t bitrate)); +}; + +const uint32_t kInitialBitrateBps = 60000; + +} // namespace + +namespace test { + +TEST(ReceiveSideCongestionControllerTest, OnReceivedPacketWithAbsSendTime) { + StrictMock packet_router; + SimulatedClock clock_(123456); + + ReceiveSideCongestionController controller(&clock_, &packet_router); + + size_t payload_size = 1000; + RTPHeader header; + header.ssrc = 0x11eb21c; + header.extension.hasAbsoluteSendTime = true; + + std::vector ssrcs; + EXPECT_CALL(packet_router, OnReceiveBitrateChanged(_, _)) + .WillRepeatedly(SaveArg<0>(&ssrcs)); + + for (int i = 0; i < 10; ++i) { + clock_.AdvanceTimeMilliseconds((1000 * payload_size) / kInitialBitrateBps); + int64_t now_ms = clock_.TimeInMilliseconds(); + header.extension.absoluteSendTime = AbsSendTime(now_ms, 1000); + controller.OnReceivedPacket(now_ms, payload_size, header); + } + + ASSERT_EQ(1u, ssrcs.size()); + EXPECT_EQ(header.ssrc, ssrcs[0]); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller.cc new file mode 100644 index 0000000000..ccca692712 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller.cc @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/include/send_side_congestion_controller.h" + +#include +#include +#include + +#include "modules/bitrate_controller/include/bitrate_controller.h" +#include "modules/congestion_controller/acknowledged_bitrate_estimator.h" +#include "modules/congestion_controller/probe_controller.h" +#include "modules/pacing/alr_detector.h" +#include "modules/remote_bitrate_estimator/include/bwe_defines.h" +#include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" +#include "rtc_base/logging.h" +#include "rtc_base/ptr_util.h" +#include "rtc_base/rate_limiter.h" +#include "rtc_base/socket.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace { + +const char kCwndExperiment[] = "WebRTC-CwndExperiment"; +const char kPacerPushbackExperiment[] = "WebRTC-PacerPushbackExperiment"; +const int64_t kDefaultAcceptedQueueMs = 250; + +bool CwndExperimentEnabled() { + std::string experiment_string = + webrtc::field_trial::FindFullName(kCwndExperiment); + // The experiment is enabled iff the field trial string begins with "Enabled". + return experiment_string.find("Enabled") == 0; +} + +bool ReadCwndExperimentParameter(int64_t* accepted_queue_ms) { + RTC_DCHECK(accepted_queue_ms); + std::string experiment_string = + webrtc::field_trial::FindFullName(kCwndExperiment); + int parsed_values = + sscanf(experiment_string.c_str(), "Enabled-%" PRId64, accepted_queue_ms); + if (parsed_values == 1) { + RTC_CHECK_GE(*accepted_queue_ms, 0) + << "Accepted must be greater than or equal to 0."; + return true; + } + return false; +} + +static const int64_t kRetransmitWindowSizeMs = 500; + +// Makes sure that the bitrate and the min, max values are in valid range. +static void ClampBitrates(int* bitrate_bps, + int* min_bitrate_bps, + int* max_bitrate_bps) { + // TODO(holmer): We should make sure the default bitrates are set to 10 kbps, + // and that we don't try to set the min bitrate to 0 from any applications. + // The congestion controller should allow a min bitrate of 0. + if (*min_bitrate_bps < congestion_controller::GetMinBitrateBps()) + *min_bitrate_bps = congestion_controller::GetMinBitrateBps(); + if (*max_bitrate_bps > 0) + *max_bitrate_bps = std::max(*min_bitrate_bps, *max_bitrate_bps); + if (*bitrate_bps > 0) + *bitrate_bps = std::max(*min_bitrate_bps, *bitrate_bps); +} + +std::vector ReceivedPacketFeedbackVector( + const std::vector& input) { + std::vector received_packet_feedback_vector; + auto is_received = [](const webrtc::PacketFeedback& packet_feedback) { + return packet_feedback.arrival_time_ms != + webrtc::PacketFeedback::kNotReceived; + }; + std::copy_if(input.begin(), input.end(), + std::back_inserter(received_packet_feedback_vector), + is_received); + return received_packet_feedback_vector; +} + +void SortPacketFeedbackVector( + std::vector* const input) { + RTC_DCHECK(input); + std::sort(input->begin(), input->end(), PacketFeedbackComparator()); +} + +} // namespace + +SendSideCongestionController::SendSideCongestionController( + const Clock* clock, + Observer* observer, + RtcEventLog* event_log, + PacedSender* pacer) + : clock_(clock), + observer_(observer), + event_log_(event_log), + pacer_(pacer), + bitrate_controller_( + BitrateController::CreateBitrateController(clock_, event_log)), + acknowledged_bitrate_estimator_( + rtc::MakeUnique()), + probe_controller_(new ProbeController(pacer_, clock_)), + retransmission_rate_limiter_( + new RateLimiter(clock, kRetransmitWindowSizeMs)), + transport_feedback_adapter_(clock_), + last_reported_bitrate_bps_(0), + last_reported_fraction_loss_(0), + last_reported_rtt_(0), + network_state_(kNetworkUp), + pause_pacer_(false), + pacer_paused_(false), + min_bitrate_bps_(congestion_controller::GetMinBitrateBps()), + delay_based_bwe_(new DelayBasedBwe(event_log_, clock_)), + in_cwnd_experiment_(CwndExperimentEnabled()), + accepted_queue_ms_(kDefaultAcceptedQueueMs), + was_in_alr_(false), + pacer_pushback_experiment_( + webrtc::field_trial::IsEnabled(kPacerPushbackExperiment)) { + delay_based_bwe_->SetMinBitrate(min_bitrate_bps_); + if (in_cwnd_experiment_ && + !ReadCwndExperimentParameter(&accepted_queue_ms_)) { + RTC_LOG(LS_WARNING) << "Failed to parse parameters for CwndExperiment " + "from field trial string. Experiment disabled."; + in_cwnd_experiment_ = false; + } +} + +SendSideCongestionController::~SendSideCongestionController() {} + +void SendSideCongestionController::RegisterPacketFeedbackObserver( + PacketFeedbackObserver* observer) { + transport_feedback_adapter_.RegisterPacketFeedbackObserver(observer); +} + +void SendSideCongestionController::DeRegisterPacketFeedbackObserver( + PacketFeedbackObserver* observer) { + transport_feedback_adapter_.DeRegisterPacketFeedbackObserver(observer); +} + +void SendSideCongestionController::RegisterNetworkObserver(Observer* observer) { + rtc::CritScope cs(&observer_lock_); + RTC_DCHECK(observer_ == nullptr); + observer_ = observer; +} + +void SendSideCongestionController::DeRegisterNetworkObserver( + Observer* observer) { + rtc::CritScope cs(&observer_lock_); + RTC_DCHECK_EQ(observer_, observer); + observer_ = nullptr; +} + +void SendSideCongestionController::SetBweBitrates(int min_bitrate_bps, + int start_bitrate_bps, + int max_bitrate_bps) { + ClampBitrates(&start_bitrate_bps, &min_bitrate_bps, &max_bitrate_bps); + bitrate_controller_->SetBitrates(start_bitrate_bps, min_bitrate_bps, + max_bitrate_bps); + + probe_controller_->SetBitrates(min_bitrate_bps, start_bitrate_bps, + max_bitrate_bps); + + { + rtc::CritScope cs(&bwe_lock_); + if (start_bitrate_bps > 0) + delay_based_bwe_->SetStartBitrate(start_bitrate_bps); + min_bitrate_bps_ = min_bitrate_bps; + delay_based_bwe_->SetMinBitrate(min_bitrate_bps_); + } + MaybeTriggerOnNetworkChanged(); +} + +// TODO(holmer): Split this up and use SetBweBitrates in combination with +// OnNetworkRouteChanged. +void SendSideCongestionController::OnNetworkRouteChanged( + const rtc::NetworkRoute& network_route, + int bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) { + ClampBitrates(&bitrate_bps, &min_bitrate_bps, &max_bitrate_bps); + // TODO(honghaiz): Recreate this object once the bitrate controller is + // no longer exposed outside SendSideCongestionController. + bitrate_controller_->ResetBitrates(bitrate_bps, min_bitrate_bps, + max_bitrate_bps); + + transport_feedback_adapter_.SetNetworkIds(network_route.local_network_id, + network_route.remote_network_id); + { + rtc::CritScope cs(&bwe_lock_); + min_bitrate_bps_ = min_bitrate_bps; + delay_based_bwe_.reset(new DelayBasedBwe(event_log_, clock_)); + acknowledged_bitrate_estimator_.reset(new AcknowledgedBitrateEstimator()); + delay_based_bwe_->SetStartBitrate(bitrate_bps); + delay_based_bwe_->SetMinBitrate(min_bitrate_bps); + } + + probe_controller_->Reset(); + probe_controller_->SetBitrates(min_bitrate_bps, bitrate_bps, max_bitrate_bps); + + MaybeTriggerOnNetworkChanged(); +} + +BitrateController* SendSideCongestionController::GetBitrateController() const { + return bitrate_controller_.get(); +} + +bool SendSideCongestionController::AvailableBandwidth( + uint32_t* bandwidth) const { + return bitrate_controller_->AvailableBandwidth(bandwidth); +} + +RtcpBandwidthObserver* SendSideCongestionController::GetBandwidthObserver() + const { + return bitrate_controller_.get(); +} + +RateLimiter* SendSideCongestionController::GetRetransmissionRateLimiter() { + return retransmission_rate_limiter_.get(); +} + +void SendSideCongestionController::EnablePeriodicAlrProbing(bool enable) { + probe_controller_->EnablePeriodicAlrProbing(enable); +} + +int64_t SendSideCongestionController::GetPacerQueuingDelayMs() const { + return IsNetworkDown() ? 0 : pacer_->QueueInMs(); +} + +int64_t SendSideCongestionController::GetFirstPacketTimeMs() const { + return pacer_->FirstSentPacketTimeMs(); +} + +TransportFeedbackObserver* +SendSideCongestionController::GetTransportFeedbackObserver() { + return this; +} + +void SendSideCongestionController::SignalNetworkState(NetworkState state) { + RTC_LOG(LS_INFO) << "SignalNetworkState " + << (state == kNetworkUp ? "Up" : "Down"); + { + rtc::CritScope cs(&network_state_lock_); + pause_pacer_ = state == kNetworkDown; + network_state_ = state; + } + probe_controller_->OnNetworkStateChanged(state); + MaybeTriggerOnNetworkChanged(); +} + +void SendSideCongestionController::SetTransportOverhead( + size_t transport_overhead_bytes_per_packet) { + transport_feedback_adapter_.SetTransportOverhead( + transport_overhead_bytes_per_packet); +} + +void SendSideCongestionController::OnSentPacket( + const rtc::SentPacket& sent_packet) { + // We're not interested in packets without an id, which may be stun packets, + // etc, sent on the same transport. + if (sent_packet.packet_id == -1) + return; + transport_feedback_adapter_.OnSentPacket(sent_packet.packet_id, + sent_packet.send_time_ms); + if (in_cwnd_experiment_) + LimitOutstandingBytes(transport_feedback_adapter_.GetOutstandingBytes()); +} + +void SendSideCongestionController::OnRttUpdate(int64_t avg_rtt_ms, + int64_t max_rtt_ms) { + rtc::CritScope cs(&bwe_lock_); + delay_based_bwe_->OnRttUpdate(avg_rtt_ms, max_rtt_ms); +} + +int64_t SendSideCongestionController::TimeUntilNextProcess() { + return bitrate_controller_->TimeUntilNextProcess(); +} + +void SendSideCongestionController::Process() { + bool pause_pacer; + // TODO(holmer): Once this class is running on a task queue we should + // replace this with a task instead. + { + rtc::CritScope lock(&network_state_lock_); + pause_pacer = pause_pacer_; + } + if (pause_pacer && !pacer_paused_) { + pacer_->Pause(); + pacer_paused_ = true; + } else if (!pause_pacer && pacer_paused_) { + pacer_->Resume(); + pacer_paused_ = false; + } + bitrate_controller_->Process(); + probe_controller_->Process(); + MaybeTriggerOnNetworkChanged(); +} + +void SendSideCongestionController::AddPacket( + uint32_t ssrc, + uint16_t sequence_number, + size_t length, + const PacedPacketInfo& pacing_info) { + transport_feedback_adapter_.AddPacket(ssrc, sequence_number, length, + pacing_info); +} + +void SendSideCongestionController::OnTransportFeedback( + const rtcp::TransportFeedback& feedback) { + RTC_DCHECK_RUNS_SERIALIZED(&worker_race_); + transport_feedback_adapter_.OnTransportFeedback(feedback); + std::vector feedback_vector = ReceivedPacketFeedbackVector( + transport_feedback_adapter_.GetTransportFeedbackVector()); + SortPacketFeedbackVector(&feedback_vector); + + bool currently_in_alr = + pacer_->GetApplicationLimitedRegionStartTime().has_value(); + if (was_in_alr_ && !currently_in_alr) { + int64_t now_ms = rtc::TimeMillis(); + acknowledged_bitrate_estimator_->SetAlrEndedTimeMs(now_ms); + probe_controller_->SetAlrEndedTimeMs(now_ms); + } + was_in_alr_ = currently_in_alr; + + acknowledged_bitrate_estimator_->IncomingPacketFeedbackVector( + feedback_vector); + DelayBasedBwe::Result result; + { + rtc::CritScope cs(&bwe_lock_); + result = delay_based_bwe_->IncomingPacketFeedbackVector( + feedback_vector, acknowledged_bitrate_estimator_->bitrate_bps()); + } + if (result.updated) { + bitrate_controller_->OnDelayBasedBweResult(result); + // Update the estimate in the ProbeController, in case we want to probe. + MaybeTriggerOnNetworkChanged(); + } + if (result.recovered_from_overuse) + probe_controller_->RequestProbe(); + if (in_cwnd_experiment_) + LimitOutstandingBytes(transport_feedback_adapter_.GetOutstandingBytes()); +} + +void SendSideCongestionController::LimitOutstandingBytes( + size_t num_outstanding_bytes) { + RTC_DCHECK(in_cwnd_experiment_); + rtc::CritScope lock(&network_state_lock_); + rtc::Optional min_rtt_ms = + transport_feedback_adapter_.GetMinFeedbackLoopRtt(); + // No valid RTT. Could be because send-side BWE isn't used, in which case + // we don't try to limit the outstanding packets. + if (!min_rtt_ms) + return; + const size_t kMinCwndBytes = 2 * 1500; + size_t max_outstanding_bytes = + std::max((*min_rtt_ms + accepted_queue_ms_) * + last_reported_bitrate_bps_ / 1000 / 8, + kMinCwndBytes); + RTC_LOG(LS_INFO) << clock_->TimeInMilliseconds() + << " Outstanding bytes: " << num_outstanding_bytes + << " pacer queue: " << pacer_->QueueInMs() + << " max outstanding: " << max_outstanding_bytes; + RTC_LOG(LS_INFO) << "Feedback rtt: " << *min_rtt_ms + << " Bitrate: " << last_reported_bitrate_bps_; + pause_pacer_ = num_outstanding_bytes > max_outstanding_bytes; +} + +std::vector +SendSideCongestionController::GetTransportFeedbackVector() const { + RTC_DCHECK_RUNS_SERIALIZED(&worker_race_); + return transport_feedback_adapter_.GetTransportFeedbackVector(); +} + +void SendSideCongestionController::MaybeTriggerOnNetworkChanged() { + uint32_t bitrate_bps; + uint8_t fraction_loss; + int64_t rtt; + bool estimate_changed = bitrate_controller_->GetNetworkParameters( + &bitrate_bps, &fraction_loss, &rtt); + if (estimate_changed) { + pacer_->SetEstimatedBitrate(bitrate_bps); + probe_controller_->SetEstimatedBitrate(bitrate_bps); + retransmission_rate_limiter_->SetMaxRate(bitrate_bps); + } + + if (!pacer_pushback_experiment_) { + bitrate_bps = IsNetworkDown() || IsSendQueueFull() ? 0 : bitrate_bps; + } else { + if (IsNetworkDown()) { + bitrate_bps = 0; + } else { + int64_t queue_length_ms = pacer_->ExpectedQueueTimeMs(); + + if (queue_length_ms == 0) { + encoding_rate_ = 1.0; + } else if (queue_length_ms > 50) { + float encoding_rate = 1.0 - queue_length_ms / 1000.0; + encoding_rate_ = std::min(encoding_rate_, encoding_rate); + encoding_rate_ = std::max(encoding_rate_, 0.0f); + } + + bitrate_bps *= encoding_rate_; + bitrate_bps = bitrate_bps < 50000 ? 0 : bitrate_bps; + } + } + + if (HasNetworkParametersToReportChanged(bitrate_bps, fraction_loss, rtt)) { + int64_t probing_interval_ms; + { + rtc::CritScope cs(&bwe_lock_); + probing_interval_ms = delay_based_bwe_->GetExpectedBwePeriodMs(); + } + { + rtc::CritScope cs(&observer_lock_); + if (observer_) { + observer_->OnNetworkChanged(bitrate_bps, fraction_loss, rtt, + probing_interval_ms); + } + } + } +} + +bool SendSideCongestionController::HasNetworkParametersToReportChanged( + uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt) { + rtc::CritScope cs(&network_state_lock_); + bool changed = + last_reported_bitrate_bps_ != bitrate_bps || + (bitrate_bps > 0 && (last_reported_fraction_loss_ != fraction_loss || + last_reported_rtt_ != rtt)); + if (changed && (last_reported_bitrate_bps_ == 0 || bitrate_bps == 0)) { + RTC_LOG(LS_INFO) << "Bitrate estimate state changed, BWE: " << bitrate_bps + << " bps."; + } + last_reported_bitrate_bps_ = bitrate_bps; + last_reported_fraction_loss_ = fraction_loss; + last_reported_rtt_ = rtt; + return changed; +} + +bool SendSideCongestionController::IsSendQueueFull() const { + return pacer_->ExpectedQueueTimeMs() > PacedSender::kMaxQueueLengthMs; +} + +bool SendSideCongestionController::IsNetworkDown() const { + rtc::CritScope cs(&network_state_lock_); + return network_state_ == kNetworkDown; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller_unittest.cc new file mode 100644 index 0000000000..71aff4a620 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/send_side_congestion_controller_unittest.cc @@ -0,0 +1,494 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "modules/bitrate_controller/include/bitrate_controller.h" +#include "modules/congestion_controller/congestion_controller_unittests_helper.h" +#include "modules/congestion_controller/include/mock/mock_congestion_observer.h" +#include "modules/congestion_controller/include/send_side_congestion_controller.h" +#include "modules/pacing/mock/mock_paced_sender.h" +#include "modules/pacing/packet_router.h" +#include "modules/remote_bitrate_estimator/include/bwe_defines.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" +#include "rtc_base/socket.h" +#include "system_wrappers/include/clock.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using testing::_; +using testing::AtLeast; +using testing::Ge; +using testing::NiceMock; +using testing::Return; +using testing::SaveArg; +using testing::StrictMock; + +namespace webrtc { + +namespace { +const webrtc::PacedPacketInfo kPacingInfo0(0, 5, 2000); +const webrtc::PacedPacketInfo kPacingInfo1(1, 8, 4000); + +const uint32_t kInitialBitrateBps = 60000; + +} // namespace + +namespace test { + +class SendSideCongestionControllerTest : public ::testing::Test { + protected: + SendSideCongestionControllerTest() + : clock_(123456), target_bitrate_observer_(this) {} + ~SendSideCongestionControllerTest() override {} + + void SetUp() override { + pacer_.reset(new NiceMock()); + controller_.reset(new SendSideCongestionController( + &clock_, &observer_, &event_log_, pacer_.get())); + bandwidth_observer_ = controller_->GetBandwidthObserver(); + + // Set the initial bitrate estimate and expect the |observer| and |pacer_| + // to be updated. + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps, _, _, _)); + EXPECT_CALL(*pacer_, SetEstimatedBitrate(kInitialBitrateBps)); + EXPECT_CALL(*pacer_, CreateProbeCluster(kInitialBitrateBps * 3)); + EXPECT_CALL(*pacer_, CreateProbeCluster(kInitialBitrateBps * 5)); + controller_->SetBweBitrates(0, kInitialBitrateBps, 5 * kInitialBitrateBps); + } + + // Custom setup - use an observer that tracks the target bitrate, without + // prescribing on which iterations it must change (like a mock would). + void TargetBitrateTrackingSetup() { + pacer_.reset(new NiceMock()); + controller_.reset(new SendSideCongestionController( + &clock_, &target_bitrate_observer_, &event_log_, pacer_.get())); + controller_->SetBweBitrates(0, kInitialBitrateBps, 5 * kInitialBitrateBps); + } + + void OnSentPacket(const PacketFeedback& packet_feedback) { + constexpr uint32_t ssrc = 0; + controller_->AddPacket(ssrc, packet_feedback.sequence_number, + packet_feedback.payload_size, + packet_feedback.pacing_info); + controller_->OnSentPacket(rtc::SentPacket(packet_feedback.sequence_number, + packet_feedback.send_time_ms)); + } + + // Allows us to track the target bitrate, without prescribing the exact + // iterations when this would hapen, like a mock would. + class TargetBitrateObserver : public SendSideCongestionController::Observer { + public: + explicit TargetBitrateObserver(SendSideCongestionControllerTest* owner) + : owner_(owner) {} + ~TargetBitrateObserver() override = default; + void OnNetworkChanged(uint32_t bitrate_bps, + uint8_t fraction_loss, // 0 - 255. + int64_t rtt_ms, + int64_t probing_interval_ms) override { + owner_->target_bitrate_bps_ = bitrate_bps; + } + + private: + SendSideCongestionControllerTest* owner_; + }; + + void PacketTransmissionAndFeedbackBlock(uint16_t* seq_num, + int64_t runtime_ms, + int64_t delay) { + int64_t delay_buildup = 0; + int64_t start_time_ms = clock_.TimeInMilliseconds(); + while (clock_.TimeInMilliseconds() - start_time_ms < runtime_ms) { + constexpr size_t kPayloadSize = 1000; + PacketFeedback packet(clock_.TimeInMilliseconds() + delay_buildup, + clock_.TimeInMilliseconds(), *seq_num, kPayloadSize, + PacedPacketInfo()); + delay_buildup += delay; // Delay has to increase, or it's just RTT. + OnSentPacket(packet); + // Create expected feedback and send into adapter. + std::unique_ptr feedback( + new rtcp::TransportFeedback()); + feedback->SetBase(packet.sequence_number, packet.arrival_time_ms * 1000); + EXPECT_TRUE(feedback->AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + rtc::Buffer raw_packet = feedback->Build(); + feedback = rtcp::TransportFeedback::ParseFrom(raw_packet.data(), + raw_packet.size()); + EXPECT_TRUE(feedback.get() != nullptr); + controller_->OnTransportFeedback(*feedback.get()); + clock_.AdvanceTimeMilliseconds(50); + controller_->Process(); + ++(*seq_num); + } + } + + SimulatedClock clock_; + StrictMock observer_; + TargetBitrateObserver target_bitrate_observer_; + NiceMock event_log_; + RtcpBandwidthObserver* bandwidth_observer_; + PacketRouter packet_router_; + std::unique_ptr> pacer_; + std::unique_ptr controller_; + + rtc::Optional target_bitrate_bps_; +}; + +TEST_F(SendSideCongestionControllerTest, OnNetworkChanged) { + // Test no change. + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); + + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps * 2, _, _, _)); + EXPECT_CALL(*pacer_, SetEstimatedBitrate(kInitialBitrateBps * 2)); + bandwidth_observer_->OnReceivedEstimatedBitrate(kInitialBitrateBps * 2); + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); + + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps, _, _, _)); + EXPECT_CALL(*pacer_, SetEstimatedBitrate(kInitialBitrateBps)); + bandwidth_observer_->OnReceivedEstimatedBitrate(kInitialBitrateBps); + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); +} + +TEST_F(SendSideCongestionControllerTest, OnSendQueueFull) { + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(PacedSender::kMaxQueueLengthMs + 1)); + + EXPECT_CALL(observer_, OnNetworkChanged(0, _, _, _)); + controller_->Process(); + + // Let the pacer not be full next time the controller checks. + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(PacedSender::kMaxQueueLengthMs - 1)); + + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps, _, _, _)); + controller_->Process(); +} + +TEST_F(SendSideCongestionControllerTest, OnSendQueueFullAndEstimateChange) { + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(PacedSender::kMaxQueueLengthMs + 1)); + EXPECT_CALL(observer_, OnNetworkChanged(0, _, _, _)); + controller_->Process(); + + // Receive new estimate but let the queue still be full. + bandwidth_observer_->OnReceivedEstimatedBitrate(kInitialBitrateBps * 2); + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(PacedSender::kMaxQueueLengthMs + 1)); + // The send pacer should get the new estimate though. + EXPECT_CALL(*pacer_, SetEstimatedBitrate(kInitialBitrateBps * 2)); + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); + + // Let the pacer not be full next time the controller checks. + // |OnNetworkChanged| should be called with the new estimate. + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(PacedSender::kMaxQueueLengthMs - 1)); + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps * 2, _, _, _)); + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); +} + +TEST_F(SendSideCongestionControllerTest, SignalNetworkState) { + EXPECT_CALL(observer_, OnNetworkChanged(0, _, _, _)); + controller_->SignalNetworkState(kNetworkDown); + + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps, _, _, _)); + controller_->SignalNetworkState(kNetworkUp); + + EXPECT_CALL(observer_, OnNetworkChanged(0, _, _, _)); + controller_->SignalNetworkState(kNetworkDown); +} + +TEST_F(SendSideCongestionControllerTest, OnNetworkRouteChanged) { + int new_bitrate = 200000; + testing::Mock::VerifyAndClearExpectations(pacer_.get()); + EXPECT_CALL(observer_, OnNetworkChanged(new_bitrate, _, _, _)); + EXPECT_CALL(*pacer_, SetEstimatedBitrate(new_bitrate)); + rtc::NetworkRoute route; + route.local_network_id = 1; + controller_->OnNetworkRouteChanged(route, new_bitrate, -1, -1); + + // If the bitrate is reset to -1, the new starting bitrate will be + // the minimum default bitrate kMinBitrateBps. + EXPECT_CALL( + observer_, + OnNetworkChanged(congestion_controller::GetMinBitrateBps(), _, _, _)); + EXPECT_CALL(*pacer_, + SetEstimatedBitrate(congestion_controller::GetMinBitrateBps())); + route.local_network_id = 2; + controller_->OnNetworkRouteChanged(route, -1, -1, -1); +} + +TEST_F(SendSideCongestionControllerTest, OldFeedback) { + int new_bitrate = 200000; + testing::Mock::VerifyAndClearExpectations(pacer_.get()); + EXPECT_CALL(observer_, OnNetworkChanged(new_bitrate, _, _, _)); + EXPECT_CALL(*pacer_, SetEstimatedBitrate(new_bitrate)); + + // Send a few packets on the first network route. + std::vector packets; + packets.push_back(PacketFeedback(0, 0, 0, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(10, 10, 1, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(20, 20, 2, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(30, 30, 3, 1500, kPacingInfo1)); + packets.push_back(PacketFeedback(40, 40, 4, 1500, kPacingInfo1)); + + for (const PacketFeedback& packet : packets) + OnSentPacket(packet); + + // Change route and then insert a number of feedback packets. + rtc::NetworkRoute route; + route.local_network_id = 1; + controller_->OnNetworkRouteChanged(route, new_bitrate, -1, -1); + + for (const PacketFeedback& packet : packets) { + rtcp::TransportFeedback feedback; + feedback.SetBase(packet.sequence_number, packet.arrival_time_ms * 1000); + + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + feedback.Build(); + controller_->OnTransportFeedback(feedback); + } + + // If the bitrate is reset to -1, the new starting bitrate will be + // the minimum default bitrate kMinBitrateBps. + EXPECT_CALL( + observer_, + OnNetworkChanged(congestion_controller::GetMinBitrateBps(), _, _, _)); + EXPECT_CALL(*pacer_, + SetEstimatedBitrate(congestion_controller::GetMinBitrateBps())); + route.local_network_id = 2; + controller_->OnNetworkRouteChanged(route, -1, -1, -1); +} + +TEST_F(SendSideCongestionControllerTest, + SignalNetworkStateAndQueueIsFullAndEstimateChange) { + // Send queue is full + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillRepeatedly(Return(PacedSender::kMaxQueueLengthMs + 1)); + EXPECT_CALL(observer_, OnNetworkChanged(0, _, _, _)); + controller_->Process(); + + // Queue is full and network is down. Expect no bitrate change. + controller_->SignalNetworkState(kNetworkDown); + controller_->Process(); + + // Queue is full but network is up. Expect no bitrate change. + controller_->SignalNetworkState(kNetworkUp); + controller_->Process(); + + // Receive new estimate but let the queue still be full. + EXPECT_CALL(*pacer_, SetEstimatedBitrate(kInitialBitrateBps * 2)); + bandwidth_observer_->OnReceivedEstimatedBitrate(kInitialBitrateBps * 2); + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); + + // Let the pacer not be full next time the controller checks. + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(PacedSender::kMaxQueueLengthMs - 1)); + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps * 2, _, _, _)); + controller_->Process(); +} + +TEST_F(SendSideCongestionControllerTest, GetPacerQueuingDelayMs) { + EXPECT_CALL(observer_, OnNetworkChanged(_, _, _, _)).Times(AtLeast(1)); + + const int64_t kQueueTimeMs = 123; + EXPECT_CALL(*pacer_, QueueInMs()).WillRepeatedly(Return(kQueueTimeMs)); + EXPECT_EQ(kQueueTimeMs, controller_->GetPacerQueuingDelayMs()); + + // Expect zero pacer delay when network is down. + controller_->SignalNetworkState(kNetworkDown); + EXPECT_EQ(0, controller_->GetPacerQueuingDelayMs()); + + // Network is up, pacer delay should be reported. + controller_->SignalNetworkState(kNetworkUp); + EXPECT_EQ(kQueueTimeMs, controller_->GetPacerQueuingDelayMs()); +} + +TEST_F(SendSideCongestionControllerTest, GetProbingInterval) { + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); + + EXPECT_CALL(observer_, OnNetworkChanged(_, _, _, testing::Ne(0))); + EXPECT_CALL(*pacer_, SetEstimatedBitrate(_)); + bandwidth_observer_->OnReceivedEstimatedBitrate(kInitialBitrateBps * 2); + clock_.AdvanceTimeMilliseconds(25); + controller_->Process(); +} + +TEST_F(SendSideCongestionControllerTest, ProbeOnRouteChange) { + testing::Mock::VerifyAndClearExpectations(pacer_.get()); + EXPECT_CALL(*pacer_, CreateProbeCluster(kInitialBitrateBps * 6)); + EXPECT_CALL(*pacer_, CreateProbeCluster(kInitialBitrateBps * 12)); + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps * 2, _, _, _)); + rtc::NetworkRoute route; + route.local_network_id = 1; + controller_->OnNetworkRouteChanged(route, 2 * kInitialBitrateBps, 0, + 20 * kInitialBitrateBps); +} + +// Estimated bitrate reduced when the feedbacks arrive with such a long delay, +// that the send-time-history no longer holds the feedbacked packets. +TEST_F(SendSideCongestionControllerTest, LongFeedbackDelays) { + TargetBitrateTrackingSetup(); + + const int64_t kFeedbackTimeoutMs = 60001; + const int kMaxConsecutiveFailedLookups = 5; + for (int i = 0; i < kMaxConsecutiveFailedLookups; ++i) { + std::vector packets; + packets.push_back( + PacketFeedback(i * 100, 2 * i * 100, 0, 1500, kPacingInfo0)); + packets.push_back( + PacketFeedback(i * 100 + 10, 2 * i * 100 + 10, 1, 1500, kPacingInfo0)); + packets.push_back( + PacketFeedback(i * 100 + 20, 2 * i * 100 + 20, 2, 1500, kPacingInfo0)); + packets.push_back( + PacketFeedback(i * 100 + 30, 2 * i * 100 + 30, 3, 1500, kPacingInfo1)); + packets.push_back( + PacketFeedback(i * 100 + 40, 2 * i * 100 + 40, 4, 1500, kPacingInfo1)); + + for (const PacketFeedback& packet : packets) + OnSentPacket(packet); + + rtcp::TransportFeedback feedback; + feedback.SetBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : packets) { + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + + feedback.Build(); + + clock_.AdvanceTimeMilliseconds(kFeedbackTimeoutMs); + PacketFeedback later_packet(kFeedbackTimeoutMs + i * 100 + 40, + kFeedbackTimeoutMs + i * 200 + 40, 5, 1500, + kPacingInfo1); + OnSentPacket(later_packet); + + controller_->OnTransportFeedback(feedback); + + // Check that packets have timed out. + for (PacketFeedback& packet : packets) { + packet.send_time_ms = -1; + packet.payload_size = 0; + packet.pacing_info = PacedPacketInfo(); + } + ComparePacketFeedbackVectors(packets, + controller_->GetTransportFeedbackVector()); + } + + controller_->Process(); + + EXPECT_EQ(kInitialBitrateBps / 2, target_bitrate_bps_); + + // Test with feedback that isn't late enough to time out. + { + std::vector packets; + packets.push_back(PacketFeedback(100, 200, 0, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(110, 210, 1, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(120, 220, 2, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(130, 230, 3, 1500, kPacingInfo1)); + packets.push_back(PacketFeedback(140, 240, 4, 1500, kPacingInfo1)); + + for (const PacketFeedback& packet : packets) + OnSentPacket(packet); + + rtcp::TransportFeedback feedback; + feedback.SetBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : packets) { + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + + feedback.Build(); + + clock_.AdvanceTimeMilliseconds(kFeedbackTimeoutMs - 1); + PacketFeedback later_packet(kFeedbackTimeoutMs + 140, + kFeedbackTimeoutMs + 240, 5, 1500, + kPacingInfo1); + OnSentPacket(later_packet); + + controller_->OnTransportFeedback(feedback); + ComparePacketFeedbackVectors(packets, + controller_->GetTransportFeedbackVector()); + } +} + +// Bandwidth estimation is updated when feedbacks are received. +// Feedbacks which show an increasing delay cause the estimation to be reduced. +TEST_F(SendSideCongestionControllerTest, UpdatesDelayBasedEstimate) { + TargetBitrateTrackingSetup(); + + const int64_t kRunTimeMs = 6000; + uint16_t seq_num = 0; + + // The test must run and insert packets/feedback long enough that the + // BWE computes a valid estimate. This is first done in an environment which + // simulates no bandwidth limitation, and therefore not built-up delay. + PacketTransmissionAndFeedbackBlock(&seq_num, kRunTimeMs, 0); + ASSERT_TRUE(target_bitrate_bps_); + + // Repeat, but this time with a building delay, and make sure that the + // estimation is adjusted downwards. + uint32_t bitrate_before_delay = *target_bitrate_bps_; + PacketTransmissionAndFeedbackBlock(&seq_num, kRunTimeMs, 50); + EXPECT_LT(*target_bitrate_bps_, bitrate_before_delay); +} + +TEST_F(SendSideCongestionControllerTest, PacerQueueEncodeRatePushback) { + ScopedFieldTrials pushback_field_trial( + "WebRTC-PacerPushbackExperiment/Enabled/"); + SetUp(); + + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()).WillOnce(Return(0)); + controller_->Process(); + + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()).WillOnce(Return(100)); + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps * 0.9, _, _, _)); + controller_->Process(); + + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()).WillOnce(Return(50)); + controller_->Process(); + + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()).WillOnce(Return(0)); + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps, _, _, _)); + controller_->Process(); + + const uint32_t kMinAdjustedBps = 50000; + int expected_queue_threshold = + 1000 - kMinAdjustedBps * 1000.0 / kInitialBitrateBps; + + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(expected_queue_threshold)); + EXPECT_CALL(observer_, OnNetworkChanged(Ge(kMinAdjustedBps), _, _, _)); + controller_->Process(); + + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()) + .WillOnce(Return(expected_queue_threshold + 1)); + EXPECT_CALL(observer_, OnNetworkChanged(0, _, _, _)); + controller_->Process(); + + EXPECT_CALL(*pacer_, ExpectedQueueTimeMs()).WillOnce(Return(0)); + EXPECT_CALL(observer_, OnNetworkChanged(kInitialBitrateBps, _, _, _)); + controller_->Process(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.cc new file mode 100644 index 0000000000..0989875df9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.cc @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/transport_feedback_adapter.h" + +#include + +#include "modules/congestion_controller/delay_based_bwe.h" +#include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/mod_ops.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +const int64_t kNoTimestamp = -1; +const int64_t kSendTimeHistoryWindowMs = 60000; +const int64_t kBaseTimestampScaleFactor = + rtcp::TransportFeedback::kDeltaScaleFactor * (1 << 8); +const int64_t kBaseTimestampRangeSizeUs = kBaseTimestampScaleFactor * (1 << 24); + +TransportFeedbackAdapter::TransportFeedbackAdapter(const Clock* clock) + : send_side_bwe_with_overhead_( + webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), + transport_overhead_bytes_per_packet_(0), + send_time_history_(clock, kSendTimeHistoryWindowMs), + clock_(clock), + current_offset_ms_(kNoTimestamp), + last_timestamp_us_(kNoTimestamp), + local_net_id_(0), + remote_net_id_(0) {} + +TransportFeedbackAdapter::~TransportFeedbackAdapter() { + RTC_DCHECK(observers_.empty()); +} + +void TransportFeedbackAdapter::RegisterPacketFeedbackObserver( + PacketFeedbackObserver* observer) { + rtc::CritScope cs(&observers_lock_); + RTC_DCHECK(observer); + RTC_DCHECK(std::find(observers_.begin(), observers_.end(), observer) == + observers_.end()); + observers_.push_back(observer); +} + +void TransportFeedbackAdapter::DeRegisterPacketFeedbackObserver( + PacketFeedbackObserver* observer) { + rtc::CritScope cs(&observers_lock_); + RTC_DCHECK(observer); + const auto it = std::find(observers_.begin(), observers_.end(), observer); + RTC_DCHECK(it != observers_.end()); + observers_.erase(it); +} + +void TransportFeedbackAdapter::AddPacket(uint32_t ssrc, + uint16_t sequence_number, + size_t length, + const PacedPacketInfo& pacing_info) { + { + rtc::CritScope cs(&lock_); + if (send_side_bwe_with_overhead_) { + length += transport_overhead_bytes_per_packet_; + } + const int64_t creation_time_ms = clock_->TimeInMilliseconds(); + send_time_history_.AddAndRemoveOld( + PacketFeedback(creation_time_ms, sequence_number, length, local_net_id_, + remote_net_id_, pacing_info)); + } + + { + rtc::CritScope cs(&observers_lock_); + for (auto observer : observers_) { + observer->OnPacketAdded(ssrc, sequence_number); + } + } +} + +void TransportFeedbackAdapter::OnSentPacket(uint16_t sequence_number, + int64_t send_time_ms) { + rtc::CritScope cs(&lock_); + send_time_history_.OnSentPacket(sequence_number, send_time_ms); +} + +void TransportFeedbackAdapter::SetTransportOverhead( + int transport_overhead_bytes_per_packet) { + rtc::CritScope cs(&lock_); + transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet; +} + +void TransportFeedbackAdapter::SetNetworkIds(uint16_t local_id, + uint16_t remote_id) { + rtc::CritScope cs(&lock_); + local_net_id_ = local_id; + remote_net_id_ = remote_id; +} + +std::vector TransportFeedbackAdapter::GetPacketFeedbackVector( + const rtcp::TransportFeedback& feedback) { + int64_t timestamp_us = feedback.GetBaseTimeUs(); + int64_t now_ms = clock_->TimeInMilliseconds(); + // Add timestamp deltas to a local time base selected on first packet arrival. + // This won't be the true time base, but makes it easier to manually inspect + // time stamps. + if (last_timestamp_us_ == kNoTimestamp) { + current_offset_ms_ = now_ms; + } else { + int64_t delta = timestamp_us - last_timestamp_us_; + + // Detect and compensate for wrap-arounds in base time. + if (std::abs(delta - kBaseTimestampRangeSizeUs) < std::abs(delta)) { + delta -= kBaseTimestampRangeSizeUs; // Wrap backwards. + } else if (std::abs(delta + kBaseTimestampRangeSizeUs) < std::abs(delta)) { + delta += kBaseTimestampRangeSizeUs; // Wrap forwards. + } + + current_offset_ms_ += delta / 1000; + } + last_timestamp_us_ = timestamp_us; + + std::vector packet_feedback_vector; + if (feedback.GetPacketStatusCount() == 0) { + RTC_LOG(LS_INFO) << "Empty transport feedback packet received."; + return packet_feedback_vector; + } + packet_feedback_vector.reserve(feedback.GetPacketStatusCount()); + int64_t feedback_rtt = -1; + { + rtc::CritScope cs(&lock_); + size_t failed_lookups = 0; + int64_t offset_us = 0; + int64_t timestamp_ms = 0; + uint16_t seq_num = feedback.GetBaseSequence(); + for (const auto& packet : feedback.GetReceivedPackets()) { + // Insert into the vector those unreceived packets which precede this + // iteration's received packet. + for (; seq_num != packet.sequence_number(); ++seq_num) { + PacketFeedback packet_feedback(PacketFeedback::kNotReceived, seq_num); + // Note: Element not removed from history because it might be reported + // as received by another feedback. + if (!send_time_history_.GetFeedback(&packet_feedback, false)) + ++failed_lookups; + if (packet_feedback.local_net_id == local_net_id_ && + packet_feedback.remote_net_id == remote_net_id_) { + packet_feedback_vector.push_back(packet_feedback); + } + } + + // Handle this iteration's received packet. + offset_us += packet.delta_us(); + timestamp_ms = current_offset_ms_ + (offset_us / 1000); + PacketFeedback packet_feedback(timestamp_ms, packet.sequence_number()); + if (!send_time_history_.GetFeedback(&packet_feedback, true)) + ++failed_lookups; + if (packet_feedback.local_net_id == local_net_id_ && + packet_feedback.remote_net_id == remote_net_id_) { + if (packet_feedback.send_time_ms >= 0) { + int64_t rtt = now_ms - packet_feedback.send_time_ms; + // max() is used to account for feedback being delayed by the + // receiver. + feedback_rtt = std::max(rtt, feedback_rtt); + } + packet_feedback_vector.push_back(packet_feedback); + } + + ++seq_num; + } + + if (failed_lookups > 0) { + RTC_LOG(LS_WARNING) << "Failed to lookup send time for " << failed_lookups + << " packet" << (failed_lookups > 1 ? "s" : "") + << ". Send time history too small?"; + } + if (feedback_rtt > -1) { + feedback_rtts_.push_back(feedback_rtt); + const size_t kFeedbackRttWindow = 32; + if (feedback_rtts_.size() > kFeedbackRttWindow) + feedback_rtts_.pop_front(); + min_feedback_rtt_.emplace( + *std::min_element(feedback_rtts_.begin(), feedback_rtts_.end())); + } + } + return packet_feedback_vector; +} + +void TransportFeedbackAdapter::OnTransportFeedback( + const rtcp::TransportFeedback& feedback) { + last_packet_feedback_vector_ = GetPacketFeedbackVector(feedback); + { + rtc::CritScope cs(&observers_lock_); + for (auto observer : observers_) { + observer->OnPacketFeedbackVector(last_packet_feedback_vector_); + } + } +} + +std::vector +TransportFeedbackAdapter::GetTransportFeedbackVector() const { + return last_packet_feedback_vector_; +} + +rtc::Optional TransportFeedbackAdapter::GetMinFeedbackLoopRtt() const { + rtc::CritScope cs(&lock_); + return min_feedback_rtt_; +} + +size_t TransportFeedbackAdapter::GetOutstandingBytes() const { + rtc::CritScope cs(&lock_); + return send_time_history_.GetOutstandingBytes(local_net_id_, remote_net_id_); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.h new file mode 100644 index 0000000000..008bab87f1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_CONGESTION_CONTROLLER_TRANSPORT_FEEDBACK_ADAPTER_H_ +#define MODULES_CONGESTION_CONTROLLER_TRANSPORT_FEEDBACK_ADAPTER_H_ + +#include +#include + +#include "modules/remote_bitrate_estimator/include/send_time_history.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +class PacketFeedbackObserver; + +namespace rtcp { +class TransportFeedback; +} // namespace rtcp + +class TransportFeedbackAdapter { + public: + explicit TransportFeedbackAdapter(const Clock* clock); + virtual ~TransportFeedbackAdapter(); + + void RegisterPacketFeedbackObserver(PacketFeedbackObserver* observer); + void DeRegisterPacketFeedbackObserver(PacketFeedbackObserver* observer); + + void AddPacket(uint32_t ssrc, + uint16_t sequence_number, + size_t length, + const PacedPacketInfo& pacing_info); + void OnSentPacket(uint16_t sequence_number, int64_t send_time_ms); + + // TODO(holmer): This method should return DelayBasedBwe::Result so that we + // can get rid of the dependency on BitrateController. Requires changes + // to the CongestionController interface. + void OnTransportFeedback(const rtcp::TransportFeedback& feedback); + std::vector GetTransportFeedbackVector() const; + rtc::Optional GetMinFeedbackLoopRtt() const; + + void SetTransportOverhead(int transport_overhead_bytes_per_packet); + + void SetNetworkIds(uint16_t local_id, uint16_t remote_id); + + size_t GetOutstandingBytes() const; + + private: + std::vector GetPacketFeedbackVector( + const rtcp::TransportFeedback& feedback); + + const bool send_side_bwe_with_overhead_; + rtc::CriticalSection lock_; + int transport_overhead_bytes_per_packet_ RTC_GUARDED_BY(&lock_); + SendTimeHistory send_time_history_ RTC_GUARDED_BY(&lock_); + const Clock* const clock_; + int64_t current_offset_ms_; + int64_t last_timestamp_us_; + std::vector last_packet_feedback_vector_; + uint16_t local_net_id_ RTC_GUARDED_BY(&lock_); + uint16_t remote_net_id_ RTC_GUARDED_BY(&lock_); + std::deque feedback_rtts_ RTC_GUARDED_BY(&lock_); + rtc::Optional min_feedback_rtt_ RTC_GUARDED_BY(&lock_); + + rtc::CriticalSection observers_lock_; + std::vector observers_ + RTC_GUARDED_BY(&observers_lock_); +}; + +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_TRANSPORT_FEEDBACK_ADAPTER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter_unittest.cc new file mode 100644 index 0000000000..1cff0d981f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/transport_feedback_adapter_unittest.cc @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "modules/bitrate_controller/include/mock/mock_bitrate_controller.h" +#include "modules/congestion_controller/congestion_controller_unittests_helper.h" +#include "modules/congestion_controller/transport_feedback_adapter.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::Invoke; + +namespace webrtc { + +namespace { +const PacedPacketInfo kPacingInfo0(0, 5, 2000); +const PacedPacketInfo kPacingInfo1(1, 8, 4000); +const PacedPacketInfo kPacingInfo2(2, 14, 7000); +const PacedPacketInfo kPacingInfo3(3, 20, 10000); +const PacedPacketInfo kPacingInfo4(4, 22, 10000); +} + +namespace test { + +class MockPacketFeedbackObserver : public webrtc::PacketFeedbackObserver { + public: + MOCK_METHOD2(OnPacketAdded, void(uint32_t ssrc, uint16_t seq_num)); + MOCK_METHOD1(OnPacketFeedbackVector, + void(const std::vector& packet_feedback_vector)); +}; + +class TransportFeedbackAdapterTest : public ::testing::Test { + public: + TransportFeedbackAdapterTest() : clock_(0) {} + + virtual ~TransportFeedbackAdapterTest() {} + + virtual void SetUp() { + adapter_.reset(new TransportFeedbackAdapter(&clock_)); + } + + virtual void TearDown() { adapter_.reset(); } + + protected: + void OnReceivedEstimatedBitrate(uint32_t bitrate) {} + + void OnReceivedRtcpReceiverReport(const ReportBlockList& report_blocks, + int64_t rtt, + int64_t now_ms) {} + + void OnSentPacket(const PacketFeedback& packet_feedback) { + adapter_->AddPacket(kSsrc, packet_feedback.sequence_number, + packet_feedback.payload_size, + packet_feedback.pacing_info); + adapter_->OnSentPacket(packet_feedback.sequence_number, + packet_feedback.send_time_ms); + } + + static constexpr uint32_t kSsrc = 8492; + + SimulatedClock clock_; + std::unique_ptr adapter_; +}; + +TEST_F(TransportFeedbackAdapterTest, ObserverSanity) { + MockPacketFeedbackObserver mock; + adapter_->RegisterPacketFeedbackObserver(&mock); + + const std::vector packets = { + PacketFeedback(100, 200, 0, 1000, kPacingInfo0), + PacketFeedback(110, 210, 1, 2000, kPacingInfo0), + PacketFeedback(120, 220, 2, 3000, kPacingInfo0) + }; + + rtcp::TransportFeedback feedback; + feedback.SetBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : packets) { + EXPECT_CALL(mock, OnPacketAdded(kSsrc, packet.sequence_number)).Times(1); + OnSentPacket(packet); + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + + EXPECT_CALL(mock, OnPacketFeedbackVector(_)).Times(1); + adapter_->OnTransportFeedback(feedback); + + adapter_->DeRegisterPacketFeedbackObserver(&mock); + + // After deregistration, the observer no longers gets indications. + EXPECT_CALL(mock, OnPacketAdded(_, _)).Times(0); + const PacketFeedback new_packet(130, 230, 3, 4000, kPacingInfo0); + OnSentPacket(new_packet); + + rtcp::TransportFeedback second_feedback; + second_feedback.SetBase(new_packet.sequence_number, + new_packet.arrival_time_ms * 1000); + EXPECT_TRUE(feedback.AddReceivedPacket(new_packet.sequence_number, + new_packet.arrival_time_ms * 1000)); + EXPECT_CALL(mock, OnPacketFeedbackVector(_)).Times(0); + adapter_->OnTransportFeedback(second_feedback); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST_F(TransportFeedbackAdapterTest, ObserverDoubleRegistrationDeathTest) { + MockPacketFeedbackObserver mock; + adapter_->RegisterPacketFeedbackObserver(&mock); + EXPECT_DEATH(adapter_->RegisterPacketFeedbackObserver(&mock), ""); + adapter_->DeRegisterPacketFeedbackObserver(&mock); +} + +TEST_F(TransportFeedbackAdapterTest, ObserverMissingDeRegistrationDeathTest) { + MockPacketFeedbackObserver mock; + adapter_->RegisterPacketFeedbackObserver(&mock); + EXPECT_DEATH(adapter_.reset(), ""); + adapter_->DeRegisterPacketFeedbackObserver(&mock); +} +#endif + +TEST_F(TransportFeedbackAdapterTest, AdaptsFeedbackAndPopulatesSendTimes) { + std::vector packets; + packets.push_back(PacketFeedback(100, 200, 0, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(110, 210, 1, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(120, 220, 2, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(130, 230, 3, 1500, kPacingInfo1)); + packets.push_back(PacketFeedback(140, 240, 4, 1500, kPacingInfo1)); + + for (const PacketFeedback& packet : packets) + OnSentPacket(packet); + + rtcp::TransportFeedback feedback; + feedback.SetBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : packets) { + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + + feedback.Build(); + + adapter_->OnTransportFeedback(feedback); + ComparePacketFeedbackVectors(packets, adapter_->GetTransportFeedbackVector()); +} + +TEST_F(TransportFeedbackAdapterTest, FeedbackVectorReportsUnreceived) { + std::vector sent_packets = { + PacketFeedback(100, 220, 0, 1500, kPacingInfo0), + PacketFeedback(110, 210, 1, 1500, kPacingInfo0), + PacketFeedback(120, 220, 2, 1500, kPacingInfo0), + PacketFeedback(130, 230, 3, 1500, kPacingInfo0), + PacketFeedback(140, 240, 4, 1500, kPacingInfo0), + PacketFeedback(150, 250, 5, 1500, kPacingInfo0), + PacketFeedback(160, 260, 6, 1500, kPacingInfo0) + }; + + for (const PacketFeedback& packet : sent_packets) + OnSentPacket(packet); + + // Note: Important to include the last packet, as only unreceived packets in + // between received packets can be inferred. + std::vector received_packets = { + sent_packets[0], sent_packets[2], sent_packets[6] + }; + + rtcp::TransportFeedback feedback; + feedback.SetBase(received_packets[0].sequence_number, + received_packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : received_packets) { + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + + feedback.Build(); + + adapter_->OnTransportFeedback(feedback); + ComparePacketFeedbackVectors(sent_packets, + adapter_->GetTransportFeedbackVector()); +} + +TEST_F(TransportFeedbackAdapterTest, HandlesDroppedPackets) { + std::vector packets; + packets.push_back(PacketFeedback(100, 200, 0, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(110, 210, 1, 1500, kPacingInfo1)); + packets.push_back(PacketFeedback(120, 220, 2, 1500, kPacingInfo2)); + packets.push_back(PacketFeedback(130, 230, 3, 1500, kPacingInfo3)); + packets.push_back(PacketFeedback(140, 240, 4, 1500, kPacingInfo4)); + + const uint16_t kSendSideDropBefore = 1; + const uint16_t kReceiveSideDropAfter = 3; + + for (const PacketFeedback& packet : packets) { + if (packet.sequence_number >= kSendSideDropBefore) + OnSentPacket(packet); + } + + rtcp::TransportFeedback feedback; + feedback.SetBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : packets) { + if (packet.sequence_number <= kReceiveSideDropAfter) { + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + } + + feedback.Build(); + + std::vector expected_packets( + packets.begin(), packets.begin() + kReceiveSideDropAfter + 1); + // Packets that have timed out on the send-side have lost the + // information stored on the send-side. + for (size_t i = 0; i < kSendSideDropBefore; ++i) { + expected_packets[i].send_time_ms = -1; + expected_packets[i].payload_size = 0; + expected_packets[i].pacing_info = PacedPacketInfo(); + } + + adapter_->OnTransportFeedback(feedback); + ComparePacketFeedbackVectors(expected_packets, + adapter_->GetTransportFeedbackVector()); +} + +TEST_F(TransportFeedbackAdapterTest, SendTimeWrapsBothWays) { + int64_t kHighArrivalTimeMs = rtcp::TransportFeedback::kDeltaScaleFactor * + static_cast(1 << 8) * + static_cast((1 << 23) - 1) / 1000; + std::vector packets; + packets.push_back( + PacketFeedback(kHighArrivalTimeMs - 64, 200, 0, 1500, PacedPacketInfo())); + packets.push_back( + PacketFeedback(kHighArrivalTimeMs + 64, 210, 1, 1500, PacedPacketInfo())); + packets.push_back( + PacketFeedback(kHighArrivalTimeMs, 220, 2, 1500, PacedPacketInfo())); + + for (const PacketFeedback& packet : packets) + OnSentPacket(packet); + + for (size_t i = 0; i < packets.size(); ++i) { + std::unique_ptr feedback( + new rtcp::TransportFeedback()); + feedback->SetBase(packets[i].sequence_number, + packets[i].arrival_time_ms * 1000); + + EXPECT_TRUE(feedback->AddReceivedPacket(packets[i].sequence_number, + packets[i].arrival_time_ms * 1000)); + + rtc::Buffer raw_packet = feedback->Build(); + feedback = rtcp::TransportFeedback::ParseFrom(raw_packet.data(), + raw_packet.size()); + + std::vector expected_packets; + expected_packets.push_back(packets[i]); + + adapter_->OnTransportFeedback(*feedback.get()); + ComparePacketFeedbackVectors(expected_packets, + adapter_->GetTransportFeedbackVector()); + } +} + +TEST_F(TransportFeedbackAdapterTest, HandlesArrivalReordering) { + std::vector packets; + packets.push_back(PacketFeedback(120, 200, 0, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(110, 210, 1, 1500, kPacingInfo0)); + packets.push_back(PacketFeedback(100, 220, 2, 1500, kPacingInfo0)); + + for (const PacketFeedback& packet : packets) + OnSentPacket(packet); + + rtcp::TransportFeedback feedback; + feedback.SetBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : packets) { + EXPECT_TRUE(feedback.AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + + feedback.Build(); + + // Adapter keeps the packets ordered by sequence number (which is itself + // assigned by the order of transmission). Reordering by some other criteria, + // eg. arrival time, is up to the observers. + adapter_->OnTransportFeedback(feedback); + ComparePacketFeedbackVectors(packets, adapter_->GetTransportFeedbackVector()); +} + +TEST_F(TransportFeedbackAdapterTest, TimestampDeltas) { + std::vector sent_packets; + const int64_t kSmallDeltaUs = + rtcp::TransportFeedback::kDeltaScaleFactor * ((1 << 8) - 1); + const int64_t kLargePositiveDeltaUs = + rtcp::TransportFeedback::kDeltaScaleFactor * + std::numeric_limits::max(); + const int64_t kLargeNegativeDeltaUs = + rtcp::TransportFeedback::kDeltaScaleFactor * + std::numeric_limits::min(); + + PacketFeedback packet_feedback(100, 200, 0, 1500, true, 0, 0, + PacedPacketInfo()); + sent_packets.push_back(packet_feedback); + + packet_feedback.send_time_ms += kSmallDeltaUs / 1000; + packet_feedback.arrival_time_ms += kSmallDeltaUs / 1000; + ++packet_feedback.sequence_number; + sent_packets.push_back(packet_feedback); + + packet_feedback.send_time_ms += kLargePositiveDeltaUs / 1000; + packet_feedback.arrival_time_ms += kLargePositiveDeltaUs / 1000; + ++packet_feedback.sequence_number; + sent_packets.push_back(packet_feedback); + + packet_feedback.send_time_ms += kLargeNegativeDeltaUs / 1000; + packet_feedback.arrival_time_ms += kLargeNegativeDeltaUs / 1000; + ++packet_feedback.sequence_number; + sent_packets.push_back(packet_feedback); + + // Too large, delta - will need two feedback messages. + packet_feedback.send_time_ms += (kLargePositiveDeltaUs + 1000) / 1000; + packet_feedback.arrival_time_ms += (kLargePositiveDeltaUs + 1000) / 1000; + ++packet_feedback.sequence_number; + + // Packets will be added to send history. + for (const PacketFeedback& packet : sent_packets) + OnSentPacket(packet); + OnSentPacket(packet_feedback); + + // Create expected feedback and send into adapter. + std::unique_ptr feedback( + new rtcp::TransportFeedback()); + feedback->SetBase(sent_packets[0].sequence_number, + sent_packets[0].arrival_time_ms * 1000); + + for (const PacketFeedback& packet : sent_packets) { + EXPECT_TRUE(feedback->AddReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + EXPECT_FALSE(feedback->AddReceivedPacket( + packet_feedback.sequence_number, packet_feedback.arrival_time_ms * 1000)); + + rtc::Buffer raw_packet = feedback->Build(); + feedback = + rtcp::TransportFeedback::ParseFrom(raw_packet.data(), raw_packet.size()); + + std::vector received_feedback; + + EXPECT_TRUE(feedback.get() != nullptr); + adapter_->OnTransportFeedback(*feedback.get()); + ComparePacketFeedbackVectors(sent_packets, + adapter_->GetTransportFeedbackVector()); + + // Create a new feedback message and add the trailing item. + feedback.reset(new rtcp::TransportFeedback()); + feedback->SetBase(packet_feedback.sequence_number, + packet_feedback.arrival_time_ms * 1000); + EXPECT_TRUE(feedback->AddReceivedPacket( + packet_feedback.sequence_number, packet_feedback.arrival_time_ms * 1000)); + raw_packet = feedback->Build(); + feedback = + rtcp::TransportFeedback::ParseFrom(raw_packet.data(), raw_packet.size()); + + EXPECT_TRUE(feedback.get() != nullptr); + adapter_->OnTransportFeedback(*feedback.get()); + { + std::vector expected_packets; + expected_packets.push_back(packet_feedback); + ComparePacketFeedbackVectors(expected_packets, + adapter_->GetTransportFeedbackVector()); + } +} +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.cc new file mode 100644 index 0000000000..eeb5edf1cb --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/trendline_estimator.h" + +#include + +#include "api/optional.h" +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { +rtc::Optional LinearFitSlope( + const std::deque>& points) { + RTC_DCHECK(points.size() >= 2); + // Compute the "center of mass". + double sum_x = 0; + double sum_y = 0; + for (const auto& point : points) { + sum_x += point.first; + sum_y += point.second; + } + double x_avg = sum_x / points.size(); + double y_avg = sum_y / points.size(); + // Compute the slope k = \sum (x_i-x_avg)(y_i-y_avg) / \sum (x_i-x_avg)^2 + double numerator = 0; + double denominator = 0; + for (const auto& point : points) { + numerator += (point.first - x_avg) * (point.second - y_avg); + denominator += (point.first - x_avg) * (point.first - x_avg); + } + if (denominator == 0) + return rtc::nullopt; + return numerator / denominator; +} +} // namespace + +enum { kDeltaCounterMax = 1000 }; + +TrendlineEstimator::TrendlineEstimator(size_t window_size, + double smoothing_coef, + double threshold_gain) + : window_size_(window_size), + smoothing_coef_(smoothing_coef), + threshold_gain_(threshold_gain), + num_of_deltas_(0), + first_arrival_time_ms(-1), + accumulated_delay_(0), + smoothed_delay_(0), + delay_hist_(), + trendline_(0) {} + +TrendlineEstimator::~TrendlineEstimator() {} + +void TrendlineEstimator::Update(double recv_delta_ms, + double send_delta_ms, + int64_t arrival_time_ms) { + const double delta_ms = recv_delta_ms - send_delta_ms; + ++num_of_deltas_; + if (num_of_deltas_ > kDeltaCounterMax) + num_of_deltas_ = kDeltaCounterMax; + if (first_arrival_time_ms == -1) + first_arrival_time_ms = arrival_time_ms; + + // Exponential backoff filter. + accumulated_delay_ += delta_ms; + BWE_TEST_LOGGING_PLOT(1, "accumulated_delay_ms", arrival_time_ms, + accumulated_delay_); + smoothed_delay_ = smoothing_coef_ * smoothed_delay_ + + (1 - smoothing_coef_) * accumulated_delay_; + BWE_TEST_LOGGING_PLOT(1, "smoothed_delay_ms", arrival_time_ms, + smoothed_delay_); + + // Simple linear regression. + delay_hist_.push_back(std::make_pair( + static_cast(arrival_time_ms - first_arrival_time_ms), + smoothed_delay_)); + if (delay_hist_.size() > window_size_) + delay_hist_.pop_front(); + if (delay_hist_.size() == window_size_) { + // Only update trendline_ if it is possible to fit a line to the data. + trendline_ = LinearFitSlope(delay_hist_).value_or(trendline_); + } + + BWE_TEST_LOGGING_PLOT(1, "trendline_slope", arrival_time_ms, trendline_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.h b/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.h new file mode 100644 index 0000000000..7ee27cdd5d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_CONGESTION_CONTROLLER_TRENDLINE_ESTIMATOR_H_ +#define MODULES_CONGESTION_CONTROLLER_TRENDLINE_ESTIMATOR_H_ + +#include +#include + +#include +#include + +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +class TrendlineEstimator { + public: + // |window_size| is the number of points required to compute a trend line. + // |smoothing_coef| controls how much we smooth out the delay before fitting + // the trend line. |threshold_gain| is used to scale the trendline slope for + // comparison to the old threshold. Once the old estimator has been removed + // (or the thresholds been merged into the estimators), we can just set the + // threshold instead of setting a gain. + TrendlineEstimator(size_t window_size, + double smoothing_coef, + double threshold_gain); + ~TrendlineEstimator(); + + // Update the estimator with a new sample. The deltas should represent deltas + // between timestamp groups as defined by the InterArrival class. + void Update(double recv_delta_ms, + double send_delta_ms, + int64_t arrival_time_ms); + + // Returns the estimated trend k multiplied by some gain. + // 0 < k < 1 -> the delay increases, queues are filling up + // k == 0 -> the delay does not change + // k < 0 -> the delay decreases, queues are being emptied + double trendline_slope() const { return trendline_ * threshold_gain_; } + + // Returns the number of deltas which the current estimator state is based on. + unsigned int num_of_deltas() const { return num_of_deltas_; } + + private: + // Parameters. + const size_t window_size_; + const double smoothing_coef_; + const double threshold_gain_; + // Used by the existing threshold. + unsigned int num_of_deltas_; + // Keep the arrival times small by using the change from the first packet. + int64_t first_arrival_time_ms; + // Exponential backoff filtering. + double accumulated_delay_; + double smoothed_delay_; + // Linear least squares regression. + std::deque> delay_hist_; + double trendline_; + + RTC_DISALLOW_COPY_AND_ASSIGN(TrendlineEstimator); +}; +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_TRENDLINE_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator_unittest.cc b/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator_unittest.cc new file mode 100644 index 0000000000..e44f5032b0 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/congestion_controller/trendline_estimator_unittest.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/congestion_controller/trendline_estimator.h" +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr size_t kWindowSize = 20; +constexpr double kSmoothing = 0.0; +constexpr double kGain = 1; +constexpr int64_t kAvgTimeBetweenPackets = 10; +constexpr size_t kPacketCount = 2 * kWindowSize + 1; + +void TestEstimator(double slope, double jitter_stddev, double tolerance) { + TrendlineEstimator estimator(kWindowSize, kSmoothing, kGain); + Random random(0x1234567); + int64_t send_times[kPacketCount]; + int64_t recv_times[kPacketCount]; + int64_t send_start_time = random.Rand(1000000); + int64_t recv_start_time = random.Rand(1000000); + for (size_t i = 0; i < kPacketCount; ++i) { + send_times[i] = send_start_time + i * kAvgTimeBetweenPackets; + double latency = i * kAvgTimeBetweenPackets / (1 - slope); + double jitter = random.Gaussian(0, jitter_stddev); + recv_times[i] = recv_start_time + latency + jitter; + } + for (size_t i = 1; i < kPacketCount; ++i) { + double recv_delta = recv_times[i] - recv_times[i - 1]; + double send_delta = send_times[i] - send_times[i - 1]; + estimator.Update(recv_delta, send_delta, recv_times[i]); + if (i < kWindowSize) + EXPECT_NEAR(estimator.trendline_slope(), 0, 0.001); + else + EXPECT_NEAR(estimator.trendline_slope(), slope, tolerance); + } +} +} // namespace + +TEST(TrendlineEstimator, PerfectLineSlopeOneHalf) { + TestEstimator(0.5, 0, 0.001); +} + +TEST(TrendlineEstimator, PerfectLineSlopeMinusOne) { + TestEstimator(-1, 0, 0.001); +} + +TEST(TrendlineEstimator, PerfectLineSlopeZero) { + TestEstimator(0, 0, 0.001); +} + +TEST(TrendlineEstimator, JitteryLineSlopeOneHalf) { + TestEstimator(0.5, kAvgTimeBetweenPackets / 3.0, 0.01); +} + +TEST(TrendlineEstimator, JitteryLineSlopeMinusOne) { + TestEstimator(-1, kAvgTimeBetweenPackets / 3.0, 0.075); +} + +TEST(TrendlineEstimator, JitteryLineSlopeZero) { + TestEstimator(0, kAvgTimeBetweenPackets / 3.0, 0.02); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/BUILD.gn b/third_party/libwebrtc/webrtc/modules/desktop_capture/BUILD.gn new file mode 100644 index 0000000000..d1033cbd71 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/BUILD.gn @@ -0,0 +1,428 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("//build/config/linux/pkg_config.gni") +import("//build/config/ui.gni") +import("../../webrtc.gni") + +if (build_with_mozilla) { + defines = [ "MULTI_MONITOR_SCREENSHARE" ] +} + +use_desktop_capture_differ_sse2 = current_cpu == "x86" || current_cpu == "x64" + +rtc_static_library("primitives") { + sources = [ + "desktop_capture_types.h", + "desktop_frame.cc", + "desktop_frame.h", + "desktop_geometry.cc", + "desktop_geometry.h", + "desktop_region.cc", + "desktop_region.h", + "shared_desktop_frame.cc", + "shared_desktop_frame.h", + "shared_memory.cc", + "shared_memory.h", + ] + + deps = [ + "../..:webrtc_common", + ] + + if (!build_with_mozilla) { + deps += [ "../../rtc_base:rtc_base" ] # TODO(kjellander): Cleanup in bugs.webrtc.org/3806. + } +} + +if (rtc_include_tests) { + rtc_source_set("desktop_capture_modules_tests") { + testonly = true + + sources = [] + deps = [] + if (rtc_desktop_capture_supported) { + deps += [ + ":desktop_capture_mock", + ":primitives", + ":screen_drawer", + "../../rtc_base:rtc_base", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../../test:test_support", + "../../test:video_test_support", + ] + sources += [ + "screen_capturer_integration_test.cc", + "screen_drawer_unittest.cc", + "window_finder_unittest.cc", + ] + } + } + + rtc_source_set("desktop_capture_unittests") { + testonly = true + + sources = [ + "blank_detector_desktop_capturer_wrapper_unittest.cc", + "cropped_desktop_frame_unittest.cc", + "desktop_and_cursor_composer_unittest.cc", + "desktop_capturer_differ_wrapper_unittest.cc", + "desktop_frame_rotation_unittest.cc", + "desktop_geometry_unittest.cc", + "desktop_region_unittest.cc", + "differ_block_unittest.cc", + "fallback_desktop_capturer_wrapper_unittest.cc", + "mouse_cursor_monitor_unittest.cc", + "rgba_color_unittest.cc", + "test_utils.cc", + "test_utils.h", + "test_utils_unittest.cc", + "win/cursor_unittest.cc", + "win/cursor_unittest_resources.h", + "win/cursor_unittest_resources.rc", + "win/screen_capture_utils_unittest.cc", + "win/screen_capturer_win_directx_unittest.cc", + ] + deps = [ + ":desktop_capture", + ":desktop_capture_mock", + ":primitives", + "../..:webrtc_common", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + "../../test:test_support", + "//testing/gmock", + ] + if (rtc_desktop_capture_supported) { + sources += [ + "screen_capturer_helper_unittest.cc", + "screen_capturer_mac_unittest.cc", + "screen_capturer_unittest.cc", + "window_capturer_unittest.cc", + ] + deps += [ ":desktop_capture_mock" ] + } + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + + source_set("screen_drawer") { + testonly = true + + public_deps = [ + ":desktop_capture", + ] + + sources = [ + "screen_drawer.cc", + "screen_drawer.h", + "screen_drawer_linux.cc", + "screen_drawer_lock_posix.cc", + "screen_drawer_lock_posix.h", + "screen_drawer_mac.cc", + "screen_drawer_win.cc", + ] + + deps = [ + ":primitives", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] + } + + source_set("desktop_capture_mock") { + testonly = true + + public_deps = [ + ":desktop_capture", + "//testing/gmock", + ] + + sources = [ + "mock_desktop_capturer_callback.cc", + "mock_desktop_capturer_callback.h", + ] + + deps = [ + ":primitives", + "../../rtc_base:rtc_base_approved", + "../../test:test_support", + ] + } +} + +rtc_source_set("desktop_capture") { + public_deps = [ + ":desktop_capture_generic", + ] + if (is_mac) { + public_deps += [ ":desktop_capture_objc" ] + } +} + +if (is_mac) { + rtc_source_set("desktop_capture_objc") { + visibility = [ ":desktop_capture" ] + sources = [ + "mac/desktop_configuration.mm", + "mouse_cursor_monitor_mac.mm", + "screen_capturer_mac.mm", + "window_capturer_mac.mm", + "window_finder_mac.h", + "window_finder_mac.mm", + ] + deps = [ + ":desktop_capture_generic", + ":primitives", + "../../rtc_base:rtc_base", + "../../rtc_base:rtc_base_approved", + ] + libs = [ + "AppKit.framework", + "IOKit.framework", + "OpenGL.framework", + ] + } +} + +rtc_static_library("desktop_capture_generic") { + visibility = [ + ":desktop_capture", + ":desktop_capture_objc", + ] + sources = [ + "blank_detector_desktop_capturer_wrapper.cc", + "blank_detector_desktop_capturer_wrapper.h", + "capture_result_desktop_capturer_wrapper.cc", + "capture_result_desktop_capturer_wrapper.h", + "cropped_desktop_frame.cc", + "cropped_desktop_frame.h", + "cropping_window_capturer.cc", + "cropping_window_capturer.h", + "cropping_window_capturer_win.cc", + "desktop_and_cursor_composer.cc", + "desktop_and_cursor_composer.h", + "desktop_capture_options.cc", + "desktop_capture_options.h", + "desktop_capturer.cc", + "desktop_capturer.h", + "desktop_capturer_differ_wrapper.cc", + "desktop_capturer_differ_wrapper.h", + "desktop_capturer_wrapper.cc", + "desktop_capturer_wrapper.h", + "desktop_frame_generator.cc", + "desktop_frame_generator.h", + "desktop_frame_rotation.cc", + "desktop_frame_rotation.h", + "desktop_frame_win.cc", + "desktop_frame_win.h", + "differ_block.cc", + "differ_block.h", + "fallback_desktop_capturer_wrapper.cc", + "fallback_desktop_capturer_wrapper.h", + "mac/desktop_configuration.h", + "mac/desktop_configuration_monitor.cc", + "mac/desktop_configuration_monitor.h", + "mac/full_screen_chrome_window_detector.cc", + "mac/full_screen_chrome_window_detector.h", + "mac/scoped_pixel_buffer_object.cc", + "mac/scoped_pixel_buffer_object.h", + "mac/window_list_utils.cc", + "mac/window_list_utils.h", + "mouse_cursor.cc", + "mouse_cursor.h", + "mouse_cursor_monitor.h", + "mouse_cursor_monitor_win.cc", + "resolution_tracker.cc", + "resolution_tracker.h", + "rgba_color.cc", + "rgba_color.h", + "screen_capture_frame_queue.h", + "screen_capturer_helper.cc", + "screen_capturer_helper.h", + "screen_capturer_win.cc", + "win/cursor.cc", + "win/cursor.h", + "win/d3d_device.cc", + "win/d3d_device.h", + "win/desktop.cc", + "win/desktop.h", + "win/display_configuration_monitor.cc", + "win/display_configuration_monitor.h", + "win/dxgi_adapter_duplicator.cc", + "win/dxgi_adapter_duplicator.h", + "win/dxgi_context.cc", + "win/dxgi_context.h", + "win/dxgi_duplicator_controller.cc", + "win/dxgi_duplicator_controller.h", + "win/dxgi_frame.cc", + "win/dxgi_frame.h", + "win/dxgi_output_duplicator.cc", + "win/dxgi_output_duplicator.h", + "win/dxgi_texture.cc", + "win/dxgi_texture.h", + "win/dxgi_texture_mapping.cc", + "win/dxgi_texture_mapping.h", + "win/dxgi_texture_staging.cc", + "win/dxgi_texture_staging.h", + "win/scoped_gdi_object.h", + "win/scoped_thread_desktop.cc", + "win/scoped_thread_desktop.h", + "win/screen_capture_utils.cc", + "win/screen_capture_utils.h", + "win/screen_capturer_win_directx.cc", + "win/screen_capturer_win_directx.h", + "win/screen_capturer_win_gdi.cc", + "win/screen_capturer_win_gdi.h", + "win/screen_capturer_win_magnifier.cc", + "win/screen_capturer_win_magnifier.h", + "win/window_capture_utils.cc", + "win/window_capture_utils.h", + "window_capturer_win.cc", + "window_finder.cc", + "window_finder.h", + "window_finder_win.cc", + "window_finder_win.h", + ] + + if (build_with_mozilla) { + sources += [ + "desktop_device_info.cc", + "desktop_device_info.h", + "win/desktop_device_info_win.cc", + "win/win_shared.cc", + ] + + if (is_mac) { + sources += [ + "mac/desktop_device_info_mac.mm", + ] + } + + include_dirs = [ "/media/libyuv/libyuv/include" ] + } else { + sources += [ + "fake_desktop_capturer.cc", + "fake_desktop_capturer.h", + ] + } + + if (use_x11 || rtc_use_pipewire) { + sources += [ + "mouse_cursor_monitor_linux.cc", + "screen_capturer_linux.cc", + "window_capturer_linux.cc", + ] + } + + if (use_x11) { + sources += [ + "linux/mouse_cursor_monitor_x11.cc", + "linux/mouse_cursor_monitor_x11.h", + "linux/screen_capturer_x11.cc", + "linux/screen_capturer_x11.h", + "linux/shared_x_display.cc", + "linux/shared_x_display.h", + "linux/window_capturer_x11.cc", + "linux/window_capturer_x11.h", + "linux/window_finder_x11.cc", + "linux/window_finder_x11.h", + "linux/window_list_utils.cc", + "linux/window_list_utils.h", + "linux/x_atom_cache.cc", + "linux/x_atom_cache.h", + "linux/x_error_trap.cc", + "linux/x_error_trap.h", + "linux/x_server_pixel_buffer.cc", + "linux/x_server_pixel_buffer.h", + ] + configs += [ "//build/config/linux:x11" ] + + if (build_with_mozilla) { + sources += [ + "linux/desktop_device_info_x11.cc", + "linux/desktop_device_info_x11.h", + "linux/shared_x_util.cc", + "linux/shared_x_util.h", + ] + } + } + + if (rtc_use_pipewire) { + sources += [ + "linux/base_capturer_pipewire.cc", + "linux/base_capturer_pipewire.h", + "linux/screen_capturer_pipewire.cc", + "linux/screen_capturer_pipewire.h", + "linux/window_capturer_pipewire.cc", + "linux/window_capturer_pipewire.h", + ] + + if (!build_with_mozilla) { + configs += [ + ":gio", + ":pipewire", + ] + } else { + defines += [ "WEBRTC_USE_PIPEWIRE" ] + include_dirs += [ "/third_party/pipewire" ] + } + } + + if (!is_win && !is_mac && !use_x11 && !rtc_use_pipewire) { + sources += [ + "mouse_cursor_monitor_null.cc", + "screen_capturer_null.cc", + "window_capturer_null.cc", + ] + } + + if (is_win) { + libs = [ + "d3d11.lib", + "dxgi.lib", + ] + } + + deps = [ + ":primitives", + "../..:webrtc_common", + "../../api:refcountedbase", + "../../rtc_base:rtc_base", # TODO(kjellander): Cleanup in bugs.webrtc.org/3806. + "../../system_wrappers", + ] + + if (build_with_mozilla) { + deps += [ "../../rtc_base:rtc_base_approved" ] + } else { + deps += [ "//third_party/libyuv" ] + } + + if (use_desktop_capture_differ_sse2) { + deps += [ ":desktop_capture_differ_sse2" ] + } +} + +if (use_desktop_capture_differ_sse2) { + # Have to be compiled as a separate target because it needs to be compiled + # with SSE2 enabled. + rtc_static_library("desktop_capture_differ_sse2") { + visibility = [ ":*" ] + sources = [ + "differ_vector_sse2.cc", + "differ_vector_sse2.h", + ] + + if (is_posix) { + cflags = [ "-msse2" ] + } + } +} diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/DEPS b/third_party/libwebrtc/webrtc/modules/desktop_capture/DEPS new file mode 100644 index 0000000000..3314b8cdfd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/DEPS @@ -0,0 +1,4 @@ +include_rules = [ + "+system_wrappers", + "+third_party/libyuv", +] diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/OWNERS b/third_party/libwebrtc/webrtc/modules/desktop_capture/OWNERS new file mode 100644 index 0000000000..e3fd87a389 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/OWNERS @@ -0,0 +1,10 @@ +# Please send the changes to zijiehe@chromium.org first. +jamiewalch@chromium.org +sergeyu@chromium.org +wez@chromium.org +zijiehe@chromium.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc new file mode 100644 index 0000000000..722f5dc53a --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" + +#include +#include + +#include "modules/desktop_capture/desktop_geometry.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +BlankDetectorDesktopCapturerWrapper::BlankDetectorDesktopCapturerWrapper( + std::unique_ptr capturer, + RgbaColor blank_pixel) + : capturer_(std::move(capturer)), + blank_pixel_(blank_pixel) { + RTC_DCHECK(capturer_); +} + +BlankDetectorDesktopCapturerWrapper::~BlankDetectorDesktopCapturerWrapper() = + default; + +void BlankDetectorDesktopCapturerWrapper::Start( + DesktopCapturer::Callback* callback) { + capturer_->Start(this); + callback_ = callback; +} + +void BlankDetectorDesktopCapturerWrapper::SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) { + capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory)); +} + +void BlankDetectorDesktopCapturerWrapper::CaptureFrame() { + RTC_DCHECK(callback_); + capturer_->CaptureFrame(); +} + +void BlankDetectorDesktopCapturerWrapper::SetExcludedWindow(WindowId window) { + capturer_->SetExcludedWindow(window); +} + +bool BlankDetectorDesktopCapturerWrapper::GetSourceList(SourceList* sources) { + return capturer_->GetSourceList(sources); +} + +bool BlankDetectorDesktopCapturerWrapper::SelectSource(SourceId id) { + return capturer_->SelectSource(id); +} + +bool BlankDetectorDesktopCapturerWrapper::FocusOnSelectedSource() { + return capturer_->FocusOnSelectedSource(); +} + +void BlankDetectorDesktopCapturerWrapper::OnCaptureResult( + Result result, + std::unique_ptr frame) { + RTC_DCHECK(callback_); + if (result != Result::SUCCESS || non_blank_frame_received_) { + callback_->OnCaptureResult(result, std::move(frame)); + return; + } + + RTC_DCHECK(frame); + + // If nothing has been changed in current frame, we do not need to check it + // again. + if (!frame->updated_region().is_empty() || is_first_frame_) { + last_frame_is_blank_ = IsBlankFrame(*frame); + is_first_frame_ = false; + } + RTC_HISTOGRAM_BOOLEAN("WebRTC.DesktopCapture.BlankFrameDetected", + last_frame_is_blank_); + if (!last_frame_is_blank_) { + non_blank_frame_received_ = true; + callback_->OnCaptureResult(Result::SUCCESS, std::move(frame)); + return; + } + + callback_->OnCaptureResult(Result::ERROR_TEMPORARY, + std::unique_ptr()); +} + +bool BlankDetectorDesktopCapturerWrapper::IsBlankFrame( + const DesktopFrame& frame) const { + // We will check 7489 pixels for a frame with 1024 x 768 resolution. + for (int i = 0; i < frame.size().width() * frame.size().height(); i += 105) { + const int x = i % frame.size().width(); + const int y = i / frame.size().width(); + if (!IsBlankPixel(frame, x, y)) { + return false; + } + } + + // We are verifying the pixel in the center as well. + return IsBlankPixel(frame, frame.size().width() / 2, + frame.size().height() / 2); +} + +bool BlankDetectorDesktopCapturerWrapper::IsBlankPixel( + const DesktopFrame& frame, + int x, + int y) const { + uint8_t* pixel_data = frame.GetFrameDataAtPos(DesktopVector(x, y)); + return RgbaColor(pixel_data) == blank_pixel_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h new file mode 100644 index 0000000000..c76ceea3f7 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ +#define MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ + +#include + +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/rgba_color.h" + +namespace webrtc { + +// A DesktopCapturer wrapper detects the return value of its owned +// DesktopCapturer implementation. If sampled pixels returned by the +// DesktopCapturer implementation all equal to the blank pixel, this wrapper +// returns ERROR_TEMPORARY. If the DesktopCapturer implementation fails for too +// many times, this wrapper returns ERROR_PERMANENT. +class BlankDetectorDesktopCapturerWrapper final + : public DesktopCapturer, + public DesktopCapturer::Callback { + public: + // Creates BlankDetectorDesktopCapturerWrapper. BlankDesktopCapturerWrapper + // takes ownership of |capturer|. The |blank_pixel| is the unmodified color + // returned by the |capturer|. + BlankDetectorDesktopCapturerWrapper(std::unique_ptr capturer, + RgbaColor blank_pixel); + ~BlankDetectorDesktopCapturerWrapper() override; + + // DesktopCapturer interface. + void Start(DesktopCapturer::Callback* callback) override; + void SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) override; + void CaptureFrame() override; + void SetExcludedWindow(WindowId window) override; + bool GetSourceList(SourceList* sources) override; + bool SelectSource(SourceId id) override; + bool FocusOnSelectedSource() override; + + private: + // DesktopCapturer::Callback interface. + void OnCaptureResult(Result result, + std::unique_ptr frame) override; + + bool IsBlankFrame(const DesktopFrame& frame) const; + + // Detects whether pixel at (x, y) equals to |blank_pixel_|. + bool IsBlankPixel(const DesktopFrame& frame, int x, int y) const; + + const std::unique_ptr capturer_; + const RgbaColor blank_pixel_; + + // Whether a non-blank frame has been received. + bool non_blank_frame_received_ = false; + + // Whether the last frame is blank. + bool last_frame_is_blank_ = false; + + // Whether current frame is the first frame. + bool is_first_frame_ = true; + + DesktopCapturer::Callback* callback_ = nullptr; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc new file mode 100644 index 0000000000..f9cfee026f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" + +#include +#include + +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/desktop_frame_generator.h" +#include "modules/desktop_capture/fake_desktop_capturer.h" +#include "test/gtest.h" + +namespace webrtc { + +class BlankDetectorDesktopCapturerWrapperTest + : public testing::Test, + public DesktopCapturer::Callback { + public: + BlankDetectorDesktopCapturerWrapperTest(); + ~BlankDetectorDesktopCapturerWrapperTest() override; + + protected: + void PerfTest(DesktopCapturer* capturer); + + const int frame_width_ = 1024; + const int frame_height_ = 768; + std::unique_ptr wrapper_; + DesktopCapturer* capturer_ = nullptr; + BlackWhiteDesktopFramePainter painter_; + int num_frames_captured_ = 0; + DesktopCapturer::Result last_result_ = DesktopCapturer::Result::SUCCESS; + std::unique_ptr last_frame_; + + private: + // DesktopCapturer::Callback interface. + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr frame) override; + + PainterDesktopFrameGenerator frame_generator_; +}; + +BlankDetectorDesktopCapturerWrapperTest:: +BlankDetectorDesktopCapturerWrapperTest() { + frame_generator_.size()->set(frame_width_, frame_height_); + frame_generator_.set_desktop_frame_painter(&painter_); + std::unique_ptr capturer(new FakeDesktopCapturer()); + FakeDesktopCapturer* fake_capturer = + static_cast(capturer.get()); + fake_capturer->set_frame_generator(&frame_generator_); + capturer_ = fake_capturer; + wrapper_.reset(new BlankDetectorDesktopCapturerWrapper( + std::move(capturer), RgbaColor(0, 0, 0, 0))); + wrapper_->Start(this); +} + +BlankDetectorDesktopCapturerWrapperTest:: +~BlankDetectorDesktopCapturerWrapperTest() = default; + +void BlankDetectorDesktopCapturerWrapperTest::OnCaptureResult( + DesktopCapturer::Result result, + std::unique_ptr frame) { + last_result_ = result; + last_frame_ = std::move(frame); + num_frames_captured_++; +} + +void BlankDetectorDesktopCapturerWrapperTest::PerfTest( + DesktopCapturer* capturer) { + for (int i = 0; i < 10000; i++) { + capturer->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, i + 1); + } +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldDetectBlankFrame) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY); + ASSERT_FALSE(last_frame_); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldPassBlankDetection) { + painter_.updated_region()->AddRect(DesktopRect::MakeXYWH(0, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, frame_height_ - 100, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 2); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(0, frame_height_ - 100, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 3); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 4); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect(DesktopRect::MakeXYWH( + (frame_width_ >> 1) - 50, (frame_height_ >> 1) - 50, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 5); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, + ShouldNotCheckAfterANonBlankFrameReceived) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY); + ASSERT_FALSE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 2); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + for (int i = 0; i < 100; i++) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, i + 3); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + } +} + +// There is no perceptible impact by using BlankDetectorDesktopCapturerWrapper. +// i.e. less than 0.2ms per frame. +// [ OK ] DISABLED_Performance (10210 ms) +// [ OK ] DISABLED_PerformanceComparison (8791 ms) +TEST_F(BlankDetectorDesktopCapturerWrapperTest, DISABLED_Performance) { + PerfTest(wrapper_.get()); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, + DISABLED_PerformanceComparison) { + capturer_->Start(this); + PerfTest(capturer_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.cc new file mode 100644 index 0000000000..b15ce2b0fd --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.cc @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/capture_result_desktop_capturer_wrapper.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +CaptureResultDesktopCapturerWrapper::CaptureResultDesktopCapturerWrapper( + std::unique_ptr base_capturer, + ResultObserver* observer) + : DesktopCapturerWrapper(std::move(base_capturer)), + observer_(observer) { + RTC_DCHECK(observer_); +} + +CaptureResultDesktopCapturerWrapper:: +~CaptureResultDesktopCapturerWrapper() = default; + +void CaptureResultDesktopCapturerWrapper::Start(Callback* callback) { + if ((callback_ == nullptr) != (callback == nullptr)) { + if (callback) { + base_capturer_->Start(this); + } else { + base_capturer_->Start(nullptr); + } + } + callback_ = callback; +} + +void CaptureResultDesktopCapturerWrapper::OnCaptureResult( + Result result, + std::unique_ptr frame) { + observer_->Observe(&result, &frame); + callback_->OnCaptureResult(result, std::move(frame)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.h new file mode 100644 index 0000000000..3f97bc5fc4 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_CAPTURE_RESULT_DESKTOP_CAPTURER_WRAPPER_H_ +#define MODULES_DESKTOP_CAPTURE_CAPTURE_RESULT_DESKTOP_CAPTURER_WRAPPER_H_ + +#include + +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/desktop_capturer_wrapper.h" + +namespace webrtc { + +// A DesktopCapturerWrapper implementation to capture the result of +// |base_capturer|. Derived classes are expected to provide a ResultObserver +// implementation to observe the DesktopFrame returned by |base_capturer_|. +class CaptureResultDesktopCapturerWrapper + : public DesktopCapturerWrapper, + public DesktopCapturer::Callback { + public: + using Callback = DesktopCapturer::Callback; + + // Provides a way to let derived classes or clients to modify the result + // returned by |base_capturer_|. + class ResultObserver { + public: + ResultObserver(); + virtual ~ResultObserver(); + + virtual void Observe(Result* result, + std::unique_ptr* frame) = 0; + }; + + // |observer| must outlive this instance and can be |this|. |observer| is + // guaranteed to be executed only after the constructor and before the + // destructor. + CaptureResultDesktopCapturerWrapper( + std::unique_ptr base_capturer, + ResultObserver* observer); + + ~CaptureResultDesktopCapturerWrapper() override; + + // DesktopCapturer implementations. + void Start(Callback* callback) final; + + private: + // DesktopCapturer::Callback implementation. + void OnCaptureResult(Result result, + std::unique_ptr frame) final; + + ResultObserver* const observer_; + Callback* callback_ = nullptr; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_CAPTURE_RESULT_DESKTOP_CAPTURER_WRAPPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.cc new file mode 100644 index 0000000000..c7ada7b61f --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/desktop_capture/cropped_desktop_frame.h" + +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// A DesktopFrame that is a sub-rect of another DesktopFrame. +class CroppedDesktopFrame : public DesktopFrame { + public: + CroppedDesktopFrame(std::unique_ptr frame, + const DesktopRect& rect); + + private: + const std::unique_ptr frame_; + + RTC_DISALLOW_COPY_AND_ASSIGN(CroppedDesktopFrame); +}; + +std::unique_ptr CreateCroppedDesktopFrame( + std::unique_ptr frame, + const DesktopRect& rect) { + RTC_DCHECK(frame); + + if (!DesktopRect::MakeSize(frame->size()).ContainsRect(rect)) { + return nullptr; + } + + if (frame->size().equals(rect.size())) { + return frame; + } + + return std::unique_ptr( + new CroppedDesktopFrame(std::move(frame), rect)); +} + +CroppedDesktopFrame::CroppedDesktopFrame(std::unique_ptr frame, + const DesktopRect& rect) + : DesktopFrame(rect.size(), + frame->stride(), + frame->GetFrameDataAtPos(rect.top_left()), + frame->shared_memory()), + frame_(std::move(frame)) { + MoveFrameInfoFrom(frame_.get()); + set_top_left(frame_->top_left().add(rect.top_left())); + mutable_updated_region()->IntersectWith(rect); + mutable_updated_region()->Translate(-rect.left(), -rect.top()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.h new file mode 100644 index 0000000000..f6cac6c7a2 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_CROPPED_DESKTOP_FRAME_H_ +#define MODULES_DESKTOP_CAPTURE_CROPPED_DESKTOP_FRAME_H_ + +#include "modules/desktop_capture/desktop_frame.h" + +namespace webrtc { + +// Creates a DesktopFrame to contain only the area of |rect| in the original +// |frame|. +// |frame| should not be nullptr. |rect| is in |frame| coordinate, i.e. +// |frame|->top_left() does not impact the area of |rect|. +// Returns nullptr frame if |rect| is not contained by the bounds of |frame|. +std::unique_ptr CreateCroppedDesktopFrame( + std::unique_ptr frame, + const DesktopRect& rect); + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_CROPPED_DESKTOP_FRAME_H_ + diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame_unittest.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame_unittest.cc new file mode 100644 index 0000000000..d15f207dea --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame_unittest.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/desktop_capture/cropped_desktop_frame.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "rtc_base/ptr_util.h" +#include "test/gtest.h" + +namespace webrtc { + +std::unique_ptr CreateTestFrame() { + return rtc::MakeUnique(DesktopSize(10, 20)); +} + +TEST(CroppedDesktopFrameTest, DoNotCreateWrapperIfSizeIsNotChanged) { + std::unique_ptr original = CreateTestFrame(); + // owned by |original| and CroppedDesktopFrame. + DesktopFrame* raw_original = original.get(); + std::unique_ptr cropped = CreateCroppedDesktopFrame( + std::move(original), DesktopRect::MakeWH(10, 20)); + ASSERT_EQ(cropped.get(), raw_original); +} + +TEST(CroppedDesktopFrameTest, ReturnNullptrIfSizeIsNotSufficient) { + ASSERT_EQ(nullptr, CreateCroppedDesktopFrame( + CreateTestFrame(), DesktopRect::MakeWH(11, 10))); +} + +TEST(CroppedDesktopFrameTest, ReturnNullIfCropRegionIsOutOfBounds) { + std::unique_ptr frame = CreateTestFrame(); + frame->set_top_left(DesktopVector(100, 200)); + ASSERT_EQ(nullptr, CreateCroppedDesktopFrame(std::move(frame), + DesktopRect::MakeLTRB(101, 203, 109, 218))); +} + +TEST(CroppedDesktopFrameTest, CropASubArea) { + std::unique_ptr cropped = CreateCroppedDesktopFrame( + CreateTestFrame(), DesktopRect::MakeLTRB(1, 2, 9, 19)); + ASSERT_EQ(cropped->size().width(), 8); + ASSERT_EQ(cropped->size().height(), 17); + ASSERT_EQ(cropped->top_left().x(), 1); + ASSERT_EQ(cropped->top_left().y(), 2); +} + +TEST(CroppedDesktopFrameTest, SetTopLeft) { + std::unique_ptr frame = CreateTestFrame(); + frame->set_top_left(DesktopVector(100, 200)); + frame = CreateCroppedDesktopFrame(std::move(frame), + DesktopRect::MakeLTRB(1, 3, 9, 18)); + ASSERT_EQ(frame->size().width(), 8); + ASSERT_EQ(frame->size().height(), 15); + ASSERT_EQ(frame->top_left().x(), 101); + ASSERT_EQ(frame->top_left().y(), 203); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.cc new file mode 100644 index 0000000000..35008caa0e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/cropping_window_capturer.h" + +#include "modules/desktop_capture/cropped_desktop_frame.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +CroppingWindowCapturer::CroppingWindowCapturer( + const DesktopCaptureOptions& options) + : options_(options), + callback_(NULL), + window_capturer_(DesktopCapturer::CreateRawWindowCapturer(options)), + selected_window_(kNullWindowId), + excluded_window_(kNullWindowId) {} + +CroppingWindowCapturer::~CroppingWindowCapturer() {} + +void CroppingWindowCapturer::Start(DesktopCapturer::Callback* callback) { + callback_ = callback; + window_capturer_->Start(callback); +} + +void CroppingWindowCapturer::SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) { + window_capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory)); +} + +void CroppingWindowCapturer::CaptureFrame() { + if (ShouldUseScreenCapturer()) { + if (!screen_capturer_.get()) { + screen_capturer_ = DesktopCapturer::CreateRawScreenCapturer(options_); + if (excluded_window_) { + screen_capturer_->SetExcludedWindow(excluded_window_); + } + screen_capturer_->Start(this); + } + screen_capturer_->CaptureFrame(); + } else { + window_capturer_->CaptureFrame(); + } +} + +void CroppingWindowCapturer::SetExcludedWindow(WindowId window) { + excluded_window_ = window; + if (screen_capturer_.get()) { + screen_capturer_->SetExcludedWindow(window); + } +} + +bool CroppingWindowCapturer::GetSourceList(SourceList* sources) { + return window_capturer_->GetSourceList(sources); +} + +bool CroppingWindowCapturer::SelectSource(SourceId id) { + if (window_capturer_->SelectSource(id)) { + selected_window_ = id; + return true; + } + return false; +} + +bool CroppingWindowCapturer::FocusOnSelectedSource() { + return window_capturer_->FocusOnSelectedSource(); +} + +void CroppingWindowCapturer::OnCaptureResult( + DesktopCapturer::Result result, + std::unique_ptr screen_frame) { + if (!ShouldUseScreenCapturer()) { + RTC_LOG(LS_INFO) << "Window no longer on top when ScreenCapturer finishes"; + window_capturer_->CaptureFrame(); + return; + } + + if (result != Result::SUCCESS) { + RTC_LOG(LS_WARNING) << "ScreenCapturer failed to capture a frame"; + callback_->OnCaptureResult(result, nullptr); + return; + } + + DesktopRect window_rect = GetWindowRectInVirtualScreen(); + if (window_rect.is_empty()) { + RTC_LOG(LS_WARNING) << "Window rect is empty"; + callback_->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr); + return; + } + + callback_->OnCaptureResult( + Result::SUCCESS, + CreateCroppedDesktopFrame(std::move(screen_frame), window_rect)); +} + +bool CroppingWindowCapturer::IsOccluded(const DesktopVector& pos) { + // Returns true if either capturer returns true. + if (window_capturer_->IsOccluded(pos)) { + return true; + } + if (screen_capturer_ != nullptr && screen_capturer_->IsOccluded(pos)) { + return true; + } + return false; +} + +#if !defined(WEBRTC_WIN) +// CroppingWindowCapturer is implemented only for windows. On other platforms +// the regular window capturer is used. +// static +std::unique_ptr CroppingWindowCapturer::CreateCapturer( + const DesktopCaptureOptions& options) { + return DesktopCapturer::CreateWindowCapturer(options); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.h new file mode 100644 index 0000000000..9fbfddaa81 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_CROPPING_WINDOW_CAPTURER_H_ +#define MODULES_DESKTOP_CAPTURE_CROPPING_WINDOW_CAPTURER_H_ + +#include + +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/desktop_capture_options.h" + +namespace webrtc { + +// WindowCapturer implementation that uses a screen capturer to capture the +// whole screen and crops the video frame to the window area when the captured +// window is on top. +class CroppingWindowCapturer : public DesktopCapturer, + public DesktopCapturer::Callback { + public: + static std::unique_ptr CreateCapturer( + const DesktopCaptureOptions& options); + + ~CroppingWindowCapturer() override; + + // DesktopCapturer implementation. + void Start(DesktopCapturer::Callback* callback) override; + void SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) override; + void CaptureFrame() override; + void SetExcludedWindow(WindowId window) override; + bool GetSourceList(SourceList* sources) override; + bool SelectSource(SourceId id) override; + bool FocusOnSelectedSource() override; + bool IsOccluded(const DesktopVector& pos) override; + + // DesktopCapturer::Callback implementation, passed to |screen_capturer_| to + // intercept the capture result. + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr frame) override; + + protected: + explicit CroppingWindowCapturer(const DesktopCaptureOptions& options); + + // The platform implementation should override these methods. + + // Returns true if it is OK to capture the whole screen and crop to the + // selected window, i.e. the selected window is opaque, rectangular, and not + // occluded. + virtual bool ShouldUseScreenCapturer() = 0; + + // Returns the window area relative to the top left of the virtual screen + // within the bounds of the virtual screen. This function should return the + // DesktopRect in full desktop coordinates, i.e. the top-left monitor starts + // from (0, 0). + virtual DesktopRect GetWindowRectInVirtualScreen() = 0; + + WindowId selected_window() const { return selected_window_; } + WindowId excluded_window() const { return excluded_window_; } + + private: + DesktopCaptureOptions options_; + DesktopCapturer::Callback* callback_; + std::unique_ptr window_capturer_; + std::unique_ptr screen_capturer_; + SourceId selected_window_; + WindowId excluded_window_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_CROPPING_WINDOW_CAPTURER_H_ + diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer_win.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer_win.cc new file mode 100644 index 0000000000..6707236b3e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer_win.cc @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/cropping_window_capturer.h" + +#include "modules/desktop_capture/win/screen_capture_utils.h" +#include "modules/desktop_capture/win/window_capture_utils.h" +#include "rtc_base/logging.h" +#include "rtc_base/win32.h" + +namespace webrtc { + +namespace { + +// Used to pass input/output data during the EnumWindow call for verifying if +// the selected window is on top. +struct TopWindowVerifierContext { + TopWindowVerifierContext(HWND selected_window, + HWND excluded_window, + DesktopRect selected_window_rect) + : selected_window(selected_window), + excluded_window(excluded_window), + selected_window_rect(selected_window_rect), + is_top_window(false) { + RTC_DCHECK_NE(selected_window, excluded_window); + } + + const HWND selected_window; + const HWND excluded_window; + const DesktopRect selected_window_rect; + bool is_top_window; +}; + +// The function is called during EnumWindow for every window enumerated and is +// responsible for verifying if the selected window is on top. +BOOL CALLBACK TopWindowVerifier(HWND hwnd, LPARAM param) { + TopWindowVerifierContext* context = + reinterpret_cast(param); + + if (hwnd == context->selected_window) { + context->is_top_window = true; + return FALSE; + } + + // Ignore the excluded window. + if (hwnd == context->excluded_window) { + return TRUE; + } + + // Ignore hidden or minimized window. + if (IsIconic(hwnd) || !IsWindowVisible(hwnd)) { + return TRUE; + } + + // Ignore descendant windows since we want to capture them. + // This check does not work for tooltips and context menus. Drop down menus + // and popup windows are fine. + // + // GA_ROOT returns the root window instead of the owner. I.e. for a dialog + // window, GA_ROOT returns the dialog window itself. GA_ROOTOWNER returns the + // application main window which opens the dialog window. Since we are sharing + // the application main window, GA_ROOT should be used here. + if (GetAncestor(hwnd, GA_ROOT) == context->selected_window) { + return TRUE; + } + + // If |hwnd| has no title and belongs to the same process, assume it's a + // tooltip or context menu from the selected window and ignore it. + // TODO(zijiehe): This check cannot cover the case where tooltip or context + // menu of the child-window is covering the main window. See + // https://bugs.chromium.org/p/webrtc/issues/detail?id=8062 for details. + const size_t kTitleLength = 32; + WCHAR window_title[kTitleLength]; + GetWindowText(hwnd, window_title, kTitleLength); + if (wcsnlen_s(window_title, kTitleLength) == 0) { + DWORD enumerated_window_process_id; + DWORD selected_window_process_id; + GetWindowThreadProcessId(hwnd, &enumerated_window_process_id); + GetWindowThreadProcessId(context->selected_window, + &selected_window_process_id); + if (selected_window_process_id == enumerated_window_process_id) { + return TRUE; + } + } + + // Checks whether current window |hwnd| intersects with + // |context|->selected_window. + // |content_rect| is preferred because, + // 1. WindowCapturerWin is using GDI capturer, which cannot capture DX output. + // So ScreenCapturer should be used as much as possible to avoid + // uncapturable cases. Note: lots of new applications are using DX output + // (hardware acceleration) to improve the performance which cannot be + // captured by WindowCapturerWin. See bug http://crbug.com/741770. + // 2. WindowCapturerWin is still useful because we do not want to expose the + // content on other windows if the target window is covered by them. + // 3. Shadow and borders should not be considered as "content" on other + // windows because they do not expose any useful information. + // + // So we can bear the false-negative cases (target window is covered by the + // borders or shadow of other windows, but we have not detected it) in favor + // of using ScreenCapturer, rather than let the false-positive cases (target + // windows is only covered by borders or shadow of other windows, but we treat + // it as overlapping) impact the user experience. + DesktopRect content_rect; + if (!GetWindowContentRect(hwnd, &content_rect)) { + // Bail out if failed to get the window area. + context->is_top_window = false; + return FALSE; + } + + content_rect.IntersectWith(context->selected_window_rect); + + // If intersection is not empty, the selected window is not on top. + if (!content_rect.is_empty()) { + context->is_top_window = false; + return FALSE; + } + // Otherwise, keep enumerating. + return TRUE; +} + +class CroppingWindowCapturerWin : public CroppingWindowCapturer { + public: + CroppingWindowCapturerWin( + const DesktopCaptureOptions& options) + : CroppingWindowCapturer(options) {} + + private: + bool ShouldUseScreenCapturer() override; + DesktopRect GetWindowRectInVirtualScreen() override; + + // The region from GetWindowRgn in the desktop coordinate if the region is + // rectangular, or the rect from GetWindowRect if the region is not set. + DesktopRect window_region_rect_; + + AeroChecker aero_checker_; +}; + +bool CroppingWindowCapturerWin::ShouldUseScreenCapturer() { + if (!rtc::IsWindows8OrLater() && aero_checker_.IsAeroEnabled()) { + return false; + } + + const HWND selected = reinterpret_cast(selected_window()); + // Check if the window is hidden or minimized. + if (IsIconic(selected) || !IsWindowVisible(selected)) { + return false; + } + + // Check if the window is a translucent layered window. + const LONG window_ex_style = GetWindowLong(selected, GWL_EXSTYLE); + if (window_ex_style & WS_EX_LAYERED) { + COLORREF color_ref_key = 0; + BYTE alpha = 0; + DWORD flags = 0; + + // GetLayeredWindowAttributes fails if the window was setup with + // UpdateLayeredWindow. We have no way to know the opacity of the window in + // that case. This happens for Stiky Note (crbug/412726). + if (!GetLayeredWindowAttributes(selected, &color_ref_key, &alpha, &flags)) + return false; + + // UpdateLayeredWindow is the only way to set per-pixel alpha and will cause + // the previous GetLayeredWindowAttributes to fail. So we only need to check + // the window wide color key or alpha. + if ((flags & LWA_COLORKEY) || ((flags & LWA_ALPHA) && (alpha < 255))) { + return false; + } + } + + if (!GetWindowRect(selected, &window_region_rect_)) { + return false; + } + + DesktopRect content_rect; + if (!GetWindowContentRect(selected, &content_rect)) { + return false; + } + + DesktopRect region_rect; + // Get the window region and check if it is rectangular. + const int region_type = + GetWindowRegionTypeWithBoundary(selected, ®ion_rect); + + // Do not use the screen capturer if the region is empty or not rectangular. + if (region_type == COMPLEXREGION || region_type == NULLREGION) { + return false; + } + + if (region_type == SIMPLEREGION) { + // The |region_rect| returned from GetRgnBox() is always in window + // coordinate. + region_rect.Translate( + window_region_rect_.left(), window_region_rect_.top()); + // MSDN: The window region determines the area *within* the window where the + // system permits drawing. + // https://msdn.microsoft.com/en-us/library/windows/desktop/dd144950(v=vs.85).aspx. + // + // |region_rect| should always be inside of |window_region_rect_|. So after + // the intersection, |window_region_rect_| == |region_rect|. If so, what's + // the point of the intersecting operations? Why cannot we directly retrieve + // |window_region_rect_| from GetWindowRegionTypeWithBoundary() function? + // TODO(zijiehe): Figure out the purpose of these intersections. + window_region_rect_.IntersectWith(region_rect); + content_rect.IntersectWith(region_rect); + } + + // Check if the client area is out of the screen area. When the window is + // maximized, only its client area is visible in the screen, the border will + // be hidden. So we are using |content_rect| here. + if (!GetFullscreenRect().ContainsRect(content_rect)) { + return false; + } + + // Check if the window is occluded by any other window, excluding the child + // windows, context menus, and |excluded_window_|. + // |content_rect| is preferred, see the comments in TopWindowVerifier() + // function. + TopWindowVerifierContext context( + selected, reinterpret_cast(excluded_window()), content_rect); + const LPARAM enum_param = reinterpret_cast(&context); + EnumWindows(&TopWindowVerifier, enum_param); + if (!context.is_top_window) { + return false; + } + + // If |selected| is not covered by other windows, check whether it is + // covered by its own child windows. Note: EnumChildWindows() enumerates child + // windows in all generations, but does not include any controls like buttons + // or textboxes. + EnumChildWindows(selected, &TopWindowVerifier, enum_param); + return context.is_top_window; +} + +DesktopRect CroppingWindowCapturerWin::GetWindowRectInVirtualScreen() { + DesktopRect window_rect; + HWND hwnd = reinterpret_cast(selected_window()); + if (!GetCroppedWindowRect(hwnd, &window_rect, /* original_rect */ nullptr)) { + RTC_LOG(LS_WARNING) << "Failed to get window info: " << GetLastError(); + return window_rect; + } + window_rect.IntersectWith(window_region_rect_); + + // Convert |window_rect| to be relative to the top-left of the virtual screen. + DesktopRect screen_rect(GetFullscreenRect()); + window_rect.IntersectWith(screen_rect); + window_rect.Translate(-screen_rect.left(), -screen_rect.top()); + return window_rect; +} + +} // namespace + +// static +std::unique_ptr CroppingWindowCapturer::CreateCapturer( + const DesktopCaptureOptions& options) { + return std::unique_ptr( + new CroppingWindowCapturerWin(options)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.cc new file mode 100644 index 0000000000..5dbf092680 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/desktop_and_cursor_composer.h" + +#include + +#include + +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/mouse_cursor.h" +#include "modules/desktop_capture/mouse_cursor_monitor.h" +#include "rtc_base/checks.h" +#include "rtc_base/constructormagic.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { + +namespace { + +// Helper function that blends one image into another. Source image must be +// pre-multiplied with the alpha channel. Destination is assumed to be opaque. +void AlphaBlend(uint8_t* dest, int dest_stride, + const uint8_t* src, int src_stride, + const DesktopSize& size) { + for (int y = 0; y < size.height(); ++y) { + for (int x = 0; x < size.width(); ++x) { + uint32_t base_alpha = 255 - src[x * DesktopFrame::kBytesPerPixel + 3]; + if (base_alpha == 255) { + continue; + } else if (base_alpha == 0) { + memcpy(dest + x * DesktopFrame::kBytesPerPixel, + src + x * DesktopFrame::kBytesPerPixel, + DesktopFrame::kBytesPerPixel); + } else { + dest[x * DesktopFrame::kBytesPerPixel] = + dest[x * DesktopFrame::kBytesPerPixel] * base_alpha / 255 + + src[x * DesktopFrame::kBytesPerPixel]; + dest[x * DesktopFrame::kBytesPerPixel + 1] = + dest[x * DesktopFrame::kBytesPerPixel + 1] * base_alpha / 255 + + src[x * DesktopFrame::kBytesPerPixel + 1]; + dest[x * DesktopFrame::kBytesPerPixel + 2] = + dest[x * DesktopFrame::kBytesPerPixel + 2] * base_alpha / 255 + + src[x * DesktopFrame::kBytesPerPixel + 2]; + } + } + src += src_stride; + dest += dest_stride; + } +} + +// DesktopFrame wrapper that draws mouse on a frame and restores original +// content before releasing the underlying frame. +class DesktopFrameWithCursor : public DesktopFrame { + public: + // Takes ownership of |frame|. + DesktopFrameWithCursor(std::unique_ptr frame, + const MouseCursor& cursor, + const DesktopVector& position); + ~DesktopFrameWithCursor() override; + + private: + const std::unique_ptr original_frame_; + + DesktopVector restore_position_; + std::unique_ptr restore_frame_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DesktopFrameWithCursor); +}; + +DesktopFrameWithCursor::DesktopFrameWithCursor( + std::unique_ptr frame, + const MouseCursor& cursor, + const DesktopVector& position) + : DesktopFrame(frame->size(), + frame->stride(), + frame->data(), + frame->shared_memory()), + original_frame_(std::move(frame)) { + MoveFrameInfoFrom(original_frame_.get()); + + DesktopVector image_pos = position.subtract(cursor.hotspot()); + DesktopRect target_rect = DesktopRect::MakeSize(cursor.image()->size()); + target_rect.Translate(image_pos); + DesktopVector target_origin = target_rect.top_left(); + target_rect.IntersectWith(DesktopRect::MakeSize(size())); + + if (target_rect.is_empty()) + return; + + // Copy original screen content under cursor to |restore_frame_|. + restore_position_ = target_rect.top_left(); + restore_frame_.reset(new BasicDesktopFrame(target_rect.size())); + restore_frame_->CopyPixelsFrom(*this, target_rect.top_left(), + DesktopRect::MakeSize(restore_frame_->size())); + + // Blit the cursor. + uint8_t* target_rect_data = reinterpret_cast(data()) + + target_rect.top() * stride() + + target_rect.left() * DesktopFrame::kBytesPerPixel; + DesktopVector origin_shift = target_rect.top_left().subtract(target_origin); + AlphaBlend(target_rect_data, stride(), + cursor.image()->data() + + origin_shift.y() * cursor.image()->stride() + + origin_shift.x() * DesktopFrame::kBytesPerPixel, + cursor.image()->stride(), + target_rect.size()); +} + +DesktopFrameWithCursor::~DesktopFrameWithCursor() { + // Restore original content of the frame. + if (restore_frame_) { + DesktopRect target_rect = DesktopRect::MakeSize(restore_frame_->size()); + target_rect.Translate(restore_position_); + CopyPixelsFrom(restore_frame_->data(), restore_frame_->stride(), + target_rect); + } +} + +} // namespace + +DesktopAndCursorComposer::DesktopAndCursorComposer( + DesktopCapturer* desktop_capturer, + MouseCursorMonitor* mouse_monitor) + : DesktopAndCursorComposer(desktop_capturer, mouse_monitor, true) {} + +DesktopAndCursorComposer::DesktopAndCursorComposer( + std::unique_ptr desktop_capturer, + const DesktopCaptureOptions& options) + : DesktopAndCursorComposer(desktop_capturer.release(), + MouseCursorMonitor::Create(options).release(), + true) {} + +DesktopAndCursorComposer::DesktopAndCursorComposer( + DesktopCapturer* desktop_capturer, + MouseCursorMonitor* mouse_monitor, + bool use_desktop_relative_cursor_position) + : desktop_capturer_(desktop_capturer), + mouse_monitor_(mouse_monitor), + use_desktop_relative_cursor_position_( + use_desktop_relative_cursor_position) { + RTC_DCHECK(desktop_capturer_); +} + +DesktopAndCursorComposer::~DesktopAndCursorComposer() = default; + +std::unique_ptr +DesktopAndCursorComposer::CreateWithoutMouseCursorMonitor( + std::unique_ptr desktop_capturer) { + return std::unique_ptr( + new DesktopAndCursorComposer(desktop_capturer.release(), nullptr)); +} + +void DesktopAndCursorComposer::Start(DesktopCapturer::Callback* callback) { + callback_ = callback; + if (mouse_monitor_) + mouse_monitor_->Init(this, MouseCursorMonitor::SHAPE_AND_POSITION); + desktop_capturer_->Start(this); +} + +void DesktopAndCursorComposer::SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) { + desktop_capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory)); +} + +void DesktopAndCursorComposer::CaptureFrame() { + if (mouse_monitor_) + mouse_monitor_->Capture(); + desktop_capturer_->CaptureFrame(); +} + +void DesktopAndCursorComposer::SetExcludedWindow(WindowId window) { + desktop_capturer_->SetExcludedWindow(window); +} + +void DesktopAndCursorComposer::OnCaptureResult( + DesktopCapturer::Result result, + std::unique_ptr frame) { + if (frame && cursor_) { + if (use_desktop_relative_cursor_position_) { + if (frame->rect().Contains(cursor_position_) && + !desktop_capturer_->IsOccluded(cursor_position_)) { + const DesktopVector relative_position = + cursor_position_.subtract(frame->top_left()); + frame = rtc::MakeUnique( + std::move(frame), *cursor_, relative_position); + } + } else { + if (cursor_state_ == MouseCursorMonitor::INSIDE) { + frame = rtc::MakeUnique( + std::move(frame), *cursor_, cursor_position_); + } + } + } + + callback_->OnCaptureResult(result, std::move(frame)); +} + +void DesktopAndCursorComposer::OnMouseCursor(MouseCursor* cursor) { + cursor_.reset(cursor); +} + +void DesktopAndCursorComposer::OnMouseCursorPosition( + MouseCursorMonitor::CursorState state, + const DesktopVector& position) { + if (!use_desktop_relative_cursor_position_) { + cursor_state_ = state; + cursor_position_ = position; + } +} + +void DesktopAndCursorComposer::OnMouseCursorPosition( + const DesktopVector& position) { + if (use_desktop_relative_cursor_position_) { + cursor_position_ = position; + } +} + +bool DesktopAndCursorComposer::FocusOnSelectedSource() { + return desktop_capturer_->FocusOnSelectedSource(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h new file mode 100644 index 0000000000..e86db18dd6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_DESKTOP_AND_CURSOR_COMPOSER_H_ +#define MODULES_DESKTOP_CAPTURE_DESKTOP_AND_CURSOR_COMPOSER_H_ + +#include + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/mouse_cursor_monitor.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +template +class DesktopAndCursorComposerTest; + +// A wrapper for DesktopCapturer that also captures mouse using specified +// MouseCursorMonitor and renders it on the generated streams. +class DesktopAndCursorComposer : public DesktopCapturer, + public DesktopCapturer::Callback, + public MouseCursorMonitor::Callback { + public: + // Creates a new comp that captures mouse cursor using |mouse_monitor| and + // renders it into the frames generated by |desktop_capturer|. If + // |mouse_monitor| is NULL the frames are passed unmodified. Takes ownership + // of both arguments. + // Deprecated: use the constructor below. + DesktopAndCursorComposer(DesktopCapturer* desktop_capturer, + MouseCursorMonitor* mouse_monitor); + + // Creates a new blender that captures mouse cursor using + // MouseCursorMonitor::Create(options) and renders it into the frames + // generated by |desktop_capturer|. + DesktopAndCursorComposer(std::unique_ptr desktop_capturer, + const DesktopCaptureOptions& options); + + ~DesktopAndCursorComposer() override; + + // Creates a new composer that relies on an external source for cursor shape + // and position information via the MouseCursorMonitor::Callback interface. + static std::unique_ptr + CreateWithoutMouseCursorMonitor( + std::unique_ptr desktop_capturer); + + // DesktopCapturer interface. + void Start(DesktopCapturer::Callback* callback) override; + void SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) override; + void CaptureFrame() override; + void SetExcludedWindow(WindowId window) override; + bool FocusOnSelectedSource() override; + + // MouseCursorMonitor::Callback interface. + void OnMouseCursor(MouseCursor* cursor) override; + void OnMouseCursorPosition(webrtc::MouseCursorMonitor::CursorState, + const webrtc::DesktopVector&) override; + void OnMouseCursorPosition(const DesktopVector& position) override; + + private: + // Allows test cases to use a fake MouseCursorMonitor implementation. + friend class DesktopAndCursorComposerTest; + friend class DesktopAndCursorComposerTest; + + // Constructor to delegate both deprecated and new constructors and allows + // test cases to use a fake MouseCursorMonitor implementation. + DesktopAndCursorComposer(DesktopCapturer* desktop_capturer, + MouseCursorMonitor* mouse_monitor, + bool use_desktop_relative_cursor_position); + + // DesktopCapturer::Callback interface. + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr frame) override; + + const std::unique_ptr desktop_capturer_; + const std::unique_ptr mouse_monitor_; + // This is a temporary flag to decide how to use the |mouse_monitor_|. + // If it's true, DesktopAndCursorComposer will use the absolute position from + // MouseCursorMonitor but ignore the MouseCursorMonitor::CursorState. + // Otherwise MouseCursorMonitor::CursorState is respected. This flag is false + // when the deprecated constructor is used, and true when the new one is used. + // This flag will be removed together with the deprecated constructor. + const bool use_desktop_relative_cursor_position_; + + DesktopCapturer::Callback* callback_; + + std::unique_ptr cursor_; + // This field is irrelevant if |use_desktop_relative_cursor_position_| is + // true. + MouseCursorMonitor::CursorState cursor_state_; + DesktopVector cursor_position_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DesktopAndCursorComposer); +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_DESKTOP_AND_CURSOR_COMPOSER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc new file mode 100644 index 0000000000..eddae1d4a6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/desktop_capture/desktop_and_cursor_composer.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/mouse_cursor.h" +#include "modules/desktop_capture/shared_desktop_frame.h" +#include "rtc_base/arraysize.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +const int kScreenWidth = 100; +const int kScreenHeight = 100; +const int kCursorWidth = 10; +const int kCursorHeight = 10; + +const int kTestCursorSize = 3; +const uint32_t kTestCursorData[kTestCursorSize][kTestCursorSize] = { + { 0xffffffff, 0x99990000, 0xaa222222, }, + { 0x88008800, 0xaa0000aa, 0xaa333333, }, + { 0x00000000, 0xaa0000aa, 0xaa333333, }, +}; + +uint32_t GetFakeFramePixelValue(const DesktopVector& p) { + uint32_t r = 100 + p.x(); + uint32_t g = 100 + p.y(); + uint32_t b = 100 + p.x() + p.y(); + return b + (g << 8) + (r << 16) + 0xff000000; +} + +uint32_t GetFramePixel(const DesktopFrame& frame, const DesktopVector& pos) { + return *reinterpret_cast(frame.GetFrameDataAtPos(pos)); +} + +// Blends two pixel values taking into account alpha. +uint32_t BlendPixels(uint32_t dest, uint32_t src) { + uint8_t alpha = 255 - ((src & 0xff000000) >> 24); + uint32_t r = + ((dest & 0x00ff0000) >> 16) * alpha / 255 + ((src & 0x00ff0000) >> 16); + uint32_t g = + ((dest & 0x0000ff00) >> 8) * alpha / 255 + ((src & 0x0000ff00) >> 8); + uint32_t b = (dest & 0x000000ff) * alpha / 255 + (src & 0x000000ff); + return b + (g << 8) + (r << 16) + 0xff000000; +} + +DesktopFrame* CreateTestFrame() { + DesktopFrame* frame = + new BasicDesktopFrame(DesktopSize(kScreenWidth, kScreenHeight)); + uint32_t* data = reinterpret_cast(frame->data()); + for (int y = 0; y < kScreenHeight; ++y) { + for (int x = 0; x < kScreenWidth; ++x) { + *(data++) = GetFakeFramePixelValue(DesktopVector(x, y)); + } + } + return frame; +} + +class FakeScreenCapturer : public DesktopCapturer { + public: + FakeScreenCapturer() {} + + void Start(Callback* callback) override { callback_ = callback; } + + void CaptureFrame() override { + callback_->OnCaptureResult( + next_frame_ ? Result::SUCCESS : Result::ERROR_TEMPORARY, + std::move(next_frame_)); + } + + void SetNextFrame(std::unique_ptr next_frame) { + next_frame_ = std::move(next_frame); + } + + bool IsOccluded(const DesktopVector& pos) override { return is_occluded_; } + + void set_is_occluded(bool value) { is_occluded_ = value; } + + private: + Callback* callback_ = nullptr; + + std::unique_ptr next_frame_; + bool is_occluded_ = false; +}; + +class FakeMouseMonitor : public MouseCursorMonitor { + public: + FakeMouseMonitor() : changed_(true) {} + + void SetState(CursorState state, const DesktopVector& pos) { + state_ = state; + position_ = pos; + } + + void SetHotspot(const DesktopVector& hotspot) { + if (!hotspot_.equals(hotspot)) + changed_ = true; + hotspot_ = hotspot; + } + + void Init(Callback* callback, Mode mode) { callback_ = callback; } + + void Capture() override { + if (changed_) { + std::unique_ptr image( + new BasicDesktopFrame(DesktopSize(kCursorWidth, kCursorHeight))); + uint32_t* data = reinterpret_cast(image->data()); + memset(data, 0, image->stride() * kCursorHeight); + + // Set four pixels near the hotspot and leave all other blank. + for (int y = 0; y < kTestCursorSize; ++y) { + for (int x = 0; x < kTestCursorSize; ++x) { + data[(hotspot_.y() + y) * kCursorWidth + (hotspot_.x() + x)] = + kTestCursorData[y][x]; + } + } + + callback_->OnMouseCursor(new MouseCursor(image.release(), hotspot_)); + } + + callback_->OnMouseCursorPosition(state_, position_); + callback_->OnMouseCursorPosition(position_); + } + + private: + Callback* callback_; + CursorState state_; + DesktopVector position_; + DesktopVector hotspot_; + bool changed_; +}; + +void VerifyFrame(const DesktopFrame& frame, + MouseCursorMonitor::CursorState state, + const DesktopVector& pos) { + // Verify that all other pixels are set to their original values. + DesktopRect image_rect = + DesktopRect::MakeWH(kTestCursorSize, kTestCursorSize); + image_rect.Translate(pos); + + for (int y = 0; y < kScreenHeight; ++y) { + for (int x = 0; x < kScreenWidth; ++x) { + DesktopVector p(x, y); + if (state == MouseCursorMonitor::INSIDE && image_rect.Contains(p)) { + EXPECT_EQ(BlendPixels(GetFakeFramePixelValue(p), + kTestCursorData[y - pos.y()][x - pos.x()]), + GetFramePixel(frame, p)); + } else { + EXPECT_EQ(GetFakeFramePixelValue(p), GetFramePixel(frame, p)); + } + } + } +} + +} // namespace + +template +class DesktopAndCursorComposerTest : public testing::Test, + public DesktopCapturer::Callback { + public: + DesktopAndCursorComposerTest() + : fake_screen_(new FakeScreenCapturer()), + fake_cursor_(new FakeMouseMonitor()), + blender_(fake_screen_, + fake_cursor_, + use_desktop_relative_cursor_position) { + blender_.Start(this); + } + + // DesktopCapturer::Callback interface + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr frame) override { + frame_ = std::move(frame); + } + + protected: + // Owned by |blender_|. + FakeScreenCapturer* fake_screen_; + FakeMouseMonitor* fake_cursor_; + + DesktopAndCursorComposer blender_; + std::unique_ptr frame_; +}; + +using DesktopAndCursorComposerWithRelativePositionTest = + DesktopAndCursorComposerTest; + +// Verify DesktopAndCursorComposer can handle the case when the screen capturer +// fails. +TEST_F(DesktopAndCursorComposerWithRelativePositionTest, Error) { + fake_cursor_->SetHotspot(DesktopVector()); + fake_cursor_->SetState(MouseCursorMonitor::INSIDE, DesktopVector()); + fake_screen_->SetNextFrame(nullptr); + + blender_.CaptureFrame(); + + EXPECT_FALSE(frame_); +} + +TEST_F(DesktopAndCursorComposerWithRelativePositionTest, Blend) { + struct { + int x, y; + int hotspot_x, hotspot_y; + bool inside; + } tests[] = { + {0, 0, 0, 0, true}, + {50, 50, 0, 0, true}, + {100, 50, 0, 0, true}, + {50, 100, 0, 0, true}, + {100, 100, 0, 0, true}, + {0, 0, 2, 5, true}, + {1, 1, 2, 5, true}, + {50, 50, 2, 5, true}, + {100, 100, 2, 5, true}, + {0, 0, 5, 2, true}, + {50, 50, 5, 2, true}, + {100, 100, 5, 2, true}, + {0, 0, 0, 0, false}, + }; + + for (size_t i = 0; i < arraysize(tests); i++) { + SCOPED_TRACE(i); + + DesktopVector hotspot(tests[i].hotspot_x, tests[i].hotspot_y); + fake_cursor_->SetHotspot(hotspot); + + MouseCursorMonitor::CursorState state = tests[i].inside + ? MouseCursorMonitor::INSIDE + : MouseCursorMonitor::OUTSIDE; + DesktopVector pos(tests[i].x, tests[i].y); + fake_cursor_->SetState(state, pos); + + std::unique_ptr frame( + SharedDesktopFrame::Wrap(CreateTestFrame())); + fake_screen_->SetNextFrame(frame->Share()); + + blender_.CaptureFrame(); + + VerifyFrame(*frame_, state, pos); + + // Verify that the cursor is erased before the frame buffer is returned to + // the screen capturer. + frame_.reset(); + VerifyFrame(*frame, MouseCursorMonitor::OUTSIDE, DesktopVector()); + } +} + +using DesktopAndCursorComposerWithAbsolutePositionTest = + DesktopAndCursorComposerTest; + +TEST_F(DesktopAndCursorComposerWithAbsolutePositionTest, + CursorShouldBeIgnoredIfItIsOutOfDesktopFrame) { + std::unique_ptr frame( + SharedDesktopFrame::Wrap(CreateTestFrame())); + frame->set_top_left(DesktopVector(100, 200)); + // The frame covers (100, 200) - (200, 300). + + struct { + int x; + int y; + } tests[] = { + { 0, 0 }, + { 50, 50 }, + { 50, 150 }, + { 100, 150 }, + { 50, 200 }, + { 99, 200 }, + { 100, 199 }, + { 200, 300 }, + { 200, 299 }, + { 199, 300 }, + { -1, -1 }, + { -10000, -10000 }, + { 10000, 10000 }, + }; + for (size_t i = 0; i < arraysize(tests); i++) { + SCOPED_TRACE(i); + + fake_screen_->SetNextFrame(frame->Share()); + // The CursorState is ignored when using absolute cursor position. + fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, + DesktopVector(tests[i].x, tests[i].y)); + blender_.CaptureFrame(); + VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector(0, 0)); + } +} + +TEST_F(DesktopAndCursorComposerWithAbsolutePositionTest, + IsOccludedShouldBeConsidered) { + std::unique_ptr frame( + SharedDesktopFrame::Wrap(CreateTestFrame())); + frame->set_top_left(DesktopVector(100, 200)); + // The frame covers (100, 200) - (200, 300). + + struct { + int x; + int y; + } tests[] = { + { 100, 200 }, + { 101, 200 }, + { 100, 201 }, + { 101, 201 }, + { 150, 250 }, + { 199, 299 }, + }; + fake_screen_->set_is_occluded(true); + for (size_t i = 0; i < arraysize(tests); i++) { + SCOPED_TRACE(i); + + fake_screen_->SetNextFrame(frame->Share()); + // The CursorState is ignored when using absolute cursor position. + fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, + DesktopVector(tests[i].x, tests[i].y)); + blender_.CaptureFrame(); + VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector()); + } +} + +TEST_F(DesktopAndCursorComposerWithAbsolutePositionTest, CursorIncluded) { + std::unique_ptr frame( + SharedDesktopFrame::Wrap(CreateTestFrame())); + frame->set_top_left(DesktopVector(100, 200)); + // The frame covers (100, 200) - (200, 300). + + struct { + int x; + int y; + } tests[] = { + { 100, 200 }, + { 101, 200 }, + { 100, 201 }, + { 101, 201 }, + { 150, 250 }, + { 199, 299 }, + }; + for (size_t i = 0; i < arraysize(tests); i++) { + SCOPED_TRACE(i); + + const DesktopVector abs_pos(tests[i].x, tests[i].y); + const DesktopVector rel_pos(abs_pos.subtract(frame->top_left())); + + fake_screen_->SetNextFrame(frame->Share()); + // The CursorState is ignored when using absolute cursor position. + fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, abs_pos); + blender_.CaptureFrame(); + VerifyFrame(*frame_, MouseCursorMonitor::INSIDE, rel_pos); + + // Verify that the cursor is erased before the frame buffer is returned to + // the screen capturer. + frame_.reset(); + VerifyFrame(*frame, MouseCursorMonitor::OUTSIDE, DesktopVector()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_differ_sse2_gn/moz.build b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_differ_sse2_gn/moz.build new file mode 100644 index 0000000000..22efa4c647 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_differ_sse2_gn/moz.build @@ -0,0 +1,184 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/differ_vector_sse2.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["CR_XCODE_VERSION"] = "0920" + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + +if CONFIG["OS_TARGET"] == "DragonFly": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2", + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2", + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2", + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("desktop_capture_differ_sse2_gn") diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_generic_gn/moz.build b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_generic_gn/moz.build new file mode 100644 index 0000000000..a0e116195e --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_generic_gn/moz.build @@ -0,0 +1,444 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### +if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk": + CXXFLAGS += CONFIG["TK_CFLAGS"] + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["MULTI_MONITOR_SCREENSHARE"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/media/libyuv/libyuv/include/", + "/third_party/libwebrtc/webrtc/" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/capture_result_desktop_capturer_wrapper.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/cropped_desktop_frame.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_and_cursor_composer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_wrapper.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_device_info.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_generator.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_rotation.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/differ_block.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/resolution_tracker.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/rgba_color.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_helper.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework Foundation" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_configuration_monitor.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_device_info_mac.mm", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mac/full_screen_chrome_window_detector.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mac/scoped_pixel_buffer_object.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mac/window_list_utils.cc" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/desktop_device_info_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/mouse_cursor_monitor_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_display.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_util.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_finder_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_list_utils.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_atom_cache.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_error_trap.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_server_pixel_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_linux.cc" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/desktop_device_info_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/mouse_cursor_monitor_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_display.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_util.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_finder_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_list_utils.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_atom_cache.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_error_trap.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_server_pixel_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_linux.cc" + ] + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt", + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/desktop_device_info_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/mouse_cursor_monitor_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_display.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_util.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_finder_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_list_utils.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_atom_cache.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_error_trap.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_server_pixel_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_linux.cc" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/desktop_device_info_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/mouse_cursor_monitor_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_display.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_util.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_finder_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_list_utils.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_atom_cache.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_error_trap.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_server_pixel_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_linux.cc" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/desktop_device_info_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/mouse_cursor_monitor_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_display.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/shared_x_util.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_finder_x11.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_list_utils.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_atom_cache.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_error_trap.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/x_server_pixel_buffer.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_linux.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_linux.cc" + ] + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "d3d11", + "dxgi", + "winmm" + ] + + SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_gdi.cc" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/cropping_window_capturer_win.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_frame_win.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_win.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_win.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/cursor.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/d3d_device.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/desktop.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/desktop_device_info_win.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/display_configuration_monitor.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_adapter_duplicator.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_context.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_duplicator_controller.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_frame.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_output_duplicator.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture_mapping.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/dxgi_texture_staging.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/scoped_thread_desktop.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capture_utils.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_directx.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/screen_capturer_win_magnifier.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/win_shared.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/win/window_capture_utils.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_win.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder_win.cc" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "FreeBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_USE_PIPEWIRE"] = True + + LOCAL_INCLUDES += [ + "/third_party/pipewire/" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/base_capturer_pipewire.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_pipewire.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_pipewire.cc" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["WEBRTC_USE_PIPEWIRE"] = True + + LOCAL_INCLUDES += [ + "/third_party/pipewire/" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/base_capturer_pipewire.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_pipewire.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_pipewire.cc" + ] + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_USE_PIPEWIRE"] = True + + LOCAL_INCLUDES += [ + "/third_party/pipewire/" + ] + + UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/base_capturer_pipewire.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/screen_capturer_pipewire.cc", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/linux/window_capturer_pipewire.cc" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "NetBSD": + + CXXFLAGS += [ + "-msse2" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "OpenBSD": + + CXXFLAGS += [ + "-msse2" + ] + +Library("desktop_capture_generic_gn") diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_gn/moz.build b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_gn/moz.build new file mode 100644 index 0000000000..daf47ae5d6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_gn/moz.build @@ -0,0 +1,231 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["NO_TCMALLOC"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + + OS_LIBS += [ + "-framework AppKit", + "-framework Foundation", + "-framework IOKit", + "-framework OpenGL" + ] + +if CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + +if CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "rt", + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + +if CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + + OS_LIBS += [ + "X11", + "X11-xcb", + "xcb", + "Xcomposite", + "Xcursor", + "Xdamage", + "Xext", + "Xfixes", + "Xi", + "Xrender" + ] + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NO_TCMALLOC"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "1" + DEFINES["UNICODE"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_CRT_SECURE_NO_WARNINGS"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_USING_V110_SDK71_"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "d3d11", + "dxgi", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "DragonFly": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "FreeBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "NetBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if not CONFIG["MOZ_DEBUG"] and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0120" + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["DISABLE_NACL"] = True + DEFINES["NO_TCMALLOC"] = True + +Library("desktop_capture_gn") diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_objc_gn/moz.build b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_objc_gn/moz.build new file mode 100644 index 0000000000..d6e22aeabc --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_objc_gn/moz.build @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["CHROMIUM_BUILD"] = True +DEFINES["NO_TCMALLOC"] = True +DEFINES["V8_DEPRECATION_WARNINGS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_MAC"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_POSIX"] = True +DEFINES["WEBRTC_RESTRICT_LOGGING"] = True +DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/ipc/glue", + "/third_party/libwebrtc/webrtc/" +] + +OS_LIBS += [ + "-framework AppKit", + "-framework Foundation", + "-framework IOKit", + "-framework OpenGL" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mac/desktop_configuration.mm", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/mouse_cursor_monitor_mac.mm", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/screen_capturer_mac.mm", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_capturer_mac.mm", + "/third_party/libwebrtc/webrtc/modules/desktop_capture/window_finder_mac.mm" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + DEFINES["_FORTIFY_SOURCE"] = "2" + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["WTF_USE_DYNAMIC_ANNOTATIONS"] = "1" + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["CR_XCODE_VERSION"] = "0120" + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["CR_XCODE_VERSION"] = "0920" + +Library("desktop_capture_objc_gn") diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.cc new file mode 100644 index 0000000000..8a33807bb3 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/desktop_capture_options.h" + +namespace webrtc { + +DesktopCaptureOptions::DesktopCaptureOptions() {} +DesktopCaptureOptions::DesktopCaptureOptions( + const DesktopCaptureOptions& options) = default; +DesktopCaptureOptions::DesktopCaptureOptions(DesktopCaptureOptions&& options) = + default; +DesktopCaptureOptions::~DesktopCaptureOptions() {} + +DesktopCaptureOptions& DesktopCaptureOptions::operator=( + const DesktopCaptureOptions& options) = default; +DesktopCaptureOptions& DesktopCaptureOptions::operator=( + DesktopCaptureOptions&& options) = default; + +// static +DesktopCaptureOptions DesktopCaptureOptions::CreateDefault() { + DesktopCaptureOptions result; +#if defined(USE_X11) + result.set_x_display(SharedXDisplay::CreateDefault()); +#endif +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + result.set_configuration_monitor(new DesktopConfigurationMonitor()); + result.set_full_screen_chrome_window_detector( + new FullScreenChromeWindowDetector()); +#endif + return result; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.h new file mode 100644 index 0000000000..316468eed1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_options.h @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_OPTIONS_H_ +#define MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_OPTIONS_H_ + +#include "rtc_base/constructormagic.h" +#include "rtc_base/scoped_ref_ptr.h" + +#if defined(USE_X11) +#include "modules/desktop_capture/linux/shared_x_display.h" +#endif + +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) +#include "modules/desktop_capture/mac/desktop_configuration_monitor.h" +#include "modules/desktop_capture/mac/full_screen_chrome_window_detector.h" +#endif + +namespace webrtc { + +// An object that stores initialization parameters for screen and window +// capturers. +class DesktopCaptureOptions { + public: + // Returns instance of DesktopCaptureOptions with default parameters. On Linux + // also initializes X window connection. x_display() will be set to null if + // X11 connection failed (e.g. DISPLAY isn't set). + static DesktopCaptureOptions CreateDefault(); + + DesktopCaptureOptions(); + DesktopCaptureOptions(const DesktopCaptureOptions& options); + DesktopCaptureOptions(DesktopCaptureOptions&& options); + ~DesktopCaptureOptions(); + + DesktopCaptureOptions& operator=(const DesktopCaptureOptions& options); + DesktopCaptureOptions& operator=(DesktopCaptureOptions&& options); + +#if defined(USE_X11) + SharedXDisplay* x_display() const { return x_display_; } + void set_x_display(rtc::scoped_refptr x_display) { + x_display_ = x_display; + } +#endif + +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + // TODO(zijiehe): Remove both DesktopConfigurationMonitor and + // FullScreenChromeWindowDetector out of DesktopCaptureOptions. It's not + // reasonable for external consumers to set these two parameters. + DesktopConfigurationMonitor* configuration_monitor() const { + return configuration_monitor_; + } + // If nullptr is set, ScreenCapturer won't work and WindowCapturer may return + // inaccurate result from IsOccluded() function. + void set_configuration_monitor( + rtc::scoped_refptr m) { + configuration_monitor_ = m; + } + + // TODO(zijiehe): Instead of FullScreenChromeWindowDetector, provide a + // FullScreenWindowDetector for external consumers to detect the target + // fullscreen window. + FullScreenChromeWindowDetector* full_screen_chrome_window_detector() const { + return full_screen_window_detector_; + } + void set_full_screen_chrome_window_detector( + rtc::scoped_refptr detector) { + full_screen_window_detector_ = detector; + } +#endif + + // Flag indicating that the capturer should use screen change notifications. + // Enables/disables use of XDAMAGE in the X11 capturer. + bool use_update_notifications() const { return use_update_notifications_; } + void set_use_update_notifications(bool use_update_notifications) { + use_update_notifications_ = use_update_notifications; + } + + // Flag indicating if desktop effects (e.g. Aero) should be disabled when the + // capturer is active. Currently used only on Windows. + bool disable_effects() const { return disable_effects_; } + void set_disable_effects(bool disable_effects) { + disable_effects_ = disable_effects; + } + + // Flag that should be set if the consumer uses updated_region() and the + // capturer should try to provide correct updated_region() for the frames it + // generates (e.g. by comparing each frame with the previous one). + bool detect_updated_region() const { return detect_updated_region_; } + void set_detect_updated_region(bool detect_updated_region) { + detect_updated_region_ = detect_updated_region; + } + +#if defined(WEBRTC_WIN) + bool allow_use_magnification_api() const { + return allow_use_magnification_api_; + } + void set_allow_use_magnification_api(bool allow) { + allow_use_magnification_api_ = allow; + } + // Allowing directx based capturer or not, this capturer works on windows 7 + // with platform update / windows 8 or upper. + bool allow_directx_capturer() const { + return allow_directx_capturer_; + } + void set_allow_directx_capturer(bool enabled) { + allow_directx_capturer_ = enabled; + } +#endif + +#if defined(WEBRTC_USE_PIPEWIRE) + bool allow_pipewire() const { return allow_pipewire_; } + void set_allow_pipewire(bool allow) { allow_pipewire_ = allow; } +#endif + + private: +#if defined(USE_X11) + rtc::scoped_refptr x_display_; +#endif + +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + rtc::scoped_refptr configuration_monitor_; + rtc::scoped_refptr + full_screen_window_detector_; +#endif + +#if defined(WEBRTC_WIN) + bool allow_use_magnification_api_ = false; + bool allow_directx_capturer_ = false; +#endif +#if defined(USE_X11) + bool use_update_notifications_ = false; +#else + bool use_update_notifications_ = true; +#endif + bool disable_effects_ = true; + bool detect_updated_region_ = false; +#if defined(WEBRTC_USE_PIPEWIRE) + bool allow_pipewire_ = true; +#endif +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_OPTIONS_H_ diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_types.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_types.h new file mode 100644 index 0000000000..47774f77ce --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capture_types.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_TYPES_H_ +#define MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_TYPES_H_ + +#ifndef XP_WIN +#include // pid_t +#endif +#include + +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +// Type used to identify windows on the desktop. Values are platform-specific: +// - On Windows: HWND cast to intptr_t. +// - On Linux (with X11): X11 Window (unsigned long) type cast to intptr_t. +// - On OSX: integer window number. +typedef intptr_t WindowId; + +const WindowId kNullWindowId = 0; + +// Type used to identify screens on the desktop. Values are platform-specific: +// - On Windows: integer display device index. +// - On OSX: CGDirectDisplayID cast to intptr_t. +// - On Linux (with X11): TBD. +// On Windows, ScreenId is implementation dependent: sending a ScreenId from one +// implementation to another usually won't work correctly. +typedef intptr_t ScreenId; + +// The screen id corresponds to all screen combined together. +const ScreenId kFullDesktopScreenId = -1; + +const ScreenId kInvalidScreenId = -2; + +typedef intptr_t ProcessId; +const ProcessId DesktopProcessId = 0; + +#ifdef XP_WIN +typedef int pid_t; +#endif + +// An integer to attach to each DesktopFrame to differentiate the generator of +// the frame. +namespace DesktopCapturerId { + constexpr uint32_t CreateFourCC(char a, char b, char c, char d) { + return ((static_cast(a)) | + (static_cast(b) << 8) | + (static_cast(c) << 16) | + (static_cast(d) << 24)); + } + + constexpr uint32_t kUnknown = 0; + constexpr uint32_t kScreenCapturerWinGdi = CreateFourCC('G', 'D', 'I', ' '); + constexpr uint32_t kScreenCapturerWinDirectx = + CreateFourCC('D', 'X', 'G', 'I'); +} // namespace DesktopCapturerId + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_TYPES_H_ diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.cc new file mode 100644 index 0000000000..5e4dcb058d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/desktop_capturer.h" + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer_differ_wrapper.h" + +#if defined(WEBRTC_USE_PIPEWIRE) || defined(USE_X11) +#include +#include +#endif + +namespace webrtc { + +DesktopCapturer::~DesktopCapturer() = default; + +void DesktopCapturer::SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) {} + +void DesktopCapturer::SetExcludedWindow(WindowId window) {} + +bool DesktopCapturer::GetSourceList(SourceList* sources) { + return true; +} + +bool DesktopCapturer::SelectSource(SourceId id) { + return false; +} + +bool DesktopCapturer::FocusOnSelectedSource() { + return false; +} + +bool DesktopCapturer::IsOccluded(const DesktopVector& pos) { + return false; +} + +// static +std::unique_ptr DesktopCapturer::CreateWindowCapturer( + const DesktopCaptureOptions& options) { + std::unique_ptr capturer = CreateRawWindowCapturer(options); + if (capturer && options.detect_updated_region()) { + capturer.reset(new DesktopCapturerDifferWrapper(std::move(capturer))); + } + + return capturer; +} + +// static +std::unique_ptr DesktopCapturer::CreateScreenCapturer( + const DesktopCaptureOptions& options) { + std::unique_ptr capturer = CreateRawScreenCapturer(options); + if (capturer && options.detect_updated_region()) { + capturer.reset(new DesktopCapturerDifferWrapper(std::move(capturer))); + } + + return capturer; +} + +// static +std::unique_ptr DesktopCapturer::CreateTabCapturer( + const DesktopCaptureOptions& options) { + std::unique_ptr capturer = CreateRawTabCapturer(options); + if (capturer && options.detect_updated_region()) { + capturer.reset(new DesktopCapturerDifferWrapper(std::move(capturer))); + } + + return capturer; +} + +#if defined(WEBRTC_USE_PIPEWIRE) || defined(USE_X11) +bool DesktopCapturer::IsRunningUnderWayland() { + const char* xdg_session_type = getenv("XDG_SESSION_TYPE"); + if (!xdg_session_type || strncmp(xdg_session_type, "wayland", 7) != 0) + return false; + + if (!(getenv("WAYLAND_DISPLAY"))) + return false; + + return true; +} +#endif // defined(WEBRTC_USE_PIPEWIRE) || defined(USE_X11) + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.h new file mode 100644 index 0000000000..461e8fd53d --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer.h @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURER_H_ +#define MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURER_H_ + +#include +#include + +#include +#include +#include +#include + +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/desktop_capture_types.h" +#include "modules/desktop_capture/shared_memory.h" + +namespace webrtc { + +class DesktopCaptureOptions; +class DesktopFrame; + +// Abstract interface for screen and window capturers. +class DesktopCapturer { + public: + enum class Result { + // The frame was captured successfully. + SUCCESS, + + // There was a temporary error. The caller should continue calling + // CaptureFrame(), in the expectation that it will eventually recover. + ERROR_TEMPORARY, + + // Capture has failed and will keep failing if the caller tries calling + // CaptureFrame() again. + ERROR_PERMANENT, + + MAX_VALUE = ERROR_PERMANENT + }; + + // Interface that must be implemented by the DesktopCapturer consumers. + class Callback { + public: + // Called after a frame has been captured. |frame| is not nullptr if and + // only if |result| is SUCCESS. + virtual void OnCaptureResult(Result result, + std::unique_ptr frame) = 0; + + protected: + virtual ~Callback() {} + }; + + typedef intptr_t SourceId; + + static_assert(std::is_same::value, + "SourceId should be a same type as ScreenId."); + + struct Source { + // The unique id to represent a Source of current DesktopCapturer. + SourceId id; + pid_t pid; + + // Title of the window or screen in UTF-8 encoding, maybe empty. This field + // should not be used to identify a source. + std::string title; + }; + + typedef std::vector SourceList; + + virtual ~DesktopCapturer(); + + // Called at the beginning of a capturing session. |callback| must remain + // valid until capturer is destroyed. + virtual void Start(Callback* callback) = 0; + + // Sets SharedMemoryFactory that will be used to create buffers for the + // captured frames. The factory can be invoked on a thread other than the one + // where CaptureFrame() is called. It will be destroyed on the same thread. + // Shared memory is currently supported only by some DesktopCapturer + // implementations. + virtual void SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory); + + // Captures next frame, and involve callback provided by Start() function. + // Pending capture requests are canceled when DesktopCapturer is deleted. + virtual void CaptureFrame() = 0; + + // Sets the window to be excluded from the captured image in the future + // Capture calls. Used to exclude the screenshare notification window for + // screen capturing. + virtual void SetExcludedWindow(WindowId window); + + // TODO(zijiehe): Following functions should be pure virtual. The default + // implementations are for backward compatibility only. Remove default + // implementations once all DesktopCapturer implementations in Chromium have + // implemented these functions. + + // Gets a list of sources current capturer supports. Returns false in case of + // a failure. + // For DesktopCapturer implementations to capture screens, this function + // should return monitors. + // For DesktopCapturer implementations to capture windows, this function + // should only return root windows owned by applications. + virtual bool GetSourceList(SourceList* sources); + + // Selects a source to be captured. Returns false in case of a failure (e.g. + // if there is no source with the specified type and id.) + virtual bool SelectSource(SourceId id); + + // Brings the selected source to the front and sets the input focus on it. + // Returns false in case of a failure or no source has been selected or the + // implementation does not support this functionality. + virtual bool FocusOnSelectedSource(); + + // Returns true if the |pos| on the selected source is covered by other + // elements on the display, and is not visible to the users. + // |pos| is in full desktop coordinates, i.e. the top-left monitor always + // starts from (0, 0). + // The return value if |pos| is out of the scope of the source is undefined. + virtual bool IsOccluded(const DesktopVector& pos); + + // Creates a DesktopCapturer instance which targets to capture windows. + static std::unique_ptr CreateWindowCapturer( + const DesktopCaptureOptions& options); + + // Creates a DesktopCapturer instance which targets to capture screens. + static std::unique_ptr CreateScreenCapturer( + const DesktopCaptureOptions& options); + + // Creates a DesktopCapturer instance which targets to capture apps. + static std::unique_ptr CreateAppCapturer( + const DesktopCaptureOptions& options); + + // Creates a DesktopCapturer instance which targets to capture tab. + static std::unique_ptr CreateTabCapturer( + const DesktopCaptureOptions& options); + +#if defined(WEBRTC_USE_PIPEWIRE) || defined(USE_X11) + static bool IsRunningUnderWayland(); +#endif // defined(WEBRTC_USE_PIPEWIRE) || defined(USE_X11) + + protected: + // CroppingWindowCapturer needs to create raw capturers without wrappers, so + // the following two functions are protected. + + // Creates a platform specific DesktopCapturer instance which targets to + // capture windows. + static std::unique_ptr CreateRawWindowCapturer( + const DesktopCaptureOptions& options); + + // Creates a platform specific DesktopCapturer instance which targets to + // capture screens. + static std::unique_ptr CreateRawScreenCapturer( + const DesktopCaptureOptions& options); + + // Creates a platform specific DesktopCapturer instance which targets to + // capture apps. + static std::unique_ptr CreateRawAppCapturer( + const DesktopCaptureOptions& options); + + // Creates a DesktopCapturer instance which targets to capture tabs + static std::unique_ptr CreateRawTabCapturer( + const DesktopCaptureOptions& options); +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURER_H_ + diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.cc new file mode 100644 index 0000000000..087ee4fc03 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.cc @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/desktop_capturer_differ_wrapper.h" + +#include + +#include +#include + +#include "modules/desktop_capture/desktop_geometry.h" +#include "modules/desktop_capture/differ_block.h" +#include "rtc_base/checks.h" +#include "rtc_base/timeutils.h" + +namespace webrtc { + +namespace { + +// Returns true if (0, 0) - (|width|, |height|) vector in |old_buffer| and +// |new_buffer| are equal. |width| should be less than 32 +// (defined by kBlockSize), otherwise BlockDifference() should be used. +bool PartialBlockDifference(const uint8_t* old_buffer, + const uint8_t* new_buffer, + int width, + int height, + int stride) { + RTC_DCHECK_LT(width, kBlockSize); + const int width_bytes = width * DesktopFrame::kBytesPerPixel; + for (int i = 0; i < height; i++) { + if (memcmp(old_buffer, new_buffer, width_bytes) != 0) { + return true; + } + old_buffer += stride; + new_buffer += stride; + } + return false; +} + +// Compares columns in the range of [|left|, |right|), in a row in the +// range of [|top|, |top| + |height|), starts from |old_buffer| and +// |new_buffer|, and outputs updated regions into |output|. |stride| is the +// DesktopFrame::stride(). +void CompareRow(const uint8_t* old_buffer, + const uint8_t* new_buffer, + const int left, + const int right, + const int top, + const int bottom, + const int stride, + DesktopRegion* const output) { + const int block_x_offset = kBlockSize * DesktopFrame::kBytesPerPixel; + const int width = right - left; + const int height = bottom - top; + const int block_count = (width - 1) / kBlockSize; + const int last_block_width = width - block_count * kBlockSize; + RTC_DCHECK_GT(last_block_width, 0); + RTC_DCHECK_LE(last_block_width, kBlockSize); + + // The first block-column in a continuous dirty area in current block-row. + int first_dirty_x_block = -1; + + // We always need to add dirty area into |output| in the last block, so handle + // it separatedly. + for (int x = 0; x < block_count; x++) { + if (BlockDifference(old_buffer, new_buffer, height, stride)) { + if (first_dirty_x_block == -1) { + // This is the first dirty block in a continuous dirty area. + first_dirty_x_block = x; + } + } else if (first_dirty_x_block != -1) { + // The block on the left is the last dirty block in a continuous + // dirty area. + output->AddRect( + DesktopRect::MakeLTRB(first_dirty_x_block * kBlockSize + left, top, + x * kBlockSize + left, bottom)); + first_dirty_x_block = -1; + } + old_buffer += block_x_offset; + new_buffer += block_x_offset; + } + + bool last_block_diff; + if (last_block_width < kBlockSize) { + // The last one is a partial vector. + last_block_diff = PartialBlockDifference(old_buffer, new_buffer, + last_block_width, height, stride); + } else { + last_block_diff = BlockDifference(old_buffer, new_buffer, height, stride); + } + if (last_block_diff) { + if (first_dirty_x_block == -1) { + first_dirty_x_block = block_count; + } + output->AddRect(DesktopRect::MakeLTRB( + first_dirty_x_block * kBlockSize + left, top, right, bottom)); + } else if (first_dirty_x_block != -1) { + output->AddRect( + DesktopRect::MakeLTRB(first_dirty_x_block * kBlockSize + left, top, + block_count * kBlockSize + left, bottom)); + } +} + +// Compares |rect| area in |old_frame| and |new_frame|, and outputs dirty +// regions into |output|. +void CompareFrames(const DesktopFrame& old_frame, + const DesktopFrame& new_frame, + DesktopRect rect, + DesktopRegion* const output) { + RTC_DCHECK(old_frame.size().equals(new_frame.size())); + RTC_DCHECK_EQ(old_frame.stride(), new_frame.stride()); + rect.IntersectWith(DesktopRect::MakeSize(old_frame.size())); + + const int y_block_count = (rect.height() - 1) / kBlockSize; + const int last_y_block_height = rect.height() - y_block_count * kBlockSize; + // Offset from the start of one block-row to the next. + const int block_y_stride = old_frame.stride() * kBlockSize; + const uint8_t* prev_block_row_start = + old_frame.GetFrameDataAtPos(rect.top_left()); + const uint8_t* curr_block_row_start = + new_frame.GetFrameDataAtPos(rect.top_left()); + + int top = rect.top(); + // The last row may have a different height, so we handle it separately. + for (int y = 0; y < y_block_count; y++) { + CompareRow(prev_block_row_start, curr_block_row_start, rect.left(), + rect.right(), top, top + kBlockSize, old_frame.stride(), output); + top += kBlockSize; + prev_block_row_start += block_y_stride; + curr_block_row_start += block_y_stride; + } + CompareRow(prev_block_row_start, curr_block_row_start, rect.left(), + rect.right(), top, top + last_y_block_height, old_frame.stride(), + output); +} + +} // namespace + +DesktopCapturerDifferWrapper::DesktopCapturerDifferWrapper( + std::unique_ptr base_capturer) + : base_capturer_(std::move(base_capturer)) { + RTC_DCHECK(base_capturer_); +} + +DesktopCapturerDifferWrapper::~DesktopCapturerDifferWrapper() {} + +void DesktopCapturerDifferWrapper::Start(DesktopCapturer::Callback* callback) { + callback_ = callback; + base_capturer_->Start(this); +} + +void DesktopCapturerDifferWrapper::SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) { + base_capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory)); +} + +void DesktopCapturerDifferWrapper::CaptureFrame() { + base_capturer_->CaptureFrame(); +} + +void DesktopCapturerDifferWrapper::SetExcludedWindow(WindowId window) { + base_capturer_->SetExcludedWindow(window); +} + +bool DesktopCapturerDifferWrapper::GetSourceList(SourceList* sources) { + return base_capturer_->GetSourceList(sources); +} + +bool DesktopCapturerDifferWrapper::SelectSource(SourceId id) { + return base_capturer_->SelectSource(id); +} + +bool DesktopCapturerDifferWrapper::FocusOnSelectedSource() { + return base_capturer_->FocusOnSelectedSource(); +} + +bool DesktopCapturerDifferWrapper::IsOccluded(const DesktopVector& pos) { + return base_capturer_->IsOccluded(pos); +} + +void DesktopCapturerDifferWrapper::OnCaptureResult( + Result result, + std::unique_ptr input_frame) { + int64_t start_time_nanos = rtc::TimeNanos(); + if (!input_frame) { + callback_->OnCaptureResult(result, nullptr); + return; + } + RTC_DCHECK(result == Result::SUCCESS); + + std::unique_ptr frame = + SharedDesktopFrame::Wrap(std::move(input_frame)); + if (last_frame_ && (last_frame_->size().width() != frame->size().width() || + last_frame_->size().height() != frame->size().height() || + last_frame_->stride() != frame->stride())) { + last_frame_.reset(); + } + + if (last_frame_) { + DesktopRegion hints; + hints.Swap(frame->mutable_updated_region()); + for (DesktopRegion::Iterator it(hints); !it.IsAtEnd(); it.Advance()) { + CompareFrames(*last_frame_, *frame, it.rect(), + frame->mutable_updated_region()); + } + } else { + frame->mutable_updated_region()->SetRect( + DesktopRect::MakeSize(frame->size())); + } + last_frame_ = frame->Share(); + + frame->set_capture_time_ms(frame->capture_time_ms() + + (rtc::TimeNanos() - start_time_nanos) / + rtc::kNumNanosecsPerMillisec); + callback_->OnCaptureResult(result, std::move(frame)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.h b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.h new file mode 100644 index 0000000000..39e1fafc77 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURER_DIFFER_WRAPPER_H_ +#define MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURER_DIFFER_WRAPPER_H_ + +#include + +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/shared_desktop_frame.h" + +namespace webrtc { + +// DesktopCapturer wrapper that calculates updated_region() by comparing frames +// content. This class always expects the underlying DesktopCapturer +// implementation returns a superset of updated regions in DestkopFrame. If a +// DesktopCapturer implementation does not know the updated region, it should +// set updated_region() to full frame. +// +// This class marks entire frame as updated if the frame size or frame stride +// has been changed. +class DesktopCapturerDifferWrapper : public DesktopCapturer, + public DesktopCapturer::Callback { + public: + // Creates a DesktopCapturerDifferWrapper with a DesktopCapturer + // implementation, and takes its ownership. + explicit DesktopCapturerDifferWrapper( + std::unique_ptr base_capturer); + + ~DesktopCapturerDifferWrapper() override; + + // DesktopCapturer interface. + void Start(DesktopCapturer::Callback* callback) override; + void SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) override; + void CaptureFrame() override; + void SetExcludedWindow(WindowId window) override; + bool GetSourceList(SourceList* screens) override; + bool SelectSource(SourceId id) override; + bool FocusOnSelectedSource() override; + bool IsOccluded(const DesktopVector& pos) override; + + private: + // DesktopCapturer::Callback interface. + void OnCaptureResult(Result result, + std::unique_ptr frame) override; + + const std::unique_ptr base_capturer_; + DesktopCapturer::Callback* callback_; + std::unique_ptr last_frame_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURER_DIFFER_WRAPPER_H_ diff --git a/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper_unittest.cc b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper_unittest.cc new file mode 100644 index 0000000000..0a08088095 --- /dev/null +++ b/third_party/libwebrtc/webrtc/modules/desktop_capture/desktop_capturer_differ_wrapper_unittest.cc @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/desktop_capturer_differ_wrapper.h" + +#include +#include +#include +#include + +#include "modules/desktop_capture/desktop_geometry.h" +#include "modules/desktop_capture/desktop_region.h" +#include "modules/desktop_capture/differ_block.h" +#include "modules/desktop_capture/fake_desktop_capturer.h" +#include "modules/desktop_capture/mock_desktop_capturer_callback.h" +#include "rtc_base/random.h" +#include "rtc_base/timeutils.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "test/gtest.h" +#include "typedefs.h" // NOLINT(build/include) + +namespace webrtc { + +namespace { + +// Compares and asserts |frame|.updated_region() equals to |rects|. This +// function does not care about the order of the |rects| and it does not expect +// DesktopRegion to return an exact area for each rectangle in |rects|. +template